diff --git a/codelist/management/commands/update_codelist.py b/codelist/management/commands/update_codelist.py index 84e43e17..66ff2249 100644 --- a/codelist/management/commands/update_codelist.py +++ b/codelist/management/commands/update_codelist.py @@ -13,7 +13,8 @@ from codelist.settings import CODELIST_CONSERVATION_OFFICE_ID, \ CODELIST_REGISTRATION_OFFICE_ID, CODELIST_BIOTOPES_ID, CODELIST_LAW_ID, CODELIST_HANDLER_ID, \ CODELIST_COMPENSATION_ACTION_ID, CODELIST_COMPENSATION_ACTION_CLASS_ID, CODELIST_COMPENSATION_ADDITIONAL_TYPE_ID, \ CODELIST_BASE_URL, CODELIST_PROCESS_TYPE_ID, CODELIST_BIOTOPES_EXTRA_CODES_ID, \ - CODELIST_COMPENSATION_ACTION_DETAIL_ID, CODELIST_288_ID, CODELIST_COMPENSATION_HANDLER_ID + CODELIST_COMPENSATION_ACTION_DETAIL_ID, CODELIST_288_ID, CODELIST_COMPENSATION_HANDLER_ID, \ + CODELIST_AFTER_STATE_BIOTOPES_ID from konova.management.commands.setup import BaseKonovaCommand from konova.settings import PROXIES @@ -33,6 +34,7 @@ class Command(BaseKonovaCommand): CODELIST_REGISTRATION_OFFICE_ID, CODELIST_288_ID, CODELIST_BIOTOPES_ID, + CODELIST_AFTER_STATE_BIOTOPES_ID, CODELIST_BIOTOPES_EXTRA_CODES_ID, CODELIST_LAW_ID, CODELIST_HANDLER_ID, diff --git a/konova/management/commands/kspMigrater/base_migrater.py b/konova/management/commands/kspMigrater/base_migrater.py index 84c7c033..16f24d48 100644 --- a/konova/management/commands/kspMigrater/base_migrater.py +++ b/konova/management/commands/kspMigrater/base_migrater.py @@ -31,7 +31,6 @@ class BaseMigrater: user=self.options["db_user"], password=self.options["db_pw"], ) - print("Connected to ksp db...") self.db_connection = conn @abstractmethod diff --git a/konova/management/commands/kspMigrater/compensation_migrater.py b/konova/management/commands/kspMigrater/compensation_migrater.py index 2c3c85c7..f836ad50 100644 --- a/konova/management/commands/kspMigrater/compensation_migrater.py +++ b/konova/management/commands/kspMigrater/compensation_migrater.py @@ -14,6 +14,7 @@ from konova.models import Deadline, DeadlineType class CompensationMigrater(BaseMigrater): def migrate(self): + self.connect_db() cursor = self.db_connection.cursor() empty_str = "''" cursor.execute( @@ -35,7 +36,7 @@ class CompensationMigrater(BaseMigrater): 'om."OKL"=7730080 and ' 'om.archiv=false and ' f'(auf."Infos" is null or auf."Infos"={empty_str}) and ' - 'om.nicht_vollstaendig=0' + 'om.nicht_vollstaendig=0 ' ) all_koms = cursor.fetchall() @@ -58,6 +59,7 @@ class CompensationMigrater(BaseMigrater): compensation.title = kom_title compensation.comment = kom_comment + compensation = self._migrate_par_7_data(compensation, kom) compensation = self._migrate_geometry(compensation, kom) compensation = self._migrate_responsibility(compensation, kom) compensation = self._migrate_compensation_type(compensation, kom) @@ -155,6 +157,9 @@ class CompensationMigrater(BaseMigrater): # garbage continue state_surface = result[1] or 0.0 + if state_surface == 0.0: + # no surface, no real state -> garbage + continue pkey_entry = f"'{result[2]}'" try: state_code = KonovaCode.objects.get( @@ -215,19 +220,19 @@ class CompensationMigrater(BaseMigrater): f'om."KENNUNG"={kom_identifier}' ) db_result = tmp_cursor.fetchall() - if len(db_result) != 1: + if len(db_result) == 0: raise AssertionError(f"{kom_identifier} has no specification on compensation type (CEF, ...)") - comp_type = db_result[0][0] - if comp_type == 705816: - # regular compensation, do nothing - pass - elif comp_type == 705815: - compensation.is_cef = True - elif comp_type == 705817: - compensation.is_coherence_keeping = True - elif comp_type == 154156555: - compensation.is_coherence_keeping = True - compensation.is_cef = True + for comp_type in db_result: + if comp_type == 705816: + # regular compensation, do nothing + pass + elif comp_type == 705815: + compensation.is_cef = True + elif comp_type == 705817: + compensation.is_coherence_keeping = True + elif comp_type == 154156555: + compensation.is_coherence_keeping = True + compensation.is_cef = True tmp_cursor.close() return compensation @@ -508,3 +513,74 @@ class CompensationMigrater(BaseMigrater): tmp_cursor.close() return compensation + + def __migrate_par_7_action(self, compensation, kom): + # Since this codelist is deprecated and rather small, we define it once for this migration as a simple map + code_map = { + 265452837: "ökologische Verbesserung bestehender land- oder forstwirtschaftlicher Bodennutzung und landschaftlicher Strukturen", + 619127441: "Erhaltung und Verbesserung von Dauergrünland, insbesondere durch Beweidung", + 255952123: "Renaturierung von Gewässern", + 125425801: "Entsiegelung und Renaturierung von nicht mehr benötigten versiegelten Flächen im Innen- und Außenbereich", + 328264164: "Schaffung und Erhaltung größerer, zusammenhängender Biotopverbundstrukturen ", + 492185802: "Entwicklung und Wiederherstellung gesetzlich geschützter Biotope einschließlich des Verbunds zwischen einzelnen, benachbarten Biotopen", + 695727364: "Herstellung eines günstigen Erhaltungszustands eines Lebensraumtyps oder eines Vorkommens einer besonders geschützten Art", + 505765917: "Grunderwerb", + } + identifier = f"'{kom[0]}'" + tmp_cursor = self.db_connection.cursor() + tmp_cursor.execute( + 'select ' + 'par7.massnahmenachpar7 ' + 'from "OBJ_MASTER" om ' + 'join massnahmennachpar7 par7 on om."GISPADID"=par7.gispadid ' + 'where ' + f'om."KENNUNG"={identifier}' + ) + fetch_results = tmp_cursor.fetchall() + for result in fetch_results: + code = result[0] + try: + code = code_map[code] + except KeyError: + code = f"Unbekannt ({code})" + compensation.comment += f"\nMaßnahmen nach § 7 Abs. 3 LNatSchG: {code}" + + tmp_cursor.close() + return compensation + + def __migrate_par_7_area_details(self, compensation, kom): + # Since this codelist is deprecated and rather small, we define it once for this migration as a simple map + code_map = { + 134510452: "Natura 2000-Gebiete", + 40451403: "Flächen für Maßnahmen zur Verbesserung des ökologischen Gewässerzustands nach Wasserrahmenrichtlinie ", + 726756838: "Flächen in geschützten Teilen von Natur und Landschaft", + 742579102: "Zu Kompensationszwecken vorgesehene Flächen in Landschafts- oder Grünordnungsplänen", + 239671518: "Bewirtschaftungs- und Pflegemaßnahmen zur dauerhaften Aufwertung des Naturhaushaltes und des Landschaftsbildes im Rahmen von produktionsintegrierten Maßnahmen (PIK) auf land- und forstwirtschaftlichen Flächen, auch außerhalb von Schutzgebieten", + 685985349: "ökologische Aufwertung von Waldbeständen für Waldrodungen", + } + identifier = f"'{kom[0]}'" + tmp_cursor = self.db_connection.cursor() + tmp_cursor.execute( + 'select ' + 'g.gebietskulisse ' + 'from "OBJ_MASTER" om ' + 'join gebietskulisse g on om."GISPADID"=g.gispadid ' + 'where ' + f'om."KENNUNG"={identifier}' + ) + fetch_results = tmp_cursor.fetchall() + for result in fetch_results: + code = result[0] + try: + code = code_map[code] + except KeyError: + code = f"Unbekannt ({code})" + compensation.comment += f"\nGebietskulisse nach § 7 Abs. 1 und 2 LNatSchG: {code}" + + tmp_cursor.close() + return compensation + + def _migrate_par_7_data(self, compensation, kom): + compensation = self.__migrate_par_7_action(compensation, kom) + compensation = self.__migrate_par_7_area_details(compensation, kom) + return compensation \ No newline at end of file diff --git a/konova/management/commands/kspMigrater/eco_account_migrater.py b/konova/management/commands/kspMigrater/eco_account_migrater.py index 57ab15e0..33a4f8d7 100644 --- a/konova/management/commands/kspMigrater/eco_account_migrater.py +++ b/konova/management/commands/kspMigrater/eco_account_migrater.py @@ -1,3 +1,5 @@ +import datetime + from django.contrib.gis.geos import MultiPolygon, Polygon from django.core.exceptions import ObjectDoesNotExist from django.db import transaction @@ -5,7 +7,7 @@ from django.db import transaction from codelist.models import KonovaCode from codelist.settings import CODELIST_CONSERVATION_OFFICE_ID, CODELIST_COMPENSATION_HANDLER_ID from compensation.models import EcoAccount, EcoAccountDocument, EcoAccountDeduction -from intervention.models import Responsibility, Handler, Intervention +from intervention.models import Responsibility, Handler, Intervention, Legal from konova.management.commands.kspMigrater.compensation_migrater import CompensationMigrater from konova.models import Geometry from konova.sub_settings.lanis_settings import DEFAULT_SRID_RLP, DEFAULT_SRID @@ -14,6 +16,7 @@ from konova.sub_settings.lanis_settings import DEFAULT_SRID_RLP, DEFAULT_SRID class EcoAccountMigrater(CompensationMigrater): def migrate(self): + self.connect_db() cursor = self.db_connection.cursor() cursor.execute( 'select ' @@ -55,6 +58,7 @@ class EcoAccountMigrater(CompensationMigrater): eco_account.title = oek_title eco_account.comment = oek_comment + eco_account = self._migrate_legal(eco_account, oek) eco_account = self._migrate_geometry(eco_account, oek) eco_account = self._migrate_responsibility(eco_account, oek) eco_account = self._migrate_states(eco_account, oek) @@ -126,6 +130,7 @@ class EcoAccountMigrater(CompensationMigrater): return instance def _migrate_responsibility(self, eco_account, oek): + fallback = "Kein Eintrag" acc_identifier = f"'{oek[0]}'" tmp_cursor = self.db_connection.cursor() tmp_cursor.execute( @@ -151,6 +156,9 @@ class EcoAccountMigrater(CompensationMigrater): handler_type = db_results[2] handler_detail = db_results[3] + if cons_file_number is None or len(cons_file_number) == 0: + cons_file_number = fallback + eco_account.responsible = eco_account.responsible or Responsibility.objects.create() try: conservation_office = KonovaCode.objects.get( @@ -236,4 +244,11 @@ class EcoAccountMigrater(CompensationMigrater): ) tmp_cursor.close() + return eco_account + + def _migrate_legal(self, eco_account, oek): + # Just add default dummy values, since there has never been data for this previously + eco_account.legal = eco_account.legal or Legal() + eco_account.legal.registration_date = datetime.date.fromisoformat("1970-01-01") + eco_account.comment += "\nKein Vereinbarungsdatum eingetragen. Platzhalter 01.01.1970 gesetzt." return eco_account \ No newline at end of file diff --git a/konova/management/commands/kspMigrater/ema_migrater.py b/konova/management/commands/kspMigrater/ema_migrater.py index 249868a3..3bb005a1 100644 --- a/konova/management/commands/kspMigrater/ema_migrater.py +++ b/konova/management/commands/kspMigrater/ema_migrater.py @@ -13,6 +13,7 @@ from konova.models import DeadlineType, Deadline class EmaMigrater(CompensationMigrater): def migrate(self): + self.connect_db() cursor = self.db_connection.cursor() cursor.execute( 'select ' @@ -157,7 +158,7 @@ class EmaMigrater(CompensationMigrater): handler_type = None ema_obj.responsible.conservation_office = conservation_office - ema_obj.responsible.conservation_file_number = cons_file_number + ema_obj.responsible.conservation_file_number = cons_file_number or "Kein Eintrag" handler = ema_obj.responsible.handler or Handler.objects.create() handler.type = handler_type handler.detail = handler_detail diff --git a/konova/management/commands/kspMigrater/intervention_migrater.py b/konova/management/commands/kspMigrater/intervention_migrater.py index 25e2af24..90d0fa5d 100644 --- a/konova/management/commands/kspMigrater/intervention_migrater.py +++ b/konova/management/commands/kspMigrater/intervention_migrater.py @@ -1,3 +1,5 @@ +import datetime + from django.core.exceptions import ObjectDoesNotExist from django.db import transaction @@ -12,12 +14,20 @@ from konova.management.commands.kspMigrater.base_migrater import BaseMigrater class InterventionMigrater(BaseMigrater): def _migrate_intervention_responsibility(self, intervention, eiv): + fallback = "Kein Eintrag" intervention.responsible = intervention.responsible or Responsibility.objects.create() intervention.responsible.handler = intervention.responsible.handler or Handler.objects.create() eiv_reg_off = eiv[7] eiv_cons_off = eiv[9] eiv_handler_type = eiv[11] eiv_handler_detail = eiv[12] + eiv_reg_file_num = eiv[8] + eiv_cons_file_num = eiv[10] + + if eiv_reg_file_num is None or len(eiv_reg_file_num) == 0: + eiv_reg_file_num = fallback + if eiv_cons_file_num is None or len(eiv_cons_file_num) == 0: + eiv_cons_file_num = fallback if eiv_reg_off is not None and eiv_reg_off != 0: try: @@ -28,8 +38,8 @@ class InterventionMigrater(BaseMigrater): ) intervention.responsible.registration_office = reg_office_code except ObjectDoesNotExist: - intervention.comment = f"{intervention.comment or ''}\nNicht migrierbare Zulassungsbehörde: {eiv_reg_off}" - intervention.responsible.registration_file_number = eiv[8] + intervention.comment = f"{intervention.comment or ''}\nUnbekannte Zulassungsbehörde: {eiv_reg_off}" + intervention.responsible.registration_file_number = eiv_reg_file_num if eiv_cons_off is not None and eiv_cons_off != 0: try: @@ -39,9 +49,9 @@ class InterventionMigrater(BaseMigrater): code_lists__in=[CODELIST_CONSERVATION_OFFICE_ID], ) except ObjectDoesNotExist: - intervention.comment = f"{intervention.comment or ''}\nNicht migrierbare Eintragungsstelle: {eiv_cons_off}" + intervention.comment = f"{intervention.comment or ''}\nUnbekannte Eintragungsstelle: {eiv_cons_off}" intervention.responsible.conservation_office = cons_office_code - intervention.responsible.conservation_file_number = eiv[10] + intervention.responsible.conservation_file_number = eiv_cons_file_num if eiv_handler_type is not None and eiv_handler_type != 0: try: @@ -66,13 +76,17 @@ class InterventionMigrater(BaseMigrater): eiv_date_registration = eiv[14] eiv_date_binding = eiv[15] - if eiv_process_type is not None and eiv_process_type != 0: - process_type_code = KonovaCode.objects.get( - atom_id=eiv_process_type, - is_leaf=True, - code_lists__in=[CODELIST_PROCESS_TYPE_ID], - ) - intervention.legal.process_type = process_type_code + if eiv_process_type is None or eiv_process_type == 0: + # Fallback "Genehmigung" + eiv_process_type = 623645439 + + process_type_code = KonovaCode.objects.get( + atom_id=eiv_process_type, + is_leaf=True, + code_lists__in=[CODELIST_PROCESS_TYPE_ID], + ) + intervention.legal.process_type = process_type_code + if eiv_law is not None and eiv_law != 0: law_code = KonovaCode.objects.get( atom_id=eiv_law, @@ -81,10 +95,17 @@ class InterventionMigrater(BaseMigrater): ) intervention.legal.laws.add(law_code) - if eiv_date_registration is not None: - intervention.legal.registration_date = eiv_date_registration - if eiv_date_binding is not None: - intervention.legal.binding_date = eiv_date_binding + fallback_date = datetime.date.fromisoformat("1970-01-01") + if eiv_date_registration is None: + eiv_date_registration = fallback_date + intervention.comment += "\nKein Eintrag für Zulassungsdatum. Platzhalter 01.01.1970 gesetzt" + + if eiv_date_binding is None: + eiv_date_binding = fallback_date + intervention.comment += "\nKein Eintrag für Bestandskraftdatum. Platzhalter 01.01.1970 gesetzt" + + intervention.legal.binding_date = eiv_date_binding + intervention.legal.registration_date = eiv_date_registration intervention.legal.save() return intervention @@ -112,6 +133,7 @@ class InterventionMigrater(BaseMigrater): return intervention def migrate(self): + self.connect_db() cursor = self.db_connection.cursor() cursor.execute( 'select ' diff --git a/konova/management/commands/update_all_parcels.py b/konova/management/commands/update_all_parcels.py index 9d96ebae..9cd3a334 100644 --- a/konova/management/commands/update_all_parcels.py +++ b/konova/management/commands/update_all_parcels.py @@ -5,6 +5,8 @@ Contact: michel.peltriaux@sgdnord.rlp.de Created on: 04.01.22 """ +from pyexpat import ExpatError + from konova.management.commands.setup import BaseKonovaCommand from konova.models import Geometry, Parcel, District @@ -14,21 +16,44 @@ class Command(BaseKonovaCommand): def handle(self, *args, **options): try: - self.update_all_parcels() + self.update_parcels() except KeyboardInterrupt: self._break_line() exit(-1) - def update_all_parcels(self): + def update_parcels(self, ids: list = None): + """ Updates all geometry-parcel relations using an official parcel WFS. + + THIS METHOD SHOULD NOT BE RUN IN PARALLEL E.G. USING CELERY DUE TO POSSIBLE PARCEL-CONFLICTS ON THE DB + + """ num_parcels_before = Parcel.objects.count() num_districts_before = District.objects.count() self._write_warning("=== Update parcels and districts ===") geometries = Geometry.objects.all().exclude( geom=None ) - self._write_warning(f"Process parcels for {geometries.count()} geometry entries now ...") + + if ids is not None: + self._write_warning(f"Retry parcel calculation for {len(ids)} geometry entries now ...") + geometries = geometries.filter(id__in=ids) + + total_count = geometries.count() + self._write_warning(f"Process parcels for {total_count} geometry entries now ...") + i = 0 + try_again_ids = [] for geometry in geometries: - geometry.update_parcels() + if i % 500 == 0: + self._write_warning(f"--- {i}/{total_count} processed") + try: + geometry.update_parcels() + except ExpatError as e: + self._write_error(f"--- {geometry.id} encountered an error: {e}. Geometry added to the retry list.") + try_again_ids.append(geometry.id) + i += 1 + + if len(try_again_ids) > 0: + self.update_parcels(try_again_ids) num_parcels_after = Parcel.objects.count() num_districts_after = District.objects.count()