#132 WIP migration

* adds KOM paragraph 7 migration into compensation comment field
* adds fallback data for missing registration and binding dates, file numbers and so on
* fixes bug where a migration class' db connection would get a timeout if not used asap after instanciation
* enhances parcel update command
    * collects geometry id in case of unsuccessful calculation attempt and rerun another try on these afterwards (errors cased server-side of WFS)

codelist/management/commands/update_codelist.py

@@ -13,7 +13,8 @@ from codelist.settings import CODELIST_CONSERVATION_OFFICE_ID, \
     CODELIST_REGISTRATION_OFFICE_ID, CODELIST_BIOTOPES_ID, CODELIST_LAW_ID, CODELIST_HANDLER_ID, \
     CODELIST_COMPENSATION_ACTION_ID, CODELIST_COMPENSATION_ACTION_CLASS_ID, CODELIST_COMPENSATION_ADDITIONAL_TYPE_ID, \
     CODELIST_BASE_URL, CODELIST_PROCESS_TYPE_ID, CODELIST_BIOTOPES_EXTRA_CODES_ID, \
-    CODELIST_COMPENSATION_ACTION_DETAIL_ID, CODELIST_288_ID, CODELIST_COMPENSATION_HANDLER_ID
+    CODELIST_COMPENSATION_ACTION_DETAIL_ID, CODELIST_288_ID, CODELIST_COMPENSATION_HANDLER_ID, \
+    CODELIST_AFTER_STATE_BIOTOPES_ID
 from konova.management.commands.setup import BaseKonovaCommand
 from konova.settings import PROXIES
 
@@ -33,6 +34,7 @@ class Command(BaseKonovaCommand):
                 CODELIST_REGISTRATION_OFFICE_ID,
                 CODELIST_288_ID,
                 CODELIST_BIOTOPES_ID,
+                CODELIST_AFTER_STATE_BIOTOPES_ID,
                 CODELIST_BIOTOPES_EXTRA_CODES_ID,
                 CODELIST_LAW_ID,
                 CODELIST_HANDLER_ID,

konova/management/commands/kspMigrater/base_migrater.py

@@ -31,7 +31,6 @@ class BaseMigrater:
             user=self.options["db_user"],
             password=self.options["db_pw"],
         )
-        print("Connected to ksp db...")
         self.db_connection = conn
 
     @abstractmethod

konova/management/commands/kspMigrater/compensation_migrater.py

@@ -14,6 +14,7 @@ from konova.models import Deadline, DeadlineType
 class CompensationMigrater(BaseMigrater):
 
     def migrate(self):
+        self.connect_db()
         cursor = self.db_connection.cursor()
         empty_str = "''"
         cursor.execute(
@@ -58,6 +59,7 @@ class CompensationMigrater(BaseMigrater):
 
                 compensation.title = kom_title
                 compensation.comment = kom_comment
+                compensation = self._migrate_par_7_data(compensation, kom)
                 compensation = self._migrate_geometry(compensation, kom)
                 compensation = self._migrate_responsibility(compensation, kom)
                 compensation = self._migrate_compensation_type(compensation, kom)
@@ -155,6 +157,9 @@ class CompensationMigrater(BaseMigrater):
                 # garbage
                 continue
             state_surface = result[1] or 0.0
+            if state_surface == 0.0:
+                # no surface, no real state -> garbage
+                continue
             pkey_entry = f"'{result[2]}'"
             try:
                 state_code = KonovaCode.objects.get(
@@ -215,9 +220,9 @@ class CompensationMigrater(BaseMigrater):
                 f'om."KENNUNG"={kom_identifier}'
         )
         db_result = tmp_cursor.fetchall()
-        if len(db_result) != 1:
+        if len(db_result) == 0:
             raise AssertionError(f"{kom_identifier} has no specification on compensation type (CEF, ...)")
-        comp_type = db_result[0][0]
+        for comp_type in db_result:
             if comp_type == 705816:
                 # regular compensation, do nothing
                 pass
@@ -508,3 +513,74 @@ class CompensationMigrater(BaseMigrater):
 
         tmp_cursor.close()
         return compensation
+
+    def __migrate_par_7_action(self, compensation, kom):
+        # Since this codelist is deprecated and rather small, we define it once for this migration as a simple map
+        code_map = {
+            265452837: "ökologische Verbesserung bestehender land- oder forstwirtschaftlicher Bodennutzung und landschaftlicher Strukturen",
+            619127441: "Erhaltung und Verbesserung von Dauergrünland, insbesondere durch Beweidung",
+            255952123: "Renaturierung von Gewässern",
+            125425801: "Entsiegelung und Renaturierung von nicht mehr benötigten versiegelten Flächen im Innen- und Außenbereich",
+            328264164: "Schaffung und Erhaltung größerer, zusammenhängender Biotopverbundstrukturen ",
+            492185802: "Entwicklung und Wiederherstellung gesetzlich geschützter Biotope einschließlich des Verbunds zwischen einzelnen, benachbarten Biotopen",
+            695727364: "Herstellung eines günstigen Erhaltungszustands eines Lebensraumtyps oder eines Vorkommens einer besonders geschützten Art",
+            505765917: "Grunderwerb",
+        }
+        identifier = f"'{kom[0]}'"
+        tmp_cursor = self.db_connection.cursor()
+        tmp_cursor.execute(
+            'select '
+                'par7.massnahmenachpar7 '
+            'from "OBJ_MASTER" om '
+                'join massnahmennachpar7 par7 on om."GISPADID"=par7.gispadid '
+            'where '
+                f'om."KENNUNG"={identifier}'
+        )
+        fetch_results = tmp_cursor.fetchall()
+        for result in fetch_results:
+            code = result[0]
+            try:
+                code = code_map[code]
+            except KeyError:
+                code = f"Unbekannt ({code})"
+            compensation.comment += f"\nMaßnahmen nach § 7 Abs. 3 LNatSchG: {code}"
+
+        tmp_cursor.close()
+        return compensation
+
+    def __migrate_par_7_area_details(self, compensation, kom):
+        # Since this codelist is deprecated and rather small, we define it once for this migration as a simple map
+        code_map = {
+            134510452: "Natura 2000-Gebiete",
+            40451403: "Flächen für Maßnahmen zur Verbesserung des ökologischen Gewässerzustands nach Wasserrahmenrichtlinie ",
+            726756838: "Flächen in geschützten Teilen von Natur und Landschaft",
+            742579102: "Zu Kompensationszwecken vorgesehene Flächen in Landschafts- oder Grünordnungsplänen",
+            239671518: "Bewirtschaftungs- und Pflegemaßnahmen zur dauerhaften Aufwertung des Naturhaushaltes und des Landschaftsbildes im Rahmen von produktionsintegrierten Maßnahmen (PIK) auf land- und forstwirtschaftlichen Flächen, auch außerhalb von Schutzgebieten",
+            685985349: "ökologische Aufwertung von Waldbeständen für Waldrodungen",
+        }
+        identifier = f"'{kom[0]}'"
+        tmp_cursor = self.db_connection.cursor()
+        tmp_cursor.execute(
+            'select '
+                'g.gebietskulisse '
+            'from "OBJ_MASTER" om '
+                'join gebietskulisse g on om."GISPADID"=g.gispadid '
+            'where '
+                f'om."KENNUNG"={identifier}'
+        )
+        fetch_results = tmp_cursor.fetchall()
+        for result in fetch_results:
+            code = result[0]
+            try:
+                code = code_map[code]
+            except KeyError:
+                code = f"Unbekannt ({code})"
+            compensation.comment += f"\nGebietskulisse nach § 7 Abs. 1 und 2 LNatSchG: {code}"
+
+        tmp_cursor.close()
+        return compensation
+
+    def _migrate_par_7_data(self, compensation, kom):
+        compensation = self.__migrate_par_7_action(compensation, kom)
+        compensation = self.__migrate_par_7_area_details(compensation, kom)
+        return compensation

konova/management/commands/kspMigrater/eco_account_migrater.py

@@ -1,3 +1,5 @@
+import datetime
+
 from django.contrib.gis.geos import MultiPolygon, Polygon
 from django.core.exceptions import ObjectDoesNotExist
 from django.db import transaction
@@ -5,7 +7,7 @@ from django.db import transaction
 from codelist.models import KonovaCode
 from codelist.settings import CODELIST_CONSERVATION_OFFICE_ID, CODELIST_COMPENSATION_HANDLER_ID
 from compensation.models import EcoAccount, EcoAccountDocument, EcoAccountDeduction
-from intervention.models import Responsibility, Handler, Intervention
+from intervention.models import Responsibility, Handler, Intervention, Legal
 from konova.management.commands.kspMigrater.compensation_migrater import CompensationMigrater
 from konova.models import Geometry
 from konova.sub_settings.lanis_settings import DEFAULT_SRID_RLP, DEFAULT_SRID
@@ -14,6 +16,7 @@ from konova.sub_settings.lanis_settings import DEFAULT_SRID_RLP, DEFAULT_SRID
 class EcoAccountMigrater(CompensationMigrater):
 
     def migrate(self):
+        self.connect_db()
         cursor = self.db_connection.cursor()
         cursor.execute(
             'select '
@@ -55,6 +58,7 @@ class EcoAccountMigrater(CompensationMigrater):
 
                 eco_account.title = oek_title
                 eco_account.comment = oek_comment
+                eco_account = self._migrate_legal(eco_account, oek)
                 eco_account = self._migrate_geometry(eco_account, oek)
                 eco_account = self._migrate_responsibility(eco_account, oek)
                 eco_account = self._migrate_states(eco_account, oek)
@@ -126,6 +130,7 @@ class EcoAccountMigrater(CompensationMigrater):
         return instance
 
     def _migrate_responsibility(self, eco_account, oek):
+        fallback = "Kein Eintrag"
         acc_identifier = f"'{oek[0]}'"
         tmp_cursor = self.db_connection.cursor()
         tmp_cursor.execute(
@@ -151,6 +156,9 @@ class EcoAccountMigrater(CompensationMigrater):
         handler_type = db_results[2]
         handler_detail = db_results[3]
 
+        if cons_file_number is None or len(cons_file_number) == 0:
+            cons_file_number = fallback
+
         eco_account.responsible = eco_account.responsible or Responsibility.objects.create()
         try:
             conservation_office = KonovaCode.objects.get(
@@ -237,3 +245,10 @@ class EcoAccountMigrater(CompensationMigrater):
 
         tmp_cursor.close()
         return eco_account
+
+    def _migrate_legal(self, eco_account, oek):
+        # Just add default dummy values, since there has never been data for this previously
+        eco_account.legal = eco_account.legal or Legal()
+        eco_account.legal.registration_date = datetime.date.fromisoformat("1970-01-01")
+        eco_account.comment += "\nKein Vereinbarungsdatum eingetragen. Platzhalter 01.01.1970 gesetzt."
+        return eco_account

konova/management/commands/kspMigrater/ema_migrater.py

@@ -13,6 +13,7 @@ from konova.models import DeadlineType, Deadline
 class EmaMigrater(CompensationMigrater):
 
     def migrate(self):
+        self.connect_db()
         cursor = self.db_connection.cursor()
         cursor.execute(
             'select '
@@ -157,7 +158,7 @@ class EmaMigrater(CompensationMigrater):
             handler_type = None
 
         ema_obj.responsible.conservation_office = conservation_office
-        ema_obj.responsible.conservation_file_number = cons_file_number
+        ema_obj.responsible.conservation_file_number = cons_file_number or "Kein Eintrag"
         handler = ema_obj.responsible.handler or Handler.objects.create()
         handler.type = handler_type
         handler.detail = handler_detail

konova/management/commands/kspMigrater/intervention_migrater.py

@@ -1,3 +1,5 @@
+import datetime
+
 from django.core.exceptions import ObjectDoesNotExist
 from django.db import transaction
 
@@ -12,12 +14,20 @@ from konova.management.commands.kspMigrater.base_migrater import BaseMigrater
 class InterventionMigrater(BaseMigrater):
 
     def _migrate_intervention_responsibility(self, intervention, eiv):
+        fallback = "Kein Eintrag"
         intervention.responsible = intervention.responsible or Responsibility.objects.create()
         intervention.responsible.handler = intervention.responsible.handler or Handler.objects.create()
         eiv_reg_off = eiv[7]
         eiv_cons_off = eiv[9]
         eiv_handler_type = eiv[11]
         eiv_handler_detail = eiv[12]
+        eiv_reg_file_num = eiv[8]
+        eiv_cons_file_num = eiv[10]
+
+        if eiv_reg_file_num is None or len(eiv_reg_file_num) == 0:
+            eiv_reg_file_num = fallback
+        if eiv_cons_file_num is None or len(eiv_cons_file_num) == 0:
+            eiv_cons_file_num = fallback
 
         if eiv_reg_off is not None and eiv_reg_off != 0:
             try:
@@ -28,8 +38,8 @@ class InterventionMigrater(BaseMigrater):
                 )
                 intervention.responsible.registration_office = reg_office_code
             except ObjectDoesNotExist:
-                intervention.comment = f"{intervention.comment or ''}\nNicht migrierbare Zulassungsbehörde: {eiv_reg_off}"
+                intervention.comment = f"{intervention.comment or ''}\nUnbekannte Zulassungsbehörde: {eiv_reg_off}"
-        intervention.responsible.registration_file_number = eiv[8]
+        intervention.responsible.registration_file_number = eiv_reg_file_num
 
         if eiv_cons_off is not None and eiv_cons_off != 0:
             try:
@@ -39,9 +49,9 @@ class InterventionMigrater(BaseMigrater):
                     code_lists__in=[CODELIST_CONSERVATION_OFFICE_ID],
                 )
             except ObjectDoesNotExist:
-                intervention.comment = f"{intervention.comment or ''}\nNicht migrierbare Eintragungsstelle: {eiv_cons_off}"
+                intervention.comment = f"{intervention.comment or ''}\nUnbekannte Eintragungsstelle: {eiv_cons_off}"
             intervention.responsible.conservation_office = cons_office_code
-        intervention.responsible.conservation_file_number = eiv[10]
+        intervention.responsible.conservation_file_number = eiv_cons_file_num
 
         if eiv_handler_type is not None and eiv_handler_type != 0:
             try:
@@ -66,13 +76,17 @@ class InterventionMigrater(BaseMigrater):
         eiv_date_registration = eiv[14]
         eiv_date_binding = eiv[15]
 
-        if eiv_process_type is not None and eiv_process_type != 0:
+        if eiv_process_type is None or eiv_process_type == 0:
+            # Fallback "Genehmigung"
+            eiv_process_type = 623645439
+
         process_type_code = KonovaCode.objects.get(
             atom_id=eiv_process_type,
             is_leaf=True,
             code_lists__in=[CODELIST_PROCESS_TYPE_ID],
         )
         intervention.legal.process_type = process_type_code
+
         if eiv_law is not None and eiv_law != 0:
             law_code = KonovaCode.objects.get(
                 atom_id=eiv_law,
@@ -81,10 +95,17 @@ class InterventionMigrater(BaseMigrater):
             )
             intervention.legal.laws.add(law_code)
 
-        if eiv_date_registration is not None:
+        fallback_date = datetime.date.fromisoformat("1970-01-01")
-            intervention.legal.registration_date = eiv_date_registration
+        if eiv_date_registration is None:
-        if eiv_date_binding is not None:
+            eiv_date_registration = fallback_date
+            intervention.comment += "\nKein Eintrag für Zulassungsdatum. Platzhalter 01.01.1970 gesetzt"
+
+        if eiv_date_binding is None:
+            eiv_date_binding = fallback_date
+            intervention.comment += "\nKein Eintrag für Bestandskraftdatum. Platzhalter 01.01.1970 gesetzt"
+
         intervention.legal.binding_date = eiv_date_binding
+        intervention.legal.registration_date = eiv_date_registration
 
         intervention.legal.save()
         return intervention
@@ -112,6 +133,7 @@ class InterventionMigrater(BaseMigrater):
         return intervention
 
     def migrate(self):
+        self.connect_db()
         cursor = self.db_connection.cursor()
         cursor.execute(
             'select '

konova/management/commands/update_all_parcels.py

@@ -5,6 +5,8 @@ Contact: michel.peltriaux@sgdnord.rlp.de
 Created on: 04.01.22
 
 """
+from pyexpat import ExpatError
+
 from konova.management.commands.setup import BaseKonovaCommand
 from konova.models import Geometry, Parcel, District
 
@@ -14,21 +16,44 @@ class Command(BaseKonovaCommand):
 
     def handle(self, *args, **options):
         try:
-            self.update_all_parcels()
+            self.update_parcels()
         except KeyboardInterrupt:
             self._break_line()
             exit(-1)
 
-    def update_all_parcels(self):
+    def update_parcels(self, ids: list = None):
+        """ Updates all geometry-parcel relations using an official parcel WFS.
+
+        THIS METHOD SHOULD NOT BE RUN IN PARALLEL E.G. USING CELERY DUE TO POSSIBLE PARCEL-CONFLICTS ON THE DB
+
+        """
         num_parcels_before = Parcel.objects.count()
         num_districts_before = District.objects.count()
         self._write_warning("=== Update parcels and districts ===")
         geometries = Geometry.objects.all().exclude(
             geom=None
         )
-        self._write_warning(f"Process parcels for {geometries.count()} geometry entries now ...")
+
+        if ids is not None:
+            self._write_warning(f"Retry parcel calculation for {len(ids)} geometry entries now ...")
+            geometries = geometries.filter(id__in=ids)
+
+        total_count = geometries.count()
+        self._write_warning(f"Process parcels for {total_count} geometry entries now ...")
+        i = 0
+        try_again_ids = []
         for geometry in geometries:
+            if i % 500 == 0:
+                self._write_warning(f"--- {i}/{total_count} processed")
+            try:
                 geometry.update_parcels()
+            except ExpatError as e:
+                self._write_error(f"--- {geometry.id} encountered an error: {e}. Geometry added to the retry list.")
+                try_again_ids.append(geometry.id)
+            i += 1
+
+        if len(try_again_ids) > 0:
+            self.update_parcels(try_again_ids)
 
         num_parcels_after = Parcel.objects.count()
         num_districts_after = District.objects.count()