#132 old entries

* WIP: further improves migration of interventions

konova/management/commands/migrate_ksp_konova.py

@@ -1,7 +1,12 @@
 import psycopg2
 from django.contrib.gis.geos import MultiPolygon, GEOSException, MultiPoint, MultiLineString, Polygon
+from django.core.exceptions import ObjectDoesNotExist
+from django.db import transaction
 
-from intervention.models import Intervention, Legal, Responsibility
+from codelist.models import KonovaCode
+from codelist.settings import CODELIST_CONSERVATION_OFFICE_ID, CODELIST_REGISTRATION_OFFICE_ID, \
+    CODELIST_PROCESS_TYPE_ID, CODELIST_LAW_ID, CODELIST_HANDLER_ID
+from intervention.models import Intervention, Legal, Responsibility, Handler
 from konova.management.commands.setup import BaseKonovaCommand
 from konova.models import Geometry
 from konova.tasks import celery_update_parcels
@@ -37,43 +42,168 @@ class Command(BaseKonovaCommand):
         self._write_success("Connected to ksp db...")
         self.db_connection = conn
 
+    def _migrate_intervention_geometry(self, intervention:Intervention, eiv: tuple):
+        try:
+            # eiv[2] == polygon, eiv[3] == line, eiv[4] == point
+            if eiv[2] is not None:
+                eiv_geom = MultiPolygon.from_ewkt(eiv[2])
+            elif eiv[3] is not None:
+                eiv_geom = MultiLineString.from_ewkt(eiv[3])
+                eiv_geom = eiv_geom.buffer(0.00001, 1)
+                if isinstance(eiv_geom, Polygon):
+                    eiv_geom = MultiPolygon(eiv_geom)
+            elif eiv[4] is not None:
+                eiv_geom = MultiPoint.from_ewkt(eiv[4])
+                eiv_geom = eiv_geom.buffer(0.00001, 1)
+                if isinstance(eiv_geom, Polygon):
+                    eiv_geom = MultiPolygon(eiv_geom)
+            else:
+                eiv_geom = None
+        except GEOSException:
+            self._write_error(f"Malicious geometry on {eiv}")
+            return
+        geom = intervention.geometry or Geometry()
+        geom.geom = eiv_geom
+        geom.save()
+        # celery_update_parcels.delay(geom.id)
+
+        intervention.geometry = geom
+        return intervention
+
+    def _migrate_intervention_responsibility(self, intervention, eiv):
+        intervention.responsible = intervention.responsible or Responsibility.objects.create()
+        intervention.responsible.handler = intervention.responsible.handler or Handler.objects.create()
+        eiv_reg_off = eiv[7]
+        eiv_cons_off = eiv[9]
+        eiv_handler_type = eiv[11]
+        eiv_handler_detail = eiv[12]
+
+        if eiv_reg_off is not None and eiv_reg_off != 0:
+            try:
+                reg_office_code = KonovaCode.objects.get(
+                    atom_id=eiv_reg_off,
+                    is_leaf=True,
+                    code_lists__in=[CODELIST_REGISTRATION_OFFICE_ID],
+                )
+                intervention.responsible.registration_office = reg_office_code
+            except ObjectDoesNotExist:
+                intervention.comment = f"{intervention.comment or ''}\nNicht migrierbare Zulassungsbehörde: {eiv_reg_off}"
+        intervention.responsible.registration_file_number = eiv[8]
+
+        if eiv_cons_off is not None and eiv_cons_off != 0:
+            try:
+                cons_office_code = KonovaCode.objects.get(
+                    atom_id=eiv_cons_off,
+                    is_leaf=True,
+                    code_lists__in=[CODELIST_CONSERVATION_OFFICE_ID],
+                )
+            except ObjectDoesNotExist:
+                intervention.comment = f"{intervention.comment or ''}\nNicht migrierbare Eintragungsstelle: {eiv_cons_off}"
+            intervention.responsible.conservation_office = cons_office_code
+        intervention.responsible.conservation_file_number = eiv[10]
+
+        if eiv_handler_type is not None and eiv_handler_type != 0:
+            try:
+                handler_type_code = KonovaCode.objects.get(
+                    atom_id=eiv_handler_type,
+                    is_leaf=True,
+                    code_lists__in=[CODELIST_HANDLER_ID]
+                )
+                intervention.responsible.handler.type = handler_type_code
+            except ObjectDoesNotExist:
+                intervention.comment = f"{intervention.comment or ''}\nNicht migrierbarer Eingriffsverursacher_TYP: {eiv_handler_type}"
+            intervention.responsible.handler.detail = eiv_handler_detail
+            intervention.responsible.handler.save()
+
+        intervention.responsible.save()
+        return intervention
+
+    def _migrate_intervention_legal(self, intervention, eiv):
+        intervention.legal = intervention.legal or Legal.objects.create()
+        eiv_process_type = eiv[5]
+        eiv_law = eiv[6]
+        eiv_date_registration = eiv[14]
+        eiv_date_binding = eiv[15]
+
+        if eiv_process_type is not None and eiv_process_type != 0:
+            process_type_code = KonovaCode.objects.get(
+                atom_id=eiv_process_type,
+                is_leaf=True,
+                code_lists__in=[CODELIST_PROCESS_TYPE_ID],
+            )
+            intervention.legal.process_type = process_type_code
+        if eiv_law is not None and eiv_law != 0:
+            law_code = KonovaCode.objects.get(
+                atom_id=eiv_law,
+                is_leaf=True,
+                code_lists__in=[CODELIST_LAW_ID],
+            )
+            intervention.legal.laws.add(law_code)
+
+        if eiv_date_registration is not None:
+            intervention.legal.registration_date = eiv_date_registration
+        if eiv_date_binding is not None:
+            intervention.legal.binding_date = eiv_date_binding
+
+        intervention.legal.save()
+        return intervention
+
     def migrate_interventions(self):
         cursor = self.db_connection.cursor()
         cursor.execute(
-            'select om."KENNUNG", linf."OBJBEZ", ST_AsEWKT(ST_Transform(ST_MakeValid(geomf.the_geom),4326)) as geomf, ST_AsEWKT(ST_Transform(ST_MakeValid(geoml.the_geom),4326)) as geoml, ST_AsEWKT(ST_Transform(ST_MakeValid(geomp.the_geom),4326)) as geomp from "OBJ_MASTER" om left join "LINFOS" linf on om."GISPADID"=linf."GISPADID" left join geometry_f geomf on om."GISPADID"=geomf.gispadid left join geometry_l geoml on om."GISPADID"=geoml.gispadid left join geometry_p geomp on om."GISPADID"=geomp.gispadid where om."OKL"=7730085 and om.archiv=false;'
+            'select '
+                'om."KENNUNG", '
+                'linf."OBJBEZ", '
+                'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geomf.the_geom,4326)), 3))) as geomf, '
+                'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geoml.the_geom,4326)), 2))) as geoml, '
+                'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geomp.the_geom,4326)), 1))) as geomp, '
+                'eiv.verfahrenstyp, '
+                'vr.verfahrensrecht, '
+                'adr.behoerde as zb, '
+                'linf."AZ" as zb_az, '
+                'adr.adr_pruef as ets, '
+                'auf."AZ" as ets_az, '
+                'adr.adressrolle as eingriffsverursacher_typ, '
+                'adr."Bemerkung" as eingriffsverursacher_bemerkung, '
+                'linf."Bemerkung" as eiv_comment, '
+                'zt.erlass as Zulassungsdatum, '
+                'zt.rechtskraft as Bestandskraftdatum '
+            'from "OBJ_MASTER" om '
+                'left join "LINFOS" linf on om."GISPADID"=linf."GISPADID" '
+                'left join eiv on om."GISPADID"=eiv.gispadid '
+                'left join verfahrensrecht vr on om."GISPADID"=vr.gispadid '
+                'left join geometry_f geomf on om."GISPADID"=geomf.gispadid '
+                'left join geometry_l geoml on om."GISPADID"=geoml.gispadid '
+                'left join geometry_p geomp on om."GISPADID"=geomp.gispadid '
+                'left join adressrolle adr on om."GISPADID"=adr."GISPADID" '
+                'left join "Aufwertung" auf on om."GISPADID"=auf."GISPADID" '
+                'left join zulassungstermin zt on eiv.zulassung=zt.id '
+            'where '
+                'om."OKL"=7730085 and '
+                'om.archiv=false and '
+                'om.nicht_vollstaendig=0;'
         )
+
         all_eivs = cursor.fetchall()
+        len_all_eivs = len(all_eivs)
+        num_processed = 0
+        self._write_warning(f"Migrate EIVs to interventions...")
+        self._write_warning(f"Found {len_all_eivs} entries. Process now...")
         for eiv in all_eivs:
-            intervention = Intervention.objects.get_or_create(
+            if num_processed % 500 == 0:
-                identifier=eiv[0]
+                self._write_warning(f"   {num_processed}/{len_all_eivs} processed")
-            )[0]
+            with transaction.atomic():
-            intervention.title = eiv[1]
+                eiv_comment = eiv[13] or ""
-            try:
+                intervention = Intervention.objects.get_or_create(
-                # eiv[2] == polygon, eiv[3] == line, eiv[4] == point
+                    identifier=eiv[0]
-                if eiv[2] is not None:
+                )[0]
-                    eiv_geom = MultiPolygon.from_ewkt(eiv[2])
+                intervention.title = eiv[1]
-                elif eiv[3] is not None:
+                intervention.comment = eiv_comment
-                    eiv_geom = MultiPoint.from_ewkt(eiv[3])
-                    eiv_geom = eiv_geom.buffer(0.00001, 1)
-                    if isinstance(eiv_geom, Polygon):
-                        eiv_geom = MultiPolygon(eiv_geom)
-                elif eiv[4] is not None:
-                    eiv_geom = MultiLineString.from_ewkt(eiv[4])
-                    eiv_geom = eiv_geom.buffer(0.00001, 1)
-                    if isinstance(eiv_geom, Polygon):
-                        eiv_geom = MultiPolygon(eiv_geom)
-                else:
-                    eiv_geom = None
-            except GEOSException:
-                self._write_error(f"Malicious geometry on {eiv}")
-                return
-            geom = intervention.geometry or Geometry()
-            geom.geom = eiv_geom
-            geom.save()
-            #celery_update_parcels.delay(geom.id)
 
-            intervention.geometry = geom
+                intervention = self._migrate_intervention_geometry(intervention, eiv)
-            intervention.legal = intervention.legal or Legal.objects.create()
+                intervention = self._migrate_intervention_legal(intervention, eiv)
-            intervention.responsible = intervention.responsible or Responsibility.objects.create()
+                intervention = self._migrate_intervention_responsibility(intervention, eiv)
-            intervention.save()
+
+                intervention.save()
+                num_processed += 1