#132 Compensations

* WIP: adds CompensationMigrater
* moves more ksp-migration logic from InterventionMigrater to BaseMigrater
* improves comment card rendering

konova/management/commands/kspMigrater/base_migrater.py

@@ -1,6 +1,10 @@
 from abc import abstractmethod
 
 import psycopg2
+from django.contrib.gis.geos import GEOSException, MultiPolygon, Polygon, MultiPoint, MultiLineString
+from django.core.files.uploadedfile import UploadedFile
+
+from konova.models import Geometry
 
 
 class BaseMigrater:
@@ -30,3 +34,75 @@ class BaseMigrater:
     @abstractmethod
     def migrate(self):
         raise NotImplementedError("Must be implemented in subclass")
+
+    def _migrate_geometry(self, instance, db_result: tuple):
+        try:
+            # db_result[2] == polygon, db_result[3] == line, db_result[4] == point
+            if db_result[2] is not None:
+                db_result_geom = MultiPolygon.from_ewkt(db_result[2])
+            elif db_result[3] is not None:
+                db_result_geom = MultiLineString.from_ewkt(db_result[3])
+                db_result_geom = db_result_geom.buffer(0.00001, 1)
+                if isinstance(db_result_geom, Polygon):
+                    db_result_geom = MultiPolygon(db_result_geom)
+            elif db_result[4] is not None:
+                db_result_geom = MultiPoint.from_ewkt(db_result[4])
+                db_result_geom = db_result_geom.buffer(0.00001, 1)
+                if isinstance(db_result_geom, Polygon):
+                    db_result_geom = MultiPolygon(db_result_geom)
+            else:
+                db_result_geom = None
+        except GEOSException:
+            print(f"Malicious geometry on {db_result}")
+            return
+        geom = instance.geometry or Geometry()
+        geom.geom = db_result_geom
+        geom.save()
+        # celery_update_parcels.delay(geom.id)
+
+        instance.geometry = geom
+        return instance
+
+    def _migrate_documents(self, instance, document_cls, db_result):
+        db_result_identifier = f"'{db_result[0]}'"
+        tmp_cursor = self.db_connection.cursor()
+        tmp_cursor.execute(
+            'select '
+                'doc.pfad, '
+                'doc."Bemerkung", '
+                'doc."Datum" '
+            'from "OBJ_MASTER" om '
+                'left join "LINFOS" linf on om."GISPADID"=linf."GISPADID" '
+                'left join "Objektphotos" doc on linf."PKEY"=doc."FKEY" '
+            'where '
+                f'om."KENNUNG"={db_result_identifier} and '
+                'doc.pfad is not null '
+        )
+        doc_results = tmp_cursor.fetchall()
+        if len(doc_results) > 0:
+            for doc_result in doc_results:
+                doc_path = doc_result[0]
+                doc_comment = doc_result[1]
+                doc_date = doc_result[2]
+                with open(doc_path, encoding="latin1") as file:
+                    file = UploadedFile(file)
+                    doc_title = "Migrierte Datei"
+                    doc_date = doc_date or "1970-01-01"
+                    doc_exists = document_cls.objects.filter(
+                        instance=instance,
+                        title=doc_title,
+                        comment=doc_comment,
+                        date_of_creation=doc_date
+                    ).exists()
+                    if doc_exists:
+                        continue
+                    doc = document_cls.objects.create(
+                        title=doc_title,
+                        comment=doc_comment,
+                        file=file,
+                        date_of_creation=doc_date,
+                        instance=instance,
+                    )
+
+        tmp_cursor.close()
+        return instance

konova/management/commands/kspMigrater/compensation_migrater.py

@@ -0,0 +1,98 @@
+from django.core.exceptions import ObjectDoesNotExist
+from django.db import transaction
+
+from compensation.models import Compensation
+from intervention.models import Responsibility, Handler, Intervention
+from konova.management.commands.kspMigrater.base_migrater import BaseMigrater
+
+
+class CompensationMigrater(BaseMigrater):
+
+    def migrate(self):
+        cursor = self.db_connection.cursor()
+        empty_str = "''"
+        cursor.execute(
+            'select '
+                'om."KENNUNG", '
+                'linf."OBJBEZ", '
+                'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geomf.the_geom,4326)), 3))) as geomf, '
+                'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geoml.the_geom,4326)), 2))) as geoml, '
+                'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geomp.the_geom,4326)), 1))) as geomp, '
+                'linf."Bemerkung" '
+            'from "OBJ_MASTER" om '
+                'left join "LINFOS" linf on om."GISPADID"=linf."GISPADID" '
+                'left join kom on om."GISPADID"=kom.gispadid '
+                'left join geometry_f geomf on om."GISPADID"=geomf.gispadid '
+                'left join geometry_l geoml on om."GISPADID"=geoml.gispadid '
+                'left join geometry_p geomp on om."GISPADID"=geomp.gispadid '
+                'left join "Aufwertung" auf on om."GISPADID"=auf."GISPADID" '
+            'where '
+                'om."OKL"=7730080 and '
+                'om.archiv=false and '
+                f'(auf."Infos" is null or auf."Infos"={empty_str}) and '
+                'om.nicht_vollstaendig=0;'
+        )
+
+        all_koms = cursor.fetchall()
+        len_all_koms = len(all_koms)
+        num_processed = 0
+        print(f"Migrate KOMs to compensations...")
+        print(f"--Found {len_all_koms} entries. Process now...")
+        unsuccessfull_compensations = {}
+        for kom in all_koms:
+            if num_processed % 500 == 0:
+                print(f"----{num_processed}/{len_all_koms} processed")
+
+            with transaction.atomic():
+                kom_identifier = kom[0]
+                kom_title = kom[1]
+                kom_comment = kom[5]
+                compensation = Compensation.objects.get_or_create(
+                    identifier=kom_identifier
+                )[0]
+
+                compensation.title = kom_title
+                compensation.comment = kom_comment
+                compensation = self._migrate_geometry(compensation, kom)
+                compensation = self._migrate_responsibility(compensation, kom)
+                try:
+                    compensation = self._migrate_interventions_reference(compensation, kom)
+                    compensation.save()
+                except ObjectDoesNotExist:
+                    unsuccessfull_compensations[kom_identifier] = "EIV does not exist"
+                    print(f"{kom_identifier} does not have any intervention. ({len(unsuccessfull_compensations)} similar problems)")
+                num_processed += 1
+        print(f"The following KOMs could not be migrated: {unsuccessfull_compensations}")
+        cursor.close()
+
+    def _migrate_interventions_reference(self, compensation, kom):
+        kom_identifier = f"'{kom[0]}'"
+        tmp_cursor = self.db_connection.cursor()
+        tmp_cursor.execute(
+            'select '
+                'ref."REFERENZ" '
+            'from "OBJ_MASTER" om '
+            'left join "LINFOS" linf on om."GISPADID"=linf."GISPADID" '
+            'left join "REFERENZ" ref on om."GISPADID"=ref."GISPADID" '
+            'where '
+                'om."OKL"=7730080 and '
+                f'om."KENNUNG"={kom_identifier};'
+        )
+        eivs = tmp_cursor.fetchall()
+        len_eivs = len(eivs)
+        if len_eivs != 1:
+            raise AssertionError(f"{kom_identifier} has {len_eivs} EIVs!")
+        eiv = eivs[0]
+        try:
+            intervention = Intervention.objects.get(
+                identifier=eiv[0]
+            )
+        except ObjectDoesNotExist:
+            raise ObjectDoesNotExist(f"{kom_identifier} could not find migrated {eiv}")
+        compensation.intervention = intervention
+        tmp_cursor.close()
+        return compensation
+
+    def _migrate_responsibility(self, compensation, kom):
+        compensation.responsible = compensation.responsible or Responsibility.objects.create()
+        return compensation

konova/management/commands/kspMigrater/intervention_migrater.py

@@ -1,6 +1,4 @@
-from django.contrib.gis.geos import GEOSException, MultiPolygon, Polygon, MultiPoint, MultiLineString
 from django.core.exceptions import ObjectDoesNotExist
-from django.core.files.uploadedfile import UploadedFile
 from django.db import transaction
 
 from codelist.models import KonovaCode
@@ -9,39 +7,10 @@ from codelist.settings import CODELIST_LAW_ID, CODELIST_PROCESS_TYPE_ID, CODELIS
 from compensation.models import Payment
 from intervention.models import Intervention, InterventionDocument, Legal, Handler, Responsibility
 from konova.management.commands.kspMigrater.base_migrater import BaseMigrater
-from konova.models import Geometry
 
 
 class InterventionMigrater(BaseMigrater):
 
-    def _migrate_intervention_geometry(self, intervention:Intervention, eiv: tuple):
-        try:
-            # eiv[2] == polygon, eiv[3] == line, eiv[4] == point
-            if eiv[2] is not None:
-                eiv_geom = MultiPolygon.from_ewkt(eiv[2])
-            elif eiv[3] is not None:
-                eiv_geom = MultiLineString.from_ewkt(eiv[3])
-                eiv_geom = eiv_geom.buffer(0.00001, 1)
-                if isinstance(eiv_geom, Polygon):
-                    eiv_geom = MultiPolygon(eiv_geom)
-            elif eiv[4] is not None:
-                eiv_geom = MultiPoint.from_ewkt(eiv[4])
-                eiv_geom = eiv_geom.buffer(0.00001, 1)
-                if isinstance(eiv_geom, Polygon):
-                    eiv_geom = MultiPolygon(eiv_geom)
-            else:
-                eiv_geom = None
-        except GEOSException:
-            print(f"Malicious geometry on {eiv}")
-            return
-        geom = intervention.geometry or Geometry()
-        geom.geom = eiv_geom
-        geom.save()
-        # celery_update_parcels.delay(geom.id)
-
-        intervention.geometry = geom
-        return intervention
-
     def _migrate_intervention_responsibility(self, intervention, eiv):
         intervention.responsible = intervention.responsible or Responsibility.objects.create()
         intervention.responsible.handler = intervention.responsible.handler or Handler.objects.create()
@@ -142,50 +111,6 @@ class InterventionMigrater(BaseMigrater):
             intervention.payments.add(payment)
         return intervention
 
-    def _migrate_intervention_documents(self, intervention, eiv):
-        eiv_identifier = f"'{eiv[0]}'"
-        tmp_cursor = self.db_connection.cursor()
-        tmp_cursor.execute(
-            'select '
-                'doc.pfad, '
-                'doc."Bemerkung", '
-                'doc."Datum" '
-            'from "OBJ_MASTER" om '
-                'left join "LINFOS" linf on om."GISPADID"=linf."GISPADID" '
-                'left join "Objektphotos" doc on linf."PKEY"=doc."FKEY" '
-            'where '
-                f'om."KENNUNG"={eiv_identifier} and '
-                'doc.pfad is not null '
-        )
-        doc_results = tmp_cursor.fetchall()
-        if len(doc_results) > 0:
-            for doc_result in doc_results:
-                doc_path = doc_result[0]
-                doc_comment = doc_result[1]
-                doc_date = doc_result[2]
-                with open(doc_path, encoding="latin1") as file:
-                    file = UploadedFile(file)
-                    doc_title = "Migrierte Datei"
-                    doc_date = doc_date or "1970-01-01"
-                    doc_exists = InterventionDocument.objects.filter(
-                        instance=intervention,
-                        title=doc_title,
-                        comment=doc_comment,
-                        date_of_creation=doc_date
-                    ).exists()
-                    if doc_exists:
-                        continue
-                    doc = InterventionDocument.objects.create(
-                        title=doc_title,
-                        comment=doc_comment,
-                        file=file,
-                        date_of_creation=doc_date,
-                        instance=intervention,
-                    )
-
-        tmp_cursor.close()
-        return intervention
-
     def migrate(self):
         cursor = self.db_connection.cursor()
         cursor.execute(
@@ -240,11 +165,11 @@ class InterventionMigrater(BaseMigrater):
                 intervention.title = eiv[1]
                 intervention.comment = eiv_comment
 
-                intervention = self._migrate_intervention_geometry(intervention, eiv)
+                intervention = self._migrate_geometry(intervention, eiv)
                 intervention = self._migrate_intervention_legal(intervention, eiv)
                 intervention = self._migrate_intervention_responsibility(intervention, eiv)
                 intervention = self._migrate_intervention_payment(intervention, eiv)
-                intervention = self._migrate_intervention_documents(intervention, eiv)
+                intervention = self._migrate_documents(intervention, InterventionDocument, eiv)
 
                 intervention.save()
                 num_processed += 1

konova/management/commands/migrate_ksp_konova.py

@@ -1,3 +1,4 @@
+from konova.management.commands.kspMigrater.compensation_migrater import CompensationMigrater
 from konova.management.commands.kspMigrater.intervention_migrater import InterventionMigrater
 from konova.management.commands.setup import BaseKonovaCommand
 
@@ -13,8 +14,12 @@ class Command(BaseKonovaCommand):
 
     def handle(self, *args, **options):
         try:
-            intervention_migrater = InterventionMigrater(options)
+            migraters = [
-            intervention_migrater.migrate()
+                #InterventionMigrater(options),
+                CompensationMigrater(options),
+            ]
+            for migrater in migraters:
+                migrater.migrate()
 
         except KeyboardInterrupt:
             self._break_line()

konova/templates/konova/includes/comment_card.html

@@ -6,7 +6,7 @@
 
 
 {% if obj.comment %}
-<div class="w-100">
+<div class="col-sm-12">
     <div class="card mt-3">
         <div class="card-header rlp-gd">
             <div class="row">