From d21baf11f30f7481ef8ad577bd02b7d0870731a3 Mon Sep 17 00:00:00 2001 From: mpeltriaux Date: Mon, 14 Mar 2022 15:58:02 +0100 Subject: [PATCH] #132 Compensations * WIP: adds CompensationMigrater * moves more ksp-migration logic from InterventionMigrater to BaseMigrater * improves comment card rendering --- .../commands/kspMigrater/base_migrater.py | 76 ++++++++++++++ .../kspMigrater/compensation_migrater.py | 98 +++++++++++++++++++ .../kspMigrater/intervention_migrater.py | 79 +-------------- .../management/commands/migrate_ksp_konova.py | 9 +- .../konova/includes/comment_card.html | 2 +- 5 files changed, 184 insertions(+), 80 deletions(-) create mode 100644 konova/management/commands/kspMigrater/compensation_migrater.py diff --git a/konova/management/commands/kspMigrater/base_migrater.py b/konova/management/commands/kspMigrater/base_migrater.py index 982250ef..e680a0e7 100644 --- a/konova/management/commands/kspMigrater/base_migrater.py +++ b/konova/management/commands/kspMigrater/base_migrater.py @@ -1,6 +1,10 @@ from abc import abstractmethod import psycopg2 +from django.contrib.gis.geos import GEOSException, MultiPolygon, Polygon, MultiPoint, MultiLineString +from django.core.files.uploadedfile import UploadedFile + +from konova.models import Geometry class BaseMigrater: @@ -30,3 +34,75 @@ class BaseMigrater: @abstractmethod def migrate(self): raise NotImplementedError("Must be implemented in subclass") + + def _migrate_geometry(self, instance, db_result: tuple): + try: + # db_result[2] == polygon, db_result[3] == line, db_result[4] == point + if db_result[2] is not None: + db_result_geom = MultiPolygon.from_ewkt(db_result[2]) + elif db_result[3] is not None: + db_result_geom = MultiLineString.from_ewkt(db_result[3]) + db_result_geom = db_result_geom.buffer(0.00001, 1) + if isinstance(db_result_geom, Polygon): + db_result_geom = MultiPolygon(db_result_geom) + elif db_result[4] is not None: + db_result_geom = MultiPoint.from_ewkt(db_result[4]) + db_result_geom = db_result_geom.buffer(0.00001, 1) + if isinstance(db_result_geom, Polygon): + db_result_geom = MultiPolygon(db_result_geom) + else: + db_result_geom = None + except GEOSException: + print(f"Malicious geometry on {db_result}") + return + geom = instance.geometry or Geometry() + geom.geom = db_result_geom + geom.save() + # celery_update_parcels.delay(geom.id) + + instance.geometry = geom + return instance + + def _migrate_documents(self, instance, document_cls, db_result): + db_result_identifier = f"'{db_result[0]}'" + tmp_cursor = self.db_connection.cursor() + tmp_cursor.execute( + 'select ' + 'doc.pfad, ' + 'doc."Bemerkung", ' + 'doc."Datum" ' + 'from "OBJ_MASTER" om ' + 'left join "LINFOS" linf on om."GISPADID"=linf."GISPADID" ' + 'left join "Objektphotos" doc on linf."PKEY"=doc."FKEY" ' + 'where ' + f'om."KENNUNG"={db_result_identifier} and ' + 'doc.pfad is not null ' + ) + doc_results = tmp_cursor.fetchall() + if len(doc_results) > 0: + for doc_result in doc_results: + doc_path = doc_result[0] + doc_comment = doc_result[1] + doc_date = doc_result[2] + with open(doc_path, encoding="latin1") as file: + file = UploadedFile(file) + doc_title = "Migrierte Datei" + doc_date = doc_date or "1970-01-01" + doc_exists = document_cls.objects.filter( + instance=instance, + title=doc_title, + comment=doc_comment, + date_of_creation=doc_date + ).exists() + if doc_exists: + continue + doc = document_cls.objects.create( + title=doc_title, + comment=doc_comment, + file=file, + date_of_creation=doc_date, + instance=instance, + ) + + tmp_cursor.close() + return instance \ No newline at end of file diff --git a/konova/management/commands/kspMigrater/compensation_migrater.py b/konova/management/commands/kspMigrater/compensation_migrater.py new file mode 100644 index 00000000..c2113650 --- /dev/null +++ b/konova/management/commands/kspMigrater/compensation_migrater.py @@ -0,0 +1,98 @@ +from django.core.exceptions import ObjectDoesNotExist +from django.db import transaction + +from compensation.models import Compensation +from intervention.models import Responsibility, Handler, Intervention +from konova.management.commands.kspMigrater.base_migrater import BaseMigrater + + +class CompensationMigrater(BaseMigrater): + + def migrate(self): + cursor = self.db_connection.cursor() + empty_str = "''" + cursor.execute( + 'select ' + 'om."KENNUNG", ' + 'linf."OBJBEZ", ' + 'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geomf.the_geom,4326)), 3))) as geomf, ' + 'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geoml.the_geom,4326)), 2))) as geoml, ' + 'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geomp.the_geom,4326)), 1))) as geomp, ' + 'linf."Bemerkung" ' + 'from "OBJ_MASTER" om ' + 'left join "LINFOS" linf on om."GISPADID"=linf."GISPADID" ' + 'left join kom on om."GISPADID"=kom.gispadid ' + 'left join geometry_f geomf on om."GISPADID"=geomf.gispadid ' + 'left join geometry_l geoml on om."GISPADID"=geoml.gispadid ' + 'left join geometry_p geomp on om."GISPADID"=geomp.gispadid ' + 'left join "Aufwertung" auf on om."GISPADID"=auf."GISPADID" ' + 'where ' + 'om."OKL"=7730080 and ' + 'om.archiv=false and ' + f'(auf."Infos" is null or auf."Infos"={empty_str}) and ' + 'om.nicht_vollstaendig=0;' + ) + + all_koms = cursor.fetchall() + len_all_koms = len(all_koms) + num_processed = 0 + print(f"Migrate KOMs to compensations...") + print(f"--Found {len_all_koms} entries. Process now...") + unsuccessfull_compensations = {} + for kom in all_koms: + if num_processed % 500 == 0: + print(f"----{num_processed}/{len_all_koms} processed") + + with transaction.atomic(): + kom_identifier = kom[0] + kom_title = kom[1] + kom_comment = kom[5] + compensation = Compensation.objects.get_or_create( + identifier=kom_identifier + )[0] + + compensation.title = kom_title + compensation.comment = kom_comment + compensation = self._migrate_geometry(compensation, kom) + compensation = self._migrate_responsibility(compensation, kom) + try: + compensation = self._migrate_interventions_reference(compensation, kom) + compensation.save() + except ObjectDoesNotExist: + unsuccessfull_compensations[kom_identifier] = "EIV does not exist" + print(f"{kom_identifier} does not have any intervention. ({len(unsuccessfull_compensations)} similar problems)") + num_processed += 1 + print(f"The following KOMs could not be migrated: {unsuccessfull_compensations}") + cursor.close() + + def _migrate_interventions_reference(self, compensation, kom): + kom_identifier = f"'{kom[0]}'" + tmp_cursor = self.db_connection.cursor() + tmp_cursor.execute( + 'select ' + 'ref."REFERENZ" ' + 'from "OBJ_MASTER" om ' + 'left join "LINFOS" linf on om."GISPADID"=linf."GISPADID" ' + 'left join "REFERENZ" ref on om."GISPADID"=ref."GISPADID" ' + 'where ' + 'om."OKL"=7730080 and ' + f'om."KENNUNG"={kom_identifier};' + ) + eivs = tmp_cursor.fetchall() + len_eivs = len(eivs) + if len_eivs != 1: + raise AssertionError(f"{kom_identifier} has {len_eivs} EIVs!") + eiv = eivs[0] + try: + intervention = Intervention.objects.get( + identifier=eiv[0] + ) + except ObjectDoesNotExist: + raise ObjectDoesNotExist(f"{kom_identifier} could not find migrated {eiv}") + compensation.intervention = intervention + tmp_cursor.close() + return compensation + + def _migrate_responsibility(self, compensation, kom): + compensation.responsible = compensation.responsible or Responsibility.objects.create() + return compensation \ No newline at end of file diff --git a/konova/management/commands/kspMigrater/intervention_migrater.py b/konova/management/commands/kspMigrater/intervention_migrater.py index de539798..4291952e 100644 --- a/konova/management/commands/kspMigrater/intervention_migrater.py +++ b/konova/management/commands/kspMigrater/intervention_migrater.py @@ -1,6 +1,4 @@ -from django.contrib.gis.geos import GEOSException, MultiPolygon, Polygon, MultiPoint, MultiLineString from django.core.exceptions import ObjectDoesNotExist -from django.core.files.uploadedfile import UploadedFile from django.db import transaction from codelist.models import KonovaCode @@ -9,39 +7,10 @@ from codelist.settings import CODELIST_LAW_ID, CODELIST_PROCESS_TYPE_ID, CODELIS from compensation.models import Payment from intervention.models import Intervention, InterventionDocument, Legal, Handler, Responsibility from konova.management.commands.kspMigrater.base_migrater import BaseMigrater -from konova.models import Geometry class InterventionMigrater(BaseMigrater): - def _migrate_intervention_geometry(self, intervention:Intervention, eiv: tuple): - try: - # eiv[2] == polygon, eiv[3] == line, eiv[4] == point - if eiv[2] is not None: - eiv_geom = MultiPolygon.from_ewkt(eiv[2]) - elif eiv[3] is not None: - eiv_geom = MultiLineString.from_ewkt(eiv[3]) - eiv_geom = eiv_geom.buffer(0.00001, 1) - if isinstance(eiv_geom, Polygon): - eiv_geom = MultiPolygon(eiv_geom) - elif eiv[4] is not None: - eiv_geom = MultiPoint.from_ewkt(eiv[4]) - eiv_geom = eiv_geom.buffer(0.00001, 1) - if isinstance(eiv_geom, Polygon): - eiv_geom = MultiPolygon(eiv_geom) - else: - eiv_geom = None - except GEOSException: - print(f"Malicious geometry on {eiv}") - return - geom = intervention.geometry or Geometry() - geom.geom = eiv_geom - geom.save() - # celery_update_parcels.delay(geom.id) - - intervention.geometry = geom - return intervention - def _migrate_intervention_responsibility(self, intervention, eiv): intervention.responsible = intervention.responsible or Responsibility.objects.create() intervention.responsible.handler = intervention.responsible.handler or Handler.objects.create() @@ -142,50 +111,6 @@ class InterventionMigrater(BaseMigrater): intervention.payments.add(payment) return intervention - def _migrate_intervention_documents(self, intervention, eiv): - eiv_identifier = f"'{eiv[0]}'" - tmp_cursor = self.db_connection.cursor() - tmp_cursor.execute( - 'select ' - 'doc.pfad, ' - 'doc."Bemerkung", ' - 'doc."Datum" ' - 'from "OBJ_MASTER" om ' - 'left join "LINFOS" linf on om."GISPADID"=linf."GISPADID" ' - 'left join "Objektphotos" doc on linf."PKEY"=doc."FKEY" ' - 'where ' - f'om."KENNUNG"={eiv_identifier} and ' - 'doc.pfad is not null ' - ) - doc_results = tmp_cursor.fetchall() - if len(doc_results) > 0: - for doc_result in doc_results: - doc_path = doc_result[0] - doc_comment = doc_result[1] - doc_date = doc_result[2] - with open(doc_path, encoding="latin1") as file: - file = UploadedFile(file) - doc_title = "Migrierte Datei" - doc_date = doc_date or "1970-01-01" - doc_exists = InterventionDocument.objects.filter( - instance=intervention, - title=doc_title, - comment=doc_comment, - date_of_creation=doc_date - ).exists() - if doc_exists: - continue - doc = InterventionDocument.objects.create( - title=doc_title, - comment=doc_comment, - file=file, - date_of_creation=doc_date, - instance=intervention, - ) - - tmp_cursor.close() - return intervention - def migrate(self): cursor = self.db_connection.cursor() cursor.execute( @@ -240,11 +165,11 @@ class InterventionMigrater(BaseMigrater): intervention.title = eiv[1] intervention.comment = eiv_comment - intervention = self._migrate_intervention_geometry(intervention, eiv) + intervention = self._migrate_geometry(intervention, eiv) intervention = self._migrate_intervention_legal(intervention, eiv) intervention = self._migrate_intervention_responsibility(intervention, eiv) intervention = self._migrate_intervention_payment(intervention, eiv) - intervention = self._migrate_intervention_documents(intervention, eiv) + intervention = self._migrate_documents(intervention, InterventionDocument, eiv) intervention.save() num_processed += 1 diff --git a/konova/management/commands/migrate_ksp_konova.py b/konova/management/commands/migrate_ksp_konova.py index 927f98bd..f6528ff5 100644 --- a/konova/management/commands/migrate_ksp_konova.py +++ b/konova/management/commands/migrate_ksp_konova.py @@ -1,3 +1,4 @@ +from konova.management.commands.kspMigrater.compensation_migrater import CompensationMigrater from konova.management.commands.kspMigrater.intervention_migrater import InterventionMigrater from konova.management.commands.setup import BaseKonovaCommand @@ -13,8 +14,12 @@ class Command(BaseKonovaCommand): def handle(self, *args, **options): try: - intervention_migrater = InterventionMigrater(options) - intervention_migrater.migrate() + migraters = [ + #InterventionMigrater(options), + CompensationMigrater(options), + ] + for migrater in migraters: + migrater.migrate() except KeyboardInterrupt: self._break_line() diff --git a/konova/templates/konova/includes/comment_card.html b/konova/templates/konova/includes/comment_card.html index a0d72deb..51a5667a 100644 --- a/konova/templates/konova/includes/comment_card.html +++ b/konova/templates/konova/includes/comment_card.html @@ -6,7 +6,7 @@ {% if obj.comment %} -
+