From ac704bc930b942536d37ecf45aa3232aaf32c78e Mon Sep 17 00:00:00 2001 From: mpeltriaux Date: Tue, 15 Mar 2022 17:04:15 +0100 Subject: [PATCH] #132 WIP: EMA * WIP: adds first implementation for EMA migration * fixes bug where not found files would case migration to fail --- .../commands/kspMigrater/base_migrater.py | 42 ++++++------- .../kspMigrater/compensation_migrater.py | 60 ++++++++++++++++++- .../management/commands/migrate_ksp_konova.py | 5 +- 3 files changed, 84 insertions(+), 23 deletions(-) diff --git a/konova/management/commands/kspMigrater/base_migrater.py b/konova/management/commands/kspMigrater/base_migrater.py index e680a0e7..0053af41 100644 --- a/konova/management/commands/kspMigrater/base_migrater.py +++ b/konova/management/commands/kspMigrater/base_migrater.py @@ -84,25 +84,27 @@ class BaseMigrater: doc_path = doc_result[0] doc_comment = doc_result[1] doc_date = doc_result[2] - with open(doc_path, encoding="latin1") as file: - file = UploadedFile(file) - doc_title = "Migrierte Datei" - doc_date = doc_date or "1970-01-01" - doc_exists = document_cls.objects.filter( - instance=instance, - title=doc_title, - comment=doc_comment, - date_of_creation=doc_date - ).exists() - if doc_exists: - continue - doc = document_cls.objects.create( - title=doc_title, - comment=doc_comment, - file=file, - date_of_creation=doc_date, - instance=instance, - ) - + try: + with open(doc_path, encoding="latin1") as file: + file = UploadedFile(file) + doc_title = "Migrierte Datei" + doc_date = doc_date or "1970-01-01" + doc_exists = document_cls.objects.filter( + instance=instance, + title=doc_title, + comment=doc_comment, + date_of_creation=doc_date + ).exists() + if doc_exists: + continue + doc = document_cls.objects.create( + title=doc_title, + comment=doc_comment, + file=file, + date_of_creation=doc_date, + instance=instance, + ) + except FileNotFoundError: + print(f"------ !!! File not found: {doc_path}") tmp_cursor.close() return instance \ No newline at end of file diff --git a/konova/management/commands/kspMigrater/compensation_migrater.py b/konova/management/commands/kspMigrater/compensation_migrater.py index 69b4a7f3..a56e3649 100644 --- a/konova/management/commands/kspMigrater/compensation_migrater.py +++ b/konova/management/commands/kspMigrater/compensation_migrater.py @@ -5,6 +5,7 @@ from codelist.models import KonovaCode from codelist.settings import CODELIST_BIOTOPES_ID, CODELIST_BIOTOPES_EXTRA_CODES_ID, \ CODELIST_COMPENSATION_ACTION_DETAIL_ID, CODELIST_COMPENSATION_ACTION_ID from compensation.models import Compensation, CompensationState, CompensationAction, UnitChoices, CompensationDocument +from ema.models import Ema, EmaDocument from intervention.models import Responsibility, Handler, Intervention from konova.management.commands.kspMigrater.base_migrater import BaseMigrater from konova.models import Deadline, DeadlineType @@ -408,4 +409,61 @@ class CompensationMigrater(BaseMigrater): tmp_cursor.close() - return compensation \ No newline at end of file + return compensation + + +class EmaMigrater(CompensationMigrater): + + def migrate(self): + cursor = self.db_connection.cursor() + cursor.execute( + 'select ' + 'om."KENNUNG", ' + 'linf."OBJBEZ", ' + 'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geomf.the_geom,4326)), 3))) as geomf, ' + 'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geoml.the_geom,4326)), 2))) as geoml, ' + 'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geomp.the_geom,4326)), 1))) as geomp, ' + 'linf."Bemerkung" ' + 'from "OBJ_MASTER" om ' + 'left join "LINFOS" linf on om."GISPADID"=linf."GISPADID" ' + 'left join kom on om."GISPADID"=kom.gispadid ' + 'left join geometry_f geomf on om."GISPADID"=geomf.gispadid ' + 'left join geometry_l geoml on om."GISPADID"=geoml.gispadid ' + 'left join geometry_p geomp on om."GISPADID"=geomp.gispadid ' + 'left join "Aufwertung" auf on om."GISPADID"=auf."GISPADID" ' + 'where ' + 'om."OKL"=7730090 and ' + 'om.archiv=false and ' + 'om.nicht_vollstaendig=0' + ) + + all_emas = cursor.fetchall() + len_all_emas = len(all_emas) + num_processed = 0 + print(f"Migrate EMAs to emas...") + print(f"--Found {len_all_emas} entries. Process now...") + for ema in all_emas: + if num_processed % 500 == 0: + print(f"----{num_processed}/{len_all_emas} processed") + + with transaction.atomic(): + ema_identifier = ema[0] + ema_title = ema[1] + ema_comment = ema[5] + ema_obj = Ema.objects.get_or_create( + identifier=ema_identifier + )[0] + + ema_obj.title = ema_title + ema_obj.comment = ema_comment + ema_obj = self._migrate_geometry(ema_obj, ema) + ema_obj = self._migrate_responsibility(ema_obj, ema) + ema_obj = self._migrate_compensation_type(ema_obj, ema) + ema_obj = self._migrate_states(ema_obj, ema) + ema_obj = self._migrate_deadlines(ema_obj, ema) + ema_obj = self._migrate_actions(ema_obj, ema) + ema_obj = self._migrate_documents(ema_obj, EmaDocument, ema) + ema_obj.save() + + num_processed += 1 + cursor.close() diff --git a/konova/management/commands/migrate_ksp_konova.py b/konova/management/commands/migrate_ksp_konova.py index f6528ff5..2ddf4591 100644 --- a/konova/management/commands/migrate_ksp_konova.py +++ b/konova/management/commands/migrate_ksp_konova.py @@ -1,4 +1,4 @@ -from konova.management.commands.kspMigrater.compensation_migrater import CompensationMigrater +from konova.management.commands.kspMigrater.compensation_migrater import CompensationMigrater, EmaMigrater from konova.management.commands.kspMigrater.intervention_migrater import InterventionMigrater from konova.management.commands.setup import BaseKonovaCommand @@ -16,7 +16,8 @@ class Command(BaseKonovaCommand): try: migraters = [ #InterventionMigrater(options), - CompensationMigrater(options), + #CompensationMigrater(options), + EmaMigrater(options), ] for migrater in migraters: migrater.migrate()