From 0c1ca4d173da32e865a045bd7cf1b95d8977c91d Mon Sep 17 00:00:00 2001 From: mpeltriaux Date: Thu, 17 Mar 2022 16:33:06 +0100 Subject: [PATCH] #132 WIP Migrate OEKs * WIP: adds oek migrations * refactors ema migration into own file --- .../kspMigrater/compensation_migrater.py | 187 ----------------- .../kspMigrater/eco_account_migrater.py | 123 +++++++++++ .../commands/kspMigrater/ema_migrater.py | 196 ++++++++++++++++++ .../management/commands/migrate_ksp_konova.py | 7 +- 4 files changed, 324 insertions(+), 189 deletions(-) create mode 100644 konova/management/commands/kspMigrater/eco_account_migrater.py create mode 100644 konova/management/commands/kspMigrater/ema_migrater.py diff --git a/konova/management/commands/kspMigrater/compensation_migrater.py b/konova/management/commands/kspMigrater/compensation_migrater.py index 3f4998cb..9e70a36f 100644 --- a/konova/management/commands/kspMigrater/compensation_migrater.py +++ b/konova/management/commands/kspMigrater/compensation_migrater.py @@ -499,190 +499,3 @@ class CompensationMigrater(BaseMigrater): tmp_cursor.close() return compensation - - -class EmaMigrater(CompensationMigrater): - - def migrate(self): - cursor = self.db_connection.cursor() - cursor.execute( - 'select ' - 'om."KENNUNG", ' - 'linf."OBJBEZ", ' - 'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geomf.the_geom,4326)), 3))) as geomf, ' - 'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geoml.the_geom,4326)), 2))) as geoml, ' - 'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geomp.the_geom,4326)), 1))) as geomp, ' - 'linf."Bemerkung" ' - 'from "OBJ_MASTER" om ' - 'left join "LINFOS" linf on om."GISPADID"=linf."GISPADID" ' - 'left join kom on om."GISPADID"=kom.gispadid ' - 'left join geometry_f geomf on om."GISPADID"=geomf.gispadid ' - 'left join geometry_l geoml on om."GISPADID"=geoml.gispadid ' - 'left join geometry_p geomp on om."GISPADID"=geomp.gispadid ' - 'left join "Aufwertung" auf on om."GISPADID"=auf."GISPADID" ' - 'where ' - 'om."OKL"=7730090 and ' - 'om.archiv=false and ' - 'om.nicht_vollstaendig=0' - ) - - all_emas = cursor.fetchall() - len_all_emas = len(all_emas) - num_processed = 0 - print(f"Migrate EMAs to emas...") - print(f"--Found {len_all_emas} entries. Process now...") - for ema in all_emas: - if num_processed % 500 == 0: - print(f"----{num_processed}/{len_all_emas} processed") - - with transaction.atomic(): - ema_identifier = ema[0] - ema_title = ema[1] - ema_comment = ema[5] - ema_obj = Ema.objects.get_or_create( - identifier=ema_identifier - )[0] - - ema_obj.title = ema_title - ema_obj.comment = ema_comment - ema_obj = self._migrate_geometry(ema_obj, ema) - ema_obj = self._migrate_responsibility(ema_obj, ema) - ema_obj = self._migrate_compensation_type(ema_obj, ema) - ema_obj = self._migrate_states(ema_obj, ema) - ema_obj = self._migrate_deadlines(ema_obj, ema) - ema_obj = self._migrate_action_control_deadlines(ema_obj, ema) - ema_obj = self._migrate_actions(ema_obj, ema) - ema_obj = self._migrate_finance_volume_to_comment(ema_obj, ema) - ema_obj = self._migrate_documents(ema_obj, EmaDocument, ema) - ema_obj.save() - - num_processed += 1 - cursor.close() - - def _migrate_deadlines(self, ema_obj, ema_result): - ema_obj.deadlines.all().delete() - ema_identifier = f"'{ema_result[0]}'" - tmp_cursor = self.db_connection.cursor() - tmp_cursor.execute( - 'select ' - 't."Terminart", ' - 't."K_Termin"::date ' - 'from "OBJ_MASTER" om ' - 'left join "Termine" t on om."GISPADID"=t."GISPADID" ' - 'where ' - f'om."KENNUNG"={ema_identifier}' - ) - db_results = tmp_cursor.fetchall() - for result in db_results: - deadline_type = result[0] - deadline_comment = None - if deadline_type == 708166: - deadline_comment = "Wiedervorlage" - elif deadline_type == 708163: - deadline_comment = "Projektbeginn" - deadline_date = result[1] - if deadline_date is None: - # Useless data - continue - - try: - deadline = ema_obj.deadlines.get( - type=DeadlineType.OTHER, - date=deadline_date, - comment=deadline_comment - ) - except ObjectDoesNotExist: - deadline = Deadline.objects.create( - type=DeadlineType.OTHER, - date=deadline_date, - comment=deadline_comment - ) - ema_obj.deadlines.add(deadline) - - return ema_obj - - def _migrate_responsibility(self, ema_obj, ema_result): - ema_identifier = f"'{ema_result[0]}'" - tmp_cursor = self.db_connection.cursor() - tmp_cursor.execute( - 'select ' - 'adr."adr_pruef" as ets, ' - 'linf."AZ", ' - 'adr.behoerde, ' - 'adr.angaben ' - 'from "OBJ_MASTER" om ' - 'left join "LINFOS" linf on om."GISPADID"=linf."GISPADID" ' - 'left join adressrolle adr on adr."GISPADID"=om."GISPADID" ' - 'where ' - f'om."KENNUNG"={ema_identifier} ' - ) - db_results = tmp_cursor.fetchall() - if len(db_results) != 1: - raise AssertionError(f"{ema_identifier} has invalid responsibilities: {db_results}") - - db_results = db_results[0] - cons_office_code = db_results[0] - cons_file_number = db_results[1] - handler_type = db_results[2] - handler_detail = db_results[3] - - responsible = ema_obj.responsible or Responsibility.objects.create() - try: - conservation_office = KonovaCode.objects.get( - atom_id=cons_office_code, - is_selectable=True, - is_archived=False, - code_lists__in=[CODELIST_CONSERVATION_OFFICE_ID] - ) - except ObjectDoesNotExist: - raise ObjectDoesNotExist(f"{ema_identifier}, {db_results}") - try: - handler_type = KonovaCode.objects.get( - atom_id=handler_type, - is_selectable=True, - is_archived=False, - code_lists__in=[CODELIST_COMPENSATION_HANDLER_ID] - ) - except ObjectDoesNotExist: - handler_type = None - responsible.conservation_file_number = db_results[1] - - ema_obj.responsible.conservation_office = conservation_office - ema_obj.responsible.conservation_file_number = cons_file_number - handler = ema_obj.responsible.handler or Handler.objects.create() - handler.type = handler_type - handler.detail = handler_detail - ema_obj.responsible.handler = handler - ema_obj.responsible.handler.save() - ema_obj.responsible.save() - - tmp_cursor.close() - return ema_obj - - def _migrate_finance_volume_to_comment(self, ema_obj, ema_result): - ema_identifier = f"'{ema_result[0]}'" - tmp_cursor = self.db_connection.cursor() - tmp_cursor.execute( - 'select ' - 'b.datum, ' - 'b.hoehe ' - 'from "OBJ_MASTER" om ' - 'left join bewilligung b on om."GISPADID"=b.gispadid ' - 'where ' - f'om."KENNUNG"={ema_identifier} ' - ) - db_results = tmp_cursor.fetchall() - for result in db_results: - payment_date = result[0] - payment_amount = result[1] - - comment_extra = f"\n\nFinanzierung bewilligt am {formats.localize(payment_date, use_l10n=True)} in Höhe von {formats.localize(payment_amount, use_l10n=True)} €" - comment = ema_obj.comment or "" - if comment_extra in comment: - # skip - continue - comment += comment_extra - ema_obj.comment = comment - - tmp_cursor.close() - return ema_obj \ No newline at end of file diff --git a/konova/management/commands/kspMigrater/eco_account_migrater.py b/konova/management/commands/kspMigrater/eco_account_migrater.py new file mode 100644 index 00000000..013f49cd --- /dev/null +++ b/konova/management/commands/kspMigrater/eco_account_migrater.py @@ -0,0 +1,123 @@ +from django.core.exceptions import ObjectDoesNotExist +from django.db import transaction + +from codelist.models import KonovaCode +from codelist.settings import CODELIST_CONSERVATION_OFFICE_ID, CODELIST_COMPENSATION_HANDLER_ID +from compensation.models import EcoAccount, EcoAccountDocument +from intervention.models import Responsibility, Handler +from konova.management.commands.kspMigrater.compensation_migrater import CompensationMigrater + + +class EcoAccountMigrater(CompensationMigrater): + + def migrate(self): + cursor = self.db_connection.cursor() + cursor.execute( + 'select ' + 'om."KENNUNG", ' + 'linf."OBJBEZ", ' + 'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geomf.the_geom,4326)), 3))) as geomf, ' + 'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geoml.the_geom,4326)), 2))) as geoml, ' + 'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geomp.the_geom,4326)), 1))) as geomp, ' + 'linf."Bemerkung" ' + 'from "OBJ_MASTER" om ' + 'left join "LINFOS" linf on om."GISPADID"=linf."GISPADID" ' + 'left join kom on om."GISPADID"=kom.gispadid ' + 'left join geometry_f geomf on om."GISPADID"=geomf.gispadid ' + 'left join geometry_l geoml on om."GISPADID"=geoml.gispadid ' + 'left join geometry_p geomp on om."GISPADID"=geomp.gispadid ' + 'left join "Aufwertung" auf on om."GISPADID"=auf."GISPADID" ' + 'where ' + 'om."OKL"=7730081 and ' + 'om.archiv=false and ' + 'om.nicht_vollstaendig=0' + ) + + all_oeks = cursor.fetchall() + len_all_oeks = len(all_oeks) + num_processed = 0 + print(f"Migrate OEKs to ecoaccounts...") + print(f"--Found {len_all_oeks} entries. Process now...") + for oek in all_oeks: + if num_processed % 500 == 0: + print(f"----{num_processed}/{len_all_oeks} processed") + + with transaction.atomic(): + oek_identifier = oek[0] + oek_title = oek[1] + oek_comment = oek[5] + eco_account = EcoAccount.objects.get_or_create( + identifier=oek_identifier + )[0] + + eco_account.title = oek_title + eco_account.comment = oek_comment + eco_account = self._migrate_geometry(eco_account, oek) + eco_account = self._migrate_responsibility(eco_account, oek) + eco_account = self._migrate_states(eco_account, oek) + eco_account = self._migrate_deadlines(eco_account, oek) + eco_account = self._migrate_action_control_deadlines(eco_account, oek) + eco_account = self._migrate_actions(eco_account, oek) + eco_account = self._migrate_documents(eco_account, EcoAccountDocument, oek) + eco_account.save() + + num_processed += 1 + cursor.close() + + def _migrate_responsibility(self, eco_account, oek): + acc_identifier = f"'{oek[0]}'" + tmp_cursor = self.db_connection.cursor() + tmp_cursor.execute( + 'select ' + 'adr."adr_pruef" as ets, ' + 'linf."AZ", ' + 'oek.traeger, ' + 'oek.bemerkungtraeger ' + 'from "OBJ_MASTER" om ' + 'left join "LINFOS" linf on om."GISPADID"=linf."GISPADID" ' + 'left join adressrolle adr on adr."GISPADID"=om."GISPADID" ' + 'left join oek on om."GISPADID"=oek.gispadid ' + 'where ' + f'om."KENNUNG"={acc_identifier}' + ) + db_results = tmp_cursor.fetchall() + if len(db_results) != 1: + raise AssertionError(f"{acc_identifier} has invalid responsibilities: {db_results}") + + db_results = db_results[0] + cons_office_code = db_results[0] + cons_file_number = db_results[1] + handler_type = db_results[2] + handler_detail = db_results[3] + + eco_account.responsible = eco_account.responsible or Responsibility.objects.create() + try: + conservation_office = KonovaCode.objects.get( + atom_id=cons_office_code, + is_selectable=True, + is_archived=False, + code_lists__in=[CODELIST_CONSERVATION_OFFICE_ID] + ) + except ObjectDoesNotExist: + raise ObjectDoesNotExist(f"{acc_identifier}, {db_results}") + try: + handler_type = KonovaCode.objects.get( + atom_id=handler_type, + is_selectable=True, + is_archived=False, + code_lists__in=[CODELIST_COMPENSATION_HANDLER_ID] + ) + except ObjectDoesNotExist: + handler_type = None + + eco_account.responsible.conservation_office = conservation_office + eco_account.responsible.conservation_file_number = cons_file_number + handler = eco_account.responsible.handler or Handler.objects.create() + handler.type = handler_type + handler.detail = handler_detail + eco_account.responsible.handler = handler + eco_account.responsible.handler.save() + eco_account.responsible.save() + + tmp_cursor.close() + return eco_account \ No newline at end of file diff --git a/konova/management/commands/kspMigrater/ema_migrater.py b/konova/management/commands/kspMigrater/ema_migrater.py new file mode 100644 index 00000000..c523f12f --- /dev/null +++ b/konova/management/commands/kspMigrater/ema_migrater.py @@ -0,0 +1,196 @@ +from django.core.exceptions import ObjectDoesNotExist +from django.db import transaction +from django.utils import formats + +from codelist.models import KonovaCode +from codelist.settings import CODELIST_COMPENSATION_HANDLER_ID, CODELIST_CONSERVATION_OFFICE_ID +from ema.models import EmaDocument, Ema +from intervention.models import Handler, Responsibility +from konova.management.commands.kspMigrater.compensation_migrater import CompensationMigrater +from konova.models import DeadlineType, Deadline + + +class EmaMigrater(CompensationMigrater): + + def migrate(self): + cursor = self.db_connection.cursor() + cursor.execute( + 'select ' + 'om."KENNUNG", ' + 'linf."OBJBEZ", ' + 'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geomf.the_geom,4326)), 3))) as geomf, ' + 'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geoml.the_geom,4326)), 2))) as geoml, ' + 'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geomp.the_geom,4326)), 1))) as geomp, ' + 'linf."Bemerkung" ' + 'from "OBJ_MASTER" om ' + 'left join "LINFOS" linf on om."GISPADID"=linf."GISPADID" ' + 'left join kom on om."GISPADID"=kom.gispadid ' + 'left join geometry_f geomf on om."GISPADID"=geomf.gispadid ' + 'left join geometry_l geoml on om."GISPADID"=geoml.gispadid ' + 'left join geometry_p geomp on om."GISPADID"=geomp.gispadid ' + 'left join "Aufwertung" auf on om."GISPADID"=auf."GISPADID" ' + 'where ' + 'om."OKL"=7730090 and ' + 'om.archiv=false and ' + 'om.nicht_vollstaendig=0' + ) + + all_emas = cursor.fetchall() + len_all_emas = len(all_emas) + num_processed = 0 + print(f"Migrate EMAs to emas...") + print(f"--Found {len_all_emas} entries. Process now...") + for ema in all_emas: + if num_processed % 500 == 0: + print(f"----{num_processed}/{len_all_emas} processed") + + with transaction.atomic(): + ema_identifier = ema[0] + ema_title = ema[1] + ema_comment = ema[5] + ema_obj = Ema.objects.get_or_create( + identifier=ema_identifier + )[0] + + ema_obj.title = ema_title + ema_obj.comment = ema_comment + ema_obj = self._migrate_geometry(ema_obj, ema) + ema_obj = self._migrate_responsibility(ema_obj, ema) + ema_obj = self._migrate_compensation_type(ema_obj, ema) + ema_obj = self._migrate_states(ema_obj, ema) + ema_obj = self._migrate_deadlines(ema_obj, ema) + ema_obj = self._migrate_action_control_deadlines(ema_obj, ema) + ema_obj = self._migrate_actions(ema_obj, ema) + ema_obj = self._migrate_finance_volume_to_comment(ema_obj, ema) + ema_obj = self._migrate_documents(ema_obj, EmaDocument, ema) + ema_obj.save() + + num_processed += 1 + cursor.close() + + def _migrate_deadlines(self, ema_obj, ema_result): + ema_obj.deadlines.all().delete() + ema_identifier = f"'{ema_result[0]}'" + tmp_cursor = self.db_connection.cursor() + tmp_cursor.execute( + 'select ' + 't."Terminart", ' + 't."K_Termin"::date ' + 'from "OBJ_MASTER" om ' + 'left join "Termine" t on om."GISPADID"=t."GISPADID" ' + 'where ' + f'om."KENNUNG"={ema_identifier}' + ) + db_results = tmp_cursor.fetchall() + for result in db_results: + deadline_type = result[0] + deadline_comment = None + if deadline_type == 708166: + deadline_comment = "Wiedervorlage" + elif deadline_type == 708163: + deadline_comment = "Projektbeginn" + deadline_date = result[1] + if deadline_date is None: + # Useless data + continue + + try: + deadline = ema_obj.deadlines.get( + type=DeadlineType.OTHER, + date=deadline_date, + comment=deadline_comment + ) + except ObjectDoesNotExist: + deadline = Deadline.objects.create( + type=DeadlineType.OTHER, + date=deadline_date, + comment=deadline_comment + ) + ema_obj.deadlines.add(deadline) + + return ema_obj + + def _migrate_responsibility(self, ema_obj, ema_result): + ema_identifier = f"'{ema_result[0]}'" + tmp_cursor = self.db_connection.cursor() + tmp_cursor.execute( + 'select ' + 'adr."adr_pruef" as ets, ' + 'linf."AZ", ' + 'adr.behoerde, ' + 'adr.angaben ' + 'from "OBJ_MASTER" om ' + 'left join "LINFOS" linf on om."GISPADID"=linf."GISPADID" ' + 'left join adressrolle adr on adr."GISPADID"=om."GISPADID" ' + 'where ' + f'om."KENNUNG"={ema_identifier} ' + ) + db_results = tmp_cursor.fetchall() + if len(db_results) != 1: + raise AssertionError(f"{ema_identifier} has invalid responsibilities: {db_results}") + + db_results = db_results[0] + cons_office_code = db_results[0] + cons_file_number = db_results[1] + handler_type = db_results[2] + handler_detail = db_results[3] + + ema_obj.responsible = ema_obj.responsible or Responsibility.objects.create() + try: + conservation_office = KonovaCode.objects.get( + atom_id=cons_office_code, + is_selectable=True, + is_archived=False, + code_lists__in=[CODELIST_CONSERVATION_OFFICE_ID] + ) + except ObjectDoesNotExist: + raise ObjectDoesNotExist(f"{ema_identifier}, {db_results}") + try: + handler_type = KonovaCode.objects.get( + atom_id=handler_type, + is_selectable=True, + is_archived=False, + code_lists__in=[CODELIST_COMPENSATION_HANDLER_ID] + ) + except ObjectDoesNotExist: + handler_type = None + + ema_obj.responsible.conservation_office = conservation_office + ema_obj.responsible.conservation_file_number = cons_file_number + handler = ema_obj.responsible.handler or Handler.objects.create() + handler.type = handler_type + handler.detail = handler_detail + ema_obj.responsible.handler = handler + ema_obj.responsible.handler.save() + ema_obj.responsible.save() + + tmp_cursor.close() + return ema_obj + + def _migrate_finance_volume_to_comment(self, ema_obj, ema_result): + ema_identifier = f"'{ema_result[0]}'" + tmp_cursor = self.db_connection.cursor() + tmp_cursor.execute( + 'select ' + 'b.datum, ' + 'b.hoehe ' + 'from "OBJ_MASTER" om ' + 'left join bewilligung b on om."GISPADID"=b.gispadid ' + 'where ' + f'om."KENNUNG"={ema_identifier} ' + ) + db_results = tmp_cursor.fetchall() + for result in db_results: + payment_date = result[0] + payment_amount = result[1] + + comment_extra = f"\n\nFinanzierung bewilligt am {formats.localize(payment_date, use_l10n=True)} in Höhe von {formats.localize(payment_amount, use_l10n=True)} €" + comment = ema_obj.comment or "" + if comment_extra in comment: + # skip + continue + comment += comment_extra + ema_obj.comment = comment + + tmp_cursor.close() + return ema_obj \ No newline at end of file diff --git a/konova/management/commands/migrate_ksp_konova.py b/konova/management/commands/migrate_ksp_konova.py index 2ddf4591..f3f1dce9 100644 --- a/konova/management/commands/migrate_ksp_konova.py +++ b/konova/management/commands/migrate_ksp_konova.py @@ -1,4 +1,6 @@ -from konova.management.commands.kspMigrater.compensation_migrater import CompensationMigrater, EmaMigrater +from konova.management.commands.kspMigrater.compensation_migrater import CompensationMigrater +from konova.management.commands.kspMigrater.eco_account_migrater import EcoAccountMigrater +from konova.management.commands.kspMigrater.ema_migrater import EmaMigrater from konova.management.commands.kspMigrater.intervention_migrater import InterventionMigrater from konova.management.commands.setup import BaseKonovaCommand @@ -17,7 +19,8 @@ class Command(BaseKonovaCommand): migraters = [ #InterventionMigrater(options), #CompensationMigrater(options), - EmaMigrater(options), + #EmaMigrater(options), + EcoAccountMigrater(options), ] for migrater in migraters: migrater.migrate()