From f621aab6e17cda5b53a96067fa22eaf7619ff62a Mon Sep 17 00:00:00 2001 From: mpeltriaux Date: Tue, 22 Mar 2022 16:36:52 +0100 Subject: [PATCH] #132 WIP OEK * adds geometry-remerging with former deductions * adds geometry-area calculation * adds fallback for old data migration on deprecated biotope code lists * fixes typo in CODELIST_AFTER_STATE_BIOTOPES_ID --- codelist/settings.py | 2 +- .../migrations/0005_auto_20220218_0917.py | 4 +- .../kspMigrater/compensation_migrater.py | 9 +- .../kspMigrater/eco_account_migrater.py | 89 +++++++++++++++---- konova/migrations/0005_auto_20220216_0856.py | 4 +- 5 files changed, 86 insertions(+), 22 deletions(-) diff --git a/codelist/settings.py b/codelist/settings.py index e1afa352..4196c061 100644 --- a/codelist/settings.py +++ b/codelist/settings.py @@ -15,7 +15,7 @@ CODELIST_CONSERVATION_OFFICE_ID = 907 # CLNaturschutzbehörden CODELIST_REGISTRATION_OFFICE_ID = 1053 # CLZulassungsbehörden CODELIST_288_ID = 288 # CL_288 (holds GISPAD related detail-biotope codes, similar to 975, but historical and only important for migration) CODELIST_BIOTOPES_ID = 654 # CL_Biotoptypen -CODELIST_AFTER_STATE_BIOTOPES__ID = 974 # CL-KSP_ZielBiotoptypen - USAGE HAS BEEN DROPPED IN 2022 IN FAVOR OF 654 +CODELIST_AFTER_STATE_BIOTOPES_ID = 974 # CL-KSP_ZielBiotoptypen - USAGE HAS BEEN DROPPED IN 2022 IN FAVOR OF 654 CODELIST_BIOTOPES_EXTRA_CODES_ID = 975 # CLZusatzbezeichnung CODELIST_LAW_ID = 1048 # CLVerfahrensrecht CODELIST_PROCESS_TYPE_ID = 44382 # CLVerfahrenstyp diff --git a/compensation/migrations/0005_auto_20220218_0917.py b/compensation/migrations/0005_auto_20220218_0917.py index 43e7db9c..00442640 100644 --- a/compensation/migrations/0005_auto_20220218_0917.py +++ b/compensation/migrations/0005_auto_20220218_0917.py @@ -3,7 +3,7 @@ from django.db import migrations, models, transaction import django.db.models.deletion -from codelist.settings import CODELIST_BIOTOPES_ID, CODELIST_AFTER_STATE_BIOTOPES__ID +from codelist.settings import CODELIST_BIOTOPES_ID, CODELIST_AFTER_STATE_BIOTOPES_ID def migrate_entries_974_to_654(apps, schema_editor): @@ -23,7 +23,7 @@ def migrate_entries_974_to_654(apps, schema_editor): state.save() old_list_states = CompensationState.objects.filter( - biotope_type__code_lists__in=[CODELIST_AFTER_STATE_BIOTOPES__ID] + biotope_type__code_lists__in=[CODELIST_AFTER_STATE_BIOTOPES_ID] ) if old_list_states.count() > 0: raise Exception("Still unmigrated values!") diff --git a/konova/management/commands/kspMigrater/compensation_migrater.py b/konova/management/commands/kspMigrater/compensation_migrater.py index 96bcc5c0..d986e7cd 100644 --- a/konova/management/commands/kspMigrater/compensation_migrater.py +++ b/konova/management/commands/kspMigrater/compensation_migrater.py @@ -3,7 +3,8 @@ from django.db import transaction from codelist.models import KonovaCode from codelist.settings import CODELIST_BIOTOPES_ID, CODELIST_BIOTOPES_EXTRA_CODES_ID, \ - CODELIST_COMPENSATION_ACTION_DETAIL_ID, CODELIST_COMPENSATION_ACTION_ID, CODELIST_COMPENSATION_HANDLER_ID, CODELIST_288_ID + CODELIST_COMPENSATION_ACTION_DETAIL_ID, CODELIST_COMPENSATION_ACTION_ID, CODELIST_COMPENSATION_HANDLER_ID, \ + CODELIST_288_ID, CODELIST_AFTER_STATE_BIOTOPES_ID from compensation.models import Compensation, CompensationState, CompensationAction, UnitChoices, CompensationDocument from intervention.models import Responsibility, Intervention from konova.management.commands.kspMigrater.base_migrater import BaseMigrater @@ -160,7 +161,11 @@ class CompensationMigrater(BaseMigrater): code_lists__in=[CODELIST_BIOTOPES_ID] ) except ObjectDoesNotExist: - raise ObjectDoesNotExist(f"{state_type}, {compensation.identifier}") + # Very old data might contain entries from this deprecated list + state_code = KonovaCode.objects.get( + atom_id=state_type, + code_lists__in=[CODELIST_AFTER_STATE_BIOTOPES_ID] + ) tmp_cursor_z_code = self.db_connection.cursor() tmp_cursor_z_code.execute(z_code_sql.format(pkey_entry)) z_code_results = tmp_cursor_z_code.fetchall() diff --git a/konova/management/commands/kspMigrater/eco_account_migrater.py b/konova/management/commands/kspMigrater/eco_account_migrater.py index 013f49cd..9b404db2 100644 --- a/konova/management/commands/kspMigrater/eco_account_migrater.py +++ b/konova/management/commands/kspMigrater/eco_account_migrater.py @@ -1,3 +1,4 @@ +from django.contrib.gis.geos import MultiPolygon, Polygon from django.core.exceptions import ObjectDoesNotExist from django.db import transaction @@ -6,6 +7,8 @@ from codelist.settings import CODELIST_CONSERVATION_OFFICE_ID, CODELIST_COMPENSA from compensation.models import EcoAccount, EcoAccountDocument from intervention.models import Responsibility, Handler from konova.management.commands.kspMigrater.compensation_migrater import CompensationMigrater +from konova.models import Geometry +from konova.sub_settings.lanis_settings import DEFAULT_SRID_RLP, DEFAULT_SRID class EcoAccountMigrater(CompensationMigrater): @@ -14,23 +17,23 @@ class EcoAccountMigrater(CompensationMigrater): cursor = self.db_connection.cursor() cursor.execute( 'select ' - 'om."KENNUNG", ' - 'linf."OBJBEZ", ' - 'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geomf.the_geom,4326)), 3))) as geomf, ' - 'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geoml.the_geom,4326)), 2))) as geoml, ' - 'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geomp.the_geom,4326)), 1))) as geomp, ' - 'linf."Bemerkung" ' + 'om."KENNUNG", ' + 'linf."OBJBEZ", ' + 'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geomf.the_geom,4326)), 3))) as geomf, ' + 'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geoml.the_geom,4326)), 2))) as geoml, ' + 'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geomp.the_geom,4326)), 1))) as geomp, ' + 'linf."Bemerkung" ' 'from "OBJ_MASTER" om ' - 'left join "LINFOS" linf on om."GISPADID"=linf."GISPADID" ' - 'left join kom on om."GISPADID"=kom.gispadid ' - 'left join geometry_f geomf on om."GISPADID"=geomf.gispadid ' - 'left join geometry_l geoml on om."GISPADID"=geoml.gispadid ' - 'left join geometry_p geomp on om."GISPADID"=geomp.gispadid ' - 'left join "Aufwertung" auf on om."GISPADID"=auf."GISPADID" ' + 'left join "LINFOS" linf on om."GISPADID"=linf."GISPADID" ' + 'left join kom on om."GISPADID"=kom.gispadid ' + 'left join geometry_f geomf on om."GISPADID"=geomf.gispadid ' + 'left join geometry_l geoml on om."GISPADID"=geoml.gispadid ' + 'left join geometry_p geomp on om."GISPADID"=geomp.gispadid ' + 'left join "Aufwertung" auf on om."GISPADID"=auf."GISPADID" ' 'where ' - 'om."OKL"=7730081 and ' - 'om.archiv=false and ' - 'om.nicht_vollstaendig=0' + 'om."OKL"=7730081 and ' + 'om.archiv=false and ' + 'om.nicht_vollstaendig=0 ' ) all_oeks = cursor.fetchall() @@ -64,6 +67,62 @@ class EcoAccountMigrater(CompensationMigrater): num_processed += 1 cursor.close() + def _migrate_geometry(self, instance, db_result: tuple): + identifier = f"'{db_result[0]}'" + empty_str = f"''" + tmp_cursor = self.db_connection.cursor() + tmp_cursor.execute( + 'select ' + 'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Union(ST_Transform(geom.the_geom,4326)))))) ' + 'from "OBJ_MASTER" om ' + 'left join "Aufwertung" auf on om."GISPADID"=auf."GISPADID" ' + 'left join "OBJ_MASTER" om_kom on auf."Infos"=om_kom."KENNUNG" ' + 'left join geometry_f geom on geom.gispadid=om_kom."GISPADID" ' + 'where ' + 'om."OKL"=7730081 and ' + 'om.archiv=false and ' + 'om_kom.archiv=false and ' + f'(auf."Infos" is not null and auf."Infos"!={empty_str}) and ' + f'om."KENNUNG"={identifier}' + ) + deduction_db_results = tmp_cursor.fetchall() + if len(deduction_db_results) != 1: + raise AssertionError(f"Unexpected number of matches: {deduction_db_results}") + db_result_geom = deduction_db_results[0][0] + if db_result_geom is not None: + deductions_db_result_geom = MultiPolygon.from_ewkt(deduction_db_results[0][0]) + else: + deductions_db_result_geom = MultiPolygon(srid=DEFAULT_SRID) + + tmp_cursor.execute( + 'select ' + 'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geomf.the_geom,4326)), 3))) as geomf ' + 'from "OBJ_MASTER" om ' + 'left join geometry_f geomf on om."GISPADID"=geomf.gispadid ' + 'where ' + f'om."KENNUNG"={identifier}' + ) + original_geom_db_results = tmp_cursor.fetchall() + if len(original_geom_db_results) != 1: + raise AssertionError(f"Unexpected number of matches: {original_geom_db_results}") + account_db_result_geom = MultiPolygon.from_ewkt(original_geom_db_results[0][0]) + + db_result_geom = account_db_result_geom.union(deductions_db_result_geom) + if isinstance(db_result_geom, Polygon): + db_result_geom = MultiPolygon(db_result_geom, srid=DEFAULT_SRID) + instance.geometry = instance.geometry or Geometry() + try: + # Calculate area by transforming + rlp_geom = db_result_geom.transform(ct=DEFAULT_SRID_RLP, clone=True) + area = int(rlp_geom.area) + instance.deductable_surface = area + instance.geometry.geom = db_result_geom if not db_result_geom.empty else None + instance.geometry.save() + except TypeError: + raise TypeError(f"{identifier}, {db_result_geom}") + tmp_cursor.close() + return instance + def _migrate_responsibility(self, eco_account, oek): acc_identifier = f"'{oek[0]}'" tmp_cursor = self.db_connection.cursor() diff --git a/konova/migrations/0005_auto_20220216_0856.py b/konova/migrations/0005_auto_20220216_0856.py index 43c518ae..567e2065 100644 --- a/konova/migrations/0005_auto_20220216_0856.py +++ b/konova/migrations/0005_auto_20220216_0856.py @@ -2,7 +2,7 @@ from django.db import migrations, transaction -from codelist.settings import CODELIST_BIOTOPES_ID, CODELIST_AFTER_STATE_BIOTOPES__ID +from codelist.settings import CODELIST_BIOTOPES_ID, CODELIST_AFTER_STATE_BIOTOPES_ID def migrate_biotopes_from_974_to_654(apps, schema_editor): @@ -23,7 +23,7 @@ def migrate_biotopes_from_974_to_654(apps, schema_editor): all_states = CompensationState.objects.all() after_state_list_elements = all_states.filter( - biotope_type__code_lists__in=[CODELIST_AFTER_STATE_BIOTOPES__ID] + biotope_type__code_lists__in=[CODELIST_AFTER_STATE_BIOTOPES_ID] ) if after_state_list_elements.count() > 0: raise Exception("Still states with wrong codelist entries!")