#132 WIP Migrate OEKs

* WIP: adds oek migrations
* refactors ema migration into own file
This commit is contained in:
mpeltriaux 2022-03-17 16:33:06 +01:00
parent 899a6240c1
commit 0c1ca4d173
4 changed files with 324 additions and 189 deletions

View File

@ -499,190 +499,3 @@ class CompensationMigrater(BaseMigrater):
tmp_cursor.close() tmp_cursor.close()
return compensation return compensation
class EmaMigrater(CompensationMigrater):
def migrate(self):
cursor = self.db_connection.cursor()
cursor.execute(
'select '
'om."KENNUNG", '
'linf."OBJBEZ", '
'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geomf.the_geom,4326)), 3))) as geomf, '
'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geoml.the_geom,4326)), 2))) as geoml, '
'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geomp.the_geom,4326)), 1))) as geomp, '
'linf."Bemerkung" '
'from "OBJ_MASTER" om '
'left join "LINFOS" linf on om."GISPADID"=linf."GISPADID" '
'left join kom on om."GISPADID"=kom.gispadid '
'left join geometry_f geomf on om."GISPADID"=geomf.gispadid '
'left join geometry_l geoml on om."GISPADID"=geoml.gispadid '
'left join geometry_p geomp on om."GISPADID"=geomp.gispadid '
'left join "Aufwertung" auf on om."GISPADID"=auf."GISPADID" '
'where '
'om."OKL"=7730090 and '
'om.archiv=false and '
'om.nicht_vollstaendig=0'
)
all_emas = cursor.fetchall()
len_all_emas = len(all_emas)
num_processed = 0
print(f"Migrate EMAs to emas...")
print(f"--Found {len_all_emas} entries. Process now...")
for ema in all_emas:
if num_processed % 500 == 0:
print(f"----{num_processed}/{len_all_emas} processed")
with transaction.atomic():
ema_identifier = ema[0]
ema_title = ema[1]
ema_comment = ema[5]
ema_obj = Ema.objects.get_or_create(
identifier=ema_identifier
)[0]
ema_obj.title = ema_title
ema_obj.comment = ema_comment
ema_obj = self._migrate_geometry(ema_obj, ema)
ema_obj = self._migrate_responsibility(ema_obj, ema)
ema_obj = self._migrate_compensation_type(ema_obj, ema)
ema_obj = self._migrate_states(ema_obj, ema)
ema_obj = self._migrate_deadlines(ema_obj, ema)
ema_obj = self._migrate_action_control_deadlines(ema_obj, ema)
ema_obj = self._migrate_actions(ema_obj, ema)
ema_obj = self._migrate_finance_volume_to_comment(ema_obj, ema)
ema_obj = self._migrate_documents(ema_obj, EmaDocument, ema)
ema_obj.save()
num_processed += 1
cursor.close()
def _migrate_deadlines(self, ema_obj, ema_result):
ema_obj.deadlines.all().delete()
ema_identifier = f"'{ema_result[0]}'"
tmp_cursor = self.db_connection.cursor()
tmp_cursor.execute(
'select '
't."Terminart", '
't."K_Termin"::date '
'from "OBJ_MASTER" om '
'left join "Termine" t on om."GISPADID"=t."GISPADID" '
'where '
f'om."KENNUNG"={ema_identifier}'
)
db_results = tmp_cursor.fetchall()
for result in db_results:
deadline_type = result[0]
deadline_comment = None
if deadline_type == 708166:
deadline_comment = "Wiedervorlage"
elif deadline_type == 708163:
deadline_comment = "Projektbeginn"
deadline_date = result[1]
if deadline_date is None:
# Useless data
continue
try:
deadline = ema_obj.deadlines.get(
type=DeadlineType.OTHER,
date=deadline_date,
comment=deadline_comment
)
except ObjectDoesNotExist:
deadline = Deadline.objects.create(
type=DeadlineType.OTHER,
date=deadline_date,
comment=deadline_comment
)
ema_obj.deadlines.add(deadline)
return ema_obj
def _migrate_responsibility(self, ema_obj, ema_result):
ema_identifier = f"'{ema_result[0]}'"
tmp_cursor = self.db_connection.cursor()
tmp_cursor.execute(
'select '
'adr."adr_pruef" as ets, '
'linf."AZ", '
'adr.behoerde, '
'adr.angaben '
'from "OBJ_MASTER" om '
'left join "LINFOS" linf on om."GISPADID"=linf."GISPADID" '
'left join adressrolle adr on adr."GISPADID"=om."GISPADID" '
'where '
f'om."KENNUNG"={ema_identifier} '
)
db_results = tmp_cursor.fetchall()
if len(db_results) != 1:
raise AssertionError(f"{ema_identifier} has invalid responsibilities: {db_results}")
db_results = db_results[0]
cons_office_code = db_results[0]
cons_file_number = db_results[1]
handler_type = db_results[2]
handler_detail = db_results[3]
responsible = ema_obj.responsible or Responsibility.objects.create()
try:
conservation_office = KonovaCode.objects.get(
atom_id=cons_office_code,
is_selectable=True,
is_archived=False,
code_lists__in=[CODELIST_CONSERVATION_OFFICE_ID]
)
except ObjectDoesNotExist:
raise ObjectDoesNotExist(f"{ema_identifier}, {db_results}")
try:
handler_type = KonovaCode.objects.get(
atom_id=handler_type,
is_selectable=True,
is_archived=False,
code_lists__in=[CODELIST_COMPENSATION_HANDLER_ID]
)
except ObjectDoesNotExist:
handler_type = None
responsible.conservation_file_number = db_results[1]
ema_obj.responsible.conservation_office = conservation_office
ema_obj.responsible.conservation_file_number = cons_file_number
handler = ema_obj.responsible.handler or Handler.objects.create()
handler.type = handler_type
handler.detail = handler_detail
ema_obj.responsible.handler = handler
ema_obj.responsible.handler.save()
ema_obj.responsible.save()
tmp_cursor.close()
return ema_obj
def _migrate_finance_volume_to_comment(self, ema_obj, ema_result):
ema_identifier = f"'{ema_result[0]}'"
tmp_cursor = self.db_connection.cursor()
tmp_cursor.execute(
'select '
'b.datum, '
'b.hoehe '
'from "OBJ_MASTER" om '
'left join bewilligung b on om."GISPADID"=b.gispadid '
'where '
f'om."KENNUNG"={ema_identifier} '
)
db_results = tmp_cursor.fetchall()
for result in db_results:
payment_date = result[0]
payment_amount = result[1]
comment_extra = f"\n\nFinanzierung bewilligt am {formats.localize(payment_date, use_l10n=True)} in Höhe von {formats.localize(payment_amount, use_l10n=True)}"
comment = ema_obj.comment or ""
if comment_extra in comment:
# skip
continue
comment += comment_extra
ema_obj.comment = comment
tmp_cursor.close()
return ema_obj

View File

@ -0,0 +1,123 @@
from django.core.exceptions import ObjectDoesNotExist
from django.db import transaction
from codelist.models import KonovaCode
from codelist.settings import CODELIST_CONSERVATION_OFFICE_ID, CODELIST_COMPENSATION_HANDLER_ID
from compensation.models import EcoAccount, EcoAccountDocument
from intervention.models import Responsibility, Handler
from konova.management.commands.kspMigrater.compensation_migrater import CompensationMigrater
class EcoAccountMigrater(CompensationMigrater):
def migrate(self):
cursor = self.db_connection.cursor()
cursor.execute(
'select '
'om."KENNUNG", '
'linf."OBJBEZ", '
'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geomf.the_geom,4326)), 3))) as geomf, '
'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geoml.the_geom,4326)), 2))) as geoml, '
'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geomp.the_geom,4326)), 1))) as geomp, '
'linf."Bemerkung" '
'from "OBJ_MASTER" om '
'left join "LINFOS" linf on om."GISPADID"=linf."GISPADID" '
'left join kom on om."GISPADID"=kom.gispadid '
'left join geometry_f geomf on om."GISPADID"=geomf.gispadid '
'left join geometry_l geoml on om."GISPADID"=geoml.gispadid '
'left join geometry_p geomp on om."GISPADID"=geomp.gispadid '
'left join "Aufwertung" auf on om."GISPADID"=auf."GISPADID" '
'where '
'om."OKL"=7730081 and '
'om.archiv=false and '
'om.nicht_vollstaendig=0'
)
all_oeks = cursor.fetchall()
len_all_oeks = len(all_oeks)
num_processed = 0
print(f"Migrate OEKs to ecoaccounts...")
print(f"--Found {len_all_oeks} entries. Process now...")
for oek in all_oeks:
if num_processed % 500 == 0:
print(f"----{num_processed}/{len_all_oeks} processed")
with transaction.atomic():
oek_identifier = oek[0]
oek_title = oek[1]
oek_comment = oek[5]
eco_account = EcoAccount.objects.get_or_create(
identifier=oek_identifier
)[0]
eco_account.title = oek_title
eco_account.comment = oek_comment
eco_account = self._migrate_geometry(eco_account, oek)
eco_account = self._migrate_responsibility(eco_account, oek)
eco_account = self._migrate_states(eco_account, oek)
eco_account = self._migrate_deadlines(eco_account, oek)
eco_account = self._migrate_action_control_deadlines(eco_account, oek)
eco_account = self._migrate_actions(eco_account, oek)
eco_account = self._migrate_documents(eco_account, EcoAccountDocument, oek)
eco_account.save()
num_processed += 1
cursor.close()
def _migrate_responsibility(self, eco_account, oek):
acc_identifier = f"'{oek[0]}'"
tmp_cursor = self.db_connection.cursor()
tmp_cursor.execute(
'select '
'adr."adr_pruef" as ets, '
'linf."AZ", '
'oek.traeger, '
'oek.bemerkungtraeger '
'from "OBJ_MASTER" om '
'left join "LINFOS" linf on om."GISPADID"=linf."GISPADID" '
'left join adressrolle adr on adr."GISPADID"=om."GISPADID" '
'left join oek on om."GISPADID"=oek.gispadid '
'where '
f'om."KENNUNG"={acc_identifier}'
)
db_results = tmp_cursor.fetchall()
if len(db_results) != 1:
raise AssertionError(f"{acc_identifier} has invalid responsibilities: {db_results}")
db_results = db_results[0]
cons_office_code = db_results[0]
cons_file_number = db_results[1]
handler_type = db_results[2]
handler_detail = db_results[3]
eco_account.responsible = eco_account.responsible or Responsibility.objects.create()
try:
conservation_office = KonovaCode.objects.get(
atom_id=cons_office_code,
is_selectable=True,
is_archived=False,
code_lists__in=[CODELIST_CONSERVATION_OFFICE_ID]
)
except ObjectDoesNotExist:
raise ObjectDoesNotExist(f"{acc_identifier}, {db_results}")
try:
handler_type = KonovaCode.objects.get(
atom_id=handler_type,
is_selectable=True,
is_archived=False,
code_lists__in=[CODELIST_COMPENSATION_HANDLER_ID]
)
except ObjectDoesNotExist:
handler_type = None
eco_account.responsible.conservation_office = conservation_office
eco_account.responsible.conservation_file_number = cons_file_number
handler = eco_account.responsible.handler or Handler.objects.create()
handler.type = handler_type
handler.detail = handler_detail
eco_account.responsible.handler = handler
eco_account.responsible.handler.save()
eco_account.responsible.save()
tmp_cursor.close()
return eco_account

View File

@ -0,0 +1,196 @@
from django.core.exceptions import ObjectDoesNotExist
from django.db import transaction
from django.utils import formats
from codelist.models import KonovaCode
from codelist.settings import CODELIST_COMPENSATION_HANDLER_ID, CODELIST_CONSERVATION_OFFICE_ID
from ema.models import EmaDocument, Ema
from intervention.models import Handler, Responsibility
from konova.management.commands.kspMigrater.compensation_migrater import CompensationMigrater
from konova.models import DeadlineType, Deadline
class EmaMigrater(CompensationMigrater):
def migrate(self):
cursor = self.db_connection.cursor()
cursor.execute(
'select '
'om."KENNUNG", '
'linf."OBJBEZ", '
'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geomf.the_geom,4326)), 3))) as geomf, '
'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geoml.the_geom,4326)), 2))) as geoml, '
'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geomp.the_geom,4326)), 1))) as geomp, '
'linf."Bemerkung" '
'from "OBJ_MASTER" om '
'left join "LINFOS" linf on om."GISPADID"=linf."GISPADID" '
'left join kom on om."GISPADID"=kom.gispadid '
'left join geometry_f geomf on om."GISPADID"=geomf.gispadid '
'left join geometry_l geoml on om."GISPADID"=geoml.gispadid '
'left join geometry_p geomp on om."GISPADID"=geomp.gispadid '
'left join "Aufwertung" auf on om."GISPADID"=auf."GISPADID" '
'where '
'om."OKL"=7730090 and '
'om.archiv=false and '
'om.nicht_vollstaendig=0'
)
all_emas = cursor.fetchall()
len_all_emas = len(all_emas)
num_processed = 0
print(f"Migrate EMAs to emas...")
print(f"--Found {len_all_emas} entries. Process now...")
for ema in all_emas:
if num_processed % 500 == 0:
print(f"----{num_processed}/{len_all_emas} processed")
with transaction.atomic():
ema_identifier = ema[0]
ema_title = ema[1]
ema_comment = ema[5]
ema_obj = Ema.objects.get_or_create(
identifier=ema_identifier
)[0]
ema_obj.title = ema_title
ema_obj.comment = ema_comment
ema_obj = self._migrate_geometry(ema_obj, ema)
ema_obj = self._migrate_responsibility(ema_obj, ema)
ema_obj = self._migrate_compensation_type(ema_obj, ema)
ema_obj = self._migrate_states(ema_obj, ema)
ema_obj = self._migrate_deadlines(ema_obj, ema)
ema_obj = self._migrate_action_control_deadlines(ema_obj, ema)
ema_obj = self._migrate_actions(ema_obj, ema)
ema_obj = self._migrate_finance_volume_to_comment(ema_obj, ema)
ema_obj = self._migrate_documents(ema_obj, EmaDocument, ema)
ema_obj.save()
num_processed += 1
cursor.close()
def _migrate_deadlines(self, ema_obj, ema_result):
ema_obj.deadlines.all().delete()
ema_identifier = f"'{ema_result[0]}'"
tmp_cursor = self.db_connection.cursor()
tmp_cursor.execute(
'select '
't."Terminart", '
't."K_Termin"::date '
'from "OBJ_MASTER" om '
'left join "Termine" t on om."GISPADID"=t."GISPADID" '
'where '
f'om."KENNUNG"={ema_identifier}'
)
db_results = tmp_cursor.fetchall()
for result in db_results:
deadline_type = result[0]
deadline_comment = None
if deadline_type == 708166:
deadline_comment = "Wiedervorlage"
elif deadline_type == 708163:
deadline_comment = "Projektbeginn"
deadline_date = result[1]
if deadline_date is None:
# Useless data
continue
try:
deadline = ema_obj.deadlines.get(
type=DeadlineType.OTHER,
date=deadline_date,
comment=deadline_comment
)
except ObjectDoesNotExist:
deadline = Deadline.objects.create(
type=DeadlineType.OTHER,
date=deadline_date,
comment=deadline_comment
)
ema_obj.deadlines.add(deadline)
return ema_obj
def _migrate_responsibility(self, ema_obj, ema_result):
ema_identifier = f"'{ema_result[0]}'"
tmp_cursor = self.db_connection.cursor()
tmp_cursor.execute(
'select '
'adr."adr_pruef" as ets, '
'linf."AZ", '
'adr.behoerde, '
'adr.angaben '
'from "OBJ_MASTER" om '
'left join "LINFOS" linf on om."GISPADID"=linf."GISPADID" '
'left join adressrolle adr on adr."GISPADID"=om."GISPADID" '
'where '
f'om."KENNUNG"={ema_identifier} '
)
db_results = tmp_cursor.fetchall()
if len(db_results) != 1:
raise AssertionError(f"{ema_identifier} has invalid responsibilities: {db_results}")
db_results = db_results[0]
cons_office_code = db_results[0]
cons_file_number = db_results[1]
handler_type = db_results[2]
handler_detail = db_results[3]
ema_obj.responsible = ema_obj.responsible or Responsibility.objects.create()
try:
conservation_office = KonovaCode.objects.get(
atom_id=cons_office_code,
is_selectable=True,
is_archived=False,
code_lists__in=[CODELIST_CONSERVATION_OFFICE_ID]
)
except ObjectDoesNotExist:
raise ObjectDoesNotExist(f"{ema_identifier}, {db_results}")
try:
handler_type = KonovaCode.objects.get(
atom_id=handler_type,
is_selectable=True,
is_archived=False,
code_lists__in=[CODELIST_COMPENSATION_HANDLER_ID]
)
except ObjectDoesNotExist:
handler_type = None
ema_obj.responsible.conservation_office = conservation_office
ema_obj.responsible.conservation_file_number = cons_file_number
handler = ema_obj.responsible.handler or Handler.objects.create()
handler.type = handler_type
handler.detail = handler_detail
ema_obj.responsible.handler = handler
ema_obj.responsible.handler.save()
ema_obj.responsible.save()
tmp_cursor.close()
return ema_obj
def _migrate_finance_volume_to_comment(self, ema_obj, ema_result):
ema_identifier = f"'{ema_result[0]}'"
tmp_cursor = self.db_connection.cursor()
tmp_cursor.execute(
'select '
'b.datum, '
'b.hoehe '
'from "OBJ_MASTER" om '
'left join bewilligung b on om."GISPADID"=b.gispadid '
'where '
f'om."KENNUNG"={ema_identifier} '
)
db_results = tmp_cursor.fetchall()
for result in db_results:
payment_date = result[0]
payment_amount = result[1]
comment_extra = f"\n\nFinanzierung bewilligt am {formats.localize(payment_date, use_l10n=True)} in Höhe von {formats.localize(payment_amount, use_l10n=True)}"
comment = ema_obj.comment or ""
if comment_extra in comment:
# skip
continue
comment += comment_extra
ema_obj.comment = comment
tmp_cursor.close()
return ema_obj

View File

@ -1,4 +1,6 @@
from konova.management.commands.kspMigrater.compensation_migrater import CompensationMigrater, EmaMigrater from konova.management.commands.kspMigrater.compensation_migrater import CompensationMigrater
from konova.management.commands.kspMigrater.eco_account_migrater import EcoAccountMigrater
from konova.management.commands.kspMigrater.ema_migrater import EmaMigrater
from konova.management.commands.kspMigrater.intervention_migrater import InterventionMigrater from konova.management.commands.kspMigrater.intervention_migrater import InterventionMigrater
from konova.management.commands.setup import BaseKonovaCommand from konova.management.commands.setup import BaseKonovaCommand
@ -17,7 +19,8 @@ class Command(BaseKonovaCommand):
migraters = [ migraters = [
#InterventionMigrater(options), #InterventionMigrater(options),
#CompensationMigrater(options), #CompensationMigrater(options),
EmaMigrater(options), #EmaMigrater(options),
EcoAccountMigrater(options),
] ]
for migrater in migraters: for migrater in migraters:
migrater.migrate() migrater.migrate()