konova/konova/management/commands/kspMigrater/compensation_migrater.py
mpeltriaux 899a6240c1 #132 EMA finance volume migration
* adds migration of finance volumes into ema comment
2022-03-17 16:08:50 +01:00

688 lines
27 KiB
Python

from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
from django.db import transaction
from django.utils import formats
from codelist.models import KonovaCode
from codelist.settings import CODELIST_BIOTOPES_ID, CODELIST_BIOTOPES_EXTRA_CODES_ID, \
CODELIST_COMPENSATION_ACTION_DETAIL_ID, CODELIST_COMPENSATION_ACTION_ID, CODELIST_CONSERVATION_OFFICE_ID, \
CODELIST_HANDLER_ID, CODELIST_COMPENSATION_HANDLER_ID
from compensation.models import Compensation, CompensationState, CompensationAction, UnitChoices, CompensationDocument
from ema.models import Ema, EmaDocument
from intervention.models import Responsibility, Handler, Intervention
from konova.management.commands.kspMigrater.base_migrater import BaseMigrater
from konova.models import Deadline, DeadlineType
class CompensationMigrater(BaseMigrater):
def migrate(self):
cursor = self.db_connection.cursor()
empty_str = "''"
cursor.execute(
'select '
'om."KENNUNG", '
'linf."OBJBEZ", '
'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geomf.the_geom,4326)), 3))) as geomf, '
'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geoml.the_geom,4326)), 2))) as geoml, '
'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geomp.the_geom,4326)), 1))) as geomp, '
'linf."Bemerkung" '
'from "OBJ_MASTER" om '
'left join "LINFOS" linf on om."GISPADID"=linf."GISPADID" '
'left join kom on om."GISPADID"=kom.gispadid '
'left join geometry_f geomf on om."GISPADID"=geomf.gispadid '
'left join geometry_l geoml on om."GISPADID"=geoml.gispadid '
'left join geometry_p geomp on om."GISPADID"=geomp.gispadid '
'left join "Aufwertung" auf on om."GISPADID"=auf."GISPADID" '
'where '
'om."OKL"=7730080 and '
'om.archiv=false and '
f'(auf."Infos" is null or auf."Infos"={empty_str}) and '
'om.nicht_vollstaendig=0'
)
all_koms = cursor.fetchall()
len_all_koms = len(all_koms)
num_processed = 0
print(f"Migrate KOMs to compensations...")
print(f"--Found {len_all_koms} entries. Process now...")
unsuccessfull_compensations = {}
for kom in all_koms:
if num_processed % 500 == 0:
print(f"----{num_processed}/{len_all_koms} processed")
with transaction.atomic():
kom_identifier = kom[0]
kom_title = kom[1]
kom_comment = kom[5]
compensation = Compensation.objects.get_or_create(
identifier=kom_identifier
)[0]
compensation.title = kom_title
compensation.comment = kom_comment
compensation = self._migrate_geometry(compensation, kom)
compensation = self._migrate_responsibility(compensation, kom)
compensation = self._migrate_compensation_type(compensation, kom)
compensation = self._migrate_states(compensation, kom)
compensation = self._migrate_deadlines(compensation, kom)
compensation = self._migrate_action_control_deadlines(compensation, kom)
compensation = self._migrate_actions(compensation, kom)
compensation = self._migrate_documents(compensation, CompensationDocument, kom)
try:
compensation = self._migrate_interventions_reference(compensation, kom)
compensation.save()
except ObjectDoesNotExist:
compensation.delete()
unsuccessfull_compensations[kom_identifier] = "EIV does not exist"
num_processed += 1
print("The following KOMs could not be migrated: ")
for kom, val in unsuccessfull_compensations:
print(kom)
cursor.close()
def _migrate_interventions_reference(self, compensation, kom):
kom_identifier = f"'{kom[0]}'"
tmp_cursor = self.db_connection.cursor()
tmp_cursor.execute(
'select '
'ref."REFERENZ" '
'from "OBJ_MASTER" om '
'left join "LINFOS" linf on om."GISPADID"=linf."GISPADID" '
'left join "REFERENZ" ref on om."GISPADID"=ref."GISPADID" '
'where '
'om."OKL"=7730080 and '
f'om."KENNUNG"={kom_identifier};'
)
eivs = tmp_cursor.fetchall()
len_eivs = len(eivs)
if len_eivs != 1:
raise AssertionError(f"{kom_identifier} has {len_eivs} EIVs!")
eiv = eivs[0]
try:
intervention = Intervention.objects.get(
identifier=eiv[0]
)
except ObjectDoesNotExist:
raise ObjectDoesNotExist(f"{kom_identifier} could not find migrated {eiv}")
compensation.intervention = intervention
tmp_cursor.close()
return compensation
def _migrate_responsibility(self, compensation, kom):
compensation.responsible = compensation.responsible or Responsibility.objects.create()
return compensation
def _migrate_states(self, compensation, kom):
kom_identifier = f"'{kom[0]}'"
before_states_sql = 'select ' \
'b."Biotoptyp", ' \
'b."Flaeche", ' \
'b."PKEY" ' \
'from "BtypHtyp" b ' \
'left join "OBJ_MASTER" om on om."GISPADID"=b."GISPADID" ' \
'where ' \
f'om."KENNUNG"={kom_identifier} and ' \
'b."Biotoptyp"!=0 and '\
'b."Biotoptyp" is not null '
zero_str = "'0'"
z_code_before_states_sql = 'select z."Zusatzcode" from "Zusatzcodes" z where z."FKEY"={}'
after_states_sql = 'select ' \
'z."Zielbiotoptyp", ' \
'z."Flaeche", ' \
'z."PKEY" ' \
'from "Zielbiotoptypen" z ' \
'left join "OBJ_MASTER" om on om."GISPADID"=z."GISPADID" ' \
'where ' \
f'om."KENNUNG"={kom_identifier} and '\
f'z."Zielbiotoptyp"!={zero_str} and '\
'z."Zielbiotoptyp" is not null '
z_code_after_states_sql = 'select z."Z_Code" from "Z_Codes" z where z."FKEY"={}'
compensation.before_states.all().delete()
compensation = self._process_state_migration(compensation, before_states_sql, z_code_before_states_sql, compensation.before_states)
compensation.after_states.all().delete()
compensation = self._process_state_migration(compensation, after_states_sql, z_code_after_states_sql, compensation.after_states)
return compensation
def _process_state_migration(self, compensation, sql, z_code_sql, state_manager):
tmp_cursor = self.db_connection.cursor()
tmp_cursor.execute(sql)
db_result = tmp_cursor.fetchall()
for result in db_result:
state_type = result[0]
if state_type is None or len(str(state_type)) == 0:
# garbage
continue
state_surface = result[1] or 0.0
pkey_entry = f"'{result[2]}'"
try:
state_code = KonovaCode.objects.get(
atom_id=state_type,
code_lists__in=[CODELIST_BIOTOPES_ID]
)
except ObjectDoesNotExist:
raise ObjectDoesNotExist(f"{state_type}, {compensation.identifier}")
tmp_cursor_z_code = self.db_connection.cursor()
tmp_cursor_z_code.execute(z_code_sql.format(pkey_entry))
z_code_results = tmp_cursor_z_code.fetchall()
z_codes = []
for z_code_result in z_code_results:
z_code = KonovaCode.objects.filter(
atom_id=z_code_result[0],
code_lists__in=[CODELIST_BIOTOPES_EXTRA_CODES_ID],
)
z_codes += z_code
tmp_cursor_z_code.close()
try:
_state_obj = state_manager.get(
biotope_type=state_code,
surface=state_surface
)
except ObjectDoesNotExist:
_state_obj = CompensationState.objects.create(
biotope_type=state_code,
surface=state_surface,
)
state_manager.add(_state_obj)
_state_obj.biotope_type_details.set(z_codes)
tmp_cursor.close()
return compensation
def _migrate_compensation_type(self, compensation, kom):
kom_identifier = f"'{kom[0]}'"
tmp_cursor = self.db_connection.cursor()
tmp_cursor.execute(
'select '
'mt.typ '
'from "OBJ_MASTER" om '
'left join massnahmetyp mt on om."GISPADID"=mt.gispadid '
'where '
f'om."KENNUNG"={kom_identifier}'
)
db_result = tmp_cursor.fetchall()
if len(db_result) != 1:
raise AssertionError(f"{kom_identifier} has no specification on compensation type (CEF, ...)")
comp_type = db_result[0][0]
if comp_type == 705816:
# regular compensation, do nothing
pass
elif comp_type == 705815:
compensation.is_cef = True
elif comp_type == 705817:
compensation.is_coherence_keeping = True
elif comp_type == 154156555:
compensation.is_coherence_keeping = True
compensation.is_cef = True
tmp_cursor.close()
return compensation
def _migrate_action_control_deadlines(self, compensation, kom):
kom_identifier = f"'{kom[0]}'"
tmp_cursor = self.db_connection.cursor()
tmp_cursor.execute(
'select '
'k."Datum", '
'k."Typ", '
'k."Ergebnis", '
'k."Stelle", '
'k."Bemerkung" '
'from "OBJ_MASTER" om '
'left join "MASSN" m on m."GISPADID"=om."GISPADID" '
'left join "Kontrolle" k on m."PKEY"=k."FKEY" '
'where '
f'om."KENNUNG"={kom_identifier}'
)
control_types = {
707175: "Zielerreichungskontrolle",
707176: "Zustandskontrolle",
707174: "Ausführungskontrolle",
}
control_results = {
153097: "nicht beurteilbar",
153954: "vollständig umgesetzt",
153956: "teilweise umgesetzt",
153957: "eher nicht umgesetzt",
153960: "mit leichten Mängeln",
153963: "ohne Mängel",
187136: "Ziel nicht erreicht",
187137: "Ziel teilweise erreicht",
187139: "Ziel vollständig umgesetzt",
}
db_results = tmp_cursor.fetchall()
for result in db_results:
control_date = result[0]
if control_date is None:
# useless data
continue
control_type = result[1]
control_result = result[2]
control_responsible = result[3]
control_comment = result[4] or ""
try:
control_type = control_types[control_type]
except KeyError:
control_type = "Unbekannt"
try:
control_result = control_results[control_result]
except KeyError:
control_result = "Unbekannt"
try:
control_responsible = KonovaCode.objects.get(
atom_id=control_responsible,
is_selectable=True,
is_archived=False,
code_lists__in=[CODELIST_COMPENSATION_HANDLER_ID]
)
control_responsible = control_responsible.long_name
except ObjectDoesNotExist:
control_responsible = "Unbekannt"
control_comment += f"\n\nKontrolltyp: {control_type}"
control_comment += f"\nKontrollergebnis: {control_result}"
control_comment += f"\nKontrollstelle: {control_responsible}"
try:
deadline = compensation.deadlines.get(
type=DeadlineType.CONTROL,
date=control_date,
comment=control_comment
)
except ObjectDoesNotExist:
deadline = Deadline.objects.create(
type=DeadlineType.CONTROL,
date=control_date,
comment=control_comment
)
compensation.deadlines.add(deadline)
tmp_cursor.close()
return compensation
def _migrate_deadlines(self, compensation, kom):
compensation.deadlines.all().delete()
kom_identifier = f"'{kom[0]}'"
tmp_cursor = self.db_connection.cursor()
tmp_cursor.execute(
'select '
'f.umsetzungbis, '
'f.unterhaltungbis, '
'f.artunterhaltungbis, '
'f.artumsetzungbis '
'from kom k '
'left join fristen f on k.fristen=f.id '
'left join "OBJ_MASTER" om on k.gispadid=om."GISPADID" '
'where '
f'om."KENNUNG"={kom_identifier}'
)
db_results = tmp_cursor.fetchall()
for result in db_results:
finish_until = result[0]
maintain_until = result[1]
maintain_until_type = result[2]
finish_until_type = result[3]
if finish_until_type == 941256187:
finish_until_type = "Termin endgültig"
elif finish_until_type == 889700393:
finish_until_type = "Termin vorläufig"
else:
finish_until_type = "Unbekannt ob Termin vorläufig oder endgültig"
if maintain_until_type == 807238388:
maintain_until_type = "Terminart 'bis zum'"
elif maintain_until_type == 23925195:
maintain_until_type = "Terminart 'dauerhaft'"
elif maintain_until_type == 322729640:
maintain_until_type = "Terminart 'nicht erforderlich'"
else:
maintain_until_type = "Unbekannte Terminart"
if finish_until is not None:
try:
deadline = compensation.deadlines.get(
type=DeadlineType.FINISHED,
date=finish_until,
comment=finish_until_type
)
except ObjectDoesNotExist:
deadline = Deadline.objects.create(
type=DeadlineType.FINISHED,
date=finish_until,
comment=finish_until_type
)
compensation.deadlines.add(deadline)
if maintain_until is not None:
try:
deadline = compensation.deadlines.get(
type=DeadlineType.MAINTAIN,
date=maintain_until,
comment=maintain_until_type
)
except ObjectDoesNotExist:
deadline = Deadline.objects.create(
type=DeadlineType.MAINTAIN,
date=maintain_until,
comment=maintain_until_type
)
compensation.deadlines.add(deadline)
tmp_cursor.close()
return compensation
def _migrate_actions(self, compensation, kom):
compensation.actions.all().delete()
kom_identifier = f"'{kom[0]}'"
tmp_cursor = self.db_connection.cursor()
tmp_cursor.execute(
'select '
'm."PKEY", '
'm."Massn_RLP" as atom_id_parent, '
'e."E_Massn" as atom_id_detail, '
'm."Proz_Anteil" as menge, '
'm."Massnahm_Typ" as einheit, '
'm."MassnBeschreib" '
'from "OBJ_MASTER" om '
'left join "MASSN" m on om."GISPADID"=m."GISPADID" '
'left join "Entwicklung" e on m."PKEY"=e."FKEY" '
'where '
f'om."KENNUNG"={kom_identifier} and '\
'm."Massn_RLP"!=0 '
)
db_results = tmp_cursor.fetchall()
for result in db_results:
if result[0] is None:
continue
action_pkey = f"'{result[0]}'"
selectable_action_parent = result[1]
selectable_action = result[2] # could be None!
amount = result[3]
unit = result[4]
comment = result[5] or ""
if selectable_action_parent in [999990, 888880, 777770, 0]:
# is garbage
selectable_action_parent = None
if selectable_action in [999999, 888888, 777777, 0]:
# is garbage
selectable_action = None
if selectable_action is None and selectable_action_parent is None:
# a giant load of garbage. Step over
continue
if unit == 710113:
unit = UnitChoices.m
elif unit == 710119:
# durchmesser - no such entries, can skip!
pass
elif unit == 710118:
unit = UnitChoices.st
elif unit == 710116:
unit = UnitChoices.ha
elif unit == 710114:
unit = UnitChoices.m2
elif unit == 710115:
unit = UnitChoices.m3
else:
unit = UnitChoices.st
comment += "\nDatenmigration: Mengeneinheit unbekannt"
if amount is None:
amount = 0.0
comment += "\nDatenmigration: Menge unbekannt"
tmp_cursor_z_code = self.db_connection.cursor()
tmp_cursor_z_code.execute(
'select '
'pe."Posi_E" as zusatzmerkmal '
'from "Posi_E" pe '
'where '
f'pe."FKEY"={action_pkey} and '
'pe."Posi_E"!=0 and '
'pe."Posi_E" is not null'
)
z_code_results = tmp_cursor_z_code.fetchall()
z_codes = []
for z_code_result in z_code_results:
try:
z_code = KonovaCode.objects.get(
atom_id=z_code_result[0],
code_lists__in=[CODELIST_COMPENSATION_ACTION_DETAIL_ID],
is_selectable=True,
is_archived=False
)
except ObjectDoesNotExist:
raise ObjectDoesNotExist(f"{z_code_result[0]}, {kom_identifier}")
z_codes.append(z_code)
action_code = selectable_action or selectable_action_parent
try:
action_code = KonovaCode.objects.get(
atom_id=action_code,
code_lists__in=[CODELIST_COMPENSATION_ACTION_ID],
)
except ObjectDoesNotExist:
raise ObjectDoesNotExist(f"{action_code}, {kom_identifier}")
try:
action = compensation.actions.get(
action_type__in=[action_code],
amount=amount,
unit=unit,
comment=comment
)
except ObjectDoesNotExist:
action = CompensationAction.objects.create(
amount=amount,
unit=unit,
comment=comment
)
compensation.actions.add(action)
action.action_type.set([action_code])
action.action_type_details.set(z_codes)
tmp_cursor_z_code.close()
tmp_cursor.close()
return compensation
class EmaMigrater(CompensationMigrater):
def migrate(self):
cursor = self.db_connection.cursor()
cursor.execute(
'select '
'om."KENNUNG", '
'linf."OBJBEZ", '
'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geomf.the_geom,4326)), 3))) as geomf, '
'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geoml.the_geom,4326)), 2))) as geoml, '
'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geomp.the_geom,4326)), 1))) as geomp, '
'linf."Bemerkung" '
'from "OBJ_MASTER" om '
'left join "LINFOS" linf on om."GISPADID"=linf."GISPADID" '
'left join kom on om."GISPADID"=kom.gispadid '
'left join geometry_f geomf on om."GISPADID"=geomf.gispadid '
'left join geometry_l geoml on om."GISPADID"=geoml.gispadid '
'left join geometry_p geomp on om."GISPADID"=geomp.gispadid '
'left join "Aufwertung" auf on om."GISPADID"=auf."GISPADID" '
'where '
'om."OKL"=7730090 and '
'om.archiv=false and '
'om.nicht_vollstaendig=0'
)
all_emas = cursor.fetchall()
len_all_emas = len(all_emas)
num_processed = 0
print(f"Migrate EMAs to emas...")
print(f"--Found {len_all_emas} entries. Process now...")
for ema in all_emas:
if num_processed % 500 == 0:
print(f"----{num_processed}/{len_all_emas} processed")
with transaction.atomic():
ema_identifier = ema[0]
ema_title = ema[1]
ema_comment = ema[5]
ema_obj = Ema.objects.get_or_create(
identifier=ema_identifier
)[0]
ema_obj.title = ema_title
ema_obj.comment = ema_comment
ema_obj = self._migrate_geometry(ema_obj, ema)
ema_obj = self._migrate_responsibility(ema_obj, ema)
ema_obj = self._migrate_compensation_type(ema_obj, ema)
ema_obj = self._migrate_states(ema_obj, ema)
ema_obj = self._migrate_deadlines(ema_obj, ema)
ema_obj = self._migrate_action_control_deadlines(ema_obj, ema)
ema_obj = self._migrate_actions(ema_obj, ema)
ema_obj = self._migrate_finance_volume_to_comment(ema_obj, ema)
ema_obj = self._migrate_documents(ema_obj, EmaDocument, ema)
ema_obj.save()
num_processed += 1
cursor.close()
def _migrate_deadlines(self, ema_obj, ema_result):
ema_obj.deadlines.all().delete()
ema_identifier = f"'{ema_result[0]}'"
tmp_cursor = self.db_connection.cursor()
tmp_cursor.execute(
'select '
't."Terminart", '
't."K_Termin"::date '
'from "OBJ_MASTER" om '
'left join "Termine" t on om."GISPADID"=t."GISPADID" '
'where '
f'om."KENNUNG"={ema_identifier}'
)
db_results = tmp_cursor.fetchall()
for result in db_results:
deadline_type = result[0]
deadline_comment = None
if deadline_type == 708166:
deadline_comment = "Wiedervorlage"
elif deadline_type == 708163:
deadline_comment = "Projektbeginn"
deadline_date = result[1]
if deadline_date is None:
# Useless data
continue
try:
deadline = ema_obj.deadlines.get(
type=DeadlineType.OTHER,
date=deadline_date,
comment=deadline_comment
)
except ObjectDoesNotExist:
deadline = Deadline.objects.create(
type=DeadlineType.OTHER,
date=deadline_date,
comment=deadline_comment
)
ema_obj.deadlines.add(deadline)
return ema_obj
def _migrate_responsibility(self, ema_obj, ema_result):
ema_identifier = f"'{ema_result[0]}'"
tmp_cursor = self.db_connection.cursor()
tmp_cursor.execute(
'select '
'adr."adr_pruef" as ets, '
'linf."AZ", '
'adr.behoerde, '
'adr.angaben '
'from "OBJ_MASTER" om '
'left join "LINFOS" linf on om."GISPADID"=linf."GISPADID" '
'left join adressrolle adr on adr."GISPADID"=om."GISPADID" '
'where '
f'om."KENNUNG"={ema_identifier} '
)
db_results = tmp_cursor.fetchall()
if len(db_results) != 1:
raise AssertionError(f"{ema_identifier} has invalid responsibilities: {db_results}")
db_results = db_results[0]
cons_office_code = db_results[0]
cons_file_number = db_results[1]
handler_type = db_results[2]
handler_detail = db_results[3]
responsible = ema_obj.responsible or Responsibility.objects.create()
try:
conservation_office = KonovaCode.objects.get(
atom_id=cons_office_code,
is_selectable=True,
is_archived=False,
code_lists__in=[CODELIST_CONSERVATION_OFFICE_ID]
)
except ObjectDoesNotExist:
raise ObjectDoesNotExist(f"{ema_identifier}, {db_results}")
try:
handler_type = KonovaCode.objects.get(
atom_id=handler_type,
is_selectable=True,
is_archived=False,
code_lists__in=[CODELIST_COMPENSATION_HANDLER_ID]
)
except ObjectDoesNotExist:
handler_type = None
responsible.conservation_file_number = db_results[1]
ema_obj.responsible.conservation_office = conservation_office
ema_obj.responsible.conservation_file_number = cons_file_number
handler = ema_obj.responsible.handler or Handler.objects.create()
handler.type = handler_type
handler.detail = handler_detail
ema_obj.responsible.handler = handler
ema_obj.responsible.handler.save()
ema_obj.responsible.save()
tmp_cursor.close()
return ema_obj
def _migrate_finance_volume_to_comment(self, ema_obj, ema_result):
ema_identifier = f"'{ema_result[0]}'"
tmp_cursor = self.db_connection.cursor()
tmp_cursor.execute(
'select '
'b.datum, '
'b.hoehe '
'from "OBJ_MASTER" om '
'left join bewilligung b on om."GISPADID"=b.gispadid '
'where '
f'om."KENNUNG"={ema_identifier} '
)
db_results = tmp_cursor.fetchall()
for result in db_results:
payment_date = result[0]
payment_amount = result[1]
comment_extra = f"\n\nFinanzierung bewilligt am {formats.localize(payment_date, use_l10n=True)} in Höhe von {formats.localize(payment_amount, use_l10n=True)}"
comment = ema_obj.comment or ""
if comment_extra in comment:
# skip
continue
comment += comment_extra
ema_obj.comment = comment
tmp_cursor.close()
return ema_obj