konova/konova/management/commands/kspMigrater/eco_account_migrater.py

import datetime

from django.contrib.auth.models import Group
from django.contrib.gis.geos import MultiPolygon, Polygon
from django.core.exceptions import ObjectDoesNotExist
from django.db import transaction
from django.utils import timezone

from codelist.models import KonovaCode
from codelist.settings import CODELIST_CONSERVATION_OFFICE_ID, CODELIST_COMPENSATION_HANDLER_ID
from compensation.models import EcoAccount, EcoAccountDocument, EcoAccountDeduction
from intervention.models import Responsibility, Handler, Intervention, Legal
from konova.management.commands.kspMigrater.compensation_migrater import CompensationMigrater
from konova.models import Geometry
from konova.settings import DEFAULT_GROUP
from konova.sub_settings.lanis_settings import DEFAULT_SRID_RLP, DEFAULT_SRID
from user.models import User, UserActionLogEntry


class EcoAccountMigrater(CompensationMigrater):

    def migrate(self):
        self.connect_db()
        cursor = self.db_connection.cursor()
        cursor.execute(
            'select '
                'om."KENNUNG", '
                'linf."OBJBEZ", '
                'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geomf.the_geom,4326)), 3))) as geomf, '
                'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geoml.the_geom,4326)), 2))) as geoml, '
                'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geomp.the_geom,4326)), 1))) as geomp, '
                'linf."Bemerkung" '
            'from "OBJ_MASTER" om '
                'left join "LINFOS" linf on om."GISPADID"=linf."GISPADID" '
                'left join kom on om."GISPADID"=kom.gispadid '
                'left join geometry_f geomf on om."GISPADID"=geomf.gispadid '
                'left join geometry_l geoml on om."GISPADID"=geoml.gispadid '
                'left join geometry_p geomp on om."GISPADID"=geomp.gispadid '
            'where '
                'om."OKL"=7730081 and '
                'om.archiv=false and '
                'om.nicht_vollstaendig=0 '
        )

        all_oeks = cursor.fetchall()
        len_all_oeks = len(all_oeks)
        num_processed = 0
        print(f"Migrate OEKs to ecoaccounts...")
        print(f"--Found {len_all_oeks} entries. Process now...")
        for oek in all_oeks:
            if num_processed % 500 == 0:
                print(f"----{num_processed}/{len_all_oeks} processed")

            with transaction.atomic():
                oek_identifier = oek[0]
                oek_title = oek[1]
                oek_comment = oek[5] or ""
                eco_account = EcoAccount.objects.get_or_create(
                    identifier=oek_identifier
                )[0]

                eco_account.title = oek_title
                eco_account.comment = oek_comment
                eco_account = self._migrate_legal(eco_account, oek)
                eco_account = self._migrate_geometry(eco_account, oek)
                eco_account = self._migrate_responsibility(eco_account, oek)
                eco_account = self._migrate_states(eco_account, oek)
                eco_account = self._migrate_deadlines(eco_account, oek)
                eco_account = self._migrate_action_control_deadlines(eco_account, oek)
                eco_account = self._migrate_actions(eco_account, oek)
                eco_account = self._migrate_log(eco_account, oek)
                eco_account = self._migrate_deductions(eco_account, oek)
                eco_account = self._migrate_documents(eco_account, EcoAccountDocument, oek)
                eco_account.save()

                num_processed += 1
        cursor.close()

    def _migrate_geometry(self, instance, db_result: tuple):
        identifier = f"'{db_result[0]}'"
        empty_str = f"''"
        tmp_cursor = self.db_connection.cursor()
        tmp_cursor.execute(
            'select '
                'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Union(ST_Transform(geom.the_geom,4326)))))) '
            'from "OBJ_MASTER" om '
                'left join "Aufwertung" auf on om."GISPADID"=auf."GISPADID" '
                'left join "OBJ_MASTER" om_kom on auf."Infos"=om_kom."KENNUNG" '
                'left join geometry_f geom on geom.gispadid=om_kom."GISPADID" '
            'where '
                'om."OKL"=7730081 and '
                'om.archiv=false and '
                'om_kom.archiv=false and '
                f'(auf."Infos" is not null and auf."Infos"!={empty_str}) and '
                f'om."KENNUNG"={identifier}'
        )
        deduction_db_results = tmp_cursor.fetchall()
        if len(deduction_db_results) != 1:
            raise AssertionError(f"Unexpected number of matches: {deduction_db_results}")
        db_result_geom = deduction_db_results[0][0]
        if db_result_geom is not None:
            deductions_db_result_geom = MultiPolygon.from_ewkt(deduction_db_results[0][0])
        else:
            deductions_db_result_geom = MultiPolygon(srid=DEFAULT_SRID)

        tmp_cursor.execute(
            'select '
                'ST_AsEWKT(ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(geomf.the_geom,4326)), 3))) as geomf '
            'from "OBJ_MASTER" om '
                'left join geometry_f geomf on om."GISPADID"=geomf.gispadid '
            'where '
                f'om."KENNUNG"={identifier}'
        )
        original_geom_db_results = tmp_cursor.fetchall()
        if len(original_geom_db_results) != 1:
            raise AssertionError(f"Unexpected number of matches: {original_geom_db_results}")
        account_db_result_geom = MultiPolygon.from_ewkt(original_geom_db_results[0][0])

        db_result_geom = account_db_result_geom.union(deductions_db_result_geom)
        if isinstance(db_result_geom, Polygon):
            db_result_geom = MultiPolygon(db_result_geom, srid=DEFAULT_SRID)
        instance.geometry = instance.geometry or Geometry()
        try:
            # Calculate area by transforming
            rlp_geom = db_result_geom.transform(ct=DEFAULT_SRID_RLP, clone=True)
            area = round(rlp_geom.area)
            instance.deductable_surface = area
            instance.geometry.geom = db_result_geom if not db_result_geom.empty else None
            instance.geometry.save()
        except TypeError:
            raise TypeError(f"{identifier}, {db_result_geom}")
        tmp_cursor.close()
        return instance

    def _migrate_responsibility(self, eco_account, oek):
        fallback = "Kein Eintrag"
        acc_identifier = f"'{oek[0]}'"
        tmp_cursor = self.db_connection.cursor()
        tmp_cursor.execute(
            'select '
                'adr."adr_pruef" as ets, '
                'linf."AZ", '
                'oek.traeger, '
                'oek.bemerkungtraeger '
            'from "OBJ_MASTER" om '
                'left join "LINFOS" linf on om."GISPADID"=linf."GISPADID" '
                'left join adressrolle adr on adr."GISPADID"=om."GISPADID" '
                'left join oek on om."GISPADID"=oek.gispadid '
            'where '
                f'om."KENNUNG"={acc_identifier}'
        )
        db_results = tmp_cursor.fetchall()
        if len(db_results) != 1:
            raise AssertionError(f"{acc_identifier} has invalid responsibilities: {db_results}")

        db_results = db_results[0]
        cons_office_code = db_results[0]
        cons_file_number = db_results[1]
        handler_type = db_results[2]
        handler_detail = db_results[3]

        if cons_file_number is None or len(cons_file_number) == 0:
            cons_file_number = fallback

        eco_account.responsible = eco_account.responsible or Responsibility.objects.create()
        try:
            conservation_office = KonovaCode.objects.get(
                atom_id=cons_office_code,
                is_selectable=True,
                is_archived=False,
                code_lists__in=[CODELIST_CONSERVATION_OFFICE_ID]
            )
            self._migrate_responsible_code_to_team(eco_account, conservation_office, "ETS")
        except ObjectDoesNotExist:
            raise ObjectDoesNotExist(f"{acc_identifier}, {db_results}")
        try:
            handler_type = KonovaCode.objects.get(
                atom_id=handler_type,
                is_selectable=True,
                is_archived=False,
                code_lists__in=[CODELIST_COMPENSATION_HANDLER_ID]
            )
        except ObjectDoesNotExist:
            handler_type = None

        eco_account.responsible.conservation_office = conservation_office
        eco_account.responsible.conservation_file_number = cons_file_number
        handler = eco_account.responsible.handler or Handler.objects.create()
        handler.type = handler_type
        handler.detail = handler_detail
        eco_account.responsible.handler = handler
        eco_account.responsible.handler.save()
        eco_account.responsible.save()

        tmp_cursor.close()
        return eco_account

    def _migrate_deductions(self, eco_account, oek):
        identifier = f"'{oek[0]}'"
        empty_str = "''"
        tmp_cursor = self.db_connection.cursor()
        tmp_cursor.execute(
            'select '
                'om_kom."KENNUNG", '
                'auf."Anteil", '
                'ref."REFERENZ" '
            'from "OBJ_MASTER" om '
                'left join "Aufwertung" auf on om."GISPADID"=auf."GISPADID" '
                'left join "OBJ_MASTER" om_kom on auf."Infos"=om_kom."KENNUNG" '
                'left join "REFERENZ" ref on om_kom."GISPADID"=ref."GISPADID" '
            'where '
                f'(auf."Infos" is not null and auf."Infos"!={empty_str}) and '
                f'(ref."REFERENZ" is not null and ref."REFERENZ"!={empty_str}) and '
                f'om."KENNUNG"={identifier}'
        )
        fetched_results = tmp_cursor.fetchall()

        eco_account.deductions.all().delete()
        for result in fetched_results:
            old_deduction_kom_identifier = result[0]
            deduction_amount_percentage = result[1]
            target_intervention_identifier = result[2]
            if target_intervention_identifier is None:
                # old garbage data - skip
                continue

            if deduction_amount_percentage is None or float(deduction_amount_percentage) > 100.0:
                deduction_amount_percentage = self.__calculate_deduction_amount_percentage(eco_account, old_deduction_kom_identifier)

            try:
                intervention = Intervention.objects.get(
                    identifier=target_intervention_identifier
                )
            except ObjectDoesNotExist:
                # old garbage data
                print(f"{identifier} has deduction for {target_intervention_identifier} which does not exist")
                continue
            deduction_amount_sqm = round(eco_account.deductable_surface * (float(deduction_amount_percentage)/100))
            rest_available_surface = eco_account.deductable_surface - eco_account.get_deductions_surface()
            rest_after_deduction = rest_available_surface - deduction_amount_sqm

            if rest_after_deduction < 0:
                print(f"{identifier} has {rest_available_surface} sqm left but old deduction {old_deduction_kom_identifier} requires {deduction_amount_sqm} sqm.")
                print(f"Increase deductable surface by {rest_after_deduction} sqm")
                eco_account.deductable_surface += abs(rest_after_deduction)
                eco_account.save()

            deduction = EcoAccountDeduction.objects.get_or_create(
                account=eco_account,
                surface=deduction_amount_sqm,
                intervention=intervention
            )[0]
            created_on = self.__fetch_deduction_create_date(old_deduction_kom_identifier)
            deduction.created = created_on
            deduction.save()

        tmp_cursor.close()
        return eco_account

    def _migrate_legal(self, eco_account, oek):
        # Just add default dummy values, since there has never been data for this previously
        eco_account.legal = eco_account.legal or Legal()
        eco_account.legal.registration_date = datetime.date.fromisoformat("1970-01-01")
        eco_account.legal.save()
        eco_account.comment += "\nKein Vereinbarungsdatum eingetragen. Platzhalter 01.01.1970 gesetzt."
        return eco_account

    def __calculate_deduction_amount_percentage(self, eco_account, kom_deduction_identifier):
        """ Calculates the amount of a deduction from an eco account in percentage.

        Depends on the geometry of the old KOM-deduction

        """
        kom_deduction_identifier = f"'{kom_deduction_identifier}'"
        result = 0.0
        tmp_cursor = self.db_connection.cursor()
        tmp_cursor.execute(
            'select '
                f'st_area(st_transform(geom.the_geom,{DEFAULT_SRID_RLP})) '
            'from "OBJ_MASTER" om '
                'join geometry_f geom on om."GISPADID"=geom.gispadid '
            'where '
                f'om."KENNUNG"={kom_deduction_identifier}'
        )
        fetch_result = tmp_cursor.fetchall()
        area_surface = fetch_result[0][0]
        tmp_cursor.close()

        result = float(area_surface / eco_account.deductable_surface) * 100

        return result

    def __fetch_deduction_create_date(self, deduction_kom_identifier):
        """ Fetches the creation timestamp for the old KOM-deduction to be used as create timestamp for
         the migrated deduction entry

        """
        deduction_kom_identifier = f"'{deduction_kom_identifier}'"
        tmp_cursor = self.db_connection.cursor()
        tmp_cursor.execute(
            'select '
                'log.erstelltam, '
                'log.erstelltvon '
            'from "OBJ_MASTER" om '
                'join log on om."GISPADID"=log.gispadid::Integer '
            'where '
                f'om."KENNUNG"={deduction_kom_identifier} '
            'order by log.erstelltam '
            'limit 1'
        )
        fetch_results = tmp_cursor.fetchall()
        if len(fetch_results) == 0:
            tmp_cursor.execute(
                'select '
                    'p.geaendertam, '
                    'p.geaendertvon '
                'from "OBJ_MASTER" om '
                    'join protokoll p on om."GISPADID"=p."FKEY" '
                'where '
                    f'om."KENNUNG"={deduction_kom_identifier} '
                'order by p.geaendertam '
                'limit 1'
            )
            fetch_results = tmp_cursor.fetchall()
            if len(fetch_results) == 0:
                return None
        create_ts = fetch_results[0][0]
        create_user = fetch_results[0][1]

        user = User.objects.get_or_create(
            username=create_user,
        )
        is_new = user[1]
        user = user[0]
        if is_new:
            user.is_active = False
            user.first_name = "MIGRIERT"
            user.last_name = "MIGRIERT"
            user.save()

        # Make sure user has at least the default group set
        default_group = Group.objects.get(name=DEFAULT_GROUP)
        user.groups.add(default_group)
        tmp_cursor.close()

        create_action = UserActionLogEntry.get_created_action(user, comment="[Migriert] Abbuchung angelegt")
        create_action.timestamp = timezone.make_aware(create_ts)
        create_action.save()
        return create_action