#132 Compensation migrater enhancements

* adds further catches for data garbage
* adds action migrater
* extends UnitChoices

compensation/models/action.py

@@ -20,7 +20,8 @@ class UnitChoices(models.TextChoices):
     cm = "cm", _("cm")
     m = "m", _("m")
     km = "km", _("km")
-    qm = "qm", _("m²")
+    m2 = "m2", _("m²")
+    m3 = "m3", _("m³")
     ha = "ha", _("ha")
     st = "pcs", _("Pieces")  # pieces
 

konova/management/commands/kspMigrater/compensation_migrater.py

@@ -2,8 +2,9 @@ from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
 from django.db import transaction
 
 from codelist.models import KonovaCode
-from codelist.settings import CODELIST_BIOTOPES_ID, CODELIST_BIOTOPES_EXTRA_CODES_ID
+from codelist.settings import CODELIST_BIOTOPES_ID, CODELIST_BIOTOPES_EXTRA_CODES_ID, \
-from compensation.models import Compensation, CompensationState
+    CODELIST_COMPENSATION_ACTION_DETAIL_ID, CODELIST_COMPENSATION_ACTION_ID
+from compensation.models import Compensation, CompensationState, CompensationAction, UnitChoices
 from intervention.models import Responsibility, Handler, Intervention
 from konova.management.commands.kspMigrater.base_migrater import BaseMigrater
 from konova.models import Deadline, DeadlineType
@@ -33,7 +34,7 @@ class CompensationMigrater(BaseMigrater):
                 'om."OKL"=7730080 and '
                 'om.archiv=false and '
                 f'(auf."Infos" is null or auf."Infos"={empty_str}) and '
-                'om.nicht_vollstaendig=0;'
+                'om.nicht_vollstaendig=0'
         )
 
         all_koms = cursor.fetchall()
@@ -61,6 +62,7 @@ class CompensationMigrater(BaseMigrater):
                 compensation = self._migrate_compensation_type(compensation, kom)
                 compensation = self._migrate_states(compensation, kom)
                 compensation = self._migrate_deadlines(compensation, kom)
+                compensation = self._migrate_actions(compensation, kom)
                 try:
                     compensation = self._migrate_interventions_reference(compensation, kom)
                     compensation.save()
@@ -114,8 +116,12 @@ class CompensationMigrater(BaseMigrater):
             'from "BtypHtyp" b ' \
                 'left join "OBJ_MASTER" om on om."GISPADID"=b."GISPADID" ' \
             'where ' \
-                f'om."KENNUNG"={kom_identifier}' 
+                f'om."KENNUNG"={kom_identifier} and ' \
-        
+                'b."Biotoptyp"!=0 and '\
+                'b."Biotoptyp" is not null '
+
+        zero_str = "'0'"
+        z_code_before_states_sql = 'select z."Zusatzcode" from "Zusatzcodes" z where z."FKEY"={}'
         after_states_sql = 'select ' \
                 'z."Zielbiotoptyp", ' \
                 'z."Flaeche", ' \
@@ -123,11 +129,15 @@ class CompensationMigrater(BaseMigrater):
             'from "Zielbiotoptypen" z ' \
                 'left join "OBJ_MASTER" om on om."GISPADID"=z."GISPADID" ' \
             'where ' \
-                f'om."KENNUNG"={kom_identifier}' 
+                f'om."KENNUNG"={kom_identifier} and '\
-        z_code_sql = 'select z."Zusatzcode" from "Zusatzcodes" z where z."FKEY"={}'
+                f'z."Zielbiotoptyp"!={zero_str} and '\
-        
+                'z."Zielbiotoptyp" is not null '
-        compensation = self._process_state_migration(compensation, before_states_sql, z_code_sql, compensation.before_states)
+        z_code_after_states_sql = 'select z."Z_Code" from "Z_Codes" z where z."FKEY"={}'
-        compensation = self._process_state_migration(compensation, after_states_sql, z_code_sql, compensation.after_states)
+
+        compensation.before_states.all().delete()
+        compensation = self._process_state_migration(compensation, before_states_sql, z_code_before_states_sql, compensation.before_states)
+        compensation.after_states.all().delete()
+        compensation = self._process_state_migration(compensation, after_states_sql, z_code_after_states_sql, compensation.after_states)
         return compensation
 
     def _process_state_migration(self, compensation, sql, z_code_sql, state_manager):
@@ -137,12 +147,18 @@ class CompensationMigrater(BaseMigrater):
 
         for result in db_result:
             state_type = result[0]
+            if state_type is None or len(str(state_type)) == 0:
+                # garbage
+                continue
             state_surface = result[1] or 0.0
             pkey_entry = f"'{result[2]}'"
-            state_code = KonovaCode.objects.get(
+            try:
-                atom_id=state_type,
+                state_code = KonovaCode.objects.get(
-                code_lists__in=[CODELIST_BIOTOPES_ID]
+                    atom_id=state_type,
-            )
+                    code_lists__in=[CODELIST_BIOTOPES_ID]
+                )
+            except ObjectDoesNotExist:
+                raise ObjectDoesNotExist(f"{state_type}, {compensation.identifier}")
             tmp_cursor_z_code = self.db_connection.cursor()
             tmp_cursor_z_code.execute(z_code_sql.format(pkey_entry))
             z_code_results = tmp_cursor_z_code.fetchall()
@@ -202,6 +218,7 @@ class CompensationMigrater(BaseMigrater):
         return compensation
 
     def _migrate_deadlines(self, compensation, kom):
+        compensation.deadlines.all().delete()
         kom_identifier = f"'{kom[0]}'"
         tmp_cursor = self.db_connection.cursor()
         tmp_cursor.execute(
@@ -270,5 +287,124 @@ class CompensationMigrater(BaseMigrater):
                     )
                     compensation.deadlines.add(deadline)
 
+        tmp_cursor.close()
+        return compensation
+
+    def _migrate_actions(self, compensation, kom):
+        compensation.actions.all().delete()
+
+        kom_identifier = f"'{kom[0]}'"
+        tmp_cursor = self.db_connection.cursor()
+        tmp_cursor.execute(
+            'select '
+                'm."PKEY", '
+                'm."Massn_RLP" as atom_id_parent, '
+                'e."E_Massn" as atom_id_detail, '
+                'm."Proz_Anteil" as menge, '
+                'm."Massnahm_Typ" as einheit, '
+                'm."MassnBeschreib" '
+            'from "OBJ_MASTER" om '
+                'left join "MASSN" m on om."GISPADID"=m."GISPADID" '
+                'left join "Entwicklung" e on m."PKEY"=e."FKEY" '
+            'where '
+                f'om."KENNUNG"={kom_identifier} and '\
+                'm."Massn_RLP"!=0 '
+        )
+        db_results = tmp_cursor.fetchall()
+
+        for result in db_results:
+            if result[0] is None:
+                continue
+            action_pkey = f"'{result[0]}'"
+            selectable_action_parent = result[1]
+            selectable_action = result[2]  # could be None!
+            amount = result[3]
+            unit = result[4]
+            comment = result[5] or ""
+
+            if selectable_action_parent in [999990, 888880, 777770, 0]:
+                # is garbage
+                selectable_action_parent = None
+            if selectable_action in [999999, 888888, 777777, 0]:
+                # is garbage
+                selectable_action = None
+
+            if selectable_action is None and selectable_action_parent is None:
+                # a giant load of garbage. Step over
+                continue
+
+            if unit == 710113:
+                unit = UnitChoices.m
+            elif unit == 710119:
+                # durchmesser - no such entries, can skip!
+                pass
+            elif unit == 710118:
+                unit = UnitChoices.st
+            elif unit == 710116:
+                unit = UnitChoices.ha
+            elif unit == 710114:
+                unit = UnitChoices.m2
+            elif unit == 710115:
+                unit = UnitChoices.m3
+            else:
+                unit = UnitChoices.st
+                comment += "\nDatenmigration: Mengeneinheit unbekannt"
+
+            if amount is None:
+                amount = 0.0
+                comment += "\nDatenmigration: Menge unbekannt"
+
+            tmp_cursor_z_code = self.db_connection.cursor()
+            tmp_cursor_z_code.execute(
+                'select '
+                    'pe."Posi_E" as zusatzmerkmal '
+                'from "Posi_E" pe '
+                'where '
+                    f'pe."FKEY"={action_pkey} and '
+                    'pe."Posi_E"!=0 and '
+                    'pe."Posi_E" is not null'
+            )
+            z_code_results = tmp_cursor_z_code.fetchall()
+            z_codes = []
+            for z_code_result in z_code_results:
+                try:
+                    z_code = KonovaCode.objects.get(
+                        atom_id=z_code_result[0],
+                        code_lists__in=[CODELIST_COMPENSATION_ACTION_DETAIL_ID],
+                        is_selectable=True,
+                        is_archived=False
+                    )
+                except ObjectDoesNotExist:
+                    raise ObjectDoesNotExist(f"{z_code_result[0]}, {kom_identifier}")
+                z_codes.append(z_code)
+
+            action_code = selectable_action or selectable_action_parent
+            try:
+                action_code = KonovaCode.objects.get(
+                    atom_id=action_code,
+                    code_lists__in=[CODELIST_COMPENSATION_ACTION_ID],
+                )
+            except ObjectDoesNotExist:
+                raise ObjectDoesNotExist(f"{action_code}, {kom_identifier}")
+            try:
+                action = compensation.actions.get(
+                    action_type__in=[action_code],
+                    amount=amount,
+                    unit=unit,
+                    comment=comment
+                )
+            except ObjectDoesNotExist:
+                action = CompensationAction.objects.create(
+                    amount=amount,
+                    unit=unit,
+                    comment=comment
+                )
+                compensation.actions.add(action)
+            action.action_type.set([action_code])
+            action.action_type_details.set(z_codes)
+
+            tmp_cursor_z_code.close()
+
+
         tmp_cursor.close()
         return compensation