#132 WIP Deduction enhances

* enhances comment messages
* adds fallback calculation for deduction-account ratio in case of missing or unlogical values
* adds creation timestamp for migrated deductions, derived from old kom-deduction log/protokoll table
* fixes bug in eco account migration where legal changes would not be saved
* changes bug in compensation where direct data changes would not be saved
* improves update_all_parcels.py by catching all types of possible exceptions during the update procedure and adding the causing geometry to the try_again_ids list

konova/management/commands/kspMigrater/base_migrater.py

@@ -113,9 +113,6 @@ class BaseMigrater:
         tmp_cursor.close()
         return instance
 
-    def _migrate_alternative_log(self, instance, db_result: tuple):
-        return instance
-
     def _migrate_log(self, instance, db_result: tuple):
         identifier = f"'{db_result[0]}'"
         tmp_cursor = self.db_connection.cursor()

konova/management/commands/kspMigrater/compensation_migrater.py

@@ -61,7 +61,6 @@ class CompensationMigrater(BaseMigrater):
                 compensation.comment = kom_comment
                 try:
                     compensation = self._migrate_interventions_reference(compensation, kom)
-                    compensation.save()
                 except ObjectDoesNotExist:
                     compensation.delete()
                     unsuccessfull_compensations[kom_identifier] = "EIV does not exist"
@@ -77,6 +76,7 @@ class CompensationMigrater(BaseMigrater):
                 compensation = self._migrate_actions(compensation, kom)
                 compensation = self._migrate_log(compensation, kom)
                 compensation = self._migrate_documents(compensation, CompensationDocument, kom)
+                compensation.save()
                 num_processed += 1
         print("The following KOMs could not be migrated: ")
         for kom, val in unsuccessfull_compensations.items():
@@ -286,11 +286,11 @@ class CompensationMigrater(BaseMigrater):
             try:
                 control_type = control_types[control_type]
             except KeyError:
-                control_type = "Unbekannt"
+                control_type = "Kein Eintrag"
             try:
                 control_result = control_results[control_result]
             except KeyError:
-                control_result = "Unbekannt"
+                control_result = "Kein Eintrag"
 
             try:
                 control_responsible = KonovaCode.objects.get(
@@ -301,7 +301,7 @@ class CompensationMigrater(BaseMigrater):
                 )
                 control_responsible = control_responsible.long_name
             except ObjectDoesNotExist:
-                control_responsible = "Unbekannt"
+                control_responsible = "Kein Eintrag"
 
             control_comment += f"\n\nKontrolltyp: {control_type}"
             control_comment += f"\nKontrollergebnis: {control_result}"
@@ -353,7 +353,7 @@ class CompensationMigrater(BaseMigrater):
             elif finish_until_type == 889700393:
                 finish_until_type = "Termin vorläufig"
             else:
-                finish_until_type = "Unbekannt ob Termin vorläufig oder endgültig"
+                finish_until_type = "Kein Eintrag"
 
             if maintain_until_type == 807238388:
                 maintain_until_type = "Terminart 'bis zum'"
@@ -362,7 +362,7 @@ class CompensationMigrater(BaseMigrater):
             elif maintain_until_type == 322729640:
                 maintain_until_type = "Terminart 'nicht erforderlich'"
             else:
-                maintain_until_type = "Unbekannte Terminart"
+                maintain_until_type = "Kein Eintrag zur Terminart"
 
             if finish_until is not None:
                 try:
@@ -455,11 +455,11 @@ class CompensationMigrater(BaseMigrater):
                 unit = UnitChoices.m3
             else:
                 unit = UnitChoices.st
-                comment += "\nDatenmigration: Mengeneinheit unbekannt"
+                comment += "\nDatenmigration: Kein Eintrag zur Mengeneinheit"
 
             if amount is None:
                 amount = 0.0
-                comment += "\nDatenmigration: Menge unbekannt"
+                comment += "\nDatenmigration: Kein Eintrag zur Menge"
 
             tmp_cursor_z_code = self.db_connection.cursor()
             tmp_cursor_z_code.execute(

konova/management/commands/kspMigrater/eco_account_migrater.py

@@ -1,8 +1,10 @@
 import datetime
 
+from django.contrib.auth.models import Group
 from django.contrib.gis.geos import MultiPolygon, Polygon
 from django.core.exceptions import ObjectDoesNotExist
 from django.db import transaction
+from django.utils import timezone
 
 from codelist.models import KonovaCode
 from codelist.settings import CODELIST_CONSERVATION_OFFICE_ID, CODELIST_COMPENSATION_HANDLER_ID
@@ -10,7 +12,9 @@ from compensation.models import EcoAccount, EcoAccountDocument, EcoAccountDeduct
 from intervention.models import Responsibility, Handler, Intervention, Legal
 from konova.management.commands.kspMigrater.compensation_migrater import CompensationMigrater
 from konova.models import Geometry
+from konova.settings import DEFAULT_GROUP
 from konova.sub_settings.lanis_settings import DEFAULT_SRID_RLP, DEFAULT_SRID
+from user.models import User, UserActionLogEntry
 
 
 class EcoAccountMigrater(CompensationMigrater):
@@ -32,7 +36,6 @@ class EcoAccountMigrater(CompensationMigrater):
                 'left join geometry_f geomf on om."GISPADID"=geomf.gispadid '
                 'left join geometry_l geoml on om."GISPADID"=geoml.gispadid '
                 'left join geometry_p geomp on om."GISPADID"=geomp.gispadid '
-                'left join "Aufwertung" auf on om."GISPADID"=auf."GISPADID" '
             'where '
                 'om."OKL"=7730081 and '
                 'om.archiv=false and '
@@ -51,7 +54,7 @@ class EcoAccountMigrater(CompensationMigrater):
             with transaction.atomic():
                 oek_identifier = oek[0]
                 oek_title = oek[1]
-                oek_comment = oek[5]
+                oek_comment = oek[5] or ""
                 eco_account = EcoAccount.objects.get_or_create(
                     identifier=oek_identifier
                 )[0]
@@ -220,6 +223,10 @@ class EcoAccountMigrater(CompensationMigrater):
             if target_intervention_identifier is None:
                 # old garbage data - skip
                 continue
+
+            if deduction_amount_percentage is None or float(deduction_amount_percentage) > 100.0:
+                deduction_amount_percentage = self.__calculate_deduction_amount_percentage(eco_account, old_deduction_kom_identifier)
+
             try:
                 intervention = Intervention.objects.get(
                     identifier=target_intervention_identifier
@@ -233,7 +240,7 @@ class EcoAccountMigrater(CompensationMigrater):
             rest_after_deduction = rest_available_surface - deduction_amount_sqm
 
             if rest_after_deduction < 0:
-                print(f"{identifier} has {eco_account.deductable_surface} sqm left but old deduction {old_deduction_kom_identifier} requires {deduction_amount_sqm} sqm.")
+                print(f"{identifier} has {rest_available_surface} sqm left but old deduction {old_deduction_kom_identifier} requires {deduction_amount_sqm} sqm.")
                 print(f"Increase deductable surface by {rest_after_deduction} sqm")
                 eco_account.deductable_surface += abs(rest_after_deduction)
                 eco_account.save()
@@ -242,7 +249,10 @@ class EcoAccountMigrater(CompensationMigrater):
                 account=eco_account,
                 surface=deduction_amount_sqm,
                 intervention=intervention
-            )
+            )[0]
+            created_on = self.__fetch_deduction_create_date(old_deduction_kom_identifier)
+            deduction.created = created_on
+            deduction.save()
 
         tmp_cursor.close()
         return eco_account
@@ -251,5 +261,89 @@ class EcoAccountMigrater(CompensationMigrater):
         # Just add default dummy values, since there has never been data for this previously
         eco_account.legal = eco_account.legal or Legal()
         eco_account.legal.registration_date = datetime.date.fromisoformat("1970-01-01")
+        eco_account.legal.save()
         eco_account.comment += "\nKein Vereinbarungsdatum eingetragen. Platzhalter 01.01.1970 gesetzt."
-        return eco_account
+        return eco_account
+
+    def __calculate_deduction_amount_percentage(self, eco_account, kom_deduction_identifier):
+        """ Calculates the amount of a deduction from an eco account in percentage.
+
+        Depends on the geometry of the old KOM-deduction
+
+        """
+        kom_deduction_identifier = f"'{kom_deduction_identifier}'"
+        result = 0.0
+        tmp_cursor = self.db_connection.cursor()
+        tmp_cursor.execute(
+            'select '
+                f'st_area(st_transform(geom.the_geom,{DEFAULT_SRID_RLP})) '
+            'from "OBJ_MASTER" om '
+                'join geometry_f geom on om."GISPADID"=geom.gispadid '
+            'where '
+                f'om."KENNUNG"={kom_deduction_identifier}'
+        )
+        fetch_result = tmp_cursor.fetchall()
+        area_surface = fetch_result[0][0]
+        tmp_cursor.close()
+
+        result = float(area_surface / eco_account.deductable_surface) * 100
+
+        return result
+
+    def __fetch_deduction_create_date(self, deduction_kom_identifier):
+        """ Fetches the creation timestamp for the old KOM-deduction to be used as create timestamp for
+         the migrated deduction entry
+
+        """
+        deduction_kom_identifier = f"'{deduction_kom_identifier}'"
+        tmp_cursor = self.db_connection.cursor()
+        tmp_cursor.execute(
+            'select '
+                'log.erstelltam, '
+                'log.erstelltvon '
+            'from "OBJ_MASTER" om '
+                'join log on om."GISPADID"=log.gispadid::Integer '
+            'where '
+                f'om."KENNUNG"={deduction_kom_identifier} '
+            'order by log.erstelltam '
+            'limit 1'
+        )
+        fetch_results = tmp_cursor.fetchall()
+        if len(fetch_results) == 0:
+            tmp_cursor.execute(
+                'select '
+                    'p.geaendertam, '
+                    'p.geaendertvon '
+                'from "OBJ_MASTER" om '
+                    'join protokoll p on om."GISPADID"=p."FKEY" '
+                'where '
+                    f'om."KENNUNG"={deduction_kom_identifier} '
+                'order by p.geaendertam '
+                'limit 1'
+            )
+            fetch_results = tmp_cursor.fetchall()
+            if len(fetch_results) == 0:
+                return None
+        create_ts = fetch_results[0][0]
+        create_user = fetch_results[0][1]
+
+        user = User.objects.get_or_create(
+            username=create_user,
+        )
+        is_new = user[1]
+        user = user[0]
+        if is_new:
+            user.is_active = False
+            user.first_name = "MIGRIERT"
+            user.last_name = "MIGRIERT"
+            user.save()
+
+        # Make sure user has at least the default group set
+        default_group = Group.objects.get(name=DEFAULT_GROUP)
+        user.groups.add(default_group)
+        tmp_cursor.close()
+
+        create_action = UserActionLogEntry.get_created_action(user, comment="[Migriert] Abbuchung angelegt")
+        create_action.timestamp = timezone.make_aware(create_ts)
+        create_action.save()
+        return create_action

konova/management/commands/update_all_parcels.py

@@ -7,6 +7,8 @@ Created on: 04.01.22
 """
 from pyexpat import ExpatError
 
+from requests.exceptions import ProxyError
+
 from konova.management.commands.setup import BaseKonovaCommand
 from konova.models import Geometry, Parcel, District
 
@@ -47,7 +49,7 @@ class Command(BaseKonovaCommand):
                 self._write_warning(f"--- {i}/{total_count} processed")
             try:
                 geometry.update_parcels()
-            except ExpatError as e:
+            except Exception as e:
                 self._write_error(f"--- {geometry.id} encountered an error: {e}. Geometry added to the retry list.")
                 try_again_ids.append(geometry.id)
             i += 1