#132 WIP migration improvements

* set automatically DEFAULT_GROUP to migrated users
* fixes bug where GISPADID could not be resolved properly, since it is suddenly named FKEY in source table (protokoll)
* adds fallback for protokoll table migration, since some logs can be found in a table called "log"
* set created and modified attributes of a migrated instance according to their first and last log entries
* automatically create teams from previous registration and conservation office access roles
    * need to be populated with users using data from natportal migration
* checks first if a KOM's EIV exists before performing all (time consuming) migration steps

konova/management/commands/kspMigrater/base_migrater.py

@@ -1,13 +1,15 @@
 from abc import abstractmethod
 
 import psycopg2
+from django.contrib.auth.models import Group
 from django.contrib.gis.geos import GEOSException, MultiPolygon, Polygon, MultiPoint, MultiLineString
 from django.core.exceptions import ObjectDoesNotExist
 from django.core.files.uploadedfile import UploadedFile
 from django.utils.timezone import make_aware
 
 from konova.models import Geometry
-from user.models import User, UserActionLogEntry, UserAction
+from konova.settings import DEFAULT_GROUP
+from user.models import User, UserActionLogEntry, UserAction, Team
 
 
 class BaseMigrater:
@@ -111,6 +113,9 @@ class BaseMigrater:
         tmp_cursor.close()
         return instance
 
+    def _migrate_alternative_log(self, instance, db_result: tuple):
+        return instance
+
     def _migrate_log(self, instance, db_result: tuple):
         identifier = f"'{db_result[0]}'"
         tmp_cursor = self.db_connection.cursor()
@@ -120,11 +125,27 @@ class BaseMigrater:
                 'p.geaendertam, '
                 'p.geaendertvon '
             'from "OBJ_MASTER" om '
-                'join protokoll p on om."GISPADID"=p."GISPADID" '
+                'join protokoll p on om."GISPADID"=p."FKEY" '
             'where '
                 f'om."KENNUNG"={identifier}'
         )
         fetch_results = tmp_cursor.fetchall()
+
+        if len(fetch_results) == 0:
+            # For whatever reason, there is another logging table. So if there are no entries on "protokoll", we
+            # need to take a look on "log" as a fallback
+            tmp_cursor.execute(
+                'select '
+                    'l.nachricht, '
+                    'l.erstelltam, '
+                    'l.erstelltvon '
+                'from "OBJ_MASTER" om '
+                    'join log l on om."GISPADID"=l.gispadid::Integer '
+                'where '
+                    f'om."KENNUNG"={identifier}'
+            )
+            fetch_results = tmp_cursor.fetchall()
+
         instance.log.all().delete()
         for result in fetch_results:
             comment = result[0]
@@ -141,6 +162,10 @@ class BaseMigrater:
                 user.last_name = "MIGRIERT"
                 user.save()
 
+            # Make sure user has at least the default group set
+            default_group = Group.objects.get(name=DEFAULT_GROUP)
+            user.groups.add(default_group)
+
             try:
                 action = instance.log.get(
                     user=user,
@@ -160,5 +185,29 @@ class BaseMigrater:
                 action.save()
             instance.log.add(action)
 
+        first_entry = instance.log.order_by("-timestamp").first()
+        last_entry = instance.log.order_by("timestamp").first()
+
+        if first_entry is not None:
+            instance.created = UserActionLogEntry.get_created_action(first_entry.user)
+            instance.created.timestamp = first_entry.timestamp
+            instance.created.save()
+        if last_entry is not None:
+            instance.modified = last_entry
+
         tmp_cursor.close()
         return instance
+
+    def _migrate_responsible_code_to_team(self, instance, responsible_code, prefix: str = "Team"):
+        name = f"{prefix} {responsible_code.long_name}"
+        if responsible_code.parent is not None:
+            name += f", {responsible_code.parent.long_name}"
+
+        description = f"Automatisch erzeugtes Team für {str(responsible_code)}"
+        team = Team.objects.get_or_create(
+            name=name,
+            description=description
+        )[0]
+        instance.share_with_team(team)
+
+        return instance

konova/management/commands/kspMigrater/compensation_migrater.py

@@ -59,6 +59,14 @@ class CompensationMigrater(BaseMigrater):
 
                 compensation.title = kom_title
                 compensation.comment = kom_comment
+                try:
+                    compensation = self._migrate_interventions_reference(compensation, kom)
+                    compensation.save()
+                except ObjectDoesNotExist:
+                    compensation.delete()
+                    unsuccessfull_compensations[kom_identifier] = "EIV does not exist"
+                    continue
+
                 compensation = self._migrate_par_7_data(compensation, kom)
                 compensation = self._migrate_geometry(compensation, kom)
                 compensation = self._migrate_responsibility(compensation, kom)
@@ -69,15 +77,9 @@ class CompensationMigrater(BaseMigrater):
                 compensation = self._migrate_actions(compensation, kom)
                 compensation = self._migrate_log(compensation, kom)
                 compensation = self._migrate_documents(compensation, CompensationDocument, kom)
-                try:
-                    compensation = self._migrate_interventions_reference(compensation, kom)
-                    compensation.save()
-                except ObjectDoesNotExist:
-                    compensation.delete()
-                    unsuccessfull_compensations[kom_identifier] = "EIV does not exist"
                 num_processed += 1
         print("The following KOMs could not be migrated: ")
-        for kom, val in unsuccessfull_compensations:
+        for kom, val in unsuccessfull_compensations.items():
             print(kom)
         cursor.close()
 

konova/management/commands/kspMigrater/eco_account_migrater.py

@@ -167,6 +167,7 @@ class EcoAccountMigrater(CompensationMigrater):
                 is_archived=False,
                 code_lists__in=[CODELIST_CONSERVATION_OFFICE_ID]
             )
+            self._migrate_responsible_code_to_team(eco_account, conservation_office, "ETS")
         except ObjectDoesNotExist:
             raise ObjectDoesNotExist(f"{acc_identifier}, {db_results}")
         try:

konova/management/commands/kspMigrater/ema_migrater.py

@@ -145,6 +145,7 @@ class EmaMigrater(CompensationMigrater):
                 is_archived=False,
                 code_lists__in=[CODELIST_CONSERVATION_OFFICE_ID]
             )
+            self._migrate_responsible_code_to_team(ema_obj, conservation_office, "ETS")
         except ObjectDoesNotExist:
             raise ObjectDoesNotExist(f"{ema_identifier}, {db_results}")
         try:

konova/management/commands/kspMigrater/intervention_migrater.py

@@ -37,6 +37,7 @@ class InterventionMigrater(BaseMigrater):
                     code_lists__in=[CODELIST_REGISTRATION_OFFICE_ID],
                 )
                 intervention.responsible.registration_office = reg_office_code
+                self._migrate_responsible_code_to_team(intervention, reg_office_code, "ZB")
             except ObjectDoesNotExist:
                 intervention.comment = f"{intervention.comment or ''}\nUnbekannte Zulassungsbehörde: {eiv_reg_off}"
         intervention.responsible.registration_file_number = eiv_reg_file_num
@@ -48,6 +49,7 @@ class InterventionMigrater(BaseMigrater):
                     is_leaf=True,
                     code_lists__in=[CODELIST_CONSERVATION_OFFICE_ID],
                 )
+                self._migrate_responsible_code_to_team(intervention, cons_office_code, "ETS")
             except ObjectDoesNotExist:
                 intervention.comment = f"{intervention.comment or ''}\nUnbekannte Eintragungsstelle: {eiv_cons_off}"
             intervention.responsible.conservation_office = cons_office_code
@@ -168,7 +170,7 @@ class InterventionMigrater(BaseMigrater):
             'where '
                 'om."OKL"=7730085 and '
                 'om.archiv=false and '
-                'om.nicht_vollstaendig=0;'
+                'om.nicht_vollstaendig=0 '
         )
 
         all_eivs = cursor.fetchall()