#132 WIP Migrate Users

* improves auto-recording of eco accounts
* adds migration of users and adding to auto-created teams

compensation/templates/compensation/detail/compensation/includes/deadlines.html

@@ -51,7 +51,7 @@
                 <td class="align-middle">{{ deadline.date|default_if_none:"---" }}</td>
                 <td class="align-middle">
                     <div class="scroll-150">
-                        {{ deadline.comment }}
+                        {{ deadline.comment|linebreaks }}
                     </div>
                 </td>
                 <td class="align-middle float-right">

compensation/templates/compensation/detail/eco_account/includes/deadlines.html

@@ -49,7 +49,7 @@
                 <td class="align-middle">{{ deadline.date|default_if_none:"---" }}</td>
                 <td class="align-middle">
                     <div class="scroll-150">
-                        {{ deadline.comment }}
+                        {{ deadline.comment|linebreaks }}
                     </div>
                 </td>
                 <td class="align-middle float-right">

konova/management/commands/kspMigrater/base_migrater.py

@@ -24,12 +24,12 @@ class BaseMigrater:
         if self.db_connection is not None:
             self.db_connection.close()
 
-    def connect_db(self):
+    def connect_db(self, db_name: str="ksp"):
         if self.options is None:
             return
         conn = psycopg2.connect(
             host=self.options["host"],
-            database="ksp",
+            database=db_name,
             user=self.options["db_user"],
             password=self.options["db_pw"],
         )
@@ -182,7 +182,23 @@ class BaseMigrater:
         tmp_cursor.close()
         return instance
 
-    def _migrate_responsible_code_to_team(self, instance, responsible_code, prefix: str = "Team"):
+    def _get_team(self, team_name: str):
+        """ Returns a team from given name
+
+        """
+        name = f"Team {team_name}"
+
+        description = f"Automatisch erzeugtes Team für {team_name}"
+        team = Team.objects.get_or_create(
+            name=name,
+            description=description
+        )[0]
+        return team
+
+    def _get_team_from_responsible_code(self, responsible_code, prefix: str = "Team"):
+        """ Returns a team from given responsible code
+
+        """
         name = f"{prefix} {responsible_code.long_name}"
         if responsible_code.parent is not None:
             name += f", {responsible_code.parent.long_name}"
@@ -192,6 +208,10 @@ class BaseMigrater:
             name=name,
             description=description
         )[0]
+        return team
+
+    def _migrate_responsible_code_to_team(self, instance, responsible_code, prefix: str = "Team"):
+        team = self._get_team_from_responsible_code(responsible_code, prefix)
         instance.share_with_team(team)
 
         return instance

konova/management/commands/kspMigrater/eco_account_migrater.py

@@ -4,7 +4,7 @@ from django.contrib.auth.models import Group
 from django.contrib.gis.geos import MultiPolygon, Polygon
 from django.core.exceptions import ObjectDoesNotExist
 from django.db import transaction
-from django.utils import timezone
+from django.utils import timezone, formats
 
 from codelist.models import KonovaCode
 from codelist.settings import CODELIST_CONSERVATION_OFFICE_ID, CODELIST_COMPENSATION_HANDLER_ID
@@ -13,7 +13,6 @@ from compensation.utils.quality import EcoAccountQualityChecker
 from intervention.models import Responsibility, Handler, Intervention, Legal
 from konova.management.commands.kspMigrater.compensation_migrater import CompensationMigrater
 from konova.models import Geometry
-from konova.settings import DEFAULT_GROUP
 from konova.sub_settings.lanis_settings import DEFAULT_SRID_RLP, DEFAULT_SRID
 from user.models import User, UserActionLogEntry
 
@@ -21,6 +20,7 @@ from user.models import User, UserActionLogEntry
 class EcoAccountMigrater(CompensationMigrater):
 
     def migrate(self):
+        el = "'OEK-1488450234228'"
         self.connect_db()
         cursor = self.db_connection.cursor()
         cursor.execute(
@@ -61,11 +61,12 @@ class EcoAccountMigrater(CompensationMigrater):
                 )[0]
 
                 eco_account.title = oek_title
+                eco_account.prevent_recording = False
                 eco_account.comment = oek_comment
                 eco_account = self._migrate_legal(eco_account, oek)
+                eco_account = self._migrate_states(eco_account, oek)
                 eco_account = self._migrate_geometry(eco_account, oek)
                 eco_account = self._migrate_responsibility(eco_account, oek)
-                eco_account = self._migrate_states(eco_account, oek)
                 eco_account = self._migrate_deadlines(eco_account, oek)
                 eco_account = self._migrate_action_control_deadlines(eco_account, oek)
                 eco_account = self._migrate_actions(eco_account, oek)
@@ -126,6 +127,20 @@ class EcoAccountMigrater(CompensationMigrater):
             # Calculate area by transforming
             rlp_geom = db_result_geom.transform(ct=DEFAULT_SRID_RLP, clone=True)
             area = round(rlp_geom.area)
+            max_state_after_area = instance.get_state_after_surface_sum()
+
+            # Check whether the geometric area is at least
+            diff = abs(area - max_state_after_area)
+            diff_perc = diff / max(area, max_state_after_area)
+            is_diff_too_high = diff_perc > 0.1
+            if is_diff_too_high:
+                print(f"      !!! {identifier} has diff of {diff_perc*100} % between geometry and after_states. Should not be recorded!")
+                instance.comment += f"\n\nÖkokonto konnte nicht automatisch verzeichnet übernommen werden: Zu große Differenz zwischen Geometriefläche ({formats.localize(area, use_l10n=True)} m²) und angegebener Zielzustandsfläche. Bitte prüfen und korrigieren bzw. eigenständig verzeichnen."
+                instance.prevent_recording = True
+                area = min(area, max_state_after_area)
+            else:
+                area = max_state_after_area
+
             instance.deductable_surface = area
             instance.geometry.geom = db_result_geom if not db_result_geom.empty else None
             instance.geometry.save()
@@ -341,7 +356,7 @@ class EcoAccountMigrater(CompensationMigrater):
         quality_checker = EcoAccountQualityChecker(instance)
         quality_checker.run_check()
 
-        if quality_checker.valid:
+        if quality_checker.valid and not instance.prevent_recording:
             identifier = f"'{db_result[0]}'"
             tmp_cursor = self.db_connection.cursor()
             tmp_cursor.execute(
@@ -358,7 +373,25 @@ class EcoAccountMigrater(CompensationMigrater):
                 'limit 1'
             )
             fetch_result = tmp_cursor.fetchone()
+            if fetch_result is None:
+                # Can happen on very old eco accounts: This data might only be found on table 'protokoll'
+                tmp_cursor.execute(
+                    'select '
+                        'p.bemerkung1, '
+                        'p.geaendertvon, '
+                        'p.geaendertam '
+                    'from "OBJ_MASTER" om '
+                        'join protokoll p on om."GISPADID"=p."FKEY" '
+                    'where '
+                        f'om."KENNUNG"={identifier} '
+                    'order by '
+                        'p.geaendertam '
+                    'limit 1'
+                )
+                fetch_result = tmp_cursor.fetchone()
+
             if fetch_result is not None:
+                # Something has been found on one of these two tables
                 recorded_by = fetch_result[1]
                 recorded_ts = timezone.make_aware(fetch_result[2])
                 user = self._get_migrate_user(recorded_by)
@@ -369,4 +402,7 @@ class EcoAccountMigrater(CompensationMigrater):
                 )
                 instance.recorded.timestamp = recorded_ts
                 instance.recorded.save()
+            tmp_cursor.close()
+        else:
+            instance.recorded = None
         return instance

konova/management/commands/kspMigrater/user_migrater.py

@@ -0,0 +1,92 @@
+from django.core.exceptions import ObjectDoesNotExist
+
+from codelist.models import KonovaCode
+from codelist.settings import CODELIST_CONSERVATION_OFFICE_ID, CODELIST_REGISTRATION_OFFICE_ID
+from konova.management.commands.kspMigrater.base_migrater import BaseMigrater
+from user.models import User
+
+
+class UserMigrater(BaseMigrater):
+
+    def migrate(self):
+        self.connect_db(db_name="natportal")
+        cursor = self.db_connection.cursor()
+
+        el = "'%§§15§§%'"
+        cursor.execute(
+            'select '
+                'kennung, '
+                'name, '
+                'vorname, '
+                'email, '
+                'anwendungsnutzer, '
+                'firmenname '
+            'from berechtigung.nutzer where '
+                'aktiv=true and '
+                f'anwendungsnutzer like {el}'
+        )
+        fetch_results = cursor.fetchall()
+
+        len_all_users = len(fetch_results)
+        print(f"Migrate users...")
+        print(f"--Found {len_all_users} entries. Process now...")
+        num_processed = 0
+        for result in fetch_results:
+            if num_processed % 500 == 0:
+                print(f"----{num_processed}/{len_all_users} processed")
+            user = User.objects.get_or_create(
+                username=result[0]
+            )[0]
+            user.last_name = result[2]
+            user.first_name = result[2]
+            user.email = result[3]
+
+            appusers = result[4].split(";;;")
+            user_teams = []
+            company_team = None
+            for tmp in appusers:
+                if "§§15§§" not in tmp:
+                    continue
+                contents = tmp.split("§§")
+                _type = contents[2]
+                _org = contents[3]
+                ets_id = "20"
+                zb_id = "26"
+                is_type_ets = _type == ets_id
+                is_type_zb = _type == zb_id
+                if not _type or not _org:
+                    # broken - unusable
+                    continue
+
+                try:
+                    if is_type_zb:
+                        _org = KonovaCode.objects.get(
+                            atom_id=_org,
+                            code_lists__in=[CODELIST_REGISTRATION_OFFICE_ID]
+                        )
+                        team = self._get_team_from_responsible_code(responsible_code=_org, prefix="ZB")
+                        user_teams.append(team)
+                    elif is_type_ets:
+                        _org = KonovaCode.objects.get(
+                            atom_id=_org,
+                            code_lists__in=[CODELIST_CONSERVATION_OFFICE_ID]
+                        )
+                        team = self._get_team_from_responsible_code(responsible_code=_org, prefix="ETS")
+                        user_teams.append(team)
+                    else:
+                        _org = result[5]
+                        if company_team is None:
+                            company_team = self._get_team(_org)
+                except ObjectDoesNotExist:
+                    # organisation code might not be valid anymore
+                    continue
+            if company_team is not None and len(user_teams) == 0:
+                # Only team is the company team
+                company_team.users.add(user)
+            else:
+                if company_team is not None:
+                    company_team.delete()
+                for team in user_teams:
+                    team.users.add(user)
+            num_processed += 1
+        cursor.close()

konova/management/commands/migrate_ksp_konova.py

@@ -2,6 +2,7 @@ from konova.management.commands.kspMigrater.compensation_migrater import Compens
 from konova.management.commands.kspMigrater.eco_account_migrater import EcoAccountMigrater
 from konova.management.commands.kspMigrater.ema_migrater import EmaMigrater
 from konova.management.commands.kspMigrater.intervention_migrater import InterventionMigrater
+from konova.management.commands.kspMigrater.user_migrater import UserMigrater
 from konova.management.commands.setup import BaseKonovaCommand
 
 
@@ -20,7 +21,8 @@ class Command(BaseKonovaCommand):
                 #InterventionMigrater(options),
                 #CompensationMigrater(options),
                 #EmaMigrater(options),
-                EcoAccountMigrater(options),
+                #EcoAccountMigrater(options),
+                UserMigrater(options),
             ]
             for migrater in migraters:
                 migrater.migrate()

user/admin.py

@@ -74,6 +74,9 @@ class TeamAdmin(admin.ModelAdmin):
         "name",
         "description",
     ]
+    filter_horizontal = [
+        "users"
+    ]
 
 
 admin.site.register(User, UserAdmin)