Konova Code fix

* adds command sync_codelist
    * provides updating of all codes to the newest version (id)
    * must be run once on staging, can be dropped afterwards since the root for the problem has been resolved on the codelist management application

codelist/management/commands/sync_codelist.py

@@ -0,0 +1,165 @@
+"""
+Author: Michel Peltriaux
+Organization: Struktur- und Genehmigungsdirektion Nord, Rhineland-Palatinate, Germany
+Contact: michel.peltriaux@sgdnord.rlp.de
+Created on: 31.05.22
+
+"""
+
+from django.db import transaction
+
+from codelist.models import KonovaCode
+from compensation.models import CompensationAction, CompensationState
+from intervention.models import Legal, Handler, Responsibility
+from konova.management.commands.setup import BaseKonovaCommand
+
+
+class Command(BaseKonovaCommand):
+    help = "Updates internal codelist by external API"
+
+    def handle(self, *args, **options):
+        try:
+            with transaction.atomic():
+                self.sync_codelist()
+        except KeyboardInterrupt:
+            self._break_line()
+            exit(-1)
+
+    def __get_newest_code(self, code):
+        code = KonovaCode.objects.filter(
+            atom_id=code.atom_id,
+            parent=code.parent,
+            code_lists__in=code.code_lists.all(),
+        ).order_by(
+            "-id"
+        ).first()
+        return code
+
+    def __migrate_compensation_action_codes(self):
+        all_actions = CompensationAction.objects.all()
+        used_codes = []
+        for action in all_actions:
+            stored_codes = action.action_type.all()
+            codes = []
+            for code in stored_codes:
+                codes.append(self.__get_newest_code(code))
+            action.action_type.set(codes)
+            used_codes += codes
+
+            stored_codes = action.action_type_details.all()
+            codes = []
+            for code in stored_codes:
+                codes.append(self.__get_newest_code(code))
+            action.action_type_details.set(codes)
+            used_codes += codes
+
+            action.save()
+        return used_codes
+
+    def __migrate_compensation_state_codes(self):
+        all_states = CompensationState.objects.all()
+        used_codes = []
+        for state in all_states:
+            code = state.biotope_type
+            if code is not None:
+                new_code = self.__get_newest_code(code)
+                state.biotope_type = new_code
+                used_codes.append(new_code)
+
+            stored_codes = state.biotope_type_details.all()
+            codes = []
+            for code in stored_codes:
+                codes.append(self.__get_newest_code(code))
+            state.biotope_type_details.set(codes)
+
+            used_codes += codes
+            state.save()
+        return used_codes
+
+    def __migrate_legal_codes(self):
+        all_legal = Legal.objects.all()
+        used_codes = []
+        for legal in all_legal:
+            code = legal.process_type
+            if code is not None:
+                new_code = self.__get_newest_code(code)
+                legal.process_type = new_code
+                used_codes.append(new_code)
+
+            stored_codes = legal.laws.all()
+            codes = []
+            for code in stored_codes:
+                codes.append(self.__get_newest_code(code))
+            legal.laws.set(codes)
+
+            used_codes += codes
+            legal.save()
+        return used_codes
+
+    def __migrate_handler_codes(apps):
+        all_handlers = Handler.objects.all()
+        used_codes = []
+        for handler in all_handlers:
+            code = handler.type
+            if code is None:
+                continue
+            new_code = apps.__get_newest_code(code)
+            handler.type = new_code
+            used_codes.append(new_code)
+            handler.save()
+        return used_codes
+
+    def __migrate_responsibility_codes(apps):
+        all_resps = Responsibility.objects.all()
+        used_codes = []
+        for responsibility in all_resps:
+            code = responsibility.registration_office
+            if code is not None:
+                new_code = apps.__get_newest_code(code)
+                responsibility.registration_office = new_code
+                used_codes.append(new_code)
+
+            code = responsibility.conservation_office
+            if code is not None:
+                new_code = apps.__get_newest_code(code)
+                responsibility.conservation_office = new_code
+                used_codes.append(new_code)
+
+            responsibility.save()
+        return used_codes
+
+    def sync_codelist(self):
+        """ Due to issues on the external codelist app there can be multiple entries of the same code
+        (atom_id, parent, list) but with different identifiers.
+
+        These issues have been resolved but already
+
+        Returns:
+
+        """
+        self._write_warning("Sync codes in usage and replace by newest entries...")
+        used_codes = []
+        used_codes += self.__migrate_compensation_action_codes()
+        used_codes += self.__migrate_compensation_state_codes()
+        used_codes += self.__migrate_legal_codes()
+        used_codes += self.__migrate_handler_codes()
+        used_codes += self.__migrate_responsibility_codes()
+        self._write_success(f"Synced {len(used_codes)} code usages!")
+
+        all_codes = KonovaCode.objects.all()
+        newest_code_ids = []
+        for code in all_codes:
+            newest_code = self.__get_newest_code(code)
+            newest_code_ids.append(newest_code.id)
+
+        code_ids_to_keep = set(newest_code_ids)
+        self._write_warning(f"Of {all_codes.count()} KonovaCodes there are {len(code_ids_to_keep)} to keep as newest versions...")
+
+        deletable_codes = KonovaCode.objects.all().exclude(
+            id__in=code_ids_to_keep
+        )
+        deletable_codes_count = deletable_codes.count()
+        self._write_warning(f"{deletable_codes_count} found which are obsolet...")
+        if deletable_codes_count > 0:
+            deletable_codes.delete()
+            self._write_success("Obsolete codes deleted!")

codelist/management/commands/update_codelist.py

@@ -6,7 +6,6 @@ Created on: 23.08.21
 
 """
 import requests
-from django.core.management import BaseCommand
 from xml.etree import ElementTree as etree
 
 from codelist.models import KonovaCode, KonovaCodeList

codelist/settings.py

@@ -14,7 +14,7 @@ CODELIST_INTERVENTION_HANDLER_ID = 903  # CLMassnahmeträger
 CODELIST_CONSERVATION_OFFICE_ID = 907  # CLNaturschutzbehörden
 CODELIST_REGISTRATION_OFFICE_ID = 1053  # CLZulassungsbehörden
 CODELIST_BIOTOPES_ID = 654  # CL_Biotoptypen
-CODELIST_AFTER_STATE_BIOTOPES__ID = 974  # CL-KSP_ZielBiotoptypen - USAGE HAS BEEN DROPPED IN 2022 IN FAVOR OF 654
+CODELIST_AFTER_STATE_BIOTOPES_ID = 974  # CL-KSP_ZielBiotoptypen - USAGE HAS BEEN DROPPED IN 2022 IN FAVOR OF 654
 CODELIST_BIOTOPES_EXTRA_CODES_ID = 975  # CLZusatzbezeichnung
 CODELIST_LAW_ID = 1048  # CLVerfahrensrecht
 CODELIST_PROCESS_TYPE_ID = 44382  # CLVerfahrenstyp

compensation/migrations/0005_auto_20220218_0917.py

@@ -3,7 +3,7 @@
 from django.db import migrations, models, transaction
 import django.db.models.deletion
 
-from codelist.settings import CODELIST_BIOTOPES_ID, CODELIST_AFTER_STATE_BIOTOPES__ID
+from codelist.settings import CODELIST_BIOTOPES_ID, CODELIST_AFTER_STATE_BIOTOPES_ID
 
 
 def migrate_entries_974_to_654(apps, schema_editor):
@@ -23,7 +23,7 @@ def migrate_entries_974_to_654(apps, schema_editor):
             state.save()
 
     old_list_states = CompensationState.objects.filter(
-        biotope_type__code_lists__in=[CODELIST_AFTER_STATE_BIOTOPES__ID]
+        biotope_type__code_lists__in=[CODELIST_AFTER_STATE_BIOTOPES_ID]
     )
     if old_list_states.count() > 0:
         raise Exception("Still unmigrated values!")

compensation/models/state.py

@@ -12,7 +12,6 @@ from codelist.models import KonovaCode
 from codelist.settings import CODELIST_BIOTOPES_ID, CODELIST_BIOTOPES_EXTRA_CODES_ID
 from compensation.managers import CompensationStateManager
 from konova.models import UuidModel
-from konova.utils.message_templates import COMPENSATION_STATE_REMOVED
 
 
 class CompensationState(UuidModel):

konova/migrations/0005_auto_20220216_0856.py

@@ -2,7 +2,7 @@
 
 from django.db import migrations, transaction
 
-from codelist.settings import CODELIST_BIOTOPES_ID, CODELIST_AFTER_STATE_BIOTOPES__ID
+from codelist.settings import CODELIST_BIOTOPES_ID, CODELIST_AFTER_STATE_BIOTOPES_ID
 
 
 def migrate_biotopes_from_974_to_654(apps, schema_editor):
@@ -23,7 +23,7 @@ def migrate_biotopes_from_974_to_654(apps, schema_editor):
 
         all_states = CompensationState.objects.all()
         after_state_list_elements = all_states.filter(
-            biotope_type__code_lists__in=[CODELIST_AFTER_STATE_BIOTOPES__ID]
+            biotope_type__code_lists__in=[CODELIST_AFTER_STATE_BIOTOPES_ID]
         )
         if after_state_list_elements.count() > 0:
             raise Exception("Still states with wrong codelist entries!")