From b0a15d9d1c036b9ebc8606d1529bc81ff5551a94 Mon Sep 17 00:00:00 2001 From: mpeltriaux Date: Tue, 31 May 2022 16:53:13 +0200 Subject: [PATCH] Konova Code fix * adds command sync_codelist * provides updating of all codes to the newest version (id) * must be run once on staging, can be dropped afterwards since the root for the problem has been resolved on the codelist management application --- codelist/management/commands/sync_codelist.py | 165 ++++++++++++++++++ .../management/commands/update_codelist.py | 1 - codelist/settings.py | 2 +- .../migrations/0005_auto_20220218_0917.py | 4 +- compensation/models/state.py | 1 - konova/migrations/0005_auto_20220216_0856.py | 4 +- 6 files changed, 170 insertions(+), 7 deletions(-) create mode 100644 codelist/management/commands/sync_codelist.py diff --git a/codelist/management/commands/sync_codelist.py b/codelist/management/commands/sync_codelist.py new file mode 100644 index 0000000..1ad7c72 --- /dev/null +++ b/codelist/management/commands/sync_codelist.py @@ -0,0 +1,165 @@ +""" +Author: Michel Peltriaux +Organization: Struktur- und Genehmigungsdirektion Nord, Rhineland-Palatinate, Germany +Contact: michel.peltriaux@sgdnord.rlp.de +Created on: 31.05.22 + +""" + +from django.db import transaction + +from codelist.models import KonovaCode +from compensation.models import CompensationAction, CompensationState +from intervention.models import Legal, Handler, Responsibility +from konova.management.commands.setup import BaseKonovaCommand + + +class Command(BaseKonovaCommand): + help = "Updates internal codelist by external API" + + def handle(self, *args, **options): + try: + with transaction.atomic(): + self.sync_codelist() + except KeyboardInterrupt: + self._break_line() + exit(-1) + + def __get_newest_code(self, code): + code = KonovaCode.objects.filter( + atom_id=code.atom_id, + parent=code.parent, + code_lists__in=code.code_lists.all(), + ).order_by( + "-id" + ).first() + return code + + def __migrate_compensation_action_codes(self): + all_actions = CompensationAction.objects.all() + used_codes = [] + for action in all_actions: + stored_codes = action.action_type.all() + codes = [] + for code in stored_codes: + codes.append(self.__get_newest_code(code)) + action.action_type.set(codes) + used_codes += codes + + stored_codes = action.action_type_details.all() + codes = [] + for code in stored_codes: + codes.append(self.__get_newest_code(code)) + action.action_type_details.set(codes) + used_codes += codes + + action.save() + return used_codes + + def __migrate_compensation_state_codes(self): + all_states = CompensationState.objects.all() + used_codes = [] + for state in all_states: + code = state.biotope_type + if code is not None: + new_code = self.__get_newest_code(code) + state.biotope_type = new_code + used_codes.append(new_code) + + stored_codes = state.biotope_type_details.all() + codes = [] + for code in stored_codes: + codes.append(self.__get_newest_code(code)) + state.biotope_type_details.set(codes) + + used_codes += codes + state.save() + return used_codes + + def __migrate_legal_codes(self): + all_legal = Legal.objects.all() + used_codes = [] + for legal in all_legal: + code = legal.process_type + if code is not None: + new_code = self.__get_newest_code(code) + legal.process_type = new_code + used_codes.append(new_code) + + stored_codes = legal.laws.all() + codes = [] + for code in stored_codes: + codes.append(self.__get_newest_code(code)) + legal.laws.set(codes) + + used_codes += codes + legal.save() + return used_codes + + def __migrate_handler_codes(apps): + all_handlers = Handler.objects.all() + used_codes = [] + for handler in all_handlers: + code = handler.type + if code is None: + continue + new_code = apps.__get_newest_code(code) + handler.type = new_code + used_codes.append(new_code) + handler.save() + return used_codes + + def __migrate_responsibility_codes(apps): + all_resps = Responsibility.objects.all() + used_codes = [] + for responsibility in all_resps: + code = responsibility.registration_office + if code is not None: + new_code = apps.__get_newest_code(code) + responsibility.registration_office = new_code + used_codes.append(new_code) + + code = responsibility.conservation_office + if code is not None: + new_code = apps.__get_newest_code(code) + responsibility.conservation_office = new_code + used_codes.append(new_code) + + responsibility.save() + return used_codes + + def sync_codelist(self): + """ Due to issues on the external codelist app there can be multiple entries of the same code + (atom_id, parent, list) but with different identifiers. + + These issues have been resolved but already + + Returns: + + """ + self._write_warning("Sync codes in usage and replace by newest entries...") + used_codes = [] + used_codes += self.__migrate_compensation_action_codes() + used_codes += self.__migrate_compensation_state_codes() + used_codes += self.__migrate_legal_codes() + used_codes += self.__migrate_handler_codes() + used_codes += self.__migrate_responsibility_codes() + self._write_success(f"Synced {len(used_codes)} code usages!") + + all_codes = KonovaCode.objects.all() + newest_code_ids = [] + for code in all_codes: + newest_code = self.__get_newest_code(code) + newest_code_ids.append(newest_code.id) + + code_ids_to_keep = set(newest_code_ids) + self._write_warning(f"Of {all_codes.count()} KonovaCodes there are {len(code_ids_to_keep)} to keep as newest versions...") + + deletable_codes = KonovaCode.objects.all().exclude( + id__in=code_ids_to_keep + ) + deletable_codes_count = deletable_codes.count() + self._write_warning(f"{deletable_codes_count} found which are obsolet...") + if deletable_codes_count > 0: + deletable_codes.delete() + self._write_success("Obsolete codes deleted!") \ No newline at end of file diff --git a/codelist/management/commands/update_codelist.py b/codelist/management/commands/update_codelist.py index 345b324..d29b272 100644 --- a/codelist/management/commands/update_codelist.py +++ b/codelist/management/commands/update_codelist.py @@ -6,7 +6,6 @@ Created on: 23.08.21 """ import requests -from django.core.management import BaseCommand from xml.etree import ElementTree as etree from codelist.models import KonovaCode, KonovaCodeList diff --git a/codelist/settings.py b/codelist/settings.py index 61652bd..3bd7032 100644 --- a/codelist/settings.py +++ b/codelist/settings.py @@ -14,7 +14,7 @@ CODELIST_INTERVENTION_HANDLER_ID = 903 # CLMassnahmeträger CODELIST_CONSERVATION_OFFICE_ID = 907 # CLNaturschutzbehörden CODELIST_REGISTRATION_OFFICE_ID = 1053 # CLZulassungsbehörden CODELIST_BIOTOPES_ID = 654 # CL_Biotoptypen -CODELIST_AFTER_STATE_BIOTOPES__ID = 974 # CL-KSP_ZielBiotoptypen - USAGE HAS BEEN DROPPED IN 2022 IN FAVOR OF 654 +CODELIST_AFTER_STATE_BIOTOPES_ID = 974 # CL-KSP_ZielBiotoptypen - USAGE HAS BEEN DROPPED IN 2022 IN FAVOR OF 654 CODELIST_BIOTOPES_EXTRA_CODES_ID = 975 # CLZusatzbezeichnung CODELIST_LAW_ID = 1048 # CLVerfahrensrecht CODELIST_PROCESS_TYPE_ID = 44382 # CLVerfahrenstyp diff --git a/compensation/migrations/0005_auto_20220218_0917.py b/compensation/migrations/0005_auto_20220218_0917.py index 43e7db9..0044264 100644 --- a/compensation/migrations/0005_auto_20220218_0917.py +++ b/compensation/migrations/0005_auto_20220218_0917.py @@ -3,7 +3,7 @@ from django.db import migrations, models, transaction import django.db.models.deletion -from codelist.settings import CODELIST_BIOTOPES_ID, CODELIST_AFTER_STATE_BIOTOPES__ID +from codelist.settings import CODELIST_BIOTOPES_ID, CODELIST_AFTER_STATE_BIOTOPES_ID def migrate_entries_974_to_654(apps, schema_editor): @@ -23,7 +23,7 @@ def migrate_entries_974_to_654(apps, schema_editor): state.save() old_list_states = CompensationState.objects.filter( - biotope_type__code_lists__in=[CODELIST_AFTER_STATE_BIOTOPES__ID] + biotope_type__code_lists__in=[CODELIST_AFTER_STATE_BIOTOPES_ID] ) if old_list_states.count() > 0: raise Exception("Still unmigrated values!") diff --git a/compensation/models/state.py b/compensation/models/state.py index 5cb8376..77026b5 100644 --- a/compensation/models/state.py +++ b/compensation/models/state.py @@ -12,7 +12,6 @@ from codelist.models import KonovaCode from codelist.settings import CODELIST_BIOTOPES_ID, CODELIST_BIOTOPES_EXTRA_CODES_ID from compensation.managers import CompensationStateManager from konova.models import UuidModel -from konova.utils.message_templates import COMPENSATION_STATE_REMOVED class CompensationState(UuidModel): diff --git a/konova/migrations/0005_auto_20220216_0856.py b/konova/migrations/0005_auto_20220216_0856.py index 43c518a..567e206 100644 --- a/konova/migrations/0005_auto_20220216_0856.py +++ b/konova/migrations/0005_auto_20220216_0856.py @@ -2,7 +2,7 @@ from django.db import migrations, transaction -from codelist.settings import CODELIST_BIOTOPES_ID, CODELIST_AFTER_STATE_BIOTOPES__ID +from codelist.settings import CODELIST_BIOTOPES_ID, CODELIST_AFTER_STATE_BIOTOPES_ID def migrate_biotopes_from_974_to_654(apps, schema_editor): @@ -23,7 +23,7 @@ def migrate_biotopes_from_974_to_654(apps, schema_editor): all_states = CompensationState.objects.all() after_state_list_elements = all_states.filter( - biotope_type__code_lists__in=[CODELIST_AFTER_STATE_BIOTOPES__ID] + biotope_type__code_lists__in=[CODELIST_AFTER_STATE_BIOTOPES_ID] ) if after_state_list_elements.count() > 0: raise Exception("Still states with wrong codelist entries!")