Codelist integration

* adds codelist app * adds KonovaCodeList and KonovaCode model for fetching and storing OSIRIS Codelisten entries * adds update_codelist command for updating and fetching codes * adds autocomplete route for using codelists in forms
2021-08-23 18:30:02 +02:00
parent c94d73c037
commit 49f7f3db53
12 changed files with 344 additions and 2 deletions
--- a/codelist/init.py
+++ b/codelist/init.py
--- a/codelist/admin.py
+++ b/codelist/admin.py
@@ -0,0 +1,36 @@
+from django.contrib import admin
+
+from codelist.models import KonovaCode, KonovaCodeList
+
+
+class KonovaCodeListAdmin(admin.ModelAdmin):
+    list_display = [
+        "id",
+    ]
+    readonly_fields = [
+        "id",
+        "codes",
+    ]
+
+
+class KonovaCodeAdmin(admin.ModelAdmin):
+    list_display = [
+        "id",
+        "parent",
+        "short_name",
+        "long_name",
+        "is_leaf",
+        "is_active",
+    ]
+
+    readonly_fields = [
+        "id",
+        "short_name",
+        "long_name",
+        "is_leaf",
+        "parent",
+    ]
+
+
+#admin.site.register(KonovaCodeList, KonovaCodeListAdmin)
+admin.site.register(KonovaCode, KonovaCodeAdmin)
--- a/codelist/apps.py
+++ b/codelist/apps.py
@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class CodelistConfig(AppConfig):
+    name = 'codelist'
--- a/codelist/management/commands/update_codelist.py
+++ b/codelist/management/commands/update_codelist.py
@@ -0,0 +1,157 @@
+"""
+Author: Michel Peltriaux
+Organization: Struktur- und Genehmigungsdirektion Nord, Rhineland-Palatinate, Germany
+Contact: michel.peltriaux@sgdnord.rlp.de
+Created on: 23.08.21
+
+"""
+import requests
+from django.core.management import BaseCommand
+from xml.etree import ElementTree as etree
+
+from codelist.models import KonovaCode, KonovaCodeList
+from codelist.settings import CODELIST_INTERVENTION_HANDLER_ID, CODELIST_CONSERVATION_OFFICE_ID, \
+    CODELIST_REGISTRATION_OFFICE_ID, CODELIST_BIOTOPES_ID, CODELIST_LAW_ID, CODELIST_COMPENSATION_HANDLER_ID, \
+    CODELIST_COMPENSATION_ACTION_ID, CODELIST_COMPENSATION_ACTION_CLASS_ID, CODELIST_COMPENSATION_ADDITIONAL_TYPE_ID, \
+    CODELIST_COMPENSATION_COMBINATION_ID, CODELIST_BASE_URL
+
+
+class Command(BaseCommand):
+    help = "Performs test on collisions using the identifier generation"
+
+    def handle(self, *args, **options):
+        try:
+            codelist_ids = [
+                CODELIST_INTERVENTION_HANDLER_ID,
+                CODELIST_CONSERVATION_OFFICE_ID,
+                CODELIST_REGISTRATION_OFFICE_ID,
+                CODELIST_BIOTOPES_ID,
+                CODELIST_LAW_ID,
+                CODELIST_COMPENSATION_HANDLER_ID,
+                CODELIST_COMPENSATION_ACTION_ID,
+                CODELIST_COMPENSATION_ACTION_CLASS_ID,
+                CODELIST_COMPENSATION_ADDITIONAL_TYPE_ID,
+                CODELIST_COMPENSATION_COMBINATION_ID,
+            ]
+            self._write_warning("Fetching codes...")
+
+            for list_id in codelist_ids:
+                _url = CODELIST_BASE_URL + "/" + str(list_id)
+                result = requests.get(url=_url)
+
+                if result.status_code != 200:
+                    self._write_error("Error on codelist url '{}'".format(_url))
+                    continue
+                self._write_warning("Fetch codes for list id {}".format(list_id))
+                content = result.content.decode("utf-8")
+                root = etree.fromstring(content)
+                items = root.findall("item")
+                self._write_warning("Found {} codes. Process now...".format(len(items)))
+
+                code_list = KonovaCodeList.objects.get_or_create(
+                    id=list_id,
+                )[0]
+                self._rec_xml_code_fetch(
+                    items=items,
+                    code_list=code_list,
+                    parent=None,
+                )
+                """
+                for element in items:
+                    atom_id = element.find("atomid").text
+                    parent = element.find("vaterid").text
+                    short_name = element.find("shortname").text
+                    long_name = element.find("longname").text
+                    is_archived = bool(element.find("archive").text)
+
+                    # If a parent has been set, we need to fetch/create this entry. Otherwise ("0") we ignore it.
+                    if parent == "0":
+                        parent = None
+                    else:
+                        parent = KonovaCode.objects.get_or_create(
+                            id=parent,
+                        )[0]
+
+                    code = KonovaCode.objects.get_or_create(
+                        id=atom_id,
+                    )
+                    created = code[1]
+                    if created:
+                        num_created += 1
+                    else:
+                        num_updated += 1
+                    code = code[0]
+                    code.short_name = short_name
+                    code.long_name = long_name
+                    code.parent = parent
+                    code.is_active = is_archived
+
+                    code.save()
+                    if code not in code_list.codes.all():
+                        code_list.codes.add(code)
+
+                """
+
+        except KeyboardInterrupt:
+            self._break_line()
+            exit(-1)
+
+    def _rec_xml_code_fetch(self, items: list, code_list: KonovaCodeList, parent: KonovaCode):
+        if items is None:
+            return
+        else:
+            for element in items:
+                children = element.find("items")
+                atom_id = element.find("atomid").text
+                short_name = element.find("shortname").text
+                long_name = element.find("longname").text
+                is_archived = bool(element.find("archive").text)
+
+                code = KonovaCode.objects.get_or_create(
+                    id=atom_id,
+                )
+                code = code[0]
+                code.short_name = short_name
+                code.long_name = long_name
+                code.parent = parent
+                code.is_active = is_archived
+                code.is_leaf = children is None
+
+                code.save()
+                if code not in code_list.codes.all():
+                    code_list.codes.add(code)
+
+                self._rec_xml_code_fetch(
+                    items=children,
+                    code_list=code_list,
+                    parent=code
+                )
+
+    def _break_line(self):
+        """ Simply prints a line break
+
+        Returns:
+
+        """
+        self.stdout.write("\n")
+
+    def _write_warning(self, txt: str):
+        self.stdout.write(
+            self.style.WARNING(
+                txt
+            )
+        )
+
+    def _write_success(self, txt: str):
+        self.stdout.write(
+            self.style.SUCCESS(
+                txt
+            )
+        )
+
+    def _write_error(self, txt: str):
+        self.stdout.write(
+            self.style.ERROR(
+                txt
+            )
+        )
--- a/codelist/models.py
+++ b/codelist/models.py
@@ -0,0 +1,65 @@
+from django.db import models
+
+
+class KonovaCode(models.Model):
+    """
+    Representation of the OSIRIS Codelisten codes.
+
+    KonovaCodes will be updated using a command regularly to provide all codes, which are provided by the
+    Codelisten application itself. To reduce traffic, we store them in our own database.
+    """
+    id = models.IntegerField(
+        primary_key=True,
+        help_text="AtomId; Identifies this code uniquely over all NatIT projects"
+    )
+    parent = models.ForeignKey(
+        "KonovaCode",
+        on_delete=models.CASCADE,
+        null=True,
+        blank=True,
+    )
+    short_name = models.CharField(
+        max_length=500,
+        null=True,
+        blank=True,
+        help_text="Short version of long name",
+    )
+    long_name = models.CharField(
+        max_length=1000,
+        null=True,
+        blank=True,
+        help_text="",
+    )
+    is_leaf = models.BooleanField(
+        default=False,
+        help_text="Whether this code has children or not"
+    )
+    is_active = models.BooleanField(
+        default=False,
+        help_text="Whether this code is archived or not"
+    )
+
+    def __str__(self):
+        if self.is_leaf and self.parent:
+            return "{} | {}".format(self.parent.long_name, self.long_name)
+        return self.long_name
+
+
+class KonovaCodeList(models.Model):
+    """
+    The codelist itself
+    """
+    id = models.IntegerField(
+        primary_key=True,
+        help_text="List identifier",
+    )
+    codes = models.ManyToManyField(
+        KonovaCode,
+        null=True,
+        blank=True,
+        help_text="Codes for this list",
+        related_name="code_lists"
+    )
+
+    def __str__(self):
+        return str(self.id)
--- a/codelist/settings.py
+++ b/codelist/settings.py
@@ -0,0 +1,23 @@
+"""
+Author: Michel Peltriaux
+Organization: Struktur- und Genehmigungsdirektion Nord, Rhineland-Palatinate, Germany
+Contact: michel.peltriaux@sgdnord.rlp.de
+Created on: 23.08.21
+
+"""
+
+# Codelist definitions
+CODELIST_BASE_URL = "https://codelisten.naturschutz.rlp.de/repository/referenzliste"
+
+# Identifier
+CODELIST_INTERVENTION_HANDLER_ID = 903  # CLMassnahmeträger
+CODELIST_CONSERVATION_OFFICE_ID = 907  # CLNaturschutzbehörden
+CODELIST_REGISTRATION_OFFICE_ID = 1053  # CLZulassungsbehörden
+CODELIST_BIOTOPES_ID = 974  # CL_EIV_Biotoptypen
+CODELIST_LAW_ID = 1048  # CLVerfahrensrecht
+
+CODELIST_COMPENSATION_HANDLER_ID = 1052  # CLEingreifer
+CODELIST_COMPENSATION_ACTION_ID = 1026  # CLMassnahmedetail
+CODELIST_COMPENSATION_ACTION_CLASS_ID = 1034  # CLMassnahmeklasse
+CODELIST_COMPENSATION_ADDITIONAL_TYPE_ID = 1028  # CLMassnahmetyp
+CODELIST_COMPENSATION_COMBINATION_ID = 1049  # CLKombimassnahme
--- a/codelist/tests.py
+++ b/codelist/tests.py
@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.
--- a/codelist/urls.py
+++ b/codelist/urls.py
@@ -0,0 +1,14 @@
+"""
+Author: Michel Peltriaux
+Organization: Struktur- und Genehmigungsdirektion Nord, Rhineland-Palatinate, Germany
+Contact: michel.peltriaux@sgdnord.rlp.de
+Created on: 23.08.21
+
+"""
+from django.urls import path
+
+
+app_name = "codelist"
+urlpatterns = [
+
+]
--- a/codelist/views.py
+++ b/codelist/views.py