Codelist integration

* adds codelist app
* adds KonovaCodeList and KonovaCode model for fetching and storing OSIRIS Codelisten entries
* adds update_codelist command for updating and fetching codes
* adds autocomplete route for using codelists in forms

codelist/admin.py

@@ -0,0 +1,36 @@
+from django.contrib import admin
+
+from codelist.models import KonovaCode, KonovaCodeList
+
+
+class KonovaCodeListAdmin(admin.ModelAdmin):
+    list_display = [
+        "id",
+    ]
+    readonly_fields = [
+        "id",
+        "codes",
+    ]
+
+
+class KonovaCodeAdmin(admin.ModelAdmin):
+    list_display = [
+        "id",
+        "parent",
+        "short_name",
+        "long_name",
+        "is_leaf",
+        "is_active",
+    ]
+
+    readonly_fields = [
+        "id",
+        "short_name",
+        "long_name",
+        "is_leaf",
+        "parent",
+    ]
+
+
+#admin.site.register(KonovaCodeList, KonovaCodeListAdmin)
+admin.site.register(KonovaCode, KonovaCodeAdmin)

codelist/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class CodelistConfig(AppConfig):
+    name = 'codelist'

codelist/management/commands/update_codelist.py

@@ -0,0 +1,157 @@
+"""
+Author: Michel Peltriaux
+Organization: Struktur- und Genehmigungsdirektion Nord, Rhineland-Palatinate, Germany
+Contact: michel.peltriaux@sgdnord.rlp.de
+Created on: 23.08.21
+
+"""
+import requests
+from django.core.management import BaseCommand
+from xml.etree import ElementTree as etree
+
+from codelist.models import KonovaCode, KonovaCodeList
+from codelist.settings import CODELIST_INTERVENTION_HANDLER_ID, CODELIST_CONSERVATION_OFFICE_ID, \
+    CODELIST_REGISTRATION_OFFICE_ID, CODELIST_BIOTOPES_ID, CODELIST_LAW_ID, CODELIST_COMPENSATION_HANDLER_ID, \
+    CODELIST_COMPENSATION_ACTION_ID, CODELIST_COMPENSATION_ACTION_CLASS_ID, CODELIST_COMPENSATION_ADDITIONAL_TYPE_ID, \
+    CODELIST_COMPENSATION_COMBINATION_ID, CODELIST_BASE_URL
+
+
+class Command(BaseCommand):
+    help = "Performs test on collisions using the identifier generation"
+
+    def handle(self, *args, **options):
+        try:
+            codelist_ids = [
+                CODELIST_INTERVENTION_HANDLER_ID,
+                CODELIST_CONSERVATION_OFFICE_ID,
+                CODELIST_REGISTRATION_OFFICE_ID,
+                CODELIST_BIOTOPES_ID,
+                CODELIST_LAW_ID,
+                CODELIST_COMPENSATION_HANDLER_ID,
+                CODELIST_COMPENSATION_ACTION_ID,
+                CODELIST_COMPENSATION_ACTION_CLASS_ID,
+                CODELIST_COMPENSATION_ADDITIONAL_TYPE_ID,
+                CODELIST_COMPENSATION_COMBINATION_ID,
+            ]
+            self._write_warning("Fetching codes...")
+
+            for list_id in codelist_ids:
+                _url = CODELIST_BASE_URL + "/" + str(list_id)
+                result = requests.get(url=_url)
+
+                if result.status_code != 200:
+                    self._write_error("Error on codelist url '{}'".format(_url))
+                    continue
+                self._write_warning("Fetch codes for list id {}".format(list_id))
+                content = result.content.decode("utf-8")
+                root = etree.fromstring(content)
+                items = root.findall("item")
+                self._write_warning("Found {} codes. Process now...".format(len(items)))
+
+                code_list = KonovaCodeList.objects.get_or_create(
+                    id=list_id,
+                )[0]
+                self._rec_xml_code_fetch(
+                    items=items,
+                    code_list=code_list,
+                    parent=None,
+                )
+                """
+                for element in items:
+                    atom_id = element.find("atomid").text
+                    parent = element.find("vaterid").text
+                    short_name = element.find("shortname").text
+                    long_name = element.find("longname").text
+                    is_archived = bool(element.find("archive").text)
+
+                    # If a parent has been set, we need to fetch/create this entry. Otherwise ("0") we ignore it.
+                    if parent == "0":
+                        parent = None
+                    else:
+                        parent = KonovaCode.objects.get_or_create(
+                            id=parent,
+                        )[0]
+
+                    code = KonovaCode.objects.get_or_create(
+                        id=atom_id,
+                    )
+                    created = code[1]
+                    if created:
+                        num_created += 1
+                    else:
+                        num_updated += 1
+                    code = code[0]
+                    code.short_name = short_name
+                    code.long_name = long_name
+                    code.parent = parent
+                    code.is_active = is_archived
+
+                    code.save()
+                    if code not in code_list.codes.all():
+                        code_list.codes.add(code)
+
+                """
+
+        except KeyboardInterrupt:
+            self._break_line()
+            exit(-1)
+
+    def _rec_xml_code_fetch(self, items: list, code_list: KonovaCodeList, parent: KonovaCode):
+        if items is None:
+            return
+        else:
+            for element in items:
+                children = element.find("items")
+                atom_id = element.find("atomid").text
+                short_name = element.find("shortname").text
+                long_name = element.find("longname").text
+                is_archived = bool(element.find("archive").text)
+
+                code = KonovaCode.objects.get_or_create(
+                    id=atom_id,
+                )
+                code = code[0]
+                code.short_name = short_name
+                code.long_name = long_name
+                code.parent = parent
+                code.is_active = is_archived
+                code.is_leaf = children is None
+
+                code.save()
+                if code not in code_list.codes.all():
+                    code_list.codes.add(code)
+
+                self._rec_xml_code_fetch(
+                    items=children,
+                    code_list=code_list,
+                    parent=code
+                )
+
+    def _break_line(self):
+        """ Simply prints a line break
+
+        Returns:
+
+        """
+        self.stdout.write("\n")
+
+    def _write_warning(self, txt: str):
+        self.stdout.write(
+            self.style.WARNING(
+                txt
+            )
+        )
+
+    def _write_success(self, txt: str):
+        self.stdout.write(
+            self.style.SUCCESS(
+                txt
+            )
+        )
+
+    def _write_error(self, txt: str):
+        self.stdout.write(
+            self.style.ERROR(
+                txt
+            )
+        )

codelist/models.py

@@ -0,0 +1,65 @@
+from django.db import models
+
+
+class KonovaCode(models.Model):
+    """
+    Representation of the OSIRIS Codelisten codes.
+
+    KonovaCodes will be updated using a command regularly to provide all codes, which are provided by the
+    Codelisten application itself. To reduce traffic, we store them in our own database.
+    """
+    id = models.IntegerField(
+        primary_key=True,
+        help_text="AtomId; Identifies this code uniquely over all NatIT projects"
+    )
+    parent = models.ForeignKey(
+        "KonovaCode",
+        on_delete=models.CASCADE,
+        null=True,
+        blank=True,
+    )
+    short_name = models.CharField(
+        max_length=500,
+        null=True,
+        blank=True,
+        help_text="Short version of long name",
+    )
+    long_name = models.CharField(
+        max_length=1000,
+        null=True,
+        blank=True,
+        help_text="",
+    )
+    is_leaf = models.BooleanField(
+        default=False,
+        help_text="Whether this code has children or not"
+    )
+    is_active = models.BooleanField(
+        default=False,
+        help_text="Whether this code is archived or not"
+    )
+
+    def __str__(self):
+        if self.is_leaf and self.parent:
+            return "{} | {}".format(self.parent.long_name, self.long_name)
+        return self.long_name
+
+
+class KonovaCodeList(models.Model):
+    """
+    The codelist itself
+    """
+    id = models.IntegerField(
+        primary_key=True,
+        help_text="List identifier",
+    )
+    codes = models.ManyToManyField(
+        KonovaCode,
+        null=True,
+        blank=True,
+        help_text="Codes for this list",
+        related_name="code_lists"
+    )
+
+    def __str__(self):
+        return str(self.id)

codelist/settings.py

@@ -0,0 +1,23 @@
+"""
+Author: Michel Peltriaux
+Organization: Struktur- und Genehmigungsdirektion Nord, Rhineland-Palatinate, Germany
+Contact: michel.peltriaux@sgdnord.rlp.de
+Created on: 23.08.21
+
+"""
+
+# Codelist definitions
+CODELIST_BASE_URL = "https://codelisten.naturschutz.rlp.de/repository/referenzliste"
+
+# Identifier
+CODELIST_INTERVENTION_HANDLER_ID = 903  # CLMassnahmeträger
+CODELIST_CONSERVATION_OFFICE_ID = 907  # CLNaturschutzbehörden
+CODELIST_REGISTRATION_OFFICE_ID = 1053  # CLZulassungsbehörden
+CODELIST_BIOTOPES_ID = 974  # CL_EIV_Biotoptypen
+CODELIST_LAW_ID = 1048  # CLVerfahrensrecht
+
+CODELIST_COMPENSATION_HANDLER_ID = 1052  # CLEingreifer
+CODELIST_COMPENSATION_ACTION_ID = 1026  # CLMassnahmedetail
+CODELIST_COMPENSATION_ACTION_CLASS_ID = 1034  # CLMassnahmeklasse
+CODELIST_COMPENSATION_ADDITIONAL_TYPE_ID = 1028  # CLMassnahmetyp
+CODELIST_COMPENSATION_COMBINATION_ID = 1049  # CLKombimassnahme

codelist/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

codelist/urls.py

@@ -0,0 +1,14 @@
+"""
+Author: Michel Peltriaux
+Organization: Struktur- und Genehmigungsdirektion Nord, Rhineland-Palatinate, Germany
+Contact: michel.peltriaux@sgdnord.rlp.de
+Created on: 23.08.21
+
+"""
+from django.urls import path
+
+
+app_name = "codelist"
+urlpatterns = [
+
+]

konova/autocompletes.py

@@ -7,6 +7,7 @@ Created on: 07.12.20
 """
 from dal_select2.views import Select2QuerySetView
 
+from codelist.models import KonovaCode
 from compensation.models import EcoAccount
 from intervention.models import Intervention
 from organisation.models import Organisation
@@ -74,3 +75,38 @@ class InterventionAutocomplete(Select2QuerySetView):
                 "identifier"
             )
         return qs
+
+
+class KonovaCodeAutocomplete(Select2QuerySetView):
+    """
+    Provides simple autocomplete functionality for codes
+
+    Parameter support:
+    * q: Search for a word inside long_name of a code
+    * c: Search inside a special codelist
+
+    """
+    def dispatch(self, request, *args, **kwargs):
+        # Retrieve 'c' for 'code' from request
+        self.c = request.GET.get("c", "")
+        return super().dispatch(request, *args, **kwargs)
+
+    def get_queryset(self):
+        if self.request.user.is_anonymous:
+            return KonovaCode.objects.none()
+        qs = KonovaCode.objects.filter(
+            is_active=True,
+            is_leaf=True,
+        )
+        if self.c:
+            qs = qs.filter(
+                code_lists__in=[self.c]
+            )
+        if self.q:
+            qs = qs.filter(
+                long_name__icontains=self.q
+            )
+            qs.order_by(
+                "long_name"
+            )
+        return qs

konova/sub_settings/django_settings.py

@@ -69,6 +69,7 @@ INSTALLED_APPS = [
     'news',
     'user',
     'ema',
+    'codelist',
 ]
 if DEBUG:
     INSTALLED_APPS += [

konova/urls.py

@@ -18,7 +18,7 @@ from django.contrib import admin
 from django.urls import path, include
 
 from konova.autocompletes import OrganisationAutocomplete, NonOfficialOrganisationAutocomplete, EcoAccountAutocomplete, \
-    InterventionAutocomplete
+    InterventionAutocomplete, KonovaCodeAutocomplete
 from konova.settings import SSO_SERVER, SSO_PUBLIC_KEY, SSO_PRIVATE_KEY, DEBUG
 from konova.sso.sso import KonovaSSOClient
 from konova.views import logout_view, home_view, get_document_view, remove_document_view, remove_deadline_view
@@ -35,6 +35,7 @@ urlpatterns = [
     path('organisation/', include("organisation.urls")),
     path('user/', include("user.urls")),
     path('news/', include("news.urls")),
+    path('news/', include("codelist.urls")),
 
     # Generic documents routes
     path('document/<id>', get_document_view, name="doc-open"),
@@ -43,11 +44,12 @@ urlpatterns = [
     # Generic deadline routes
     path('deadline/<id>/remove', remove_deadline_view, name="deadline-remove"),
 
-    # Autocomplete paths
+    # Autocomplete paths for all apps
     path("atcmplt/orgs", OrganisationAutocomplete.as_view(), name="orgs-autocomplete"),
     path("atcmplt/orgs/other", NonOfficialOrganisationAutocomplete.as_view(), name="other-orgs-autocomplete"),
     path("atcmplt/eco-accounts", EcoAccountAutocomplete.as_view(), name="accounts-autocomplete"),
     path("atcmplt/interventions", InterventionAutocomplete.as_view(), name="interventions-autocomplete"),
+    path("atcmplt/codes", KonovaCodeAutocomplete.as_view(), name="codes-autocomplete"),
 ]
 
 if DEBUG: