IT-Naturschutz/konova
7
1
Fork 0
Code Issues 6 Pull Requests 1 Actions Projects Releases 28 Wiki Activity

Codelist integration

Browse Source 
* adds codelist app
* adds KonovaCodeList and KonovaCode model for fetching and storing OSIRIS Codelisten entries
* adds update_codelist command for updating and fetching codes
* adds autocomplete route for using codelists in forms
This commit is contained in:
mipel 2021-08-23 18:30:02 +02:00
parent 631b5d0ea6
commit 41c9ed106c
12 changed files with 344 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
codelist/__init__.py Normal file
View File

36
codelist/admin.py Normal file
View File

@ -0,0 +1,36 @@
from django.contrib import admin
from codelist.models import KonovaCode, KonovaCodeList
class KonovaCodeListAdmin(admin.ModelAdmin):
list_display = [
"id",
]
readonly_fields = [
"id",
"codes",
]
class KonovaCodeAdmin(admin.ModelAdmin):
list_display = [
"id",
"parent",
"short_name",
"long_name",
"is_leaf",
"is_active",
]
readonly_fields = [
"id",
"short_name",
"long_name",
"is_leaf",
"parent",
]
#admin.site.register(KonovaCodeList, KonovaCodeListAdmin)
admin.site.register(KonovaCode, KonovaCodeAdmin)

5
codelist/apps.py Normal file
View File

@ -0,0 +1,5 @@
from django.apps import AppConfig
class CodelistConfig(AppConfig):
name = 'codelist'

157
codelist/management/commands/update_codelist.py Normal file
View File

@ -0,0 +1,157 @@
"""
Author: Michel Peltriaux
Organization: Struktur- und Genehmigungsdirektion Nord, Rhineland-Palatinate, Germany
Contact: michel.peltriaux@sgdnord.rlp.de
Created on: 23.08.21
"""
import requests
from django.core.management import BaseCommand
from xml.etree import ElementTree as etree
from codelist.models import KonovaCode, KonovaCodeList
from codelist.settings import CODELIST_INTERVENTION_HANDLER_ID, CODELIST_CONSERVATION_OFFICE_ID, \
CODELIST_REGISTRATION_OFFICE_ID, CODELIST_BIOTOPES_ID, CODELIST_LAW_ID, CODELIST_COMPENSATION_HANDLER_ID, \
CODELIST_COMPENSATION_ACTION_ID, CODELIST_COMPENSATION_ACTION_CLASS_ID, CODELIST_COMPENSATION_ADDITIONAL_TYPE_ID, \
CODELIST_COMPENSATION_COMBINATION_ID, CODELIST_BASE_URL
class Command(BaseCommand):
help = "Performs test on collisions using the identifier generation"
def handle(self, *args, **options):
try:
codelist_ids = [
CODELIST_INTERVENTION_HANDLER_ID,
CODELIST_CONSERVATION_OFFICE_ID,
CODELIST_REGISTRATION_OFFICE_ID,
CODELIST_BIOTOPES_ID,
CODELIST_LAW_ID,
CODELIST_COMPENSATION_HANDLER_ID,
CODELIST_COMPENSATION_ACTION_ID,
CODELIST_COMPENSATION_ACTION_CLASS_ID,
CODELIST_COMPENSATION_ADDITIONAL_TYPE_ID,
CODELIST_COMPENSATION_COMBINATION_ID,
]
self._write_warning("Fetching codes...")
for list_id in codelist_ids:
_url = CODELIST_BASE_URL + "/" + str(list_id)
result = requests.get(url=_url)
if result.status_code != 200:
self._write_error("Error on codelist url '{}'".format(_url))
continue
self._write_warning("Fetch codes for list id {}".format(list_id))
content = result.content.decode("utf-8")
root = etree.fromstring(content)
items = root.findall("item")
self._write_warning("Found {} codes. Process now...".format(len(items)))
code_list = KonovaCodeList.objects.get_or_create(
id=list_id,
)[0]
self._rec_xml_code_fetch(
items=items,
code_list=code_list,
parent=None,
)
"""
for element in items:
atom_id = element.find("atomid").text
parent = element.find("vaterid").text
short_name = element.find("shortname").text
long_name = element.find("longname").text
is_archived = bool(element.find("archive").text)
# If a parent has been set, we need to fetch/create this entry. Otherwise ("0") we ignore it.
if parent == "0":
parent = None
else:
parent = KonovaCode.objects.get_or_create(
id=parent,
)[0]
code = KonovaCode.objects.get_or_create(
id=atom_id,
)
created = code[1]
if created:
num_created += 1
else:
num_updated += 1
code = code[0]
code.short_name = short_name
code.long_name = long_name
code.parent = parent
code.is_active = is_archived
code.save()
if code not in code_list.codes.all():
code_list.codes.add(code)
"""
except KeyboardInterrupt:
self._break_line()
exit(-1)
def _rec_xml_code_fetch(self, items: list, code_list: KonovaCodeList, parent: KonovaCode):
if items is None:
return
else:
for element in items:
children = element.find("items")
atom_id = element.find("atomid").text
short_name = element.find("shortname").text
long_name = element.find("longname").text
is_archived = bool(element.find("archive").text)
code = KonovaCode.objects.get_or_create(
id=atom_id,
)
code = code[0]
code.short_name = short_name
code.long_name = long_name
code.parent = parent
code.is_active = is_archived
code.is_leaf = children is None
code.save()
if code not in code_list.codes.all():
code_list.codes.add(code)
self._rec_xml_code_fetch(
items=children,
code_list=code_list,
parent=code
)
def _break_line(self):
""" Simply prints a line break
Returns:
"""
self.stdout.write("\n")
def _write_warning(self, txt: str):
self.stdout.write(
self.style.WARNING(
txt
)
)
def _write_success(self, txt: str):
self.stdout.write(
self.style.SUCCESS(
txt
)
)
def _write_error(self, txt: str):
self.stdout.write(
self.style.ERROR(
txt
)
)

65
codelist/models.py Normal file
View File

@ -0,0 +1,65 @@
from django.db import models
class KonovaCode(models.Model):
"""
Representation of the OSIRIS Codelisten codes.
KonovaCodes will be updated using a command regularly to provide all codes, which are provided by the
Codelisten application itself. To reduce traffic, we store them in our own database.
"""
id = models.IntegerField(
primary_key=True,
help_text="AtomId; Identifies this code uniquely over all NatIT projects"
)
parent = models.ForeignKey(
"KonovaCode",
on_delete=models.CASCADE,
null=True,
blank=True,
)
short_name = models.CharField(
max_length=500,
null=True,
blank=True,
help_text="Short version of long name",
)
long_name = models.CharField(
max_length=1000,
null=True,
blank=True,
help_text="",
)
is_leaf = models.BooleanField(
default=False,
help_text="Whether this code has children or not"
)
is_active = models.BooleanField(
default=False,
help_text="Whether this code is archived or not"
)
def __str__(self):
if self.is_leaf and self.parent:
return "{} | {}".format(self.parent.long_name, self.long_name)
return self.long_name
class KonovaCodeList(models.Model):
"""
The codelist itself
"""
id = models.IntegerField(
primary_key=True,
help_text="List identifier",
)
codes = models.ManyToManyField(
KonovaCode,
null=True,
blank=True,
help_text="Codes for this list",
related_name="code_lists"
)
def __str__(self):
return str(self.id)

23
codelist/settings.py Normal file
View File

@ -0,0 +1,23 @@
"""
Author: Michel Peltriaux
Organization: Struktur- und Genehmigungsdirektion Nord, Rhineland-Palatinate, Germany
Contact: michel.peltriaux@sgdnord.rlp.de
Created on: 23.08.21
"""
# Codelist definitions
CODELIST_BASE_URL = "https://codelisten.naturschutz.rlp.de/repository/referenzliste"
# Identifier
CODELIST_INTERVENTION_HANDLER_ID = 903 # CLMassnahmeträger
CODELIST_CONSERVATION_OFFICE_ID = 907 # CLNaturschutzbehörden
CODELIST_REGISTRATION_OFFICE_ID = 1053 # CLZulassungsbehörden
CODELIST_BIOTOPES_ID = 974 # CL_EIV_Biotoptypen
CODELIST_LAW_ID = 1048 # CLVerfahrensrecht
CODELIST_COMPENSATION_HANDLER_ID = 1052 # CLEingreifer
CODELIST_COMPENSATION_ACTION_ID = 1026 # CLMassnahmedetail
CODELIST_COMPENSATION_ACTION_CLASS_ID = 1034 # CLMassnahmeklasse
CODELIST_COMPENSATION_ADDITIONAL_TYPE_ID = 1028 # CLMassnahmetyp
CODELIST_COMPENSATION_COMBINATION_ID = 1049 # CLKombimassnahme

3
codelist/tests.py Normal file
View File

@ -0,0 +1,3 @@
from django.test import TestCase
# Create your tests here.

14
codelist/urls.py Normal file
View File

@ -0,0 +1,14 @@
"""
Author: Michel Peltriaux
Organization: Struktur- und Genehmigungsdirektion Nord, Rhineland-Palatinate, Germany
Contact: michel.peltriaux@sgdnord.rlp.de
Created on: 23.08.21
"""
from django.urls import path
app_name = "codelist"
urlpatterns = [
]

0
codelist/views.py Normal file
View File

36
konova/autocompletes.py
View File

@ -7,6 +7,7 @@ Created on: 07.12.20
"""
from dal_select2.views import Select2QuerySetView
from codelist.models import KonovaCode
from compensation.models import EcoAccount
from intervention.models import Intervention
from organisation.models import Organisation
@ -74,3 +75,38 @@ class InterventionAutocomplete(Select2QuerySetView):
"identifier"
)
return qs
class KonovaCodeAutocomplete(Select2QuerySetView):
"""
Provides simple autocomplete functionality for codes
Parameter support:
* q: Search for a word inside long_name of a code
* c: Search inside a special codelist
"""
def dispatch(self, request, *args, **kwargs):
# Retrieve 'c' for 'code' from request
self.c = request.GET.get("c", "")
return super().dispatch(request, *args, **kwargs)
def get_queryset(self):
if self.request.user.is_anonymous:
return KonovaCode.objects.none()
qs = KonovaCode.objects.filter(
is_active=True,
is_leaf=True,
)
if self.c:
qs = qs.filter(
code_lists__in=[self.c]
)
if self.q:
qs = qs.filter(
long_name__icontains=self.q
)
qs.order_by(
"long_name"
)
return qs

1
konova/sub_settings/django_settings.py
View File

@ -69,6 +69,7 @@ INSTALLED_APPS = [
'news',
'user',
'ema',
'codelist',
]
if DEBUG:
INSTALLED_APPS += [

6
konova/urls.py
View File

@ -18,7 +18,7 @@ from django.contrib import admin
from django.urls import path, include
from konova.autocompletes import OrganisationAutocomplete, NonOfficialOrganisationAutocomplete, EcoAccountAutocomplete, \
InterventionAutocomplete
InterventionAutocomplete, KonovaCodeAutocomplete
from konova.settings import SSO_SERVER, SSO_PUBLIC_KEY, SSO_PRIVATE_KEY, DEBUG
from konova.sso.sso import KonovaSSOClient
from konova.views import logout_view, home_view, get_document_view, remove_document_view, remove_deadline_view
@ -35,6 +35,7 @@ urlpatterns = [
path('organisation/', include("organisation.urls")),
path('user/', include("user.urls")),
path('news/', include("news.urls")),
path('news/', include("codelist.urls")),
# Generic documents routes
path('document/<id>', get_document_view, name="doc-open"),
@ -43,11 +44,12 @@ urlpatterns = [
# Generic deadline routes
path('deadline/<id>/remove', remove_deadline_view, name="deadline-remove"),
# Autocomplete paths
# Autocomplete paths for all apps
path("atcmplt/orgs", OrganisationAutocomplete.as_view(), name="orgs-autocomplete"),
path("atcmplt/orgs/other", NonOfficialOrganisationAutocomplete.as_view(), name="other-orgs-autocomplete"),
path("atcmplt/eco-accounts", EcoAccountAutocomplete.as_view(), name="accounts-autocomplete"),
path("atcmplt/interventions", InterventionAutocomplete.as_view(), name="interventions-autocomplete"),
path("atcmplt/codes", KonovaCodeAutocomplete.as_view(), name="codes-autocomplete"),
]
if DEBUG: