From 06ad0fdc2dd7d9354bfdca40bc31df8f9bf2f0c4 Mon Sep 17 00:00:00 2001 From: mpeltriaux Date: Tue, 8 Mar 2022 11:54:26 +0100 Subject: [PATCH 1/3] #131 WIP: EGON exporter * adds incomplete WIP implementation of an EGON exporter --- intervention/utils/egon_export.py | 150 ++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 intervention/utils/egon_export.py diff --git a/intervention/utils/egon_export.py b/intervention/utils/egon_export.py new file mode 100644 index 00000000..c3e62aba --- /dev/null +++ b/intervention/utils/egon_export.py @@ -0,0 +1,150 @@ +""" +Author: Michel Peltriaux +Organization: Struktur- und Genehmigungsdirektion Nord, Rhineland-Palatinate, Germany +Contact: michel.peltriaux@sgdnord.rlp.de +Created on: 07.03.22 + +""" +import xmltodict +from django.contrib.gis.gdal import OGRGeometry +from django.db.models import Sum +from django.utils import formats + +from intervention.models import Intervention +from xml.etree import ElementTree as etree + +from konova.sub_settings.lanis_settings import DEFAULT_SRID_RLP + + +class EgonExporter: + """ + EGON is the payment management system of SNU RLP. Due to compatibility reasons we need to provide the old style + of data transmission between KSP and EGON: + 1. Create GML from intervention object + 2. Send created GML to the appropriate RabbitMQ channel + """ + intervention = None + gml_builder = None + + def __init__(self, intervention: Intervention): + self.intervention = intervention + self.gml_builder = EgonGmlBuilder(intervention) + + def export_to_rabbitmq(self): + raise NotImplementedError("ToDo!") + + +class EgonGmlBuilder: + """ + Creates the GML for EGON export + """ + intervention = None + gml = None + + def __init__(self, intervention: Intervention): + self.intervention = intervention + self.gml = self.build_gml() + + def build_gml(self): + all_payments = self.intervention.payments.aggregate( + summed=Sum("amount") + )["summed"] + + comp_type = "Ersatzzahlung" + if self.intervention.compensations.exists(): + comp_type += " und Kompensation" + + geom = self.intervention.geometry.geom + geom.transform(DEFAULT_SRID_RLP) + geoms_list = [ + { + "gml:polygonMember": { + "gml:Polygon": { + "gml:exterior": { + "gml:LinearRing": { + "gml:posList": " ".join([f"{str(coord[0])},{str(coord[1])}" for coord in coords[0]]) + } + } + } + } + } for coords in geom.coords + ] + + parcels = self.intervention.get_underlying_parcels() + spatial_reference_list = [ + { + "oneo:Raumreferenz": { + "oneo:datumAbgleich": None, + "oneo:ortsangabe": { + "oneo:Ortsangaben": { + "oneo:kreisSchluessel": parcel.district.krs, + "oneo:gemeindeSchluessel": parcel.district.gmnd, + "oneo:verbandsgemeindeSchluessel": parcel.gmrkng, + "oneo:flurstuecksKennzeichen": None, + } + }, + } + } for parcel in parcels + ] + + t = { + "wfs:FeatureCollection": { + "oneo:Eingriffsverfahren": { + "@gml:id": self.intervention.identifier, + "oneo:azEintragungsstelle": self.intervention.responsible.conservation_file_number, + "oneo:azZulassungsstelle": self.intervention.responsible.registration_file_number, + "oneo:bemerkungZulassungsstelle": None, + "oneo:eintragungsstelle": self.intervention.responsible.conservation_office.long_name, + "oneo:zulassungsstelle": self.intervention.responsible.registration_office.long_name, + "oneo:ersatzzahlung": all_payments, + "oneo:kompensationsart": comp_type, + "oneo:verfahrensrecht": self.intervention.legal.laws.first().short_name, + "oneo:verfahrenstyp": self.intervention.legal.process_type.long_name, + "oneo:eingreifer": { + "oneo:Eingreifer": { + "oneo:art": self.intervention.responsible.handler.type.long_name, + "oneo:bemerkung": self.intervention.responsible.handler.type.long_name, + } + }, + "oneo:erfasser": { + "oneo:Erfasser": { + "oneo:name": None, + "oneo:bemerkung": None, + } + }, + "oneo:zulassung": { + "oneo:Zulassungstermin": { + "oneo:bauBeginn": formats.localize(self.intervention.payments.first().due_on), + "oneo:erlass": formats.localize(self.intervention.legal.registration_date), + "oneo:rechtsKraft": formats.localize(self.intervention.legal.binding_date), + } + }, + "oneo:geometrie": { + "gml:multiSurfaceProperty": { + "gml:MultiPolygon": { + "@srsName": f"http://www.opengis.net/gml/srs/epsg.xml#{DEFAULT_SRID_RLP}", + "#text": geoms_list, + } + }, + }, + "oneo:kennung": self.intervention.identifier, + "oneo:bezeichnung": self.intervention.title, + "oneo:bemerkung": self.intervention.comment, + "oneo:verantwortlicheStelle": None, + "oneo:veroffentlichtAm": None, + "oneo:raumreferenz": spatial_reference_list, + "oneo:foto": { + "oneo:Foto": { + "oneo:aufnahmezeitpunkt": None, + "oneo:bemerkung": None, + "oneo:fotoverweis": None, + "oneo:dateiname": None, + "oneo:hauptfoto": False, + } + }, + } + }, + } + gml = xmltodict.unparse(t, pretty=True) + print(gml) + return gml \ No newline at end of file From 17c954e844613dad2af605b3047d8cffcec00de1 Mon Sep 17 00:00:00 2001 From: mpeltriaux Date: Wed, 9 Mar 2022 08:34:26 +0100 Subject: [PATCH 2/3] #131 EGON exporter * enhances EGON exporter code structure --- intervention/utils/egon_export.py | 99 ++++++++++++++++++++----------- 1 file changed, 63 insertions(+), 36 deletions(-) diff --git a/intervention/utils/egon_export.py b/intervention/utils/egon_export.py index c3e62aba..fe66626b 100644 --- a/intervention/utils/egon_export.py +++ b/intervention/utils/egon_export.py @@ -5,13 +5,13 @@ Contact: michel.peltriaux@sgdnord.rlp.de Created on: 07.03.22 """ +import base64 + import xmltodict -from django.contrib.gis.gdal import OGRGeometry from django.db.models import Sum from django.utils import formats from intervention.models import Intervention -from xml.etree import ElementTree as etree from konova.sub_settings.lanis_settings import DEFAULT_SRID_RLP @@ -45,49 +45,83 @@ class EgonGmlBuilder: self.intervention = intervention self.gml = self.build_gml() - def build_gml(self): + def _gen_flurstuecksKennzeichen(self, parcel): + """ Generates oneo:flurstuecksKennzeichen to provide backwards compatibility + + Args: + parcel (Parcel): The requested parcel + + Returns: + str + """ + gmrkng_code = "000000" + flr_code = "{0:03d}".format(int(parcel.flr or 0)) + flrstckzhlr_code = "{0:05d}".format(int(parcel.flrstck_zhlr or 0)) + flrstcknnr_code = "{0:06d}".format(int(parcel.flrstck_nnr or 0)) + return gmrkng_code + flr_code + flrstckzhlr_code + flrstcknnr_code + + def _sum_all_payments(self): all_payments = self.intervention.payments.aggregate( summed=Sum("amount") )["summed"] + return all_payments + def _gen_kompensationsArt(self): comp_type = "Ersatzzahlung" if self.intervention.compensations.exists(): comp_type += " und Kompensation" + return comp_type + def _gen_geometry_list(self): geom = self.intervention.geometry.geom geom.transform(DEFAULT_SRID_RLP) geoms_list = [ { - "gml:polygonMember": { - "gml:Polygon": { - "gml:exterior": { - "gml:LinearRing": { - "gml:posList": " ".join([f"{str(coord[0])},{str(coord[1])}" for coord in coords[0]]) - } + "gml:Polygon": { + "gml:exterior": { + "gml:LinearRing": { + "gml:posList": " ".join([f"{str(coord[0])},{str(coord[1])}" for coord in coords[0]]) } } } } for coords in geom.coords ] + return geoms_list + def _gen_raumreferenz(self): parcels = self.intervention.get_underlying_parcels() spatial_reference_list = [ { - "oneo:Raumreferenz": { - "oneo:datumAbgleich": None, - "oneo:ortsangabe": { - "oneo:Ortsangaben": { - "oneo:kreisSchluessel": parcel.district.krs, - "oneo:gemeindeSchluessel": parcel.district.gmnd, - "oneo:verbandsgemeindeSchluessel": parcel.gmrkng, - "oneo:flurstuecksKennzeichen": None, - } - }, - } + "oneo:datumAbgleich": None, + "oneo:ortsangabe": { + "oneo:Ortsangaben": { + "oneo:kreisSchluessel": parcel.district.krs, + "oneo:gemeindeSchluessel": parcel.district.gmnd, + "oneo:verbandsgemeindeSchluessel": parcel.gmrkng, + "oneo:flurstuecksKennzeichen": self._gen_flurstuecksKennzeichen(parcel), + } + }, } for parcel in parcels ] + return spatial_reference_list - t = { + def _gen_foto(self): + revoc_docs, regular_docs = self.intervention.get_documents() + docs_list = [ + { + "oneo:Foto": { + "oneo:aufnahmezeitpunkt": formats.localize(doc.date_of_creation), + "oneo:bemerkung": doc.comment, + "oneo:fotoverweis": base64.b64encode(doc.file.read()).decode("utf-8"), + "oneo:dateiname": doc.title, + "oneo:hauptfoto": False, + } + } for doc in regular_docs + ] + return docs_list + + def build_gml(self): + xml_dict = { "wfs:FeatureCollection": { "oneo:Eingriffsverfahren": { "@gml:id": self.intervention.identifier, @@ -96,8 +130,8 @@ class EgonGmlBuilder: "oneo:bemerkungZulassungsstelle": None, "oneo:eintragungsstelle": self.intervention.responsible.conservation_office.long_name, "oneo:zulassungsstelle": self.intervention.responsible.registration_office.long_name, - "oneo:ersatzzahlung": all_payments, - "oneo:kompensationsart": comp_type, + "oneo:ersatzzahlung": self._sum_all_payments(), + "oneo:kompensationsart": self._gen_kompensationsArt(), "oneo:verfahrensrecht": self.intervention.legal.laws.first().short_name, "oneo:verfahrenstyp": self.intervention.legal.process_type.long_name, "oneo:eingreifer": { @@ -123,7 +157,7 @@ class EgonGmlBuilder: "gml:multiSurfaceProperty": { "gml:MultiPolygon": { "@srsName": f"http://www.opengis.net/gml/srs/epsg.xml#{DEFAULT_SRID_RLP}", - "#text": geoms_list, + "gml:polygonMember": self._gen_geometry_list(), } }, }, @@ -132,19 +166,12 @@ class EgonGmlBuilder: "oneo:bemerkung": self.intervention.comment, "oneo:verantwortlicheStelle": None, "oneo:veroffentlichtAm": None, - "oneo:raumreferenz": spatial_reference_list, - "oneo:foto": { - "oneo:Foto": { - "oneo:aufnahmezeitpunkt": None, - "oneo:bemerkung": None, - "oneo:fotoverweis": None, - "oneo:dateiname": None, - "oneo:hauptfoto": False, - } + "oneo:raumreferenz": { + "oneo:Raumreferenz": self._gen_raumreferenz(), }, + "oneo:foto": self._gen_foto(), } }, } - gml = xmltodict.unparse(t, pretty=True) - print(gml) - return gml \ No newline at end of file + gml = xmltodict.unparse(xml_dict) + return gml From 7689e0b80d69bec61a066679e3c12364ec03f9aa Mon Sep 17 00:00:00 2001 From: mpeltriaux Date: Mon, 21 Mar 2022 12:14:55 +0100 Subject: [PATCH 3/3] #131 EGON export * finishes egon compatible (tested) data export * moves egon export into celery process * adds export of data in case of intervention recording * adds _RABBITMQ_ settings for intervention/settings.py * adds new dependency for requirements.txt --- intervention/models/intervention.py | 12 ++++ intervention/settings.py | 9 ++- intervention/tasks.py | 18 +++++ intervention/utils/egon_export.py | 103 ++++++++++++++++++++++------ requirements.txt | 1 + 5 files changed, 120 insertions(+), 23 deletions(-) create mode 100644 intervention/tasks.py diff --git a/intervention/models/intervention.py b/intervention/models/intervention.py index 167c27af..c215a139 100644 --- a/intervention/models/intervention.py +++ b/intervention/models/intervention.py @@ -13,6 +13,7 @@ from django.db.models.fields.files import FieldFile from django.urls import reverse from django.utils import timezone +from intervention.tasks import celery_export_to_egon from user.models import User from django.db import models, transaction from django.db.models import QuerySet @@ -131,9 +132,20 @@ class Intervention(BaseObject, ShareableObjectMixin, RecordableObjectMixin, Chec self.add_log_entry_to_compensations(log_entry) return log_entry + def send_data_to_egon(self): + """ Performs the export to rabbitmq of this intervention's data + + FOLLOWING BACKWARDS COMPATIBILITY LOGIC + + Returns: + + """ + celery_export_to_egon.delay(self.id) + def set_recorded(self, user: User) -> UserActionLogEntry: log_entry = super().set_recorded(user) self.add_log_entry_to_compensations(log_entry) + self.send_data_to_egon() return log_entry def add_log_entry_to_compensations(self, log_entry: UserActionLogEntry): diff --git a/intervention/settings.py b/intervention/settings.py index 2b6ebee8..a5c974d4 100644 --- a/intervention/settings.py +++ b/intervention/settings.py @@ -6,4 +6,11 @@ Created on: 30.11.20 """ INTERVENTION_IDENTIFIER_LENGTH = 6 -INTERVENTION_IDENTIFIER_TEMPLATE = "EIV-{}" \ No newline at end of file +INTERVENTION_IDENTIFIER_TEMPLATE = "EIV-{}" + +# EGON connection settings via rabbitmq +# NEEDED FOR BACKWARDS COMPATIBILITY +EGON_RABBITMQ_HOST = "CHANGE_ME" +EGON_RABBITMQ_PORT = "CHANGE_ME" +EGON_RABBITMQ_USER = "CHANGE_ME" +EGON_RABBITMQ_PW = "CHANGE_ME" diff --git a/intervention/tasks.py b/intervention/tasks.py new file mode 100644 index 00000000..3488e189 --- /dev/null +++ b/intervention/tasks.py @@ -0,0 +1,18 @@ +""" +Author: Michel Peltriaux +Organization: Struktur- und Genehmigungsdirektion Nord, Rhineland-Palatinate, Germany +Contact: michel.peltriaux@sgdnord.rlp.de +Created on: 21.03.22 + +""" +from celery import shared_task + +from intervention.utils.egon_export import EgonExporter + + +@shared_task +def celery_export_to_egon(intervention_id: str): + from intervention.models import Intervention + intervention = Intervention.objects.get(id=intervention_id) + egon_exporter = EgonExporter(intervention) + egon_exporter.export_to_rabbitmq() diff --git a/intervention/utils/egon_export.py b/intervention/utils/egon_export.py index fe66626b..39a4c318 100644 --- a/intervention/utils/egon_export.py +++ b/intervention/utils/egon_export.py @@ -6,12 +6,14 @@ Created on: 07.03.22 """ import base64 +import json +import pika import xmltodict from django.db.models import Sum -from django.utils import formats -from intervention.models import Intervention +from intervention.settings import EGON_RABBITMQ_HOST, EGON_RABBITMQ_USER, EGON_RABBITMQ_PW, EGON_RABBITMQ_PORT +from konova.sub_settings.django_settings import DEFAULT_DATE_FORMAT from konova.sub_settings.lanis_settings import DEFAULT_SRID_RLP @@ -26,12 +28,36 @@ class EgonExporter: intervention = None gml_builder = None - def __init__(self, intervention: Intervention): + def __init__(self, intervention): self.intervention = intervention self.gml_builder = EgonGmlBuilder(intervention) def export_to_rabbitmq(self): - raise NotImplementedError("ToDo!") + """ Sends the exporter gml to message broker rabbitmq to be fetched by EGON application from there + + Returns: + + """ + msg = { + "nachricht": self.gml_builder.gml, + } + msg = json.dumps(msg) + print(msg) + credentials = pika.PlainCredentials(EGON_RABBITMQ_USER, EGON_RABBITMQ_PW) + params = pika.ConnectionParameters( + EGON_RABBITMQ_HOST, + EGON_RABBITMQ_PORT, + "/", + credentials + ) + conn = pika.BlockingConnection(params) + channel = conn.channel() + channel.basic_publish( + exchange="", + routing_key="KSP_EGON", + body=msg.encode("utf-8"), + ) + conn.close() class EgonGmlBuilder: @@ -41,7 +67,7 @@ class EgonGmlBuilder: intervention = None gml = None - def __init__(self, intervention: Intervention): + def __init__(self, intervention): self.intervention = intervention self.gml = self.build_gml() @@ -66,11 +92,13 @@ class EgonGmlBuilder: )["summed"] return all_payments - def _gen_kompensationsArt(self): + def _gen_kompensationsArt(self) -> (str, int): comp_type = "Ersatzzahlung" + comp_type_code = 774898901 if self.intervention.compensations.exists(): comp_type += " und Kompensation" - return comp_type + comp_type_code = 771655351 + return comp_type, comp_type_code def _gen_geometry_list(self): geom = self.intervention.geometry.geom @@ -80,7 +108,7 @@ class EgonGmlBuilder: "gml:Polygon": { "gml:exterior": { "gml:LinearRing": { - "gml:posList": " ".join([f"{str(coord[0])},{str(coord[1])}" for coord in coords[0]]) + "gml:posList": " ".join([f"{str(coord[0])} {str(coord[1])}" for coord in coords[0]]) } } } @@ -95,9 +123,15 @@ class EgonGmlBuilder: "oneo:datumAbgleich": None, "oneo:ortsangabe": { "oneo:Ortsangaben": { - "oneo:kreisSchluessel": parcel.district.krs, - "oneo:gemeindeSchluessel": parcel.district.gmnd, - "oneo:verbandsgemeindeSchluessel": parcel.gmrkng, + "oneo:kreisSchluessel": { + "xlink:href": f"http://register.naturschutz.rlp.de/repository/services/referenzliste/588/{parcel.district.krs}", + }, + "oneo:gemeindeSchluessel": { + "xlink:href": f"http://register.naturschutz.rlp.de/repository/services/referenzliste/910/{parcel.district.gmnd}", + }, + "oneo:verbandsgemeindeSchluessel": { + "xlink:href": f"http://register.naturschutz.rlp.de/repository/services/referenzliste/589/{parcel.gmrkng}", + }, "oneo:flurstuecksKennzeichen": self._gen_flurstuecksKennzeichen(parcel), } }, @@ -110,7 +144,7 @@ class EgonGmlBuilder: docs_list = [ { "oneo:Foto": { - "oneo:aufnahmezeitpunkt": formats.localize(doc.date_of_creation), + "oneo:aufnahmezeitpunkt": doc.date_of_creation.strftime(DEFAULT_DATE_FORMAT), "oneo:bemerkung": doc.comment, "oneo:fotoverweis": base64.b64encode(doc.file.read()).decode("utf-8"), "oneo:dateiname": doc.title, @@ -121,23 +155,48 @@ class EgonGmlBuilder: return docs_list def build_gml(self): + comp_type, comp_type_code = self._gen_kompensationsArt() xml_dict = { "wfs:FeatureCollection": { + "@xmlns:wfs": "http://www.opengis.net/wfs", + "@xmlns:xlink": "http://www.w3.org/1999/xlink", + "@xmlns:oneo": "http://www.osiris-projekt.rlp.de/oneo", + "@xmlns:gmlexr": "http://www.opengis.net/gml/3.3/exr", + "@xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", + "@xmlns:gml": "http://www.opengis.net/gml/3.2", "oneo:Eingriffsverfahren": { "@gml:id": self.intervention.identifier, "oneo:azEintragungsstelle": self.intervention.responsible.conservation_file_number, "oneo:azZulassungsstelle": self.intervention.responsible.registration_file_number, "oneo:bemerkungZulassungsstelle": None, - "oneo:eintragungsstelle": self.intervention.responsible.conservation_office.long_name, - "oneo:zulassungsstelle": self.intervention.responsible.registration_office.long_name, + "oneo:eintragungsstelle": { + "@xlink:href": f"http://register.naturschutz.rlp.de/repository/services/referenzliste/907/{self.intervention.responsible.conservation_office.atom_id}", + "#text": self.intervention.responsible.conservation_office.long_name + }, + "oneo:zulassungsstelle": { + "@xlink:href": f"http://register.naturschutz.rlp.de/repository/services/referenzliste/1053/{self.intervention.responsible.registration_office.atom_id}", + "#text": self.intervention.responsible.registration_office.long_name + }, "oneo:ersatzzahlung": self._sum_all_payments(), - "oneo:kompensationsart": self._gen_kompensationsArt(), - "oneo:verfahrensrecht": self.intervention.legal.laws.first().short_name, - "oneo:verfahrenstyp": self.intervention.legal.process_type.long_name, + "oneo:kompensationsart": { + "@xlink:href": f"http://register.naturschutz.rlp.de/repository/services/referenzliste/88140/{comp_type_code}", + "#text": comp_type + }, + "oneo:verfahrensrecht": { + "@xlink:href": f"http://register.naturschutz.rlp.de/repository/services/referenzliste/1048/{self.intervention.legal.laws.first().atom_id}", + "#text": self.intervention.legal.laws.first().short_name + }, + "oneo:verfahrenstyp": { + "@xlink:href": f"http://register.naturschutz.rlp.de/repository/services/referenzliste/44382/{self.intervention.legal.process_type.atom_id}", + "#text": self.intervention.legal.process_type.long_name, + }, "oneo:eingreifer": { "oneo:Eingreifer": { - "oneo:art": self.intervention.responsible.handler.type.long_name, - "oneo:bemerkung": self.intervention.responsible.handler.type.long_name, + "oneo:art": { + "@xlink:href": f"http://register.naturschutz.rlp.de/repository/services/referenzliste/1053/{self.intervention.responsible.handler.type.atom_id}", + "#text": self.intervention.responsible.handler.type.long_name, + }, + "oneo:bemerkung": self.intervention.responsible.handler.detail, } }, "oneo:erfasser": { @@ -148,9 +207,9 @@ class EgonGmlBuilder: }, "oneo:zulassung": { "oneo:Zulassungstermin": { - "oneo:bauBeginn": formats.localize(self.intervention.payments.first().due_on), - "oneo:erlass": formats.localize(self.intervention.legal.registration_date), - "oneo:rechtsKraft": formats.localize(self.intervention.legal.binding_date), + "oneo:bauBeginn": self.intervention.payments.first().due_on.strftime(DEFAULT_DATE_FORMAT), + "oneo:erlass": self.intervention.legal.registration_date.strftime(DEFAULT_DATE_FORMAT), + "oneo:rechtsKraft": self.intervention.legal.binding_date.strftime(DEFAULT_DATE_FORMAT), } }, "oneo:geometrie": { diff --git a/requirements.txt b/requirements.txt index e2f43108..79a479ed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,6 +28,7 @@ kombu==5.2.3 openpyxl==3.0.9 OWSLib==0.25.0 packaging==21.3 +pika==1.2.0 prompt-toolkit==3.0.24 psycopg2-binary==2.9.1 pyparsing==3.0.6