#49 Update all parcels command

* adds update_all_parcels command which can be used e.g. with cronjob once a month to update all parcels and districts

konova/admin.py

@@ -7,7 +7,7 @@ Created on: 22.07.21
 """
 from django.contrib import admin
 
-from konova.models import Geometry, Deadline, GeometryConflict
+from konova.models import Geometry, Deadline, GeometryConflict, Parcel, District
 
 
 class GeometryAdmin(admin.ModelAdmin):
@@ -17,6 +17,25 @@ class GeometryAdmin(admin.ModelAdmin):
     ]
 
 
+class ParcelAdmin(admin.ModelAdmin):
+    list_display = [
+        "id",
+        "flr",
+        "flrstck_nnr",
+        "flrstck_zhlr",
+        "updated_on",
+    ]
+
+
+class DistrictAdmin(admin.ModelAdmin):
+    list_display = [
+        "id",
+        "gmrkng",
+        "gmnd",
+        "krs",
+    ]
+
+
 class GeometryConflictAdmin(admin.ModelAdmin):
     list_display = [
         "conflicting_geometry",
@@ -52,5 +71,7 @@ class BaseObjectAdmin(admin.ModelAdmin):
 
 
 admin.site.register(Geometry, GeometryAdmin)
+admin.site.register(Parcel, ParcelAdmin)
+admin.site.register(District, DistrictAdmin)
 admin.site.register(GeometryConflict, GeometryConflictAdmin)
 admin.site.register(Deadline, DeadlineAdmin)

konova/forms.py

@@ -287,6 +287,7 @@ class SimpleGeomForm(BaseForm):
             geometry = self.instance.geometry
             geometry.geom = self.cleaned_data.get("geom", MultiPolygon(srid=DEFAULT_SRID))
             geometry.modified = action
+            geometry.update_parcels()
             geometry.save()
         except LookupError:
             # No geometry or linked instance holding a geometry exist --> create a new one!

konova/management/commands/sanitize_db.py

@@ -9,7 +9,7 @@ from compensation.models import CompensationState, Compensation, EcoAccount, Com
 from ema.models import Ema
 from intervention.models import Intervention
 from konova.management.commands.setup import BaseKonovaCommand
-from konova.models import Deadline, Geometry
+from konova.models import Deadline, Geometry, Parcel, District
 from user.models import UserActionLogEntry
 
 
@@ -23,6 +23,7 @@ class Command(BaseKonovaCommand):
             self.sanitize_actions()
             self.sanitize_deadlines()
             self.sanitize_geometries()
+            self.sanitize_parcels_and_districts()
         except KeyboardInterrupt:
             self._break_line()
             exit(-1)
@@ -266,3 +267,34 @@ class Command(BaseKonovaCommand):
             self._write_success("No unused states found.")
         self._break_line()
 
+    def sanitize_parcels_and_districts(self):
+        """ Removes unattached parcels and districts
+
+        Returns:
+
+        """
+        self._write_warning("=== Sanitize parcels and districts ===")
+        unrelated_parcels = Parcel.objects.filter(
+            geometries=None,
+        )
+        num_unrelated_parcels = unrelated_parcels.count()
+        if num_unrelated_parcels > 0:
+            self._write_error(f"Found {num_unrelated_parcels} unrelated parcel entries. Delete now...")
+            unrelated_parcels.delete()
+            self._write_success("Unrelated parcels deleted.")
+        else:
+            self._write_success("No unrelated parcels found.")
+
+        unrelated_districts = District.objects.filter(
+            parcels=None,
+        )
+        num_unrelated_districts = unrelated_districts.count()
+        if num_unrelated_districts > 0:
+            self._write_error(f"Found {num_unrelated_districts} unrelated district entries. Delete now...")
+            unrelated_districts.delete()
+            self._write_success("Unrelated districts deleted.")
+        else:
+            self._write_success("No unrelated districts found.")
+
+        self._break_line()
+

konova/management/commands/update_all_parcels.py

@@ -0,0 +1,41 @@
+"""
+Author: Michel Peltriaux
+Organization: Struktur- und Genehmigungsdirektion Nord, Rhineland-Palatinate, Germany
+Contact: michel.peltriaux@sgdnord.rlp.de
+Created on: 04.01.22
+
+"""
+from konova.management.commands.setup import BaseKonovaCommand
+from konova.models import Geometry, Parcel, District
+
+
+class Command(BaseKonovaCommand):
+    help = "Checks the database' sanity and removes unused entries"
+
+    def handle(self, *args, **options):
+        try:
+            self.update_all_parcels()
+        except KeyboardInterrupt:
+            self._break_line()
+            exit(-1)
+
+    def update_all_parcels(self):
+        num_parcels_before = Parcel.objects.count()
+        num_districts_before = District.objects.count()
+        self._write_warning("=== Update parcels and districts ===")
+        geometries = Geometry.objects.all().exclude(
+            geom=None
+        )
+        self._write_warning(f"Process parcels for {geometries.count()} geometry entries now ...")
+        for geometry in geometries:
+            geometry.update_parcels()
+
+        num_parcels_after = Parcel.objects.count()
+        num_districts_after = District.objects.count()
+        if num_parcels_after != num_parcels_before:
+            self._write_error(f"Parcels have changed: {num_parcels_before} to {num_parcels_after} entries. You should run the sanitize command.")
+        if num_districts_after != num_districts_before:
+            self._write_error(f"Districts have changed: {num_districts_before} to {num_districts_after} entries. You should run the sanitize command.")
+
+        self._write_success("Updating parcels done!")
+        self._break_line()

konova/models/geometry.py

@@ -7,8 +7,10 @@ Created on: 15.11.21
 """
 from django.contrib.gis.db.models import MultiPolygonField
 from django.db import models
+from django.utils import timezone
 
 from konova.models import BaseResource, UuidModel
+from konova.utils.wfs.spatial import ParcelWFSFetcher
 
 
 class Geometry(BaseResource):
@@ -21,7 +23,6 @@ class Geometry(BaseResource):
     def save(self, *args, **kwargs):
         super().save(*args, **kwargs)
         self.check_for_conflicts()
-        self.update_parcels()
 
     def check_for_conflicts(self):
         """ Checks for new geometry overlaps
@@ -93,8 +94,38 @@ class Geometry(BaseResource):
         return objs
 
     def update_parcels(self):
-        # ToDo
+        """ Updates underlying parcel information
-        pass
+
+        Returns:
+
+        """
+        from konova.models import Parcel, District
+        parcel_fetcher = ParcelWFSFetcher(
+            geometry_id=self.id,
+        )
+        typename = "ave:Flurstueck"
+        fetched_parcels = parcel_fetcher.get_features(
+            typename
+        )
+        underlying_parcels = []
+        for result in fetched_parcels:
+            fetched_parcel = result[typename]
+            parcel_obj = Parcel.objects.get_or_create(
+                flr=fetched_parcel["ave:flur"],
+                flrstck_nnr=fetched_parcel['ave:flstnrnen'],
+                flrstck_zhlr=fetched_parcel['ave:flstnrzae'],
+            )[0]
+            district = District.objects.get_or_create(
+                gmrkng=fetched_parcel["ave:gemarkung"],
+                gmnd=fetched_parcel["ave:gemeinde"],
+                krs=fetched_parcel["ave:kreis"],
+            )[0]
+            parcel_obj.district = district
+            parcel_obj.updated_on = timezone.now()
+            parcel_obj.save()
+            underlying_parcels.append(parcel_obj)
+
+        self.parcels.set(underlying_parcels)
 
 
 class GeometryConflict(UuidModel):

konova/models/parcel.py

@@ -22,28 +22,30 @@ class Parcel(UuidModel):
     To avoid conflicts due to german Umlaute, the field names are shortened and vocals are dropped.
 
     """
-    geometries = models.ManyToManyField("konova.Geometry", related_name="parcels", null=True, blank=True)
+    geometries = models.ManyToManyField("konova.Geometry", related_name="parcels", blank=True)
     district = models.ForeignKey("konova.District", on_delete=models.SET_NULL, null=True, blank=True, related_name="parcels")
     flrstck_nnr = models.CharField(
         max_length=1000,
-        help_text="Flurstücksnenner"
+        help_text="Flurstücksnenner",
+        null=True,
+        blank=True,
     )
     flrstck_zhlr = models.CharField(
         max_length=1000,
-        help_text="Flurstückszähler"
+        help_text="Flurstückszähler",
+        null=True,
+        blank=True,
     )
     flr = models.CharField(
         max_length=1000,
-        help_text="Flur"
+        help_text="Flur",
-    )
+        null=True,
-    gmrkng = models.CharField(
+        blank=True,
-        max_length=1000,
-        help_text="Gemarkung"
     )
     updated_on = models.DateTimeField(auto_now_add=True)
 
     def __str__(self):
-        return f"{self.gmrkng} | {self.flr} | {self.flrstck_nnr} | {self.flrstck_zhlr}"
+        return f"{self.flr} | {self.flrstck_nnr} | {self.flrstck_zhlr}"
 
 
 class District(UuidModel):
@@ -54,18 +56,24 @@ class District(UuidModel):
     District.
 
     """
+    gmrkng = models.CharField(
+        max_length=1000,
+        help_text="Gemarkung",
+        null=True,
+        blank=True,
+    )
     gmnd = models.CharField(
         max_length=1000,
-        help_text="Gemeinde"
+        help_text="Gemeinde",
-    )
+        null=True,
-    vg = models.CharField(
+        blank=True,
-        max_length=1000,
-        help_text="Verbandsgemeinde",
     )
     krs = models.CharField(
         max_length=1000,
-        help_text="Kreis"
+        help_text="Kreis",
+        null=True,
+        blank=True,
     )
 
     def __str__(self):
-        return f"{self.krs} | {self.vg} | {self.gmnd}"
+        return f"{self.gmrkng} | {self.gmnd} | {self.krs}"

konova/settings.py

@@ -80,3 +80,8 @@ LANIS_ZOOM_LUT = {
     1000: 30,
     500: 31,
 }
+
+# Parcel WFS settings
+PARCEL_WFS_BASE_URL = "https://www.geoportal.rlp.de/registry/wfs/519"
+PARCEL_WFS_USER = "ksp"
+PARCEL_WFS_PW = "CHANGE_ME"

konova/tests/test_geometries.py

@@ -9,7 +9,7 @@ from django.contrib.gis.db.models.functions import Translate
 
 from konova.models import Geometry, GeometryConflict
 from konova.tests.test_views import BaseTestCase
-from konova.utils.wfs.spatial import SpatialWFSFetcher
+from konova.utils.wfs.spatial import ParcelWFSFetcher
 
 
 class GeometryTestCase(BaseTestCase):
@@ -24,6 +24,22 @@ class GeometryTestCase(BaseTestCase):
             geom=geom,
         )
 
+    def test_geometry_parcel_caluclation(self):
+        """ Tests whether newly created geometries already have parcels calculated during save
+
+        Returns:
+
+        """
+        has_parcels = self.geom_1.parcels.all().exists()
+        self.assertFalse(has_parcels, msg=f"{self.geom_1.id} has parcels but should not!")
+        self.geom_1.update_parcels()
+        self.geom_1.refresh_from_db()
+        parcels = self.geom_1.parcels.all()
+        has_parcels = parcels.exists()
+        parcel_districts = parcels.values_list("district", flat=True)
+        self.assertTrue(has_parcels, msg=f"{self.geom_1.id} has no parcels but should!")
+        self.assertEqual(parcels.count(), len(parcel_districts), msg=f"Not every parcel has exactly one district!")
+
     def test_geometry_conflict(self):
         """ Tests whether a geometry conflict will be present in case of identical/overlaying geometries and
         if the conflict will be resolved if one geometry is edited.
@@ -47,19 +63,17 @@ class GeometryTestCase(BaseTestCase):
         self.assertEqual(0, num_conflict)
 
     def test_wfs_fetch(self):
-        """ Tests the fetching functionality of SpatialWFSFetcher
+        """ Tests the fetching functionality of ParcelWFSFetcher
 
-        +++ Test relies on the availability of the RLP Gemarkung WFS +++
+        +++ Test relies on the availability of the RLP Flurstück WFS +++
 
         Returns:
 
         """
-        fetcher = SpatialWFSFetcher(
+        fetcher = ParcelWFSFetcher(
-            base_url="http://geo5.service24.rlp.de/wfs/verwaltungsgrenzen_rp.fcgi",
+            geometry_id=self.geom_1.id,
-            version="1.1.0",
-            geometry=self.geom_1
         )
         features = fetcher.get_features(
-            "vermkv:fluren_rlp"
+            "ave:Flurstueck",
         )
         self.assertNotEqual(0, len(features), msg="Spatial wfs get feature did not work!")

konova/tests/test_views.py

@@ -214,7 +214,7 @@ class BaseTestCase(TestCase):
         Returns:
 
         """
-        polygon = Polygon.from_bbox((7.157593, 49.882247, 7.816772, 50.266521))
+        polygon = Polygon.from_bbox((7.592449, 50.359385, 7.593382, 50.359874))
         polygon.srid = 4326
         polygon = polygon.transform(3857, clone=True)
         return MultiPolygon(polygon, srid=3857)  # 3857 is the default srid used for MultiPolygonField in the form

konova/utils/wfs/spatial.py

@@ -10,83 +10,167 @@ from abc import abstractmethod
 import requests
 import xmltodict
 from django.contrib.gis.db.models.functions import AsGML, Transform
-from django.contrib.gis.geos import MultiPolygon
+from requests.auth import HTTPDigestAuth
-from owslib.wfs import WebFeatureService
 
-from konova.models import Geometry
+from konova.settings import DEFAULT_SRID_RLP, PARCEL_WFS_USER, PARCEL_WFS_PW
-from konova.settings import DEFAULT_SRID_RLP
 
 
-class BaseWFSFetcher:
+class AbstractWFSFetcher:
+    """ Base class for fetching WFS data
+
+    """
     # base_url represents not the capabilities url but the parameter-free base url
-    base_url = ""
+    base_url = None
-    version = ""
+    version = None
     wfs = None
+    auth_user = None
+    auth_pw = None
+    auth_digest_obj = None
 
-    def __init__(self, base_url: str, version: str = "1.1.0", *args, **kwargs):
+    class Meta:
+        abstract = True
+
+    def __init__(self, base_url: str, version: str = "1.1.0", auth_user: str = None, auth_pw: str = None, *args, **kwargs):
         self.base_url = base_url
         self.version = version
-        self.wfs = WebFeatureService(
+        self.auth_pw = auth_pw
-            url=base_url,
+        self.auth_user = auth_user
-            version=version,
+
+        self._create_auth_obj()
+
+    def _create_auth_obj(self):
+        if self.auth_pw is not None and self.auth_user is not None:
+            self.auth_digest_obj = HTTPDigestAuth(
+                self.auth_user,
+                self.auth_pw
             )
 
     @abstractmethod
-    def get_features(self, feature_identifier: str, filter: str):
+    def get_features(self, feature_identifier: str, filter_str: str):
         raise NotImplementedError
 
 
-class SpatialWFSFetcher(BaseWFSFetcher):
+class ParcelWFSFetcher(AbstractWFSFetcher):
-    """ Fetches features from a parcel WFS
+    """ Fetches features from a special parcel WFS
 
     """
-    geometry = None
+    geometry_id = None
-    geometry_property_name = ""
+    geometry_property_name = None
+    count = 100
 
-    def __init__(self, geometry: MultiPolygon, geometry_property_name: str = "msGeometry", *args, **kwargs):
+    def __init__(self, geometry_id: str, geometry_property_name: str = "msGeometry", *args, **kwargs):
-        super().__init__(*args, **kwargs)
+        super().__init__(
-        self.geometry = geometry
+            version="2.0.0",
+            base_url="https://www.geoportal.rlp.de/registry/wfs/519",
+            auth_user=PARCEL_WFS_USER,
+            auth_pw=PARCEL_WFS_PW,
+            *args,
+            **kwargs
+        )
+        self.geometry_id = geometry_id
         self.geometry_property_name = geometry_property_name
 
-    def _create_geometry_filter(self, geometry_operation: str, filter_srid: str = None):
+    def _create_spatial_filter(self,
-        """ Creates an
+                               geometry_operation: str,
+                               filter_srid: str = None):
+        """ Creates a xml spatial filter according to the WFS filter specification
 
         Args:
-            geometry_operation ():
+            geometry_operation (str): One of the WFS supported spatial filter operations (according to capabilities)
-            filter_srid ():
+            filter_srid (str): Used to transform the geometry into the spatial reference system identified by this srid
 
         Returns:
-
+            spatial_filter (str): The spatial filter element
         """
+        from konova.models import Geometry
         if filter_srid is None:
             filter_srid = DEFAULT_SRID_RLP
         geom_gml = Geometry.objects.filter(
-            id=self.geometry.id
+            id=self.geometry_id
         ).annotate(
             transformed=Transform(srid=filter_srid, expression="geom")
         ).annotate(
             gml=AsGML('transformed')
         ).first().gml
-        _filter = f"<Filter><{geometry_operation}><PropertyName>{self.geometry_property_name}</PropertyName>{geom_gml}</{geometry_operation}></Filter>"
+        spatial_filter = f"<Filter><{geometry_operation}><PropertyName>{self.geometry_property_name}</PropertyName>{geom_gml}</{geometry_operation}></Filter>"
+        return spatial_filter
+
+    def _create_post_data(self,
+                          geometry_operation: str,
+                          filter_srid: str = None,
+                          typenames: str = None,
+                          start_index: int = 0,
+                          ):
+        """ Creates a POST body content for fetching features
+
+        Args:
+            geometry_operation (str): One of the WFS supported spatial filter operations (according to capabilities)
+            filter_srid (str): Used to transform the geometry into the spatial reference system identified by this srid
+
+        Returns:
+            _filter (str): A proper xml WFS filter
+        """
+        start_index = str(start_index)
+        spatial_filter = self._create_spatial_filter(
+            geometry_operation,
+            filter_srid
+        )
+        _filter = f'<wfs:GetFeature service="WFS" version="{self.version}" xmlns:wfs="http://www.opengis.net/wfs/2.0" xmlns:fes="http://www.opengis.net/fes/2.0" xmlns:myns="http://www.someserver.com/myns" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.opengis.net/wfs/2.0 http://schemas.opengis.net/wfs/2.0.0/wfs.xsd" count="{self.count}" startindex="{start_index}"><wfs:Query typeNames="{typenames}">{spatial_filter}</wfs:Query></wfs:GetFeature>'
         return _filter
 
-    def get_features(self, typenames: str, geometry_operation: str = "Intersects", filter_srid: str = None, filter: str = None):
+    def get_features(self,
-        if filter is None:
+                     typenames: str,
-            filter = self._create_geometry_filter(geometry_operation, filter_srid)
+                     spatial_operator: str = "Intersects",
-        response = requests.post(
+                     filter_srid: str = None,
-            url=f"{self.base_url}?request=GetFeature&service=WFS&version={self.version}&typenames={typenames}&count=10",
+                     start_index: int = 0,
-            data={
+                     ):
-                "filter": filter,
+        """ Fetches features from the WFS using POST
-            }
+
+        POST is required since GET has a character limit around 4000. Having a larger filter would result in errors,
+        which do not occur in case of POST.
+
+        Args:
+            typenames (str): References to parameter 'typenames' in a WFS GetFeature request
+            spatial_operator (str): Defines the spatial operation for filtering
+            filter_srid (str): Defines the spatial reference system, the geometry shall be transformed into for filtering
+            start_index (str): References to parameter 'startindex' in a
+
+        Returns:
+            features (list): A list of returned features
+        """
+        features = []
+        while start_index is not None:
+            post_body = self._create_post_data(
+                spatial_operator,
+                filter_srid,
+                typenames,
+                start_index
             )
+            response = requests.post(
+                url=self.base_url,
+                data=post_body,
+                auth=self.auth_digest_obj
+            )
+
             content = response.content.decode("utf-8")
             content = xmltodict.parse(content)
-        features = content.get(
+            collection = content.get(
                 "wfs:FeatureCollection",
                 {},
-        ).get(
+            )
-            "gml:featureMember",
+
+            members = collection.get(
+                "wfs:member",
                 [],
             )
-        return features
+            if len(members) > 1:
+                features += members
+            else:
+                features += [members]
 
+            if collection.get("@next", None) is not None:
+                start_index += self.count
+            else:
+                start_index = None
+
+        return features