#49 Parcels and Districts

* refactors WFS fetching to proper POST handling
* adds authentication support to WFS handling
* reduces dummy geometry for tests to a small area to reduce test network traffic overhead

konova/settings.py

@@ -80,3 +80,8 @@ LANIS_ZOOM_LUT = {
     1000: 30,
     500: 31,
 }
+
+# Parcel WFS settings
+PARCEL_WFS_BASE_URL = "https://www.geoportal.rlp.de/registry/wfs/519"
+PARCEL_WFS_USER = "ksp"
+PARCEL_WFS_PW = "CHANGE_ME"

konova/tests/test_geometries.py

@@ -8,8 +8,9 @@ Created on: 15.12.21
 from django.contrib.gis.db.models.functions import Translate
 
 from konova.models import Geometry, GeometryConflict
+from konova.settings import PARCEL_WFS_USER, PARCEL_WFS_PW
 from konova.tests.test_views import BaseTestCase
-from konova.utils.wfs.spatial import SpatialWFSFetcher
+from konova.utils.wfs.spatial import ParcelWFSFetcher
 
 
 class GeometryTestCase(BaseTestCase):
@@ -47,19 +48,21 @@ class GeometryTestCase(BaseTestCase):
         self.assertEqual(0, num_conflict)
 
     def test_wfs_fetch(self):
-        """ Tests the fetching functionality of SpatialWFSFetcher
+        """ Tests the fetching functionality of ParcelWFSFetcher
 
         +++ Test relies on the availability of the RLP Gemarkung WFS +++
 
         Returns:
 
         """
-        fetcher = SpatialWFSFetcher(
+        fetcher = ParcelWFSFetcher(
-            base_url="http://geo5.service24.rlp.de/wfs/verwaltungsgrenzen_rp.fcgi",
+            base_url="https://www.geoportal.rlp.de/registry/wfs/519",
-            version="1.1.0",
+            version="2.0.0",
-            geometry=self.geom_1
+            geometry=self.geom_1,
+            auth_user=PARCEL_WFS_USER,
+            auth_pw=PARCEL_WFS_PW
         )
         features = fetcher.get_features(
-            "vermkv:fluren_rlp"
+            "ave:Flurstueck",
         )
         self.assertNotEqual(0, len(features), msg="Spatial wfs get feature did not work!")

konova/tests/test_views.py

@@ -214,7 +214,7 @@ class BaseTestCase(TestCase):
         Returns:
 
         """
-        polygon = Polygon.from_bbox((7.157593, 49.882247, 7.816772, 50.266521))
+        polygon = Polygon.from_bbox((7.5971391556, 50.3600032354, 7.5993975756, 50.3612420894))
         polygon.srid = 4326
         polygon = polygon.transform(3857, clone=True)
         return MultiPolygon(polygon, srid=3857)  # 3857 is the default srid used for MultiPolygonField in the form

konova/utils/wfs/spatial.py

@@ -11,52 +11,68 @@ import requests
 import xmltodict
 from django.contrib.gis.db.models.functions import AsGML, Transform
 from django.contrib.gis.geos import MultiPolygon
-from owslib.wfs import WebFeatureService
+from requests.auth import HTTPDigestAuth
 
 from konova.models import Geometry
 from konova.settings import DEFAULT_SRID_RLP
 
 
-class BaseWFSFetcher:
+class AbstractWFSFetcher:
-    # base_url represents not the capabilities url but the parameter-free base url
+    """ Base class for fetching WFS data
-    base_url = ""
-    version = ""
-    wfs = None
 
-    def __init__(self, base_url: str, version: str = "1.1.0", *args, **kwargs):
+    """
+    # base_url represents not the capabilities url but the parameter-free base url
+    base_url = None
+    version = None
+    wfs = None
+    auth_user = None
+    auth_pw = None
+    auth_digest_obj = None
+
+    class Meta:
+        abstract = True
+
+    def __init__(self, base_url: str, version: str = "1.1.0", auth_user: str = None, auth_pw: str = None, *args, **kwargs):
         self.base_url = base_url
         self.version = version
-        self.wfs = WebFeatureService(
+        self.auth_pw = auth_pw
-            url=base_url,
+        self.auth_user = auth_user
-            version=version,
+
-        )
+        if self.auth_pw is not None and self.auth_user is not None:
+            self.auth_digest_obj = HTTPDigestAuth(
+                self.auth_user,
+                self.auth_pw
+            )
 
     @abstractmethod
-    def get_features(self, feature_identifier: str, filter: str):
+    def get_features(self, feature_identifier: str, filter_str: str):
         raise NotImplementedError
 
 
-class SpatialWFSFetcher(BaseWFSFetcher):
+class ParcelWFSFetcher(AbstractWFSFetcher):
-    """ Fetches features from a parcel WFS
+    """ Fetches features from a special parcel WFS
 
     """
     geometry = None
-    geometry_property_name = ""
+    geometry_property_name = None
+    count = 100
 
     def __init__(self, geometry: MultiPolygon, geometry_property_name: str = "msGeometry", *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.geometry = geometry
         self.geometry_property_name = geometry_property_name
 
-    def _create_geometry_filter(self, geometry_operation: str, filter_srid: str = None):
+    def _create_spatial_filter(self,
-        """ Creates an
+                               geometry_operation: str,
+                               filter_srid: str = None):
+        """ Creates a xml spatial filter according to the WFS filter specification
 
         Args:
-            geometry_operation ():
+            geometry_operation (str): One of the WFS supported spatial filter operations (according to capabilities)
-            filter_srid ():
+            filter_srid (str): Used to transform the geometry into the spatial reference system identified by this srid
 
         Returns:
-
+            spatial_filter (str): The spatial filter element
         """
         if filter_srid is None:
             filter_srid = DEFAULT_SRID_RLP
@@ -67,26 +83,78 @@ class SpatialWFSFetcher(BaseWFSFetcher):
         ).annotate(
             gml=AsGML('transformed')
         ).first().gml
-        _filter = f"<Filter><{geometry_operation}><PropertyName>{self.geometry_property_name}</PropertyName>{geom_gml}</{geometry_operation}></Filter>"
+        spatial_filter = f"<Filter><{geometry_operation}><PropertyName>{self.geometry_property_name}</PropertyName>{geom_gml}</{geometry_operation}></Filter>"
+        return spatial_filter
+
+    def _create_post_data(self,
+                          geometry_operation: str,
+                          filter_srid: str = None,
+                          typenames: str = None,
+                          start_index: int = 0,
+                          ):
+        """ Creates a POST body content for fetching features
+
+        Args:
+            geometry_operation (str): One of the WFS supported spatial filter operations (according to capabilities)
+            filter_srid (str): Used to transform the geometry into the spatial reference system identified by this srid
+
+        Returns:
+            _filter (str): A proper xml WFS filter
+        """
+        start_index = str(start_index)
+        spatial_filter = self._create_spatial_filter(
+            geometry_operation,
+            filter_srid
+        )
+        _filter = f'<wfs:GetFeature service="WFS" version="{self.version}" xmlns:wfs="http://www.opengis.net/wfs/2.0" xmlns:fes="http://www.opengis.net/fes/2.0" xmlns:myns="http://www.someserver.com/myns" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.opengis.net/wfs/2.0 http://schemas.opengis.net/wfs/2.0.0/wfs.xsd" count="{self.count}" startindex="{start_index}"><wfs:Query typeNames="{typenames}">{spatial_filter}</wfs:Query></wfs:GetFeature>'
         return _filter
 
-    def get_features(self, typenames: str, geometry_operation: str = "Intersects", filter_srid: str = None, filter: str = None):
+    def get_features(self,
-        if filter is None:
+                     typenames: str,
-            filter = self._create_geometry_filter(geometry_operation, filter_srid)
+                     spatial_operator: str = "Intersects",
-        response = requests.post(
+                     filter_srid: str = None,
-            url=f"{self.base_url}?request=GetFeature&service=WFS&version={self.version}&typenames={typenames}&count=10",
+                     start_index: int = 0,
-            data={
+                     ):
-                "filter": filter,
+        """ Fetches features from the WFS using POST
-            }
-        )
-        content = response.content.decode("utf-8")
-        content = xmltodict.parse(content)
-        features = content.get(
-            "wfs:FeatureCollection",
-            {},
-        ).get(
-            "gml:featureMember",
-            [],
-        )
-        return features
 
+        POST is required since GET has a character limit around 4000. Having a larger filter would result in errors,
+        which do not occur in case of POST.
+
+        Args:
+            typenames (str): References to parameter 'typenames' in a WFS GetFeature request
+            spatial_operator (str): Defines the spatial operation for filtering
+            filter_srid (str): Defines the spatial reference system, the geometry shall be transformed into for filtering
+
+        Returns:
+            features (list): A list of returned features
+        """
+        features = []
+        while start_index is not None:
+            post_body = self._create_post_data(
+                spatial_operator,
+                filter_srid,
+                typenames,
+                start_index
+            )
+            response = requests.post(
+                url=self.base_url,
+                data=post_body,
+                auth=self.auth_digest_obj
+            )
+
+            content = response.content.decode("utf-8")
+            content = xmltodict.parse(content)
+            collection = content.get(
+                "wfs:FeatureCollection",
+                {},
+            )
+            features += collection.get(
+                "wfs:member",
+                [],
+            )
+            if collection.get("@next", None) is not None:
+                start_index += self.count
+            else:
+                start_index = None
+
+        return features