From 62030c4dcc6a7d12f14f820dcc4122518bfb1c55 Mon Sep 17 00:00:00 2001 From: mpeltriaux Date: Tue, 4 Jan 2022 13:03:21 +0100 Subject: [PATCH] #49 Parcels and Districts * refactors WFS fetching to proper POST handling * adds authentication support to WFS handling * reduces dummy geometry for tests to a small area to reduce test network traffic overhead --- konova/settings.py | 5 ++ konova/tests/test_geometries.py | 17 ++-- konova/tests/test_views.py | 2 +- konova/utils/wfs/spatial.py | 148 +++++++++++++++++++++++--------- 4 files changed, 124 insertions(+), 48 deletions(-) diff --git a/konova/settings.py b/konova/settings.py index c80c54fc..35b3f722 100644 --- a/konova/settings.py +++ b/konova/settings.py @@ -80,3 +80,8 @@ LANIS_ZOOM_LUT = { 1000: 30, 500: 31, } + +# Parcel WFS settings +PARCEL_WFS_BASE_URL = "https://www.geoportal.rlp.de/registry/wfs/519" +PARCEL_WFS_USER = "ksp" +PARCEL_WFS_PW = "CHANGE_ME" \ No newline at end of file diff --git a/konova/tests/test_geometries.py b/konova/tests/test_geometries.py index cf06e79c..da5e82d0 100644 --- a/konova/tests/test_geometries.py +++ b/konova/tests/test_geometries.py @@ -8,8 +8,9 @@ Created on: 15.12.21 from django.contrib.gis.db.models.functions import Translate from konova.models import Geometry, GeometryConflict +from konova.settings import PARCEL_WFS_USER, PARCEL_WFS_PW from konova.tests.test_views import BaseTestCase -from konova.utils.wfs.spatial import SpatialWFSFetcher +from konova.utils.wfs.spatial import ParcelWFSFetcher class GeometryTestCase(BaseTestCase): @@ -47,19 +48,21 @@ class GeometryTestCase(BaseTestCase): self.assertEqual(0, num_conflict) def test_wfs_fetch(self): - """ Tests the fetching functionality of SpatialWFSFetcher + """ Tests the fetching functionality of ParcelWFSFetcher +++ Test relies on the availability of the RLP Gemarkung WFS +++ Returns: """ - fetcher = SpatialWFSFetcher( - base_url="http://geo5.service24.rlp.de/wfs/verwaltungsgrenzen_rp.fcgi", - version="1.1.0", - geometry=self.geom_1 + fetcher = ParcelWFSFetcher( + base_url="https://www.geoportal.rlp.de/registry/wfs/519", + version="2.0.0", + geometry=self.geom_1, + auth_user=PARCEL_WFS_USER, + auth_pw=PARCEL_WFS_PW ) features = fetcher.get_features( - "vermkv:fluren_rlp" + "ave:Flurstueck", ) self.assertNotEqual(0, len(features), msg="Spatial wfs get feature did not work!") diff --git a/konova/tests/test_views.py b/konova/tests/test_views.py index ae5d4524..72cbbea7 100644 --- a/konova/tests/test_views.py +++ b/konova/tests/test_views.py @@ -214,7 +214,7 @@ class BaseTestCase(TestCase): Returns: """ - polygon = Polygon.from_bbox((7.157593, 49.882247, 7.816772, 50.266521)) + polygon = Polygon.from_bbox((7.5971391556, 50.3600032354, 7.5993975756, 50.3612420894)) polygon.srid = 4326 polygon = polygon.transform(3857, clone=True) return MultiPolygon(polygon, srid=3857) # 3857 is the default srid used for MultiPolygonField in the form diff --git a/konova/utils/wfs/spatial.py b/konova/utils/wfs/spatial.py index c763d530..c53a6b7f 100644 --- a/konova/utils/wfs/spatial.py +++ b/konova/utils/wfs/spatial.py @@ -11,52 +11,68 @@ import requests import xmltodict from django.contrib.gis.db.models.functions import AsGML, Transform from django.contrib.gis.geos import MultiPolygon -from owslib.wfs import WebFeatureService +from requests.auth import HTTPDigestAuth from konova.models import Geometry from konova.settings import DEFAULT_SRID_RLP -class BaseWFSFetcher: - # base_url represents not the capabilities url but the parameter-free base url - base_url = "" - version = "" - wfs = None +class AbstractWFSFetcher: + """ Base class for fetching WFS data - def __init__(self, base_url: str, version: str = "1.1.0", *args, **kwargs): + """ + # base_url represents not the capabilities url but the parameter-free base url + base_url = None + version = None + wfs = None + auth_user = None + auth_pw = None + auth_digest_obj = None + + class Meta: + abstract = True + + def __init__(self, base_url: str, version: str = "1.1.0", auth_user: str = None, auth_pw: str = None, *args, **kwargs): self.base_url = base_url self.version = version - self.wfs = WebFeatureService( - url=base_url, - version=version, - ) + self.auth_pw = auth_pw + self.auth_user = auth_user + + if self.auth_pw is not None and self.auth_user is not None: + self.auth_digest_obj = HTTPDigestAuth( + self.auth_user, + self.auth_pw + ) @abstractmethod - def get_features(self, feature_identifier: str, filter: str): + def get_features(self, feature_identifier: str, filter_str: str): raise NotImplementedError -class SpatialWFSFetcher(BaseWFSFetcher): - """ Fetches features from a parcel WFS +class ParcelWFSFetcher(AbstractWFSFetcher): + """ Fetches features from a special parcel WFS """ geometry = None - geometry_property_name = "" + geometry_property_name = None + count = 100 def __init__(self, geometry: MultiPolygon, geometry_property_name: str = "msGeometry", *args, **kwargs): super().__init__(*args, **kwargs) self.geometry = geometry self.geometry_property_name = geometry_property_name - def _create_geometry_filter(self, geometry_operation: str, filter_srid: str = None): - """ Creates an + def _create_spatial_filter(self, + geometry_operation: str, + filter_srid: str = None): + """ Creates a xml spatial filter according to the WFS filter specification Args: - geometry_operation (): - filter_srid (): + geometry_operation (str): One of the WFS supported spatial filter operations (according to capabilities) + filter_srid (str): Used to transform the geometry into the spatial reference system identified by this srid Returns: - + spatial_filter (str): The spatial filter element """ if filter_srid is None: filter_srid = DEFAULT_SRID_RLP @@ -67,26 +83,78 @@ class SpatialWFSFetcher(BaseWFSFetcher): ).annotate( gml=AsGML('transformed') ).first().gml - _filter = f"<{geometry_operation}>{self.geometry_property_name}{geom_gml}" + spatial_filter = f"<{geometry_operation}>{self.geometry_property_name}{geom_gml}" + return spatial_filter + + def _create_post_data(self, + geometry_operation: str, + filter_srid: str = None, + typenames: str = None, + start_index: int = 0, + ): + """ Creates a POST body content for fetching features + + Args: + geometry_operation (str): One of the WFS supported spatial filter operations (according to capabilities) + filter_srid (str): Used to transform the geometry into the spatial reference system identified by this srid + + Returns: + _filter (str): A proper xml WFS filter + """ + start_index = str(start_index) + spatial_filter = self._create_spatial_filter( + geometry_operation, + filter_srid + ) + _filter = f'{spatial_filter}' return _filter - def get_features(self, typenames: str, geometry_operation: str = "Intersects", filter_srid: str = None, filter: str = None): - if filter is None: - filter = self._create_geometry_filter(geometry_operation, filter_srid) - response = requests.post( - url=f"{self.base_url}?request=GetFeature&service=WFS&version={self.version}&typenames={typenames}&count=10", - data={ - "filter": filter, - } - ) - content = response.content.decode("utf-8") - content = xmltodict.parse(content) - features = content.get( - "wfs:FeatureCollection", - {}, - ).get( - "gml:featureMember", - [], - ) - return features + def get_features(self, + typenames: str, + spatial_operator: str = "Intersects", + filter_srid: str = None, + start_index: int = 0, + ): + """ Fetches features from the WFS using POST + POST is required since GET has a character limit around 4000. Having a larger filter would result in errors, + which do not occur in case of POST. + + Args: + typenames (str): References to parameter 'typenames' in a WFS GetFeature request + spatial_operator (str): Defines the spatial operation for filtering + filter_srid (str): Defines the spatial reference system, the geometry shall be transformed into for filtering + + Returns: + features (list): A list of returned features + """ + features = [] + while start_index is not None: + post_body = self._create_post_data( + spatial_operator, + filter_srid, + typenames, + start_index + ) + response = requests.post( + url=self.base_url, + data=post_body, + auth=self.auth_digest_obj + ) + + content = response.content.decode("utf-8") + content = xmltodict.parse(content) + collection = content.get( + "wfs:FeatureCollection", + {}, + ) + features += collection.get( + "wfs:member", + [], + ) + if collection.get("@next", None) is not None: + start_index += self.count + else: + start_index = None + + return features