#156 Parcel WFS as geojson

* refactors fetching of parcels via wfs from xml to json for easier and faster processing

konova/models/geometry.py

@@ -113,32 +113,38 @@ class Geometry(BaseResource):
         _now = timezone.now()
         underlying_parcels = []
         for result in fetched_parcels:
-            fetched_parcel = result[typename]
+            parcel_properties = result["properties"]
             # There could be parcels which include the word 'Flur',
             # which needs to be deleted and just keep the numerical values
             ## THIS CAN BE REMOVED IN THE FUTURE, WHEN 'Flur' WON'T OCCUR ANYMORE!
-            flr_val = fetched_parcel["ave:flur"].replace("Flur ", "")
+            flr_val = parcel_properties["flur"].replace("Flur ", "")
             district = District.objects.get_or_create(
-                key=fetched_parcel["ave:kreisschl"],
+                key=parcel_properties["kreisschl"],
-                name=fetched_parcel["ave:kreis"],
+                name=parcel_properties["kreis"],
             )[0]
             municipal = Municipal.objects.get_or_create(
-                key=fetched_parcel["ave:gmdschl"],
+                key=parcel_properties["gmdschl"],
-                name=fetched_parcel["ave:gemeinde"],
+                name=parcel_properties["gemeinde"],
                 district=district,
             )[0]
             parcel_group = ParcelGroup.objects.get_or_create(
-                key=fetched_parcel["ave:gemaschl"],
+                key=parcel_properties["gemaschl"],
-                name=fetched_parcel["ave:gemarkung"],
+                name=parcel_properties["gemarkung"],
                 municipal=municipal,
             )[0]
+            flrstck_nnr = parcel_properties['flstnrnen']
+            if not flrstck_nnr:
+                flrstck_nnr = None
+            flrstck_zhlr = parcel_properties['flstnrzae']
+            if not flrstck_zhlr:
+                flrstck_zhlr = None
             parcel_obj = Parcel.objects.get_or_create(
                 district=district,
                 municipal=municipal,
                 parcel_group=parcel_group,
                 flr=flr_val,
-                flrstck_nnr=fetched_parcel['ave:flstnrnen'],
+                flrstck_nnr=flrstck_nnr,
-                flrstck_zhlr=fetched_parcel['ave:flstnrzae'],
+                flrstck_zhlr=flrstck_zhlr,
             )[0]
             parcel_obj.district = district
             parcel_obj.updated_on = _now

konova/utils/wfs/spatial.py

@@ -5,11 +5,12 @@ Contact: michel.peltriaux@sgdnord.rlp.de
 Created on: 17.12.21
 
 """
+import json
 from abc import abstractmethod
+from json import JSONDecodeError
 from time import sleep
 
 import requests
-import xmltodict
 from django.contrib.gis.db.models.functions import AsGML, Transform
 from requests.auth import HTTPDigestAuth
 
@@ -115,7 +116,7 @@ class ParcelWFSFetcher(AbstractWFSFetcher):
             geometry_operation,
             filter_srid
         )
-        _filter = f'<wfs:GetFeature service="WFS" version="{self.version}" xmlns:wfs="http://www.opengis.net/wfs/2.0" xmlns:fes="http://www.opengis.net/fes/2.0" xmlns:myns="http://www.someserver.com/myns" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.opengis.net/wfs/2.0 http://schemas.opengis.net/wfs/2.0.0/wfs.xsd" count="{self.count}" startindex="{start_index}"><wfs:Query typeNames="{typenames}">{spatial_filter}</wfs:Query></wfs:GetFeature>'
+        _filter = f'<wfs:GetFeature service="WFS" version="{self.version}" xmlns:wfs="http://www.opengis.net/wfs/2.0" xmlns:fes="http://www.opengis.net/fes/2.0" xmlns:myns="http://www.someserver.com/myns" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.opengis.net/wfs/2.0 http://schemas.opengis.net/wfs/2.0.0/wfs.xsd" count="{self.count}" startindex="{start_index}" outputFormat="application/json; subtype=geojson"><wfs:Query typeNames="{typenames}">{spatial_filter}</wfs:Query></wfs:GetFeature>'
         return _filter
 
     def get_features(self,
@@ -139,7 +140,7 @@ class ParcelWFSFetcher(AbstractWFSFetcher):
         Returns:
             features (list): A list of returned features
         """
-        features = []
+        found_features = []
         while start_index is not None:
             post_body = self._create_post_data(
                 spatial_operator,
@@ -155,19 +156,11 @@ class ParcelWFSFetcher(AbstractWFSFetcher):
             )
 
             content = response.content.decode("utf-8")
-            content = xmltodict.parse(content)
+            try:
-            collection = content.get(
-                "wfs:FeatureCollection",
-                {},
-            )
-
                 # Check if collection is an exception and does not contain the requested data
-            if len(collection) == 0:
+                content = json.loads(content)
-                exception = content.get(
+            except JSONDecodeError as e:
-                    "ows:ExceptionReport",
+                if rerun_on_exception:
-                    {}
-                )
-                if len(exception) > 0 and rerun_on_exception:
                     # Wait a second before another try
                     sleep(1)
                     self.get_features(
@@ -177,22 +170,21 @@ class ParcelWFSFetcher(AbstractWFSFetcher):
                         start_index,
                         rerun_on_exception=False
                     )
-
+                else:
-            members = collection.get(
+                    e.msg += content
-                "wfs:member",
+                    raise e
-                None,
+            fetched_features = content.get(
+                "features",
+                {},
             )
-            if members is not None:
-                if len(members) > 1:
-                    # extend feature list with found list of new feature members
-                    features += members
-                else:
-                    # convert single found feature member into list and extent feature list
-                    features += [members]
 
-            if collection.get("@next", None) is not None:
+            found_features += fetched_features
-                start_index += self.count
+
-            else:
+            if len(fetched_features) < self.count:
+                # The response was not 'full', so we got everything to fetch
                 start_index = None
+            else:
+                # If a 'full' response returned, there might be more to fetch. Increase the start_index!
+                start_index += self.count
 
-        return features
+        return found_features