#156 Parcel WFS as geojson

* refactors fetching of parcels via wfs from xml to json for easier and faster processing

konova/utils/wfs/spatial.py

@@ -5,11 +5,12 @@ Contact: michel.peltriaux@sgdnord.rlp.de
 Created on: 17.12.21
 
 """
+import json
 from abc import abstractmethod
+from json import JSONDecodeError
 from time import sleep
 
 import requests
-import xmltodict
 from django.contrib.gis.db.models.functions import AsGML, Transform
 from requests.auth import HTTPDigestAuth
 
@@ -115,7 +116,7 @@ class ParcelWFSFetcher(AbstractWFSFetcher):
             geometry_operation,
             filter_srid
         )
-        _filter = f'<wfs:GetFeature service="WFS" version="{self.version}" xmlns:wfs="http://www.opengis.net/wfs/2.0" xmlns:fes="http://www.opengis.net/fes/2.0" xmlns:myns="http://www.someserver.com/myns" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.opengis.net/wfs/2.0 http://schemas.opengis.net/wfs/2.0.0/wfs.xsd" count="{self.count}" startindex="{start_index}"><wfs:Query typeNames="{typenames}">{spatial_filter}</wfs:Query></wfs:GetFeature>'
+        _filter = f'<wfs:GetFeature service="WFS" version="{self.version}" xmlns:wfs="http://www.opengis.net/wfs/2.0" xmlns:fes="http://www.opengis.net/fes/2.0" xmlns:myns="http://www.someserver.com/myns" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.opengis.net/wfs/2.0 http://schemas.opengis.net/wfs/2.0.0/wfs.xsd" count="{self.count}" startindex="{start_index}" outputFormat="application/json; subtype=geojson"><wfs:Query typeNames="{typenames}">{spatial_filter}</wfs:Query></wfs:GetFeature>'
         return _filter
 
     def get_features(self,
@@ -139,7 +140,7 @@ class ParcelWFSFetcher(AbstractWFSFetcher):
         Returns:
             features (list): A list of returned features
         """
-        features = []
+        found_features = []
         while start_index is not None:
             post_body = self._create_post_data(
                 spatial_operator,
@@ -155,19 +156,11 @@ class ParcelWFSFetcher(AbstractWFSFetcher):
             )
 
             content = response.content.decode("utf-8")
-            content = xmltodict.parse(content)
-            collection = content.get(
-                "wfs:FeatureCollection",
-                {},
-            )
-
-            # Check if collection is an exception and does not contain the requested data
-            if len(collection) == 0:
-                exception = content.get(
-                    "ows:ExceptionReport",
-                    {}
-                )
-                if len(exception) > 0 and rerun_on_exception:
+            try:
+                # Check if collection is an exception and does not contain the requested data
+                content = json.loads(content)
+            except JSONDecodeError as e:
+                if rerun_on_exception:
                     # Wait a second before another try
                     sleep(1)
                     self.get_features(
@@ -177,22 +170,21 @@ class ParcelWFSFetcher(AbstractWFSFetcher):
                         start_index,
                         rerun_on_exception=False
                     )
-
-            members = collection.get(
-                "wfs:member",
-                None,
-            )
-            if members is not None:
-                if len(members) > 1:
-                    # extend feature list with found list of new feature members
-                    features += members
                 else:
-                    # convert single found feature member into list and extent feature list
-                    features += [members]
+                    e.msg += content
+                    raise e
+            fetched_features = content.get(
+                "features",
+                {},
+            )
 
-            if collection.get("@next", None) is not None:
-                start_index += self.count
-            else:
+            found_features += fetched_features
+
+            if len(fetched_features) < self.count:
+                # The response was not 'full', so we got everything to fetch
                 start_index = None
+            else:
+                # If a 'full' response returned, there might be more to fetch. Increase the start_index!
+                start_index += self.count
 
-        return features
+        return found_features