From c9c918bae12f5b2753cc54ea7eed5d077b33a57c Mon Sep 17 00:00:00 2001 From: mpeltriaux Date: Wed, 27 Apr 2022 12:12:56 +0200 Subject: [PATCH] #156 Parcel WFS as geojson * refactors fetching of parcels via wfs from xml to json for easier and faster processing --- konova/models/geometry.py | 26 +++++++++++------- konova/utils/wfs/spatial.py | 54 ++++++++++++++++--------------------- 2 files changed, 39 insertions(+), 41 deletions(-) diff --git a/konova/models/geometry.py b/konova/models/geometry.py index a71a2af..283f850 100644 --- a/konova/models/geometry.py +++ b/konova/models/geometry.py @@ -113,32 +113,38 @@ class Geometry(BaseResource): _now = timezone.now() underlying_parcels = [] for result in fetched_parcels: - fetched_parcel = result[typename] + parcel_properties = result["properties"] # There could be parcels which include the word 'Flur', # which needs to be deleted and just keep the numerical values ## THIS CAN BE REMOVED IN THE FUTURE, WHEN 'Flur' WON'T OCCUR ANYMORE! - flr_val = fetched_parcel["ave:flur"].replace("Flur ", "") + flr_val = parcel_properties["flur"].replace("Flur ", "") district = District.objects.get_or_create( - key=fetched_parcel["ave:kreisschl"], - name=fetched_parcel["ave:kreis"], + key=parcel_properties["kreisschl"], + name=parcel_properties["kreis"], )[0] municipal = Municipal.objects.get_or_create( - key=fetched_parcel["ave:gmdschl"], - name=fetched_parcel["ave:gemeinde"], + key=parcel_properties["gmdschl"], + name=parcel_properties["gemeinde"], district=district, )[0] parcel_group = ParcelGroup.objects.get_or_create( - key=fetched_parcel["ave:gemaschl"], - name=fetched_parcel["ave:gemarkung"], + key=parcel_properties["gemaschl"], + name=parcel_properties["gemarkung"], municipal=municipal, )[0] + flrstck_nnr = parcel_properties['flstnrnen'] + if not flrstck_nnr: + flrstck_nnr = None + flrstck_zhlr = parcel_properties['flstnrzae'] + if not flrstck_zhlr: + flrstck_zhlr = None parcel_obj = Parcel.objects.get_or_create( district=district, municipal=municipal, parcel_group=parcel_group, flr=flr_val, - flrstck_nnr=fetched_parcel['ave:flstnrnen'], - flrstck_zhlr=fetched_parcel['ave:flstnrzae'], + flrstck_nnr=flrstck_nnr, + flrstck_zhlr=flrstck_zhlr, )[0] parcel_obj.district = district parcel_obj.updated_on = _now diff --git a/konova/utils/wfs/spatial.py b/konova/utils/wfs/spatial.py index 8a7bf36..2f1452d 100644 --- a/konova/utils/wfs/spatial.py +++ b/konova/utils/wfs/spatial.py @@ -5,11 +5,12 @@ Contact: michel.peltriaux@sgdnord.rlp.de Created on: 17.12.21 """ +import json from abc import abstractmethod +from json import JSONDecodeError from time import sleep import requests -import xmltodict from django.contrib.gis.db.models.functions import AsGML, Transform from requests.auth import HTTPDigestAuth @@ -115,7 +116,7 @@ class ParcelWFSFetcher(AbstractWFSFetcher): geometry_operation, filter_srid ) - _filter = f'{spatial_filter}' + _filter = f'{spatial_filter}' return _filter def get_features(self, @@ -139,7 +140,7 @@ class ParcelWFSFetcher(AbstractWFSFetcher): Returns: features (list): A list of returned features """ - features = [] + found_features = [] while start_index is not None: post_body = self._create_post_data( spatial_operator, @@ -155,19 +156,11 @@ class ParcelWFSFetcher(AbstractWFSFetcher): ) content = response.content.decode("utf-8") - content = xmltodict.parse(content) - collection = content.get( - "wfs:FeatureCollection", - {}, - ) - - # Check if collection is an exception and does not contain the requested data - if len(collection) == 0: - exception = content.get( - "ows:ExceptionReport", - {} - ) - if len(exception) > 0 and rerun_on_exception: + try: + # Check if collection is an exception and does not contain the requested data + content = json.loads(content) + except JSONDecodeError as e: + if rerun_on_exception: # Wait a second before another try sleep(1) self.get_features( @@ -177,22 +170,21 @@ class ParcelWFSFetcher(AbstractWFSFetcher): start_index, rerun_on_exception=False ) - - members = collection.get( - "wfs:member", - None, - ) - if members is not None: - if len(members) > 1: - # extend feature list with found list of new feature members - features += members else: - # convert single found feature member into list and extent feature list - features += [members] + e.msg += content + raise e + fetched_features = content.get( + "features", + {}, + ) - if collection.get("@next", None) is not None: - start_index += self.count - else: + found_features += fetched_features + + if len(fetched_features) < self.count: + # The response was not 'full', so we got everything to fetch start_index = None + else: + # If a 'full' response returned, there might be more to fetch. Increase the start_index! + start_index += self.count - return features + return found_features