IT-Naturschutz/konova
7
0
Fork 0
Code Issues 6 Pull Requests 1 Actions Projects Releases 37 Wiki Activity

Merge pull request 'geom_parcel_improvements' (#384) from geom_parcel_improvements into master

Browse Source 
Reviewed-on: SGD-Nord/konova#384
This commit is contained in:
mpeltriaux 2024-02-16 08:44:38 +01:00
commit 5b157254b4
32 changed files with 592 additions and 1965 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
compensation/forms/compensation.py
View File

@ -213,7 +213,6 @@ class EditCompensationForm(NewCompensationForm):
action = UserActionLogEntry.get_edited_action(user)
# Fetch data from cleaned POST values
identifier = self.cleaned_data.get("identifier", None)
title = self.cleaned_data.get("title", None)
intervention = self.cleaned_data.get("intervention", None)
is_cef = self.cleaned_data.get("is_cef", None)
@ -221,7 +220,6 @@ class EditCompensationForm(NewCompensationForm):
is_pik = self.cleaned_data.get("is_pik", None)
comment = self.cleaned_data.get("comment", None)
self.instance.identifier = identifier
self.instance.title = title
self.instance.intervention = intervention
self.instance.is_cef = is_cef

2
compensation/forms/eco_account.py
View File

@ -192,7 +192,6 @@ class EditEcoAccountForm(NewEcoAccountForm):
def save(self, user: User, geom_form: SimpleGeomForm):
with transaction.atomic():
# Fetch data from cleaned POST values
identifier = self.cleaned_data.get("identifier", None)
title = self.cleaned_data.get("title", None)
registration_date = self.cleaned_data.get("registration_date", None)
handler_type = self.cleaned_data.get("handler_type", None)
@ -219,7 +218,6 @@ class EditEcoAccountForm(NewEcoAccountForm):
self.instance.legal.save()
# Update main oject data
self.instance.identifier = identifier
self.instance.title = title
self.instance.deductable_surface = surface
self.instance.comment = comment

1
compensation/models/compensation.py
View File

@ -315,7 +315,6 @@ class Compensation(AbstractCompensation, CEFMixin, CoherenceMixin, PikMixin):
def get_detail_url_absolute(self):
return BASE_URL + self.get_detail_url()
def save(self, *args, **kwargs):
if self.identifier is None or len(self.identifier) == 0:
# Create new identifier is none was given

14
compensation/tests/compensation/test_workflow.py
View File

@ -125,10 +125,16 @@ class CompensationWorkflowTestCase(BaseWorkflowTestCase):
self.compensation = self.fill_out_compensation(self.compensation)
pre_edit_log_count = self.compensation.log.count()
self.assertTrue(self.compensation.is_shared_with(self.superuser))
old_identifier = self.compensation.identifier
new_title = self.create_dummy_string()
new_identifier = self.create_dummy_string()
new_comment = self.create_dummy_string()
new_geometry = MultiPolygon(srid=4326) # Create an empty geometry
new_geometry = MultiPolygon(
self.compensation.geometry.geom.buffer(10),
srid=self.compensation.geometry.geom.srid
) # Create a geometry which differs from the stored one
geojson = self.create_geojson(new_geometry)
check_on_elements = {
@ -151,19 +157,21 @@ class CompensationWorkflowTestCase(BaseWorkflowTestCase):
check_on_elements = {
self.compensation.title: new_title,
self.compensation.identifier: new_identifier,
self.compensation.comment: new_comment,
}
for k, v in check_on_elements.items():
self.assertEqual(k, v)
self.assert_equal_geometries(self.compensation.geometry.geom, new_geometry)
# Expect identifier to not be editable
self.assertEqual(self.compensation.identifier, old_identifier, msg="Identifier was editable!")
# Expect logs to be set
self.assertEqual(pre_edit_log_count + 1, self.compensation.log.count())
self.assertEqual(self.compensation.log.first().action, UserAction.EDITED)
self.assert_equal_geometries(self.compensation.geometry.geom, new_geometry)
def test_checkability(self):
"""
This tests if the checkability of the compensation (which is defined by the linked intervention's checked

3
compensation/tests/ecoaccount/test_workflow.py
View File

@ -82,6 +82,7 @@ class EcoAccountWorkflowTestCase(BaseWorkflowTestCase):
url = reverse("compensation:acc:edit", args=(self.eco_account.id,))
pre_edit_log_count = self.eco_account.log.count()
old_identifier = self.eco_account.identifier
new_title = self.create_dummy_string()
new_identifier = self.create_dummy_string()
new_comment = self.create_dummy_string()
@ -114,7 +115,6 @@ class EcoAccountWorkflowTestCase(BaseWorkflowTestCase):
check_on_elements = {
self.eco_account.title: new_title,
self.eco_account.identifier: new_identifier,
self.eco_account.deductable_surface: test_deductable_surface,
self.eco_account.deductable_rest: test_deductable_surface - deductions_surface,
self.eco_account.comment: new_comment,
@ -123,6 +123,7 @@ class EcoAccountWorkflowTestCase(BaseWorkflowTestCase):
for k, v in check_on_elements.items():
self.assertEqual(k, v)
self.assertEqual(self.eco_account.identifier, old_identifier)
self.assert_equal_geometries(self.eco_account.geometry.geom, new_geometry)
# Expect logs to be set

2
ema/forms.py
View File

@ -133,7 +133,6 @@ class EditEmaForm(NewEmaForm):
def save(self, user: User, geom_form: SimpleGeomForm):
with transaction.atomic():
# Fetch data from cleaned POST values
identifier = self.cleaned_data.get("identifier", None)
title = self.cleaned_data.get("title", None)
handler_type = self.cleaned_data.get("handler_type", None)
handler_detail = self.cleaned_data.get("handler_detail", None)
@ -154,7 +153,6 @@ class EditEmaForm(NewEmaForm):
self.instance.responsible.save()
# Update main oject data
self.instance.identifier = identifier
self.instance.title = title
self.instance.comment = comment
self.instance.is_pik = is_pik

3
ema/tests/test_workflow.py
View File

@ -80,6 +80,7 @@ class EmaWorkflowTestCase(BaseWorkflowTestCase):
self.ema = self.fill_out_ema(self.ema)
pre_edit_log_count = self.ema.log.count()
old_identifier = self.ema.identifier
new_title = self.create_dummy_string()
new_identifier = self.create_dummy_string()
new_comment = self.create_dummy_string()
@ -106,13 +107,13 @@ class EmaWorkflowTestCase(BaseWorkflowTestCase):
check_on_elements = {
self.ema.title: new_title,
self.ema.identifier: new_identifier,
self.ema.comment: new_comment,
}
for k, v in check_on_elements.items():
self.assertEqual(k, v)
self.assertEqual(self.ema.identifier, old_identifier)
self.assert_equal_geometries(self.ema.geometry.geom, new_geometry)
# Expect logs to be set

2
ema/tests/unit/test_forms.py
View File

@ -130,7 +130,7 @@ class EditEmaFormTestCase(BaseTestCase):
self.assertIsNotNone(obj.responsible.handler)
self.assertEqual(obj.responsible.conservation_office, data["conservation_office"])
self.assertEqual(obj.responsible.conservation_file_number, data["conservation_file_number"])
self.assertEqual(obj.identifier, data["identifier"])
self.assertNotEqual(obj.identifier, data["identifier"], msg="Identifier editable via form!")
self.assertEqual(obj.comment, data["comment"])
last_log = obj.log.first()

2
intervention/forms/intervention.py
View File

@ -345,7 +345,6 @@ class EditInterventionForm(NewInterventionForm):
"""
with transaction.atomic():
identifier = self.cleaned_data.get("identifier", None)
title = self.cleaned_data.get("title", None)
process_type = self.cleaned_data.get("type", None)
laws = self.cleaned_data.get("laws", None)
@ -379,7 +378,6 @@ class EditInterventionForm(NewInterventionForm):
self.instance.log.add(user_action)
self.instance.identifier = identifier
self.instance.title = title
self.instance.comment = comment
self.instance.modified = user_action

2
intervention/forms/modals/check.py
View File

@ -33,7 +33,7 @@ class CheckModalForm(BaseModalForm):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.form_title = _("Run check")
self.form_caption = _("I, {} {}, confirm that all necessary control steps have been performed by myself.").format(self.user.first_name, self.user.last_name)
self.form_caption = _("The necessary control steps have been performed:").format(self.user.first_name, self.user.last_name)
self.valid = False
def _are_deductions_valid(self):

5
intervention/tables.py
View File

@ -33,6 +33,11 @@ class InterventionTable(BaseTable, TableRenderMixin, TableOrderMixin):
verbose_name=_("Parcel gmrkng"),
orderable=False,
accessor="geometry",
attrs={
"th": {
"class": "w-25",
}
}
)
c = tables.Column(
verbose_name=_("Checked"),

2
intervention/views/intervention.py
View File

@ -39,7 +39,7 @@ def index_view(request: HttpRequest):
"""
template = "generic_index.html"
# Filtering by user access is performed in table filter inside of InterventionTableFilter class
# Filtering by user access is performed in table filter inside InterventionTableFilter class
interventions = Intervention.objects.filter(
deleted=None, # not deleted
).select_related(

2
konova/admin.py
View File

@ -151,7 +151,7 @@ class ResubmissionAdmin(BaseResourceAdmin):
# Outcommented for a cleaner admin backend on production
#admin.site.register(Geometry, GeometryAdmin)
admin.site.register(Geometry, GeometryAdmin)
#admin.site.register(Parcel, ParcelAdmin)
#admin.site.register(District, DistrictAdmin)
#admin.site.register(Municipal, MunicipalAdmin)

26
konova/forms/geometry_form.py
View File

@ -98,12 +98,14 @@ class SimpleGeomForm(BaseForm):
if g.geom_type not in accepted_ogr_types:
self.add_error("geom", _("Only surfaces allowed. Points or lines must be buffered."))
is_valid = False
is_valid &= False
return is_valid
is_valid &= self.__is_area_valid(g)
polygon = Polygon.from_ewkt(g.ewkt)
is_valid = polygon.valid
if not is_valid:
is_valid &= polygon.valid
if not polygon.valid:
self.add_error("geom", polygon.valid_reason)
return is_valid
@ -137,6 +139,24 @@ class SimpleGeomForm(BaseForm):
return num_vertices <= GEOM_MAX_VERTICES
def __is_area_valid(self, geom: gdal.OGRGeometry):
""" Checks whether the area is at least > 1m²
Returns:
"""
is_area_valid = geom.area > 1 # > 1m² (SRID:25832)
if not is_area_valid:
self.add_error(
"geom",
_("Geometry must be greater than 1m². Currently is {}m²").format(
float(geom.area)
)
)
return is_area_valid
def __simplify_geometry(self, geom, max_vert: int):
""" Simplifies a geometry

2
konova/forms/modals/record_form.py
View File

@ -27,7 +27,7 @@ class RecordModalForm(BaseModalForm):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.form_title = _("Record data")
self.form_caption = _("I, {} {}, confirm that all necessary control steps have been performed by myself.").format(self.user.first_name, self.user.last_name)
self.form_caption = _("The necessary control steps have been performed:").format(self.user.first_name, self.user.last_name)
# Disable automatic w-100 setting for this type of modal form. Looks kinda strange
self.fields["confirm"].widget.attrs["class"] = ""

107
konova/management/commands/sanitize_db.py
View File

@ -61,14 +61,24 @@ class Command(BaseKonovaCommand):
action=UserAction.CREATED
)
intervention_log_entries_ids = self.get_all_log_entries_ids(Intervention)
attached_log_entries_id = intervention_log_entries_ids.union(
self.get_all_log_entries_ids(Compensation),
self.get_all_log_entries_ids(EcoAccount),
self.get_all_log_entries_ids(Ema),
)
EIV_log_entries_ids = self.get_all_log_entries_ids(Intervention)
self._write_warning(f" EIV: {EIV_log_entries_ids.count()} attached log entries")
KOM_log_entries_ids = self.get_all_log_entries_ids(Compensation)
self._write_warning(f" KOM: {KOM_log_entries_ids.count()} attached log entries")
OEK_log_entries_ids = self.get_all_log_entries_ids(EcoAccount)
self._write_warning(f" OEK: {OEK_log_entries_ids.count()} attached log entries")
EMA_log_entries_ids = self.get_all_log_entries_ids(Ema)
self._write_warning(f" EMA: {EMA_log_entries_ids.count()} attached log entries")
unattached_log_entries = all_log_entries.exclude(id__in=attached_log_entries_id)
unattached_log_entries = all_log_entries.exclude(
id__in=EIV_log_entries_ids
).exclude(
id__in=KOM_log_entries_ids
).exclude(
id__in=OEK_log_entries_ids
).exclude(
id__in=EMA_log_entries_ids
)
num_entries = unattached_log_entries.count()
if num_entries > 0:
@ -108,13 +118,20 @@ class Command(BaseKonovaCommand):
self._write_warning("=== Sanitize compensation actions ===")
all_actions = CompensationAction.objects.all()
compensation_action_ids = self.get_all_action_ids(Compensation)
attached_action_ids = compensation_action_ids.union(
self.get_all_action_ids(EcoAccount),
self.get_all_action_ids(Ema),
)
kom_action_ids = self.get_all_action_ids(Compensation)
self._write_warning(f" KOM: {kom_action_ids.count()} attached actions")
oek_action_ids = self.get_all_action_ids(EcoAccount)
self._write_warning(f" OEK: {oek_action_ids.count()} attached actions")
ema_action_ids = self.get_all_action_ids(Ema)
self._write_warning(f" EMA: {ema_action_ids.count()} attached actions")
unattached_actions = all_actions.exclude(id__in=attached_action_ids)
unattached_actions = all_actions.exclude(
id__in=kom_action_ids
).exclude(
id__in=oek_action_ids
).exclude(
id__in=ema_action_ids
)
num_entries = unattached_actions.count()
if num_entries > 0:
@ -125,7 +142,7 @@ class Command(BaseKonovaCommand):
self._write_success("No unattached actions found.")
self._break_line()
def get_all_deadline_ids(self, cls):
def _get_all_deadline_ids(self, cls):
""" Getter for all deadline ids of a model
Args:
@ -154,13 +171,20 @@ class Command(BaseKonovaCommand):
self._write_warning("=== Sanitize deadlines ===")
all_deadlines = Deadline.objects.all()
compensation_deadline_ids = self.get_all_deadline_ids(Compensation)
attached_deadline_ids = compensation_deadline_ids.union(
self.get_all_deadline_ids(EcoAccount),
self.get_all_deadline_ids(Ema),
)
kom_deadline_ids = self._get_all_deadline_ids(Compensation)
self._write_warning(f" KOM: {kom_deadline_ids.count()} attached deadlines")
oek_deadline_ids = self._get_all_deadline_ids(EcoAccount)
self._write_warning(f" OEK: {kom_deadline_ids.count()} attached deadlines")
ema_deadline_ids = self._get_all_deadline_ids(Ema)
self._write_warning(f" EMA: {kom_deadline_ids.count()} attached deadlines")
unattached_deadlines = all_deadlines.exclude(id__in=attached_deadline_ids)
unattached_deadlines = all_deadlines.exclude(
id__in=kom_deadline_ids
).exclude(
id__in=oek_deadline_ids
).exclude(
id__in=ema_deadline_ids
)
num_entries = unattached_deadlines.count()
if num_entries > 0:
@ -171,7 +195,7 @@ class Command(BaseKonovaCommand):
self._write_success("No unattached deadlines found.")
self._break_line()
def get_all_geometry_ids(self, cls):
def _get_all_geometry_ids(self, cls):
""" Getter for all geometry ids of a model
Args:
@ -200,14 +224,24 @@ class Command(BaseKonovaCommand):
self._write_warning("=== Sanitize geometries ===")
all_geometries = Geometry.objects.all()
compensation_geometry_ids = self.get_all_geometry_ids(Compensation)
attached_geometry_ids = compensation_geometry_ids.union(
self.get_all_geometry_ids(Intervention),
self.get_all_geometry_ids(EcoAccount),
self.get_all_geometry_ids(Ema),
)
kom_geometry_ids = self._get_all_geometry_ids(Compensation)
self._write_warning(f" KOM: {kom_geometry_ids.count()} attached geometries")
eiv_geometry_ids = self._get_all_geometry_ids(Intervention)
self._write_warning(f" EIV: {eiv_geometry_ids.count()} attached geometries")
oek_geometry_ids = self._get_all_geometry_ids(EcoAccount)
self._write_warning(f" OEK: {oek_geometry_ids.count()} attached geometries")
ema_geometry_ids = self._get_all_geometry_ids(Ema)
self._write_warning(f" EMA: {ema_geometry_ids.count()} attached geometries")
unattached_geometries = all_geometries.exclude(id__in=attached_geometry_ids)
unattached_geometries = all_geometries.exclude(
id__in=kom_geometry_ids
).exclude(
id__in=eiv_geometry_ids
).exclude(
id__in=oek_geometry_ids
).exclude(
id__in=ema_geometry_ids
)
num_entries = unattached_geometries.count()
if num_entries > 0:
@ -218,7 +252,7 @@ class Command(BaseKonovaCommand):
self._write_success("No unattached geometries found.")
self._break_line()
def get_all_state_ids(self, cls):
def _get_all_state_ids(self, cls):
""" Getter for all states (before and after) of a class
Args:
@ -254,14 +288,19 @@ class Command(BaseKonovaCommand):
"""
self._write_warning("=== Sanitize compensation states ===")
all_states = CompensationState.objects.all()
compensation_state_ids = self.get_all_state_ids(Compensation)
account_state_ids = self.get_all_state_ids(EcoAccount)
ema_state_ids = self.get_all_state_ids(Ema)
attached_state_ids = compensation_state_ids.union(account_state_ids, ema_state_ids)
kom_state_ids = self._get_all_state_ids(Compensation)
oek_state_ids = self._get_all_state_ids(EcoAccount)
ema_state_ids = self._get_all_state_ids(Ema)
unattached_states = all_states.exclude(
id__in=attached_state_ids
id__in=kom_state_ids
).exclude(
id__in=oek_state_ids
).exclude(
id__in=ema_state_ids
)
num_unattached_states = unattached_states.count()
if num_unattached_states > 0:
self._write_error(f"Found {num_unattached_states} unused compensation states. Delete now...")

23
konova/migrations/0015_geometry_parcel_calculation_end_and_more.py Normal file
View File

@ -0,0 +1,23 @@
# Generated by Django 5.0.1 on 2024-01-09 10:38
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('konova', '0014_resubmission'),
]
operations = [
migrations.AddField(
model_name='geometry',
name='parcel_update_end',
field=models.DateTimeField(blank=True, db_comment='When the last parcel calculation finished', help_text='When the last parcel calculation finished', null=True),
),
migrations.AddField(
model_name='geometry',
name='parcel_update_start',
field=models.DateTimeField(blank=True, db_comment='When the last parcel calculation started', help_text='When the last parcel calculation started', null=True),
),
]

17
konova/migrations/0016_remove_parcelintersection_calculated_on.py Normal file
View File

@ -0,0 +1,17 @@
# Generated by Django 5.0.1 on 2024-02-16 07:34
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('konova', '0015_geometry_parcel_calculation_end_and_more'),
]
operations = [
migrations.RemoveField(
model_name='parcelintersection',
name='calculated_on',
),
]

308
konova/models/geometry.py
View File

@ -8,19 +8,31 @@ Created on: 15.11.21
import json
from django.contrib.gis.db.models import MultiPolygonField
from django.core.exceptions import ObjectDoesNotExist
from django.db import models, transaction
from django.utils import timezone
from konova.models import BaseResource, UuidModel
from konova.sub_settings.lanis_settings import DEFAULT_SRID_RLP
from konova.utils.schneider.fetcher import ParcelFetcher
from konova.utils.wfs.spatial import ParcelWFSFetcher
class Geometry(BaseResource):
"""
Geometry model
"""
parcel_update_start = models.DateTimeField(
blank=True,
null=True,
db_comment="When the last parcel calculation started",
help_text="When the last parcel calculation started"
)
parcel_update_end = models.DateTimeField(
blank=True,
null=True,
db_comment="When the last parcel calculation finished",
help_text="When the last parcel calculation finished",
)
geom = MultiPolygonField(null=True, blank=True, srid=DEFAULT_SRID_RLP)
def __str__(self):
@ -109,82 +121,14 @@ class Geometry(BaseResource):
objs += set_objs
return objs
@transaction.atomic
def update_parcels_wfs(self):
""" Updates underlying parcel information using the WFS of LVermGeo
Returns:
def get_data_object(self):
"""
from konova.models import Parcel, District, ParcelIntersection, Municipal, ParcelGroup
if self.geom.empty:
# Nothing to do
return
parcel_fetcher = ParcelWFSFetcher(
geometry_id=self.id,
)
typename = "ave:Flurstueck"
fetched_parcels = parcel_fetcher.get_features(
typename
)
_now = timezone.now()
underlying_parcels = []
for result in fetched_parcels:
parcel_properties = result["properties"]
# There could be parcels which include the word 'Flur',
# which needs to be deleted and just keep the numerical values
## THIS CAN BE REMOVED IN THE FUTURE, WHEN 'Flur' WON'T OCCUR ANYMORE!
flr_val = parcel_properties["flur"].replace("Flur ", "")
district = District.objects.get_or_create(
key=parcel_properties["kreisschl"],
name=parcel_properties["kreis"],
)[0]
municipal = Municipal.objects.get_or_create(
key=parcel_properties["gmdschl"],
name=parcel_properties["gemeinde"],
district=district,
)[0]
parcel_group = ParcelGroup.objects.get_or_create(
key=parcel_properties["gemaschl"],
name=parcel_properties["gemarkung"],
municipal=municipal,
)[0]
flrstck_nnr = parcel_properties['flstnrnen']
if not flrstck_nnr:
flrstck_nnr = None
flrstck_zhlr = parcel_properties['flstnrzae']
if not flrstck_zhlr:
flrstck_zhlr = None
parcel_obj = Parcel.objects.get_or_create(
district=district,
municipal=municipal,
parcel_group=parcel_group,
flr=flr_val,
flrstck_nnr=flrstck_nnr,
flrstck_zhlr=flrstck_zhlr,
)[0]
parcel_obj.district = district
parcel_obj.updated_on = _now
parcel_obj.save()
underlying_parcels.append(parcel_obj)
# Update the linked parcels
self.parcels.clear()
self.parcels.set(underlying_parcels)
# Set the calculated_on intermediate field, so this related data will be found on lookups
intersections_without_ts = self.parcelintersection_set.filter(
parcel__in=self.parcels.all(),
calculated_on__isnull=True,
)
for entry in intersections_without_ts:
entry.calculated_on = _now
ParcelIntersection.objects.bulk_update(
intersections_without_ts,
["calculated_on"]
)
Getter for the specific data object which is related to this geometry
"""
objs = self.get_data_objects()
assert (len(objs) <= 1)
result = objs.pop()
return result
def update_parcels(self):
""" Updates underlying parcel information
@ -192,72 +136,141 @@ class Geometry(BaseResource):
Returns:
"""
from konova.models import Parcel, District, ParcelIntersection, Municipal, ParcelGroup
if self.geom.empty:
# Nothing to do
return
self._set_parcel_update_start_time()
self._perform_parcel_update()
self._set_parcel_update_end_time()
def _perform_parcel_update(self):
"""
Performs the main logic of parcel updating.
"""
from konova.models import Parcel, District, Municipal, ParcelGroup
parcel_fetcher = ParcelFetcher(
geometry=self
)
fetched_parcels = parcel_fetcher.get_parcels()
_now = timezone.now()
underlying_parcels = []
districts = {}
municipals = {}
parcel_groups = {}
parcels_to_update = []
parcels_to_create = []
for result in fetched_parcels:
# There could be parcels which include the word 'Flur',
# which needs to be deleted and just keep the numerical values
## THIS CAN BE REMOVED IN THE FUTURE, WHEN 'Flur' WON'T OCCUR ANYMORE!
flr_val = result["flur"].replace("Flur ", "")
district = District.objects.get_or_create(
key=result["kreisschl"],
name=result["kreis"],
)[0]
municipal = Municipal.objects.get_or_create(
key=result["gmdschl"],
name=result["gemeinde"],
district=district,
)[0]
parcel_group = ParcelGroup.objects.get_or_create(
key=result["gemaschl"],
name=result["gemarkung"],
municipal=municipal,
)[0]
flrstck_nnr = result['flstnrnen']
if not flrstck_nnr:
flrstck_nnr = None
flrstck_zhlr = result['flstnrzae']
if not flrstck_zhlr:
flrstck_zhlr = None
parcel_obj = Parcel.objects.get_or_create(
district=district,
municipal=municipal,
parcel_group=parcel_group,
flr=flr_val,
flrstck_nnr=flrstck_nnr,
flrstck_zhlr=flrstck_zhlr,
)[0]
parcel_obj.district = district
parcel_obj.updated_on = _now
parcel_obj.save()
underlying_parcels.append(parcel_obj)
# Update the linked parcels
self.parcels.clear()
# Get district (cache in dict)
try:
district = districts["kreisschl"]
except KeyError:
district = District.objects.get_or_create(
key=result["kreisschl"],
name=result["kreis"],
)[0]
districts[district.key] = district
# Get municipal (cache in dict)
try:
municipal = municipals["gmdschl"]
except KeyError:
municipal = Municipal.objects.get_or_create(
key=result["gmdschl"],
name=result["gemeinde"],
district=district,
)[0]
municipals[municipal.key] = municipal
# Get parcel group (cache in dict)
try:
parcel_group = parcel_groups["gemaschl"]
except KeyError:
parcel_group = ParcelGroup.objects.get_or_create(
key=result["gemaschl"],
name=result["gemarkung"],
municipal=municipal,
)[0]
parcel_groups[parcel_group.key] = parcel_group
# Preprocess parcel data
flrstck_nnr = result['flstnrnen']
match flrstck_nnr:
case "":
flrstck_nnr = None
flrstck_zhlr = result['flstnrzae']
match flrstck_zhlr:
case "":
flrstck_zhlr = None
try:
# Try to fetch parcel from db. If it already exists, just update timestamp.
parcel_obj = Parcel.objects.get(
district=district,
municipal=municipal,
parcel_group=parcel_group,
flr=flr_val,
flrstck_nnr=flrstck_nnr,
flrstck_zhlr=flrstck_zhlr,
)
parcel_obj.updated_on = _now
parcels_to_update.append(parcel_obj)
except ObjectDoesNotExist:
# If not existing, create object but do not commit, yet
parcel_obj = Parcel(
district=district,
municipal=municipal,
parcel_group=parcel_group,
flr=flr_val,
flrstck_nnr=flrstck_nnr,
flrstck_zhlr=flrstck_zhlr,
updated_on=_now,
)
parcels_to_create.append(parcel_obj)
# Create new parcels
Parcel.objects.bulk_create(
parcels_to_create,
batch_size=500
)
# Update existing parcels
Parcel.objects.bulk_update(
parcels_to_update,
[
"updated_on"
],
batch_size=500
)
# Update linking to geometry
parcel_ids = [x.id for x in parcels_to_update] + [x.id for x in parcels_to_create]
underlying_parcels = Parcel.objects.filter(id__in=parcel_ids)
self.parcels.set(underlying_parcels)
# Set the calculated_on intermediate field, so this related data will be found on lookups
intersections_without_ts = self.parcelintersection_set.filter(
parcel__in=self.parcels.all(),
calculated_on__isnull=True,
)
for entry in intersections_without_ts:
entry.calculated_on = _now
ParcelIntersection.objects.bulk_update(
intersections_without_ts,
["calculated_on"]
)
@transaction.atomic
def _set_parcel_update_start_time(self):
"""
Sets the current time for the parcel calculation begin
"""
self.parcel_update_start = timezone.now()
self.parcel_update_end = None
self.save()
@transaction.atomic
def _set_parcel_update_end_time(self):
"""
Sets the current time for the parcel calculation end
"""
self.parcel_update_end = timezone.now()
self.save()
def get_underlying_parcels(self):
""" Getter for related parcels and their districts
@ -265,9 +278,7 @@ class Geometry(BaseResource):
Returns:
parcels (QuerySet): The related parcels as queryset
"""
parcels = self.parcels.filter(
parcelintersection__calculated_on__isnull=False,
).prefetch_related(
parcels = self.parcels.prefetch_related(
"district",
"municipal",
).order_by(
@ -292,17 +303,6 @@ class Geometry(BaseResource):
municipals = Municipal.objects.filter(id__in=municipals).order_by("name")
return municipals
def count_underlying_parcels(self):
""" Getter for number of underlying parcels
Returns:
"""
num_parcels = self.parcels.filter(
parcelintersection__calculated_on__isnull=False,
).count()
return num_parcels
def as_feature_collection(self, srid=DEFAULT_SRID_RLP):
""" Returns a FeatureCollection structure holding all polygons of the MultiPolygon as single features
@ -337,6 +337,36 @@ class Geometry(BaseResource):
}
return geojson
@property
def complexity_factor(self) -> float:
""" Calculates a factor to estimate the complexity of a Geometry
0 = very low complexity
1 = very high complexity
ASSUMPTION:
The envelope is the bounding box of a geometry. If the geometry's area is similar to the area of it's bounding
box, it is considered as rather simple, since it seems to be a closer shape like a simple box.
If the geometry has a very big bounding box, but the geometry's own area is rather small,
compared to the one of the bounding box, the complexity can be higher.
Example:
geometry area similar to bounding box --> geometry / bounding_box ~ 1
geometry area far smaller than bb --> geometry / bounding_box ~ 0
Result is being inverted for better understanding of 'low' and 'high' complexity.
Returns:
complexity_factor (float): The estimated complexity
"""
if self.geom.empty:
return 0
geom_envelope = self.geom.envelope
diff = geom_envelope - self.geom
complexity_factor = 1 - self.geom.area / diff.area
return complexity_factor
class GeometryConflict(UuidModel):
"""

11
konova/models/object.py
View File

@ -672,17 +672,6 @@ class GeoReferencedMixin(models.Model):
result = self.geometry.get_underlying_parcels()
return result
def count_underlying_parcels(self):
""" Getter for number of underlying parcels
Returns:
"""
result = 0
if self.geometry is not None:
result = self.geometry.count_underlying_parcels()
return result
def set_geometry_conflict_message(self, request: HttpRequest):
if self.geometry is None:
return request

14
konova/models/parcel.py
View File

@ -160,19 +160,9 @@ class Parcel(UuidModel):
class ParcelIntersection(UuidModel):
""" ParcelIntersection is an intermediary model, which is used to configure the
"""
ParcelIntersection is an intermediary model, which is used to add extras to the
M2M relation between Parcel and Geometry.
Based on uuids, we will not have (practically) any problems on outrunning primary keys
and extending the model with calculated_on timestamp, we can 'hide' entries while they
are being recalculated and keep track on the last time they have been calculated this
way.
Please note: The calculated_on describes when the relation between the Parcel and the Geometry
has been established. The updated_on field of Parcel describes when this Parcel has been
changed the last time.
"""
parcel = models.ForeignKey(Parcel, on_delete=models.CASCADE)
geometry = models.ForeignKey("konova.Geometry", on_delete=models.CASCADE)
calculated_on = models.DateTimeField(auto_now_add=True, null=True, blank=True)

4
konova/settings.py
View File

@ -46,4 +46,8 @@ DEFAULT_GROUP = "Default"
ZB_GROUP = "Registration office"
ETS_GROUP = "Conservation office"
# GEOMETRY
## Max number of allowed vertices. Geometries larger will be simplified until they reach this threshold
GEOM_MAX_VERTICES = 10000
## Max seconds to wait for a parcel calculation result before a new request will be started (default: 30 minutes)
GEOM_THRESHOLD_RECALCULATION_SECONDS = 60 * 30

10
konova/tasks.py
View File

@ -10,15 +10,9 @@ def celery_update_parcels(geometry_id: str, recheck: bool = True):
from konova.models import Geometry, ParcelIntersection
try:
geom = Geometry.objects.get(id=geometry_id)
objs = geom.parcelintersection_set.all()
for obj in objs:
obj.calculated_on = None
ParcelIntersection.objects.bulk_update(
objs,
["calculated_on"]
)
geom.parcels.clear()
geom.update_parcels()
except ObjectDoesNotExist:
if recheck:
sleep(5)

8
konova/tests/test_views.py
View File

@ -146,7 +146,6 @@ class BaseTestCase(TestCase):
geometry = Geometry.objects.create()
# Finally create main object, holding the other objects
intervention = Intervention.objects.create(
identifier="TEST",
title="Test_title",
responsible=responsibility_data,
legal=legal_data,
@ -174,7 +173,6 @@ class BaseTestCase(TestCase):
geometry = Geometry.objects.create()
# Finally create main object, holding the other objects
compensation = Compensation.objects.create(
identifier="TEST",
title="Test_title",
intervention=interv,
created=action,
@ -200,10 +198,8 @@ class BaseTestCase(TestCase):
responsible_data.handler = handler
responsible_data.save()
identifier = EcoAccount().generate_new_identifier()
# Finally create main object, holding the other objects
eco_account = EcoAccount.objects.create(
identifier=identifier,
title="Test_title",
deductable_surface=500,
legal=lega_data,
@ -230,7 +226,6 @@ class BaseTestCase(TestCase):
responsible_data.save()
# Finally create main object, holding the other objects
ema = Ema.objects.create(
identifier="TEST",
title="Test_title",
responsible=responsible_data,
created=action,
@ -474,7 +469,7 @@ class BaseTestCase(TestCase):
eco_account.save()
return eco_account
def assert_equal_geometries(self, geom1: MultiPolygon, geom2: MultiPolygon):
def assert_equal_geometries(self, geom1: MultiPolygon, geom2: MultiPolygon, tolerance = 0.001):
""" Assert for geometries to be equal
Transforms the geometries to matching srids before checking
@ -491,7 +486,6 @@ class BaseTestCase(TestCase):
self.assertTrue(True)
return
tolerance = 0.001
if geom1.srid != geom2.srid:
# Due to prior possible transformation of any of these geometries, we need to make sure there exists a
# transformation from one coordinate system into the other, which is valid

2
konova/tests/unit/test_forms.py
View File

@ -152,7 +152,7 @@ class RecordModalFormTestCase(BaseTestCase):
)
self.assertEqual(form.form_title, str(_("Record data")))
self.assertEqual(form.form_caption, str(
_("I, {} {}, confirm that all necessary control steps have been performed by myself.").format(
_("The necessary control steps have been performed:").format(
self.user.first_name,
self.user.last_name
)

6
konova/utils/schneider/fetcher.py
View File

@ -28,7 +28,11 @@ class ParcelFetcher:
self.geometry = geometry
# Reduce size of geometry to avoid "intersections" because of exact border matching
geom = geometry.geom.buffer(-0.001)
buffer_threshold = 0.001
geom = geometry.geom.buffer(-buffer_threshold)
if geom.area < buffer_threshold:
# Fallback for malicious geometries which are way too small and would disappear on negative buffering
geom = geometry.geom
self.geojson = geom.ewkt
self.results = []

4
konova/utils/tables.py
View File

@ -173,9 +173,13 @@ class TableRenderMixin:
Returns:
"""
value_orig = value
max_length = 75
if len(value) > max_length:
value = f"{value[:max_length]}..."
value = format_html(
f'<div title="{value_orig}">{value}</div>'
)
return value
def render_d(self, value, record: GeoReferencedMixin):

189
konova/utils/wfs/spatial.py
View File

@ -1,189 +0,0 @@
"""
Author: Michel Peltriaux
Organization: Struktur- und Genehmigungsdirektion Nord, Rhineland-Palatinate, Germany
Contact: michel.peltriaux@sgdnord.rlp.de
Created on: 17.12.21
"""
import json
from abc import abstractmethod
from json import JSONDecodeError
from time import sleep
import requests
from django.contrib.gis.db.models.functions import AsGML, MakeValid
from django.db.models import Func, F
from requests.auth import HTTPDigestAuth
from konova.settings import PARCEL_WFS_USER, PARCEL_WFS_PW, PROXIES
class AbstractWFSFetcher:
""" Base class for fetching WFS data
"""
# base_url represents not the capabilities url but the parameter-free base url
base_url = None
version = None
auth_user = None
auth_pw = None
auth_digest_obj = None
class Meta:
abstract = True
def __init__(self, base_url: str, version: str = "1.1.0", auth_user: str = None, auth_pw: str = None, *args, **kwargs):
self.base_url = base_url
self.version = version
self.auth_pw = auth_pw
self.auth_user = auth_user
self._create_auth_obj()
def _create_auth_obj(self):
if self.auth_pw is not None and self.auth_user is not None:
self.auth_digest_obj = HTTPDigestAuth(
self.auth_user,
self.auth_pw
)
@abstractmethod
def get_features(self, feature_identifier: str, filter_str: str):
raise NotImplementedError
class ParcelWFSFetcher(AbstractWFSFetcher):
""" Fetches features from a special parcel WFS
"""
geometry_id = None
geometry_property_name = None
count = 100
def __init__(self, geometry_id: str, geometry_property_name: str = "msGeometry", *args, **kwargs):
super().__init__(
version="2.0.0",
base_url="https://www.geoportal.rlp.de/registry/wfs/519",
auth_user=PARCEL_WFS_USER,
auth_pw=PARCEL_WFS_PW,
*args,
**kwargs
)
self.geometry_id = geometry_id
self.geometry_property_name = geometry_property_name
def _create_spatial_filter(self,
geometry_operation: str):
""" Creates a xml spatial filter according to the WFS filter specification
The geometry needs to be shrinked by a very small factor (-0.01) before a GML can be created for intersection
checking. Otherwise perfect parcel outline placement on top of a neighbouring parcel would result in an
intersection hit, despite the fact they do not truly intersect just because their vertices match.
Args:
geometry_operation (str): One of the WFS supported spatial filter operations (according to capabilities)
Returns:
spatial_filter (str): The spatial filter element
"""
from konova.models import Geometry
geom = Geometry.objects.filter(
id=self.geometry_id
).annotate(
smaller=Func(F('geom'), -0.001, function="ST_Buffer") # same as geometry.geom_small_buffered but for QuerySet
).annotate(
gml=AsGML(MakeValid('smaller'))
).first()
geom_gml = geom.gml
spatial_filter = f"<Filter><{geometry_operation}><PropertyName>{self.geometry_property_name}</PropertyName>{geom_gml}</{geometry_operation}></Filter>"
return spatial_filter
def _create_post_data(self,
geometry_operation: str,
typenames: str = None,
start_index: int = 0,
):
""" Creates a POST body content for fetching features
Args:
geometry_operation (str): One of the WFS supported spatial filter operations (according to capabilities)
Returns:
_filter (str): A proper xml WFS filter
"""
start_index = str(start_index)
spatial_filter = self._create_spatial_filter(
geometry_operation
)
_filter = f'<wfs:GetFeature service="WFS" version="{self.version}" xmlns:wfs="http://www.opengis.net/wfs/2.0" xmlns:fes="http://www.opengis.net/fes/2.0" xmlns:myns="http://www.someserver.com/myns" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.opengis.net/wfs/2.0 http://schemas.opengis.net/wfs/2.0.0/wfs.xsd" count="{self.count}" startindex="{start_index}" outputFormat="application/json; subtype=geojson"><wfs:Query typeNames="{typenames}">{spatial_filter}</wfs:Query></wfs:GetFeature>'
return _filter
def get_features(self,
typenames: str,
spatial_operator: str = "Intersects",
filter_srid: str = None,
start_index: int = 0,
rerun_on_exception: bool = True
):
""" Fetches features from the WFS using POST
POST is required since GET has a character limit around 4000. Having a larger filter would result in errors,
which do not occur in case of POST.
Args:
typenames (str): References to parameter 'typenames' in a WFS GetFeature request
spatial_operator (str): Defines the spatial operation for filtering
filter_srid (str): Defines the spatial reference system, the geometry shall be transformed into for filtering
start_index (str): References to parameter 'startindex' in a
Returns:
features (list): A list of returned features
"""
found_features = []
while start_index is not None:
post_body = self._create_post_data(
spatial_operator,
typenames,
start_index
)
response = requests.post(
url=self.base_url,
data=post_body,
auth=self.auth_digest_obj,
proxies=PROXIES,
)
content = response.content.decode("utf-8")
try:
# Check if collection is an exception and does not contain the requested data
content = json.loads(content)
except JSONDecodeError as e:
if rerun_on_exception:
# Wait a second before another try
sleep(1)
self.get_features(
typenames,
spatial_operator,
filter_srid,
start_index,
rerun_on_exception=False
)
else:
e.msg += content
raise e
fetched_features = content.get(
"features",
{},
)
found_features += fetched_features
if len(fetched_features) < self.count:
# The response was not 'full', so we got everything to fetch
start_index = None
else:
# If a 'full' response returned, there might be more to fetch. Increase the start_index!
start_index += self.count
return found_features

61
konova/views/geometry.py
View File

@ -10,10 +10,13 @@ from django.contrib.gis.geos import MultiPolygon
from django.http import HttpResponse, HttpRequest
from django.shortcuts import get_object_or_404
from django.template.loader import render_to_string
from django.utils import timezone
from django.views import View
from konova.models import Geometry
from konova.settings import GEOM_THRESHOLD_RECALCULATION_SECONDS
from konova.sub_settings.lanis_settings import DEFAULT_SRID_RLP
from konova.tasks import celery_update_parcels
class GeomParcelsView(LoginRequiredMixin, View):
@ -30,24 +33,43 @@ class GeomParcelsView(LoginRequiredMixin, View):
Returns:
A rendered piece of HTML
"""
# HTTP code 286 states that the HTMX should stop polling for updates
# https://htmx.org/docs/#polling
status_code = 286
template = "konova/includes/parcels/parcel_table_frame.html"
geom = get_object_or_404(Geometry, id=id)
parcels = geom.get_underlying_parcels()
geos_geom = geom.geom or MultiPolygon(srid=DEFAULT_SRID_RLP)
geometry_exists = not geos_geom.empty
parcels_are_currently_calculated = geometry_exists and geos_geom.area > 0 and len(parcels) == 0
parcels_available = len(parcels) > 0
geom = get_object_or_404(Geometry, id=id)
geos_geom = geom.geom or MultiPolygon(srid=DEFAULT_SRID_RLP)
geometry_exists = not geos_geom.empty and geos_geom.area > 0
geom_parcel_update_started = geom.parcel_update_start is not None
geom_parcel_update_finished = geom.parcel_update_end is not None
parcels = geom.get_underlying_parcels()
parcels_are_available = len(parcels) > 0
waiting_too_long = self._check_waiting_too_long(geom)
if geometry_exists and not parcels_are_available and waiting_too_long:
# Trigger calculation again - process may have failed silently
celery_update_parcels.delay(geom.id)
parcels_are_currently_calculated = True
else:
parcels_are_currently_calculated = (
geometry_exists and
not parcels_are_available and
geom_parcel_update_started and
not geom_parcel_update_finished
)
if parcels_are_currently_calculated:
# Parcels are being calculated right now. Change the status code, so polling stays active for fetching
# resutls after the calculation
# results after the calculation
status_code = 200
else:
# HTTP code 286 states that the HTMX should stop polling for updates
# https://htmx.org/docs/#polling
status_code = 286
if parcels_available or not geometry_exists:
if parcels_are_available or not geometry_exists:
# Default case: Parcels are calculated or there is no geometry at all
# (so there will be no parcels to expect)
municipals = geom.get_underlying_municipals(parcels)
rpp = 100
@ -69,6 +91,23 @@ class GeomParcelsView(LoginRequiredMixin, View):
else:
return HttpResponse(None, status=404)
def _check_waiting_too_long(self, geom: Geometry):
""" Check whether the client is waiting too long for a parcel calculation result
Depending on the geometry's modified attribute
"""
# Scale time to wait longer with increasing geometry complexity
complexity_factor = geom.complexity_factor + 1
wait_for_seconds = int(GEOM_THRESHOLD_RECALCULATION_SECONDS * complexity_factor)
try:
pcs_diff = (timezone.now() - geom.parcel_update_start).seconds
except TypeError:
pcs_diff = wait_for_seconds
waiting_too_long = (pcs_diff >= wait_for_seconds)
return waiting_too_long
class GeomParcelsContentView(LoginRequiredMixin, View):

BIN
locale/de/LC_MESSAGES/django.mo
View File

Binary file not shown.

1722
locale/de/LC_MESSAGES/django.po
View File

File diff suppressed because it is too large Load Diff

1
requirements.txt
View File

@ -34,7 +34,6 @@ pika==1.3.2
prompt-toolkit==3.0.43
psycopg==3.1.16
psycopg-binary==3.1.16
psycopg2-binary==2.9.9
pyparsing==3.1.1
pypng==0.20220715.0
pyproj==3.6.1