IT-Naturschutz/konova
7
1
Fork 0
Code Issues 6 Pull Requests 1 Actions Projects Releases 28 Wiki Activity

#86 Parcel-Geometry improvement

Browse Source 
* improves the way parcel-geometry relations are stored on the DB
    * instead of a numerical sequence we switched to UUID, so no sequence will run out at anytime (new model: ParcelIntersection)
    * instead of dropping all M2M relations between parcel and geometry on each calculation, we keep the ones that still exist, drop the ones that do not exist and add new ones (if new ones exist)
This commit is contained in:
mpeltriaux 2022-02-09 09:18:35 +01:00
parent e3fbe60fce
commit 591e35a0e2
8 changed files with 132 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
compensation/tables.py
View File

@ -133,7 +133,9 @@ class CompensationTable(BaseTable, TableRenderMixin):
Returns:
"""
parcels = value.parcels.values_list(
parcels = value.parcels.filter(
parcelintersection__calculated_on__isnull=False,
).values_list(
"gmrkng",
flat=True
).distinct()
@ -294,7 +296,9 @@ class EcoAccountTable(BaseTable, TableRenderMixin):
Returns:
"""
parcels = value.parcels.values_list(
parcels = value.parcels.filter(
parcelintersection__calculated_on__isnull=False,
).values_list(
"gmrkng",
flat=True
).distinct()

4
ema/tables.py
View File

@ -103,7 +103,9 @@ class EmaTable(BaseTable, TableRenderMixin):
Returns:
"""
parcels = value.parcels.values_list(
parcels = value.parcels.filter(
parcelintersection__calculated_on__isnull=False,
).values_list(
"gmrkng",
flat=True
).distinct()

4
intervention/tables.py
View File

@ -130,7 +130,9 @@ class InterventionTable(BaseTable, TableRenderMixin):
Returns:
"""
parcels = value.parcels.values_list(
parcels = value.parcels.filter(
parcelintersection__calculated_on__isnull=False,
).values_list(
"gmrkng",
flat=True
).distinct()

54
konova/migrations/0003_auto_20220208_1801.py Normal file
View File

@ -0,0 +1,54 @@
# Generated by Django 3.1.3 on 2022-02-08 17:01
from django.db import migrations, models
import django.db.models.deletion
import uuid
def migrate_parcels(apps, schema_editor):
Geometry = apps.get_model('konova', 'Geometry')
SpatialIntersection = apps.get_model('konova', 'SpatialIntersection')
all_geoms = Geometry.objects.all()
for geom in all_geoms:
SpatialIntersection.objects.bulk_create([
SpatialIntersection(geometry=geom, parcel=parcel)
for parcel in geom.parcels.all()
])
class Migration(migrations.Migration):
dependencies = [
('konova', '0002_auto_20220114_0936'),
]
operations = [
migrations.CreateModel(
name='SpatialIntersection',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('calculated_on', models.DateTimeField(auto_now_add=True, null=True)),
('geometry', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='konova.geometry')),
('parcel', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='konova.parcel')),
],
options={
'abstract': False,
},
),
migrations.RunPython(migrate_parcels),
migrations.AddField(
model_name='parcel',
name='geometries_tmp',
field=models.ManyToManyField(blank=True, related_name='parcels', through='konova.SpatialIntersection', to='konova.Geometry'),
),
migrations.RemoveField(
model_name='parcel',
name='geometries',
),
migrations.RenameField(
model_name='parcel',
old_name='geometries_tmp',
new_name='geometries',
),
]

17
konova/migrations/0004_auto_20220209_0839.py Normal file
View File

@ -0,0 +1,17 @@
# Generated by Django 3.1.3 on 2022-02-09 07:39
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('konova', '0003_auto_20220208_1801'),
]
operations = [
migrations.RenameModel(
old_name='SpatialIntersection',
new_name='ParcelIntersection',
),
]

23
konova/models/geometry.py
View File

@ -99,7 +99,7 @@ class Geometry(BaseResource):
Returns:
"""
from konova.models import Parcel, District
from konova.models import Parcel, District, ParcelIntersection
parcel_fetcher = ParcelWFSFetcher(
geometry_id=self.id,
)
@ -107,6 +107,7 @@ class Geometry(BaseResource):
fetched_parcels = parcel_fetcher.get_features(
typename
)
_now = timezone.now()
underlying_parcels = []
for result in fetched_parcels:
fetched_parcel = result[typename]
@ -125,19 +126,35 @@ class Geometry(BaseResource):
krs=fetched_parcel["ave:kreis"],
)[0]
parcel_obj.district = district
parcel_obj.updated_on = timezone.now()
parcel_obj.updated_on = _now
parcel_obj.save()
underlying_parcels.append(parcel_obj)
# Update the linked parcels
self.parcels.set(underlying_parcels)
# Set the calculated_on intermediate field, so this related data will be found on lookups
intersections_without_ts = self.parcelintersection_set.filter(
parcel__in=self.parcels.all(),
calculated_on__isnull=True,
)
for entry in intersections_without_ts:
entry.calculated_on = _now
ParcelIntersection.objects.bulk_update(
intersections_without_ts,
["calculated_on"]
)
def get_underlying_parcels(self):
""" Getter for related parcels and their districts
Returns:
parcels (QuerySet): The related parcels as queryset
"""
parcels = self.parcels.all().prefetch_related(
parcels = self.parcels.filter(
parcelintersection__calculated_on__isnull=False,
).prefetch_related(
"district"
).order_by(
"gmrkng",

21
konova/models/parcel.py
View File

@ -22,7 +22,7 @@ class Parcel(UuidModel):
To avoid conflicts due to german Umlaute, the field names are shortened and vocals are dropped.
"""
geometries = models.ManyToManyField("konova.Geometry", related_name="parcels", blank=True)
geometries = models.ManyToManyField("konova.Geometry", blank=True, related_name="parcels", through='ParcelIntersection')
district = models.ForeignKey("konova.District", on_delete=models.SET_NULL, null=True, blank=True, related_name="parcels")
gmrkng = models.CharField(
max_length=1000,
@ -77,3 +77,22 @@ class District(UuidModel):
def __str__(self):
return f"{self.gmnd} | {self.krs}"
class ParcelIntersection(UuidModel):
""" ParcelIntersection is an intermediary model, which is used to configure the
M2M relation between Parcel and Geometry.
Based on uuids, we will not have (practically) any problems on outrunning primary keys
and extending the model with calculated_on timestamp, we can 'hide' entries while they
are being recalculated and keep track on the last time they have been calculated this
way.
Please note: The calculated_on describes when the relation between the Parcel and the Geometry
has been established. The updated_on field of Parcel describes when this Parcel has been
changed the last time.
"""
parcel = models.ForeignKey(Parcel, on_delete=models.CASCADE)
geometry = models.ForeignKey("konova.Geometry", on_delete=models.CASCADE)
calculated_on = models.DateTimeField(auto_now_add=True, null=True, blank=True)

12
konova/tasks.py
View File

@ -4,13 +4,19 @@ from celery import shared_task
from django.core.exceptions import ObjectDoesNotExist
@shared_task
def celery_update_parcels(geometry_id: str, recheck: bool = True):
from konova.models import Geometry
from konova.models import Geometry, ParcelIntersection
try:
geom = Geometry.objects.get(id=geometry_id)
geom.parcels.clear()
objs = geom.parcelintersection_set.all()
for obj in objs:
obj.calculated_on = None
ParcelIntersection.objects.bulk_update(
objs,
["calculated_on"]
)
geom.update_parcels()
except ObjectDoesNotExist:
if recheck: