From 591e35a0e2a4a28c9012d6cb46474cb81f11f4ca Mon Sep 17 00:00:00 2001 From: mpeltriaux Date: Wed, 9 Feb 2022 09:18:35 +0100 Subject: [PATCH] #86 Parcel-Geometry improvement * improves the way parcel-geometry relations are stored on the DB * instead of a numerical sequence we switched to UUID, so no sequence will run out at anytime (new model: ParcelIntersection) * instead of dropping all M2M relations between parcel and geometry on each calculation, we keep the ones that still exist, drop the ones that do not exist and add new ones (if new ones exist) --- compensation/tables.py | 8 ++- ema/tables.py | 4 +- intervention/tables.py | 4 +- konova/migrations/0003_auto_20220208_1801.py | 54 ++++++++++++++++++++ konova/migrations/0004_auto_20220209_0839.py | 17 ++++++ konova/models/geometry.py | 23 +++++++-- konova/models/parcel.py | 21 +++++++- konova/tasks.py | 12 +++-- 8 files changed, 132 insertions(+), 11 deletions(-) create mode 100644 konova/migrations/0003_auto_20220208_1801.py create mode 100644 konova/migrations/0004_auto_20220209_0839.py diff --git a/compensation/tables.py b/compensation/tables.py index 6f29e0dd..5a3da24b 100644 --- a/compensation/tables.py +++ b/compensation/tables.py @@ -133,7 +133,9 @@ class CompensationTable(BaseTable, TableRenderMixin): Returns: """ - parcels = value.parcels.values_list( + parcels = value.parcels.filter( + parcelintersection__calculated_on__isnull=False, + ).values_list( "gmrkng", flat=True ).distinct() @@ -294,7 +296,9 @@ class EcoAccountTable(BaseTable, TableRenderMixin): Returns: """ - parcels = value.parcels.values_list( + parcels = value.parcels.filter( + parcelintersection__calculated_on__isnull=False, + ).values_list( "gmrkng", flat=True ).distinct() diff --git a/ema/tables.py b/ema/tables.py index 20ceb456..bf3709de 100644 --- a/ema/tables.py +++ b/ema/tables.py @@ -103,7 +103,9 @@ class EmaTable(BaseTable, TableRenderMixin): Returns: """ - parcels = value.parcels.values_list( + parcels = value.parcels.filter( + parcelintersection__calculated_on__isnull=False, + ).values_list( "gmrkng", flat=True ).distinct() diff --git a/intervention/tables.py b/intervention/tables.py index fd39ddcf..96d17c59 100644 --- a/intervention/tables.py +++ b/intervention/tables.py @@ -130,7 +130,9 @@ class InterventionTable(BaseTable, TableRenderMixin): Returns: """ - parcels = value.parcels.values_list( + parcels = value.parcels.filter( + parcelintersection__calculated_on__isnull=False, + ).values_list( "gmrkng", flat=True ).distinct() diff --git a/konova/migrations/0003_auto_20220208_1801.py b/konova/migrations/0003_auto_20220208_1801.py new file mode 100644 index 00000000..d1d9b5a0 --- /dev/null +++ b/konova/migrations/0003_auto_20220208_1801.py @@ -0,0 +1,54 @@ +# Generated by Django 3.1.3 on 2022-02-08 17:01 + +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +def migrate_parcels(apps, schema_editor): + Geometry = apps.get_model('konova', 'Geometry') + SpatialIntersection = apps.get_model('konova', 'SpatialIntersection') + + all_geoms = Geometry.objects.all() + for geom in all_geoms: + SpatialIntersection.objects.bulk_create([ + SpatialIntersection(geometry=geom, parcel=parcel) + for parcel in geom.parcels.all() + ]) + + +class Migration(migrations.Migration): + + dependencies = [ + ('konova', '0002_auto_20220114_0936'), + ] + + operations = [ + migrations.CreateModel( + name='SpatialIntersection', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('calculated_on', models.DateTimeField(auto_now_add=True, null=True)), + ('geometry', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='konova.geometry')), + ('parcel', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='konova.parcel')), + ], + options={ + 'abstract': False, + }, + ), + migrations.RunPython(migrate_parcels), + migrations.AddField( + model_name='parcel', + name='geometries_tmp', + field=models.ManyToManyField(blank=True, related_name='parcels', through='konova.SpatialIntersection', to='konova.Geometry'), + ), + migrations.RemoveField( + model_name='parcel', + name='geometries', + ), + migrations.RenameField( + model_name='parcel', + old_name='geometries_tmp', + new_name='geometries', + ), + ] diff --git a/konova/migrations/0004_auto_20220209_0839.py b/konova/migrations/0004_auto_20220209_0839.py new file mode 100644 index 00000000..fe41eada --- /dev/null +++ b/konova/migrations/0004_auto_20220209_0839.py @@ -0,0 +1,17 @@ +# Generated by Django 3.1.3 on 2022-02-09 07:39 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('konova', '0003_auto_20220208_1801'), + ] + + operations = [ + migrations.RenameModel( + old_name='SpatialIntersection', + new_name='ParcelIntersection', + ), + ] diff --git a/konova/models/geometry.py b/konova/models/geometry.py index 0a380b48..bec89c39 100644 --- a/konova/models/geometry.py +++ b/konova/models/geometry.py @@ -99,7 +99,7 @@ class Geometry(BaseResource): Returns: """ - from konova.models import Parcel, District + from konova.models import Parcel, District, ParcelIntersection parcel_fetcher = ParcelWFSFetcher( geometry_id=self.id, ) @@ -107,6 +107,7 @@ class Geometry(BaseResource): fetched_parcels = parcel_fetcher.get_features( typename ) + _now = timezone.now() underlying_parcels = [] for result in fetched_parcels: fetched_parcel = result[typename] @@ -125,19 +126,35 @@ class Geometry(BaseResource): krs=fetched_parcel["ave:kreis"], )[0] parcel_obj.district = district - parcel_obj.updated_on = timezone.now() + parcel_obj.updated_on = _now parcel_obj.save() underlying_parcels.append(parcel_obj) + # Update the linked parcels self.parcels.set(underlying_parcels) + # Set the calculated_on intermediate field, so this related data will be found on lookups + intersections_without_ts = self.parcelintersection_set.filter( + parcel__in=self.parcels.all(), + calculated_on__isnull=True, + ) + for entry in intersections_without_ts: + entry.calculated_on = _now + ParcelIntersection.objects.bulk_update( + intersections_without_ts, + ["calculated_on"] + ) + def get_underlying_parcels(self): """ Getter for related parcels and their districts Returns: parcels (QuerySet): The related parcels as queryset """ - parcels = self.parcels.all().prefetch_related( + + parcels = self.parcels.filter( + parcelintersection__calculated_on__isnull=False, + ).prefetch_related( "district" ).order_by( "gmrkng", diff --git a/konova/models/parcel.py b/konova/models/parcel.py index 487225e6..9c887f1a 100644 --- a/konova/models/parcel.py +++ b/konova/models/parcel.py @@ -22,7 +22,7 @@ class Parcel(UuidModel): To avoid conflicts due to german Umlaute, the field names are shortened and vocals are dropped. """ - geometries = models.ManyToManyField("konova.Geometry", related_name="parcels", blank=True) + geometries = models.ManyToManyField("konova.Geometry", blank=True, related_name="parcels", through='ParcelIntersection') district = models.ForeignKey("konova.District", on_delete=models.SET_NULL, null=True, blank=True, related_name="parcels") gmrkng = models.CharField( max_length=1000, @@ -77,3 +77,22 @@ class District(UuidModel): def __str__(self): return f"{self.gmnd} | {self.krs}" + + +class ParcelIntersection(UuidModel): + """ ParcelIntersection is an intermediary model, which is used to configure the + M2M relation between Parcel and Geometry. + + Based on uuids, we will not have (practically) any problems on outrunning primary keys + and extending the model with calculated_on timestamp, we can 'hide' entries while they + are being recalculated and keep track on the last time they have been calculated this + way. + + Please note: The calculated_on describes when the relation between the Parcel and the Geometry + has been established. The updated_on field of Parcel describes when this Parcel has been + changed the last time. + + """ + parcel = models.ForeignKey(Parcel, on_delete=models.CASCADE) + geometry = models.ForeignKey("konova.Geometry", on_delete=models.CASCADE) + calculated_on = models.DateTimeField(auto_now_add=True, null=True, blank=True) diff --git a/konova/tasks.py b/konova/tasks.py index 4c528038..c74a2bd7 100644 --- a/konova/tasks.py +++ b/konova/tasks.py @@ -4,13 +4,19 @@ from celery import shared_task from django.core.exceptions import ObjectDoesNotExist - @shared_task def celery_update_parcels(geometry_id: str, recheck: bool = True): - from konova.models import Geometry + from konova.models import Geometry, ParcelIntersection try: geom = Geometry.objects.get(id=geometry_id) - geom.parcels.clear() + objs = geom.parcelintersection_set.all() + for obj in objs: + obj.calculated_on = None + ParcelIntersection.objects.bulk_update( + objs, + ["calculated_on"] + ) + geom.update_parcels() except ObjectDoesNotExist: if recheck: