Merge pull request '# Bugfix: Recalculate_parcels command' (#448) from bugfixing into master

Reviewed-on: #448

codelist/migrations/0003_alter_konovacode_long_name_and_more.py

@@ -0,0 +1,25 @@
+# Generated by Django 5.0.8 on 2024-08-26 16:47
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('codelist', '0002_migrate_975_to_288'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='konovacode',
+            name='long_name',
+            field=models.CharField(blank=True, default="", max_length=1000),
+            preserve_default=False,
+        ),
+        migrations.AlterField(
+            model_name='konovacode',
+            name='short_name',
+            field=models.CharField(blank=True, default="", help_text='Short version of long name', max_length=500),
+            preserve_default=False,
+        ),
+    ]

compensation/migrations/0016_alter_compensationstate_biotope_type_details.py

@@ -0,0 +1,19 @@
+# Generated by Django 5.0.8 on 2024-08-26 16:47
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('codelist', '0003_alter_konovacode_long_name_and_more'),
+        ('compensation', '0015_alter_compensation_after_states_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='compensationstate',
+            name='biotope_type_details',
+            field=models.ManyToManyField(blank=True, limit_choices_to={'code_lists__in': [288], 'is_archived': False, 'is_selectable': True}, related_name='+', to='codelist.konovacode'),
+        ),
+    ]

konova/management/commands/generate_report.py

@@ -6,11 +6,11 @@ Created on: 26.10.22
 
 """
 import zipfile
+from datetime import datetime
 from io import BytesIO
 
 from django.core.mail import EmailMessage
 from django.utils import timezone
-from django.utils.datetime_safe import datetime
 
 from analysis.utils.excel.excel import TempExcelFile
 from analysis.utils.report import TimespanReport

konova/management/commands/recalculate_parcels.py

@@ -0,0 +1,86 @@
+"""
+Author: Michel Peltriaux
+Organization: Struktur- und Genehmigungsdirektion Nord, Rhineland-Palatinate, Germany
+Contact: michel.peltriaux@sgdnord.rlp.de
+Created on: 04.01.22
+
+"""
+import datetime
+
+from django.contrib.gis.db.models.functions import Area
+
+from konova.management.commands.setup import BaseKonovaCommand
+from konova.models import Geometry, ParcelIntersection
+
+
+class Command(BaseKonovaCommand):
+    help = "Recalculates parcels for entries with geometry but missing parcel information"
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--force-all",
+            action="store_true",
+            default=False,
+            help="If Attribute set, all entries parcels will be recalculated"
+        )
+
+    def handle(self, *args, **options):
+        try:
+            self.recalculate_parcels(options)
+        except KeyboardInterrupt:
+            self._break_line()
+            exit(-1)
+
+    def recalculate_parcels(self, options: dict):
+        force_all = options.get("force_all", False)
+
+        geometry_objects = Geometry.objects.all().exclude(
+            geom=None
+        )
+
+        if not force_all:
+            # Fetch all intersections
+            intersection_objs = ParcelIntersection.objects.filter(
+                geometry__in=geometry_objects
+            )
+            # Just take the geometry ids, which seem to have intersections
+            geom_with_intersection_ids = intersection_objs.values_list(
+                "geometry__id",
+                flat=True
+            )
+            # ... and resolve into Geometry objects again ...
+            intersected_geom_objs = Geometry.objects.filter(
+                id__in=geom_with_intersection_ids
+            )
+            # ... to be able to use the way more efficient difference() function ...
+            geometry_objects_ids = geometry_objects.difference(intersected_geom_objs).values_list("id", flat=True)
+            # ... so we can resolve these into proper Geometry objects again for further annotation usage
+            geometry_objects = Geometry.objects.filter(id__in=geometry_objects_ids)
+
+        self._write_warning("=== Update parcels and districts ===")
+        # Order geometries by size to process smaller once at first
+        geometries = geometry_objects.annotate(
+            area=Area("geom")
+        ).order_by(
+            'area'
+        )
+        self._write_warning(f"Process parcels for {geometries.count()} geometry entries now ...")
+        i = 0
+        num_geoms = geometries.count()
+        geoms_with_errors = {}
+        for geometry in geometries:
+            self._write_warning(f"--- {datetime.datetime.now()} Process {geometry.id} now ...")
+            try:
+                geometry.update_parcels()
+                self._write_warning(f"--- Processed {geometry.get_underlying_parcels().count()} underlying parcels")
+            except Exception as e:
+                geoms_with_errors[geometry.id] = str(e)
+            i += 1
+            self._write_warning(f"--- {i}/{num_geoms} processed")
+
+        self._write_success("Updating parcels done!")
+
+        for key, val in geoms_with_errors.items():
+            self._write_error(f"    Error on {key}: {val}")
+        self._write_success(f"{num_geoms - len(geoms_with_errors)} geometries successfuly recalculated!")
+        self._break_line()

konova/management/commands/update_all_parcels.py

@@ -1,54 +0,0 @@
-"""
-Author: Michel Peltriaux
-Organization: Struktur- und Genehmigungsdirektion Nord, Rhineland-Palatinate, Germany
-Contact: michel.peltriaux@sgdnord.rlp.de
-Created on: 04.01.22
-
-"""
-import datetime
-
-from django.contrib.gis.db.models.functions import Area
-
-from konova.management.commands.setup import BaseKonovaCommand
-from konova.models import Geometry, Parcel, District
-
-
-class Command(BaseKonovaCommand):
-    help = "Checks the database' sanity and removes unused entries"
-
-    def handle(self, *args, **options):
-        try:
-            self.update_all_parcels()
-        except KeyboardInterrupt:
-            self._break_line()
-            exit(-1)
-
-    def update_all_parcels(self):
-        num_parcels_before = Parcel.objects.count()
-        num_districts_before = District.objects.count()
-        self._write_warning("=== Update parcels and districts ===")
-        # Order geometries by size to process smaller once at first
-        geometries = Geometry.objects.all().exclude(
-            geom=None
-        ).annotate(area=Area("geom")).order_by(
-            'area'
-        )
-        self._write_warning(f"Process parcels for {geometries.count()} geometry entries now ...")
-        i = 0
-        num_geoms = geometries.count()
-        for geometry in geometries:
-            self._write_warning(f"--- {datetime.datetime.now()} Process {geometry.id} now ...")
-            geometry.update_parcels()
-            self._write_warning(f"--- Processed {geometry.get_underlying_parcels().count()} underlying parcels")
-            i += 1
-            self._write_warning(f"--- {i}/{num_geoms} processed")
-
-        num_parcels_after = Parcel.objects.count()
-        num_districts_after = District.objects.count()
-        if num_parcels_after != num_parcels_before:
-            self._write_error(f"Parcels have changed: {num_parcels_before} to {num_parcels_after} entries. You should run the sanitize command.")
-        if num_districts_after != num_districts_before:
-            self._write_error(f"Districts have changed: {num_districts_before} to {num_districts_after} entries. You should run the sanitize command.")
-
-        self._write_success("Updating parcels done!")
-        self._break_line()

konova/models/geometry.py

@@ -8,7 +8,7 @@ Created on: 15.11.21
 import json
 
 from django.contrib.gis.db.models import MultiPolygonField
-from django.core.exceptions import ObjectDoesNotExist
+from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
 from django.db import models, transaction
 from django.utils import timezone
 
@@ -223,6 +223,17 @@ class Geometry(BaseResource):
                 )
                 parcel_obj.updated_on = _now
                 parcels_to_update.append(parcel_obj)
+            except MultipleObjectsReturned:
+                parcel_obj = Parcel.make_unique(
+                    district=district,
+                    municipal=municipal,
+                    parcel_group=parcel_group,
+                    flr=flr_val,
+                    flrstck_nnr=flrstck_nnr,
+                    flrstck_zhlr=flrstck_zhlr,
+                )
+                parcel_obj.updated_on = _now
+                parcels_to_update.append(parcel_obj)
             except ObjectDoesNotExist:
                 # If not existing, create object but do not commit, yet
                 parcel_obj = Parcel(
@@ -366,11 +377,10 @@ class Geometry(BaseResource):
         diff = geom_envelope - self.geom
 
         if diff.area == 0:
-            ratio = 1
+            complexity_factor = 1
         else:
-            ratio = self.geom.area / diff.area
+            complexity_factor = self.geom.area / diff.area
 
-        complexity_factor = 1 - ratio
         return complexity_factor
 
 

konova/models/parcel.py

@@ -5,7 +5,7 @@ Contact: michel.peltriaux@sgdnord.rlp.de
 Created on: 16.12.21
 
 """
-from django.db import models
+from django.db import models, transaction
 
 from konova.models import UuidModel
 
@@ -158,6 +158,46 @@ class Parcel(UuidModel):
     def __str__(self):
         return f"{self.parcel_group} | {self.flr} | {self.flrstck_zhlr} | {self.flrstck_nnr}"
 
+    @classmethod
+    def make_unique(cls, **kwargs):
+        """ Checks for duplicates of a Parcel, choose a (now) unique one,
+        repairs relations for ParcelIntersection and removes duplicates.
+
+        Args:
+            **kwargs ():
+
+        Returns:
+            unique_true (Parcel): The new unique 'true one'
+        """
+        parcel_objs = Parcel.objects.filter(**kwargs)
+
+        if not parcel_objs.exists():
+            return None
+
+        # Get one of the found parcels and use it as new 'true one'
+        unique_parcel = parcel_objs.first()
+        # separate it from the rest
+        parcel_objs = parcel_objs.exclude(id=unique_parcel.id)
+
+        if not parcel_objs.exists():
+            # There are no duplicates - all good, just return
+            return unique_parcel
+
+        # Fetch existing intersections, which still point on the duplicated parcels
+        intersection_objs = ParcelIntersection.objects.filter(
+            parcel__in=parcel_objs
+        )
+
+        # Change each intersection, so they point on the 'true one' parcel from now on
+        for intersection in intersection_objs:
+            intersection.parcel = unique_parcel
+            intersection.save()
+
+        # Remove the duplicated parcels
+        parcel_objs.delete()
+
+        return unique_parcel
+
 
 class ParcelIntersection(UuidModel):
     """

konova/utils/schneider/fetcher.py

@@ -55,11 +55,11 @@ class ParcelFetcher:
             content = json.loads(response.content.decode("utf-8"))
         except JSONDecodeError:
             content = {}
-        next = content.get("next", None)
+        _next = content.get("next", None)
         fetched_parcels = content.get("results", [])
         self.results += fetched_parcels
 
-        if next:
+        if _next:
-            self.get_parcels(next)
+            self.get_parcels(_next)
 
         return self.results

user/views/propagate.py

@@ -35,9 +35,9 @@ class PropagateUserView(View):
     def post(self, request: HttpRequest, *args, **kwargs):
         # Decrypt
         encrypted_body = request.body
-        hash = hashlib.md5()
+        _hash = hashlib.md5()
-        hash.update(OAUTH_CLIENT_ID.encode("utf-8"))
+        _hash.update(OAUTH_CLIENT_ID.encode("utf-8"))
-        key = base64.urlsafe_b64encode(hash.hexdigest().encode("utf-8"))
+        key = base64.urlsafe_b64encode(_hash.hexdigest().encode("utf-8"))
         fernet = Fernet(key)
         body = fernet.decrypt(encrypted_body).decode("utf-8")
         body = json.loads(body)