Merge pull request '# Mass delete command' (#562) from 561_Mass_delete_command_via_identifier into master

Reviewed-on: #562

api/utils/serializer/serializer.py

@@ -6,6 +6,7 @@ Created on: 24.01.22
 
 """
 import json
+import uuid
 from abc import abstractmethod
 
 from django.contrib.gis import geos
@@ -155,6 +156,9 @@ class AbstractModelAPISerializer:
         if isinstance(geojson, dict):
             geojson = json.dumps(geojson)
         geometry = geos.fromstr(geojson)
+        is_4326 = Geometry.is_valid_4326(geometry)
+        if not is_4326:
+            raise ValueError("Geometry not in EPSG:4326 (WGS84). Unknown spatial reference system.")
         geometry = Geometry.cast_to_rlp_srid(geometry)
         geometry = Geometry.cast_to_multipolygon(geometry)
         return geometry
@@ -171,13 +175,20 @@ class AbstractModelAPISerializer:
         Returns:
 
         """
+        if not id:
+            raise ValueError("No id provided. The id is expected to live in the URL!")
+
         # First if there is an external identifier linked to an internal one, so we can continue with the internal
         try:
             ext_id = ExternalIdentifier.objects.get(external_id=id)
             id = ext_id.internal_id
         except ObjectDoesNotExist:
-            # No external id found - let's hope the given id exists internally
-            pass
+            # Not found as external id - let's check whether this is a valid uuid and therefore
+            # potentially an internal id
+            try:
+                uuid.UUID(id)
+            except ValueError:
+                raise AssertionError(f"'{id}' is neither a known external identifier nor a valid uuid.")
 
         obj = self.model.objects.get(
             id=id,
@@ -202,3 +213,33 @@ class AbstractModelAPISerializer:
             obj (Intervention)
         """
         raise NotImplementedError("Must be implemented in subclasses")
+
+
+    def _geometry_conflicts_to_list(self, geometry) -> list:
+        """ Serializes geometry conflict ids into dict
+
+        Args:
+            geometry (Geometry): The geometry to fetch geometry conflicts from
+
+        Returns:
+            ids (list): Serialized geometry conflicts as dict objects inside a list
+        """
+        ids = []
+        conflict_geometries = geometry.get_conflict_geometries()
+        for geom in conflict_geometries:
+            try:
+                data = geom.get_data_objects(["identifier", "id"])
+                if len(data) == 0:
+                    # expected behaviour in case of deleted data object
+                    continue
+                data = data[0]
+            except KeyError:
+                raise AssertionError(f"Geometry {geom.id} is not attached to any entries. Contact an admin!")
+            ids.append(
+                {
+                    "identifier": data["identifier"],
+                    "id": data["id"],
+                }
+            )
+
+        return ids

api/utils/serializer/v1/serializer.py

@@ -54,6 +54,7 @@ class AbstractModelAPISerializerV1(AbstractModelAPISerializer):
             "created_on": self._created_on_to_json(entry),
             "modified_on": self._modified_on_to_json(entry),
             "external_identifiers": ext_ids,
+            "geometry_conflicts": self._geometry_conflicts_to_list(entry.geometry)
         }
         self._extend_properties_data(entry)
         geo_json["properties"] = self.properties_data

api/views/v1/views.py

@@ -10,6 +10,7 @@ from json import JSONDecodeError
 
 from django.core.exceptions import ObjectDoesNotExist
 from django.http import JsonResponse, HttpRequest
+from django.views.decorators.csrf import csrf_exempt
 
 from api.utils.serializer.v1.compensation import CompensationAPISerializerV1
 from api.utils.serializer.v1.deduction import DeductionAPISerializerV1
@@ -28,6 +29,20 @@ class AbstractAPIViewV1(AbstractAPIView):
         super().__init__(*args, **kwargs)
         self.serializer = self.serializer()
 
+    @csrf_exempt
+    def dispatch(self, request, *args, **kwargs):
+        try:
+            return super().dispatch(request, *args, **kwargs)
+        except (JSONDecodeError,
+                AssertionError,
+                ValueError,
+                PermissionError) as e:
+            return self._return_error_response(e, 400)
+        except ObjectDoesNotExist as e:
+            return self._return_error_response(e, 404)
+        except Exception as e:
+            return self._return_error_response(e, 500)
+
     def get(self, request: HttpRequest, id=None):
         """ Handles the GET request
 
@@ -40,17 +55,12 @@ class AbstractAPIViewV1(AbstractAPIView):
         Returns:
             response (JsonResponse)
         """
-        try:
         self.rpp = int(request.GET.get("rpp", self.rpp))
         self.page_number = int(request.GET.get("p", self.page_number))
-
         self.serializer.rpp = self.rpp
         self.serializer.page_number = self.page_number
-
         self.serializer.prepare_lookup(id, self.user)
         data = self.serializer.fetch_and_serialize()
-        except Exception as e:
-            return self._return_error_response(e, 500)
         return self._return_response(request, data)
 
     def post(self, request: HttpRequest):
@@ -64,16 +74,9 @@ class AbstractAPIViewV1(AbstractAPIView):
         Returns:
             response (JsonResponse)
         """
-        try:
         body = request.body.decode("utf-8")
         body = json.loads(body)
         created_id = self.serializer.create_model_from_json(body, self.user)
-        except (JSONDecodeError,
-                AssertionError,
-                ValueError,
-                PermissionError,
-                ObjectDoesNotExist) as e:
-            return self._return_error_response(e, 400)
         return JsonResponse({"id": created_id})
 
     def put(self, request: HttpRequest, id=None):
@@ -88,12 +91,9 @@ class AbstractAPIViewV1(AbstractAPIView):
         Returns:
             response (JsonResponse)
         """
-        try:
         body = request.body.decode("utf-8")
         body = json.loads(body)
         updated_id = self.serializer.update_model_from_json(id, body, self.user)
-        except Exception as e:
-            return self._return_error_response(e, 500)
         return JsonResponse({"id": updated_id})
 
     def delete(self, request: HttpRequest, id=None):
@@ -107,10 +107,7 @@ class AbstractAPIViewV1(AbstractAPIView):
             response (JsonResponse)
         """
 
-        try:
         success = self.serializer.delete_entry(id, self.user)
-        except Exception as e:
-            return self._return_error_response(e, 500)
         return JsonResponse(
             {
                 "success": success,

compensation/models/compensation.py

@@ -325,8 +325,8 @@ class Compensation(AbstractCompensation, CEFMixin, CoherenceMixin, PikMixin):
             self.identifier = self.generate_new_identifier()
         super().save(*args, **kwargs)
 
-    def mark_as_deleted(self, user, send_mail: bool = True):
-        super().mark_as_deleted(user, send_mail)
+    def mark_as_deleted(self, user, send_mail: bool = True, comment: str|None = None):
+        super().mark_as_deleted(user, send_mail, comment)
         if user is not None:
             self.intervention.mark_as_edited(user, edit_comment=COMPENSATION_REMOVED_TEMPLATE.format(self.identifier))
 

intervention/models/intervention.py

@@ -306,7 +306,7 @@ class Intervention(BaseObject,
             self.set_unchecked()
         return action
 
-    def mark_as_deleted(self, user, send_mail: bool = True):
+    def mark_as_deleted(self, user, send_mail: bool = True, comment: str|None = None):
         """ Extends base mark_as_delete functionality
 
         Removes related deductions from the database, which results in updating the deductable_rest of the
@@ -315,11 +315,12 @@ class Intervention(BaseObject,
         Args:
             user (User): The performing user
             send_mail (bool): Whether to send an info mail
+            comment (str|None): The delete comment
 
         Returns:
 
         """
-        super().mark_as_deleted(user, send_mail)
+        super().mark_as_deleted(user, send_mail, comment)
 
         # Remove pending deductions to free booked capacities
         deductions = self.deductions.all()

konova/management/commands/mass_delete_entries.py

@@ -0,0 +1,113 @@
+"""
+Author: Michel Peltriaux
+Created on: 16.07.26
+
+"""
+from django.db.models import QuerySet
+
+from compensation.models import Compensation, EcoAccount
+from ema.models import Ema
+from intervention.models import Intervention
+from konova.management.commands.setup import BaseKonovaCommand
+from user.models import User
+
+
+class Command(BaseKonovaCommand):
+    help = "Mass-deletes entries according to user input"
+
+    _DELETE_CLS = None
+    _DELETE_COMMENT = None
+    _DELETE_IDENTIFIER_ILIKE_PARAM = None
+    _DELETE_USER = None
+
+    def handle(self, *args, **options):
+        self._get_delete_user()
+        self._get_object_type()
+        self._get_identifier_ilike_param()
+        self._get_delete_comment()
+        self._show_config()
+        entries = self._collect_entries()
+        self._process_delete(entries)
+
+    def _get_delete_user(self):
+        self._DELETE_USER = User.objects.filter(
+            is_superuser=True,
+            is_staff=True
+        ).order_by(
+            "id"
+        ).first()
+
+        if not self._DELETE_USER:
+            self._write_error("No admin-staff user could be found. Please check the admin users!")
+
+    def _get_object_type(self):
+        object_types = {
+            "eiv": Intervention,
+            "kom": Compensation,
+            "oek": EcoAccount,
+            "ema": Ema,
+        }
+        object_types_str = "|".join(object_types.keys())
+        object_type_select = input(f"Which object type shall be processed? ({object_types_str}): ").lower()
+        try:
+            self._DELETE_CLS = object_types[object_type_select]
+            if self._DELETE_CLS == EcoAccount:
+                oek_deduction_remove_understood = input(f"Please be aware that deleting OEKs will result in TOTAL REMOVING of their deductions without any chance to restore them. Do you want to continue? (y|n): ")
+                oek_deduction_remove_understood = oek_deduction_remove_understood.lower() == "y"
+                if not oek_deduction_remove_understood:
+                    self._write_error("Removing of OEK canceled. Abort.")
+                    exit(-1)
+
+        except KeyError:
+            self._write_error(f"'{object_type_select}' not a valid option of {object_types_str}. Abort.")
+            exit(-1)
+
+    def _get_identifier_ilike_param(self):
+        self._DELETE_IDENTIFIER_ILIKE_PARAM = input("Use this substring to search in identifiers: ")
+
+    def _get_delete_comment(self):
+        self._DELETE_COMMENT = input("Comment to store in delete action: ")
+
+    def _show_config(self):
+        assert self._DELETE_CLS is not None
+        assert self._DELETE_IDENTIFIER_ILIKE_PARAM is not None
+        assert self._DELETE_COMMENT is not None
+
+        self._write_warning("You are about to delete entries with:")
+        self._write_warning(f"    Object type: {self._DELETE_CLS.__name__}")
+        self._write_warning(f"    Identifier pattern: '*{self._DELETE_IDENTIFIER_ILIKE_PARAM}*'")
+        self._write_warning(f"    Comment for delete action: '{self._DELETE_COMMENT}'")
+        self._write_warning(f"    Delete performing user: '{self._DELETE_USER}'")
+
+    def _collect_entries(self):
+        return self._DELETE_CLS.objects.filter(
+            deleted__isnull=True,
+            identifier__contains=self._DELETE_IDENTIFIER_ILIKE_PARAM
+        )
+
+    def _process_delete(self, entries: QuerySet):
+        entries_count = entries.count()
+        if entries_count == 0:
+            self._write_success(f"Found {entries_count} matches to delete. Nothing to do here. Bye.")
+            exit(0)
+
+        self._write_warning(f"Found {entries_count} matches to delete. Please double-check whether these samples from the results are what you expected: ")
+        first_three_elements = entries[:3]
+        for element in first_three_elements:
+            self._write_warning(f"    {element.identifier} ({element.id})")
+
+        delete_input = input("Are you ready to delete them now? (y|n): ").lower()
+        delete_now = delete_input == "y"
+        if not delete_now:
+            self._write_error(f"Entered unknown '{delete_input}'. Abort.")
+            exit(-1)
+        else:
+            i = 0
+            for element in entries:
+                if i % 10 == 0:
+                    self._write_warning(f"    Deleted {i}/{entries_count}")
+                element.mark_as_deleted(user=self._DELETE_USER, send_mail=False, comment=self._DELETE_COMMENT)
+                i += 1
+
+            self._write_warning(f"    Deleted {entries_count}/{entries_count}")
+            self._write_success("Entries deleted. Bye.")

konova/models/geometry.py

@@ -113,7 +113,7 @@ class Geometry(BaseResource):
             objs (list): The list of objects
         """
         objs = []
-
+        stop_searching = False
         # Some related data sets can be processed rather easily
         regular_sets = [
             self.intervention_set,
@@ -125,28 +125,44 @@ class Geometry(BaseResource):
                 deleted=None
             )
             if limit_to_attrs:
-                objs += set_objs.values_list(*limit_to_attrs, flat=True)
+                objs += set_objs.values(*limit_to_attrs)
             else:
                 objs += set_objs
 
+            stop_searching = len(objs) > 0
+            if stop_searching:
+                break
+
+        if not stop_searching:
             # ... but we need a special treatment for compensations, since they can be deleted directly OR inherit their
             # de-facto-deleted status from their deleted parent intervention
             comp_objs = self.compensation_set.filter(
-            Q(deleted=None) & Q(intervention__deleted=None)
+                Q(deleted=None) | Q(intervention__deleted=None)
             )
             if limit_to_attrs:
-            objs += comp_objs.values_list(*limit_to_attrs, flat=True)
+                objs += comp_objs.values(*limit_to_attrs)
             else:
                 objs += comp_objs
         return objs
 
-    def get_data_object(self):
+    def get_data_object(self, limit_to_attrs: list = None):
         """
-        Getter for the specific data object which is related to this geometry
+        Getter for the specific data object which is related to this geometry.
+
+        !!! Only returns undeleted entries !!!
+
+        Returns:
+            result (str|None): Returns the desired attributes or None if the data object is marked as deleted
+
         """
-        objs = self.get_data_objects()
+        objs = self.get_data_objects(limit_to_attrs)
         assert (len(objs) <= 1)
+        try:
             result = objs.pop()
+        except IndexError:
+            # If this happens, we just processed a GeometryConflict with an entry which is marked as deleted.
+            # Therefore we return None
+            result = None
         return result
 
     def update_parcels(self):
@@ -436,6 +452,36 @@ class Geometry(BaseResource):
             output_geom.transform(DEFAULT_SRID_RLP)
         return output_geom
 
+    def get_conflict_geometries(self):
+        """ Getter for geometry ids which conflict with this geometry or are conflicted by this one
+
+        Returns:
+            geom_ids (list): List of geometry ids
+        """
+        conflict_geoms_id = GeometryConflict.get_conflict_geometries(self)
+        conflict_geoms = Geometry.objects.filter(id__in=conflict_geoms_id)
+        return conflict_geoms
+
+    @staticmethod
+    def is_valid_4326(geometry):
+        """ Checks whether a given geometry's coordinates are in a valid range to be of EPSG:4326
+
+        Args:
+            geometry: The geometry
+
+        Returns:
+            ret_val (bool): Whether the geometry is valid EPSG:4326
+        """
+        if not geometry.centroid.coords:
+            # No coordinates at all found, therefore technically proper 4326
+            return True
+
+        try:
+            lat,lon = geometry.centroid.coords
+            return (-90.0 <= lat <= 90.0) and (-180.0 <= lon <= 180.0)
+        except IndexError:
+            return False
+
 
 class GeometryConflict(UuidModel):
     """
@@ -459,3 +505,23 @@ class GeometryConflict(UuidModel):
 
     def __str__(self):
         return f"{self.conflicting_geometry.id} conflicts with {self.affected_geometry.id}"
+
+    @staticmethod
+    def get_conflict_geometries(geometry: Geometry):
+        """ Getter for geometries which conflict in one or another way with the given one
+
+        Args:
+            geometry (Geometry): The geometry which shall be checked
+
+        Returns:
+            conflict_geometries (QuerySet): QuerySet of geometries which have conflicts with the given geometry
+        """
+        conflict_geometries = GeometryConflict.objects.filter(
+            affected_geometry=geometry.id,
+        ).values_list("conflicting_geometry__id", flat=True)
+        conflict_geometries = conflict_geometries.union(
+            GeometryConflict.objects.filter(
+                conflicting_geometry=geometry.id,
+            ).values_list("affected_geometry__id", flat=True)
+        )
+        return conflict_geometries

konova/models/object.py

@@ -92,13 +92,15 @@ class DeletableObjectMixin(models.Model):
     class Meta:
         abstract = True
 
-    def mark_as_deleted(self, user, send_mail: bool = True):
+    def mark_as_deleted(self, user, send_mail: bool = True, comment: str|None = None):
         """ Mark an entry as deleted
 
         Does not delete from database but sets a timestamp for being deleted on and which user deleted the object
 
         Args:
             user (User): The performing user
+            send_mail (bool): Whether to send mails to users and teams
+            comment (str|None): The comment for delete action
 
         Returns:
 
@@ -109,7 +111,7 @@ class DeletableObjectMixin(models.Model):
             return
 
         with transaction.atomic():
-            action = UserActionLogEntry.get_deleted_action(user)
+            action = UserActionLogEntry.get_deleted_action(user, comment)
             self.deleted = action
             self.log.add(action)
 
@@ -676,24 +678,17 @@ class GeoReferencedMixin(models.Model):
         if self.geometry is None:
             return request
 
-        instance_objs = []
-        needed_data_object_attrs = [
-            "identifier"
-        ]
-        conflicts = self.geometry.conflicts_geometries.iterator()
+        conflicting_geometries = self.geometry.get_conflict_geometries()
+        data_object_identifiers = []
+        for conflicting_geom in conflicting_geometries:
+            data_obj_id = conflicting_geom.get_data_object(["identifier"])
+            if data_obj_id:
+                data_object_identifiers.append(data_obj_id)
 
-        for conflict in conflicts:
-            # Only check the affected geometry of this conflict, since we know the conflicting geometry is self.geometry
-            instance_objs += conflict.affected_geometry.get_data_objects(needed_data_object_attrs)
-
-        conflicts = self.geometry.conflicted_by_geometries.iterator()
-        for conflict in conflicts:
-            # Only check the conflicting geometry of this conflict, since we know the affected geometry is self.geometry
-            instance_objs += conflict.conflicting_geometry.get_data_objects(needed_data_object_attrs)
-
-        add_message = len(instance_objs) > 0
+        add_message = len(data_object_identifiers) > 0
         if add_message:
-            instance_identifiers = ", ".join(instance_objs)
+            data_object_identifiers = [x["identifier"] for x in data_object_identifiers]
+            instance_identifiers = ", ".join(data_object_identifiers)
             message_str = GEOMETRY_CONFLICT_WITH_TEMPLATE.format(instance_identifiers)
             messages.info(request, message_str)
         return request

konova/tests/test_geometries.py

@@ -85,12 +85,6 @@ class GeometryTestCase(BaseTestCase):
         )
 
     def test_get_data_objects(self):
-        num_objs_with_geom = 0
-        self.assertEqual(
-            len(self.geom_1.get_data_objects()),
-            num_objs_with_geom
-        )
-
         objs = [
             self.intervention,
             self.compensation,
@@ -98,16 +92,13 @@ class GeometryTestCase(BaseTestCase):
             self.ema,
         ]
         for obj in objs:
-            obj.geometry = self.geom_1
+            if not obj.geometry:
+                obj.geometry = Geometry.objects.create(geom=self.create_dummy_geometry())
                 obj.save()
 
-            num_objs_with_geom += 1
-            geom_objs = self.geom_1.get_data_objects()
-            self.assertEqual(
-                len(geom_objs),
-                num_objs_with_geom
-            )
-            self.assertIn(obj, geom_objs)
+            data_objs = obj.geometry.get_data_objects()
+            self.assertEqual(len(data_objs), 1)
+            self.assertIn(obj, data_objs)
 
     def test_as_feature_collection(self):
         geometry = self.geom_1.geom

templates/map/client/config.json

@@ -56,11 +56,9 @@
 	],
 	"layers":
 	[
-		{ "id": "webatlas_farbe", "folder": "bg", "type": "WMS", "order": -1, "title": "WebatlasRP farbig", "attribution": "LVermGeo", "url": "https://maps.service24.rlp.de/gisserver/services/RP/RP_WebAtlasRP/MapServer/WmsServer?", "name": "RP_WebAtlasRP", "active": true},
-		{ "id": "webatlas_grau", "folder": "bg", "type": "WMS", "order": -1, "title": "WebatlasRP grau", "attribution": "LVermGeo", "url": "https://maps.service24.rlp.de/gisserver/services/RP/RP_ETRS_Gt/MapServer/WmsServer?", "name": "0", "active": false },
 		{ "id": "luftbilder", "folder": "bg", "type": "WMS", "order": -1, "title": "Luftbilder", "attribution": "LVermGeo", "url": "https://geo4.service24.rlp.de/wms/rp_dop20.fcgi?", "name": "rp_dop20", "active": false },
 		{ "id": "basemap_farbe", "folder": "bg", "type": "WMS", "order": -1, "title": "BasemapDE farbig", "attribution": "BKG", "url": "https://sgx.geodatenzentrum.de/wms_basemapde?", "name": "de_basemapde_web_raster_farbe", "active": false },
-		{ "id": "basemap_grau", "folder": "bg", "type": "WMS", "order": -1, "title": "BasemapDE grau", "attribution": "BKG", "url": "https://sgx.geodatenzentrum.de/wms_basemapde?", "name": "de_basemapde_web_raster_grau", "active": false },
+		{ "id": "basemap_grau", "folder": "bg", "type": "WMS", "order": -1, "title": "BasemapDE grau", "attribution": "BKG", "url": "https://sgx.geodatenzentrum.de/wms_basemapde?", "name": "de_basemapde_web_raster_grau", "active": true },
 		{ "id": "dtk_farbe", "folder": "bg", "type": "WMS", "order": -1, "title": "DTK5 farbig", "attribution": "LVermGeo", "url": "https://geo4.service24.rlp.de/wms/dtk5_rp.fcgi?", "name": "rp_dtk5", "active": false },
 		{ "id": "dtk_grau", "folder": "bg", "type": "WMS", "order": -1, "title": "DTK5 grau", "attribution": "LVermGeo", "url": "https://geo4.service24.rlp.de/wms/dtk5_rp.fcgi?", "name": "rp_dtk5_grau", "active": false },