Merge pull request '# Improve exception reporting for API' (#515) from improve_exception_reporting into master

Reviewed-on: #515

Dockerfile

@@ -1,36 +0,0 @@
-# Nutze ein schlankes Python-Image
-FROM python:3.11-slim-bullseye
-
-ENV PYTHONUNBUFFERED 1
-
-WORKDIR /konova
-
-# Installiere System-Abhängigkeiten
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    gdal-bin redis-server nginx \
-    && rm -rf /var/lib/apt/lists/*  # Platz sparen
-
-# Erstelle benötigte Verzeichnisse & setze Berechtigungen
-RUN mkdir -p /var/log/nginx /var/log/gunicorn /var/lib/nginx /tmp/nginx_client_body \
-    && touch /var/log/nginx/access.log /var/log/nginx/error.log \
-    && chown -R root:root /var/log/nginx /var/lib/nginx /tmp/nginx_client_body
-
-# Kopiere und installiere Python-Abhängigkeiten
-COPY ./requirements.txt /konova/
-RUN pip install --upgrade pip && pip install --no-cache-dir -r requirements.txt
-
-# Entferne Standard-Nginx-Site und ersetze sie durch eigene Config
-RUN rm -rf /etc/nginx/sites-enabled/default
-COPY ./nginx.conf /etc/nginx/conf.d
-
-# Kopiere restliche Projektdateien
-COPY . /konova/
-
-# Sammle statische Dateien
-RUN python manage.py collectstatic --noinput
-
-# Exponiere Ports
-#EXPOSE 80 6379 8000
-
-# Setze Entrypoint
-ENTRYPOINT ["/konova/docker-entrypoint.sh"]

README.md

@@ -4,7 +4,6 @@ the database postgresql and the css library bootstrap as well as the icon packag
 fontawesome for a modern look, following best practices from the industry.
 
 ## Background processes
-### !!! For non-docker run
 Konova uses celery for background processing. To start the worker you need to run
 ```shell
 $ celery -A konova worker -l INFO
@@ -19,58 +18,3 @@ Technical documention is provided in the projects git wiki.
 A user documentation is not available (and not needed, yet). 
 
 
-# Docker 
-To run the docker-compose as expected, you need to take the following steps:
-
-1. Create a database containing docker, using an appropriate Dockerfile, e.g. the following
-```
-version: '3.3'
-services:
-  postgis:
-    image: postgis/postgis
-    restart: always
-    container_name: postgis-docker
-    ports: 
-      - 5433:5432
-    volumes:
-      - db-volume:/var/lib/postgresql/data
-    environment:
-      - POSTGRES_PASSWORD=postgres
-      - POSTGRES_USER=postgres
-    networks:
-      - db-network-bridge
-
-networks:
-  db-network-bridge:
-    driver: "bridge"
-
-volumes:
-  db-volume:
-```
-This Dockerfile creates a Docker container running postgresql and postgis, creates the default superuser postgres, 
-creates a named volume for persisting the database and creates a new network bridge, which **must be used by any other
-container, which wants to write/read on this database**. 
-
-2. Make sure the name of the network bridge above matches the network in the konova docker-compose.yml  
-3. Get into the running postgis container (`docker exec -it postgis-docker bash`) and create new databases, users and so on. Make sure the database `konova` exists now!
-4. Replace all `CHANGE_ME_xy` values inside of konova/docker-compose.yml for your installation. Make sure the `SSO_HOST` holds the proper SSO host, e.g. for the arnova project `arnova.example.org` (Arnova must be installed and the webserver configured as well, of course)
-5. Take a look on konova/settings.py and konova/sub_settings/django_settings.py. Again: Replace all occurences of `CHANGE_ME` with proper values for your installation.
-   1. Make sure you have the proper host strings added to `ALLOWED_HOSTS` inside of django_settings.py.  
-6. Build and run the docker setup using `docker-compose build` and `docker-compose start` from the main directory of this project (where the docker-compose.yml lives)
-7. Run migrations! To do so, get into the konova service container (`docker exec -it konova-docker bash`) and run the needed commands (`python manage.py makemigrations LIST_OF_ALL_MIGRATABLE_APPS`, then `python manage.py migrate`)
-8. Run the setup command `python manage.py setup` and follow the instructions on the CLI
-9. To enable **SMTP** mail support, make sure your host machine (the one where the docker container run) has the postfix service configured properly. Make sure the `mynetworks` variable is xtended using the docker network bridge ip, created in the postgis container and used by the konova services.
-   1. **Hint**: You can find out this easily by trying to perform a test mail in the running konova web application (which will fail, of course). Then take a look to the latest entries in `/var/log/mail.log` on your host machine. The failed IP will be displayed there.
-   2. **Please note**: This installation guide is based on SMTP using postfix! 
-   3. Restart the postfix service on your host machine to reload the new configuration (`service postfix restart`)
-10. Finally, make sure your host machine webserver passes incoming requests properly to the docker nginx webserver of konova. A proper nginx config for the host machine may look like this:
-```
-server {
-    server_name konova.domain.org;
-
-    location / {
-        proxy_pass http://localhost:KONOVA_NGINX_DOCKER_PORT/;
-        proxy_set_header Host $host;
-    }
-}
-```

api/tests/v1/create/test_api_create.py

@@ -71,7 +71,7 @@ class APIV1CreateTestCase(BaseAPIV1TestCase):
         # Expect this first request to fail, since user has no shared access on the intervention, we want to create
         # a compensation for
         response = self._run_create_request(url, post_body)
-        self.assertEqual(response.status_code, 500, msg=response.content)
+        self.assertEqual(response.status_code, 400, msg=response.content)
         content = json.loads(response.content)
         self.assertGreater(len(content.get("errors", [])), 0, msg=response.content)
 

api/views/v1/views.py

@@ -6,7 +6,9 @@ Created on: 21.01.22
 
 """
 import json
+from json import JSONDecodeError
 
+from django.core.exceptions import ObjectDoesNotExist
 from django.http import JsonResponse, HttpRequest
 
 from api.utils.serializer.v1.compensation import CompensationAPISerializerV1
@@ -66,8 +68,12 @@ class AbstractAPIViewV1(AbstractAPIView):
             body = request.body.decode("utf-8")
             body = json.loads(body)
             created_id = self.serializer.create_model_from_json(body, self.user)
-        except Exception as e:
-            return self._return_error_response(e, 500)
+        except (JSONDecodeError,
+                AssertionError,
+                ValueError,
+                PermissionError,
+                ObjectDoesNotExist) as e:
+            return self._return_error_response(e, 400)
         return JsonResponse({"id": created_id})
 
     def put(self, request: HttpRequest, id=None):

api/views/views.py

@@ -81,9 +81,7 @@ class AbstractAPIView(View):
         Returns:
 
         """
-        content = [error.__str__()]
-        if hasattr(error, "messages"):
-            content = error.messages
+        content = [f"{error.__class__.__name__}: {str(error)}"]
         return JsonResponse(
             {
                 "errors": content

compensation/templates/compensation/detail/eco_account/includes/deductions.html

@@ -53,7 +53,7 @@
                 </td>
                 <td class="align-middle">
                     {% if deduction.intervention.recorded %}
-                        <em title="{% trans 'Recorded on' %} {{obj.recorded.timestamp}} {% trans 'by' %} {{obj.recorded.user}}" class='fas fa-bookmark registered-bookmark'></em>
+                        <em title="{% trans 'Recorded on' %} {{deduction.intervention.recorded.timestamp}} {% trans 'by' %} {{deduction.intervention.recorded.user}}" class='fas fa-bookmark registered-bookmark'></em>
                     {% else %}
                         <em title="{% trans 'Not recorded yet' %}" class='far fa-bookmark'></em>
                     {% endif %}

docker-compose.yml

@@ -1,23 +0,0 @@
-version: '3.3'
-
-services:
-  konova:
-    external_links:
-      - postgis:db
-      - arnova-nginx-server:arnova
-    build: .
-    image: "ksp/konova:1.8"
-    container_name: "konova-docker"
-    command: ./docker-entrypoint.sh
-    restart: always
-    volumes:
-      - /data/apps/konova/uploaded_files:/konova_uploaded_files
-    ports:
-      - "1337:80"
-
-# Instead of an own, new network, we need to connect to the existing one, which is provided by the postgis container
-# NOTE: THIS NETWORK MUST EXIST
-networks:
-  default:
-    external:
-      name: postgis_nat_it_backend

docker-entrypoint.sh

@@ -1,27 +0,0 @@
-#!/bin/bash
-
-set -e  # Beende Skript bei Fehlern
-set -o pipefail  # Fehler in Pipelines nicht ignorieren
-
-# Starte Redis
-redis-server --daemonize yes
-
-# Starte Celery Worker im Hintergrund
-celery -A konova worker --loglevel=info &
-
-# Starte Nginx als Hintergrundprozess
-nginx -g "daemon off;" &
-
-# Setze Gunicorn Worker-Anzahl (Standard: (2*CPUs)+1)
-WORKERS=${GUNICORN_WORKERS:-$((2 * $(nproc) + 1))}
-
-# Stelle sicher, dass Logs existieren
-mkdir -p /var/log/gunicorn
-touch /var/log/gunicorn/access.log /var/log/gunicorn/error.log
-
-# Starte Gunicorn als Hauptprozess
-exec gunicorn --workers="$WORKERS" konova.wsgi:application \
-    --bind=0.0.0.0:8000 \
-    --access-logfile /var/log/gunicorn/access.log \
-    --error-logfile /var/log/gunicorn/error.log \
-    --access-logformat '%({x-real-ip}i)s via %(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"'

konova/forms/geometry_form.py

@@ -35,6 +35,7 @@ class SimpleGeomForm(BaseForm):
         disabled=False,
     )
     _num_geometries_ignored: int = 0
+    empty = False
 
     def __init__(self, *args, **kwargs):
         self.read_only = kwargs.pop("read_only", True)
@@ -49,11 +50,11 @@ class SimpleGeomForm(BaseForm):
                 raise AttributeError
 
             geojson = self.instance.geometry.as_feature_collection(srid=DEFAULT_SRID_RLP)
-            self._set_geojson_properties(geojson, title=self.instance.identifier or None)
+            geojson = self._set_geojson_properties(geojson, title=self.instance.identifier or None)
             geom = json.dumps(geojson)
         except AttributeError:
             # If no geometry exists for this form, we simply set the value to None and zoom to the maximum level
-            geom = ""
+            geom = json.dumps({})
             self.empty = True
 
         self.initialize_form_field("output", geom)
@@ -62,17 +63,17 @@ class SimpleGeomForm(BaseForm):
         super().is_valid()
         is_valid = True
 
-        # Get geojson from form
-        geom = self.data.get("output", None)
-        if geom is None or len(geom) == 0:
-            # empty geometry is a valid geometry
-            self.cleaned_data["output"] = MultiPolygon(srid=DEFAULT_SRID_RLP).ewkt
-            return is_valid
-        geom = json.loads(geom)
+        # Make sure invalid geometry is properly rendered again to the user
+        # Therefore: write submitted data back into form field
+        # (does not matter whether we know if it is valid or invalid)
+        submitted_data = self.data["output"]
+        submitted_data = json.loads(submitted_data)
+        submitted_data = self._set_geojson_properties(submitted_data)
+        self.initialize_form_field("output", json.dumps(submitted_data))
 
-        # Write submitted data back into form field to make sure invalid geometry
-        # will be rendered again on failed submit
-        self.initialize_form_field("output", self.data["output"])
+        # Get geojson from form for validity checking
+        geom = self.data.get("output", json.dumps({}))
+        geom = json.loads(geom)
 
         # Initialize features list with empty MultiPolygon, so that an empty input will result in a
         # proper empty MultiPolygon object
@@ -84,20 +85,23 @@ class SimpleGeomForm(BaseForm):
             "MultiPolygon",
             "MultiPolygon25D",
         ]
+        # Check validity for each feature of the geometry
         for feature in features_json:
             feature_geom = feature.get("geometry", feature)
             if feature_geom is None:
                 # Fallback for rare cases where a feature does not contain any geometry
                 continue
 
+            # Try to create a geometry object from the single feature
             feature_geom = json.dumps(feature_geom)
             g = gdal.OGRGeometry(feature_geom, srs=DEFAULT_SRID_RLP)
 
-            flatten_geometry = g.coord_dim > 2
-            if flatten_geometry:
+            geometry_has_unwanted_dimensions = g.coord_dim > 2
+            if geometry_has_unwanted_dimensions:
                 g = self.__flatten_geom_to_2D(g)
 
-            if g.geom_type not in accepted_ogr_types:
+            geometry_type_is_accepted = g.geom_type not in accepted_ogr_types
+            if geometry_type_is_accepted:
                 self.add_error("output", _("Only surfaces allowed. Points or lines must be buffered."))
                 is_valid &= False
                 return is_valid
@@ -109,27 +113,33 @@ class SimpleGeomForm(BaseForm):
                 self._num_geometries_ignored += 1
                 continue
 
+            # Whatever this geometry object is -> try to create a Polygon from it
+            # The resulting polygon object automatically detects whether a valid polygon has been created or not
             g = Polygon.from_ewkt(g.ewkt)
             is_valid &= g.valid
             if not g.valid:
                 self.add_error("output", g.valid_reason)
                 return is_valid
 
+            # If the resulting polygon is just a single polygon, we add it to the list of properly casted features
             if isinstance(g, Polygon):
                 features.append(g)
             elif isinstance(g, MultiPolygon):
+                # The resulting polygon could be of type MultiPolygon (due to multiple surfaces)
+                # If so, we extract all polygons from the MultiPolygon and extend the casted features list
                 features.extend(list(g))
 
-        # Unionize all geometry features into one new MultiPolygon
+        # Unionize all polygon features into one new MultiPolygon
         if features:
             form_geom = MultiPolygon(*features, srid=DEFAULT_SRID_RLP).unary_union
         else:
+            # If no features have been processed, this indicates an empty geometry - so we store an empty geometry
             form_geom = MultiPolygon(srid=DEFAULT_SRID_RLP)
 
         # Make sure to convert into a MultiPolygon. Relevant if a single Polygon is provided.
         form_geom = Geometry.cast_to_multipolygon(form_geom)
 
-        # Write unioned Multipolygon into cleaned data
+        # Write unionized Multipolygon back into cleaned data
         if self.cleaned_data is None:
             self.cleaned_data = {}
         self.cleaned_data["output"] = form_geom.ewkt
@@ -252,6 +262,8 @@ class SimpleGeomForm(BaseForm):
         """
         features = geojson.get("features", [])
         for feature in features:
+            if not feature.get("properties", None):
+                feature["properties"] = {}
             feature["properties"]["editable"] = not self.read_only
             if title:
                 feature["properties"]["title"] = title

konova/models/geometry.py

@@ -10,6 +10,7 @@ import json
 from django.contrib.gis.db.models import MultiPolygonField
 from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
 from django.db import models, transaction
+from django.db.models import Q
 from django.utils import timezone
 from django.contrib.gis.geos import MultiPolygon
 
@@ -109,17 +110,26 @@ class Geometry(BaseResource):
             objs (list): The list of objects
         """
         objs = []
-        sets = [
+
+        # Some related data sets can be processed rather easily
+        regular_sets = [
             self.intervention_set,
-            self.compensation_set,
             self.ema_set,
             self.ecoaccount_set,
         ]
-        for _set in sets:
+        for _set in regular_sets:
             set_objs = _set.filter(
                 deleted=None
             )
             objs += set_objs
+
+        # ... but we need a special treatment for compensations, since they can be deleted directly OR inherit their
+        # de-facto-deleted status from their deleted parent intervention
+        comp_objs = self.compensation_set.filter(
+            Q(deleted=None) & Q(intervention__deleted=None)
+        )
+        objs += comp_objs
+
         return objs
 
     def get_data_object(self):
@@ -397,7 +407,10 @@ class Geometry(BaseResource):
         """
         output_geom = input_geom
         if not isinstance(input_geom, MultiPolygon):
-            output_geom = MultiPolygon(input_geom, srid=DEFAULT_SRID_RLP)
+            try:
+                output_geom = MultiPolygon(input_geom, srid=DEFAULT_SRID_RLP)
+            except TypeError as e:
+                raise AssertionError(f"Only (Multi)Polygon allowed! Could not convert {input_geom.geom_type} to MultiPolygon")
         return output_geom
 
     @staticmethod

konova/models/object.py

@@ -677,12 +677,12 @@ class GeoReferencedMixin(models.Model):
             return request
 
         instance_objs = []
-        conflicts = self.geometry.conflicts_geometries.all()
+        conflicts = self.geometry.conflicts_geometries.iterator()
 
         for conflict in conflicts:
             instance_objs += conflict.affected_geometry.get_data_objects()
 
-        conflicts = self.geometry.conflicted_by_geometries.all()
+        conflicts = self.geometry.conflicted_by_geometries.iterator()
         for conflict in conflicts:
             instance_objs += conflict.conflicting_geometry.get_data_objects()
 

konova/sub_settings/context_settings.py

@@ -11,4 +11,4 @@ BASE_TITLE = "KSP - Kompensationsverzeichnis Service Portal"
 BASE_FRONTEND_TITLE = "Kompensationsverzeichnis Service Portal"
 TAB_TITLE_IDENTIFIER = "tab_title"
 HELP_LINK = "https://dienste.naturschutz.rlp.de/doku/doku.php?id=ksp2:start"
-IMPRESSUM_LINK = "https://naturschutz.rlp.de/index.php?q=impressum"
+IMPRESSUM_LINK = "https://naturschutz.rlp.de/ueber-uns/impressum"

konova/sub_settings/django_settings.py

@@ -191,10 +191,11 @@ STATICFILES_DIRS = [
 ]
 
 # EMAIL (see https://docs.djangoproject.com/en/dev/topics/email/)
+
+# CHANGE_ME !!! ONLY FOR DEVELOPMENT !!!
 if DEBUG:
-    # ONLY FOR DEVELOPMENT NEEDED
     EMAIL_BACKEND = 'django.core.mail.backends.filebased.EmailBackend'
-    EMAIL_FILE_PATH = '/tmp/app-messages'
+    EMAIL_FILE_PATH = '/tmp/app-messages' # change this to a proper location
 
 DEFAULT_FROM_EMAIL = env("DEFAULT_FROM_EMAIL")  # The default email address for the 'from' element
 SERVER_EMAIL = DEFAULT_FROM_EMAIL  # The default email sender address, which is used by Django to send errors via mail

konova/utils/exception_reporter.py

@@ -5,6 +5,9 @@ Contact: ksp-servicestelle@sgdnord.rlp.de
 Created on: 11.12.23
 
 """
+import json
+from json import JSONDecodeError
+
 from django.views.debug import ExceptionReporter
 
 
@@ -30,7 +33,7 @@ class KonovaExceptionReporter(ExceptionReporter):
         """
         whitelist = [
             "is_email",
-            "unicdoe_hint",
+            "unicode_hint",
             "frames",
             "request",
             "user_str",
@@ -39,6 +42,8 @@ class KonovaExceptionReporter(ExceptionReporter):
             "raising_view_name",
             "exception_type",
             "exception_value",
+            "filtered_GET_items",
+            "filtered_POST_items",
         ]
         clean_data = dict()
         for entry in whitelist:
@@ -56,7 +61,28 @@ class KonovaExceptionReporter(ExceptionReporter):
         """
         tb_data = super().get_traceback_data()
 
+        return_data = tb_data
         if self.is_email:
-            tb_data = self._filter_traceback_data(tb_data)
+            filtered_data = dict()
+            filtered_data.update(self._filter_traceback_data(tb_data))
+            filtered_data.update(self._filter_POST_body(tb_data))
+            return_data = filtered_data
+        return return_data
 
-        return tb_data
+    def _filter_POST_body(self, tb_data: dict):
+        """ Filters POST body from traceback data
+
+        """
+        post_data = tb_data.get("request", None)
+        if post_data:
+            post_data = post_data.body
+            try:
+                post_data = json.loads(post_data)
+            except JSONDecodeError:
+                pass
+        post_data = {
+            "filtered_POST_items": [
+                ("body", post_data),
+            ]
+        }
+        return post_data

nginx.conf

@@ -1,25 +0,0 @@
-server {
-    listen 80;
-    client_max_body_size 25M;
-
-    location / {
-        proxy_pass http://127.0.0.1:8000;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header Host $host;
-        proxy_redirect off;
-        proxy_cache_bypass $http_upgrade;
-    }
-
-    location /static/ {
-        alias /konova/static/;
-        access_log /var/log/nginx/access.log;
-        autoindex off;
-        types {
-            text/css css;
-            application/javascript js;
-        }
-    }
-
-    error_log /var/log/nginx/error.log;
-}

requirements.txt

@@ -48,7 +48,7 @@ pytz==2024.2
 PyYAML==6.0.2
 qrcode==7.3.1
 redis==5.1.0b6
-requests<2.32.0
+requests==2.32.3
 six==1.16.0
 soupsieve==2.5
 sqlparse==0.5.1

templates/map/client/config.json

@@ -188,6 +188,7 @@
 	{
 		"title": "Ebene hinzufügen",
 		"preview": true,
+		"editable": true,
 		"wms_options": [ "https://sgx.geodatenzentrum.de/wms_topplus_open" ],
 		"wfs_options": [ "http://213.139.159.34:80/geoserver/uesg/wfs" ],
 		"wfs_proxy": "/client/proxy?",

templates/map/geom_form.html

@@ -13,9 +13,9 @@
     </div>
 {% endif %}
 
-{% if geom_form.geom.errors %}
+{% if geom_form.output.errors %}
     <div class="alert-danger p-2">
-    {% for error in geom_form.geom.errors %}
+    {% for error in geom_form.output.errors %}
             <strong class="invalid">{{ error }}</strong>
             <br>
     {% endfor %}