Merge pull request 'Master' (#543) from master into Docker

Reviewed-on: #543

Dockerfile

@@ -0,0 +1,36 @@
+# Nutze ein schlankes Python-Image
+FROM python:3.13-slim-bullseye
+
+ENV PYTHONUNBUFFERED 1
+
+WORKDIR /konova
+
+# Installiere System-Abhängigkeiten
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    gdal-bin redis-server nginx \
+    && rm -rf /var/lib/apt/lists/*  # Platz sparen
+
+# Erstelle benötigte Verzeichnisse & setze Berechtigungen
+RUN mkdir -p /var/log/nginx /var/log/gunicorn /var/lib/nginx /tmp/nginx_client_body \
+    && touch /var/log/nginx/access.log /var/log/nginx/error.log \
+    && chown -R root:root /var/log/nginx /var/lib/nginx /tmp/nginx_client_body
+
+# Kopiere und installiere Python-Abhängigkeiten
+COPY ./requirements.txt /konova/
+RUN pip install --upgrade pip && pip install --no-cache-dir -r requirements.txt
+
+# Entferne Standard-Nginx-Site und ersetze sie durch eigene Config
+RUN rm -rf /etc/nginx/sites-enabled/default
+COPY ./nginx.conf /etc/nginx/conf.d
+
+# Kopiere restliche Projektdateien
+COPY . /konova/
+
+# Sammle statische Dateien
+RUN python manage.py collectstatic --noinput
+
+# Exponiere Ports
+#EXPOSE 80 6379 8000
+
+# Setze Entrypoint
+ENTRYPOINT ["/konova/docker-entrypoint.sh"]

README.md

@@ -4,6 +4,7 @@ the database postgresql and the css library bootstrap as well as the icon packag
 fontawesome for a modern look, following best practices from the industry.
 
 ## Background processes
+### !!! For non-docker run
 Konova uses celery for background processing. To start the worker you need to run
 ```shell
 $ celery -A konova worker -l INFO
@@ -18,3 +19,58 @@ Technical documention is provided in the projects git wiki.
 A user documentation is not available (and not needed, yet). 
 
 
+# Docker 
+To run the docker-compose as expected, you need to take the following steps:
+
+1. Create a database containing docker, using an appropriate Dockerfile, e.g. the following
+```
+version: '3.3'
+services:
+  postgis:
+    image: postgis/postgis
+    restart: always
+    container_name: postgis-docker
+    ports: 
+      - 5433:5432
+    volumes:
+      - db-volume:/var/lib/postgresql/data
+    environment:
+      - POSTGRES_PASSWORD=postgres
+      - POSTGRES_USER=postgres
+    networks:
+      - db-network-bridge
+
+networks:
+  db-network-bridge:
+    driver: "bridge"
+
+volumes:
+  db-volume:
+```
+This Dockerfile creates a Docker container running postgresql and postgis, creates the default superuser postgres, 
+creates a named volume for persisting the database and creates a new network bridge, which **must be used by any other
+container, which wants to write/read on this database**. 
+
+2. Make sure the name of the network bridge above matches the network in the konova docker-compose.yml  
+3. Get into the running postgis container (`docker exec -it postgis-docker bash`) and create new databases, users and so on. Make sure the database `konova` exists now!
+4. Replace all `CHANGE_ME_xy` values inside of konova/docker-compose.yml for your installation. Make sure the `SSO_HOST` holds the proper SSO host, e.g. for the arnova project `arnova.example.org` (Arnova must be installed and the webserver configured as well, of course)
+5. Take a look on konova/settings.py and konova/sub_settings/django_settings.py. Again: Replace all occurences of `CHANGE_ME` with proper values for your installation.
+   1. Make sure you have the proper host strings added to `ALLOWED_HOSTS` inside of django_settings.py.  
+6. Build and run the docker setup using `docker-compose build` and `docker-compose start` from the main directory of this project (where the docker-compose.yml lives)
+7. Run migrations! To do so, get into the konova service container (`docker exec -it konova-docker bash`) and run the needed commands (`python manage.py makemigrations LIST_OF_ALL_MIGRATABLE_APPS`, then `python manage.py migrate`)
+8. Run the setup command `python manage.py setup` and follow the instructions on the CLI
+9. To enable **SMTP** mail support, make sure your host machine (the one where the docker container run) has the postfix service configured properly. Make sure the `mynetworks` variable is xtended using the docker network bridge ip, created in the postgis container and used by the konova services.
+   1. **Hint**: You can find out this easily by trying to perform a test mail in the running konova web application (which will fail, of course). Then take a look to the latest entries in `/var/log/mail.log` on your host machine. The failed IP will be displayed there.
+   2. **Please note**: This installation guide is based on SMTP using postfix! 
+   3. Restart the postfix service on your host machine to reload the new configuration (`service postfix restart`)
+10. Finally, make sure your host machine webserver passes incoming requests properly to the docker nginx webserver of konova. A proper nginx config for the host machine may look like this:
+```
+server {
+    server_name konova.domain.org;
+
+    location / {
+        proxy_pass http://localhost:KONOVA_NGINX_DOCKER_PORT/;
+        proxy_set_header Host $host;
+    }
+}
+```

api/utils/serializer/serializer.py

@@ -6,7 +6,6 @@ Created on: 24.01.22
 
 """
 import json
-import uuid
 from abc import abstractmethod
 
 from django.contrib.gis import geos
@@ -172,20 +171,13 @@ class AbstractModelAPISerializer:
         Returns:
 
         """
-        if not id:
-            raise ValueError("No id provided. The id is expected to live in the URL!")
-
         # First if there is an external identifier linked to an internal one, so we can continue with the internal
         try:
             ext_id = ExternalIdentifier.objects.get(external_id=id)
             id = ext_id.internal_id
         except ObjectDoesNotExist:
-            # Not found as external id - let's check whether this is a valid uuid and therefore
+            # No external id found - let's hope the given id exists internally
-            # potentially an internal id
+            pass
-            try:
-                uuid.UUID(id)
-            except ValueError:
-                raise AssertionError(f"'{id}' is neither a known external identifier nor a valid uuid.")
 
         obj = self.model.objects.get(
             id=id,
@@ -210,33 +202,3 @@ class AbstractModelAPISerializer:
             obj (Intervention)
         """
         raise NotImplementedError("Must be implemented in subclasses")
-
-
-    def _geometry_conflicts_to_list(self, geometry) -> list:
-        """ Serializes geometry conflict ids into dict
-
-        Args:
-            geometry (Geometry): The geometry to fetch geometry conflicts from
-
-        Returns:
-            ids (list): Serialized geometry conflicts as dict objects inside a list
-        """
-        ids = []
-        conflict_geometries = geometry.get_conflict_geometries()
-        for geom in conflict_geometries:
-            try:
-                data = geom.get_data_objects(["identifier", "id"])
-                if len(data) == 0:
-                    # expected behaviour in case of deleted data object
-                    continue
-                data = data[0]
-            except KeyError:
-                raise AssertionError(f"Geometry {geom.id} is not attached to any entries. Contact an admin!")
-            ids.append(
-                {
-                    "identifier": data["identifier"],
-                    "id": data["id"],
-                }
-            )
-
-        return ids

api/utils/serializer/v1/serializer.py

@@ -54,7 +54,6 @@ class AbstractModelAPISerializerV1(AbstractModelAPISerializer):
             "created_on": self._created_on_to_json(entry),
             "modified_on": self._modified_on_to_json(entry),
             "external_identifiers": ext_ids,
-            "geometry_conflicts": self._geometry_conflicts_to_list(entry.geometry)
         }
         self._extend_properties_data(entry)
         geo_json["properties"] = self.properties_data

api/views/v1/views.py

@@ -10,7 +10,6 @@ from json import JSONDecodeError
 
 from django.core.exceptions import ObjectDoesNotExist
 from django.http import JsonResponse, HttpRequest
-from django.views.decorators.csrf import csrf_exempt
 
 from api.utils.serializer.v1.compensation import CompensationAPISerializerV1
 from api.utils.serializer.v1.deduction import DeductionAPISerializerV1
@@ -29,20 +28,6 @@ class AbstractAPIViewV1(AbstractAPIView):
         super().__init__(*args, **kwargs)
         self.serializer = self.serializer()
 
-    @csrf_exempt
-    def dispatch(self, request, *args, **kwargs):
-        try:
-            return super().dispatch(request, *args, **kwargs)
-        except (JSONDecodeError,
-                AssertionError,
-                ValueError,
-                PermissionError) as e:
-            return self._return_error_response(e, 400)
-        except ObjectDoesNotExist as e:
-            return self._return_error_response(e, 404)
-        except Exception as e:
-            return self._return_error_response(e, 500)
-
     def get(self, request: HttpRequest, id=None):
         """ Handles the GET request
 
@@ -55,12 +40,17 @@ class AbstractAPIViewV1(AbstractAPIView):
         Returns:
             response (JsonResponse)
         """
-        self.rpp = int(request.GET.get("rpp", self.rpp))
+        try:
-        self.page_number = int(request.GET.get("p", self.page_number))
+            self.rpp = int(request.GET.get("rpp", self.rpp))
-        self.serializer.rpp = self.rpp
+            self.page_number = int(request.GET.get("p", self.page_number))
-        self.serializer.page_number = self.page_number
+
-        self.serializer.prepare_lookup(id, self.user)
+            self.serializer.rpp = self.rpp
-        data = self.serializer.fetch_and_serialize()
+            self.serializer.page_number = self.page_number
+
+            self.serializer.prepare_lookup(id, self.user)
+            data = self.serializer.fetch_and_serialize()
+        except Exception as e:
+            return self._return_error_response(e, 500)
         return self._return_response(request, data)
 
     def post(self, request: HttpRequest):
@@ -74,9 +64,16 @@ class AbstractAPIViewV1(AbstractAPIView):
         Returns:
             response (JsonResponse)
         """
-        body = request.body.decode("utf-8")
+        try:
-        body = json.loads(body)
+            body = request.body.decode("utf-8")
-        created_id = self.serializer.create_model_from_json(body, self.user)
+            body = json.loads(body)
+            created_id = self.serializer.create_model_from_json(body, self.user)
+        except (JSONDecodeError,
+                AssertionError,
+                ValueError,
+                PermissionError,
+                ObjectDoesNotExist) as e:
+            return self._return_error_response(e, 400)
         return JsonResponse({"id": created_id})
 
     def put(self, request: HttpRequest, id=None):
@@ -91,9 +88,12 @@ class AbstractAPIViewV1(AbstractAPIView):
         Returns:
             response (JsonResponse)
         """
-        body = request.body.decode("utf-8")
+        try:
-        body = json.loads(body)
+            body = request.body.decode("utf-8")
-        updated_id = self.serializer.update_model_from_json(id, body, self.user)
+            body = json.loads(body)
+            updated_id = self.serializer.update_model_from_json(id, body, self.user)
+        except Exception as e:
+            return self._return_error_response(e, 500)
         return JsonResponse({"id": updated_id})
 
     def delete(self, request: HttpRequest, id=None):
@@ -107,7 +107,10 @@ class AbstractAPIViewV1(AbstractAPIView):
             response (JsonResponse)
         """
 
-        success = self.serializer.delete_entry(id, self.user)
+        try:
+            success = self.serializer.delete_entry(id, self.user)
+        except Exception as e:
+            return self._return_error_response(e, 500)
         return JsonResponse(
             {
                 "success": success,

docker-compose.yml

@@ -0,0 +1,21 @@
+services:
+  konova:
+    external_links:
+      - postgis:db
+      - arnova-nginx-server:arnova
+    build: .
+    image: "ksp/konova:x.y"
+    container_name: "konova-docker"
+    command: ./docker-entrypoint.sh
+    restart: always
+    volumes:
+      - /data/apps/konova/uploaded_files:/konova_uploaded_files
+    ports:
+      - "1337:80"
+
+# Instead of an own, new network, we need to connect to the existing one, which is provided by the postgis container
+# NOTE: THIS NETWORK MUST EXIST
+networks:
+  default:
+    name: postgis_nat_it_backend
+    external: true

docker-entrypoint.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -e  # Beende Skript bei Fehlern
+set -o pipefail  # Fehler in Pipelines nicht ignorieren
+
+# Starte Redis
+redis-server --daemonize yes
+
+# Starte Celery Worker im Hintergrund
+celery -A konova worker --loglevel=info &
+
+# Starte Nginx als Hintergrundprozess
+nginx -g "daemon off;" &
+
+# Setze Gunicorn Worker-Anzahl (Standard: (2*CPUs)+1)
+WORKERS=${GUNICORN_WORKERS:-$((2 * $(nproc) + 1))}
+
+# Stelle sicher, dass Logs existieren
+mkdir -p /var/log/gunicorn
+touch /var/log/gunicorn/access.log /var/log/gunicorn/error.log
+
+# Starte Gunicorn als Hauptprozess
+exec gunicorn --workers="$WORKERS" konova.wsgi:application \
+    --bind=0.0.0.0:8000 \
+    --access-logfile /var/log/gunicorn/access.log \
+    --error-logfile /var/log/gunicorn/error.log \
+    --access-logformat '%({x-real-ip}i)s via %(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"'

konova/models/geometry.py

@@ -113,7 +113,7 @@ class Geometry(BaseResource):
             objs (list): The list of objects
         """
         objs = []
-        stop_searching = False
+
         # Some related data sets can be processed rather easily
         regular_sets = [
             self.intervention_set,
@@ -125,44 +125,28 @@ class Geometry(BaseResource):
                 deleted=None
             )
             if limit_to_attrs:
-                objs += set_objs.values(*limit_to_attrs)
+                objs += set_objs.values_list(*limit_to_attrs, flat=True)
             else:
                 objs += set_objs
 
-            stop_searching = len(objs) > 0
+        # ... but we need a special treatment for compensations, since they can be deleted directly OR inherit their
-            if stop_searching:
+        # de-facto-deleted status from their deleted parent intervention
-                break
+        comp_objs = self.compensation_set.filter(
-
+            Q(deleted=None) & Q(intervention__deleted=None)
-        if not stop_searching:
+        )
-            # ... but we need a special treatment for compensations, since they can be deleted directly OR inherit their
+        if limit_to_attrs:
-            # de-facto-deleted status from their deleted parent intervention
+            objs += comp_objs.values_list(*limit_to_attrs, flat=True)
-            comp_objs = self.compensation_set.filter(
+        else:
-                Q(deleted=None) | Q(intervention__deleted=None)
+            objs += comp_objs
-            )
-            if limit_to_attrs:
-                objs += comp_objs.values(*limit_to_attrs)
-            else:
-                objs += comp_objs
         return objs
 
-    def get_data_object(self, limit_to_attrs: list = None):
+    def get_data_object(self):
         """
-        Getter for the specific data object which is related to this geometry.
+        Getter for the specific data object which is related to this geometry
-
-        !!! Only returns undeleted entries !!!
-
-        Returns:
-            result (str|None): Returns the desired attributes or None if the data object is marked as deleted
-
         """
-        objs = self.get_data_objects(limit_to_attrs)
+        objs = self.get_data_objects()
         assert (len(objs) <= 1)
-        try:
+        result = objs.pop()
-            result = objs.pop()
-        except IndexError:
-            # If this happens, we just processed a GeometryConflict with an entry which is marked as deleted.
-            # Therefore we return None
-            result = None
         return result
 
     def update_parcels(self):
@@ -452,16 +436,6 @@ class Geometry(BaseResource):
             output_geom.transform(DEFAULT_SRID_RLP)
         return output_geom
 
-    def get_conflict_geometries(self):
-        """ Getter for geometry ids which conflict with this geometry or are conflicted by this one
-
-        Returns:
-            geom_ids (list): List of geometry ids
-        """
-        conflict_geoms_id = GeometryConflict.get_conflict_geometries(self)
-        conflict_geoms = Geometry.objects.filter(id__in=conflict_geoms_id)
-        return conflict_geoms
-
 
 class GeometryConflict(UuidModel):
     """
@@ -485,23 +459,3 @@ class GeometryConflict(UuidModel):
 
     def __str__(self):
         return f"{self.conflicting_geometry.id} conflicts with {self.affected_geometry.id}"
-
-    @staticmethod
-    def get_conflict_geometries(geometry: Geometry):
-        """ Getter for geometries which conflict in one or another way with the given one
-
-        Args:
-            geometry (Geometry): The geometry which shall be checked
-
-        Returns:
-            conflict_geometries (QuerySet): QuerySet of geometries which have conflicts with the given geometry
-        """
-        conflict_geometries = GeometryConflict.objects.filter(
-            affected_geometry=geometry.id,
-        ).values_list("conflicting_geometry__id", flat=True)
-        conflict_geometries = conflict_geometries.union(
-            GeometryConflict.objects.filter(
-                conflicting_geometry=geometry.id,
-            ).values_list("affected_geometry__id", flat=True)
-        )
-        return conflict_geometries

konova/models/object.py

@@ -676,17 +676,24 @@ class GeoReferencedMixin(models.Model):
         if self.geometry is None:
             return request
 
-        conflicting_geometries = self.geometry.get_conflict_geometries()
+        instance_objs = []
-        data_object_identifiers = []
+        needed_data_object_attrs = [
-        for conflicting_geom in conflicting_geometries:
+            "identifier"
-            data_obj_id = conflicting_geom.get_data_object(["identifier"])
+        ]
-            if data_obj_id:
+        conflicts = self.geometry.conflicts_geometries.iterator()
-                data_object_identifiers.append(data_obj_id)
 
-        add_message = len(data_object_identifiers) > 0
+        for conflict in conflicts:
+            # Only check the affected geometry of this conflict, since we know the conflicting geometry is self.geometry
+            instance_objs += conflict.affected_geometry.get_data_objects(needed_data_object_attrs)
+
+        conflicts = self.geometry.conflicted_by_geometries.iterator()
+        for conflict in conflicts:
+            # Only check the conflicting geometry of this conflict, since we know the affected geometry is self.geometry
+            instance_objs += conflict.conflicting_geometry.get_data_objects(needed_data_object_attrs)
+
+        add_message = len(instance_objs) > 0
         if add_message:
-            data_object_identifiers = [x["identifier"] for x in data_object_identifiers]
+            instance_identifiers = ", ".join(instance_objs)
-            instance_identifiers = ", ".join(data_object_identifiers)
             message_str = GEOMETRY_CONFLICT_WITH_TEMPLATE.format(instance_identifiers)
             messages.info(request, message_str)
         return request

konova/sub_settings/django_settings.py

@@ -191,11 +191,10 @@ STATICFILES_DIRS = [
 ]
 
 # EMAIL (see https://docs.djangoproject.com/en/dev/topics/email/)
-
-# CHANGE_ME !!! ONLY FOR DEVELOPMENT !!!
 if DEBUG:
+    # ONLY FOR DEVELOPMENT NEEDED
     EMAIL_BACKEND = 'django.core.mail.backends.filebased.EmailBackend'
-    EMAIL_FILE_PATH = '/tmp/app-messages' # change this to a proper location
+    EMAIL_FILE_PATH = '/tmp/app-messages'
 
 DEFAULT_FROM_EMAIL = env("DEFAULT_FROM_EMAIL")  # The default email address for the 'from' element
 SERVER_EMAIL = DEFAULT_FROM_EMAIL  # The default email sender address, which is used by Django to send errors via mail

konova/tests/test_geometries.py

@@ -85,6 +85,12 @@ class GeometryTestCase(BaseTestCase):
         )
 
     def test_get_data_objects(self):
+        num_objs_with_geom = 0
+        self.assertEqual(
+            len(self.geom_1.get_data_objects()),
+            num_objs_with_geom
+        )
+
         objs = [
             self.intervention,
             self.compensation,
@@ -92,13 +98,16 @@ class GeometryTestCase(BaseTestCase):
             self.ema,
         ]
         for obj in objs:
-            if not obj.geometry:
+            obj.geometry = self.geom_1
-                obj.geometry = Geometry.objects.create(geom=self.create_dummy_geometry())
+            obj.save()
-                obj.save()
 
-            data_objs = obj.geometry.get_data_objects()
+            num_objs_with_geom += 1
-            self.assertEqual(len(data_objs), 1)
+            geom_objs = self.geom_1.get_data_objects()
-            self.assertIn(obj, data_objs)
+            self.assertEqual(
+                len(geom_objs),
+                num_objs_with_geom
+            )
+            self.assertIn(obj, geom_objs)
 
     def test_as_feature_collection(self):
         geometry = self.geom_1.geom

nginx.conf

@@ -0,0 +1,25 @@
+server {
+    listen 80;
+    client_max_body_size 25M;
+
+    location / {
+        proxy_pass http://127.0.0.1:8000;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header Host $host;
+        proxy_redirect off;
+        proxy_cache_bypass $http_upgrade;
+    }
+
+    location /static/ {
+        alias /konova/static/;
+        access_log /var/log/nginx/access.log;
+        autoindex off;
+        types {
+            text/css css;
+            application/javascript js;
+        }
+    }
+
+    error_log /var/log/nginx/error.log;
+}

templates/map/client/config.json

@@ -56,9 +56,11 @@
 	],
 	"layers":
 	[
+		{ "id": "webatlas_farbe", "folder": "bg", "type": "WMS", "order": -1, "title": "WebatlasRP farbig", "attribution": "LVermGeo", "url": "https://maps.service24.rlp.de/gisserver/services/RP/RP_WebAtlasRP/MapServer/WmsServer?", "name": "RP_WebAtlasRP", "active": true},
+		{ "id": "webatlas_grau", "folder": "bg", "type": "WMS", "order": -1, "title": "WebatlasRP grau", "attribution": "LVermGeo", "url": "https://maps.service24.rlp.de/gisserver/services/RP/RP_ETRS_Gt/MapServer/WmsServer?", "name": "0", "active": false },
 		{ "id": "luftbilder", "folder": "bg", "type": "WMS", "order": -1, "title": "Luftbilder", "attribution": "LVermGeo", "url": "https://geo4.service24.rlp.de/wms/rp_dop20.fcgi?", "name": "rp_dop20", "active": false },
 		{ "id": "basemap_farbe", "folder": "bg", "type": "WMS", "order": -1, "title": "BasemapDE farbig", "attribution": "BKG", "url": "https://sgx.geodatenzentrum.de/wms_basemapde?", "name": "de_basemapde_web_raster_farbe", "active": false },
-		{ "id": "basemap_grau", "folder": "bg", "type": "WMS", "order": -1, "title": "BasemapDE grau", "attribution": "BKG", "url": "https://sgx.geodatenzentrum.de/wms_basemapde?", "name": "de_basemapde_web_raster_grau", "active": true },
+		{ "id": "basemap_grau", "folder": "bg", "type": "WMS", "order": -1, "title": "BasemapDE grau", "attribution": "BKG", "url": "https://sgx.geodatenzentrum.de/wms_basemapde?", "name": "de_basemapde_web_raster_grau", "active": false },
 		{ "id": "dtk_farbe", "folder": "bg", "type": "WMS", "order": -1, "title": "DTK5 farbig", "attribution": "LVermGeo", "url": "https://geo4.service24.rlp.de/wms/dtk5_rp.fcgi?", "name": "rp_dtk5", "active": false },
 		{ "id": "dtk_grau", "folder": "bg", "type": "WMS", "order": -1, "title": "DTK5 grau", "attribution": "LVermGeo", "url": "https://geo4.service24.rlp.de/wms/dtk5_rp.fcgi?", "name": "rp_dtk5_grau", "active": false },