Merge pull request 'Update docker' (#569) from master into Docker

Reviewed-on: #569

Dockerfile

@@ -0,0 +1,36 @@
+# Nutze ein schlankes Python-Image
+FROM python:3.13-slim-bullseye
+
+ENV PYTHONUNBUFFERED 1
+
+WORKDIR /konova
+
+# Installiere System-Abhängigkeiten
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    gdal-bin redis-server nginx \
+    && rm -rf /var/lib/apt/lists/*  # Platz sparen
+
+# Erstelle benötigte Verzeichnisse & setze Berechtigungen
+RUN mkdir -p /var/log/nginx /var/log/gunicorn /var/lib/nginx /tmp/nginx_client_body \
+    && touch /var/log/nginx/access.log /var/log/nginx/error.log \
+    && chown -R root:root /var/log/nginx /var/lib/nginx /tmp/nginx_client_body
+
+# Kopiere und installiere Python-Abhängigkeiten
+COPY ./requirements.txt /konova/
+RUN pip install --upgrade pip && pip install --no-cache-dir -r requirements.txt
+
+# Entferne Standard-Nginx-Site und ersetze sie durch eigene Config
+RUN rm -rf /etc/nginx/sites-enabled/default
+COPY ./nginx.conf /etc/nginx/conf.d
+
+# Kopiere restliche Projektdateien
+COPY . /konova/
+
+# Sammle statische Dateien
+RUN python manage.py collectstatic --noinput
+
+# Exponiere Ports
+#EXPOSE 80 6379 8000
+
+# Setze Entrypoint
+ENTRYPOINT ["/konova/docker-entrypoint.sh"]

README.md

@@ -4,6 +4,7 @@ the database postgresql and the css library bootstrap as well as the icon packag
 fontawesome for a modern look, following best practices from the industry.
 
 ## Background processes
+### !!! For non-docker run
 Konova uses celery for background processing. To start the worker you need to run
 ```shell
 $ celery -A konova worker -l INFO
@@ -18,3 +19,58 @@ Technical documention is provided in the projects git wiki.
 A user documentation is not available (and not needed, yet). 
 
 
+# Docker 
+To run the docker-compose as expected, you need to take the following steps:
+
+1. Create a database containing docker, using an appropriate Dockerfile, e.g. the following
+```
+version: '3.3'
+services:
+  postgis:
+    image: postgis/postgis
+    restart: always
+    container_name: postgis-docker
+    ports: 
+      - 5433:5432
+    volumes:
+      - db-volume:/var/lib/postgresql/data
+    environment:
+      - POSTGRES_PASSWORD=postgres
+      - POSTGRES_USER=postgres
+    networks:
+      - db-network-bridge
+
+networks:
+  db-network-bridge:
+    driver: "bridge"
+
+volumes:
+  db-volume:
+```
+This Dockerfile creates a Docker container running postgresql and postgis, creates the default superuser postgres, 
+creates a named volume for persisting the database and creates a new network bridge, which **must be used by any other
+container, which wants to write/read on this database**. 
+
+2. Make sure the name of the network bridge above matches the network in the konova docker-compose.yml  
+3. Get into the running postgis container (`docker exec -it postgis-docker bash`) and create new databases, users and so on. Make sure the database `konova` exists now!
+4. Replace all `CHANGE_ME_xy` values inside of konova/docker-compose.yml for your installation. Make sure the `SSO_HOST` holds the proper SSO host, e.g. for the arnova project `arnova.example.org` (Arnova must be installed and the webserver configured as well, of course)
+5. Take a look on konova/settings.py and konova/sub_settings/django_settings.py. Again: Replace all occurences of `CHANGE_ME` with proper values for your installation.
+   1. Make sure you have the proper host strings added to `ALLOWED_HOSTS` inside of django_settings.py.  
+6. Build and run the docker setup using `docker-compose build` and `docker-compose start` from the main directory of this project (where the docker-compose.yml lives)
+7. Run migrations! To do so, get into the konova service container (`docker exec -it konova-docker bash`) and run the needed commands (`python manage.py makemigrations LIST_OF_ALL_MIGRATABLE_APPS`, then `python manage.py migrate`)
+8. Run the setup command `python manage.py setup` and follow the instructions on the CLI
+9. To enable **SMTP** mail support, make sure your host machine (the one where the docker container run) has the postfix service configured properly. Make sure the `mynetworks` variable is xtended using the docker network bridge ip, created in the postgis container and used by the konova services.
+   1. **Hint**: You can find out this easily by trying to perform a test mail in the running konova web application (which will fail, of course). Then take a look to the latest entries in `/var/log/mail.log` on your host machine. The failed IP will be displayed there.
+   2. **Please note**: This installation guide is based on SMTP using postfix! 
+   3. Restart the postfix service on your host machine to reload the new configuration (`service postfix restart`)
+10. Finally, make sure your host machine webserver passes incoming requests properly to the docker nginx webserver of konova. A proper nginx config for the host machine may look like this:
+```
+server {
+    server_name konova.domain.org;
+
+    location / {
+        proxy_pass http://localhost:KONOVA_NGINX_DOCKER_PORT/;
+        proxy_set_header Host $host;
+    }
+}
+```

api/admin.py

@@ -1,6 +1,5 @@
 from django.contrib import admin
 
-from api.models import ExternalIdentifier
 from api.models.token import APIUserToken, OAuthToken
 
 
@@ -29,17 +28,6 @@ class OAuthTokenAdmin(admin.ModelAdmin):
         "refresh_token",
     ]
 
-class ExternalIdentifierAdmin(admin.ModelAdmin):
-    list_display = [
-        "external_id",
-        "internal_id",
-        "created",
-    ]
-    search_fields = [
-        "external_id",
-        "internal_id",
-    ]
 
 admin.site.register(APIUserToken, APITokenAdmin)
 admin.site.register(OAuthToken, OAuthTokenAdmin)
-admin.site.register(ExternalIdentifier, ExternalIdentifierAdmin)

compensation/templates/compensation/report/eco_account/report.html

@@ -57,9 +57,7 @@
                                         </a>
                                     </td>
                                     <td>{{ deduction.surface|floatformat:2 }} m²</td>
-                                </tr>
                                 {% empty %}
-                                <tr>
                                     <td>
                                         {% translate 'None' %}
                                     </td>
@@ -68,6 +66,7 @@
                                     </td>
                                 </tr>
                                 {% endfor %}
+
                             </table>
                         </td>
                     </tr>

docker-compose.yml

@@ -0,0 +1,21 @@
+services:
+  konova:
+    external_links:
+      - postgis:db
+      - arnova-nginx-server:arnova
+    build: .
+    image: "ksp/konova:x.y"
+    container_name: "konova-docker"
+    command: ./docker-entrypoint.sh
+    restart: always
+    volumes:
+      - /data/apps/konova/uploaded_files:/konova_uploaded_files
+    ports:
+      - "1337:80"
+
+# Instead of an own, new network, we need to connect to the existing one, which is provided by the postgis container
+# NOTE: THIS NETWORK MUST EXIST
+networks:
+  default:
+    name: postgis_nat_it_backend
+    external: true

docker-entrypoint.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -e  # Beende Skript bei Fehlern
+set -o pipefail  # Fehler in Pipelines nicht ignorieren
+
+# Starte Redis
+redis-server --daemonize yes
+
+# Starte Celery Worker im Hintergrund
+celery -A konova worker --loglevel=info &
+
+# Starte Nginx als Hintergrundprozess
+nginx -g "daemon off;" &
+
+# Setze Gunicorn Worker-Anzahl (Standard: (2*CPUs)+1)
+WORKERS=${GUNICORN_WORKERS:-$((2 * $(nproc) + 1))}
+
+# Stelle sicher, dass Logs existieren
+mkdir -p /var/log/gunicorn
+touch /var/log/gunicorn/access.log /var/log/gunicorn/error.log
+
+# Starte Gunicorn als Hauptprozess
+exec gunicorn --workers="$WORKERS" konova.wsgi:application \
+    --bind=0.0.0.0:8000 \
+    --access-logfile /var/log/gunicorn/access.log \
+    --error-logfile /var/log/gunicorn/error.log \
+    --access-logformat '%({x-real-ip}i)s via %(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"'

intervention/admin.py

@@ -1,6 +1,6 @@
 from django.contrib import admin
 
-from intervention.models import Intervention
+from intervention.models import Intervention, Responsibility, Legal, Revocation, InterventionDocument
 from konova.admin import AbstractDocumentAdmin, BaseObjectAdmin
 
 

konova/admin.py

@@ -7,7 +7,10 @@ Created on: 22.07.21
 """
 from django.contrib import admin
 
+from konova.models import Geometry, Deadline, GeometryConflict, Parcel, District, Municipal, ParcelGroup, Resubmission
 from konova.sub_settings.lanis_settings import DEFAULT_SRID_RLP
+from konova.utils.message_templates import COMPENSATION_REMOVED_TEMPLATE
+from user.models import UserAction
 
 
 class GeometryAdmin(admin.ModelAdmin):
@@ -120,7 +123,6 @@ class BaseResourceAdmin(admin.ModelAdmin):
 
 class BaseObjectAdmin(BaseResourceAdmin, DeletableObjectMixinAdmin):
     search_fields = [
-        "id",
         "identifier",
         "title",
     ]

konova/management/commands/mass_delete_entries.py

@@ -17,16 +17,16 @@ class Command(BaseKonovaCommand):
 
     _DELETE_CLS = None
     _DELETE_COMMENT = None
+    _DELETE_IDENTIFIER_ILIKE_PARAM = None
     _DELETE_USER = None
-    _DELETE_FROM_DB = False
 
     def handle(self, *args, **options):
         self._get_delete_user()
         self._get_object_type()
-        filter = self._get_filter()
+        self._get_identifier_ilike_param()
         self._get_delete_comment()
-        self._show_config(filter)
+        self._show_config()
-        entries = self._collect_entries(filter)
+        entries = self._collect_entries()
         self._process_delete(entries)
 
     def _get_delete_user(self):
@@ -62,144 +62,36 @@ class Command(BaseKonovaCommand):
             self._write_error(f"'{object_type_select}' not a valid option of {object_types_str}. Abort.")
             exit(-1)
 
-    def _get_filter(self):
+    def _get_identifier_ilike_param(self):
-        filter_attribute = input("What do you want to filter entries by? (identifier|created_by_user): ").lower()
+        self._DELETE_IDENTIFIER_ILIKE_PARAM = input("Use this substring to search in identifiers: ")
-        filter = {}
-        try:
-            assert filter_attribute in ["identifier", "created_by_user"]
-            if filter_attribute == "identifier":
-                filter_attribute = "identifier__contains"
-            elif filter_attribute == "created_by_user":
-                filter_attribute = "created__user__username"
-        except AssertionError:
-            self._write_error(f"'{filter_attribute}' unknown. Abort")
-            exit(-1)
-
-        filter_val = input("Use this value to search for: ")
-        try:
-            assert len(filter_val) > 0
-        except AssertionError:
-            self._write_error(f"'{filter_val}' is empty. Abort.")
-            exit(-1)
-
-        filter[filter_attribute] = filter_val
-        return filter
 
     def _get_delete_comment(self):
         self._DELETE_COMMENT = input("Comment to store in delete action: ")
 
-    def _show_config(self, filter: dict):
+    def _show_config(self):
         assert self._DELETE_CLS is not None
+        assert self._DELETE_IDENTIFIER_ILIKE_PARAM is not None
         assert self._DELETE_COMMENT is not None
 
         self._write_warning("You are about to delete entries with:")
         self._write_warning(f"    Object type: {self._DELETE_CLS.__name__}")
-        self._write_warning(f"    Filter by: '{filter}'")
+        self._write_warning(f"    Identifier pattern: '*{self._DELETE_IDENTIFIER_ILIKE_PARAM}*'")
+        self._write_warning(f"    Comment for delete action: '{self._DELETE_COMMENT}'")
         self._write_warning(f"    Delete performing user: '{self._DELETE_USER}'")
 
-    def _collect_entries(self, filter: dict):
+    def _collect_entries(self):
         return self._DELETE_CLS.objects.filter(
-            deleted__isnull=True
+            deleted__isnull=True,
-        ).filter(
+            identifier__contains=self._DELETE_IDENTIFIER_ILIKE_PARAM
-            **filter
         )
 
     def _process_delete(self, entries: QuerySet):
-        """ Wrapping delete method. Forces user to decide whether entries will be marked as deleted or shall
-        be removed completely from database
-
-        Args:
-            entries (QuerySet): Entries to be processed
-
-        Returns:
-
-        """
-        delete_type = input("Do you want 1) mark entries as deleted or 2) remove them from the database? (1|2): ")
-        try:
-            delete_type = int(delete_type)
-            assert delete_type in (1, 2)
-            self._DELETE_FROM_DB = delete_type == 2
-        except (ValueError, AssertionError) as e:
-            self._write_error(f"Entered unknown '{delete_type}'. Abort.")
-            exit(-1)
-        if self._DELETE_FROM_DB:
-            self._process_delete_from_db(entries)
-        else:
-            self._process_mark_as_deleted(entries)
-
-
-    def _process_mark_as_deleted(self, entries: QuerySet):
-        """ Method do mark chosen entries as deleted
-
-        Args:
-            entries (QuerySet): Entries to be processed
-
-        Returns:
-
-        """
-        self._write_warning(f"    Comment for delete action: '{self._DELETE_COMMENT}'")
-
-        self._check_entries_sample(entries)
-
-        entries_count = entries.count()
-        i = 0
-        for element in entries:
-            if i % 10 == 0:
-                self._write_warning(f"    Deleted {i}/{entries_count}")
-            element.mark_as_deleted(user=self._DELETE_USER, send_mail=False, comment=self._DELETE_COMMENT)
-            i += 1
-        self._write_warning(f"    Deleted {entries_count}/{entries_count}")
-        self._write_success("Entries deleted. Bye.")
-
-    def _process_delete_from_db(self, entries: QuerySet):
-        """ Method do REMOVE entries from the database
-
-        Args:
-            entries (QuerySet): Entries to be processed
-
-        Returns:
-
-        """
-        self._check_entries_sample(entries)
-
-        entries_count = entries.count()
-        input_num_entries = input(f"THIS IS THE FINAL WARNING! {entries_count} ENTRIES WILL BE REMOVED FROM THE DATABASE! ENTER THE NUMBER OF ENTRIES TO BE DELETED TO CONTINUE: ")
-        try:
-            input_num_entries = int(input_num_entries)
-        except ValueError:
-            self._write_error(f"'{input_num_entries}' is not an integer. Abort.")
-            exit(-1)
-        if entries_count == input_num_entries:
-            i = 0
-            for element in entries:
-                if i % 10 == 0:
-                    self._write_warning(f"    Deleted {i}/{entries_count}")
-                #element.delete()  # call individual delete routine to enforce custom delete logic (not called on bulk deleting!)
-                i += 1
-
-            self._write_warning(f"    Deleted {entries_count}/{entries_count}")
-            self._write_success("Entries deleted. Bye.")
-        else:
-            self._write_error(f"'{input_num_entries}' was not correct. Abort.")
-            exit(-1)
-
-    def _check_entries_sample(self, entries):
-        """ Shows a sample from the entries on the terminal, so the user can decide whether the results are as expected
-
-        Args:
-            entries (QuerySet): Entries to be processed
-
-        Returns:
-
-        """
-
         entries_count = entries.count()
         if entries_count == 0:
             self._write_success(f"Found {entries_count} matches to delete. Nothing to do here. Bye.")
             exit(0)
 
-        self._write_warning(
+        self._write_warning(f"Found {entries_count} matches to delete. Please double-check whether these samples from the results are what you expected: ")
-            f"Found {entries_count} matches to delete. Please double-check whether these samples from the results are what you expected: ")
         first_three_elements = entries[:3]
         for element in first_three_elements:
             self._write_warning(f"    {element.identifier} ({element.id})")
@@ -207,5 +99,15 @@ class Command(BaseKonovaCommand):
         delete_input = input("Are you ready to delete them now? (y|n): ").lower()
         delete_now = delete_input == "y"
         if not delete_now:
-            self._write_error(f"Entered '{delete_input}'. Abort.")
+            self._write_error(f"Entered unknown '{delete_input}'. Abort.")
             exit(-1)
+        else:
+            i = 0
+            for element in entries:
+                if i % 10 == 0:
+                    self._write_warning(f"    Deleted {i}/{entries_count}")
+                element.mark_as_deleted(user=self._DELETE_USER, send_mail=False, comment=self._DELETE_COMMENT)
+                i += 1
+
+            self._write_warning(f"    Deleted {entries_count}/{entries_count}")
+            self._write_success("Entries deleted. Bye.")

konova/models/object.py

@@ -146,33 +146,6 @@ class BaseObject(BaseResource, DeletableObjectMixin):
     class Meta:
         abstract = True
 
-    def delete(self, using=None, keep_parents=False):
-        """ Base deleting of a BaseObject instance
-
-        Args:
-            using:
-            keep_parents:
-
-        Returns:
-
-        """
-        self.__delete_external_identifier()
-        super().delete(using, keep_parents)
-
-    def __delete_external_identifier(self):
-        """ Checks on existing external identifier linking and drops them
-
-        Returns:
-
-        """
-        from api.models import ExternalIdentifier
-        try:
-            external_identifier = ExternalIdentifier.objects.filter(internal_id=self.id)
-            external_identifier.delete()
-        except ObjectDoesNotExist:
-            pass
-
-
     @abstractmethod
     def set_status_messages(self, request: HttpRequest):
         raise NotImplementedError

konova/sub_settings/django_settings.py

@@ -191,11 +191,10 @@ STATICFILES_DIRS = [
 ]
 
 # EMAIL (see https://docs.djangoproject.com/en/dev/topics/email/)
-
-# CHANGE_ME !!! ONLY FOR DEVELOPMENT !!!
 if DEBUG:
+    # ONLY FOR DEVELOPMENT NEEDED
     EMAIL_BACKEND = 'django.core.mail.backends.filebased.EmailBackend'
-    EMAIL_FILE_PATH = '/tmp/app-messages' # change this to a proper location
+    EMAIL_FILE_PATH = '/tmp/app-messages'
 
 DEFAULT_FROM_EMAIL = env("DEFAULT_FROM_EMAIL")  # The default email address for the 'from' element
 SERVER_EMAIL = DEFAULT_FROM_EMAIL  # The default email sender address, which is used by Django to send errors via mail

nginx.conf

@@ -0,0 +1,25 @@
+server {
+    listen 80;
+    client_max_body_size 25M;
+
+    location / {
+        proxy_pass http://127.0.0.1:8000;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header Host $host;
+        proxy_redirect off;
+        proxy_cache_bypass $http_upgrade;
+    }
+
+    location /static/ {
+        alias /konova/static/;
+        access_log /var/log/nginx/access.log;
+        autoindex off;
+        types {
+            text/css css;
+            application/javascript js;
+        }
+    }
+
+    error_log /var/log/nginx/error.log;
+}