Merge pull request '# .env fix' (#415) from env_fix into master

Reviewed-on: #415

.env.sample

@@ -0,0 +1,46 @@
+# General
+SECRET_KEY=CHANGE_ME
+DEBUG=True
+ALLOWED_HOSTS=127.0.0.1,localhost,example.org
+BASE_URL=http://localhost:8002
+ADMINS=Admin1:mail@example.org,Admin2:mail2@example.org
+
+# Database
+DB_USER=postgres
+DB_PASSWORD=
+DB_NAME=konova
+DB_HOST=127.0.0.1
+DB_PORT=5432
+
+# Redis (for celery)
+REDIS_HOST=CHANGE_ME
+REDIS_PORT=CHANGE_ME
+
+# E-Mail
+SMTP_HOST=localhost
+SMTP_PORT=25
+REPLY_TO_ADDR=ksp-servicestelle@sgdnord.rlp.de
+DEFAULT_FROM_EMAIL=service@ksp.de
+
+# Proxy
+PROXY=CHANGE_ME
+GEOPORTAL_RLP_USER=CHANGE_ME
+GEOPORTAL_RLP_PASSWORD=CHANGE_ME
+
+# Schneider
+SCHNEIDER_BASE_URL=https://schneider.naturschutz.rlp.de
+SCHNEIDER_AUTH_TOKEN=CHANGE_ME
+SCHNEIDER_AUTH_HEADER=auth
+
+# SSO
+SSO_SERVER_BASE_URL=https://login.naturschutz.rlp.de
+OAUTH_CODE_VERIFIER=CHANGE_ME
+OAUTH_CLIENT_ID=CHANGE_ME
+OAUTH_CLIENT_SECRET=CHANGE_ME
+
+# RabbitMQ
+## For connections to EGON
+EGON_RABBITMQ_HOST=CHANGE_ME
+EGON_RABBITMQ_PORT=CHANGE_ME
+EGON_RABBITMQ_USER=CHANGE_ME
+EGON_RABBITMQ_PW=CHANGE_ME

.gitignore

@@ -3,3 +3,4 @@
 /.idea/
 /.coverage
 /htmlcov/
+/.env

intervention/settings.py

@@ -5,6 +5,8 @@ Contact: michel.peltriaux@sgdnord.rlp.de
 Created on: 30.11.20
 
 """
+from konova.sub_settings.django_settings import env
+
 INTERVENTION_IDENTIFIER_LENGTH = 6
 INTERVENTION_IDENTIFIER_TEMPLATE = "EIV-{}"
 
@@ -14,7 +16,7 @@ INTERVENTION_LANIS_LAYER_NAME_UNRECORDED_OLD_ENTRY = "eiv_unrecorded_old_entries
 
 # EGON connection settings via rabbitmq
 # NEEDED FOR BACKWARDS COMPATIBILITY
-EGON_RABBITMQ_HOST = "CHANGE_ME"
+EGON_RABBITMQ_HOST = env("EGON_RABBITMQ_HOST")
-EGON_RABBITMQ_PORT = "CHANGE_ME"
+EGON_RABBITMQ_PORT = env("EGON_RABBITMQ_PORT")
-EGON_RABBITMQ_USER = "CHANGE_ME"
+EGON_RABBITMQ_USER = env("EGON_RABBITMQ_USER")
-EGON_RABBITMQ_PW = "CHANGE_ME"
+EGON_RABBITMQ_PW = env("EGON_RABBITMQ_PW")

konova/celery.py

@@ -1,6 +1,7 @@
 import os
 
 from celery import Celery
+from konova.sub_settings.django_settings import env
 
 # Set the default Django settings module for the 'celery' program.
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'konova.settings')
@@ -17,7 +18,7 @@ app.config_from_object('django.conf:settings', namespace='CELERY')
 app.autodiscover_tasks()
 
 # Declare redis as broker
-app.conf.broker_url = 'redis://localhost:6379/0'
+app.conf.broker_url = f'redis://{env("REDIS_HOST")}:{env.int("REDIS_PORT")}/0'
 
 
 @app.task(bind=True)

konova/settings.py

@@ -18,7 +18,6 @@ from konova.sub_settings.proxy_settings import *
 from konova.sub_settings.sso_settings import *
 from konova.sub_settings.table_settings import *
 from konova.sub_settings.lanis_settings import *
-from konova.sub_settings.wfs_parcel_settings import *
 from konova.sub_settings.logging_settings import *
 
 # Max upload size for POST forms

konova/sso/sso.py

@@ -1,78 +0,0 @@
-"""
-Author: Michel Peltriaux
-Organization: Struktur- und Genehmigungsdirektion Nord, Rhineland-Palatinate, Germany
-Contact: michel.peltriaux@sgdnord.rlp.de
-Created on: 17.08.21
-
-"""
-from django.http import HttpResponse
-from django.urls import re_path
-from django.views import View
-from django.views.decorators.csrf import csrf_exempt
-from itsdangerous import TimedSerializer
-from simple_sso.sso_client.client import Client
-
-from user.models import User
-
-
-class PropagateView(View):
-    """ View used to receive propagated sso-server user data
-
-    """
-    client = None
-    signer = None
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.signer = TimedSerializer(self.client.private_key)
-
-    @csrf_exempt
-    def dispatch(self, request, *args, **kwargs):
-        return super().dispatch(request, *args, **kwargs)
-
-    def post(self, request):
-        user_data = request.body
-        user_data = self.signer.loads(user_data)
-        self.client.build_user(user_data)
-        return HttpResponse(status=200)
-
-
-class KonovaSSOClient(Client):
-    """ Konova specialized derivative of general sso.Client.
-
-    Adds some custom behaviour for konova usage.
-
-    """
-    propagate_view = PropagateView
-
-    def get_urls(self):
-        urls = super().get_urls()
-        urls += re_path(r'^propagate/$', self.propagate_view.as_view(client=self), name='simple-sso-propagate'),
-        return urls
-
-    def build_user(self, user_data):
-        """ Creates a user or updates user data
-
-        Args:
-            user_data ():
-
-        Returns:
-
-        """
-        try:
-            user = User.objects.get(username=user_data['username'])
-            # Update user data, excluding some changes
-            skipable_attrs = {
-                "username",
-                "is_staff",
-                "is_superuser",
-            }
-            for _attr, _val in user_data.items():
-                if _attr in skipable_attrs:
-                    continue
-                setattr(user, _attr, _val)
-        except User.DoesNotExist:
-            user = User(**user_data)
-        user.set_unusable_password()
-        user.save()
-        return user

konova/sub_settings/django_settings.py

@@ -10,6 +10,8 @@ For the full list of settings and their values, see
 https://docs.djangoproject.com/en/3.1/ref/settings/
 """
 import os
+
+import environ
 from django.utils.translation import gettext_lazy as _
 from django.conf.locale.de import formats as de_formats
 
@@ -24,28 +26,24 @@ BASE_DIR = os.path.dirname(
     )
 )
 
-# Quick-start development settings - unsuitable for production
+env = environ.Env()
-# See https://docs.djangoproject.com/en/3.1/howto/deployment/checklist/
+# Take environment variables from .env.dev file
+environ.Env.read_env(os.path.join(BASE_DIR, '.env'))
 
 # SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY = '5=9-)2)h$u9=!zrhia9=lj-2#cpcb8=#$7y+)l$5tto$3q(n_+'
+SECRET_KEY = env("SECRET_KEY")
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = True
+DEBUG = env.bool("DEBUG", default=False)
 
-ADMINS = [
+ADMINS = [x.split(':') for x in env.list('ADMINS')]
-    ('KSP-Servicestelle', 'ksp-servicestelle@sgdnord.rlp.de'),
-]
 
-BASE_URL = "http://localhost:8001"
+ALLOWED_HOSTS = env.list("ALLOWED_HOSTS")
 
-ALLOWED_HOSTS = [
+BASE_URL = env("BASE_URL")
-    "127.0.0.1",
-    "localhost",
-]
 
 CSRF_TRUSTED_ORIGINS = [
-    "http://localhost",  # not only host but schema (http/s) as well!
+    BASE_URL
 ]
 
 # Authentication settings
@@ -83,10 +81,6 @@ INSTALLED_APPS = [
     'analysis',
     'api',
 ]
-if DEBUG:
-    INSTALLED_APPS += [
-        'debug_toolbar',
-    ]
 
 MIDDLEWARE = [
     'django.middleware.security.SecurityMiddleware',
@@ -98,10 +92,6 @@ MIDDLEWARE = [
     'django.contrib.messages.middleware.MessageMiddleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
 ]
-if DEBUG:
-    MIDDLEWARE += [
-        "debug_toolbar.middleware.DebugToolbarMiddleware",
-    ]
 
 ROOT_URLCONF = 'konova.urls'
 
@@ -131,10 +121,11 @@ WSGI_APPLICATION = 'konova.wsgi.application'
 DATABASES = {
     'default': {
         'ENGINE': 'django.contrib.gis.db.backends.postgis',
-        'NAME': 'konova',
+        'NAME': env("DB_NAME"),
-        'USER': 'postgres',
+        'USER': env("DB_USER"),
-        'HOST': '127.0.0.1',
+        'PASSWORD': env("DB_PASSWORD"),
-        'PORT': '5432',
+        'HOST': env("DB_HOST"),
+        'PORT': env("DB_PORT"),
     }
 }
 DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
@@ -201,28 +192,6 @@ STATICFILES_DIRS = [
     os.path.join(BASE_DIR, 'templates/map/client/libs'),    # NETGIS map client files
 ]
 
-# DJANGO DEBUG TOOLBAR
-INTERNAL_IPS = [
-    "127.0.0.1"
-]
-DEBUG_TOOLBAR_CONFIG = {
-    "DISABLE_PANELS": {
-        'debug_toolbar.panels.versions.VersionsPanel',
-        'debug_toolbar.panels.timer.TimerPanel',
-        'debug_toolbar.panels.settings.SettingsPanel',
-        'debug_toolbar.panels.headers.HeadersPanel',
-        'debug_toolbar.panels.request.RequestPanel',
-        'debug_toolbar.panels.sql.SQLPanel',
-        'debug_toolbar.panels.staticfiles.StaticFilesPanel',
-        'debug_toolbar.panels.templates.TemplatesPanel',
-        'debug_toolbar.panels.cache.CachePanel',
-        'debug_toolbar.panels.signals.SignalsPanel',
-        'debug_toolbar.panels.logging.LoggingPanel',
-        'debug_toolbar.panels.redirects.RedirectsPanel',
-        'debug_toolbar.panels.profiling.ProfilingPanel',
-    }
-}
-
 # EMAIL (see https://docs.djangoproject.com/en/dev/topics/email/)
 
 # CHANGE_ME !!! ONLY FOR DEVELOPMENT !!!
@@ -230,13 +199,10 @@ if DEBUG:
     EMAIL_BACKEND = 'django.core.mail.backends.filebased.EmailBackend'
     EMAIL_FILE_PATH = '/tmp/app-messages' # change this to a proper location
 
-DEFAULT_FROM_EMAIL = "service@ksp.de"  # The default email address for the 'from' element
+DEFAULT_FROM_EMAIL = env("DEFAULT_FROM_EMAIL")  # The default email address for the 'from' element
 SERVER_EMAIL = DEFAULT_FROM_EMAIL  # The default email sender address, which is used by Django to send errors via mail
-EMAIL_HOST = "localhost"
+EMAIL_HOST = env("SMTP_HOST")
-EMAIL_REPLY_TO = "ksp-servicestelle@sgdnord.rlp.de"
+EMAIL_REPLY_TO = env("REPLY_TO_ADDR")
-SUPPORT_MAIL_RECIPIENT = EMAIL_REPLY_TO
+EMAIL_PORT = env("SMTP_PORT")
-EMAIL_PORT = "25"
-#EMAIL_HOST_USER = ""
-#EMAIL_HOST_PASSWORD = ""
 EMAIL_USE_TLS = False
 EMAIL_USE_SSL = False

konova/sub_settings/proxy_settings.py

@@ -5,12 +5,13 @@ Contact: michel.peltriaux@sgdnord.rlp.de
 Created on: 31.01.22
 
 """
+from konova.sub_settings.django_settings import env
 
-proxy = ""
+proxy = env("PROXY")
 PROXIES = {
     "http": proxy,
     "https": proxy,
 }
 
-CLIENT_PROXY_AUTH_USER = "CHANGE_ME"
+GEOPORTAL_RLP_USER = env("GEOPORTAL_RLP_USER")
-CLIENT_PROXY_AUTH_PASSWORD = "CHANGE_ME"
+GEOPORTAL_RLP_PASSWORD = env("GEOPORTAL_RLP_PASSWORD")

konova/sub_settings/schneider_settings.py

@@ -5,7 +5,8 @@ Contact: ksp-servicestelle@sgdnord.rlp.de
 Created on: 14.12.22
 
 """
+from konova.sub_settings.django_settings import env
 
-base_url = "http://127.0.0.1:8002"
+base_url = env("SCHNEIDER_BASE_URL")
-auth_header = "auth"
+auth_header = env("SCHNEIDER_AUTH_HEADER")
-auth_header_token = "CHANGE_ME"
+auth_header_token = env("SCHNEIDER_AUTH_TOKEN")

konova/sub_settings/sso_settings.py

@@ -5,17 +5,14 @@ Contact: michel.peltriaux@sgdnord.rlp.de
 Created on: 31.01.22
 
 """
-import random
+from konova.sub_settings.django_settings import env
-import string
 
-# Django-simple-SSO settings
+# SSO settings
-SSO_SERVER_BASE = "http://127.0.0.1:8000/"
+SSO_SERVER_BASE = env("SSO_SERVER_BASE_URL")
 SSO_SERVER = f"{SSO_SERVER_BASE}sso/"
-SSO_PRIVATE_KEY = "CHANGE_ME"
-SSO_PUBLIC_KEY = "CHANGE_ME"
 
 # OAuth settings
-OAUTH_CODE_VERIFIER = "CHANGE_ME"
+OAUTH_CODE_VERIFIER = env("OAUTH_CODE_VERIFIER")
 
-OAUTH_CLIENT_ID = "CHANGE_ME"
+OAUTH_CLIENT_ID = env("OAUTH_CLIENT_ID")
-OAUTH_CLIENT_SECRET = "CHANGE_ME"
+OAUTH_CLIENT_SECRET = env("OAUTH_CLIENT_SECRET")

konova/sub_settings/wfs_parcel_settings.py

@@ -1,12 +0,0 @@
-"""
-Author: Michel Peltriaux
-Organization: Struktur- und Genehmigungsdirektion Nord, Rhineland-Palatinate, Germany
-Contact: michel.peltriaux@sgdnord.rlp.de
-Created on: 31.01.22
-
-"""
-
-# Parcel WFS settings
-PARCEL_WFS_BASE_URL = "https://www.geoportal.rlp.de/registry/wfs/519"
-PARCEL_WFS_USER = "ksp"
-PARCEL_WFS_PW = "CHANGE_ME"

konova/tasks.py

@@ -7,7 +7,7 @@ from django.core.exceptions import ObjectDoesNotExist
 
 @shared_task
 def celery_update_parcels(geometry_id: str, recheck: bool = True):
-    from konova.models import Geometry, ParcelIntersection
+    from konova.models import Geometry
     try:
         geom = Geometry.objects.get(id=geometry_id)
         geom.parcels.clear()

konova/urls.py

@@ -13,22 +13,17 @@ Including another URLconf
     1. Import the include() function: from django.urls import include, path
     2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
 """
-import debug_toolbar
 from django.contrib import admin
 from django.urls import path, include
 
-from konova.settings import SSO_SERVER, SSO_PUBLIC_KEY, SSO_PRIVATE_KEY, DEBUG
-from konova.sso.sso import KonovaSSOClient
 from konova.views.logout import LogoutView
 from konova.views.geometry import GeomParcelsView, GeomParcelsContentView
 from konova.views.home import HomeView
 from konova.views.map_proxy import ClientProxyParcelSearch, ClientProxyParcelWFS
 from konova.views.oauth import OAuthLoginView, OAuthCallbackView
 
-sso_client = KonovaSSOClient(SSO_SERVER, SSO_PUBLIC_KEY, SSO_PRIVATE_KEY)
 urlpatterns = [
     path('admin/', admin.site.urls),
-    path('login/', include(sso_client.get_urls())),
     path('oauth/callback/', OAuthCallbackView.as_view(), name="oauth-callback"),
     path('oauth/login/', OAuthLoginView.as_view(), name="oauth-login"),
     path('logout/', LogoutView.as_view(), name="logout"),
@@ -47,10 +42,5 @@ urlpatterns = [
     path('client/proxy/wfs', ClientProxyParcelWFS.as_view(), name="client-proxy-wfs"),
 ]
 
-if DEBUG:
-    urlpatterns += [
-        path('__debug__/', include(debug_toolbar.urls)),
-    ]
-
 handler404 = "konova.views.error.get_404_view"
 handler500 = "konova.views.error.get_500_view"

konova/utils/mailer.py

@@ -9,7 +9,7 @@ from django.core.mail import send_mail
 from django.template.loader import render_to_string
 from django.utils.translation import gettext_lazy as _
 
-from konova.sub_settings.django_settings import DEFAULT_FROM_EMAIL, EMAIL_REPLY_TO, SUPPORT_MAIL_RECIPIENT
+from konova.sub_settings.django_settings import DEFAULT_FROM_EMAIL, EMAIL_REPLY_TO
 
 
 class Mailer:
@@ -416,7 +416,7 @@ class Mailer:
             "EMAIL_REPLY_TO": EMAIL_REPLY_TO,
         }
         msg = render_to_string("email/api/verify_token.html", context)
-        user_mail_address = [SUPPORT_MAIL_RECIPIENT]
+        user_mail_address = [EMAIL_REPLY_TO]
         self.send(
             user_mail_address,
             _("Request for new API token"),

konova/utils/schneider/fetcher.py

@@ -11,6 +11,7 @@ from json import JSONDecodeError
 import requests
 
 from konova.sub_settings import schneider_settings
+from konova.sub_settings.proxy_settings import PROXIES
 
 
 class ParcelFetcher:
@@ -43,6 +44,7 @@ class ParcelFetcher:
 
         response = requests.post(
             url=post_url,
+            proxies=PROXIES,
             data=self.geojson,
             headers={
                 self.auth_header: self.auth_header_token

konova/views/map_proxy.py

@@ -18,7 +18,7 @@ from django.utils.translation import gettext_lazy as _
 
 from requests.auth import HTTPDigestAuth
 
-from konova.sub_settings.proxy_settings import PROXIES, CLIENT_PROXY_AUTH_USER, CLIENT_PROXY_AUTH_PASSWORD
+from konova.sub_settings.proxy_settings import PROXIES, GEOPORTAL_RLP_USER, GEOPORTAL_RLP_PASSWORD
 
 
 class BaseClientProxyView(View):
@@ -90,7 +90,7 @@ class ClientProxyParcelWFS(BaseClientProxyView):
         url = f"{base_url}?{urlencode(params, doseq=True)}"
 
         url = url.replace("typename", "typenames")
-        auth = HTTPDigestAuth(CLIENT_PROXY_AUTH_USER, CLIENT_PROXY_AUTH_PASSWORD)
+        auth = HTTPDigestAuth(GEOPORTAL_RLP_USER, GEOPORTAL_RLP_PASSWORD)
 
         content, response_code = self.perform_url_call(url, auth=auth)
         error_detected = response_code != 200

requirements.txt

@@ -4,43 +4,43 @@ async-timeout==4.0.3
 beautifulsoup4==4.13.0b2
 billiard==4.2.0
 cached-property==1.5.2
-celery==5.4.0rc2
+celery==5.4.0
-certifi==2024.2.2
+certifi==2024.6.2
-cffi==1.16.0
+cffi==1.17.0rc1
 chardet==5.2.0
 charset-normalizer==3.3.2
 click==8.1.7
 click-didyoumean==0.3.1
 click-plugins==1.1.1
 click-repl==0.3.0
-coverage==7.4.4
+coverage==7.5.3
-cryptography==42.0.5
+cryptography==42.0.8
 Deprecated==1.2.14
-Django==5.0.4
+Django==5.0.6
 django-autocomplete-light==3.11.0
 django-bootstrap-modal-forms==3.0.4
-django-bootstrap4==24.1
+django-bootstrap4==24.3
-django-debug-toolbar==4.3.0
 django-environ==0.11.2
 django-filter==24.2
 django-fontawesome-5==1.0.18
-django-oauth-toolkit==2.3.0
+django-oauth-toolkit==2.4.0
 django-simple-sso==1.2.0
 django-tables2==2.7.0
 et-xmlfile==1.1.0
+gunicorn==22.0.0
 idna==3.7
-importlib_metadata==7.1.0
 itsdangerous==0.24
+importlib_metadata==7.1.0
 jwcrypto==1.5.6
 kombu==5.3.7
 oauthlib==3.2.2
 openpyxl==3.2.0b1
-packaging==24.0
+packaging==24.1
 pika==1.3.2
-pillow==10.2.0
+pillow==10.3.0
-prompt-toolkit==3.0.43
+prompt_toolkit==3.0.47
-psycopg==3.1.18
+psycopg==3.1.19
-psycopg-binary==3.1.18
+psycopg-binary==3.1.19
 pycparser==2.22
 pyparsing==3.1.2
 pypng==0.20220715.0
@@ -49,17 +49,16 @@ python-dateutil==2.9.0.post0
 pytz==2024.1
 PyYAML==6.0.1
 qrcode==7.3.1
-redis==5.1.0b4
+redis==5.1.0b6
-requests==2.31.0
+requests==2.32.3
 six==1.16.0
 soupsieve==2.5
-sqlparse==0.4.4
+sqlparse==0.5.0
-typing_extensions==4.11.0
+typing_extensions==4.12.2
 tzdata==2024.1
 urllib3==2.2.1
 vine==5.1.0
 wcwidth==0.2.13
-webservices==0.7
 wrapt==1.16.0
 xmltodict==0.13.0
-zipp==3.18.1
+zipp==3.19.2

templates/map/client/config.json

@@ -112,7 +112,7 @@
 	},
 	"import":
 	{
-		"geopackageLibURL": "/libs/geopackage/4.2.3/"
+		"geopackageLibURL": "/static/libs/geopackage/4.2.3/"
 	},
 	"export":
 	{