Merge pull request 'oauth_fix' (#453) from oauth_fix into master

Reviewed-on: #453

.env.sample

@@ -24,6 +24,7 @@ DEFAULT_FROM_EMAIL=service@ksp.de
 
 # Proxy
 PROXY=CHANGE_ME
+MAP_PROXY_HOST_WHITELIST=CHANGE_ME_1,CHANGE_ME_2
 GEOPORTAL_RLP_USER=CHANGE_ME
 GEOPORTAL_RLP_PASSWORD=CHANGE_ME
 
@@ -37,6 +38,7 @@ SSO_SERVER_BASE_URL=https://login.naturschutz.rlp.de
 OAUTH_CODE_VERIFIER=CHANGE_ME
 OAUTH_CLIENT_ID=CHANGE_ME
 OAUTH_CLIENT_SECRET=CHANGE_ME
+PROPAGATION_SECRET=CHANGE_ME
 
 # RabbitMQ
 ## For connections to EGON

api/models/token.py

@@ -155,3 +155,25 @@ class OAuthToken(UuidModel):
 
         return user
 
+    def revoke(self) -> int:
+        """ Revokes the OAuth2 token of the user
+
+        (/o/revoke_token/ indeed removes the corresponding access token on provider side and invalidates the
+        submitted refresh token in one step)
+
+        Returns:
+            revocation_status_code (int): HTTP status code for revocation of refresh_token
+        """
+        revoke_url = f"{SSO_SERVER_BASE}o/revoke_token/"
+        token = self.refresh_token
+        revocation_status_code = requests.post(
+                revoke_url,
+                data={
+                    'token': token,
+                    'token_type_hint': "refresh_token",
+                },
+                auth=(OAUTH_CLIENT_ID, OAUTH_CLIENT_SECRET),
+            ).status_code
+
+        return revocation_status_code
+

codelist/migrations/0003_alter_konovacode_long_name_and_more.py

@@ -0,0 +1,25 @@
+# Generated by Django 5.0.8 on 2024-08-26 16:47
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('codelist', '0002_migrate_975_to_288'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='konovacode',
+            name='long_name',
+            field=models.CharField(blank=True, default="", max_length=1000),
+            preserve_default=False,
+        ),
+        migrations.AlterField(
+            model_name='konovacode',
+            name='short_name',
+            field=models.CharField(blank=True, default="", help_text='Short version of long name', max_length=500),
+            preserve_default=False,
+        ),
+    ]

compensation/migrations/0016_alter_compensationstate_biotope_type_details.py

@@ -0,0 +1,19 @@
+# Generated by Django 5.0.8 on 2024-08-26 16:47
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('codelist', '0003_alter_konovacode_long_name_and_more'),
+        ('compensation', '0015_alter_compensation_after_states_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='compensationstate',
+            name='biotope_type_details',
+            field=models.ManyToManyField(blank=True, limit_choices_to={'code_lists__in': [288], 'is_archived': False, 'is_selectable': True}, related_name='+', to='codelist.konovacode'),
+        ),
+    ]

konova/management/commands/generate_report.py

@@ -6,11 +6,11 @@ Created on: 26.10.22
 
 """
 import zipfile
+from datetime import datetime
 from io import BytesIO
 
 from django.core.mail import EmailMessage
 from django.utils import timezone
-from django.utils.datetime_safe import datetime
 
 from analysis.utils.excel.excel import TempExcelFile
 from analysis.utils.report import TimespanReport

konova/management/commands/recalculate_parcels.py

@@ -0,0 +1,86 @@
+"""
+Author: Michel Peltriaux
+Organization: Struktur- und Genehmigungsdirektion Nord, Rhineland-Palatinate, Germany
+Contact: michel.peltriaux@sgdnord.rlp.de
+Created on: 04.01.22
+
+"""
+import datetime
+
+from django.contrib.gis.db.models.functions import Area
+
+from konova.management.commands.setup import BaseKonovaCommand
+from konova.models import Geometry, ParcelIntersection
+
+
+class Command(BaseKonovaCommand):
+    help = "Recalculates parcels for entries with geometry but missing parcel information"
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--force-all",
+            action="store_true",
+            default=False,
+            help="If Attribute set, all entries parcels will be recalculated"
+        )
+
+    def handle(self, *args, **options):
+        try:
+            self.recalculate_parcels(options)
+        except KeyboardInterrupt:
+            self._break_line()
+            exit(-1)
+
+    def recalculate_parcels(self, options: dict):
+        force_all = options.get("force_all", False)
+
+        geometry_objects = Geometry.objects.all().exclude(
+            geom=None
+        )
+
+        if not force_all:
+            # Fetch all intersections
+            intersection_objs = ParcelIntersection.objects.filter(
+                geometry__in=geometry_objects
+            )
+            # Just take the geometry ids, which seem to have intersections
+            geom_with_intersection_ids = intersection_objs.values_list(
+                "geometry__id",
+                flat=True
+            )
+            # ... and resolve into Geometry objects again ...
+            intersected_geom_objs = Geometry.objects.filter(
+                id__in=geom_with_intersection_ids
+            )
+            # ... to be able to use the way more efficient difference() function ...
+            geometry_objects_ids = geometry_objects.difference(intersected_geom_objs).values_list("id", flat=True)
+            # ... so we can resolve these into proper Geometry objects again for further annotation usage
+            geometry_objects = Geometry.objects.filter(id__in=geometry_objects_ids)
+
+        self._write_warning("=== Update parcels and districts ===")
+        # Order geometries by size to process smaller once at first
+        geometries = geometry_objects.annotate(
+            area=Area("geom")
+        ).order_by(
+            'area'
+        )
+        self._write_warning(f"Process parcels for {geometries.count()} geometry entries now ...")
+        i = 0
+        num_geoms = geometries.count()
+        geoms_with_errors = {}
+        for geometry in geometries:
+            self._write_warning(f"--- {datetime.datetime.now()} Process {geometry.id} now ...")
+            try:
+                geometry.update_parcels()
+                self._write_warning(f"--- Processed {geometry.get_underlying_parcels().count()} underlying parcels")
+            except Exception as e:
+                geoms_with_errors[geometry.id] = str(e)
+            i += 1
+            self._write_warning(f"--- {i}/{num_geoms} processed")
+
+        self._write_success("Updating parcels done!")
+
+        for key, val in geoms_with_errors.items():
+            self._write_error(f"    Error on {key}: {val}")
+        self._write_success(f"{num_geoms - len(geoms_with_errors)} geometries successfuly recalculated!")
+        self._break_line()

konova/management/commands/update_all_parcels.py

@@ -1,54 +0,0 @@
-"""
-Author: Michel Peltriaux
-Organization: Struktur- und Genehmigungsdirektion Nord, Rhineland-Palatinate, Germany
-Contact: michel.peltriaux@sgdnord.rlp.de
-Created on: 04.01.22
-
-"""
-import datetime
-
-from django.contrib.gis.db.models.functions import Area
-
-from konova.management.commands.setup import BaseKonovaCommand
-from konova.models import Geometry, Parcel, District
-
-
-class Command(BaseKonovaCommand):
-    help = "Checks the database' sanity and removes unused entries"
-
-    def handle(self, *args, **options):
-        try:
-            self.update_all_parcels()
-        except KeyboardInterrupt:
-            self._break_line()
-            exit(-1)
-
-    def update_all_parcels(self):
-        num_parcels_before = Parcel.objects.count()
-        num_districts_before = District.objects.count()
-        self._write_warning("=== Update parcels and districts ===")
-        # Order geometries by size to process smaller once at first
-        geometries = Geometry.objects.all().exclude(
-            geom=None
-        ).annotate(area=Area("geom")).order_by(
-            'area'
-        )
-        self._write_warning(f"Process parcels for {geometries.count()} geometry entries now ...")
-        i = 0
-        num_geoms = geometries.count()
-        for geometry in geometries:
-            self._write_warning(f"--- {datetime.datetime.now()} Process {geometry.id} now ...")
-            geometry.update_parcels()
-            self._write_warning(f"--- Processed {geometry.get_underlying_parcels().count()} underlying parcels")
-            i += 1
-            self._write_warning(f"--- {i}/{num_geoms} processed")
-
-        num_parcels_after = Parcel.objects.count()
-        num_districts_after = District.objects.count()
-        if num_parcels_after != num_parcels_before:
-            self._write_error(f"Parcels have changed: {num_parcels_before} to {num_parcels_after} entries. You should run the sanitize command.")
-        if num_districts_after != num_districts_before:
-            self._write_error(f"Districts have changed: {num_districts_before} to {num_districts_after} entries. You should run the sanitize command.")
-
-        self._write_success("Updating parcels done!")
-        self._break_line()

konova/models/geometry.py

@@ -8,7 +8,7 @@ Created on: 15.11.21
 import json
 
 from django.contrib.gis.db.models import MultiPolygonField
-from django.core.exceptions import ObjectDoesNotExist
+from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
 from django.db import models, transaction
 from django.utils import timezone
 
@@ -223,6 +223,17 @@ class Geometry(BaseResource):
                 )
                 parcel_obj.updated_on = _now
                 parcels_to_update.append(parcel_obj)
+            except MultipleObjectsReturned:
+                parcel_obj = Parcel.make_unique(
+                    district=district,
+                    municipal=municipal,
+                    parcel_group=parcel_group,
+                    flr=flr_val,
+                    flrstck_nnr=flrstck_nnr,
+                    flrstck_zhlr=flrstck_zhlr,
+                )
+                parcel_obj.updated_on = _now
+                parcels_to_update.append(parcel_obj)
             except ObjectDoesNotExist:
                 # If not existing, create object but do not commit, yet
                 parcel_obj = Parcel(
@@ -366,11 +377,10 @@ class Geometry(BaseResource):
         diff = geom_envelope - self.geom
 
         if diff.area == 0:
-            ratio = 1
+            complexity_factor = 1
         else:
-            ratio = self.geom.area / diff.area
+            complexity_factor = self.geom.area / diff.area
 
-        complexity_factor = 1 - ratio
         return complexity_factor
 
 

konova/models/parcel.py

@@ -5,7 +5,7 @@ Contact: michel.peltriaux@sgdnord.rlp.de
 Created on: 16.12.21
 
 """
-from django.db import models
+from django.db import models, transaction
 
 from konova.models import UuidModel
 
@@ -158,6 +158,46 @@ class Parcel(UuidModel):
     def __str__(self):
         return f"{self.parcel_group} | {self.flr} | {self.flrstck_zhlr} | {self.flrstck_nnr}"
 
+    @classmethod
+    def make_unique(cls, **kwargs):
+        """ Checks for duplicates of a Parcel, choose a (now) unique one,
+        repairs relations for ParcelIntersection and removes duplicates.
+
+        Args:
+            **kwargs ():
+
+        Returns:
+            unique_true (Parcel): The new unique 'true one'
+        """
+        parcel_objs = Parcel.objects.filter(**kwargs)
+
+        if not parcel_objs.exists():
+            return None
+
+        # Get one of the found parcels and use it as new 'true one'
+        unique_parcel = parcel_objs.first()
+        # separate it from the rest
+        parcel_objs = parcel_objs.exclude(id=unique_parcel.id)
+
+        if not parcel_objs.exists():
+            # There are no duplicates - all good, just return
+            return unique_parcel
+
+        # Fetch existing intersections, which still point on the duplicated parcels
+        intersection_objs = ParcelIntersection.objects.filter(
+            parcel__in=parcel_objs
+        )
+
+        # Change each intersection, so they point on the 'true one' parcel from now on
+        for intersection in intersection_objs:
+            intersection.parcel = unique_parcel
+            intersection.save()
+
+        # Remove the duplicated parcels
+        parcel_objs.delete()
+
+        return unique_parcel
+
 
 class ParcelIntersection(UuidModel):
     """

konova/sub_settings/lanis_settings.py

@@ -5,6 +5,7 @@ Contact: michel.peltriaux@sgdnord.rlp.de
 Created on: 31.01.22
 
 """
+from konova.sub_settings.django_settings import env
 
 # MAPS
 DEFAULT_LAT = 50.00
@@ -28,3 +29,6 @@ LANIS_ZOOM_LUT = {
     1000: 30,
     500: 31,
 }
+
+MAP_PROXY_HOST_WHITELIST = env.list("MAP_PROXY_HOST_WHITELIST")
+i = 0

konova/sub_settings/sso_settings.py

@@ -16,3 +16,5 @@ OAUTH_CODE_VERIFIER = env("OAUTH_CODE_VERIFIER")
 
 OAUTH_CLIENT_ID = env("OAUTH_CLIENT_ID")
 OAUTH_CLIENT_SECRET = env("OAUTH_CLIENT_SECRET")
+
+PROPAGATION_SECRET = env("PROPAGATION_SECRET")

konova/utils/schneider/fetcher.py

@@ -55,11 +55,11 @@ class ParcelFetcher:
             content = json.loads(response.content.decode("utf-8"))
         except JSONDecodeError:
             content = {}
-        next = content.get("next", None)
+        _next = content.get("next", None)
         fetched_parcels = content.get("results", [])
         self.results += fetched_parcels
 
-        if next:
-            self.get_parcels(next)
+        if _next:
+            self.get_parcels(_next)
 
         return self.results

konova/views/logout.py

@@ -24,5 +24,10 @@ class LogoutView(View):
         Returns:
             A redirect
         """
+        user = request.user
+        oauth_token = user.oauth_token
+        if oauth_token:
+            oauth_token.revoke()
+
         logout(request)
         return redirect(SSO_SERVER_BASE)

konova/views/map_proxy.py

@@ -9,6 +9,7 @@ import json
 from json import JSONDecodeError
 
 import requests
+import urllib3.util
 from django.contrib.auth.decorators import login_required
 from django.http import JsonResponse, HttpRequest
 from django.utils.decorators import method_decorator
@@ -18,6 +19,7 @@ from django.utils.translation import gettext_lazy as _
 
 from requests.auth import HTTPDigestAuth
 
+from konova.sub_settings.lanis_settings import MAP_PROXY_HOST_WHITELIST
 from konova.sub_settings.proxy_settings import PROXIES, GEOPORTAL_RLP_USER, GEOPORTAL_RLP_PASSWORD
 
 
@@ -32,6 +34,13 @@ class BaseClientProxyView(View):
     def dispatch(self, request, *args, **kwargs):
         return super().dispatch(request, *args, **kwargs)
 
+    def _check_with_whitelist(self, url):
+        parsed_url = urllib3.util.parse_url(url)
+        parsed_url_host = parsed_url.host
+        whitelist = set(MAP_PROXY_HOST_WHITELIST)
+        is_allowed = parsed_url_host in whitelist
+        return is_allowed
+
     def perform_url_call(self, url, headers={}, auth=None):
         """ Generic proxied call
 
@@ -59,6 +68,11 @@ class ClientProxyParcelSearch(BaseClientProxyView):
 
     def get(self, request: HttpRequest):
         url = request.META.get("QUERY_STRING")
+
+        is_url_allowed = self._check_with_whitelist(url)
+        if not is_url_allowed:
+            raise PermissionError(f"Proxied url '{url}' is not allowed!")
+
         content, response_code = self.perform_url_call(url)
         try:
             body = json.loads(content)

konova/views/oauth.py

@@ -115,10 +115,10 @@ class OAuthCallbackView(View):
         if status_code_invalid:
             raise RuntimeError(f"OAuth access token could not be fetched: {access_code_response.text}")
 
-        oauth_access_token = OAuthToken.from_access_token_response(access_code_response_body, received_on)
-        oauth_access_token.save()
-        user = oauth_access_token.update_and_get_user()
-        user.oauth_replace_token(oauth_access_token)
+        oauth_token = OAuthToken.from_access_token_response(access_code_response_body, received_on)
+        oauth_token.save()
+        user = oauth_token.update_and_get_user()
+        user.oauth_replace_token(oauth_token)
 
         login(request, user)
         return redirect("home")

requirements.txt

@@ -1,65 +1,65 @@
-amqp==5.2.0
+amqp==5.3.1
 asgiref==3.8.1
-async-timeout==4.0.3
+async-timeout==5.0.1
 beautifulsoup4==4.13.0b2
-billiard==4.2.0
-cached-property==1.5.2
+billiard==4.2.1
+cached-property==2.0.1
 celery==5.4.0
-certifi==2024.7.4
-cffi==1.17.0
+certifi==2024.12.14
+cffi==1.17.1
 chardet==5.2.0
-charset-normalizer==3.3.2
-click==8.1.7
+charset-normalizer==3.4.0
+click==8.1.8
 click-didyoumean==0.3.1
 click-plugins==1.1.1
 click-repl==0.3.0
-coverage==7.5.4
-cryptography==43.0.0
-Deprecated==1.2.14
-Django==5.0.8
+coverage==7.6.9
+cryptography==44.0.0
+Deprecated==1.2.15
+Django==5.1.4
 django-autocomplete-light==3.11.0
-django-bootstrap-modal-forms==3.0.4
-django-bootstrap4==24.3
+django-bootstrap-modal-forms==3.0.5
+django-bootstrap4==24.4
 django-environ==0.11.2
 django-filter==24.3
 django-fontawesome-5==1.0.18
-django-oauth-toolkit==2.4.0
+django-oauth-toolkit==3.0.1
 django-simple-sso==1.2.0
-django-tables2==2.7.0
-et-xmlfile==1.1.0
-gunicorn==22.0.0
-idna==3.7
-importlib_metadata==8.2.0
+django-tables2==2.7.1
+et_xmlfile==2.0.0
+gunicorn==23.0.0
+idna==3.10
+importlib_metadata==8.5.0
 itsdangerous==0.24
 jwcrypto==1.5.6
 kombu==5.4.0rc1
 oauthlib==3.2.2
 openpyxl==3.2.0b1
-packaging==24.1
+packaging==24.2
 pika==1.3.2
-pillow==10.4.0
-prompt_toolkit==3.0.47
-psycopg==3.2.1
-psycopg-binary==3.2.1
+pillow==11.0.0
+prompt_toolkit==3.0.48
+psycopg==3.2.3
+psycopg-binary==3.2.3
 pycparser==2.22
-pyparsing==3.1.2
+pyparsing==3.2.0
 pypng==0.20220715.0
-pyproj==3.6.1
+pyproj==3.7.0
 python-dateutil==2.9.0.post0
-pytz==2024.1
+pytz==2024.2
 PyYAML==6.0.2
 qrcode==7.3.1
 redis==5.1.0b6
-requests<2.32.0  # kombu 5.4.0rc1 depends on requests<2.32.0
+requests==2.32.3
 six==1.16.0
 soupsieve==2.5
 sqlparse==0.5.1
 typing_extensions==4.12.2
-tzdata==2024.1
-urllib3==2.2.2
+tzdata==2024.2
+urllib3==2.3.0
 vine==5.1.0
 wcwidth==0.2.13
 webservices==0.7
 wrapt==1.16.0
-xmltodict==0.13.0
-zipp==3.19.2
+xmltodict==0.14.2
+zipp==3.21.0

user/views/propagate.py

@@ -16,7 +16,7 @@ from django.utils.decorators import method_decorator
 from django.views import View
 from django.views.decorators.csrf import csrf_exempt
 
-from konova.sub_settings.sso_settings import OAUTH_CLIENT_ID
+from konova.sub_settings.sso_settings import PROPAGATION_SECRET
 from user.models import User
 
 
@@ -35,9 +35,9 @@ class PropagateUserView(View):
     def post(self, request: HttpRequest, *args, **kwargs):
         # Decrypt
         encrypted_body = request.body
-        hash = hashlib.md5()
-        hash.update(OAUTH_CLIENT_ID.encode("utf-8"))
-        key = base64.urlsafe_b64encode(hash.hexdigest().encode("utf-8"))
+        _hash = hashlib.md5()
+        _hash.update(PROPAGATION_SECRET.encode("utf-8"))
+        key = base64.urlsafe_b64encode(_hash.hexdigest().encode("utf-8"))
         fernet = Fernet(key)
         body = fernet.decrypt(encrypted_body).decode("utf-8")
         body = json.loads(body)