Merge pull request '# Hotfix' (#458) from oauth_fix into master

Reviewed-on: #458

.env.sample

@@ -24,6 +24,7 @@ DEFAULT_FROM_EMAIL=service@ksp.de
 
 # Proxy
 PROXY=CHANGE_ME
+MAP_PROXY_HOST_WHITELIST=CHANGE_ME_1,CHANGE_ME_2
 GEOPORTAL_RLP_USER=CHANGE_ME
 GEOPORTAL_RLP_PASSWORD=CHANGE_ME
 
@@ -37,6 +38,7 @@ SSO_SERVER_BASE_URL=https://login.naturschutz.rlp.de
 OAUTH_CODE_VERIFIER=CHANGE_ME
 OAUTH_CLIENT_ID=CHANGE_ME
 OAUTH_CLIENT_SECRET=CHANGE_ME
+PROPAGATION_SECRET=CHANGE_ME
 
 # RabbitMQ
 ## For connections to EGON

api/models/token.py

@@ -155,3 +155,25 @@ class OAuthToken(UuidModel):
 
         return user
 
+    def revoke(self) -> int:
+        """ Revokes the OAuth2 token of the user
+
+        (/o/revoke_token/ indeed removes the corresponding access token on provider side and invalidates the
+        submitted refresh token in one step)
+
+        Returns:
+            revocation_status_code (int): HTTP status code for revocation of refresh_token
+        """
+        revoke_url = f"{SSO_SERVER_BASE}o/revoke_token/"
+        token = self.refresh_token
+        revocation_status_code = requests.post(
+                revoke_url,
+                data={
+                    'token': token,
+                    'token_type_hint': "refresh_token",
+                },
+                auth=(OAUTH_CLIENT_ID, OAUTH_CLIENT_SECRET),
+            ).status_code
+
+        return revocation_status_code
+

konova/management/commands/recalculate_parcels.py

@@ -8,10 +8,9 @@ Created on: 04.01.22
 import datetime
 
 from django.contrib.gis.db.models.functions import Area
-from django.utils.timezone import now
 
 from konova.management.commands.setup import BaseKonovaCommand
-from konova.models import Geometry, Parcel, District
+from konova.models import Geometry, ParcelIntersection
 
 
 class Command(BaseKonovaCommand):
@@ -35,32 +34,53 @@ class Command(BaseKonovaCommand):
     def recalculate_parcels(self, options: dict):
         force_all = options.get("force_all", False)
 
-        if force_all:
+        geometry_objects = Geometry.objects.all().exclude(
-            geometry_objects = Geometry.objects.all()
+            geom=None
-        else:
+        )
-            _today = now().date()
+
-            _date_threshold = _today - datetime.timedelta(days=1)
+        if not force_all:
-            geometry_objects = Geometry.objects.filter(
+            # Fetch all intersections
-                parcel_update_start__date__lte=_date_threshold,
+            intersection_objs = ParcelIntersection.objects.filter(
-                parcel_update_end__isnull=True
+                geometry__in=geometry_objects
             )
+            # Just take the geometry ids, which seem to have intersections
+            geom_with_intersection_ids = intersection_objs.values_list(
+                "geometry__id",
+                flat=True
+            )
+            # ... and resolve into Geometry objects again ...
+            intersected_geom_objs = Geometry.objects.filter(
+                id__in=geom_with_intersection_ids
+            )
+            # ... to be able to use the way more efficient difference() function ...
+            geometry_objects_ids = geometry_objects.difference(intersected_geom_objs).values_list("id", flat=True)
+            # ... so we can resolve these into proper Geometry objects again for further annotation usage
+            geometry_objects = Geometry.objects.filter(id__in=geometry_objects_ids)
 
         self._write_warning("=== Update parcels and districts ===")
         # Order geometries by size to process smaller once at first
-        geometries = geometry_objects.exclude(
+        geometries = geometry_objects.annotate(
-            geom=None
+            area=Area("geom")
-        ).annotate(area=Area("geom")).order_by(
+        ).order_by(
             'area'
         )
         self._write_warning(f"Process parcels for {geometries.count()} geometry entries now ...")
         i = 0
         num_geoms = geometries.count()
+        geoms_with_errors = {}
         for geometry in geometries:
             self._write_warning(f"--- {datetime.datetime.now()} Process {geometry.id} now ...")
-            geometry.update_parcels()
+            try:
-            self._write_warning(f"--- Processed {geometry.get_underlying_parcels().count()} underlying parcels")
+                geometry.update_parcels()
+                self._write_warning(f"--- Processed {geometry.get_underlying_parcels().count()} underlying parcels")
+            except Exception as e:
+                geoms_with_errors[geometry.id] = str(e)
             i += 1
             self._write_warning(f"--- {i}/{num_geoms} processed")
 
         self._write_success("Updating parcels done!")
+
+        for key, val in geoms_with_errors.items():
+            self._write_error(f"    Error on {key}: {val}")
+        self._write_success(f"{num_geoms - len(geoms_with_errors)} geometries successfuly recalculated!")
         self._break_line()

konova/models/geometry.py

@@ -8,7 +8,7 @@ Created on: 15.11.21
 import json
 
 from django.contrib.gis.db.models import MultiPolygonField
-from django.core.exceptions import ObjectDoesNotExist
+from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
 from django.db import models, transaction
 from django.utils import timezone
 
@@ -223,6 +223,17 @@ class Geometry(BaseResource):
                 )
                 parcel_obj.updated_on = _now
                 parcels_to_update.append(parcel_obj)
+            except MultipleObjectsReturned:
+                parcel_obj = Parcel.make_unique(
+                    district=district,
+                    municipal=municipal,
+                    parcel_group=parcel_group,
+                    flr=flr_val,
+                    flrstck_nnr=flrstck_nnr,
+                    flrstck_zhlr=flrstck_zhlr,
+                )
+                parcel_obj.updated_on = _now
+                parcels_to_update.append(parcel_obj)
             except ObjectDoesNotExist:
                 # If not existing, create object but do not commit, yet
                 parcel_obj = Parcel(
@@ -366,11 +377,10 @@ class Geometry(BaseResource):
         diff = geom_envelope - self.geom
 
         if diff.area == 0:
-            ratio = 1
+            complexity_factor = 1
         else:
-            ratio = self.geom.area / diff.area
+            complexity_factor = self.geom.area / diff.area
 
-        complexity_factor = 1 - ratio
         return complexity_factor
 
 

konova/models/parcel.py

@@ -5,7 +5,7 @@ Contact: michel.peltriaux@sgdnord.rlp.de
 Created on: 16.12.21
 
 """
-from django.db import models
+from django.db import models, transaction
 
 from konova.models import UuidModel
 
@@ -158,6 +158,46 @@ class Parcel(UuidModel):
     def __str__(self):
         return f"{self.parcel_group} | {self.flr} | {self.flrstck_zhlr} | {self.flrstck_nnr}"
 
+    @classmethod
+    def make_unique(cls, **kwargs):
+        """ Checks for duplicates of a Parcel, choose a (now) unique one,
+        repairs relations for ParcelIntersection and removes duplicates.
+
+        Args:
+            **kwargs ():
+
+        Returns:
+            unique_true (Parcel): The new unique 'true one'
+        """
+        parcel_objs = Parcel.objects.filter(**kwargs)
+
+        if not parcel_objs.exists():
+            return None
+
+        # Get one of the found parcels and use it as new 'true one'
+        unique_parcel = parcel_objs.first()
+        # separate it from the rest
+        parcel_objs = parcel_objs.exclude(id=unique_parcel.id)
+
+        if not parcel_objs.exists():
+            # There are no duplicates - all good, just return
+            return unique_parcel
+
+        # Fetch existing intersections, which still point on the duplicated parcels
+        intersection_objs = ParcelIntersection.objects.filter(
+            parcel__in=parcel_objs
+        )
+
+        # Change each intersection, so they point on the 'true one' parcel from now on
+        for intersection in intersection_objs:
+            intersection.parcel = unique_parcel
+            intersection.save()
+
+        # Remove the duplicated parcels
+        parcel_objs.delete()
+
+        return unique_parcel
+
 
 class ParcelIntersection(UuidModel):
     """

konova/sub_settings/lanis_settings.py

@@ -5,6 +5,7 @@ Contact: michel.peltriaux@sgdnord.rlp.de
 Created on: 31.01.22
 
 """
+from konova.sub_settings.django_settings import env
 
 # MAPS
 DEFAULT_LAT = 50.00
@@ -28,3 +29,6 @@ LANIS_ZOOM_LUT = {
     1000: 30,
     500: 31,
 }
+
+MAP_PROXY_HOST_WHITELIST = env.list("MAP_PROXY_HOST_WHITELIST")
+i = 0

konova/sub_settings/sso_settings.py

@@ -16,3 +16,5 @@ OAUTH_CODE_VERIFIER = env("OAUTH_CODE_VERIFIER")
 
 OAUTH_CLIENT_ID = env("OAUTH_CLIENT_ID")
 OAUTH_CLIENT_SECRET = env("OAUTH_CLIENT_SECRET")
+
+PROPAGATION_SECRET = env("PROPAGATION_SECRET")

konova/views/logout.py

@@ -24,5 +24,13 @@ class LogoutView(View):
         Returns:
             A redirect
         """
+        user = request.user
+        try:
+            oauth_token = user.oauth_token
+            if oauth_token:
+                oauth_token.revoke()
+        except AttributeError:
+            pass
+
         logout(request)
         return redirect(SSO_SERVER_BASE)

konova/views/map_proxy.py

@@ -9,6 +9,7 @@ import json
 from json import JSONDecodeError
 
 import requests
+import urllib3.util
 from django.contrib.auth.decorators import login_required
 from django.http import JsonResponse, HttpRequest
 from django.utils.decorators import method_decorator
@@ -18,6 +19,7 @@ from django.utils.translation import gettext_lazy as _
 
 from requests.auth import HTTPDigestAuth
 
+from konova.sub_settings.lanis_settings import MAP_PROXY_HOST_WHITELIST
 from konova.sub_settings.proxy_settings import PROXIES, GEOPORTAL_RLP_USER, GEOPORTAL_RLP_PASSWORD
 
 
@@ -32,6 +34,13 @@ class BaseClientProxyView(View):
     def dispatch(self, request, *args, **kwargs):
         return super().dispatch(request, *args, **kwargs)
 
+    def _check_with_whitelist(self, url):
+        parsed_url = urllib3.util.parse_url(url)
+        parsed_url_host = parsed_url.host
+        whitelist = set(MAP_PROXY_HOST_WHITELIST)
+        is_allowed = parsed_url_host in whitelist
+        return is_allowed
+
     def perform_url_call(self, url, headers={}, auth=None):
         """ Generic proxied call
 
@@ -59,6 +68,11 @@ class ClientProxyParcelSearch(BaseClientProxyView):
 
     def get(self, request: HttpRequest):
         url = request.META.get("QUERY_STRING")
+
+        is_url_allowed = self._check_with_whitelist(url)
+        if not is_url_allowed:
+            raise PermissionError(f"Proxied url '{url}' is not allowed!")
+
         content, response_code = self.perform_url_call(url)
         try:
             body = json.loads(content)

konova/views/oauth.py

@@ -115,10 +115,10 @@ class OAuthCallbackView(View):
         if status_code_invalid:
             raise RuntimeError(f"OAuth access token could not be fetched: {access_code_response.text}")
 
-        oauth_access_token = OAuthToken.from_access_token_response(access_code_response_body, received_on)
+        oauth_token = OAuthToken.from_access_token_response(access_code_response_body, received_on)
-        oauth_access_token.save()
+        oauth_token.save()
-        user = oauth_access_token.update_and_get_user()
+        user = oauth_token.update_and_get_user()
-        user.oauth_replace_token(oauth_access_token)
+        user.oauth_replace_token(oauth_token)
 
         login(request, user)
         return redirect("home")

requirements.txt

@@ -1,65 +1,65 @@
-amqp==5.2.0
+amqp==5.3.1
 asgiref==3.8.1
-async-timeout==4.0.3
+async-timeout==5.0.1
 beautifulsoup4==4.13.0b2
-billiard==4.2.0
+billiard==4.2.1
-cached-property==1.5.2
+cached-property==2.0.1
 celery==5.4.0
-certifi==2024.7.4
+certifi==2024.12.14
-cffi==1.17.0
+cffi==1.17.1
 chardet==5.2.0
-charset-normalizer==3.3.2
+charset-normalizer==3.4.0
-click==8.1.7
+click==8.1.8
 click-didyoumean==0.3.1
 click-plugins==1.1.1
 click-repl==0.3.0
-coverage==7.5.4
+coverage==7.6.9
-cryptography==43.0.0
+cryptography==44.0.0
-Deprecated==1.2.14
+Deprecated==1.2.15
-Django==5.0.8
+Django==5.1.4
 django-autocomplete-light==3.11.0
-django-bootstrap-modal-forms==3.0.4
+django-bootstrap-modal-forms==3.0.5
-django-bootstrap4==24.3
+django-bootstrap4==24.4
 django-environ==0.11.2
 django-filter==24.3
 django-fontawesome-5==1.0.18
-django-oauth-toolkit==2.4.0
+django-oauth-toolkit==3.0.1
 django-simple-sso==1.2.0
-django-tables2==2.7.0
+django-tables2==2.7.1
-et-xmlfile==1.1.0
+et_xmlfile==2.0.0
-gunicorn==22.0.0
+gunicorn==23.0.0
-idna==3.7
+idna==3.10
-importlib_metadata==8.2.0
+importlib_metadata==8.5.0
 itsdangerous==0.24
 jwcrypto==1.5.6
 kombu==5.4.0rc1
 oauthlib==3.2.2
 openpyxl==3.2.0b1
-packaging==24.1
+packaging==24.2
 pika==1.3.2
-pillow==10.4.0
+pillow==11.0.0
-prompt_toolkit==3.0.47
+prompt_toolkit==3.0.48
-psycopg==3.2.1
+psycopg==3.2.3
-psycopg-binary==3.2.1
+psycopg-binary==3.2.3
 pycparser==2.22
-pyparsing==3.1.2
+pyparsing==3.2.0
 pypng==0.20220715.0
-pyproj==3.6.1
+pyproj==3.7.0
 python-dateutil==2.9.0.post0
-pytz==2024.1
+pytz==2024.2
 PyYAML==6.0.2
 qrcode==7.3.1
 redis==5.1.0b6
-requests<2.32.0  # kombu 5.4.0rc1 depends on requests<2.32.0
+requests==2.32.3
 six==1.16.0
 soupsieve==2.5
 sqlparse==0.5.1
 typing_extensions==4.12.2
-tzdata==2024.1
+tzdata==2024.2
-urllib3==2.2.2
+urllib3==2.3.0
 vine==5.1.0
 wcwidth==0.2.13
 webservices==0.7
 wrapt==1.16.0
-xmltodict==0.13.0
+xmltodict==0.14.2
-zipp==3.19.2
+zipp==3.21.0

user/views/propagate.py

@@ -16,7 +16,7 @@ from django.utils.decorators import method_decorator
 from django.views import View
 from django.views.decorators.csrf import csrf_exempt
 
-from konova.sub_settings.sso_settings import OAUTH_CLIENT_ID
+from konova.sub_settings.sso_settings import PROPAGATION_SECRET
 from user.models import User
 
 
@@ -36,7 +36,7 @@ class PropagateUserView(View):
         # Decrypt
         encrypted_body = request.body
         _hash = hashlib.md5()
-        _hash.update(OAUTH_CLIENT_ID.encode("utf-8"))
+        _hash.update(PROPAGATION_SECRET.encode("utf-8"))
         key = base64.urlsafe_b64encode(_hash.hexdigest().encode("utf-8"))
         fernet = Fernet(key)
         body = fernet.decrypt(encrypted_body).decode("utf-8")