Merge pull request '535_Support_foreign_key_lookup_on_API_requests' (#536) from 535_Support_foreign_key_lookup_on_API_requests into master

Reviewed-on: #536

api/migrations/0004_externalidentifier.py

@@ -0,0 +1,23 @@
+# Generated by Django 6.0.5 on 2026-05-10 07:18
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0003_oauthtoken'),
+        ('user', '0010_user_sso_identifier'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ExternalIdentifier',
+            fields=[
+                ('external_id', models.CharField(db_comment='Identifier from a source system', max_length=255, primary_key=True, serialize=False)),
+                ('internal_id', models.UUIDField(db_comment='Identifier in konova')),
+                ('created', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='user.useractionlogentry')),
+            ],
+        ),
+    ]

api/models/__init__.py

@@ -6,3 +6,4 @@ Created on: 21.01.22
 
 """
 from .token import *
+from .external_identifier import *

api/models/external_identifier.py

@@ -0,0 +1,33 @@
+"""
+Author: Michel Peltriaux
+Created on: 10.05.26
+
+"""
+from django.db import models
+
+
+class ExternalIdentifier(models.Model):
+    """ Holds a lookup to match a given external identifier against the internal identifier in konova.
+
+    Relevant in cases of API transmitted entries, which are updates using external identifiers instead of
+    the internal ones directly.
+
+    """
+    external_id = models.CharField(
+        max_length=255,
+        primary_key=True,
+        db_comment="Identifier from a source system"
+    )
+    internal_id = models.UUIDField(
+        db_comment="Identifier in konova"
+    )
+    created = models.ForeignKey(
+        "user.UserActionLogEntry",
+        on_delete=models.SET_NULL,
+        null=True,
+        blank=True,
+        related_name='+'
+    )
+
+    def __str__(self):
+        return f"{self.external_id} -> {self.internal_id}"

api/tests/v1/create/compensation_create_post_body.json

@@ -8,6 +8,7 @@
         "is_coherence_keeping": false,
         "is_pik": false,
         "intervention": "MUST_BE_SET_IN_TEST",
+        "external_identifier": "LOREMIPSUM-123",
         "before_states": [
         ],
         "after_states": [

api/tests/v1/create/ecoaccount_create_post_body.json

@@ -4,6 +4,7 @@
     ],
     "properties": {
         "title": "Test_ecoaccount",
+        "external_identifier": "LOREMIPSUM-1234",
         "deductable_surface": 10000.0,
         "is_pik": false,
         "responsible": {

api/tests/v1/create/ema_create_post_body.json

@@ -4,6 +4,7 @@
     ],
     "properties": {
         "title": "Test_ema",
+        "external_identifier": "LOREMIPSUM-1235",
         "is_pik": false,
         "responsible": {
             "conservation_office": null,

api/tests/v1/create/intervention_create_post_body.json

@@ -4,6 +4,7 @@
     ],
     "properties": {
         "title": "Test_intervention",
+        "external_identifier": "LOREMIPSUM-1236",
         "responsible": {
             "registration_office": null,
             "registration_file_number": null,

api/tests/v1/create/test_api_create.py

@@ -9,6 +9,7 @@ import json
 
 from django.urls import reverse
 
+from api.models import ExternalIdentifier
 from api.tests.v1.share.test_api_sharing import BaseAPIV1TestCase
 
 
@@ -42,7 +43,22 @@ class APIV1CreateTestCase(BaseAPIV1TestCase):
         response = self._run_create_request(url, post_body)
         self.assertEqual(response.status_code, 200, msg=response.content)
         content = json.loads(response.content)
-        self.assertIsNotNone(content.get("id", None), msg=response.content)
+        _id = content.get("id", None)
+        self.assertIsNotNone(_id, msg=response.content)
+        return _id
+
+    def _test_external_identifier_created(self, internal_id, external_id):
+        """ Tests whether an external identifier has been created
+
+        Args:
+            internal_id ():
+            external_id ():
+
+        Returns:
+
+        """
+        external_identifier = ExternalIdentifier.objects.get(internal_id=internal_id)
+        self.assertEqual(external_identifier.external_id, external_id)
 
     def test_create_intervention(self):
         """ Tests api creation
@@ -54,7 +70,8 @@ class APIV1CreateTestCase(BaseAPIV1TestCase):
         json_file_path = "api/tests/v1/create/intervention_create_post_body.json"
         with open(json_file_path) as json_file:
             post_body = json.load(fp=json_file)
-        self._test_create_object(url, post_body)
+        internal_id = self._test_create_object(url, post_body)
+        self._test_external_identifier_created(internal_id, post_body["properties"]["external_identifier"])
 
     def test_create_compensation(self):
         """ Tests api creation
@@ -77,7 +94,8 @@ class APIV1CreateTestCase(BaseAPIV1TestCase):
 
         # Add the user to the shared users of the intervention and try again! Now everything should work as expected.
         self.intervention.users.add(self.superuser)
-        self._test_create_object(url, post_body)
+        internal_id = self._test_create_object(url, post_body)
+        self._test_external_identifier_created(internal_id, post_body["properties"]["external_identifier"])
 
     def test_create_eco_account(self):
         """ Tests api creation
@@ -89,7 +107,8 @@ class APIV1CreateTestCase(BaseAPIV1TestCase):
         json_file_path = "api/tests/v1/create/ecoaccount_create_post_body.json"
         with open(json_file_path) as json_file:
             post_body = json.load(fp=json_file)
-        self._test_create_object(url, post_body)
+        internal_id = self._test_create_object(url, post_body)
+        self._test_external_identifier_created(internal_id, post_body["properties"]["external_identifier"])
 
     def test_create_ema(self):
         """ Tests api creation
@@ -101,7 +120,8 @@ class APIV1CreateTestCase(BaseAPIV1TestCase):
         json_file_path = "api/tests/v1/create/ema_create_post_body.json"
         with open(json_file_path) as json_file:
             post_body = json.load(fp=json_file)
-        self._test_create_object(url, post_body)
+        internal_id = self._test_create_object(url, post_body)
+        self._test_external_identifier_created(internal_id, post_body["properties"]["external_identifier"])
 
     def test_create_deduction(self):
         """ Tests api creation

api/tests/v1/update/compensation_update_put_body.json

@@ -44,6 +44,7 @@
         ],
     "properties": {
         "title": "TEST_compensation_CHANGED",
+        "external_identifier": "LOREMIPSUM-123_CHANGED",
         "is_cef": true,
         "is_coherence_keeping": true,
         "is_pik": true,

api/tests/v1/update/ecoaccount_update_put_body.json

@@ -44,6 +44,7 @@
         ],
     "properties": {
         "title": "TEST_account_CHANGED",
+        "external_identifier": "LOREMIPSUM-1234_CHANGED",
         "deductable_surface": "100000.0",
         "is_pik": true,
         "responsible": {

api/tests/v1/update/ema_update_put_body.json

@@ -44,6 +44,7 @@
         ],
     "properties": {
         "title": "TEST_EMA_CHANGED",
+        "external_identifier": "LOREMIPSUM-1235_CHANGED",
         "responsible": {
             "conservation_office": null,
             "conservation_file_number": "TEST_CHANGED",

api/tests/v1/update/intervention_update_put_body.json

@@ -44,6 +44,7 @@
         ],
     "properties": {
         "title": "Test_intervention_CHANGED",
+        "external_identifier": "LOREMIPSUM-1236_CHANGED",
         "responsible": {
             "registration_office": null,
             "registration_file_number": "CHANGED",

api/utils/serializer/serializer.py

@@ -10,9 +10,11 @@ from abc import abstractmethod
 
 from django.contrib.gis import geos
 from django.contrib.gis.geos import GEOSGeometry
+from django.core.exceptions import ObjectDoesNotExist
 from django.core.paginator import Paginator
 from django.db.models import Q
 
+from api.models import ExternalIdentifier
 from konova.models import Geometry
 from konova.utils.message_templates import DATA_UNSHARED
 
@@ -76,6 +78,14 @@ class AbstractModelAPISerializer:
             del self.lookup["id"]
         else:
             # Return certain object
+            ## But first check, whether this is an external identifier ...
+            try:
+                ## If we can find this _id on our ExternalIdentifier model, we need to map it on the internal id
+                ext_id = ExternalIdentifier.objects.get(external_id=_id)
+                _id = ext_id.internal_id
+            except ObjectDoesNotExist:
+                # If we did not find it, we assume that this is already an internal id. (Or it does not exist at all)
+                pass
             self.lookup["id"] = _id
 
         self.shared_lookup = Q(
@@ -161,6 +171,14 @@ class AbstractModelAPISerializer:
         Returns:
 
         """
+        # First if there is an external identifier linked to an internal one, so we can continue with the internal
+        try:
+            ext_id = ExternalIdentifier.objects.get(external_id=id)
+            id = ext_id.internal_id
+        except ObjectDoesNotExist:
+            # No external id found - let's hope the given id exists internally
+            pass
+
         obj = self.model.objects.get(
             id=id,
             deleted__isnull=True,

api/utils/serializer/v1/compensation.py

@@ -88,6 +88,11 @@ class CompensationAPISerializerV1(AbstractModelAPISerializerV1, AbstractCompensa
             # Nothing to do here
             return obj
 
+        # Transform a potential external identifier into an internal one
+        intervention_ext_id = self._get_external_identifier(intervention_id)
+        if intervention_ext_id:
+            intervention_id = intervention_ext_id.internal_id
+
         intervention = Intervention.objects.get(
             id=intervention_id,
         )
@@ -114,6 +119,10 @@ class CompensationAPISerializerV1(AbstractModelAPISerializerV1, AbstractCompensa
 
             # Fill in data to objects
             properties = json_model["properties"]
+
+            external_identifier = properties.get("external_identifier", None)
+            self._check_external_identifier_on_entry_creation(external_identifier)
+
             obj.identifier = obj.generate_new_identifier()
             obj.title = properties["title"]
             obj.is_cef = properties["is_cef"]
@@ -129,6 +138,7 @@ class CompensationAPISerializerV1(AbstractModelAPISerializerV1, AbstractCompensa
             obj = self._set_compensation_states(obj, properties["after_states"], obj.after_states)
             obj = self._set_deadlines(obj, properties["deadlines"])
 
+        self._set_external_identifier(obj.id, properties.get("external_identifier", None), obj.created)
         obj.log.add(obj.created)
 
         celery_update_parcels.delay(obj.geometry.id)
@@ -170,6 +180,7 @@ class CompensationAPISerializerV1(AbstractModelAPISerializerV1, AbstractCompensa
             obj = self._set_compensation_states(obj, properties["after_states"], obj.after_states)
             obj = self._set_deadlines(obj, properties["deadlines"])
 
+        self._set_external_identifier(obj.id, properties.get("external_identifier", None), update_action)
         obj.log.add(update_action)
 
         celery_update_parcels.delay(obj.geometry.id)

api/utils/serializer/v1/deduction.py

@@ -62,6 +62,14 @@ class DeductionAPISerializerV1(AbstractModelAPISerializerV1,
         if surface <= 0:
             raise ValueError("Surface must be > 0 m²")
 
+        # Check if external identifiers need to be mapped onto internal ones
+        acc_ext_id = self._get_external_identifier(acc_id)
+        intervention_ext_id = self._get_external_identifier(intervention_id)
+        if acc_ext_id:
+            acc_id = acc_ext_id.internal_id
+        if intervention_ext_id:
+            intervention_id = intervention_ext_id.internal_id
+
         acc = EcoAccount.objects.get(
             id=acc_id,
             deleted__isnull=True,

api/utils/serializer/v1/ecoaccount.py

@@ -121,7 +121,9 @@ class EcoAccountAPISerializerV1(AbstractModelAPISerializerV1,
             obj = self._initialize_objects(json_model, user)
 
             # Fill in data to objects
-            properties = json_model["properties"]
+            properties = json_model.get("properties", None)
+            if not properties:
+                raise AssertionError("No 'properties' found in payload!")
             obj.identifier = obj.generate_new_identifier()
             obj.title = properties["title"]
             obj.is_pik = properties.get("is_pik", False)
@@ -147,6 +149,7 @@ class EcoAccountAPISerializerV1(AbstractModelAPISerializerV1,
             obj = self._set_compensation_states(obj, properties["after_states"], obj.after_states)
             obj = self._set_deadlines(obj, properties["deadlines"])
 
+        self._set_external_identifier(obj.id, properties.get("external_identifier", None), obj.created)
         obj.log.add(obj.created)
         obj.users.add(user)
 
@@ -172,6 +175,10 @@ class EcoAccountAPISerializerV1(AbstractModelAPISerializerV1,
 
             # Fill in data to objects
             properties = json_model["properties"]
+
+            external_identifier = properties.get("external_identifier", None)
+            self._check_external_identifier_on_entry_creation(external_identifier)
+
             obj.title = properties["title"]
             obj.is_pik = properties.get("is_pik", False)
             obj.deductable_surface = float(properties["deductable_surface"])
@@ -192,6 +199,7 @@ class EcoAccountAPISerializerV1(AbstractModelAPISerializerV1,
             obj = self._set_compensation_states(obj, properties["after_states"], obj.after_states)
             obj = self._set_deadlines(obj, properties["deadlines"])
 
+        self._set_external_identifier(obj.id, external_identifier, update_action)
         obj.log.add(update_action)
 
         celery_update_parcels.delay(obj.geometry.id)

api/utils/serializer/v1/ema.py

@@ -104,6 +104,10 @@ class EmaAPISerializerV1(AbstractModelAPISerializerV1, AbstractCompensationAPISe
 
             # Fill in data to objects
             properties = json_model["properties"]
+
+            external_identifier = properties.get("external_identifier", None)
+            self._check_external_identifier_on_entry_creation(external_identifier)
+
             obj.identifier = obj.generate_new_identifier()
             obj.title = properties["title"]
             obj.is_pik = properties.get("is_pik", False)
@@ -119,6 +123,7 @@ class EmaAPISerializerV1(AbstractModelAPISerializerV1, AbstractCompensationAPISe
             obj = self._set_compensation_states(obj, properties["after_states"], obj.after_states)
             obj = self._set_deadlines(obj, properties["deadlines"])
 
+        self._set_external_identifier(obj.id, external_identifier, obj.created)
         obj.log.add(obj.created)
         obj.users.add(user)
 
@@ -161,6 +166,7 @@ class EmaAPISerializerV1(AbstractModelAPISerializerV1, AbstractCompensationAPISe
             obj = self._set_compensation_states(obj, properties["after_states"], obj.after_states)
             obj = self._set_deadlines(obj, properties["deadlines"])
 
+        self._set_external_identifier(obj.id, properties.get("external_identifier", None), update_action)
         obj.log.add(update_action)
 
         celery_update_parcels.delay(obj.geometry.id)

api/utils/serializer/v1/intervention.py

@@ -150,10 +150,14 @@ class InterventionAPISerializerV1(AbstractModelAPISerializerV1,
 
             # Fill in data to objects
             properties = json_model["properties"]
+
+            external_identifier = properties.get("external_identifier", None)
+            self._check_external_identifier_on_entry_creation(external_identifier)
+
             obj.identifier = obj.generate_new_identifier()
-            obj.title = properties["title"]
+            obj.title = properties.get("title", None)
-            self._set_responsibility(obj, properties["responsible"])
+            self._set_responsibility(obj, properties.get("responsible", None))
-            self._set_legal(obj, properties["legal"])
+            self._set_legal(obj, properties.get("legal", None))
 
             obj.responsible.handler.save()
             obj.responsible.save()
@@ -161,6 +165,7 @@ class InterventionAPISerializerV1(AbstractModelAPISerializerV1,
             obj.legal.save()
             obj.save()
 
+        self._set_external_identifier(obj.id, external_identifier, obj.created)
         obj.users.add(user)
         obj.log.add(obj.created)
 
@@ -186,7 +191,7 @@ class InterventionAPISerializerV1(AbstractModelAPISerializerV1,
 
             # Fill in data to objects
             properties = json_model["properties"]
-            obj.title = properties["title"]
+            obj.title = properties.get("title")
             self._set_responsibility(obj, properties.get("responsible", None))
             self._set_legal(obj, properties.get("legal", None))
             self._set_payments(obj, properties.get("payments", None))
@@ -200,6 +205,7 @@ class InterventionAPISerializerV1(AbstractModelAPISerializerV1,
             obj.save()
 
         obj.mark_as_edited(user, edit_comment="API update")
+        self._set_external_identifier(obj.id, properties.get("external_identifier", None), update_action)
         obj.send_data_to_egon()
 
         celery_update_parcels.delay(obj.geometry.id)

api/utils/serializer/v1/serializer.py

@@ -12,6 +12,7 @@ from django.contrib.gis.geos import MultiPolygon
 from django.core.exceptions import ObjectDoesNotExist
 from django.db.models import QuerySet
 
+from api.models import ExternalIdentifier
 from api.utils.serializer.serializer import AbstractModelAPISerializer
 from codelist.models import KonovaCode
 from codelist.settings import CODELIST_COMPENSATION_ACTION_ID, CODELIST_BIOTOPES_ID, CODELIST_PROCESS_TYPE_ID, \
@@ -39,12 +40,20 @@ class AbstractModelAPISerializerV1(AbstractModelAPISerializer):
         else:
             geom = MultiPolygon().geojson
         geo_json = json.loads(geom)
+        ext_ids = list(
+            ExternalIdentifier.objects.filter(
+                internal_id=entry.id
+            ).values_list(
+                "external_id", flat=True
+            )
+        )
         self.properties_data = {
             "id": entry.id,
             "identifier": entry.identifier,
             "title": entry.title,
             "created_on": self._created_on_to_json(entry),
             "modified_on": self._modified_on_to_json(entry),
+            "external_identifiers": ext_ids,
         }
         self._extend_properties_data(entry)
         geo_json["properties"] = self.properties_data
@@ -137,6 +146,63 @@ class AbstractModelAPISerializerV1(AbstractModelAPISerializer):
         success = entry.deleted is not None
         return success
 
+    def _set_external_identifier(self, internal_identifier, external_identifier, log_entry):
+        """ If an external identifier was provided in the payload, we set it
+        in the database
+
+        Args:
+            internal_identifier (BaseObject): The already processed konova object (EIV, KOM, ...)
+            external_identifier (any): The external identifier taken from the payload
+
+        Returns:
+
+        """
+        if external_identifier is None:
+            return None
+
+        ext_id_obj = ExternalIdentifier.objects.get_or_create(
+            internal_id=internal_identifier,
+            external_id=external_identifier
+        )[0]
+        if not ext_id_obj.created:
+            ext_id_obj.created = log_entry
+            ext_id_obj.save()
+
+        return ext_id_obj
+
+    def _get_external_identifier(self, external_identifier):
+        """ Checks whether a linkage based on an external identifier already exists and returns it if so.
+
+        Args:
+            external_identifier (any): The external identifier according to payload
+
+        Returns:
+            ExternalIdentifier | None
+        """
+        if external_identifier:
+            try:
+                obj = ExternalIdentifier.objects.get(external_id=external_identifier)
+                return obj
+            except ObjectDoesNotExist:
+                pass
+        return None
+
+    def _check_external_identifier_on_entry_creation(self, external_identifier):
+        """ Special check for POST processing:
+        Checks whether an external identifier already exists on the database. This hints that
+        the entry already has been created in the past. Instead of POST, the PUT method shall be used
+        to avoid creating duplicates.
+
+        Args:
+            external_identifier (any): The external identifier according to payload
+
+        Returns:
+
+        """
+        persisted_external_identifier = self._get_external_identifier(external_identifier)
+        if persisted_external_identifier:
+            raise AssertionError(f"{external_identifier} has already been initially created! Use PUT for updates!")
+
 
 class DeductableAPISerializerV1Mixin:
     class Meta:

requirements.txt

@@ -4,62 +4,62 @@ async-timeout==5.0.1
 beautifulsoup4==4.14.3
 billiard==4.2.4
 cached-property==2.0.1
-celery==5.6.2
+celery==5.6.3
-certifi==2026.2.25
+certifi==2026.4.22
 cffi==2.0.0
-chardet==6.0.0.post1
+chardet==7.4.3
-charset-normalizer==3.4.4
+charset-normalizer==3.4.7
-click==8.3.1
+click==8.3.3
 click-didyoumean==0.3.1
 click-plugins==1.1.1.2
 click-repl==0.3.0
-coverage==7.13.4
+coverage==7.13.5
-cryptography==46.0.5
+cryptography==48.0.0
 Deprecated==1.3.1
-Django==6.0.2
+Django==6.0.5
-django-autocomplete-light==3.12.1
+django-autocomplete-light==4.0.0
 django-bootstrap-modal-forms==3.0.5
 django-bootstrap4==26.1
 django-environ==0.13.0
 django-filter==25.2
 django-fontawesome-5==1.0.18
 django-oauth-toolkit==3.2.0
-django-tables2==2.8.0
+django-tables2==3.0.0
 et_xmlfile==2.0.0
-gunicorn==25.1.0
+gunicorn==26.0.0
-idna==3.11
+idna==3.13
-importlib_metadata==8.7.1
+importlib_metadata==9.0.0
 itsdangerous==2.2.0
-jwcrypto==1.5.6
+jwcrypto==1.5.7
 kombu==5.6.2
 oauthlib==3.3.1
 openpyxl==3.2.0b1
-packaging==26.0
+packaging==26.2
-pika==1.3.2
+pika==1.4.0
-pillow==12.1.1
+pillow==12.2.0
 prompt_toolkit==3.0.52
-psycopg==3.3.3
+psycopg==3.3.4
-psycopg-binary==3.3.3
+psycopg-binary==3.3.4
 pycparser==3.0
 pyparsing==3.3.2
 pypng==0.20220715.0
 pyproj==3.7.2
 python-dateutil==2.9.0.post0
-pytz==2025.2
+pytz==2026.2
 PyYAML==6.0.3
 qrcode==8.2
-redis==7.2.1
+redis==7.4.0
-requests==2.32.5
+requests==2.33.1
 six==1.17.0
 soupsieve==2.8.3
 sqlparse==0.5.5
 typing_extensions==4.15.0
-tzdata==2025.3
+tzdata==2026.2
 tzlocal==5.3.1
-urllib3==2.6.3
+urllib3==2.7.0
 vine==5.1.0
-wcwidth==0.6.0
+wcwidth==0.7.0
 webservices==0.7
-wrapt==2.1.1
+wrapt==2.1.2
 xmltodict==1.0.4
-zipp==3.23.0
+zipp==3.23.1

templates/generic_index.html

@@ -70,7 +70,7 @@
                         </div>
                         <div class="dropdown-menu dropdown-menu-right">
                             {% for rpp_option in table.results_per_page_choices %}
-                                <a class="dropdown-item {% if table.results_per_page_chosen == rpp_option %}selected{% endif %}" href="{% querystring table.results_per_page_parameter=rpp_option %}">
+                                <a class="dropdown-item {% if table.results_per_page_chosen == rpp_option %}selected{% endif %}" href="{% querystring %}&{{ table.results_per_page_parameter}}={{rpp_option }}">
                                     {{ rpp_option }}
                                 </a>
                             {% endfor %}