From b1fe9ed9cb41db6622e1a28603c1683fbbe59087 Mon Sep 17 00:00:00 2001 From: mpeltriaux Date: Wed, 26 Oct 2022 10:35:15 +0200 Subject: [PATCH 1/6] Bugfix * fixes bug in excel report creation * fixes order in laws of generated excel sheet --- analysis/utils/report.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/analysis/utils/report.py b/analysis/utils/report.py index 63a06b84..9bbd7d3d 100644 --- a/analysis/utils/report.py +++ b/analysis/utils/report.py @@ -17,6 +17,7 @@ from compensation.models import Compensation, Payment, EcoAccountDeduction, EcoA from intervention.models import Intervention from konova.models import Geometry from konova.sub_settings.django_settings import BASE_DIR, DEFAULT_DATE_FORMAT +from konova.sub_settings.lanis_settings import DEFAULT_SRID_RLP class TimespanReport: @@ -103,9 +104,9 @@ class TimespanReport: "iterable": self.evaluated_laws, "attrs": [ "short_name", + "num", "num_checked", "num_recorded", - "num", ] }, "i_laws_checked": self.law_sum_checked, @@ -333,7 +334,7 @@ class TimespanReport: return Geometry.objects.filter( id__in=ids ).annotate( - geom_cast=Cast("geom", MultiPolygonField()) + geom_cast=Cast("geom", MultiPolygonField(srid=DEFAULT_SRID_RLP)) ).annotate( num=NumGeometries("geom_cast") ).aggregate( From 7650e0bf73d983e0a16b0b874d7fc21bcfff0ce4 Mon Sep 17 00:00:00 2001 From: mpeltriaux Date: Wed, 26 Oct 2022 15:44:33 +0200 Subject: [PATCH 2/6] Command * adds new 'generate_report' command * generates TimeSpanReports for given conservation offices * zips reports into archive * sents archive to ADMINS mails --- konova/management/commands/generate_report.py | 134 ++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 konova/management/commands/generate_report.py diff --git a/konova/management/commands/generate_report.py b/konova/management/commands/generate_report.py new file mode 100644 index 00000000..be0531ff --- /dev/null +++ b/konova/management/commands/generate_report.py @@ -0,0 +1,134 @@ +""" +Author: Michel Peltriaux +Organization: Struktur- und Genehmigungsdirektion Nord, Rhineland-Palatinate, Germany +Contact: michel.peltriaux@sgdnord.rlp.de +Created on: 26.10.22 + +""" +import zipfile +from io import BytesIO + +from django.core.mail import EmailMessage +from django.utils import timezone +from django.utils.datetime_safe import datetime + +from analysis.utils.excel.excel import TempExcelFile +from analysis.utils.report import TimespanReport +from codelist.models import KonovaCode +from codelist.settings import CODELIST_CONSERVATION_OFFICE_ID +from konova.management.commands.setup import BaseKonovaCommand +from konova.sub_settings.django_settings import DEFAULT_FROM_EMAIL, ADMINS + + +class Command(BaseKonovaCommand): + help = "Generates reports for conservation offices" + + __from_key = "from" + __to_key = "to" + __for_key = "for" + + from_date = None + to_date = None + offices = None + + def add_arguments(self, parser): + try: + parser.add_argument(f"--{self.__from_key}", type=str, nargs="+") + parser.add_argument(f"--{self.__to_key}", type=str, nargs="+") + parser.add_argument(f"--{self.__for_key}", type=str, nargs="*") + except ValueError as e: + self._write_error(f"Argument error: {e}") + exit(-1) + + def handle(self, *args, **options): + self.__check_arguments(options) + in_memory_zipped_reports = self.generate_reports_in_memory_zipped() + self.send_mail_to_admins(in_memory_zipped_reports) + + def generate_reports_in_memory_zipped(self): + """ + Generates in-memory reports and zips into in-memory zip file. + + Returns: + + """ + memory_zip = BytesIO() + with zipfile.ZipFile(memory_zip, "w", compression=zipfile.ZIP_DEFLATED) as zf: + for office in self.offices: + self._write_warning(f" Process report for {office.long_name}...") + report = TimespanReport(office.id, self.from_date, self.to_date) + excel_file = TempExcelFile(report.excel_template_path, report.excel_map) + zf.writestr(zinfo_or_arcname=f"{office.long_name}_{self.from_date.date()}_{self.to_date.date()}.xlsx", data=excel_file.stream) + self._write_success(f"Reports generated for {self.offices.count()} offices and zipped.") + return memory_zip.getvalue() + + def __check_arguments(self, options): + """ + Checks given parameters for validity + + Args: + options (): + + Returns: + + """ + _from_value = options.get(self.__from_key, [None])[0] + _to_value = options.get(self.__to_key, [None])[0] + _for_value = options.get(self.__for_key, []) + + # Check ISO dates + try: + _from_date = timezone.make_aware(datetime.fromisoformat(_from_value)) + _to_date = timezone.make_aware(datetime.fromisoformat(_to_value)) + except Exception as e: + self._write_warning(f"One of the dates is not in ISO format (YYYY-MM-DD). {e}") + exit(-1) + + # Check conservation office IDs + _filter = { + "is_archived": False, + "is_leaf": True, + "code_lists__in": [CODELIST_CONSERVATION_OFFICE_ID], + } + offices = KonovaCode.objects.filter(**_filter) + if _for_value is not None and len(_for_value) != 0: + # Specifc offices requested + offices = offices.filter(short_name__in=_for_value) + all_requested_offices_exist = offices.count() == len(_for_value) + if not all_requested_offices_exist: + offices_short_name = set(offices.values_list("short_name", flat=True)) + missing_ids = [] + for val in _for_value: + if val not in offices_short_name: + missing_ids.append(val) + self._write_warning( + f"Unknown offices: {missing_ids}" + ) + exit(-1) + + self.offices = offices + self.from_date = _from_date + self.to_date = _to_date + + def send_mail_to_admins(self, attachment): + office_names = [office.long_name for office in self.offices] + admin_mails = [admin[1] for admin in ADMINS] + + date_from_date = self.from_date.date() + date_to_date = self.to_date.date() + + mail = EmailMessage( + subject="Konova reports generated", + body=f"Zipped reports attached from {date_from_date} to {date_to_date} for {office_names}.", + from_email=DEFAULT_FROM_EMAIL, + to=admin_mails, + attachments=[ + ( + f"reports_{date_from_date}_{date_to_date}.zip", + attachment, + "application/zip" + ) + ] + ) + success = mail.send() + self._write_success(f"Mails with zip as attachment sent to {admin_mails}.") From e048d44c95d157e543ad7dfaa27aaf7d410e110d Mon Sep 17 00:00:00 2001 From: mpeltriaux Date: Fri, 25 Nov 2022 08:27:42 +0100 Subject: [PATCH 3/6] #249 Created as modified * fills modified attribute on new entries with created value automatically * adds default ordering by last modified on table overviews --- api/utils/serializer/v1/compensation.py | 1 + api/utils/serializer/v1/ecoaccount.py | 1 + api/utils/serializer/v1/ema.py | 1 + api/utils/serializer/v1/intervention.py | 1 + compensation/forms/compensation.py | 1 + compensation/forms/eco_account.py | 1 + compensation/views/compensation/compensation.py | 2 ++ compensation/views/eco_account/eco_account.py | 2 ++ ema/forms.py | 1 + ema/views/ema.py | 2 ++ intervention/forms/intervention.py | 1 + intervention/views/intervention.py | 2 ++ 12 files changed, 16 insertions(+) diff --git a/api/utils/serializer/v1/compensation.py b/api/utils/serializer/v1/compensation.py index fbdbba62..5b38b4cd 100644 --- a/api/utils/serializer/v1/compensation.py +++ b/api/utils/serializer/v1/compensation.py @@ -64,6 +64,7 @@ class CompensationAPISerializerV1(AbstractModelAPISerializerV1, AbstractCompensa obj = Compensation() created = create_action obj.created = created + obj.modified = created obj.geometry = geometry return obj diff --git a/api/utils/serializer/v1/ecoaccount.py b/api/utils/serializer/v1/ecoaccount.py index c3d27867..f2def5ce 100644 --- a/api/utils/serializer/v1/ecoaccount.py +++ b/api/utils/serializer/v1/ecoaccount.py @@ -103,6 +103,7 @@ class EcoAccountAPISerializerV1(AbstractModelAPISerializerV1, obj.legal = Legal() created = create_action obj.created = created + obj.modified = created obj.geometry = geometry return obj diff --git a/api/utils/serializer/v1/ema.py b/api/utils/serializer/v1/ema.py index 4bbb6d9e..8191597c 100644 --- a/api/utils/serializer/v1/ema.py +++ b/api/utils/serializer/v1/ema.py @@ -85,6 +85,7 @@ class EmaAPISerializerV1(AbstractModelAPISerializerV1, AbstractCompensationAPISe ) created = create_action obj.created = created + obj.modified = created obj.geometry = geometry return obj diff --git a/api/utils/serializer/v1/intervention.py b/api/utils/serializer/v1/intervention.py index dca53d72..5f06ccd2 100644 --- a/api/utils/serializer/v1/intervention.py +++ b/api/utils/serializer/v1/intervention.py @@ -76,6 +76,7 @@ class InterventionAPISerializerV1(AbstractModelAPISerializerV1, created = create_action obj.legal = legal obj.created = created + obj.modified = created obj.geometry = geometry obj.responsible = resp return obj diff --git a/compensation/forms/compensation.py b/compensation/forms/compensation.py index 647051d3..d4dd1297 100644 --- a/compensation/forms/compensation.py +++ b/compensation/forms/compensation.py @@ -156,6 +156,7 @@ class NewCompensationForm(AbstractCompensationForm, title=title, intervention=intervention, created=action, + modified=action, is_cef=is_cef, is_coherence_keeping=is_coherence_keeping, is_pik=is_pik, diff --git a/compensation/forms/eco_account.py b/compensation/forms/eco_account.py index dd167c6e..af5f63b5 100644 --- a/compensation/forms/eco_account.py +++ b/compensation/forms/eco_account.py @@ -117,6 +117,7 @@ class NewEcoAccountForm(AbstractCompensationForm, CompensationResponsibleFormMix responsible=responsible, deductable_surface=surface, created=action, + modified=action, comment=comment, is_pik=is_pik, legal=legal diff --git a/compensation/views/compensation/compensation.py b/compensation/views/compensation/compensation.py index 57646895..1e3553a8 100644 --- a/compensation/views/compensation/compensation.py +++ b/compensation/views/compensation/compensation.py @@ -46,6 +46,8 @@ def index_view(request: HttpRequest): compensations = Compensation.objects.filter( deleted=None, # only show those which are not deleted individually intervention__deleted=None, # and don't show the ones whose intervention has been deleted + ).order_by( + "-modified__timestamp" ) table = CompensationTable( request=request, diff --git a/compensation/views/eco_account/eco_account.py b/compensation/views/eco_account/eco_account.py index 1070dfe3..077df148 100644 --- a/compensation/views/eco_account/eco_account.py +++ b/compensation/views/eco_account/eco_account.py @@ -41,6 +41,8 @@ def index_view(request: HttpRequest): template = "generic_index.html" eco_accounts = EcoAccount.objects.filter( deleted=None, + ).order_by( + "-modified__timestamp" ) table = EcoAccountTable( request=request, diff --git a/ema/forms.py b/ema/forms.py index bca72247..b1b59da1 100644 --- a/ema/forms.py +++ b/ema/forms.py @@ -81,6 +81,7 @@ class NewEmaForm(AbstractCompensationForm, CompensationResponsibleFormMixin, Pik title=title, responsible=responsible, created=action, + modified=action, comment=comment, is_pik=is_pik, ) diff --git a/ema/views/ema.py b/ema/views/ema.py index b4ad6fd0..bf171f26 100644 --- a/ema/views/ema.py +++ b/ema/views/ema.py @@ -39,6 +39,8 @@ def index_view(request: HttpRequest): template = "generic_index.html" emas = Ema.objects.filter( deleted=None, + ).order_by( + "-modified__timestamp" ) table = EmaTable( diff --git a/intervention/forms/intervention.py b/intervention/forms/intervention.py index 8086fd7f..10b84e66 100644 --- a/intervention/forms/intervention.py +++ b/intervention/forms/intervention.py @@ -270,6 +270,7 @@ class NewInterventionForm(BaseForm): responsible=responsibility_data, legal=legal_data, created=action, + modified=action, comment=comment, ) diff --git a/intervention/views/intervention.py b/intervention/views/intervention.py index 1bca8968..329e7d5f 100644 --- a/intervention/views/intervention.py +++ b/intervention/views/intervention.py @@ -45,6 +45,8 @@ def index_view(request: HttpRequest): deleted=None, # not deleted ).select_related( "legal" + ).order_by( + "-modified__timestamp" ) table = InterventionTable( request=request, From f2c5e7ae014800e2ca4883905fc5fe5201f5572e Mon Sep 17 00:00:00 2001 From: mpeltriaux Date: Fri, 25 Nov 2022 09:05:06 +0100 Subject: [PATCH 4/6] Tests * extends test for new behaviour of newly created entries --- compensation/tests/compensation/test_workflow.py | 1 + compensation/tests/ecoaccount/test_workflow.py | 1 + ema/tests/test_workflow.py | 1 + intervention/tests/test_workflow.py | 1 + 4 files changed, 4 insertions(+) diff --git a/compensation/tests/compensation/test_workflow.py b/compensation/tests/compensation/test_workflow.py index bba12334..d14fdabd 100644 --- a/compensation/tests/compensation/test_workflow.py +++ b/compensation/tests/compensation/test_workflow.py @@ -71,6 +71,7 @@ class CompensationWorkflowTestCase(BaseWorkflowTestCase): self.assertEqual(new_compensation.title, test_title) self.assert_equal_geometries(new_compensation.geometry.geom, test_geom) self.assertEqual(new_compensation.log.count(), 1) + self.assertEqual(new_compensation.created, new_compensation.modified) # Expect logs to be set self.assertEqual(pre_creation_intervention_log_count + 1, self.intervention.log.count()) diff --git a/compensation/tests/ecoaccount/test_workflow.py b/compensation/tests/ecoaccount/test_workflow.py index b1c9a8f4..4792120c 100644 --- a/compensation/tests/ecoaccount/test_workflow.py +++ b/compensation/tests/ecoaccount/test_workflow.py @@ -65,6 +65,7 @@ class EcoAccountWorkflowTestCase(BaseWorkflowTestCase): self.assertEqual(acc.deductable_rest, test_deductable_surface) self.assert_equal_geometries(acc.geometry.geom, test_geom) self.assertEqual(acc.log.count(), 1) + self.assertEqual(acc.created, acc.modified) # Expect logs to be set self.assertEqual(acc.log.count(), 1) diff --git a/ema/tests/test_workflow.py b/ema/tests/test_workflow.py index 4391fdc6..3e5bd291 100644 --- a/ema/tests/test_workflow.py +++ b/ema/tests/test_workflow.py @@ -62,6 +62,7 @@ class EmaWorkflowTestCase(BaseWorkflowTestCase): self.assertEqual(ema.title, test_title) self.assert_equal_geometries(ema.geometry.geom, test_geom) self.assertEqual(ema.log.count(), 1) + self.assertEqual(ema.created, ema.modified) # Expect logs to be set self.assertEqual(ema.log.count(), 1) diff --git a/intervention/tests/test_workflow.py b/intervention/tests/test_workflow.py index be885c96..d44c6d42 100644 --- a/intervention/tests/test_workflow.py +++ b/intervention/tests/test_workflow.py @@ -78,6 +78,7 @@ class InterventionWorkflowTestCase(BaseWorkflowTestCase): self.assertEqual(1, obj.log.count()) self.assertEqual(obj.log.first().action, UserAction.CREATED) self.assertEqual(obj.log.first().user, self.superuser) + self.assertEqual(obj.created, obj.modified) except ObjectDoesNotExist: # Fail if there is no such object self.fail() From b8d9343682e436afd4f0a3907464981cc722e2c2 Mon Sep 17 00:00:00 2001 From: mpeltriaux Date: Fri, 25 Nov 2022 09:17:15 +0100 Subject: [PATCH 5/6] Command response dynamic * adds a check whether the mail could be sent properly or not and changes the resulting response --- konova/management/commands/generate_report.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/konova/management/commands/generate_report.py b/konova/management/commands/generate_report.py index be0531ff..51091871 100644 --- a/konova/management/commands/generate_report.py +++ b/konova/management/commands/generate_report.py @@ -131,4 +131,7 @@ class Command(BaseKonovaCommand): ] ) success = mail.send() - self._write_success(f"Mails with zip as attachment sent to {admin_mails}.") + if success == 1: + self._write_success(f"Mails with zip as attachment sent to {admin_mails}.") + else: + self._write_error(f"Something went wrong during mail sending. Returned sending code was '{success}'") From 90fffb957656a4c9771d74d4aa3c582b4d395077 Mon Sep 17 00:00:00 2001 From: mpeltriaux Date: Mon, 28 Nov 2022 07:28:09 +0100 Subject: [PATCH 6/6] File number public reports * removes file numbers from public reports --- .../templates/compensation/report/eco_account/report.html | 4 ---- ema/templates/ema/report/report.html | 4 ---- intervention/templates/intervention/report/report.html | 8 -------- 3 files changed, 16 deletions(-) diff --git a/compensation/templates/compensation/report/eco_account/report.html b/compensation/templates/compensation/report/eco_account/report.html index e2147f69..07f5a6cc 100644 --- a/compensation/templates/compensation/report/eco_account/report.html +++ b/compensation/templates/compensation/report/eco_account/report.html @@ -16,10 +16,6 @@ {% trans 'Conservation office' %} {{obj.responsible.conservation_office.str_as_office|default_if_none:""}} - - {% trans 'Conservation office file number' %} - {{obj.responsible.conservation_file_number|default_if_none:""}} - {% trans 'Is PIK' %} diff --git a/ema/templates/ema/report/report.html b/ema/templates/ema/report/report.html index fd166596..c03c2fa6 100644 --- a/ema/templates/ema/report/report.html +++ b/ema/templates/ema/report/report.html @@ -16,10 +16,6 @@ {% trans 'Conservation office' %} {{obj.responsible.conservation_office.str_as_office|default_if_none:""}} - - {% trans 'Conservation office file number' %} - {{obj.responsible.conservation_file_number|default_if_none:""}} - {% trans 'Is PIK' %} diff --git a/intervention/templates/intervention/report/report.html b/intervention/templates/intervention/report/report.html index 1c1016d0..130bb3cd 100644 --- a/intervention/templates/intervention/report/report.html +++ b/intervention/templates/intervention/report/report.html @@ -29,18 +29,10 @@ {% trans 'Registration office' %} {{obj.responsible.registration_office.str_as_office|default_if_none:""}} - - {% trans 'Registration office file number' %} - {{obj.responsible.registration_file_number|default_if_none:""}} - {% trans 'Conservation office' %} {{obj.responsible.conservation_office.str_as_office|default_if_none:""}} - - {% trans 'Conservation office file number' %} - {{obj.responsible.conservation_file_number|default_if_none:""}} - {% trans 'Compensations' %}