#26 Annual conservation report

* Adds download as excel sheet
* improves db access performance

analysis/templates/analysis/reports/detail.html

@@ -9,13 +9,13 @@
         </div>
         <div class="col-sm-12 col-md-12 col-lg-6">
             <div class="d-flex justify-content-end">
-                <div class=" menu-elem dropdown">
+                <div class="dropdown">
                     <div class="btn btn" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
                         <button class="btn btn-default" title="{% trans 'Download' %}">
                             {% fa5_icon 'download' %}
                         </button>
                     </div>
-                    <div class="dropdown-menu">
+                    <div class="dropdown-menu dropdown-menu-right">
                         <a href="{{request.url}}?format=excel&{{request.GET.urlencode}}">
                             <button class="dropdown-item" title="Excel">
                                 {% fa5_icon 'file-excel' %} Excel

analysis/templates/analysis/reports/includes/compensation/amount.html

@@ -24,31 +24,31 @@
         <tbody>
             <tr>
                 <td>{% trans 'Conservation office by law' %}</td>
-                <td>{{report.compensation_report.queryset_registration_office_unb_checked.count|default_if_zero:"-"}}</td>
+                <td>{{report.compensation_report.queryset_registration_office_unb_checked_count|default_if_zero:"-"}}</td>
-                <td>{{report.compensation_report.queryset_registration_office_unb_recorded.count|default_if_zero:"-"}}</td>
+                <td>{{report.compensation_report.queryset_registration_office_unb_recorded_count|default_if_zero:"-"}}</td>
                 <td>{{report.compensation_report.num_single_surfaces_total_unb|default_if_zero:"-"}}</td>
-                <td>{{report.compensation_report.queryset_registration_office_unb.count|default_if_zero:"-"}}</td>
+                <td>{{report.compensation_report.queryset_registration_office_unb_count|default_if_zero:"-"}}</td>
             </tr>
             <tr>
                 <td>{% trans 'Land-use planning' %}</td>
-                <td>{{report.compensation_report.queryset_registration_office_tbp_checked.count|default_if_zero:"-"}}</td>
+                <td>{{report.compensation_report.queryset_registration_office_tbp_checked_count|default_if_zero:"-"}}</td>
-                <td>{{report.compensation_report.queryset_registration_office_tbp_recorded.count|default_if_zero:"-"}}</td>
+                <td>{{report.compensation_report.queryset_registration_office_tbp_recorded_count|default_if_zero:"-"}}</td>
                 <td>{{report.compensation_report.num_single_surfaces_total_tbp|default_if_zero:"-"}}</td>
-                <td>{{report.compensation_report.queryset_registration_office_tbp.count|default_if_zero:"-"}}</td>
+                <td>{{report.compensation_report.queryset_registration_office_tbp_count|default_if_zero:"-"}}</td>
             </tr>
             <tr>
                 <td>{% trans 'Other registration office' %}</td>
-                <td>{{report.compensation_report.queryset_registration_office_other_checked.count|default_if_zero:"-"}}</td>
+                <td>{{report.compensation_report.queryset_registration_office_other_checked_count|default_if_zero:"-"}}</td>
-                <td>{{report.compensation_report.queryset_registration_office_other_recorded.count|default_if_zero:"-"}}</td>
+                <td>{{report.compensation_report.queryset_registration_office_other_recorded_count|default_if_zero:"-"}}</td>
                 <td>{{report.compensation_report.num_single_surfaces_total_other|default_if_zero:"-"}}</td>
-                <td>{{report.compensation_report.queryset_registration_office_other.count|default_if_zero:"-"}}</td>
+                <td>{{report.compensation_report.queryset_registration_office_other_count|default_if_zero:"-"}}</td>
             </tr>
             <tr>
                 <td><strong>{% trans 'Total' %}</strong></td>
-                <td><strong>{{report.compensation_report.queryset_checked.count|default_if_zero:"-"}}</strong></td>
+                <td><strong>{{report.compensation_report.queryset_checked_count|default_if_zero:"-"}}</strong></td>
-                <td><strong>{{report.compensation_report.queryset_recorded.count|default_if_zero:"-"}}</strong></td>
+                <td><strong>{{report.compensation_report.queryset_recorded_count|default_if_zero:"-"}}</strong></td>
                 <td><strong>{{report.compensation_report.num_single_surfaces_total|default_if_zero:"-"}}</strong></td>
-                <td><strong>{{report.compensation_report.queryset.count|default_if_zero:"-"}}</strong></td>
+                <td><strong>{{report.compensation_report.queryset_count|default_if_zero:"-"}}</strong></td>
             </tr>
         </tbody>
     </table>

analysis/templates/analysis/reports/includes/eco_account/amount.html

@@ -22,10 +22,10 @@
         </thead>
         <tbody>
             <tr>
-                <td>{{report.eco_account_report.queryset_old.count|default_if_zero:"-"}}</td>
+                <td>{{report.eco_account_report.queryset_old_count|default_if_zero:"-"}}</td>
-                <td>{{report.eco_account_report.queryset.count|default_if_zero:"-"}}</td>
+                <td>{{report.eco_account_report.queryset_count|default_if_zero:"-"}}</td>
-                <td>{{report.eco_account_report.queryset_recorded.count|default_if_zero:"-"}}</td>
+                <td>{{report.eco_account_report.queryset_recorded_count|default_if_zero:"-"}}</td>
-                <td>{{report.eco_account_report.queryset_total.count|default_if_zero:"-"}}</td>
+                <td>{{report.eco_account_report.queryset_total_count|default_if_zero:"-"}}</td>
             </tr>
         </tbody>
     </table>

analysis/templates/analysis/reports/includes/eco_account/deductions.html

@@ -13,9 +13,9 @@
         </thead>
         <tbody>
             <tr>
-                <td>{{report.eco_account_report.queryset_deductions_recorded.count|default_if_zero:"-"}}</td>
+                <td>{{report.eco_account_report.queryset_deductions_recorded_count|default_if_zero:"-"}}</td>
                 <td>{{report.eco_account_report.recorded_deductions_sq_m|default_if_zero:"-"}}m²</td>
-                <td>{{report.eco_account_report.queryset_deductions_recorded.count|default_if_zero:"-"}}</td>
+                <td>{{report.eco_account_report.queryset_deductions_count|default_if_zero:"-"}}</td>
                 <td>{{report.eco_account_report.deductions_sq_m|default_if_zero:"-"}}m²</td>
             </tr>
         </tbody>

analysis/templates/analysis/reports/includes/intervention/amount.html

@@ -21,9 +21,9 @@
         </thead>
         <tbody>
             <tr>
-                <td>{{report.intervention_report.queryset_checked.count|default_if_zero:"-"}}</td>
+                <td>{{report.intervention_report.queryset_checked_count|default_if_zero:"-"}}</td>
-                <td>{{report.intervention_report.queryset_recorded.count|default_if_zero:"-"}}</td>
+                <td>{{report.intervention_report.queryset_recorded_count|default_if_zero:"-"}}</td>
-                <td>{{report.intervention_report.queryset.count|default_if_zero:"-"}}</td>
+                <td>{{report.intervention_report.queryset_count|default_if_zero:"-"}}</td>
             </tr>
         </tbody>
     </table>

analysis/templates/analysis/reports/includes/old_intervention/amount.html

@@ -21,9 +21,9 @@
         </thead>
         <tbody>
             <tr>
-                <td>{{report.old_intervention_report.queryset_checked.count|default_if_zero:"-"}}</td>
+                <td>{{report.old_intervention_report.queryset_checked_count|default_if_zero:"-"}}</td>
-                <td>{{report.old_intervention_report.queryset_recorded.count|default_if_zero:"-"}}</td>
+                <td>{{report.old_intervention_report.queryset_recorded_count|default_if_zero:"-"}}</td>
-                <td>{{report.old_intervention_report.queryset.count|default_if_zero:"-"}}</td>
+                <td>{{report.old_intervention_report.queryset_count|default_if_zero:"-"}}</td>
             </tr>
         </tbody>
     </table>

analysis/utils/excel/excel.py

@@ -0,0 +1,112 @@
+"""
+Author: Michel Peltriaux
+Organization: Struktur- und Genehmigungsdirektion Nord, Rhineland-Palatinate, Germany
+Contact: michel.peltriaux@sgdnord.rlp.de
+Created on: 21.10.21
+
+"""
+from django.core.files.temp import NamedTemporaryFile
+from openpyxl import load_workbook
+
+
+class TempExcelFile:
+    """ A temporary excel sheet which will not be saved on the hard drive permanently.
+
+    Using a template file and a template_map dictionary, this class can be used to fill in automatically
+    predefined values into certain cells.
+
+    Can be used to create excel files from data and sending it as a response like
+        _file = TempExcelFile()
+        response = HttpResponse(
+            content=file.stream,
+            content_type="application/ms-excel",
+        )
+        response['Content-Disposition'] = 'attachment; filename=my_file.xlsx'
+        return response
+
+    """
+    stream = None
+    _template_file_path = None
+    _template_map = {}
+    _data_obj = None
+
+    def __init__(self, template_file_path: str = None, template_map: dict = None):
+        self._template_map = template_map or {}
+        self._template_file_path = template_file_path
+        self._workbook = load_workbook(template_file_path)
+        self._file = NamedTemporaryFile()
+
+        self._replace_template_placeholders()
+
+    def _replace_template_placeholders(self, start_row: int = 0):
+        """ Replaces all placeholder inside the template according to _template_map
+
+        Args:
+            start_row (int): Defines where to start
+
+        Returns:
+
+        """
+        ws = self._workbook.active
+        # Always activate sheet protection
+        ws.protection.sheet = True
+        ws.protection.enable()
+        _rows = ws.iter_rows(start_row)
+        for row in _rows:
+            for cell in row:
+                val = cell.value
+                if val in self._template_map:
+                    attr = self._template_map[val]
+                    # If keyword '_iter' can be found inside the placeholder value it's an iterable and we
+                    # need to process it differently
+                    if isinstance(attr, dict):
+                        # Read the iterable object and related attributes from the dict
+                        _iter_obj = attr.get("iterable", None)
+                        _attrs = attr.get("attrs", [])
+                        self._add_cells_from_iterable(ws, cell, _iter_obj, _attrs)
+                        # Since the sheet length did change now, we need to rerun this function starting with the new
+                        # row counter
+                        self._replace_template_placeholders(start_row=cell.row + len(_iter_obj))
+                    else:
+                        cell.value = attr
+        self._workbook.save(self._file.name)
+        self._file.seek(0)
+        self.stream = self._file.read()
+
+    def _add_cells_from_iterable(self, ws, start_cell, _iter_obj: iter, _attrs: list):
+        """
+        Adds iterable data defined by _template_map like
+            ...
+            "some_placeholder_iter": {
+                "iterable": iterable_object,
+                "attrs": [
+                    "attr1",
+                    "attr2",
+                    "attr3",
+                    ...
+                ]
+            },
+            ...
+
+        Args:
+            ws (Workbook): The active workbook
+            _iter_obj (dict): Iterable definitions from template_map
+
+        Returns:
+
+        """
+        # Save border style
+        border_style = start_cell.border.copy()
+        # Drop current row, since it is just placeholder
+        ws.delete_rows(start_cell.row)
+        # Add enoug empty rows for the data
+        ws.insert_rows(start_cell.row, len(_iter_obj))
+
+        i = 0
+        for _iter_entry in _iter_obj:
+            j = 0
+            for _iter_attr in _attrs:
+                _new_cell = ws.cell(start_cell.row + i, start_cell.column + j, getattr(_iter_entry, _iter_attr))
+                _new_cell.border = border_style
+                j += 1
+            i += 1

analysis/utils/report.py

@@ -5,13 +5,10 @@ Contact: michel.peltriaux@sgdnord.rlp.de
 Created on: 18.10.21
 
 """
-from tempfile import NamedTemporaryFile
-
 from django.contrib.gis.db.models import MultiPolygonField
 from django.contrib.gis.db.models.functions import NumGeometries
 from django.db.models import Count, Sum, Q
 from django.db.models.functions import Cast
-from openpyxl import Workbook
 
 from analysis.settings import LKOMPVZVO_PUBLISH_DATE
 from codelist.models import KonovaCode
@@ -19,7 +16,7 @@ from codelist.settings import CODELIST_LAW_ID
 from compensation.models import Compensation, Payment, EcoAccountDeduction, EcoAccount
 from intervention.models import Intervention
 from konova.models import Geometry
-from konova.utils.generators import generate_random_string
+from konova.sub_settings.django_settings import BASE_DIR, DEFAULT_DATE_FORMAT
 
 
 class TimespanReport:
@@ -30,11 +27,19 @@ class TimespanReport:
     date_from = -1
     date_to = -1
 
+    # Excel map is used to map a cell value ("A1") to an attribute
+    excel_map = {}
+    excel_template_path = f"{BASE_DIR}/analysis/utils/excel/excel_report.xlsx"
+
     class InterventionReport:
         queryset = Intervention.objects.none()
         queryset_checked = Intervention.objects.none()
         queryset_recorded = Intervention.objects.none()
 
+        queryset_count = -1
+        queryset_checked_count = -1
+        queryset_recorded_count = -1
+
         # Law related
         law_sum = -1
         law_sum_checked = -1
@@ -66,6 +71,10 @@ class TimespanReport:
             self.queryset_recorded = self.queryset.filter(
                 recorded__isnull=False
             )
+            self.queryset_count = self.queryset.count()
+            self.queryset_checked_count = self.queryset_checked.count()
+            self.queryset_recorded_count = self.queryset_recorded.count()
+
             self._create_report()
 
         def _create_report(self):
@@ -143,20 +152,32 @@ class TimespanReport:
         queryset = Compensation.objects.none()
         queryset_checked = Compensation.objects.none()
         queryset_recorded = Compensation.objects.none()
+        queryset_count = -1
+        queryset_checked_count = -1
+        queryset_recorded_count = -1
 
         queryset_registration_office_unb = Compensation.objects.none()
         queryset_registration_office_unb_checked = Compensation.objects.none()
         queryset_registration_office_unb_recorded = Compensation.objects.none()
+        queryset_registration_office_unb_count = -1
+        queryset_registration_office_unb_checked_count = -1
+        queryset_registration_office_unb_recorded_count = -1
         num_single_surfaces_total_unb = -1
 
         queryset_registration_office_tbp = Compensation.objects.none()
         queryset_registration_office_tbp_checked = Compensation.objects.none()
         queryset_registration_office_tbp_recorded = Compensation.objects.none()
+        queryset_registration_office_tbp = -1
+        queryset_registration_office_tbp_checked = -1
+        queryset_registration_office_tbp_recorded = -1
         num_single_surfaces_total_tbp = -1
 
         queryset_registration_office_other = Compensation.objects.none()
         queryset_registration_office_other_checked = Compensation.objects.none()
         queryset_registration_office_other_recorded = Compensation.objects.none()
+        queryset_registration_office_other = -1
+        queryset_registration_office_other_checked = -1
+        queryset_registration_office_other_recorded = -1
         num_single_surfaces_total_other = -1
 
         num_single_surfaces_total = -1
@@ -183,6 +204,11 @@ class TimespanReport:
             self.queryset_recorded = self.queryset.filter(
                 intervention__recorded__isnull=False
             )
+
+            self.queryset_count = self.queryset.count()
+            self.queryset_checked_count = self.queryset_checked.count()
+            self.queryset_recorded_count = self.queryset_recorded.count()
+
             self._create_report()
 
         def _create_report(self):
@@ -260,6 +286,9 @@ class TimespanReport:
             self.queryset_registration_office_unb_checked = self.queryset_registration_office_unb.filter(
                 intervention__checked__isnull=False,
             )
+            self.queryset_registration_office_unb_count = self.queryset_registration_office_unb.count()
+            self.queryset_registration_office_unb_checked_count = self.queryset_registration_office_unb_checked.count()
+            self.queryset_registration_office_unb_recorded_count = self.queryset_registration_office_unb_recorded.count()
 
             self.queryset_registration_office_tbp = self.queryset.filter(
                 intervention__responsible__registration_office__parent__id=self.id_tbp
@@ -270,6 +299,9 @@ class TimespanReport:
             self.queryset_registration_office_tbp_checked = self.queryset_registration_office_tbp.filter(
                 intervention__checked__isnull=False,
             )
+            self.queryset_registration_office_tbp_count = self.queryset_registration_office_tbp.count()
+            self.queryset_registration_office_tbp_checked_count = self.queryset_registration_office_tbp_checked.count()
+            self.queryset_registration_office_tbp_recorded_count = self.queryset_registration_office_tbp_recorded.count()
 
             self.queryset_registration_office_other = self.queryset.exclude(
                 Q(id__in=self.queryset_registration_office_tbp) | Q(id__in=self.queryset_registration_office_unb)
@@ -280,14 +312,26 @@ class TimespanReport:
             self.queryset_registration_office_other_checked = self.queryset_registration_office_other.filter(
                 intervention__checked__isnull=False,
             )
+            self.queryset_registration_office_other_count = self.queryset_registration_office_other.count()
+            self.queryset_registration_office_other_checked_count = self.queryset_registration_office_other_checked.count()
+            self.queryset_registration_office_other_recorded_count = self.queryset_registration_office_other_recorded.count()
 
     class EcoAccountReport:
+        queryset_total = EcoAccount.objects.none()
         queryset = EcoAccount.objects.none()
         queryset_recorded = EcoAccount.objects.none()
         queryset_old = EcoAccount.objects.none()
+        queryset_total_count = -1
+        queryset_count = -1
+        queryset_recorded_count = -1
+        queryset_old_count = -1
+
         queryset_deductions = EcoAccountDeduction.objects.none()
         queryset_deductions_recorded = EcoAccountDeduction.objects.none()
         queryset_has_deductions = EcoAccountDeduction.objects.none()
+        queryset_deductions_count = -1
+        queryset_deductions_recorded_count = -1
+        queryset_has_deductions_count = -1
 
         # Total size of deductions
         deductions_sq_m = -1
@@ -320,6 +364,15 @@ class TimespanReport:
             self.queryset_deductions_recorded = self.queryset_deductions.filter(
                 intervention__recorded__isnull=False
             )
+
+            self.queryset_total_count = self.queryset_total.count()
+            self.queryset_count = self.queryset.count()
+            self.queryset_old_count = self.queryset_old.count()
+            self.queryset_recorded = self.queryset_recorded.count()
+            self.queryset_deductions_count = self.queryset_deductions.count()
+            self.queryset_deductions_recorded_count = self.queryset_deductions_recorded.count()
+            self.queryset_has_deductions_count = self.queryset_has_deductions.count()
+
             self._create_report()
 
         def _create_report(self):
@@ -343,6 +396,10 @@ class TimespanReport:
         queryset_checked = Compensation.objects.none()
         queryset_recorded = Compensation.objects.none()
 
+        queryset_count = -1
+        queryset_checked_count = -1
+        queryset_recorded_count = -1
+
         def __init__(self, id: str, date_from: str, date_to: str):
             self.queryset = Intervention.objects.filter(
                 legal__registration_date__lte=LKOMPVZVO_PUBLISH_DATE,
@@ -357,6 +414,9 @@ class TimespanReport:
             self.queryset_recorded = self.queryset.filter(
                 recorded__isnull=False
             )
+            self.queryset_count = self.queryset.count()
+            self.queryset_checked = self.queryset_checked.count()
+            self.queryset_recorded = self.queryset_recorded.count()
 
     def __init__(self, office_id: str, date_from: str, date_to: str):
         self.office_id = office_id
@@ -368,11 +428,32 @@ class TimespanReport:
         self.eco_account_report = self.EcoAccountReport(self.office_id, date_from, date_to)
         self.old_intervention_report = self.OldInterventionReport(self.office_id, date_from, date_to)
 
-    def to_excel_file(self):
+        # Build excel map
-        workbook = Workbook()
+        self.excel_map = {
-        tmp_file = NamedTemporaryFile()
+            "date_from": date_from.strftime(DEFAULT_DATE_FORMAT),
-        ws = workbook.active
+            "date_to": date_to.strftime(DEFAULT_DATE_FORMAT),
-        ws["A1"] = "TEST"
+            "i_checked": self.intervention_report.queryset_checked_count,
-        workbook.save(tmp_file.name)
+            "i_recorded": self.intervention_report.queryset_recorded_count,
-        tmp_file.seek(0)
+            "i_total": self.intervention_report.queryset_count,
-        return tmp_file
+            "i_compensations_checked": self.intervention_report.compensation_sum_checked,
+            "i_compensations_recorded": self.intervention_report.compensation_sum_recorded,
+            "i_compensations_total": self.intervention_report.compensation_sum,
+            "i_payments_recorded": self.intervention_report.payment_sum_recorded,
+            "i_payments_checked": self.intervention_report.payment_sum_checked,
+            "i_payments_total": self.intervention_report.payment_sum,
+            "i_deductions_recorded": self.intervention_report.deduction_sum_recorded,
+            "i_deductions_checked": self.intervention_report.deduction_sum_checked,
+            "i_deductions_total": self.intervention_report.deduction_sum,
+            "i_laws_iter": {
+                "iterable": self.intervention_report.evaluated_laws,
+                "attrs": [
+                    "short_name",
+                    "num_checked",
+                    "num_recorded",
+                    "num",
+                ]
+            },
+            "i_laws_checked": self.intervention_report.law_sum_checked,
+            "i_laws_recorded": self.intervention_report.law_sum_recorded,
+            "i_laws_total": self.intervention_report.law_sum,
+        }

analysis/views.py

@@ -1,10 +1,11 @@
 from django.contrib import messages
 from django.contrib.auth.decorators import login_required
-from django.http import HttpRequest, FileResponse
+from django.http import HttpRequest, HttpResponse
 from django.shortcuts import render, redirect, get_object_or_404
 from django.utils import timezone
 
 from analysis.forms import TimespanReportForm
+from analysis.utils.excel.excel import TempExcelFile
 from analysis.utils.report import TimespanReport
 from codelist.models import KonovaCode
 from konova.contexts import BaseContext
@@ -86,11 +87,12 @@ def detail_report_view(request: HttpRequest, id: str):
         context = BaseContext(request, context).context
         return render(request, template, context)
     elif format_param == "excel":
-        stream = report.to_excel_file()
+        file = TempExcelFile(report.excel_template_path, report.excel_map)
-        response = FileResponse(
+        response = HttpResponse(
-            filename=stream.name,
+            content=file.stream,
+            content_type="application/ms-excel",
         )
-        response['Content-Disposition'] = f'attachement;"'
+        response['Content-Disposition'] = f'attachment; filename={cons_office.long_name}_{df}_{dt}.xlsx'
         return response
     else:
         raise NotImplementedError