mpeltriaux
5922d5ce06
* adds another validity check to SimpleGeomForm (is_size_valid) to make sure the area of the entered geometry is somehow rational (>= 1m²) * optimizes performance of django command sanitize_db * extends Geometry model with two new attributes, holding timestamps when a parcel calculation has been started and ended * finally drops unused update_parcel_wfs in favor of update_parcels in Geometry model * refactors update_parcel method * adds geometry buffer fallback in schneider/fetcher.py to avoid emptying of geometries when parcels shall be fetched * finally removes utils/wfs/spatial.py * extends GeomParcelsView according to #381 * updates translations * removes redundant psycopg2-binary requirement
352 lines
12 KiB
Python
352 lines
12 KiB
Python
"""
|
|
Author: Michel Peltriaux
|
|
Organization: Struktur- und Genehmigungsdirektion Nord, Rhineland-Palatinate, Germany
|
|
Contact: michel.peltriaux@sgdnord.rlp.de
|
|
Created on: 16.11.21
|
|
|
|
"""
|
|
from compensation.models import CompensationState, Compensation, EcoAccount, CompensationAction
|
|
from ema.models import Ema
|
|
from intervention.models import Intervention
|
|
from konova.management.commands.setup import BaseKonovaCommand
|
|
from konova.models import Deadline, Geometry, Parcel, District, Municipal, ParcelGroup
|
|
from user.models import UserActionLogEntry, UserAction
|
|
|
|
|
|
class Command(BaseKonovaCommand):
|
|
help = "Checks the database' sanity and removes unused entries"
|
|
|
|
def handle(self, *args, **options):
|
|
try:
|
|
self.sanitize_log_entries()
|
|
self.sanitize_compensation_states()
|
|
self.sanitize_actions()
|
|
self.sanitize_deadlines()
|
|
self.sanitize_geometries()
|
|
self.sanitize_parcels_and_districts()
|
|
except KeyboardInterrupt:
|
|
self._break_line()
|
|
exit(-1)
|
|
|
|
def get_all_log_entries_ids(self, cls):
|
|
""" Getter for all log entry ids of a model
|
|
|
|
Args:
|
|
cls (Intervention|Compensation|Ema|EcoAccount): The model class
|
|
|
|
Returns:
|
|
|
|
"""
|
|
all_objects = cls.objects.all().prefetch_related(
|
|
"log",
|
|
)
|
|
all_ids = all_objects.exclude(
|
|
log__isnull=True,
|
|
).values_list(
|
|
"log__id",
|
|
flat=True,
|
|
)
|
|
return all_ids
|
|
|
|
def sanitize_log_entries(self):
|
|
""" Removes log entries which are not attached to any logs
|
|
|
|
Returns:
|
|
|
|
"""
|
|
self._write_warning("=== Sanitize log entries ===")
|
|
# Exclude created log entries from being cleaned, since they can be part of objects which do not have logs
|
|
# Being in a log (or not) is essential for this cleanup
|
|
all_log_entries = UserActionLogEntry.objects.all().exclude(
|
|
action=UserAction.CREATED
|
|
)
|
|
|
|
EIV_log_entries_ids = self.get_all_log_entries_ids(Intervention)
|
|
self._write_warning(f" EIV: {EIV_log_entries_ids.count()} attached log entries")
|
|
KOM_log_entries_ids = self.get_all_log_entries_ids(Compensation)
|
|
self._write_warning(f" KOM: {KOM_log_entries_ids.count()} attached log entries")
|
|
OEK_log_entries_ids = self.get_all_log_entries_ids(EcoAccount)
|
|
self._write_warning(f" OEK: {OEK_log_entries_ids.count()} attached log entries")
|
|
EMA_log_entries_ids = self.get_all_log_entries_ids(Ema)
|
|
self._write_warning(f" EMA: {EMA_log_entries_ids.count()} attached log entries")
|
|
|
|
unattached_log_entries = all_log_entries.exclude(
|
|
id__in=EIV_log_entries_ids
|
|
).exclude(
|
|
id__in=KOM_log_entries_ids
|
|
).exclude(
|
|
id__in=OEK_log_entries_ids
|
|
).exclude(
|
|
id__in=EMA_log_entries_ids
|
|
)
|
|
|
|
num_entries = unattached_log_entries.count()
|
|
if num_entries > 0:
|
|
self._write_error(f"Found {num_entries} log entries not attached to anything. Delete now...")
|
|
unattached_log_entries.delete()
|
|
self._write_success("Log entries deleted.")
|
|
else:
|
|
self._write_success("No unattached log entries found.")
|
|
self._break_line()
|
|
|
|
def get_all_action_ids(self, cls):
|
|
""" Getter for all action ids of a model
|
|
|
|
Args:
|
|
cls (Compensation|Ema|EcoAccount): The model class
|
|
|
|
Returns:
|
|
|
|
"""
|
|
all_objects = cls.objects.all().prefetch_related(
|
|
"actions",
|
|
)
|
|
all_ids = all_objects.exclude(
|
|
actions__isnull=True,
|
|
).values_list(
|
|
"actions__id",
|
|
flat=True,
|
|
)
|
|
return all_ids
|
|
|
|
def sanitize_actions(self):
|
|
""" Removes actions which are not attached to any entries
|
|
|
|
Returns:
|
|
|
|
"""
|
|
self._write_warning("=== Sanitize compensation actions ===")
|
|
all_actions = CompensationAction.objects.all()
|
|
|
|
kom_action_ids = self.get_all_action_ids(Compensation)
|
|
self._write_warning(f" KOM: {kom_action_ids.count()} attached actions")
|
|
oek_action_ids = self.get_all_action_ids(EcoAccount)
|
|
self._write_warning(f" OEK: {oek_action_ids.count()} attached actions")
|
|
ema_action_ids = self.get_all_action_ids(Ema)
|
|
self._write_warning(f" EMA: {ema_action_ids.count()} attached actions")
|
|
|
|
unattached_actions = all_actions.exclude(
|
|
id__in=kom_action_ids
|
|
).exclude(
|
|
id__in=oek_action_ids
|
|
).exclude(
|
|
id__in=ema_action_ids
|
|
)
|
|
|
|
num_entries = unattached_actions.count()
|
|
if num_entries > 0:
|
|
self._write_error(f"Found {num_entries} actions not attached to anything. Delete now...")
|
|
unattached_actions.delete()
|
|
self._write_success("Actions deleted.")
|
|
else:
|
|
self._write_success("No unattached actions found.")
|
|
self._break_line()
|
|
|
|
def _get_all_deadline_ids(self, cls):
|
|
""" Getter for all deadline ids of a model
|
|
|
|
Args:
|
|
cls (Compensation|Ema|EcoAccount): The model class
|
|
|
|
Returns:
|
|
|
|
"""
|
|
all_objects = cls.objects.all().prefetch_related(
|
|
"deadlines",
|
|
)
|
|
all_ids = all_objects.exclude(
|
|
deadlines__isnull=True,
|
|
).values_list(
|
|
"deadlines__id",
|
|
flat=True,
|
|
)
|
|
return all_ids
|
|
|
|
def sanitize_deadlines(self):
|
|
""" Removes deadlines which are not attached to any entries
|
|
|
|
Returns:
|
|
|
|
"""
|
|
self._write_warning("=== Sanitize deadlines ===")
|
|
all_deadlines = Deadline.objects.all()
|
|
|
|
kom_deadline_ids = self._get_all_deadline_ids(Compensation)
|
|
self._write_warning(f" KOM: {kom_deadline_ids.count()} attached deadlines")
|
|
oek_deadline_ids = self._get_all_deadline_ids(EcoAccount)
|
|
self._write_warning(f" OEK: {kom_deadline_ids.count()} attached deadlines")
|
|
ema_deadline_ids = self._get_all_deadline_ids(Ema)
|
|
self._write_warning(f" EMA: {kom_deadline_ids.count()} attached deadlines")
|
|
|
|
unattached_deadlines = all_deadlines.exclude(
|
|
id__in=kom_deadline_ids
|
|
).exclude(
|
|
id__in=oek_deadline_ids
|
|
).exclude(
|
|
id__in=ema_deadline_ids
|
|
)
|
|
|
|
num_entries = unattached_deadlines.count()
|
|
if num_entries > 0:
|
|
self._write_error(f"Found {num_entries} deadlines not attached to anything. Delete now...")
|
|
unattached_deadlines.delete()
|
|
self._write_success("Deadlines deleted.")
|
|
else:
|
|
self._write_success("No unattached deadlines found.")
|
|
self._break_line()
|
|
|
|
def _get_all_geometry_ids(self, cls):
|
|
""" Getter for all geometry ids of a model
|
|
|
|
Args:
|
|
cls (Intervention|Compensation|Ema|EcoAccount): The model class
|
|
|
|
Returns:
|
|
|
|
"""
|
|
all_objects = cls.objects.all().prefetch_related(
|
|
"geometry",
|
|
)
|
|
all_ids = all_objects.exclude(
|
|
geometry__isnull=True,
|
|
).values_list(
|
|
"geometry__id",
|
|
flat=True,
|
|
)
|
|
return all_ids
|
|
|
|
def sanitize_geometries(self):
|
|
""" Removes geometries which are not attached to any entries
|
|
|
|
Returns:
|
|
|
|
"""
|
|
self._write_warning("=== Sanitize geometries ===")
|
|
all_geometries = Geometry.objects.all()
|
|
|
|
kom_geometry_ids = self._get_all_geometry_ids(Compensation)
|
|
self._write_warning(f" EMA: {kom_geometry_ids.count()} attached geometries")
|
|
eiv_geometry_ids = self._get_all_geometry_ids(Intervention)
|
|
self._write_warning(f" EMA: {eiv_geometry_ids.count()} attached geometries")
|
|
oek_geometry_ids = self._get_all_geometry_ids(EcoAccount)
|
|
self._write_warning(f" EMA: {oek_geometry_ids.count()} attached geometries")
|
|
ema_geometry_ids = self._get_all_geometry_ids(Ema)
|
|
self._write_warning(f" EMA: {ema_geometry_ids.count()} attached geometries")
|
|
|
|
unattached_geometries = all_geometries.exclude(
|
|
id__in=kom_geometry_ids
|
|
).exclude(
|
|
id__in=eiv_geometry_ids
|
|
).exclude(
|
|
id__in=oek_geometry_ids
|
|
).exclude(
|
|
id__in=ema_geometry_ids
|
|
)
|
|
|
|
num_entries = unattached_geometries.count()
|
|
if num_entries > 0:
|
|
self._write_error(f"Found {num_entries} geometries not attached to anything. Delete now...")
|
|
unattached_geometries.delete()
|
|
self._write_success("Geometries deleted.")
|
|
else:
|
|
self._write_success("No unattached geometries found.")
|
|
self._break_line()
|
|
|
|
def _get_all_state_ids(self, cls):
|
|
""" Getter for all states (before and after) of a class
|
|
|
|
Args:
|
|
cls ():
|
|
|
|
Returns:
|
|
|
|
"""
|
|
all_objects = cls.objects.all().prefetch_related(
|
|
"before_states",
|
|
"after_states",
|
|
)
|
|
before_state_ids = all_objects.exclude(
|
|
before_states__isnull=True,
|
|
).values_list(
|
|
"before_states__id",
|
|
flat=True,
|
|
)
|
|
after_state_ids = all_objects.exclude(
|
|
after_states__isnull=True,
|
|
).values_list(
|
|
"after_states__id",
|
|
flat=True,
|
|
)
|
|
all_ids = after_state_ids.union(before_state_ids)
|
|
return all_ids
|
|
|
|
def sanitize_compensation_states(self):
|
|
""" Removes unattached compensation states
|
|
|
|
Returns:
|
|
|
|
"""
|
|
self._write_warning("=== Sanitize compensation states ===")
|
|
all_states = CompensationState.objects.all()
|
|
|
|
kom_state_ids = self._get_all_state_ids(Compensation)
|
|
oek_state_ids = self._get_all_state_ids(EcoAccount)
|
|
ema_state_ids = self._get_all_state_ids(Ema)
|
|
|
|
unattached_states = all_states.exclude(
|
|
id__in=kom_state_ids
|
|
).exclude(
|
|
id__in=oek_state_ids
|
|
).exclude(
|
|
id__in=ema_state_ids
|
|
)
|
|
|
|
num_unattached_states = unattached_states.count()
|
|
if num_unattached_states > 0:
|
|
self._write_error(f"Found {num_unattached_states} unused compensation states. Delete now...")
|
|
unattached_states.delete()
|
|
self._write_success("Unused states deleted.")
|
|
else:
|
|
self._write_success("No unused states found.")
|
|
self._break_line()
|
|
|
|
def __sanitize_parcel_sub_type(self, cls):
|
|
unrelated_entries = cls.objects.filter(
|
|
parcels=None,
|
|
)
|
|
num_unrelated_entries = unrelated_entries.count()
|
|
cls_name = cls.__name__
|
|
if num_unrelated_entries > 0:
|
|
self._write_error(f"Found {num_unrelated_entries} unrelated {cls_name} entries. Delete now...")
|
|
unrelated_entries.delete()
|
|
self._write_success(f"Unrelated {cls_name} deleted.")
|
|
else:
|
|
self._write_success(f"No unrelated {cls_name} found.")
|
|
|
|
def sanitize_parcels_and_districts(self):
|
|
""" Removes unattached parcels and districts
|
|
|
|
Returns:
|
|
|
|
"""
|
|
self._write_warning("=== Sanitize administrative spatial references ===")
|
|
unrelated_parcels = Parcel.objects.filter(
|
|
geometries=None,
|
|
)
|
|
num_unrelated_parcels = unrelated_parcels.count()
|
|
if num_unrelated_parcels > 0:
|
|
self._write_error(f"Found {num_unrelated_parcels} unrelated parcel entries. Delete now...")
|
|
unrelated_parcels.delete()
|
|
self._write_success("Unrelated parcels deleted.")
|
|
else:
|
|
self._write_success("No unrelated parcels found.")
|
|
|
|
sub_types = [
|
|
District,
|
|
Municipal,
|
|
ParcelGroup
|
|
]
|
|
for sub_type in sub_types:
|
|
self.__sanitize_parcel_sub_type(sub_type)
|
|
|
|
self._break_line() |