From 22a97de19526266d4c4e02df0358af15ed0c19d5 Mon Sep 17 00:00:00 2001 From: mpeltriaux Date: Wed, 17 Nov 2021 09:32:11 +0100 Subject: [PATCH] #35 Sanity command * improves performance * adds sanity actions for compensation state and compensation action --- konova/management/commands/sanitize_db.py | 124 ++++++++++++++++------ 1 file changed, 94 insertions(+), 30 deletions(-) diff --git a/konova/management/commands/sanitize_db.py b/konova/management/commands/sanitize_db.py index 6605f14..30e3aa3 100644 --- a/konova/management/commands/sanitize_db.py +++ b/konova/management/commands/sanitize_db.py @@ -5,10 +5,9 @@ Contact: michel.peltriaux@sgdnord.rlp.de Created on: 16.11.21 """ -from django.db.models import QuerySet - -from compensation.models import CompensationState, Compensation, EcoAccount, AbstractCompensation +from compensation.models import CompensationState, Compensation, EcoAccount, CompensationAction from ema.models import Ema +from intervention.models import Intervention from konova.management.commands.setup import BaseKonovaCommand from user.models import UserActionLogEntry @@ -18,38 +17,105 @@ class Command(BaseKonovaCommand): def handle(self, *args, **options): try: - #self.sanitize_log_entries() + self.sanitize_log_entries() self.sanitize_compensation_states() + self.sanitize_actions() except KeyboardInterrupt: self._break_line() exit(-1) + def get_all_log_entries_ids(self, cls): + """ Getter for all log entry ids of a model + + Args: + cls (Intervention|Compensation|Ema|EcoAccount): The model class + + Returns: + + """ + all_objects = cls.objects.all().prefetch_related( + "log", + ) + all_ids = all_objects.exclude( + log__isnull=True, + ).values_list( + "log__id", + flat=True, + ) + return all_ids + def sanitize_log_entries(self): """ Removes log entries which are not attached to any logs Returns: """ + self._write_warning("=== Sanitize log entries ===") all_log_entries = UserActionLogEntry.objects.all() - delete_ids = [] - for entry in all_log_entries: - logs_intervention = entry.intervention_set.all() - logs_compensation = entry.compensation_set.all() - logs_account = entry.ecoaccount_set.all() - logs_ema = entry.ema_set.all() - logs_something = logs_intervention or logs_compensation or logs_account or logs_ema - if not logs_something: - delete_ids.append(entry.id) - entries_to_delete = UserActionLogEntry.objects.filter( - id__in=delete_ids + + intervention_log_entries_ids = self.get_all_log_entries_ids(Intervention) + attached_log_entries_id = intervention_log_entries_ids.union( + self.get_all_log_entries_ids(Compensation), + self.get_all_log_entries_ids(EcoAccount), + self.get_all_log_entries_ids(Ema), ) - num_entries = entries_to_delete.count() + + unattached_log_entries = all_log_entries.exclude(id__in=attached_log_entries_id) + + num_entries = unattached_log_entries.count() if num_entries > 0: self._write_warning(f"Found {num_entries} log entries not attached to anything. Delete now...") - entries_to_delete.delete() + unattached_log_entries.delete() self._write_success("Log entries deleted.") else: - self._write_success("No unattached log entries found. Everything clear.") + self._write_success("No unattached log entries found.") + self._break_line() + + def get_all_action_ids(self, cls): + """ Getter for all action ids of a model + + Args: + cls (Compensation|Ema|EcoAccount): The model class + + Returns: + + """ + all_objects = cls.objects.all().prefetch_related( + "actions", + ) + all_ids = all_objects.exclude( + actions__isnull=True, + ).values_list( + "actions__id", + flat=True, + ) + return all_ids + + def sanitize_actions(self): + """ Removes log entries which are not attached to any logs + + Returns: + + """ + self._write_warning("=== Sanitize compensation actions ===") + all_actions = CompensationAction.objects.all() + + compensation_action_ids = self.get_all_action_ids(Compensation) + attached_action_ids = compensation_action_ids.union( + self.get_all_action_ids(EcoAccount), + self.get_all_action_ids(Ema), + ) + + unattached_actions = all_actions.exclude(id__in=attached_action_ids) + + num_entries = unattached_actions.count() + if num_entries > 0: + self._write_warning(f"Found {num_entries} actions not attached to anything. Delete now...") + unattached_actions.delete() + self._write_success("Actions deleted.") + else: + self._write_success("No unattached actions found.") + self._break_line() def get_all_state_ids(self, cls): """ Getter for all states (before and after) of a class @@ -60,9 +126,6 @@ class Command(BaseKonovaCommand): Returns: """ - if not isinstance(cls, AbstractCompensation): - return QuerySet.none() - all_objects = cls.objects.all().prefetch_related( "before_states", "after_states", @@ -88,21 +151,22 @@ class Command(BaseKonovaCommand): Returns: """ + self._write_warning("=== Sanitize compensation states ===") all_states = CompensationState.objects.all() compensation_state_ids = self.get_all_state_ids(Compensation) account_state_ids = self.get_all_state_ids(EcoAccount) ema_state_ids = self.get_all_state_ids(Ema) - all_state_ids = compensation_state_ids.union(account_state_ids, ema_state_ids) + attached_state_ids = compensation_state_ids.union(account_state_ids, ema_state_ids) - all_states = all_states.exclude( - id__in=all_state_ids + unattached_states = all_states.exclude( + id__in=attached_state_ids ) - num_all_states = all_states.count() - if num_all_states > 0: - self._write_warning(f"Found {num_all_states} unused compensation states. Delete now...") - all_states.delete() + num_unattached_states = unattached_states.count() + if num_unattached_states > 0: + self._write_warning(f"Found {num_unattached_states} unused compensation states. Delete now...") + unattached_states.delete() self._write_success("Unused states deleted.") else: - self._write_success("No unused states found!") - + self._write_success("No unused states found.") + self._break_line()