diff --git a/data/config/mosquitto/public/default-dynamic-security.json b/data/config/mosquitto/public/default-dynamic-security.json index 10ed378e2f..308341f675 100644 --- a/data/config/mosquitto/public/default-dynamic-security.json +++ b/data/config/mosquitto/public/default-dynamic-security.json @@ -183,6 +183,12 @@ "priority": 0, "allow": true }, + { + "acltype": "publishClientReceive", + "topic": "openWB/system/log_totals_generation_finished", + "priority": 0, + "allow": true + }, { "acltype": "publishClientReceive", "topic": "openWB/system/usage_terms_acknowledged", diff --git a/packages/helpermodules/measurement_logging/generate_totals_subprocess.py b/packages/helpermodules/measurement_logging/generate_totals_subprocess.py new file mode 100644 index 0000000000..b78e46889c --- /dev/null +++ b/packages/helpermodules/measurement_logging/generate_totals_subprocess.py @@ -0,0 +1,106 @@ +import fcntl +from pathlib import Path +from datetime import date + +from typing import List +from helpermodules import pub +from helpermodules.measurement_logging.process_log import (save_daily_source_totals, + save_monthly_source_totals) + +import logging +from logging.handlers import RotatingFileHandler + + +BASE_PATH = Path(__file__).resolve().parents[3] +LOG_DIR = BASE_PATH / "ramdisk" +LOG_FILE = LOG_DIR / "generate_totals.log" + +LOG_DIR.mkdir(parents=True, exist_ok=True) + +handler = RotatingFileHandler( + filename=LOG_FILE, + maxBytes=5 * 1024 * 1024, # 5 MB, + backupCount=1, +) + +logging.basicConfig( + handlers=[handler], + level=logging.DEBUG, + format="%(asctime)s [%(levelname)s] %(message)s", +) + +log = logging.getLogger(__name__) + +LOCK_FILE = BASE_PATH / "data" / "generate_totals.lock" + + +def get_all_days_to_calc(): + try: + daily_log_dir = BASE_PATH / "data" / "daily_log" + if not daily_log_dir.is_dir(): + return None + + today_stem = date.today().strftime("%Y%m%d") + daily_log_files = sorted( + [p for p in daily_log_dir.glob("*.json") + if p.stem.isdigit() and p.stem < today_stem], + key=lambda p: p.stem, + ) + + log.debug(f"Anzahl zu berechnender Tageslogs: {len(daily_log_files)}") + return daily_log_files + except Exception: + log.debug("Fehler beim Abrufen der Tageslogs. Es werden keine Tageslogs berechnet.") + return None + + +def get_all_months_to_calc(days: List[str] = None): + if days is None: + return None + current_month = date.today().strftime("%Y%m") + months = sorted({p.stem[:6] for p in days if len(p.stem) == 8 and p.stem[:6] < current_month}) + log.debug(f"Anzahl zu berechnender Monatslogs: {len(months)}") + return months + + +def _generate_totals(): + + errors = 0 + days_to_calc = get_all_days_to_calc() or [] + months_to_calc = get_all_months_to_calc(days_to_calc) or [] + + for day in days_to_calc: + try: + save_daily_source_totals(day.stem, saving=True) + except Exception: + log.exception(f"Fehler beim Generieren der Tageserträge für {day.stem}, Tag wird übersprungen.") + errors += 1 + continue + + for month in months_to_calc: + try: + save_monthly_source_totals(month, saving=True) + except Exception: + log.exception(f"Fehler beim Generieren der Monatswerte für {month}, Monat wird übersprungen.") + errors += 1 + continue + + log.info(f"Totals-Migration abgeschlossen. Fehlerhafte Logs: {errors}.") + pub.Pub().pub("openWB/set/system/log_totals_generation_finished", True) + + +def generate_totals(): + LOCK_FILE.parent.mkdir(parents=True, exist_ok=True) + + with LOCK_FILE.open("w") as lock_file: + try: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except Exception: + log.exception("generate_totals_subprocess läuft bereits. " + "Der zweite Prozess wird beendet.") + return + _generate_totals() + + +if __name__ == "__main__": + generate_totals() diff --git a/packages/helpermodules/measurement_logging/generate_totals_subprocess_test.py b/packages/helpermodules/measurement_logging/generate_totals_subprocess_test.py new file mode 100644 index 0000000000..81d03cb521 --- /dev/null +++ b/packages/helpermodules/measurement_logging/generate_totals_subprocess_test.py @@ -0,0 +1,43 @@ +from unittest.mock import Mock + +from helpermodules.measurement_logging import generate_totals_subprocess + + +def test_generate_totals_calls_generator_when_lock_is_acquired(monkeypatch, tmp_path): + # setup + lock_file = tmp_path / "locks" / "generate_totals.lock" + monkeypatch.setattr(generate_totals_subprocess, "LOCK_FILE", lock_file) + + flock_mock = Mock() + generate_mock = Mock() + + monkeypatch.setattr(generate_totals_subprocess.fcntl, "flock", flock_mock) + monkeypatch.setattr(generate_totals_subprocess, "_generate_totals", generate_mock) + + # execution + generate_totals_subprocess.generate_totals() + + # evaluation + flock_mock.assert_called_once() + generate_mock.assert_called_once() + assert lock_file.parent.is_dir() + + +def test_generate_totals_skips_generator_when_lock_is_not_acquired(monkeypatch, tmp_path): + # setup + lock_file = tmp_path / "locks" / "generate_totals.lock" + monkeypatch.setattr(generate_totals_subprocess, "LOCK_FILE", lock_file) + + flock_mock = Mock(side_effect=RuntimeError("already locked")) + generate_mock = Mock() + + monkeypatch.setattr(generate_totals_subprocess.fcntl, "flock", flock_mock) + monkeypatch.setattr(generate_totals_subprocess, "_generate_totals", generate_mock) + + # execution + generate_totals_subprocess.generate_totals() + + # evaluation + flock_mock.assert_called_once() + generate_mock.assert_not_called() + assert lock_file.parent.is_dir() diff --git a/packages/helpermodules/measurement_logging/process_log.py b/packages/helpermodules/measurement_logging/process_log.py index 24eab8feff..ff3b195648 100644 --- a/packages/helpermodules/measurement_logging/process_log.py +++ b/packages/helpermodules/measurement_logging/process_log.py @@ -1,12 +1,11 @@ from enum import Enum +from copy import deepcopy import json import logging from pathlib import Path from typing import Dict, List, Optional, Tuple, Union - +from datetime import datetime from helpermodules import timecheck -from helpermodules.measurement_logging.write_log import (LegacySmartHomeLogData, LogType, create_entry, - get_previous_entry) from helpermodules.messaging import MessageType, pub_system_message from helpermodules.utils.precision_math import decimal_add, decimal_divide, decimal_multiply, decimal_subtract @@ -244,9 +243,15 @@ def _collect_daily_log_data(date: str): with open(str(parent_file / (date+".json")), "r") as json_file: log_data = json.load(json_file) if date == timecheck.create_timestamp_YYYYMMDD(): + # Behebt Circular-Import-Fehler + from helpermodules.measurement_logging.write_log import ( + LegacySmartHomeLogData, + create_entry, + get_previous_entry + ) # beim aktuellen Tag den aktuellen Datensatz ergänzen - log_data["entries"].append(create_entry( - LogType.DAILY, LegacySmartHomeLogData(), get_previous_entry(parent_file, log_data))) + log_data["entries"].append(create_entry(LegacySmartHomeLogData(), + get_previous_entry(parent_file, log_data))) else: # bei älteren als letzten Datensatz den des nächsten Tags try: @@ -263,122 +268,115 @@ def _collect_daily_log_data(date: str): def get_monthly_log(date: str): - data = _collect_monthly_log_data(date) - data["entries"] = _process_entries(data["entries"], CalculationType.ENERGY) - data["totals"] = get_totals(data["entries"], False) - data = _analyse_energy_source(data) - return data - -def _collect_monthly_log_data(date: str): - try: - with open(f"{_get_data_folder_path()}/monthly_log/{date}.json", "r") as jsonFile: - log_data = json.load(jsonFile) - this_month = timecheck.create_timestamp_YYYYMM() - if date == this_month: - # add last entry of current day, if current month is requested - try: - today = timecheck.create_timestamp_YYYYMMDD() - with open(f"{_get_data_folder_path()}/daily_log/{today}.json", - "r") as todayJsonFile: - today_log_data = json.load(todayJsonFile) - if len(today_log_data["entries"]) > 0: - log_data["entries"].append(today_log_data["entries"][-1]) - except FILE_ERRORS: - pass - else: - # add first entry of next month - try: - next_date = timecheck.get_relative_date_string(date, month_offset=1) - with open(f"{_get_data_folder_path()}/monthly_log/{next_date}.json", - "r") as nextJsonFile: - next_log_data = json.load(nextJsonFile) - log_data["entries"].append(next_log_data["entries"][0]) - except FILE_ERRORS: - pass - except FILE_ERRORS: - log_data = {"entries": [], "names": {}} - return log_data + if not (len(date) == 6 and date.isdigit()): + log.debug(f"Ungültiges Datum für Monats-Summen: {date}") + return {"entries": [], "names": {}, "colors": {}, "totals": {}} + # Nur Logs ab dem ältesten Tageslog auswerten + # Sonst werden unnötige Totals-Werte gespeichert + oldest_log_day = _oldest_log_day() + if (oldest_log_day is None + or date < oldest_log_day[:6]): # Jahr und Monat + return {"entries": [], "names": {}, "colors": {}, "totals": {}} + + monthly_entries = [] + monthly_names = {} + monthly_colors = {} + + today = timecheck.create_timestamp_YYYYMMDD() + day = f"{date}01" + + while day.startswith(date): + # Zukunftstage/-monate nicht verarbeiten, um keine leeren daily_totals zu erzeugen. + if day > today: + break + if day < oldest_log_day: + day = timecheck.get_relative_date_string(day, day_offset=1) + continue + + content = load_daily_source_totals_content(day) + if content is None: + # Dürfte eigentlich nie passieren... + content = save_daily_source_totals(day, saving=(day != today)) + + if isinstance(content, dict): + daily_totals = content.get("totals") + daily_entry = content.get("entry") + + if isinstance(daily_totals, dict) and isinstance(daily_entry, dict) and len(daily_entry) > 0: + daily_entry = deepcopy(daily_entry) + daily_entry["date"] = day + _apply_source_totals(daily_entry, daily_totals) + + monthly_entries.append(daily_entry) + if isinstance(content.get("names"), dict): + monthly_names.update(content["names"]) + if isinstance(content.get("colors"), dict): + monthly_colors.update(content["colors"]) + + day = timecheck.get_relative_date_string(day, day_offset=1) + + if len(monthly_entries) > 0: + data = {"entries": monthly_entries, "names": monthly_names, "colors": monthly_colors} + data["totals"] = get_totals(data["entries"], False) + data["totals"] = analyse_percentage_totals(data["entries"], data["totals"]) + return data + + # Fallback, wenn keine Daten vorhanden sind + return {"entries": [], "names": {}, "colors": {}, "totals": {}} def get_yearly_log(year: str): - data = _collect_yearly_log_data(year) - data["entries"] = _process_entries(data["entries"], CalculationType.ENERGY) - data["totals"] = get_totals(data["entries"], False) - data = _analyse_energy_source(data) - return data + # Nur Logs ab dem ältesten Tageslog auswerten + # Sonst werden unnötige totals Werte gespeichert + oldest_log_day = _oldest_log_day() + if oldest_log_day is None or year < oldest_log_day[:4]: + return {"entries": [], "names": {}, "colors": {}, "totals": {}} + + monthly_entries = [] + monthly_names = {} + monthly_colors = {} + this_month = timecheck.create_timestamp_YYYYMM() + month = f"{year}01" + while month.startswith(year): + if month > this_month: + break -def _collect_yearly_log_data(year: str): - def add_monthly_log(month: str, check_next_month: bool = False) -> None: - monthly_log_path = Path(__file__).resolve().parents[3]/"data"/"monthly_log" - try: - with open(monthly_log_path / f"{month}.json", "r") as jsonFile: - content = json.load(jsonFile) - entries.append(content["entries"][0]) - # add last entry of current file if next file is missing - if check_next_month: - next_month = timecheck.get_relative_date_string(month, month_offset=1) - if not (monthly_log_path / (next_month+".json")).is_file(): - entries.append(content["entries"][-1]) - log.debug(f"Keine Logdatei für Monat {next_month} gefunden, " - f"füge letzten Datensatz von {month} ein: {entries[-1]['date']}") - names.update(content["names"]) - except FILE_ERRORS: - log.debug(f"Kein Log für Monat {month} gefunden.") - - def add_daily_log(day: str) -> None: - try: - with open(f"{_get_data_folder_path()}/daily_log/{day}.json", "r") as dayJsonFile: - day_log_data = json.load(dayJsonFile) - if len(day_log_data["entries"]) > 0: - entries.append(day_log_data["entries"][-1]) - except FILE_ERRORS: - pass - - entries = [] - names = {} - dates = [] - - # we have to find a valid data range - this_year = timecheck.create_timestamp_YYYY() - this_month = timecheck.create_timestamp_YYYYMM() - if year < this_year: - # if the requested year is in the past, just add all possible months - for month in range(1, 13): - dates.append(f"{year}{month:02}") - else: - # add all months until current month - for month in range(1, int(this_month[-2:])+1): - dates.append(f"{year}{month:02}") - # add data for month range - for date in dates: - try: - log.debug(f"add regular month: {date}") - add_monthly_log(date, date != this_month) - except Exception: - log.exception(f"Fehler beim Zusammenstellen der Jahresdaten für Monat {date}") + if month < oldest_log_day[:6]: + month = timecheck.get_relative_date_string(month, month_offset=1) + continue - # now we have to find a valid "next" entry for proper calculation - if year == this_year: # current year - # add todays last entry - this_day = timecheck.create_timestamp_YYYYMMDD() - try: - log.debug(f"add today: {this_day}") - add_daily_log(this_day) - except Exception: - log.exception(f"Fehler beim Zusammenstellen der Jahresdaten für den aktuellen Tag {this_day}") - else: - # no special handling here, just add first entry of next month - next_date = f"{int(year)+1}01" - try: - log.debug(f"add next month: {next_date}") - add_monthly_log(next_date) - except Exception: - log.exception(f"Fehler beim Zusammenstellen der Jahresdaten für Monat {next_date}") + content = load_monthly_source_totals_content(month) + if content is None: + # Dürfte eigentlich nie passieren... + content = save_monthly_source_totals(month, None, saving=(month != this_month)) + + if isinstance(content, dict): + monthly_totals = content.get("totals") + monthly_entry = content.get("entry") + + if isinstance(monthly_totals, dict) and isinstance(monthly_entry, dict) and len(monthly_entry) > 0: + monthly_entry = deepcopy(monthly_entry) + monthly_entry["date"] = month + _apply_source_totals(monthly_entry, monthly_totals) + + monthly_entries.append(monthly_entry) + if isinstance(content.get("names"), dict): + monthly_names.update(content["names"]) + if isinstance(content.get("colors"), dict): + monthly_colors.update(content["colors"]) - # return our data - return {"entries": entries, "names": names} + month = timecheck.get_relative_date_string(month, month_offset=1) + + if len(monthly_entries) > 0: + data = {"entries": monthly_entries, "names": monthly_names, "colors": monthly_colors} + data["totals"] = get_totals(data["entries"], False) + data["totals"] = analyse_percentage_totals(data["entries"], data["totals"]) + return data + + # Fallback, wenn keine Daten vorhanden sind + return {"entries": [], "names": {}, "colors": {}, "totals": {}} def _analyse_energy_source(data, calc_cp: Optional[str] = None) -> Dict: @@ -530,33 +528,59 @@ def calc_energy_imported_by_source(entry, names, message_key_filter: Optional[st def analyse_percentage_totals(entries, totals): - for section in ("hc", "cp"): - if "all" not in totals[section].keys(): - totals[section]["all"] = {} - for source in ("grid", "pv", "bat", "cp"): - totals["hc"]["all"].update({f"energy_imported_{source}": 0}) - for entry in entries: - if "hc" in entry.keys() and "all" in entry["hc"].keys(): - current_value = totals["hc"]["all"][f"energy_imported_{source}"] - add_value = entry["hc"]["all"].get(f"energy_imported_{source}", 0) - totals["hc"]["all"][f"energy_imported_{source}"] = decimal_add( - current_value, add_value) - for key in entry["cp"].keys(): - if f"energy_imported_{source}" in entry["cp"][key].keys(): - if totals["cp"][key].get(f"energy_imported_{source}") is None: - totals["cp"][key].update({f"energy_imported_{source}": 0}) - current_value = totals["cp"][key][f"energy_imported_{source}"] - add_value = entry["cp"][key][f"energy_imported_{source}"] - totals["cp"][key][f"energy_imported_{source}"] = decimal_add( - current_value, add_value) - for key, counter in entry["counter"].items(): - if counter["grid"] is False: - if totals["counter"][key].get(f"energy_imported_{source}") is None: - totals["counter"][key].update({f"energy_imported_{source}": 0}) - current_value = totals["counter"][key][f"energy_imported_{source}"] - add_value = counter[f"energy_imported_{source}"] - totals["counter"][key][f"energy_imported_{source}"] = decimal_add( + sources = ("grid", "pv", "bat", "cp") + + def ensure_zero_source_keys(module_totals: Dict): + if isinstance(module_totals, dict): + for source in sources: + module_totals[f"energy_imported_{source}"] = 0 + + try: + for section in ("hc", "cp"): + if "all" not in totals[section].keys(): + totals[section]["all"] = {} + for source in sources: + totals["hc"]["all"].update({f"energy_imported_{source}": 0}) + for entry in entries: + if "hc" in entry.keys() and "all" in entry["hc"].keys(): + current_value = totals["hc"]["all"][f"energy_imported_{source}"] + add_value = entry["hc"]["all"].get(f"energy_imported_{source}", 0) + totals["hc"]["all"][f"energy_imported_{source}"] = decimal_add( current_value, add_value) + for key in entry["cp"].keys(): + if f"energy_imported_{source}" in entry["cp"][key].keys(): + if totals["cp"][key].get(f"energy_imported_{source}") is None: + totals["cp"][key].update({f"energy_imported_{source}": 0}) + current_value = totals["cp"][key][f"energy_imported_{source}"] + add_value = entry["cp"][key][f"energy_imported_{source}"] + totals["cp"][key][f"energy_imported_{source}"] = decimal_add( + current_value, add_value) + for key, counter in entry["counter"].items(): + if counter["grid"] is False: + if totals["counter"][key].get(f"energy_imported_{source}") is None: + totals["counter"][key].update({f"energy_imported_{source}": 0}) + current_value = totals["counter"][key][f"energy_imported_{source}"] + add_value = counter[f"energy_imported_{source}"] + totals["counter"][key][f"energy_imported_{source}"] = decimal_add( + current_value, add_value) + except Exception: + log.exception("Fehler beim Berechnen der Summen der Energiequellen") + # Im Fehlerfall werden die Totals auf 0 gesetzt + # -> dann wird nur die Ladung/Entladung-Leistung angezeigt. + # -> sprich keine Aufteilung der Energiequellen, sondern nur die Gesamtwerte. + totals.setdefault("hc", {}) + totals["hc"].setdefault("all", {}) + ensure_zero_source_keys(totals["hc"]["all"]) + + totals.setdefault("cp", {}) + for key, module_totals in totals["cp"].items(): + ensure_zero_source_keys(module_totals) + + totals.setdefault("counter", {}) + for key, module_totals in totals["counter"].items(): + ensure_zero_source_keys(module_totals) + + return totals return totals @@ -597,13 +621,26 @@ def process_entry(entry: dict, next_entry: dict, calculation: CalculationType): new_data = {} if "imported" in entry[type][module].keys() or "exported" in entry[type][module].keys(): def get_current_and_next(value_key: str) -> Tuple[float, float]: - def get_single_value(source: dict, default: int = 0) -> float: + def get_single_value(source: dict) -> Optional[float]: try: - return source[type][module][value_key] + value = source[type][module][value_key] + if isinstance(value, (int, float)): + return float(value) except KeyError: - return default + pass + return None + current_value = get_single_value(entry) - return current_value, get_single_value(next_entry, current_value) + next_value = get_single_value(next_entry) + + # Keep meter deltas neutral if one side is invalid/missing. + if current_value is None and next_value is None: + return 0.0, 0.0 + if current_value is None: + return next_value, next_value + if next_value is None: + return current_value, current_value + return current_value, next_value value_imported, next_value_imported = get_current_and_next("imported") value_exported, next_value_exported = get_current_and_next("exported") if calculation in [CalculationType.POWER, CalculationType.ALL]: @@ -665,3 +702,187 @@ def _calculate_average_power(time_diff: float, current_imported: float = 0, next def _get_data_folder_path() -> str: return str(Path(__file__).resolve().parents[3] / "data") + + +def save_daily_source_totals(date: str, saving: bool = True): + try: + if not (len(date) == 8 and date.isdigit()): + log.debug(f"Ungültiges Datum für Tages-Summen: {date}") + return None + data = _collect_daily_log_data(date) + source_entries = data.get("entries", []) + + # Keep one source snapshot for the daily output entry and process entries in place to avoid full-copy peaks. + source_daily_entry = _get_last_entry_for_period(source_entries, date, "%Y%m%d") + processed_entries = _process_entries(source_entries, calculation=CalculationType.ENERGY) + + totals = get_totals(processed_entries, process_entries=False) + analysed_data = _analyse_energy_source({ + "entries": processed_entries, + "totals": totals, + "names": data.get("names", {}) + }) + totals = analysed_data["totals"] + + daily_entry = {} + if source_daily_entry is not None: + daily_entry = deepcopy(source_daily_entry) + daily_entry["date"] = date + _apply_source_totals(daily_entry, totals) + + # Erzeugt Ordner daily_totals, falls nicht vorhanden + totals_dir = Path(_get_data_folder_path()) / "daily_totals" + filepath = totals_dir / f"{date}_totals.json" + + content = { + "date": date, + "totals": totals, + "entry": daily_entry, + "names": data.get("names", {}), + "colors": data.get("colors", {}) + } + + if saving: + totals_dir.mkdir(parents=True, exist_ok=True) + with open(str(filepath), "w") as jsonFile: + json.dump(content, jsonFile, ensure_ascii=False, indent=2) + + log.debug(f"Tages-Summen für {date} gespeichert in {filepath}") + + return content + + except FILE_ERRORS: + log.exception(f"Fehler beim Speichern der Tages-Summen für {date}") + + +def load_daily_source_totals_content(date: str): + try: + if not (len(date) == 8 and date.isdigit()): + log.debug(f"Ungültiges Datum für Tages-Summen: {date}") + return None + + filepath = f"{_get_data_folder_path()}/daily_totals/{date}_totals.json" + if not Path(filepath).is_file(): + log.debug(f"Keine Tages-Summen-Datei gefunden: {filepath}") + return None + + with open(str(filepath), "r") as jsonFile: + content = json.load(jsonFile) + + log.debug(f"Tages-Summen für {date} geladen aus {filepath}") + return content + + except FILE_ERRORS: + log.exception(f"Fehler beim Laden der Tages-Summen für {date}") + + +def save_monthly_source_totals(date: str, data: Optional[Dict] = None, saving: bool = True): + try: + # Hauptsächlich für Midnight-Handler + # Wenn keine Daten übergeben werden, dann die Monatswerte berechnen + if data is None: + data = get_monthly_log(date) + + totals = data["totals"] + source_entries = data.get("entries", []) + monthly_entry = {} + source_monthly_entry = _get_last_entry_for_period(source_entries, date, "%Y%m") + if source_monthly_entry is not None: + # Nur den letzten Eintrag des Monats nehmen + monthly_entry = deepcopy(source_monthly_entry) + monthly_entry["date"] = date + + # Erzeugt Ordner monthly_totals, falls nicht vorhanden + totals_dir = Path(_get_data_folder_path()) / "monthly_totals" + filepath = totals_dir / f"{date}_totals.json" + content = { + "date": date, + "totals": totals, + "entry": monthly_entry, + "names": data.get("names", {}), + "colors": data.get("colors", {}) + } + + if saving: + totals_dir.mkdir(parents=True, exist_ok=True) + with open(str(filepath), "w") as jsonFile: + json.dump(content, jsonFile, ensure_ascii=False, indent=2) + + log.debug(f"Monats-Summen für {date} gespeichert in {filepath}") + return content + + except FILE_ERRORS: + log.exception(f"Fehler beim Speichern der Monats-Summen für {date}") + + +def load_monthly_source_totals_content(date: str): + try: + filepath = f"{_get_data_folder_path()}/monthly_totals/{date}_totals.json" + if not Path(filepath).is_file(): + log.debug(f"Keine Monats-Summen-Datei gefunden: {filepath}") + return None + + with open(str(filepath), "r") as jsonFile: + content = json.load(jsonFile) + + log.debug(f"Monats-Summen für {date} geladen aus {filepath}") + return content + + except FILE_ERRORS: + log.exception(f"Fehler beim Laden der Monats-Summen für {date}") + + +def _get_last_entry_for_period(entries: List, period: str, period_format: str) -> Optional[Dict]: + # Suche den letzten Eintrag in der Liste, der dem angegebenen Zeitraum entspricht. + for entry in reversed(entries): + if isinstance(entry, dict) and isinstance(entry.get("timestamp"), (int, float)): + entry_period = datetime.fromtimestamp(entry["timestamp"]).strftime(period_format) + if entry_period == period: + return entry + + # Fallback: Falls kein passender Zeitstempel gefunden wird, letzten gueltigen Eintrag verwenden. + for entry in reversed(entries): + if isinstance(entry, dict): + return entry + return None + + +def _apply_source_totals(entry: Dict, daily_totals: Dict): + for section, section_totals in daily_totals.items(): + section_data = entry.get(section) + if not isinstance(section_totals, dict): + continue + + if not isinstance(section_data, dict): + section_data = {} + entry[section] = section_data + + for module, module_totals in section_totals.items(): + module_data = section_data.get(module) + if not isinstance(module_totals, dict): + continue + + if not isinstance(module_data, dict): + module_data = {} + section_data[module] = module_data + + # Alle vorhandenen Summenfelder des Moduls mit den Tages-Summen ueberschreiben. + module_data.update(module_totals) + return entry + + +def _oldest_log_day() -> Optional[str]: + try: + daily_log_dir = Path(_get_data_folder_path()) / "daily_log" + if not daily_log_dir.is_dir(): + return None + + daily_log_files = [p for p in daily_log_dir.glob("*.json") if p.stem.isdigit()] + if not daily_log_files: + return None + + oldest_file = min(daily_log_files, key=lambda f: f.stem) + return oldest_file.stem + except Exception: + log.exception("Fehler beim Ermitteln des ältesten Tageslogs") + return None diff --git a/packages/helpermodules/measurement_logging/process_log_unit_test.py b/packages/helpermodules/measurement_logging/process_log_unit_test.py index 30b56ec065..63341875e8 100644 --- a/packages/helpermodules/measurement_logging/process_log_unit_test.py +++ b/packages/helpermodules/measurement_logging/process_log_unit_test.py @@ -11,6 +11,8 @@ process_entry, get_totals, _collect_daily_log_data, + _get_last_entry_for_period, + _apply_source_totals, calc_energy_imported_by_source, analyse_percentage_totals, CalculationType) @@ -295,10 +297,10 @@ def test_collect_daily_log_data_current_day(monkeypatch): monkeypatch.setattr('helpermodules.measurement_logging.process_log.json.load', mock_json_load) mock_create_entry = Mock(return_value=mock_current_entry) - monkeypatch.setattr('helpermodules.measurement_logging.process_log.create_entry', mock_create_entry) + monkeypatch.setattr('helpermodules.measurement_logging.write_log.create_entry', mock_create_entry) mock_get_previous_entry = Mock(return_value={"timestamp": 1234567800, "data": "previous"}) - monkeypatch.setattr('helpermodules.measurement_logging.process_log.get_previous_entry', mock_get_previous_entry) + monkeypatch.setattr('helpermodules.measurement_logging.write_log.get_previous_entry', mock_get_previous_entry) monkeypatch.setattr('builtins.open', mock_open(read_data=json.dumps(mock_log_data))) @@ -392,3 +394,78 @@ def test_collect_daily_log_data_json_decode_error(monkeypatch): # evaluation expected_result = {"entries": [], "names": {}} assert result == expected_result + + +def test_apply_source_totals_updates_existing_and_creates_missing_sections(): + # setup + entry = { + "cp": { + "all": {"energy_imported": 5, "keep": "x"}, + "cp9": {"keep_cp": True} + }, + "meta": {"unchanged": True} + } + daily_totals = { + "cp": { + "all": {"energy_imported": 12, "energy_exported": 3}, + "cp1": {"energy_imported": 7}, + "cp_invalid": 99 + }, + "hc": { + "all": {"energy_imported": 2} + }, + "invalid_section": "ignore_me" + } + + # execution + result = _apply_source_totals(entry, daily_totals) + + # evaluation + # in-place behavior + assert result is entry + + # existing module gets overwritten/extended, unrelated fields stay + assert result["cp"]["all"]["energy_imported"] == 12 + assert result["cp"]["all"]["energy_exported"] == 3 + assert result["cp"]["all"]["keep"] == "x" + + # missing module and section are created + assert result["cp"]["cp1"]["energy_imported"] == 7 + assert result["hc"]["all"]["energy_imported"] == 2 + + # invalid totals are ignored + assert "cp_invalid" not in result["cp"] + assert "invalid_section" not in result + + # unrelated data stays unchanged + assert result["cp"]["cp9"]["keep_cp"] is True + assert result["meta"]["unchanged"] is True + + +@pytest.mark.parametrize( + "entries, period, period_format, expected", + [ + ( + [ + {"timestamp": 1711929600, "value": "april_1"}, + {"timestamp": 1712016000, "value": "april_2"}, + {"timestamp": 1714608000, "value": "may_2"}, + ], + "202404", + "%Y%m", + {"timestamp": 1712016000, "value": "april_2"}, + ), + ( + [], + "202404", + "%Y%m", + None, + ), + ], +) +def test_get_last_entry_for_period(entries, period, period_format, expected): + # execution + result = _get_last_entry_for_period(entries, period, period_format) + + # evaluation + assert result == expected diff --git a/packages/helpermodules/measurement_logging/update_yields.py b/packages/helpermodules/measurement_logging/update_yields.py index 23b1a8a251..dc73d27ea2 100644 --- a/packages/helpermodules/measurement_logging/update_yields.py +++ b/packages/helpermodules/measurement_logging/update_yields.py @@ -1,11 +1,14 @@ -import json import logging from pathlib import Path from typing import Dict from control import data from helpermodules import timecheck -from helpermodules.measurement_logging.process_log import get_totals +from helpermodules.measurement_logging.process_log import (get_totals, + load_daily_source_totals_content, + load_monthly_source_totals_content, + save_daily_source_totals, + get_monthly_log) log = logging.getLogger(__name__) @@ -17,6 +20,7 @@ def update_daily_yields(entries): totals = get_totals(entries) [update_module_yields(type, totals) for type in ("bat", "counter", "cp", "pv")] data.data.counter_all_data.data.set.daily_yield_home_consumption = totals["hc"]["all"]["energy_imported"] + return totals except Exception: log.exception("Fehler beim Veröffentlichen der Tageserträge.") @@ -40,72 +44,115 @@ def update_module_yields(module: str, totals: Dict) -> None: log.exception(f"Fehler beim Aktualisieren der Tageserträge für Modul {m} vom Typ {module}.") -def update_pv_monthly_yearly_yields(): - """ veröffentlicht die monatlichen und jährlichen Erträge für PV +def update_pv_monthly_yearly_yields(daily_totals: Dict) -> None: """ - _update_pv_monthly_yields() - _update_pv_yearly_yields() + veröffentlicht die monatlichen und jährlichen Erträge für PV + """ + # pv_data und pv_all_data werden nur aktualisiert, wenn das Update der Logfiles abgeschlossen ist. + if not data.data.system_data["system"].data.get("log_totals_generation_finished", False): + log.debug("Update der Logfiles läuft noch. Monatliche und jährliche PV-Erträge werden nicht aktualisiert.") + return + + monthly_totals = _get_pv_monthly_yields(daily_totals) + yearly_totals = _get_pv_yearly_yields(monthly_totals) + + pv_all_monthly_yield = 0 + pv_all_yearly_yield = 0 + + for pv_key, pv_module in data.data.pv_data.items(): + if pv_key == "all" or not hasattr(pv_module, "num"): + continue + + # Was wurde im Monat/Jahr exportiert + monthly_yield = monthly_totals.get(f"pv{pv_module.num}", {}).get("energy_exported", 0) + yearly_yield = yearly_totals.get(f"pv{pv_module.num}", {}).get("energy_exported", 0) + + data.data.pv_data[f"pv{pv_module.num}"].data.get.monthly_exported = monthly_yield + data.data.pv_data[f"pv{pv_module.num}"].data.get.yearly_exported = yearly_yield + + # Summe über alle Module für pv_all + pv_all_monthly_yield += monthly_yield + pv_all_yearly_yield += yearly_yield + + data.data.pv_all_data.data.get.monthly_exported = pv_all_monthly_yield + data.data.pv_all_data.data.get.yearly_exported = pv_all_yearly_yield -def _update_pv_monthly_yields(): - """ veröffentlicht die monatlichen Erträge für PV - für pv_all nicht die Differenz aus den Logs nehmen, sondern die Summe der Module. Wenn im laufenden Monat ein Modul - gelöscht wurde und keins oder eines mit niedrigerem Zählerstand hinzugefügt wird, wird sonst ein negativer Wert - ermittelt. +def _get_pv_monthly_yields(daily_totals: Dict) -> Dict: """ - try: - pv_all_monthly_yield = 0 - with open(f"data/monthly_log/{timecheck.create_timestamp_YYYYMM()}.json", "r") as f: - monthly_log = json.load(f) - for pv_module in data.data.pv_data.values(): - for entry in monthly_log["entries"]: - if entry["pv"].get(f"pv{pv_module.num}"): - monthly_yield = data.data.pv_data[f"pv{pv_module.num}"].data.get.exported - \ - entry["pv"][f"pv{pv_module.num}"]["exported"] - pv_all_monthly_yield += monthly_yield - data.data.pv_data[f"pv{pv_module.num}"].data.get.monthly_exported = monthly_yield - break - data.data.pv_all_data.data.get.monthly_exported = pv_all_monthly_yield - except FileNotFoundError: - # am Tag der Ersteinrichtung gibt es noch kein Monatslog-File, das wird erst um Mitternacht erstellt. - log.debug("No monthly logfile found for calculation of monthly yield") - except Exception: - log.exception("Fehler beim Veröffentlichen der monatlichen Erträge für PV") + Berechnet den Unterschied zwischen dem Zählerstand vom ersten Tag des aktuellen Monats bis zum aktuellen Tag. + """ + + this_month = timecheck.create_timestamp_YYYYMM() + today = timecheck.create_timestamp_YYYYMMDD() + + pv_totals = {} + + daily_log_path = _get_parent_path()/"data"/"daily_log" + + for logfile in sorted(daily_log_path.glob(f"{this_month}*.json")): + day = logfile.stem + totals = {} + if day == today: + continue # Der aktuelle Tag wird später behandelt + else: + # Lade alle vergangenen Tage des Monats aus dem daily_totals-Logfile, um die Tageserträge zu ermitteln + content = load_daily_source_totals_content(day) + if content is None: + # Fallback/Migration: fehlende Tages-Totals aus den Tageslogs berechnen und speichern. + content = save_daily_source_totals(day, saving=True) + if isinstance(content, dict): + totals = content.get("totals", {}) + + # Totals aufsummieren + _add_pv_totals(pv_totals, totals.get("pv", {})) + # aktueller Tag ergänzen + _add_pv_totals(pv_totals, daily_totals.get("pv", {})) -def _update_pv_yearly_yields(): - """ veröffentlicht die jährlichen Erträge für PV - für pv_all nicht die Differenz aus den Logs nehmen, sondern die Summe der Module. Wenn unterjährig ein Modul - gelöscht wurde und keins oder eines mit niedrigerem Zählerstand hinzugefügt wird, wird sonst ein negativer Wert - ermittelt. + return pv_totals + + +def _get_pv_yearly_yields(current_monthly_totals: Dict) -> Dict: """ - try: - pv_all_yearly_yield = 0 - path_list = list(Path(_get_parent_path()/"data"/"monthly_log").glob(f"{timecheck.create_timestamp_YYYY()}*")) - sorted_path_list = sorted([str(p) for p in path_list]) - for pv_module in data.data.pv_data.values(): - found_pv = False - for path in sorted_path_list: - with open(path, "r") as f: - monthly_log = json.load(f) - for entry in monthly_log["entries"]: - # erster Eintrag mit PV im Jahr, falls WR erst im laufenden Jahr hinzugefügt wurden - if entry["pv"].get(f"pv{pv_module.num}"): - yearly_yield = data.data.pv_data[f"pv{pv_module.num}"].data.get.exported - \ - entry["pv"][f"pv{pv_module.num}"]["exported"] - pv_all_yearly_yield += yearly_yield - data.data.pv_data[f"pv{pv_module.num}"].data.get.yearly_exported = yearly_yield - found_pv = True - break - if found_pv: - break - else: - # am Tag der Ersteinrichtung gibt es noch kein Monatslog-File, das wird erst um Mitternacht erstellt. - log.debug("No monthly logfile found for calculation of yearly yield") - data.data.pv_all_data.data.get.yearly_exported = pv_all_yearly_yield - except Exception: - log.exception("Fehler beim Veröffentlichen der jährlichen Erträge für PV") + Berechnet den Unterschied zwischen dem Zählerstand vom ersten Monat des aktuellen Jahres bis zum aktuellen Monat. + """ + this_year = timecheck.create_timestamp_YYYY() + this_month = timecheck.create_timestamp_YYYYMM() + + pv_totals = {} + + month = f"{this_year}01" + while month < this_month: + totals = {} + content = load_monthly_source_totals_content(month) + if content is None: + # Fallback/Migration: fehlende Monats-Totals berechnen und speichern. + content = get_monthly_log(month) + if isinstance(content, dict): + totals = content.get("totals", {}) + + # Totals aufsummieren + _add_pv_totals(pv_totals, totals.get("pv", {})) + month = timecheck.get_relative_date_string(month, month_offset=1) + + # aktueller Monat ergänzen + _add_pv_totals(pv_totals, current_monthly_totals) + + return pv_totals def _get_parent_path() -> Path: return Path(__file__).resolve().parents[3] + + +def _add_pv_totals(target: Dict, source: Dict) -> None: + for pv_key, values in source.items(): + energy_exported = values.get("energy_exported", 0) + + # Wenn das PV-Modul noch nicht im target ist, initialisiere es mit 0 + if pv_key not in target: + target[pv_key] = {"energy_exported": 0} + + # Addiere die energy_exported Werte für das PV-Modul + target[pv_key]["energy_exported"] += energy_exported diff --git a/packages/helpermodules/measurement_logging/update_yields_test.py b/packages/helpermodules/measurement_logging/update_yields_test.py index 35a3fd7b81..2909508522 100644 --- a/packages/helpermodules/measurement_logging/update_yields_test.py +++ b/packages/helpermodules/measurement_logging/update_yields_test.py @@ -1,5 +1,11 @@ +from unittest.mock import Mock + from control import data -from helpermodules.measurement_logging.update_yields import update_module_yields +from helpermodules.measurement_logging import update_yields +from helpermodules.measurement_logging.update_yields import ( + update_module_yields, + update_pv_monthly_yearly_yields, +) def test_update_module_yields(daily_log_totals, mock_pub): @@ -21,3 +27,67 @@ def test_update_module_yields(daily_log_totals, mock_pub): data.data.cp_data["cp6"].data.get.daily_exported = 0.0 data.data.pv_all_data.data.get.daily_exported = 251.0 data.data.pv_data["pv1"].data.get.daily_exported = 251.0 + + +def test_update_pv_monthly_yearly_yields_skips_when_generation_not_finished(monkeypatch): + # setup + data.data.system_data = {"system": Mock(data={"log_totals_generation_finished": False})} + get_monthly_mock = Mock() + get_yearly_mock = Mock() + + monkeypatch.setattr(update_yields, "_get_pv_monthly_yields", get_monthly_mock) + monkeypatch.setattr(update_yields, "_get_pv_yearly_yields", get_yearly_mock) + + # execution + update_pv_monthly_yearly_yields({"pv": {"pv1": {"energy_exported": 12}}}) + + # evaluation + get_monthly_mock.assert_not_called() + get_yearly_mock.assert_not_called() + + +def test_update_pv_monthly_yearly_yields_with_daily_and_monthly_fallback(monkeypatch, tmp_path): + # setup + data.data.system_data = {"system": Mock(data={"log_totals_generation_finished": True})} + + parent_path = tmp_path + daily_log_path = parent_path / "data" / "daily_log" + daily_log_path.mkdir(parents=True) + (daily_log_path / "20260201.json").write_text("{}") + (daily_log_path / "20260214.json").write_text("{}") + (daily_log_path / "20260215.json").write_text("{}") + + monkeypatch.setattr(update_yields, "_get_parent_path", lambda: parent_path) + monkeypatch.setattr(update_yields.timecheck, "create_timestamp_YYYY", lambda: "2026") + monkeypatch.setattr(update_yields.timecheck, "create_timestamp_YYYYMM", lambda: "202602") + monkeypatch.setattr(update_yields.timecheck, "create_timestamp_YYYYMMDD", lambda: "20260215") + monkeypatch.setattr(update_yields.timecheck, "get_relative_date_string", lambda _m, month_offset=1: "202602") + + load_daily_mock = Mock(side_effect=[ + {"totals": {"pv": {"pv1": {"energy_exported": 10}}}}, + None, + ]) + save_daily_mock = Mock(return_value={"totals": {"pv": {"pv1": {"energy_exported": 20}}}}) + load_monthly_mock = Mock(return_value=None) + get_monthly_mock = Mock(return_value={"totals": {"pv": {"pv1": {"energy_exported": 100}}}}) + + monkeypatch.setattr(update_yields, "load_daily_source_totals_content", load_daily_mock) + monkeypatch.setattr(update_yields, "save_daily_source_totals", save_daily_mock) + monkeypatch.setattr(update_yields, "load_monthly_source_totals_content", load_monthly_mock) + monkeypatch.setattr(update_yields, "get_monthly_log", get_monthly_mock) + + daily_totals = {"pv": {"pv1": {"energy_exported": 5}}} + + # execution + update_pv_monthly_yearly_yields(daily_totals) + + # evaluation + assert data.data.pv_data["pv1"].data.get.monthly_exported == 35 + assert data.data.pv_data["pv1"].data.get.yearly_exported == 135 + assert data.data.pv_all_data.data.get.monthly_exported == 35 + assert data.data.pv_all_data.data.get.yearly_exported == 135 + + assert load_daily_mock.call_count == 2 + save_daily_mock.assert_called_once_with("20260214", saving=True) + load_monthly_mock.assert_called_once_with("202601") + get_monthly_mock.assert_called_once_with("202601") diff --git a/packages/helpermodules/measurement_logging/write_log.py b/packages/helpermodules/measurement_logging/write_log.py index fd8d51aa4b..ea86577707 100644 --- a/packages/helpermodules/measurement_logging/write_log.py +++ b/packages/helpermodules/measurement_logging/write_log.py @@ -1,4 +1,3 @@ -from enum import Enum import os import json import logging @@ -97,11 +96,6 @@ # } -class LogType(Enum): - DAILY = "daily" - MONTHLY = "monthly" - - class LegacySmartHomeLogData: def __init__(self) -> None: self.all_received_topics: Dict = {} @@ -133,20 +127,11 @@ def on_message(self, client: MqttClient, userdata, msg: MQTTMessage): self.all_received_topics.update({msg.topic: msg.payload}) -def save_log(log_type: LogType): - """ Parameter - --------- - folder: str - gibt an, ob ein Tages-oder Monats-Log-Eintrag erstellt werden soll. - """ +def save_log(): try: - parent_file = Path(__file__).resolve().parents[3] / "data" / \ - ("daily_log" if log_type == LogType.DAILY else "monthly_log") + parent_file = Path(__file__).resolve().parents[3] / "data" / "daily_log" parent_file.mkdir(mode=0o755, parents=True, exist_ok=True) - if log_type == LogType.DAILY: - file_name = timecheck.create_timestamp_YYYYMMDD() - else: - file_name = timecheck.create_timestamp_YYYYMM() + file_name = timecheck.create_timestamp_YYYYMMDD() filepath = str(parent_file / f"{file_name}.json") try: @@ -162,7 +147,7 @@ def save_log(log_type: LogType): previous_entry = get_previous_entry(parent_file, content) sh_log_data = LegacySmartHomeLogData() - new_entry = create_entry(log_type, sh_log_data, previous_entry) + new_entry = create_entry(sh_log_data, previous_entry) # json-Objekt in Datei einfügen @@ -194,11 +179,8 @@ def get_previous_entry(parent_file: Path, content: Dict) -> Optional[Dict]: return previous_entry -def create_entry(log_type: LogType, sh_log_data: LegacySmartHomeLogData, previous_entry: Optional[Dict]) -> Dict: - if log_type == LogType.DAILY: - date = timecheck.create_timestamp_HH_MM() - else: - date = timecheck.create_timestamp_YYYYMMDD() +def create_entry(sh_log_data: LegacySmartHomeLogData, previous_entry: Optional[Dict]) -> Dict: + date = timecheck.create_timestamp_HH_MM() current_timestamp = int(timecheck.create_timestamp()) try: diff --git a/packages/helpermodules/mosquitto_dynsec/missing_role_topics_test.py b/packages/helpermodules/mosquitto_dynsec/missing_role_topics_test.py index b06822ab14..cd6f53ab21 100644 --- a/packages/helpermodules/mosquitto_dynsec/missing_role_topics_test.py +++ b/packages/helpermodules/mosquitto_dynsec/missing_role_topics_test.py @@ -99,6 +99,7 @@ def _get_data_path() -> Path: 'openWB/optional/int_display/pin_active', 'openWB/optional/int_display/pin_code', 'openWB/system/datastore_version', + 'openWB/system/file_operation_version', 'openWB/system/device/+/component/+/simulation', 'openWB/system/device/+/component/+/simulation/power_present', 'openWB/system/device/+/component/+/simulation/present_exported', diff --git a/packages/helpermodules/setdata.py b/packages/helpermodules/setdata.py index 7c03072896..61e600c560 100644 --- a/packages/helpermodules/setdata.py +++ b/packages/helpermodules/setdata.py @@ -1045,6 +1045,7 @@ def process_system_topic(self, msg: mqtt.MQTTMessage): "openWB/set/system/perform_update" in msg.topic or "openWB/set/system/wizard_done" in msg.topic or "openWB/set/system/update_in_progress" in msg.topic or + "openWB/set/system/log_totals_generation_finished" in msg.topic or "openWB/set/system/backup_cloud/backup_before_update" in msg.topic or "openWB/set/system/installAssistantDone" in msg.topic or "openWB/set/system/dataprotection_acknowledged" in msg.topic or @@ -1059,6 +1060,8 @@ def process_system_topic(self, msg: mqtt.MQTTMessage): self._validate_value(msg, float) elif "openWB/set/system/datastore_version" in msg.topic: self._validate_value(msg, int, [(0, UpdateConfig.DATASTORE_VERSION)], collection=list) + elif "openWB/set/system/file_operation_version" in msg.topic: + self._validate_value(msg, int, [(0, UpdateConfig.FILE_OPERATION_VERSION)], collection=list) elif "openWB/set/system/GetRemoteSupport" in msg.topic: # Server-Topic enthält kein json-Payload. payload = msg.payload.decode("utf-8") diff --git a/packages/helpermodules/update_config.py b/packages/helpermodules/update_config.py index 48d2f72b1e..9a1d511879 100644 --- a/packages/helpermodules/update_config.py +++ b/packages/helpermodules/update_config.py @@ -9,6 +9,8 @@ from pathlib import Path import re import time +import sys +import subprocess from typing import List, Optional from paho.mqtt.client import Client as MqttClient, MQTTMessage @@ -60,6 +62,8 @@ class UpdateConfig: DATASTORE_VERSION = 137 + FILE_OPERATION_VERSION = 0 + valid_topic = [ "^openWB/bat/config/bat_control_activated$", "^openWB/bat/config/power_limit_mode$", @@ -526,10 +530,12 @@ class UpdateConfig: "^openWB/system/device/[0-9]+/component/[0-9]+/simulation/timestamp_present$", "^openWB/system/device/[0-9]+/config$", "^openWB/system/device/module_update_completed$", + "^openWB/system/file_operation_version$", "^openWB/system/hostname$", "^openWB/system/io/[0-9]+/config$", "^openWB/system/ip_address$", "^openWB/system/lastlivevaluesJson$", + "^openWB/system/log_totals_generation_finished$", "^openWB/system/mac_address$", "^openWB/system/mqtt/bridge/[0-9]+$", "^openWB/system/mqtt/valid_partner_ids$", @@ -727,6 +733,7 @@ def update(self): try: # erst breaking changes auflösen, sonst sind alte Topics schon gelöscht self.__solve_breaking_changes() + self.__solve_breaking_changes_filesystem() self.__remove_outdated_topics() self._remove_invalid_topics() self.__pub_missing_defaults() @@ -827,6 +834,67 @@ def __solve_breaking_changes(self) -> None: pub_system_message( {}, "Fehler bei der Aktualisierung der Konfiguration des Brokers.", MessageType.ERROR) + def __solve_breaking_changes_filesystem(self) -> None: + """Führt dateisystembezogene Migrationen anhand der file_operation_version aus.""" + file_operation_version = decode_payload(self.all_received_topics.get("openWB/system/file_operation_version")) + if file_operation_version is None: + # Neues Topic (z.B. bei Upgrade von älteren Versionen): alle File-Operation-Upgrades ausführen. + file_operation_version = [] + self.__update_topic("openWB/system/file_operation_version", file_operation_version) + elif isinstance(file_operation_version, int): + # Legacy-Format (int): bereits ausgeführte Upgrades als Liste abbilden. + file_operation_version = list(range(file_operation_version)) + self.__update_topic("openWB/system/file_operation_version", file_operation_version) + + log.debug(f"current file operation version: {file_operation_version}") + log.debug(f"target file operation version: {self.FILE_OPERATION_VERSION}") + for version in list(range(self.FILE_OPERATION_VERSION+1)): + try: + operation_required = version not in file_operation_version + if version == 0: + # Version 0 erneut ausführen, solange die Hintergrund-Generierung der Tages-/Monatssummen + # noch nicht erfolgreich abgeschlossen ist (Flag ist nicht True). + log_totals_generation_finished = decode_payload( + self.all_received_topics.get("openWB/system/log_totals_generation_finished")) + operation_required = operation_required or log_totals_generation_finished is not True + + if operation_required: + log.debug(f"upgrading File Operation version '{version}'") + getattr(self, f"upgrade_file_operation_{version}")() + except AttributeError: + log.error(f"missing upgrade function! '{version}'") + except Exception: + log.exception("Fehler bei der Aktualisierung des Brokers.") + pub_system_message( + {}, "Fehler bei der Aktualisierung der Konfiguration des Brokers.", MessageType.ERROR) + + def upgrade_file_operation_0(self) -> None: + """ + Generiere die Totals-Summen für Tage und Monate + """ + + self.__update_topic("openWB/system/log_totals_generation_finished", False) + try: + _generate_totals_subprocess = subprocess.Popen( + [ + sys.executable, + "-m", + "helpermodules.measurement_logging.generate_totals_subprocess" + ], + cwd="/var/www/html/openWB/packages", + start_new_session=True, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + close_fds=True + ) + + log.debug("generate_totals_subprocess gestartet, PID: %s", _generate_totals_subprocess.pid) + self._append_file_operation_version(0) + except Exception: + _generate_totals_subprocess = None + log.exception("Fehler beim Starten des generate_totals_subprocess.") + def _loop_all_received_topics(self, callback) -> None: modified_topics = {} for topic, payload in self.all_received_topics.items(): @@ -845,6 +913,12 @@ def _append_datastore_version(self, version: int) -> None: datastore_versions.append(version) self.__update_topic("openWB/system/datastore_version", datastore_versions) + def _append_file_operation_version(self, version: int) -> None: + file_operation_versions = decode_payload(self.all_received_topics.get("openWB/system/file_operation_version")) + if version not in file_operation_versions: + file_operation_versions.append(version) + self.__update_topic("openWB/system/file_operation_version", file_operation_versions) + def upgrade_datastore_0(self) -> None: def upgrade(topic: str, payload) -> Optional[dict]: modified_topics = {} diff --git a/packages/helpermodules/update_config_test.py b/packages/helpermodules/update_config_test.py index 18e036b162..a2af7d3af7 100644 --- a/packages/helpermodules/update_config_test.py +++ b/packages/helpermodules/update_config_test.py @@ -233,3 +233,20 @@ def test_upgrade_datastore_125_is_idempotent_for_already_converted_values(mock_p assert uc.all_received_topics["openWB/optional/ep/grid_fee/provider"] == expected_grid_fee assert uc.all_received_topics["openWB/system/datastore_version"] == [123, 124, 125] assert mock_pub.pub.call_count == 1 # einmal publishen für Upgrade der Datastore-Version + + +@pytest.mark.parametrize("file_operation_version, finished, expected_calls", [ + ([], False, 1), # erster Start + ([0], True, 0), # bereits fertig -> kein Neustart + ([0], False, 1), # angefangen, aber nicht fertig -> Neustart +]) +def test_file_operation_0_start_behavior(file_operation_version, finished, expected_calls): + update_config = UpdateConfig() + update_config.all_received_topics = { + "openWB/system/file_operation_version": file_operation_version, + "openWB/system/log_totals_generation_finished": finished + } + + with patch.object(update_config, "upgrade_file_operation_0") as upgrade_mock: + update_config._UpdateConfig__solve_breaking_changes_filesystem() + assert upgrade_mock.call_count == expected_calls diff --git a/packages/main.py b/packages/main.py index 3385139ba9..e2a2d6d066 100755 --- a/packages/main.py +++ b/packages/main.py @@ -27,7 +27,7 @@ from helpermodules.changed_values_handler import ChangedValuesContext from helpermodules.mosquitto_dynsec.mosquitto_dynsec import check_roles_at_start from helpermodules.measurement_logging.update_yields import update_daily_yields, update_pv_monthly_yearly_yields -from helpermodules.measurement_logging.write_log import LogType, save_log +from helpermodules.measurement_logging.write_log import save_log from helpermodules.modbusserver import start_modbus_server from helpermodules.pub import Pub from modules import configuration, loadvars, update_soc @@ -37,6 +37,7 @@ from modules.utils import wait_for_module_update_completed from smarthome.smarthome import readmq, smarthome_handler +from helpermodules.measurement_logging.process_log import save_daily_source_totals, save_monthly_source_totals class HandlerAlgorithm: def __init__(self): @@ -180,9 +181,11 @@ def handler5MinAlgorithm(self): """ try: with ChangedValuesContext(loadvars_.event_module_update_completed): - totals = save_log(LogType.DAILY) - update_daily_yields(totals) - update_pv_monthly_yearly_yields() + entries = save_log() + daily_totals = update_daily_yields(entries) + if daily_totals is not None: + update_pv_monthly_yearly_yields(daily_totals) + for cp in data.data.cp_data.values(): calc_energy_costs(cp) data.data.general_data.grid_protection() @@ -230,7 +233,15 @@ def handler5Min(self): @__with_handler_lock(error_threshold=60) def handler_midnight(self): try: - save_log(LogType.MONTHLY) + today = timecheck.create_timestamp_YYYYMMDD() + previous_day = timecheck.get_relative_date_string(today, day_offset=-1) + save_daily_source_totals(previous_day) + + prev_month = timecheck.get_relative_date_string(today, month_offset=-1)[:6] + # Neuer Monat hat angefangen, daher Monats Totals speichern + if today[6:8] == "01": + save_monthly_source_totals(prev_month, None, saving=True) + thread_errors_path = Path(Path(__file__).resolve().parents[1]/"ramdisk"/"thread_errors.log") with thread_errors_path.open("w") as f: f.write("")