From 495b70e2ec5bf8709d34ccb52dd8169a8594e088 Mon Sep 17 00:00:00 2001 From: Niko Leskinen Date: Thu, 18 Dec 2025 23:19:01 +0200 Subject: [PATCH 1/3] Refactor monitoring cli --- scripts/monitor | 6 + src/monitoring/README.md | 30 +- src/monitoring/__main__.py | 96 ------ src/monitoring/cli.py | 279 ++++++++++++++++ src/monitoring/config.py | 5 + .../halo_doppler_lidar.py | 267 ++++++++------- src/monitoring/monitor.py | 42 +-- src/monitoring/monitoring_file.py | 55 ++-- src/monitoring/period.py | 311 +++++++++++++----- src/monitoring/plot_utils.py | 13 +- src/monitoring/product.py | 8 +- src/monitoring/py.typed | 0 src/monitoring/utils.py | 59 +++- tests/unit/test_utils_module.py | 42 +-- 14 files changed, 805 insertions(+), 408 deletions(-) create mode 100755 scripts/monitor delete mode 100644 src/monitoring/__main__.py create mode 100644 src/monitoring/cli.py create mode 100644 src/monitoring/config.py rename src/monitoring/{instruments => instrument}/halo_doppler_lidar.py (57%) delete mode 100644 src/monitoring/py.typed diff --git a/scripts/monitor b/scripts/monitor new file mode 100755 index 00000000..b9f45e74 --- /dev/null +++ b/scripts/monitor @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 + +from monitoring.cli import main + +if __name__ == "__main__": + main() diff --git a/src/monitoring/README.md b/src/monitoring/README.md index 2f72f975..1808af06 100644 --- a/src/monitoring/README.md +++ b/src/monitoring/README.md @@ -2,14 +2,28 @@ ## Usage -Check available products from `/api/monitoring-products` +```bash +# Monitor current period by default +./scripts/monitor {day,week,month,year,all} -Run monitoring: +# Common options +# start and stop arguments have format YYYY-MM-DD, YYYY-VV, YYYY-MM or YYYY +# depending on the subcommand +--start START Monitor periods starting from START +--stop STOP Monitor periods until STOP +--product [PRODUCT ...] Monitor only these products +--site [SITE ...] Monitor only these sites -``` -python -m monitoring INSTRUMENT_UUID SITE all PRODUCT_ID -python -m monitoring INSTRUMENT_UUID SITE year:DATE PRODUCT_ID -python -m monitoring INSTRUMENT_UUID SITE month:DATE PRODUCT_ID -python -m monitoring INSTRUMENT_UUID SITE week:DATE PRODUCT_ID -python -m monitoring INSTRUMENT_UUID SITE day:DATE PRODUCT_ID +# Subcommand specific options +## day +--day [YYYY-MM-DD ...] + +## week +--week [YYYY-VV ...] + +## month +--month [YYYY-MM ...] + +## year +--year [YYYY ...] ``` diff --git a/src/monitoring/__main__.py b/src/monitoring/__main__.py deleted file mode 100644 index 4ca67171..00000000 --- a/src/monitoring/__main__.py +++ /dev/null @@ -1,96 +0,0 @@ -import argparse -from typing import Callable - -from cloudnet_api_client import APIClient -from cloudnet_api_client.containers import Instrument, Site - -from monitoring.monitor import monitor -from monitoring.period import Period, period_from_str -from monitoring.product import MonitoringProduct -from processing.config import Config - - -def main() -> None: - config = Config() - client = APIClient(f"{config.dataportal_url}/api/") - parser = argparse.ArgumentParser() - parser.add_argument("instrument", type=_instrument_from_client(client)) - parser.add_argument("site", type=_site_from_client(client)) - parser.add_argument("period", type=_period_from_str) - parser.add_argument("product", type=_product_from_client(client)) - args = parser.parse_args() - monitor( - site=args.site, - instrument=args.instrument, - period=args.period, - product=args.product, - ) - - -def _period_from_str(s: str) -> Period: - try: - return period_from_str(s) - except ValueError as err: - raise argparse.ArgumentTypeError(err) from err - - -def _site_from_client(client: APIClient) -> Callable[[str], Site]: - sites = client.sites() - site_dict = {site.id: site for site in sites} - - def validate_id(id_: str) -> Site: - if id_ not in site_dict: - print("Invalid site ID. Available sites:") - _print_sites(sites) - raise argparse.ArgumentTypeError("Invalid site ID.") - return site_dict[id_] - - return validate_id - - -def _product_from_client(client: APIClient) -> Callable[[str], MonitoringProduct]: - data = client.session.get(f"{client.base_url}monitoring-products/variables").json() - products = [MonitoringProduct.from_dict(d) for d in data] - product_dict = {p.id: p for p in products} - - def validate_product(id_: str) -> MonitoringProduct: - if id_ not in product_dict: - print("Invalid product ID. Available products:") - _print_products(products) - raise argparse.ArgumentTypeError(f"Invalid product ID: '{id_}'") - return product_dict[id_] - - return validate_product - - -def _instrument_from_client(client: APIClient) -> Callable[[str], Instrument]: - instruments = client.instruments() - instrument_dict = {str(inst.uuid): inst for inst in instruments} - - def validate_uuid(uuid: str) -> Instrument: - if uuid not in instrument_dict: - print("Invalid instrument UUID. Available instruments:") - _print_instruments(instruments) - raise argparse.ArgumentTypeError(f"Invalid instrument UUID: '{uuid}'") - return instrument_dict[uuid] - - return validate_uuid - - -def _print_products(products: list[MonitoringProduct]) -> None: - for p in products: - print(p.id) - - -def _print_sites(sites: list[Site]) -> None: - for s in sorted(sites, key=lambda x: x.id): - print(f"{s.id:<20} {s.human_readable_name}, {s.country}") - - -def _print_instruments(instruments: list[Instrument]) -> None: - for inst in sorted(instruments, key=lambda x: (x.type, x.name)): - print(f"{inst.type:<40} {inst.name:<40} {inst.uuid} {inst.pid}") - - -if __name__ == "__main__": - main() diff --git a/src/monitoring/cli.py b/src/monitoring/cli.py new file mode 100644 index 00000000..fcffaa7c --- /dev/null +++ b/src/monitoring/cli.py @@ -0,0 +1,279 @@ +import argparse +import itertools +import logging +from argparse import ArgumentParser, Namespace +from typing import Callable, Iterable, TypeVar + +import monitoring.period as period_module +from monitoring.config import CONFIG +from monitoring.monitor import monitor +from monitoring.period import ( + All, + Day, + Month, + PeriodProtocol, + PeriodType, + Week, + Year, + period_cls_from_str, + period_str_from_cls, +) +from monitoring.product import MonitoringProduct +from monitoring.utils import RawFilesPayload, get_api_client, get_md_api + +T = TypeVar("T", bound=PeriodProtocol) +PeriodList = list[All] | list[Day] | list[Month] | list[Week] | list[Year] + + +def main() -> None: + logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") + args = _get_args() + period_cls = period_cls_from_str(args.cmd) + periods = build_periods(period_cls, args) + periods_with_products = build_products(period_cls, periods, args.product) + periods_with_products_sites_and_instruments = build_instruments( + periods_with_products, args.site + ) + validated_periods_with_products_sites_and_instruments = validate_products( + periods_with_products_sites_and_instruments + ) + + for ( + period, + product, + site, + instrument_uuid, + ) in validated_periods_with_products_sites_and_instruments: + try: + monitor(period, product, site, instrument_uuid) + except ValueError as err: + logging.warning(err) + + +def build_periods(period_cls: type[PeriodType], args: Namespace) -> PeriodList: + match period_cls: + case period_module.Day: + return _resolve_periods(args.start, args.stop, args.day, Day.now, Day.range) + case period_module.Week: + return _resolve_periods( + args.start, args.stop, args.week, Week.now, Week.range + ) + case period_module.Month: + return _resolve_periods( + args.start, args.stop, args.month, Month.now, Month.range + ) + case period_module.Year: + return _resolve_periods( + args.start, args.stop, args.year, Year.now, Year.range + ) + case period_module.All: + return [All()] + case _: + raise ValueError + + +def build_products( + period_cls: type[PeriodType], + period_list: PeriodList, + select_products: list[str] | None, +) -> list[tuple[PeriodType, str]]: + period_str = period_str_from_cls(period_cls) + + products: list[str] = [] + for product, allowed_periods in CONFIG.items(): + if period_str in allowed_periods: + products.append(product) + if select_products: + products = [p for p in products if p in select_products] + return list(itertools.product(period_list, products)) + + +def build_instruments( + product_list: list[tuple[PeriodType, str]], sites: list[str] | None +) -> list[tuple[PeriodType, str, str, str]]: + list_with_sites_and_instruments = [] + for period, product in product_list: + for site, instrument_uuid in get_available_instruments(period, product, sites): + list_with_sites_and_instruments.append( + (period, product, site, instrument_uuid) + ) + return list_with_sites_and_instruments + + +def validate_products( + product_list: list[tuple[PeriodType, str, str, str]], +) -> list[tuple[PeriodType, MonitoringProduct, str, str]]: + available_products = get_available_products() + validated = [] + for period, product_str, site, instrument_uuid in product_list: + if not product_str in available_products: + raise ValueError(f"Invalid product '{product_str}'") + validated.append( + (period, available_products[product_str], site, instrument_uuid) + ) + return validated + + +def get_available_products() -> dict[str, MonitoringProduct]: + api = get_md_api() + data = api.get("api/monitoring-products/variables") + return {entry["id"]: MonitoringProduct.from_dict(entry) for entry in data} + + +def get_available_instruments( + period: PeriodType, product: str, sites: list[str] | None +) -> list[tuple[str, str]]: + client = get_api_client() + payload: RawFilesPayload + + match product: + case "halo-doppler-lidar_housekeeping": + payload = { + "instrument_id": "halo-doppler-lidar", + "filename_prefix": "system_parameters_", + "filename_suffix": ".txt", + } + case "halo-doppler-lidar_background": + payload = { + "instrument_id": "halo-doppler-lidar", + "filename_prefix": "Background_", + "filename_suffix": ".txt", + } + case "halo-doppler-lidar_signal": + payload = { + "instrument_id": "halo-doppler-lidar", + "filename_suffix": ".hpl", + } + case _: + raise ValueError + match period: + case Day() | Week() | Month() | Year(): + if product == "halo-doppler-lidar_housekeeping": + start, stop = period.to_interval_padded(days=31) + else: + start, stop = period.to_interval() + payload.update({"date_from": str(start), "date_to": str(stop)}) + case All(): + pass + if sites and len(sites) == 1: + payload.update({"site_id": sites[0]}) + + records = client.raw_files(**payload) + sites_and_uuids = sorted( + list(set((r.site.id, str(r.instrument.uuid)) for r in records)) + ) + if sites: + sites_and_uuids = [ + (site, uuid) for site, uuid in sites_and_uuids if site in sites + ] + return sites_and_uuids + + +def _resolve_periods( + start: T | None, + stop: T | None, + explicit: list[T] | None, + default_factory: Callable[[], T], + range_func: Callable[[T, T], Iterable[T]], +) -> list[T]: + if not start and not stop and not explicit: + return [default_factory()] + if not start and not stop and explicit is not None: + return sorted(list(set(explicit))) # type: ignore[type-var] + if start is None: + raise ValueError + if stop is None: + stop = default_factory() + range_ = list(range_func(start, stop)) + combined = set(range_) + if explicit: + combined.update(explicit) + return sorted(list(combined)) # type: ignore[type-var] + + +def _get_args() -> Namespace: + parser = ArgumentParser() + subp = parser.add_subparsers(dest="cmd", required=True) + _build_day_args(subp.add_parser("day")) + _build_week_args(subp.add_parser("week")) + _build_month_args(subp.add_parser("month")) + _build_year_args(subp.add_parser("year")) + _build_all_args(subp.add_parser("all")) + return parser.parse_args() + + +def _build_day_args(parser: ArgumentParser) -> None: + parser.add_argument("--start", type=_parse_day) + parser.add_argument("--stop", type=_parse_day) + parser.add_argument("--day", type=_parse_day, nargs="*") + _common(parser) + + +def _build_week_args(parser: ArgumentParser) -> None: + parser.add_argument("--start", type=_parse_week) + parser.add_argument("--stop", type=_parse_week) + parser.add_argument("--week", type=_parse_week, nargs="*") + _common(parser) + + +def _build_month_args(parser: ArgumentParser) -> None: + parser.add_argument("--start", type=_parse_month) + parser.add_argument("--stop", type=_parse_month) + parser.add_argument("--month", type=_parse_month, nargs="*") + _common(parser) + + +def _build_year_args(parser: ArgumentParser) -> None: + parser.add_argument("--start", type=_parse_year) + parser.add_argument("--stop", type=_parse_year) + parser.add_argument("--year", type=_parse_year, nargs="*") + _common(parser) + + +def _build_all_args(parser: ArgumentParser) -> None: + _common(parser) + + +def _common(parser: ArgumentParser) -> None: + parser.add_argument("--product", nargs="*") + parser.add_argument("--site", nargs="*") + + +def _parse_year(year_str: str) -> Year: + try: + return Year.from_str(year_str) + except ValueError: + raise argparse.ArgumentTypeError( + f"Invalid year format: '{year_str}'. Expected format: YYYY" + ) + + +def _parse_month(month_str: str) -> Month: + try: + return Month.from_str(month_str) + except ValueError: + raise argparse.ArgumentTypeError( + f"Invalid month format: '{month_str}'. Expected format: YYYY-MM" + ) + + +def _parse_week(week_str: str) -> Week: + try: + return Week.from_str(week_str) + except ValueError: + raise argparse.ArgumentTypeError( + f"Invalid week format: '{week_str}'. Expected format: YYYY-VV" + ) + + +def _parse_day(date_str: str) -> Day: + try: + return Day.from_str(date_str) + except ValueError: + raise argparse.ArgumentTypeError( + f"Invalid date: '{date_str}'. Expected format: YYYY-MM-DD" + ) + + +if __name__ == "__main__": + main() diff --git a/src/monitoring/config.py b/src/monitoring/config.py new file mode 100644 index 00000000..39c048d4 --- /dev/null +++ b/src/monitoring/config.py @@ -0,0 +1,5 @@ +CONFIG: dict[str, list[str]] = { + "halo-doppler-lidar_housekeeping": ["all", "year", "month", "week", "day"], + "halo-doppler-lidar_background": ["all", "year", "month", "week", "day"], + "halo-doppler-lidar_signal": ["week", "day"], +} diff --git a/src/monitoring/instruments/halo_doppler_lidar.py b/src/monitoring/instrument/halo_doppler_lidar.py similarity index 57% rename from src/monitoring/instruments/halo_doppler_lidar.py rename to src/monitoring/instrument/halo_doppler_lidar.py index 31da5f46..2ac23f05 100644 --- a/src/monitoring/instruments/halo_doppler_lidar.py +++ b/src/monitoring/instrument/halo_doppler_lidar.py @@ -1,20 +1,18 @@ import datetime from collections import Counter +from pathlib import Path from tempfile import TemporaryDirectory import matplotlib.pyplot as plt import numpy as np -from cloudnet_api_client import APIClient -from cloudnet_api_client.containers import Instrument, Site -from doppy.raw import HaloBg, HaloHpl, HaloSysParams -from numpy.typing import NDArray - -from monitoring.monitoring_file import ( - Dimensions, - MonitoringFile, - MonitoringVisualization, -) -from monitoring.period import Period, PeriodWithRange +from cloudnet_api_client.client import APIClient +from cloudnetpy.plotting.plotting import Dimensions +from doppy.raw import HaloSysParams +from doppy.raw.halo_bg import HaloBg +from doppy.raw.halo_hpl import HaloHpl + +from monitoring.monitoring_file import MonitoringFile, MonitoringVisualization +from monitoring.period import All, PeriodType from monitoring.plot_utils import ( SCATTER_OPTS, add_colorbar, @@ -26,71 +24,85 @@ set_xlim_for_period, ) from monitoring.product import MonitoringProduct, MonitoringVariable -from monitoring.utils import range_from_period +from monitoring.utils import ( + RawFilesDatePayload, + get_storage_api, + instrument_uuid_to_pid, +) +from processing.storage_api import StorageApi def monitor( - client: APIClient, - site: Site, - instrument: Instrument, - period: Period, + period: PeriodType, product: MonitoringProduct, + site: str, + instrument_uuid: str, + api_client: APIClient, + storage_api: StorageApi, ) -> None: match product.id: case "halo-doppler-lidar_housekeeping": - monitor_housekeeping(client, site, instrument, period, product) + monitor_housekeeping( + period, product, site, instrument_uuid, api_client, storage_api + ) case "halo-doppler-lidar_background": - monitor_background(client, site, instrument, period, product) + monitor_background( + period, product, site, instrument_uuid, api_client, storage_api + ) case "halo-doppler-lidar_signal": - monitor_signal(client, site, instrument, period, product) - - case _: - raise NotImplementedError( - f"Monitoring product '{product.id}' not implemented for '{instrument.instrument_id}'" + monitor_signal( + period, product, site, instrument_uuid, api_client, storage_api ) def monitor_housekeeping( - client: APIClient, - site: Site, - instrument: Instrument, - period: Period, + period: PeriodType, product: MonitoringProduct, + site: str, + instrument_uuid: str, + api_client: APIClient, + storage_api: StorageApi, ) -> None: - start, end = range_from_period(period) - - start -= datetime.timedelta(days=32) - end += datetime.timedelta(days=32) - raw_files = client.raw_files( - site_id=site.id, - instrument_pid=instrument.pid, - date_from=start, - date_to=end, + pid = instrument_uuid_to_pid(api_client, instrument_uuid) + date_opts: RawFilesDatePayload = {} + if not isinstance(period, All): + start, stop = period.to_interval_padded(days=31) + date_opts = {"date_from": start, "date_to": stop} + + records = api_client.raw_files( + site_id=site, + instrument_pid=pid, filename_prefix="system_parameters_", filename_suffix=".txt", + **date_opts, ) - if not raw_files: + if not records: raise ValueError( - f"Not raw files found for {period} {instrument.name} {product.id}" + f"No raw files for monitoring period {period} {product.id} {site} {pid}" ) + with TemporaryDirectory() as tempdir: - paths = client.download(raw_files, tempdir, progress=False) + (paths, _uuids) = storage_api.download_raw_data(records, Path(tempdir)) sys_params_list = [HaloSysParams.from_src(p) for p in paths] sys_params = ( HaloSysParams.merge(sys_params_list) .sorted_by_time() .non_strictly_increasing_timesteps_removed() ) - if isinstance(period, PeriodWithRange): - start_time = np.datetime64(period.start_date).astype(sys_params.time.dtype) - end_time = np.datetime64(period.end_date + datetime.timedelta(days=1)).astype( - sys_params.time.dtype - ) - select = (start_time <= sys_params.time) & (sys_params.time < end_time) + if not isinstance(period, All): + start, stop = period.to_interval() + dtype = sys_params.time.dtype + start_time = np.datetime64(start).astype(dtype) + stop_time = np.datetime64(stop + datetime.timedelta(days=1)).astype(dtype) + select = (start_time <= sys_params.time) & (sys_params.time < stop_time) sys_params = sys_params[select] + if len(sys_params.time) == 0: + raise ValueError( + f"No timestamps for monitoring period {period} {product.id} {site} {pid}" + ) monitoring_file = MonitoringFile( - instrument, + instrument_uuid, site, period, product, @@ -100,7 +112,7 @@ def monitor_housekeeping( def monitor_housekeeping_plots( - sys_params: HaloSysParams, period: Period, product: MonitoringProduct + sys_params: HaloSysParams, period: PeriodType, product: MonitoringProduct ) -> list[MonitoringVisualization]: plots = [] for variable in product.variables: @@ -110,7 +122,7 @@ def monitor_housekeeping_plots( def plot_housekeeping_variable( sys_params: HaloSysParams, - period: Period, + period: PeriodType, variable: MonitoringVariable, ) -> MonitoringVisualization: fig, ax = plt.subplots() @@ -120,50 +132,59 @@ def plot_housekeeping_variable( format_time_axis(ax) pretty_ax(ax, grid="y") fig_ = save_fig(fig) - return MonitoringVisualization(fig_.bytes, variable, Dimensions(fig, [ax])) + vis = MonitoringVisualization(fig_.bytes, variable, Dimensions(fig, [ax])) + plt.close(fig) + return vis def monitor_background( - client: APIClient, - site: Site, - instrument: Instrument, - period: Period, + period: PeriodType, product: MonitoringProduct, + site: str, + instrument_uuid: str, + api_client: APIClient, + storage_api: StorageApi, ) -> None: - start, end = range_from_period(period) - start -= datetime.timedelta(days=1) - end += datetime.timedelta(days=1) - raw_files = client.raw_files( - site_id=site.id, - instrument_pid=instrument.pid, - date_from=start, - date_to=end, + storage_api = get_storage_api() + pid = instrument_uuid_to_pid(api_client, instrument_uuid) + + date_opts: RawFilesDatePayload = {} + if not isinstance(period, All): + start, stop = period.to_interval_padded(days=1) + date_opts = {"date_from": start, "date_to": stop} + + records = api_client.raw_files( + site_id=site, + instrument_pid=pid, filename_prefix="Background_", filename_suffix=".txt", + **date_opts, ) - raw_files = [r for r in raw_files if "cross" not in r.tags] - if not raw_files: + if not records: raise ValueError( - f"Not raw files found for {period} {instrument.name} {product.id}" + f"No raw files for monitoring period {period} {product.id} {site} {pid}" ) + with TemporaryDirectory() as tempdir: - paths = client.download(raw_files, tempdir, progress=False) + (paths, _uuids) = storage_api.download_raw_data(records, Path(tempdir)) bgs = HaloBg.from_srcs(paths) counter = Counter((bg.signal.shape[1] for bg in bgs)) most_common_ngates = counter.most_common()[0][0] bgs = [bg for bg in bgs if bg.signal.shape[1] == most_common_ngates] bg = HaloBg.merge(bgs).sorted_by_time().non_strictly_increasing_timesteps_removed() - - if isinstance(period, PeriodWithRange): - start_time = np.datetime64(period.start_date).astype(bg.time.dtype) - end_time = np.datetime64(period.end_date + datetime.timedelta(days=1)).astype( - bg.time.dtype - ) - select = (start_time <= bg.time) & (bg.time < end_time) + if not isinstance(period, All): + start, stop = period.to_interval() + dtype = bg.time.dtype + start_time = np.datetime64(start).astype(dtype) + stop_time = np.datetime64(stop + datetime.timedelta(days=1)).astype(dtype) + select = (start_time <= bg.time) & (bg.time < stop_time) bg = bg[select] - + if len(bg.time) == 0: + raise ValueError( + f"No timestamps for monitoring period {period} {product.id} {site} {pid}" + ) monitoring_file = MonitoringFile( - instrument, + instrument_uuid, site, period, product, @@ -173,7 +194,7 @@ def monitor_background( def monitor_background_plots( - bg: HaloBg, period: Period, product: MonitoringProduct + bg: HaloBg, period: PeriodType, product: MonitoringProduct ) -> list[MonitoringVisualization]: plots = [] for variable in product.variables: @@ -182,7 +203,7 @@ def monitor_background_plots( def plot_background_variable( - bg: HaloBg, period: Period, variable: MonitoringVariable + bg: HaloBg, period: PeriodType, variable: MonitoringVariable ) -> MonitoringVisualization: match variable.id: case "background-profile": @@ -198,7 +219,7 @@ def plot_background_variable( def plot_background_profile( - bg: HaloBg, period: Period, variable: MonitoringVariable + bg: HaloBg, period: PeriodType, variable: MonitoringVariable ) -> MonitoringVisualization: fig, ax = plt.subplots() vmin, vmax = np.percentile(bg.signal.ravel(), [5, 95]) @@ -211,11 +232,13 @@ def plot_background_profile( format_time_axis(ax) pretty_ax_2d(ax) fig_ = save_fig(fig) - return MonitoringVisualization(fig_.bytes, variable, Dimensions(fig, [ax])) + vis = MonitoringVisualization(fig_.bytes, variable, Dimensions(fig, [ax])) + plt.close(fig) + return vis def plot_background_variance( - bg: HaloBg, period: Period, variable: MonitoringVariable + bg: HaloBg, period: PeriodType, variable: MonitoringVariable ) -> MonitoringVisualization: fig, ax = plt.subplots() var = bg.signal.var(axis=1) @@ -224,11 +247,13 @@ def plot_background_variance( format_time_axis(ax) pretty_ax(ax, grid="y") fig_ = save_fig(fig) - return MonitoringVisualization(fig_.bytes, variable, Dimensions(fig, [ax])) + vis = MonitoringVisualization(fig_.bytes, variable, Dimensions(fig, [ax])) + plt.close(fig) + return vis def plot_time_averaged_background_profile( - bg: HaloBg, _: Period, variable: MonitoringVariable + bg: HaloBg, _: PeriodType, variable: MonitoringVariable ) -> MonitoringVisualization: fig, ax = plt.subplots() mean = bg.signal.mean(axis=0) @@ -244,34 +269,41 @@ def plot_time_averaged_background_profile( pretty_ax(ax, grid="y") fig_ = save_fig(fig) - return MonitoringVisualization(fig_.bytes, variable, Dimensions(fig, [ax])) + vis = MonitoringVisualization(fig_.bytes, variable, Dimensions(fig, [ax])) + plt.close(fig) + return vis def monitor_signal( - client: APIClient, - site: Site, - instrument: Instrument, - period: Period, + period: PeriodType, product: MonitoringProduct, + site: str, + instrument_uuid: str, + api_client: APIClient, + storage_api: StorageApi, ) -> None: - start, end = range_from_period(period) - - start -= datetime.timedelta(days=1) - end += datetime.timedelta(days=1) - raw_files = client.raw_files( - site_id=site.id, - instrument_pid=instrument.pid, - date_from=start, - date_to=end, + pid = instrument_uuid_to_pid(api_client, instrument_uuid) + + date_opts: RawFilesDatePayload = {} + if not isinstance(period, All): + start, stop = period.to_interval() + date_opts = {"date_from": start, "date_to": stop} + + records = api_client.raw_files( + site_id=site, + instrument_pid=pid, filename_suffix=".hpl", + **date_opts, ) - raw_files = [r for r in raw_files if "cross" not in r.tags] - if not raw_files: + + records = [r for r in records if "cross" not in r.tags] + if not records: raise ValueError( - f"No raw files found for {period} {instrument.name} {product.id}" + f"No raw files for monitoring period {period} {product.id} {site} {pid}" ) + with TemporaryDirectory() as tempdir: - paths = client.download(raw_files, tempdir, progress=False) + (paths, _uuids) = storage_api.download_raw_data(records, Path(tempdir)) raws = HaloHpl.from_srcs(paths) def is_stare(raw: HaloHpl) -> bool: @@ -286,7 +318,7 @@ def is_stare(raw: HaloHpl) -> bool: raws = [r for r in raws if is_stare(r)] if not raws: raise ValueError( - f"No raw stare files found for {period} {instrument.name} {product.id}" + f"No raw stare files found for {period} {instrument_uuid} {product.id}" ) counter = Counter((raw.intensity.shape[1] for raw in raws)) most_common_ngates = counter.most_common()[0][0] @@ -294,16 +326,21 @@ def is_stare(raw: HaloHpl) -> bool: raw = ( HaloHpl.merge(raws).sorted_by_time().non_strictly_increasing_timesteps_removed() ) - if isinstance(period, PeriodWithRange): - start_time = np.datetime64(period.start_date).astype(raw.time.dtype) - end_time = np.datetime64(period.end_date + datetime.timedelta(days=1)).astype( - raw.time.dtype - ) - select = (start_time <= raw.time) & (raw.time < end_time) + if not isinstance(period, All): + start, stop = period.to_interval() + dtype = raw.time.dtype + start_time = np.datetime64(start).astype(dtype) + stop_time = np.datetime64(stop + datetime.timedelta(days=1)).astype(dtype) + select = (start_time <= raw.time) & (raw.time < stop_time) raw = raw[select] + if len(raw.time) == 0: + raise ValueError( + f"No timestamps for monitoring period {period} {product.id} {site} {pid}" + ) + monitoring_file = MonitoringFile( - instrument, + instrument_uuid, site, period, product, @@ -313,7 +350,7 @@ def is_stare(raw: HaloHpl) -> bool: def monitor_signal_plots( - raw: HaloHpl, period: Period, product: MonitoringProduct + raw: HaloHpl, period: PeriodType, product: MonitoringProduct ) -> list[MonitoringVisualization]: plots = [] for variable in product.variables: @@ -322,7 +359,7 @@ def monitor_signal_plots( def plot_signal_variable( - raw: HaloHpl, period: Period, variable: MonitoringVariable + raw: HaloHpl, period: PeriodType, variable: MonitoringVariable ) -> MonitoringVisualization: match variable.id: case "radial-velocity-histogram": @@ -336,7 +373,7 @@ def plot_signal_variable( def plot_radial_velocity_histogram( - raw: HaloHpl, _: Period, variable: MonitoringVariable + raw: HaloHpl, _: PeriodType, variable: MonitoringVariable ) -> MonitoringVisualization: fig, ax = plt.subplots() bins = _compute_radial_velocity_bins(raw) @@ -346,7 +383,9 @@ def plot_radial_velocity_histogram( ax.set_ylabel("Count") pretty_ax(ax, grid="both") fig_ = save_fig(fig) - return MonitoringVisualization(fig_.bytes, variable, Dimensions(fig, [ax])) + vis = MonitoringVisualization(fig_.bytes, variable, Dimensions(fig, [ax])) + plt.close(fig) + return vis def _compute_radial_velocity_bins(raw: HaloHpl) -> list[float] | int: @@ -363,7 +402,7 @@ def _compute_radial_velocity_bins(raw: HaloHpl) -> list[float] | int: def plot_signal_radial_velocity( - raw: HaloHpl, _: Period, variable: MonitoringVariable + raw: HaloHpl, _: PeriodType, variable: MonitoringVariable ) -> MonitoringVisualization: fig, ax = plt.subplots() vmin, vmax = np.percentile(raw.intensity.ravel(), [2, 95]) @@ -375,4 +414,6 @@ def plot_signal_radial_velocity( ax.set_ylabel("Radial velocity") pretty_ax(ax) fig_ = save_fig(fig) - return MonitoringVisualization(fig_.bytes, variable, Dimensions(fig, [ax])) + vis = MonitoringVisualization(fig_.bytes, variable, Dimensions(fig, [ax])) + plt.close(fig) + return vis diff --git a/src/monitoring/monitor.py b/src/monitoring/monitor.py index 3f930706..86b05c22 100644 --- a/src/monitoring/monitor.py +++ b/src/monitoring/monitor.py @@ -1,36 +1,20 @@ import logging -from cloudnet_api_client import APIClient -from cloudnet_api_client.containers import Instrument, Site - -from monitoring.instruments import halo_doppler_lidar -from monitoring.period import Period +from monitoring.instrument.halo_doppler_lidar import ( + monitor as monitor_halo, +) +from monitoring.period import PeriodType from monitoring.product import MonitoringProduct -from processing.config import Config - - -class C: - RESET = "\033[0m" - RED = "\033[31m" - GREEN = "\033[32m" - BLUE = "\033[34m" - CYAN = "\033[36m" - MAGENTA = "\033[35m" - YELLOW = "\033[33m" - BOLD = "\033[1m" +from monitoring.utils import get_api_client, get_storage_api def monitor( - site: Site, instrument: Instrument, period: Period, product: MonitoringProduct + period: PeriodType, product: MonitoringProduct, site: str, instrument_uuid: str ) -> None: - config = Config() - client = APIClient(f"{config.dataportal_url}/api/") - msg = ( - f"Monitoring: {C.RED}{site.human_readable_name} " - f"{C.BLUE}{instrument.name}({instrument.uuid}) " - f"{C.YELLOW}{period} {C.MAGENTA}{product}{C.RESET}" - ) - logging.info(msg) - match instrument.instrument_id: - case "halo-doppler-lidar": - halo_doppler_lidar.monitor(client, site, instrument, period, product) + api_client = get_api_client() + storage_api = get_storage_api() + if product.id.startswith("halo-doppler-lidar"): + monitor_halo(period, product, site, instrument_uuid, api_client, storage_api) + logging.info(f"Monitored {period!r} {product} {site}") + else: + raise ValueError(f"Unexpected product: '{product}'") diff --git a/src/monitoring/monitoring_file.py b/src/monitoring/monitoring_file.py index eb56dfc5..852fb7a9 100644 --- a/src/monitoring/monitoring_file.py +++ b/src/monitoring/monitoring_file.py @@ -2,14 +2,12 @@ from json import JSONDecodeError from pathlib import Path from tempfile import NamedTemporaryFile -from typing import Any -from cloudnet_api_client.containers import Instrument, Site from cloudnetpy.plotting.plotting import Dimensions -from monitoring.period import AllPeriod, Period, PeriodWithRange +from monitoring.period import All, PeriodProtocol, PeriodType from monitoring.product import MonitoringProduct, MonitoringVariable -from monitoring.utils import get_apis +from monitoring.utils import get_md_api, get_storage_api @dataclass @@ -32,26 +30,27 @@ def _dimensions_as_payload(dim: Dimensions) -> dict[str, int]: @dataclass class MonitoringFile: - instrument: Instrument - site: Site - period: Period + instrument_uuid: str + site: str + period: PeriodType product: MonitoringProduct visualisations: list[MonitoringVisualization] def upload(self) -> None: if not self.visualisations: raise ValueError( - f"No visualisations for product {self.product.id}, {self.site.id}, {self.instrument.name}, {self.period}" + f"No visualisations for product {self.product.id}, {self.site}, {self.instrument_uuid}, {self.period}" ) - md_api, storage_api = get_apis() + md_api = get_md_api() + storage_api = get_storage_api() payload = { - "periodType": self.period.period, - "siteId": self.site.id, + "periodType": self.period.to_str(), + "siteId": self.site, "productId": self.product.id, - "instrumentUuid": str(self.instrument.uuid), + "instrumentUuid": self.instrument_uuid, } - if isinstance(self.period, PeriodWithRange): - payload["startDate"] = str(self.period.start_date) + if isinstance(self.period, PeriodProtocol): + payload["startDate"] = str(self.period.start()) res = md_api.post("monitoring-files", payload=payload) if not res.ok: raise RuntimeError(f"Could not post file: {payload}") @@ -65,7 +64,7 @@ def upload(self) -> None: file_uuid = data["uuid"] for vis in self.visualisations: s3key = generate_s3_key( - self.site, self.instrument, self.product, vis.variable, self.period + self.site, self.instrument_uuid, self.product, vis.variable, self.period ) with NamedTemporaryFile() as tempfile: tempfile.write(vis.fig) @@ -82,29 +81,29 @@ def upload(self) -> None: def generate_s3_key( - site: Site, - instrument: Instrument, + site: str, + instrument_uuid: str, product: MonitoringProduct, variable: MonitoringVariable, - period: Period, + period: PeriodType, ) -> str: - instrument_uuid_short = str(instrument.uuid)[:8] + instrument_uuid_short = instrument_uuid[:8] period_str = _period_for_s3key(period) - return f"monitoring/{period_str}_{site.id}_{product.id}_{variable.id}_{instrument_uuid_short}.png" + return f"monitoring/{period_str}_{site}_{product.id}_{variable.id}_{instrument_uuid_short}.png" -def _period_for_s3key(p: Period) -> str: - if isinstance(p, AllPeriod): +def _period_for_s3key(p: PeriodType) -> str: + if isinstance(p, All): return "all" - period_str = p.period - match p.period: + period_str = p.to_str() + match period_str: case "year": - date_str = p.start_date.strftime("%Y") + date_str = p.start().strftime("%Y") case "month": - date_str = p.start_date.strftime("%Y-%m") + date_str = p.start().strftime("%Y-%m") case "week": - date_str = f"{p.start_date.isocalendar().week:02d}" + date_str = f"{p.start().isocalendar().week:02d}" case "day": - date_str = p.start_date.isoformat() + date_str = p.start().isoformat() return f"{period_str}{date_str}" diff --git a/src/monitoring/period.py b/src/monitoring/period.py index 68f768f1..70f2c64d 100644 --- a/src/monitoring/period.py +++ b/src/monitoring/period.py @@ -1,100 +1,241 @@ +from __future__ import annotations + import calendar import datetime from dataclasses import dataclass -from typing import Literal, cast +from typing import Generator, Iterable, Literal, Protocol, Self, runtime_checkable + +PeriodName = Literal["day", "week", "month", "year", "all"] + + +@runtime_checkable +class PeriodProtocol(Protocol): + @classmethod + def now(cls) -> Self: ... + + @classmethod + def range(cls, start: Self, stop: Self) -> Iterable[Self]: ... + + @classmethod + def to_str(cls) -> PeriodName: ... + + def to_interval(self) -> tuple[datetime.date, datetime.date]: ... + def to_interval_padded(self, days: int) -> tuple[datetime.date, datetime.date]: ... + + def start(self) -> datetime.date: ... + + +@dataclass(order=True, frozen=True) +class Day: + date: datetime.date + @classmethod + def from_str(cls, date_str: str) -> Day: + return cls(datetime.date.fromisoformat(date_str)) -class AllPeriod: - period = "all" + @classmethod + def to_str(cls) -> Literal["day"]: + return "day" + + @classmethod + def range(cls, start: Day, stop: Day) -> Generator[Day, None, None]: + current = start.date + while current <= stop.date: + yield Day(current) + current += datetime.timedelta(days=1) + + @classmethod + def now(cls) -> Day: + return cls(datetime.date.today()) + + def start(self) -> datetime.date: + return self.date + + def to_interval(self) -> tuple[datetime.date, datetime.date]: + return self.date, self.date + + def to_interval_padded(self, days: int) -> tuple[datetime.date, datetime.date]: + return pad_interval(self.to_interval(), days) def __repr__(self) -> str: - return "AllPeriod" + return f"Day({self.date})" + + +@dataclass(order=True, frozen=True) +class Week: + year: int + week: int + + @classmethod + def from_str(cls, week_str: str) -> Week: + dt = datetime.datetime.strptime(week_str + "-1", "%G-%V-%u") + cal = dt.isocalendar() + return cls(cal.year, cal.week) + + @classmethod + def to_str(cls) -> Literal["week"]: + return "week" + + @classmethod + def range(cls, start: Week, stop: Week) -> Generator[Week, None, None]: + current_date = datetime.date.fromisocalendar(start.year, start.week, 1) + stop_date = datetime.date.fromisocalendar(stop.year, stop.week, 1) + while current_date <= stop_date: + iso_year, iso_week, _ = current_date.isocalendar() + yield cls(iso_year, iso_week) + current_date += datetime.timedelta(weeks=1) + + @classmethod + def now(cls) -> Week: + today = datetime.date.today() + cal = today.isocalendar() + return cls(cal.year, cal.week) + + def start(self) -> datetime.date: + return datetime.date.fromisocalendar(self.year, self.week, 1) + + def to_interval(self) -> tuple[datetime.date, datetime.date]: + start = self.start() + stop = datetime.date.fromisocalendar(self.year, self.week, 7) + return start, stop + + def to_interval_padded(self, days: int) -> tuple[datetime.date, datetime.date]: + return pad_interval(self.to_interval(), days) - def __str__(self) -> str: - return "all" + def __repr__(self) -> str: + return f"Week({self.year}-{self.week:02})" + + +@dataclass(order=True, frozen=True) +class Month: + year: int + month: int + @classmethod + def from_str(cls, month_str: str) -> Month: + dt = datetime.date.fromisoformat(month_str + "-01") + return cls(dt.year, dt.month) -ALL_PERIOD = AllPeriod() + @classmethod + def to_str(cls) -> Literal["month"]: + return "month" -PeriodWithRangeType = Literal["year", "month", "week", "day"] + @classmethod + def range(cls, start: Month, stop: Month) -> Generator[Month, None, None]: + start_months = start.year * 12 + start.month - 1 + stop_months = stop.year * 12 + stop.month - 1 + for month in range(start_months, stop_months + 1): + y, m = divmod(month, 12) + yield cls(y, m + 1) + @classmethod + def now(cls) -> Month: + today = datetime.date.today() + return cls(today.year, today.month) -@dataclass -class PeriodWithRange: - period: PeriodWithRangeType - start_date: datetime.date + def start(self) -> datetime.date: + return datetime.date(self.year, self.month, 1) + + def to_interval(self) -> tuple[datetime.date, datetime.date]: + start = self.start() + _, ndays = calendar.monthrange(self.year, self.month) + stop = datetime.date(self.year, self.month, ndays) + return start, stop + + def to_interval_padded(self, days: int) -> tuple[datetime.date, datetime.date]: + return pad_interval(self.to_interval(), days) def __repr__(self) -> str: - return f"{self.period.capitalize()}({self.start_date})" - - @property - def end_date(self) -> datetime.date: - start = self.start_date - match self.period: - case "year": - return datetime.date(start.year, 12, 31) - case "month": - last_day = calendar.monthrange(start.year, start.month)[1] - return datetime.date(start.year, start.month, last_day) - case "week": - return start + datetime.timedelta(days=6) - case "day": - return start - - def as_interval(self) -> tuple[datetime.date, datetime.date]: - return (self.start_date, self.end_date) - - -Period = AllPeriod | PeriodWithRange - - -def period_from_str(s: str, normalise: bool = True) -> Period: - s = s.lower().strip() - - if s == "all": - return ALL_PERIOD - try: - period_type, start_str = s.split(":") - except ValueError as err: - raise ValueError( - f"Invalid period format '{s}'. Expected format: " - "'year|month|week|day:YYYY-MM-DD' or 'all'." - ) from err - if period_type not in ("year", "month", "week", "day"): - raise ValueError( - f"Invalid period type '{period_type}'. " - "Expected period types: all|month|week|day" - ) - period_literal = cast(PeriodWithRangeType, period_type) - try: - start_date = datetime.date.fromisoformat(start_str) - except ValueError as err: - raise ValueError( - f"Invalid start date format: '{start_str}'. Expected format: 'YYYY-MM-DD'" - ) from err - period = PeriodWithRange(period=period_literal, start_date=start_date) - if normalise: - return to_normalised_period(period) - return period - - -def to_normalised_period(period: PeriodWithRange) -> PeriodWithRange: - func = { - "year": normalise_year, - "month": normalise_month, - "week": normalise_week, - "day": lambda d: d, - } - return PeriodWithRange(period.period, func[period.period](period.start_date)) - - -def normalise_year(d: datetime.date) -> datetime.date: - return datetime.date(d.year, 1, 1) - - -def normalise_month(d: datetime.date) -> datetime.date: - return datetime.date(d.year, d.month, 1) - - -def normalise_week(d: datetime.date) -> datetime.date: - return d - datetime.timedelta(days=d.weekday()) + return f"Month({self.year}-{self.month:02})" + + +@dataclass(order=True, frozen=True) +class Year: + year: int + + @classmethod + def from_str(cls, year_str: str) -> Year: + dt = datetime.date.fromisoformat(year_str + "-01-01") + return cls(dt.year) + + @classmethod + def to_str(cls) -> Literal["year"]: + return "year" + + @classmethod + def range(cls, start: Year, stop: Year) -> Generator[Year, None, None]: + for year in range(start.year, stop.year): + yield cls(year) + + @classmethod + def now(cls) -> Year: + return cls(datetime.date.today().year) + + def start(self) -> datetime.date: + return datetime.date(self.year, 1, 1) + + def to_interval(self) -> tuple[datetime.date, datetime.date]: + start = self.start() + stop = datetime.date(self.year, 12, 31) + return start, stop + + def to_interval_padded(self, days: int) -> tuple[datetime.date, datetime.date]: + return pad_interval(self.to_interval(), days) + + def __repr__(self) -> str: + return f"Year({self.year})" + + +class All: + @classmethod + def to_str(cls) -> Literal["all"]: + return "all" + + def __repr__(self) -> str: + return f"All" + + +PeriodType = PeriodProtocol | All + + +def period_cls_from_str(x: str) -> type[PeriodType]: + match x: + case "day": + return Day + case "week": + return Week + case "month": + return Month + case "year": + return Year + case "all": + return All + case _: + raise ValueError + + +def period_str_from_cls( + cls: type[PeriodType], +) -> PeriodName: + if cls is Day: + return "day" + elif cls is Week: + return "week" + elif cls is Month: + return "month" + elif cls is Year: + return "year" + elif cls is All: + return "all" + else: + raise ValueError + + +def pad_interval( + interval_in: tuple[datetime.date, datetime.date], days: int +) -> tuple[datetime.date, datetime.date]: + start_in, stop_in = interval_in + padding = datetime.timedelta(days=days) + start_out = start_in - padding + stop_out = stop_in + padding + return start_out, stop_out diff --git a/src/monitoring/plot_utils.py b/src/monitoring/plot_utils.py index 8e5156c0..b617dc20 100644 --- a/src/monitoring/plot_utils.py +++ b/src/monitoring/plot_utils.py @@ -12,7 +12,7 @@ from matplotlib.figure import Figure from numpy.typing import NDArray -from monitoring.period import Period, PeriodWithRange +from monitoring.period import PeriodProtocol, PeriodType FONT_SIZE = 30 DPI = 400 @@ -110,13 +110,12 @@ def pretty_ax_2d(ax: Axes) -> None: def set_xlim_for_period( - ax: Axes, period: Period, time: NDArray[np.datetime64], pad: float = 0 + ax: Axes, period: PeriodType, time: NDArray[np.datetime64], pad: float = 0 ) -> None: - if isinstance(period, PeriodWithRange): - start_lim = np.datetime64(period.start_date).astype(time.dtype) - end_lim = np.datetime64(period.end_date + datetime.timedelta(days=1)).astype( - time.dtype - ) + if isinstance(period, PeriodProtocol): + start, stop = period.to_interval() + start_lim = np.datetime64(start).astype(time.dtype) + end_lim = np.datetime64(stop + datetime.timedelta(days=1)).astype(time.dtype) delta = (end_lim - start_lim) * pad ax.set_xlim(start_lim - delta, end_lim + delta) # type: ignore[arg-type] diff --git a/src/monitoring/product.py b/src/monitoring/product.py index 0d0d7e2e..40057c84 100644 --- a/src/monitoring/product.py +++ b/src/monitoring/product.py @@ -5,7 +5,7 @@ @dataclass class MonitoringVariable: id: str - humanReadableName: str + human_readable_name: str order: int = field(default=0) def __repr__(self) -> str: @@ -15,7 +15,7 @@ def __repr__(self) -> str: def from_dict(data: dict[str, Any]) -> "MonitoringVariable": return MonitoringVariable( id=data["id"], - humanReadableName=data["humanReadableName"], + human_readable_name=data["humanReadableName"], order=data.get("order", 0), ) @@ -23,7 +23,7 @@ def from_dict(data: dict[str, Any]) -> "MonitoringVariable": @dataclass class MonitoringProduct: id: str - humanReadableName: str + human_readable_name: str variables: list[MonitoringVariable] def __repr__(self) -> str: @@ -36,6 +36,6 @@ def from_dict(data: dict[str, Any]) -> "MonitoringProduct": ] return MonitoringProduct( id=data["id"], - humanReadableName=data["humanReadableName"], + human_readable_name=data["humanReadableName"], variables=variables, ) diff --git a/src/monitoring/py.typed b/src/monitoring/py.typed deleted file mode 100644 index e69de29b..00000000 diff --git a/src/monitoring/utils.py b/src/monitoring/utils.py index ab90c7f4..57fab077 100644 --- a/src/monitoring/utils.py +++ b/src/monitoring/utils.py @@ -1,26 +1,51 @@ -import datetime +from typing import TypedDict + +from cloudnet_api_client import APIClient +from cloudnet_api_client.client import DateParam -from monitoring.period import AllPeriod, Period, PeriodWithRange from processing.config import Config from processing.metadata_api import MetadataApi from processing.storage_api import StorageApi from processing.utils import make_session -def get_apis() -> tuple[MetadataApi, StorageApi]: +def instrument_uuid_to_pid(client: APIClient, uuid: str) -> str: + res = client._get(f"instrument-pids/{uuid}") + if len(res) == 0: + raise ValueError(f"Could not find pid for uuid '{uuid}'") + if len(res) > 1: + raise ValueError(f"Pid for uuid '{uuid}' is not unique") + return res[0]["pid"] + + +def get_api_client() -> APIClient: + config = Config() + client = APIClient(base_url=f"{config.dataportal_url}/api") + return client + + +def get_storage_api() -> StorageApi: + config = Config() + session = make_session() + return StorageApi(config, session) + + +def get_md_api() -> MetadataApi: config = Config() session = make_session() - md_api = MetadataApi(config, session) - storage_api = StorageApi(config, session) - return md_api, storage_api - - -def range_from_period(period: Period) -> tuple[datetime.date, datetime.date]: - match period: - case AllPeriod(): - start = datetime.date(1900, 1, 1) - end = datetime.date.today() + datetime.timedelta(days=1) - case PeriodWithRange(): - start = period.start_date - end = period.end_date - return start, end + return MetadataApi(config, session) + + +class RawFilesDatePayload(TypedDict, total=False): + date_from: DateParam + date_to: DateParam + + +class RawFilesPayload(TypedDict, total=False): + site_id: str + date_from: DateParam + date_to: DateParam + instrument_id: str + instrument_pid: str + filename_prefix: str + filename_suffix: str diff --git a/tests/unit/test_utils_module.py b/tests/unit/test_utils_module.py index 4856830f..887027f3 100644 --- a/tests/unit/test_utils_module.py +++ b/tests/unit/test_utils_module.py @@ -129,28 +129,28 @@ def test_are_identical_nc_files_real_data() -> None: NCDiff.NONE, ), ( - ma.masked_array([1, 2], mask=[0, 1]), + ma.masked_array([1, 2], mask=[False, True]), {"fill_value": 99}, {}, - ma.masked_array([1, 3], mask=[0, 1]), + ma.masked_array([1, 3], mask=[False, True]), {"fill_value": 999}, {}, NCDiff.NONE, ), ( - ma.masked_array([1, 2], mask=[1, 1]), + ma.masked_array([1, 2], mask=[True, True]), {"fill_value": 99}, {}, - ma.masked_array([1, 3], mask=[1, 1]), + ma.masked_array([1, 3], mask=[True, True]), {"fill_value": 999}, {}, NCDiff.NONE, ), ( - ma.masked_array([23, 23], mask=[1, 1]), + ma.masked_array([23, 23], mask=[True, True]), {}, {}, - ma.masked_array([1, 3], mask=[1, 1]), + ma.masked_array([1, 3], mask=[True, True]), {}, {}, NCDiff.NONE, @@ -159,7 +159,7 @@ def test_are_identical_nc_files_real_data() -> None: np.array([23, 23]), {}, {}, - ma.masked_array([23, 23], mask=[1, 0]), + ma.masked_array([23, 23], mask=[True, False]), {}, {}, NCDiff.MAJOR, @@ -168,7 +168,7 @@ def test_are_identical_nc_files_real_data() -> None: np.array([23, 23]), {}, {}, - ma.masked_array([23, 23], mask=[0, 0]), + ma.masked_array([23, 23], mask=[False, False]), {}, {}, NCDiff.NONE, @@ -177,43 +177,43 @@ def test_are_identical_nc_files_real_data() -> None: np.array([23, 23]), {}, {}, - ma.masked_array([23, 23], mask=[1, 1]), + ma.masked_array([23, 23], mask=[True, True]), {}, {}, NCDiff.MAJOR, ), ( - ma.masked_array([1.0, 2.0, 3.0], mask=[0, 0, 0]), + ma.masked_array([1.0, 2.0, 3.0], mask=[False, False, False]), {}, {}, - ma.masked_array([1.0, 2.0, 3.0], mask=[0, 0, 0]), + ma.masked_array([1.0, 2.0, 3.0], mask=[False, False, False]), {}, {}, NCDiff.NONE, ), ( - ma.masked_array([1.0, 2.0, 3.0], mask=[0, 0, 1]), + ma.masked_array([1.0, 2.0, 3.0], mask=[False, False, True]), {}, {}, - ma.masked_array([1.0, 2.0, 4.0], mask=[0, 0, 1]), + ma.masked_array([1.0, 2.0, 4.0], mask=[False, False, True]), {}, {}, NCDiff.NONE, ), ( - ma.masked_array([1.0, 2.0, 3.0], mask=[0, 0, 0]), + ma.masked_array([1.0, 2.0, 3.0], mask=[False, False, False]), {}, {}, - ma.masked_array([1.0, 2.0, 3.0], mask=[0, 0, 1]), + ma.masked_array([1.0, 2.0, 3.0], mask=[False, False, True]), {}, {}, NCDiff.MAJOR, ), ( - ma.masked_array([1.0, 2.0, 3.0], mask=[0, 1, 0]), + ma.masked_array([1.0, 2.0, 3.0], mask=[False, True, False]), {}, {}, - ma.masked_array([1.0, 2.0, 3.0], mask=[0, 0, 1]), + ma.masked_array([1.0, 2.0, 3.0], mask=[False, False, True]), {}, {}, NCDiff.MAJOR, @@ -347,13 +347,13 @@ def test_are_identical_nc_files_global_attributes( (ma.masked_array([1, 2, 3]), ma.masked_array([1, 2, 3]), NCDiff.NONE), (ma.masked_array([1, 2, 3]), ma.masked_array([1, 2, 4]), NCDiff.MAJOR), ( - ma.masked_array([1.0, 2.0, 3.0], mask=[0, 0, 0]), - ma.masked_array([1.0, 2.0, 3.0], mask=[0, 0, 0]), + ma.masked_array([1.0, 2.0, 3.0], mask=[False, False, False]), + ma.masked_array([1.0, 2.0, 3.0], mask=[False, False, False]), NCDiff.NONE, ), ( - ma.masked_array([1.0, 2.0, 3.0], mask=[0, 0, 0]), - ma.masked_array([1.0, 2.0, 3.0], mask=[0, 0, 1]), + ma.masked_array([1.0, 2.0, 3.0], mask=[False, False, False]), + ma.masked_array([1.0, 2.0, 3.0], mask=[False, False, True]), NCDiff.MAJOR, ), ], From cbefa875118a98037c8ff4b3ec5ab9d715296653 Mon Sep 17 00:00:00 2001 From: Niko Leskinen Date: Fri, 9 Jan 2026 11:55:31 +0200 Subject: [PATCH 2/3] refactor monitorin opts --- src/monitoring/cli.py | 54 +++++-- .../instrument/halo_doppler_lidar.py | 153 +++++++----------- src/monitoring/monitor.py | 18 +-- src/monitoring/monitor_options.py | 19 +++ src/monitoring/monitoring_file.py | 15 +- src/monitoring/utils.py | 23 --- 6 files changed, 138 insertions(+), 144 deletions(-) create mode 100644 src/monitoring/monitor_options.py diff --git a/src/monitoring/cli.py b/src/monitoring/cli.py index fcffaa7c..d440e69b 100644 --- a/src/monitoring/cli.py +++ b/src/monitoring/cli.py @@ -4,9 +4,12 @@ from argparse import ArgumentParser, Namespace from typing import Callable, Iterable, TypeVar +from cloudnet_api_client import APIClient + import monitoring.period as period_module from monitoring.config import CONFIG from monitoring.monitor import monitor +from monitoring.monitor_options import MonitorOptions from monitoring.period import ( All, Day, @@ -19,23 +22,29 @@ period_str_from_cls, ) from monitoring.product import MonitoringProduct -from monitoring.utils import RawFilesPayload, get_api_client, get_md_api +from monitoring.utils import RawFilesPayload +from processing.config import Config +from processing.metadata_api import MetadataApi +from processing.storage_api import StorageApi +from processing.utils import make_session T = TypeVar("T", bound=PeriodProtocol) PeriodList = list[All] | list[Day] | list[Month] | list[Week] | list[Year] def main() -> None: + api_client, md_api, storage_api = build_clients() + logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") args = _get_args() period_cls = period_cls_from_str(args.cmd) periods = build_periods(period_cls, args) periods_with_products = build_products(period_cls, periods, args.product) periods_with_products_sites_and_instruments = build_instruments( - periods_with_products, args.site + api_client, periods_with_products, args.site ) validated_periods_with_products_sites_and_instruments = validate_products( - periods_with_products_sites_and_instruments + md_api, periods_with_products_sites_and_instruments ) for ( @@ -45,11 +54,31 @@ def main() -> None: instrument_uuid, ) in validated_periods_with_products_sites_and_instruments: try: - monitor(period, product, site, instrument_uuid) + monitor( + MonitorOptions( + period, + product, + site, + instrument_uuid, + api_client, + storage_api, + md_api, + ) + ) except ValueError as err: logging.warning(err) +def build_clients() -> tuple[APIClient, MetadataApi, StorageApi]: + config = Config() + session = make_session() + return ( + APIClient(base_url=f"{config.dataportal_url}/api"), + MetadataApi(config, session), + StorageApi(config, session), + ) + + def build_periods(period_cls: type[PeriodType], args: Namespace) -> PeriodList: match period_cls: case period_module.Day: @@ -89,11 +118,15 @@ def build_products( def build_instruments( - product_list: list[tuple[PeriodType, str]], sites: list[str] | None + client: APIClient, + product_list: list[tuple[PeriodType, str]], + sites: list[str] | None, ) -> list[tuple[PeriodType, str, str, str]]: list_with_sites_and_instruments = [] for period, product in product_list: - for site, instrument_uuid in get_available_instruments(period, product, sites): + for site, instrument_uuid in get_available_instruments( + client, period, product, sites + ): list_with_sites_and_instruments.append( (period, product, site, instrument_uuid) ) @@ -101,9 +134,10 @@ def build_instruments( def validate_products( + api: MetadataApi, product_list: list[tuple[PeriodType, str, str, str]], ) -> list[tuple[PeriodType, MonitoringProduct, str, str]]: - available_products = get_available_products() + available_products = get_available_products(api) validated = [] for period, product_str, site, instrument_uuid in product_list: if not product_str in available_products: @@ -114,16 +148,14 @@ def validate_products( return validated -def get_available_products() -> dict[str, MonitoringProduct]: - api = get_md_api() +def get_available_products(api: MetadataApi) -> dict[str, MonitoringProduct]: data = api.get("api/monitoring-products/variables") return {entry["id"]: MonitoringProduct.from_dict(entry) for entry in data} def get_available_instruments( - period: PeriodType, product: str, sites: list[str] | None + client: APIClient, period: PeriodType, product: str, sites: list[str] | None ) -> list[tuple[str, str]]: - client = get_api_client() payload: RawFilesPayload match product: diff --git a/src/monitoring/instrument/halo_doppler_lidar.py b/src/monitoring/instrument/halo_doppler_lidar.py index 2ac23f05..f551e1ca 100644 --- a/src/monitoring/instrument/halo_doppler_lidar.py +++ b/src/monitoring/instrument/halo_doppler_lidar.py @@ -5,12 +5,12 @@ import matplotlib.pyplot as plt import numpy as np -from cloudnet_api_client.client import APIClient from cloudnetpy.plotting.plotting import Dimensions from doppy.raw import HaloSysParams from doppy.raw.halo_bg import HaloBg from doppy.raw.halo_hpl import HaloHpl +from monitoring.monitor_options import MonitorOptions from monitoring.monitoring_file import MonitoringFile, MonitoringVisualization from monitoring.period import All, PeriodType from monitoring.plot_utils import ( @@ -26,51 +26,29 @@ from monitoring.product import MonitoringProduct, MonitoringVariable from monitoring.utils import ( RawFilesDatePayload, - get_storage_api, instrument_uuid_to_pid, ) -from processing.storage_api import StorageApi -def monitor( - period: PeriodType, - product: MonitoringProduct, - site: str, - instrument_uuid: str, - api_client: APIClient, - storage_api: StorageApi, -) -> None: - match product.id: +def monitor(opts: MonitorOptions) -> None: + match opts.product.id: case "halo-doppler-lidar_housekeeping": - monitor_housekeeping( - period, product, site, instrument_uuid, api_client, storage_api - ) + monitor_housekeeping(opts) case "halo-doppler-lidar_background": - monitor_background( - period, product, site, instrument_uuid, api_client, storage_api - ) + monitor_background(opts) case "halo-doppler-lidar_signal": - monitor_signal( - period, product, site, instrument_uuid, api_client, storage_api - ) + monitor_signal(opts) -def monitor_housekeeping( - period: PeriodType, - product: MonitoringProduct, - site: str, - instrument_uuid: str, - api_client: APIClient, - storage_api: StorageApi, -) -> None: - pid = instrument_uuid_to_pid(api_client, instrument_uuid) +def monitor_housekeeping(opts: MonitorOptions) -> None: + pid = instrument_uuid_to_pid(opts.api_client, opts.instrument_uuid) date_opts: RawFilesDatePayload = {} - if not isinstance(period, All): - start, stop = period.to_interval_padded(days=31) + if not isinstance(opts.period, All): + start, stop = opts.period.to_interval_padded(days=31) date_opts = {"date_from": start, "date_to": stop} - records = api_client.raw_files( - site_id=site, + records = opts.api_client.raw_files( + site_id=opts.site, instrument_pid=pid, filename_prefix="system_parameters_", filename_suffix=".txt", @@ -78,19 +56,19 @@ def monitor_housekeeping( ) if not records: raise ValueError( - f"No raw files for monitoring period {period} {product.id} {site} {pid}" + f"No raw files for monitoring period {opts.period} {opts.product.id} {opts.site} {pid}" ) with TemporaryDirectory() as tempdir: - (paths, _uuids) = storage_api.download_raw_data(records, Path(tempdir)) + (paths, _uuids) = opts.storage_api.download_raw_data(records, Path(tempdir)) sys_params_list = [HaloSysParams.from_src(p) for p in paths] sys_params = ( HaloSysParams.merge(sys_params_list) .sorted_by_time() .non_strictly_increasing_timesteps_removed() ) - if not isinstance(period, All): - start, stop = period.to_interval() + if not isinstance(opts.period, All): + start, stop = opts.period.to_interval() dtype = sys_params.time.dtype start_time = np.datetime64(start).astype(dtype) stop_time = np.datetime64(stop + datetime.timedelta(days=1)).astype(dtype) @@ -98,15 +76,17 @@ def monitor_housekeeping( sys_params = sys_params[select] if len(sys_params.time) == 0: raise ValueError( - f"No timestamps for monitoring period {period} {product.id} {site} {pid}" + f"No timestamps for monitoring period {opts.period} {opts.product.id} {opts.site} {pid}" ) monitoring_file = MonitoringFile( - instrument_uuid, - site, - period, - product, - monitor_housekeeping_plots(sys_params, period, product), + opts.instrument_uuid, + opts.site, + opts.period, + opts.product, + monitor_housekeeping_plots(sys_params, opts.period, opts.product), + opts.md_api, + opts.storage_api, ) monitoring_file.upload() @@ -137,24 +117,16 @@ def plot_housekeeping_variable( return vis -def monitor_background( - period: PeriodType, - product: MonitoringProduct, - site: str, - instrument_uuid: str, - api_client: APIClient, - storage_api: StorageApi, -) -> None: - storage_api = get_storage_api() - pid = instrument_uuid_to_pid(api_client, instrument_uuid) +def monitor_background(opts: MonitorOptions) -> None: + pid = instrument_uuid_to_pid(opts.api_client, opts.instrument_uuid) date_opts: RawFilesDatePayload = {} - if not isinstance(period, All): - start, stop = period.to_interval_padded(days=1) + if not isinstance(opts.period, All): + start, stop = opts.period.to_interval_padded(days=1) date_opts = {"date_from": start, "date_to": stop} - records = api_client.raw_files( - site_id=site, + records = opts.api_client.raw_files( + site_id=opts.site, instrument_pid=pid, filename_prefix="Background_", filename_suffix=".txt", @@ -162,18 +134,18 @@ def monitor_background( ) if not records: raise ValueError( - f"No raw files for monitoring period {period} {product.id} {site} {pid}" + f"No raw files for monitoring period {opts.period} {opts.product.id} {opts.site} {pid}" ) with TemporaryDirectory() as tempdir: - (paths, _uuids) = storage_api.download_raw_data(records, Path(tempdir)) + (paths, _uuids) = opts.storage_api.download_raw_data(records, Path(tempdir)) bgs = HaloBg.from_srcs(paths) counter = Counter((bg.signal.shape[1] for bg in bgs)) most_common_ngates = counter.most_common()[0][0] bgs = [bg for bg in bgs if bg.signal.shape[1] == most_common_ngates] bg = HaloBg.merge(bgs).sorted_by_time().non_strictly_increasing_timesteps_removed() - if not isinstance(period, All): - start, stop = period.to_interval() + if not isinstance(opts.period, All): + start, stop = opts.period.to_interval() dtype = bg.time.dtype start_time = np.datetime64(start).astype(dtype) stop_time = np.datetime64(stop + datetime.timedelta(days=1)).astype(dtype) @@ -181,14 +153,16 @@ def monitor_background( bg = bg[select] if len(bg.time) == 0: raise ValueError( - f"No timestamps for monitoring period {period} {product.id} {site} {pid}" + f"No timestamps for monitoring period {opts.period} {opts.product.id} {opts.site} {pid}" ) monitoring_file = MonitoringFile( - instrument_uuid, - site, - period, - product, - [p for p in monitor_background_plots(bg, period, product) if p], + opts.instrument_uuid, + opts.site, + opts.period, + opts.product, + [p for p in monitor_background_plots(bg, opts.period, opts.product) if p], + opts.md_api, + opts.storage_api, ) monitoring_file.upload() @@ -274,23 +248,16 @@ def plot_time_averaged_background_profile( return vis -def monitor_signal( - period: PeriodType, - product: MonitoringProduct, - site: str, - instrument_uuid: str, - api_client: APIClient, - storage_api: StorageApi, -) -> None: - pid = instrument_uuid_to_pid(api_client, instrument_uuid) +def monitor_signal(opts: MonitorOptions) -> None: + pid = instrument_uuid_to_pid(opts.api_client, opts.instrument_uuid) date_opts: RawFilesDatePayload = {} - if not isinstance(period, All): - start, stop = period.to_interval() + if not isinstance(opts.period, All): + start, stop = opts.period.to_interval() date_opts = {"date_from": start, "date_to": stop} - records = api_client.raw_files( - site_id=site, + records = opts.api_client.raw_files( + site_id=opts.site, instrument_pid=pid, filename_suffix=".hpl", **date_opts, @@ -299,11 +266,11 @@ def monitor_signal( records = [r for r in records if "cross" not in r.tags] if not records: raise ValueError( - f"No raw files for monitoring period {period} {product.id} {site} {pid}" + f"No raw files for monitoring period {opts.period} {opts.product.id} {opts.site} {pid}" ) with TemporaryDirectory() as tempdir: - (paths, _uuids) = storage_api.download_raw_data(records, Path(tempdir)) + (paths, _uuids) = opts.storage_api.download_raw_data(records, Path(tempdir)) raws = HaloHpl.from_srcs(paths) def is_stare(raw: HaloHpl) -> bool: @@ -318,7 +285,7 @@ def is_stare(raw: HaloHpl) -> bool: raws = [r for r in raws if is_stare(r)] if not raws: raise ValueError( - f"No raw stare files found for {period} {instrument_uuid} {product.id}" + f"No raw stare files found for {opts.period} {opts.instrument_uuid} {opts.product.id}" ) counter = Counter((raw.intensity.shape[1] for raw in raws)) most_common_ngates = counter.most_common()[0][0] @@ -326,8 +293,8 @@ def is_stare(raw: HaloHpl) -> bool: raw = ( HaloHpl.merge(raws).sorted_by_time().non_strictly_increasing_timesteps_removed() ) - if not isinstance(period, All): - start, stop = period.to_interval() + if not isinstance(opts.period, All): + start, stop = opts.period.to_interval() dtype = raw.time.dtype start_time = np.datetime64(start).astype(dtype) stop_time = np.datetime64(stop + datetime.timedelta(days=1)).astype(dtype) @@ -336,15 +303,17 @@ def is_stare(raw: HaloHpl) -> bool: if len(raw.time) == 0: raise ValueError( - f"No timestamps for monitoring period {period} {product.id} {site} {pid}" + f"No timestamps for monitoring period {opts.period} {opts.product.id} {opts.site} {pid}" ) monitoring_file = MonitoringFile( - instrument_uuid, - site, - period, - product, - monitor_signal_plots(raw, period, product), + opts.instrument_uuid, + opts.site, + opts.period, + opts.product, + monitor_signal_plots(raw, opts.period, opts.product), + opts.md_api, + opts.storage_api, ) monitoring_file.upload() diff --git a/src/monitoring/monitor.py b/src/monitoring/monitor.py index 86b05c22..a8636a86 100644 --- a/src/monitoring/monitor.py +++ b/src/monitoring/monitor.py @@ -3,18 +3,12 @@ from monitoring.instrument.halo_doppler_lidar import ( monitor as monitor_halo, ) -from monitoring.period import PeriodType -from monitoring.product import MonitoringProduct -from monitoring.utils import get_api_client, get_storage_api +from monitoring.monitor_options import MonitorOptions -def monitor( - period: PeriodType, product: MonitoringProduct, site: str, instrument_uuid: str -) -> None: - api_client = get_api_client() - storage_api = get_storage_api() - if product.id.startswith("halo-doppler-lidar"): - monitor_halo(period, product, site, instrument_uuid, api_client, storage_api) - logging.info(f"Monitored {period!r} {product} {site}") +def monitor(opts: MonitorOptions) -> None: + if opts.product.id.startswith("halo-doppler-lidar"): + monitor_halo(opts) + logging.info(f"Monitored {opts.period!r} {opts.product} {opts.site}") else: - raise ValueError(f"Unexpected product: '{product}'") + raise ValueError(f"Unexpected product: '{opts.product}'") diff --git a/src/monitoring/monitor_options.py b/src/monitoring/monitor_options.py new file mode 100644 index 00000000..4f876568 --- /dev/null +++ b/src/monitoring/monitor_options.py @@ -0,0 +1,19 @@ +from dataclasses import dataclass + +from cloudnet_api_client.client import APIClient + +from monitoring.period import PeriodType +from monitoring.product import MonitoringProduct +from processing.metadata_api import MetadataApi +from processing.storage_api import StorageApi + + +@dataclass +class MonitorOptions: + period: PeriodType + product: MonitoringProduct + site: str + instrument_uuid: str + api_client: APIClient + storage_api: StorageApi + md_api: MetadataApi diff --git a/src/monitoring/monitoring_file.py b/src/monitoring/monitoring_file.py index 852fb7a9..45e5f908 100644 --- a/src/monitoring/monitoring_file.py +++ b/src/monitoring/monitoring_file.py @@ -7,7 +7,8 @@ from monitoring.period import All, PeriodProtocol, PeriodType from monitoring.product import MonitoringProduct, MonitoringVariable -from monitoring.utils import get_md_api, get_storage_api +from processing.metadata_api import MetadataApi +from processing.storage_api import StorageApi @dataclass @@ -35,14 +36,14 @@ class MonitoringFile: period: PeriodType product: MonitoringProduct visualisations: list[MonitoringVisualization] + md_api: MetadataApi + storage_api: StorageApi def upload(self) -> None: if not self.visualisations: raise ValueError( f"No visualisations for product {self.product.id}, {self.site}, {self.instrument_uuid}, {self.period}" ) - md_api = get_md_api() - storage_api = get_storage_api() payload = { "periodType": self.period.to_str(), "siteId": self.site, @@ -51,7 +52,7 @@ def upload(self) -> None: } if isinstance(self.period, PeriodProtocol): payload["startDate"] = str(self.period.start()) - res = md_api.post("monitoring-files", payload=payload) + res = self.md_api.post("monitoring-files", payload=payload) if not res.ok: raise RuntimeError(f"Could not post file: {payload}") try: @@ -68,14 +69,16 @@ def upload(self) -> None: ) with NamedTemporaryFile() as tempfile: tempfile.write(vis.fig) - storage_api.upload_image(full_path=Path(tempfile.name), s3key=s3key) + self.storage_api.upload_image( + full_path=Path(tempfile.name), s3key=s3key + ) payload_vis: dict[str, str | int] = { "s3key": s3key, "sourceFileUuid": file_uuid, "variableId": vis.variable.id, **_dimensions_as_payload(vis.dimensions), } - res = md_api.post("monitoring-visualizations", payload=payload_vis) + res = self.md_api.post("monitoring-visualizations", payload=payload_vis) if not res.ok: raise RuntimeError(f"Could not post visualisation: {payload_vis}") diff --git a/src/monitoring/utils.py b/src/monitoring/utils.py index 57fab077..9eb58664 100644 --- a/src/monitoring/utils.py +++ b/src/monitoring/utils.py @@ -3,11 +3,6 @@ from cloudnet_api_client import APIClient from cloudnet_api_client.client import DateParam -from processing.config import Config -from processing.metadata_api import MetadataApi -from processing.storage_api import StorageApi -from processing.utils import make_session - def instrument_uuid_to_pid(client: APIClient, uuid: str) -> str: res = client._get(f"instrument-pids/{uuid}") @@ -18,24 +13,6 @@ def instrument_uuid_to_pid(client: APIClient, uuid: str) -> str: return res[0]["pid"] -def get_api_client() -> APIClient: - config = Config() - client = APIClient(base_url=f"{config.dataportal_url}/api") - return client - - -def get_storage_api() -> StorageApi: - config = Config() - session = make_session() - return StorageApi(config, session) - - -def get_md_api() -> MetadataApi: - config = Config() - session = make_session() - return MetadataApi(config, session) - - class RawFilesDatePayload(TypedDict, total=False): date_from: DateParam date_to: DateParam From 4e700c950223f0ce80655f2d951518a10b2d43c2 Mon Sep 17 00:00:00 2001 From: Niko Leskinen Date: Fri, 9 Jan 2026 13:13:57 +0200 Subject: [PATCH 3/3] handle storage api errors --- src/monitoring/cli.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/monitoring/cli.py b/src/monitoring/cli.py index d440e69b..3ca1405f 100644 --- a/src/monitoring/cli.py +++ b/src/monitoring/cli.py @@ -25,7 +25,7 @@ from monitoring.utils import RawFilesPayload from processing.config import Config from processing.metadata_api import MetadataApi -from processing.storage_api import StorageApi +from processing.storage_api import StorageApi, StorageApiError from processing.utils import make_session T = TypeVar("T", bound=PeriodProtocol) @@ -35,7 +35,9 @@ def main() -> None: api_client, md_api, storage_api = build_clients() - logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") + logging.basicConfig( + level=logging.INFO, format="Monitoring:%(levelname)s: %(message)s" + ) args = _get_args() period_cls = period_cls_from_str(args.cmd) periods = build_periods(period_cls, args) @@ -65,8 +67,10 @@ def main() -> None: md_api, ) ) - except ValueError as err: - logging.warning(err) + except (ValueError, StorageApiError) as err: + logging.warning(f"{period!r} {product} {site}: {err}") + except StorageApiError as err: + logging.error(f"{period!r} {product} {site}: {err}") def build_clients() -> tuple[APIClient, MetadataApi, StorageApi]: