From 85f6a9df3e3916a24438975073450394356ec84a Mon Sep 17 00:00:00 2001 From: kraysent Date: Mon, 27 Jul 2026 00:08:41 +0100 Subject: [PATCH 1/9] re-slice internal structure of a project --- app/adminapi/cli.py | 18 ++++++++ app/{commands => }/adminapi/command.py | 5 +-- app/adminapi/domain/__init__.py | 15 +++++++ .../adminapi => adminapi/domain}/actions.py | 4 +- .../adminapi => adminapi/domain}/catalogs.py | 2 +- .../domain}/crossmatch.py | 2 +- .../domain}/layer1_write.py | 2 +- .../adminapi => adminapi/domain}/login.py | 2 +- .../adminapi => adminapi/domain}/mock.py | 4 +- .../adminapi => adminapi/domain}/pgc.py | 2 +- .../adminapi => adminapi/domain}/sources.py | 2 +- .../domain}/table_stats.py | 2 +- .../domain}/table_upload.py | 4 +- app/adminapi/presentation/__init__.py | 5 +++ .../presentation}/interface.py | 2 +- .../presentation}/records.py | 0 .../presentation}/server.py | 2 +- app/cli.py | 44 +++---------------- app/commands/__init__.py | 0 app/commands/adminapi/__init__.py | 6 --- app/commands/dataapi/__init__.py | 3 -- app/commands/serve_tasks/__init__.py | 3 -- app/dataapi/cli.py | 18 ++++++++ app/{commands => }/dataapi/command.py | 4 +- app/dataapi/domain/__init__.py | 5 +++ .../dataapi => dataapi/domain}/actions.py | 6 +-- .../domain}/parameterized_query.py | 4 +- .../dataapi => dataapi/domain}/tap_types.py | 0 app/dataapi/presentation/__init__.py | 5 +++ .../presentation}/interface.py | 2 +- .../presentation}/server.py | 2 +- .../dataapi => dataapi/presentation}/tap.py | 0 app/dataapi/responders/__init__.py | 3 ++ .../responders/interface.py | 0 .../responders/structured_responder.py | 4 +- app/domain/__init__.py | 0 app/domain/adminapi/__init__.py | 15 ------- app/domain/dataapi/__init__.py | 5 --- app/domain/responders/__init__.py | 3 -- app/presentation/__init__.py | 0 app/presentation/adminapi/__init__.py | 5 --- app/presentation/dataapi/__init__.py | 5 --- app/tasks/cli.py | 14 ++++++ .../serve_tasks => tasks}/command.py | 2 +- app/{commands/serve_tasks => tasks}/flows.py | 0 tests/env_test.py | 4 +- tests/integration/assign_record_pgcs_test.py | 4 +- tests/integration/catalogs_api_test.py | 6 +-- tests/integration/create_table_test.py | 5 +-- tests/integration/merge_pgcs_test.py | 4 +- tests/integration/metadata_api_test.py | 8 ++-- tests/integration/rawdata_table_test.py | 5 +-- tests/regression/patch_table_metadata.py | 2 +- tests/regression/upload_simple_table.py | 2 +- tests/unit/commands/serve_tasks_test.py | 2 +- tests/unit/domain/create_source_test.py | 2 +- tests/unit/domain/parameterized_query_test.py | 2 +- tests/unit/domain/table_stats_test.py | 2 +- tests/unit/domain/table_upload_test.py | 7 ++- 59 files changed, 143 insertions(+), 143 deletions(-) create mode 100644 app/adminapi/cli.py rename app/{commands => }/adminapi/command.py (96%) create mode 100644 app/adminapi/domain/__init__.py rename app/{domain/adminapi => adminapi/domain}/actions.py (96%) rename app/{domain/adminapi => adminapi/domain}/catalogs.py (98%) rename app/{domain/adminapi => adminapi/domain}/crossmatch.py (99%) rename app/{domain/adminapi => adminapi/domain}/layer1_write.py (97%) rename app/{domain/adminapi => adminapi/domain}/login.py (92%) rename app/{domain/adminapi => adminapi/domain}/mock.py (89%) rename app/{domain/adminapi => adminapi/domain}/pgc.py (95%) rename app/{domain/adminapi => adminapi/domain}/sources.py (94%) rename app/{domain/adminapi => adminapi/domain}/table_stats.py (96%) rename app/{domain/adminapi => adminapi/domain}/table_upload.py (99%) create mode 100644 app/adminapi/presentation/__init__.py rename app/{presentation/adminapi => adminapi/presentation}/interface.py (99%) rename app/{presentation/adminapi => adminapi/presentation}/records.py (100%) rename app/{presentation/adminapi => adminapi/presentation}/server.py (99%) delete mode 100644 app/commands/__init__.py delete mode 100644 app/commands/adminapi/__init__.py delete mode 100644 app/commands/dataapi/__init__.py delete mode 100644 app/commands/serve_tasks/__init__.py create mode 100644 app/dataapi/cli.py rename app/{commands => }/dataapi/command.py (95%) create mode 100644 app/dataapi/domain/__init__.py rename app/{domain/dataapi => dataapi/domain}/actions.py (96%) rename app/{domain/dataapi => dataapi/domain}/parameterized_query.py (97%) rename app/{domain/dataapi => dataapi/domain}/tap_types.py (100%) create mode 100644 app/dataapi/presentation/__init__.py rename app/{presentation/dataapi => dataapi/presentation}/interface.py (99%) rename app/{presentation/dataapi => dataapi/presentation}/server.py (98%) rename app/{presentation/dataapi => dataapi/presentation}/tap.py (100%) create mode 100644 app/dataapi/responders/__init__.py rename app/{domain => dataapi}/responders/interface.py (100%) rename app/{domain => dataapi}/responders/structured_responder.py (99%) delete mode 100644 app/domain/__init__.py delete mode 100644 app/domain/adminapi/__init__.py delete mode 100644 app/domain/dataapi/__init__.py delete mode 100644 app/domain/responders/__init__.py delete mode 100644 app/presentation/__init__.py delete mode 100644 app/presentation/adminapi/__init__.py delete mode 100644 app/presentation/dataapi/__init__.py create mode 100644 app/tasks/cli.py rename app/{commands/serve_tasks => tasks}/command.py (93%) rename app/{commands/serve_tasks => tasks}/flows.py (100%) diff --git a/app/adminapi/cli.py b/app/adminapi/cli.py new file mode 100644 index 00000000..cb8cd092 --- /dev/null +++ b/app/adminapi/cli.py @@ -0,0 +1,18 @@ +import os + +import click + +from app.adminapi.command import AdminAPICommand +from app.lib import commands + + +@click.command(short_help=AdminAPICommand.help()) +@click.option( + "-c", + "--config", + type=str, + default=lambda: os.environ.get("CONFIG", ""), + help="Path to configuration file", +) +def adminapi(config: str) -> None: + commands.run(AdminAPICommand(config)) diff --git a/app/commands/adminapi/command.py b/app/adminapi/command.py similarity index 96% rename from app/commands/adminapi/command.py rename to app/adminapi/command.py index dbf19137..1ecceaf4 100644 --- a/app/commands/adminapi/command.py +++ b/app/adminapi/command.py @@ -8,14 +8,13 @@ import structlog import yaml +from app.adminapi import domain, presentation +from app.adminapi.domain import table_stats from app.data import repositories -from app.domain import adminapi as domain -from app.domain.adminapi import table_stats from app.lib import audit, auth, cache, clients, commands, config, tracing from app.lib.storage import postgres from app.lib.tracing import TracingConfig from app.lib.web import server -from app.presentation import adminapi as presentation log: structlog.stdlib.BoundLogger = structlog.get_logger() diff --git a/app/adminapi/domain/__init__.py b/app/adminapi/domain/__init__.py new file mode 100644 index 00000000..d63e8661 --- /dev/null +++ b/app/adminapi/domain/__init__.py @@ -0,0 +1,15 @@ +from app.adminapi.domain.actions import Actions +from app.adminapi.domain.crossmatch import CrossmatchManager +from app.adminapi.domain.login import LoginManager +from app.adminapi.domain.mock import get_mock_actions +from app.adminapi.domain.sources import SourceManager +from app.adminapi.domain.table_upload import TableUploadManager + +__all__ = [ + "Actions", + "CrossmatchManager", + "get_mock_actions", + "LoginManager", + "TableUploadManager", + "SourceManager", +] diff --git a/app/domain/adminapi/actions.py b/app/adminapi/domain/actions.py similarity index 96% rename from app/domain/adminapi/actions.py rename to app/adminapi/domain/actions.py index 5b7548be..6e06706e 100644 --- a/app/domain/adminapi/actions.py +++ b/app/adminapi/domain/actions.py @@ -1,9 +1,9 @@ from typing import final +from app.adminapi import presentation as adminapi +from app.adminapi.domain import catalogs, crossmatch, layer1_write, login, pgc, sources, table_upload from app.data import repositories -from app.domain.adminapi import catalogs, crossmatch, layer1_write, login, pgc, sources, table_upload from app.lib import auth, cache, clients -from app.presentation import adminapi @final diff --git a/app/domain/adminapi/catalogs.py b/app/adminapi/domain/catalogs.py similarity index 98% rename from app/domain/adminapi/catalogs.py rename to app/adminapi/domain/catalogs.py index 672b6ede..5f5bfd57 100644 --- a/app/domain/adminapi/catalogs.py +++ b/app/adminapi/domain/catalogs.py @@ -1,7 +1,7 @@ from typing import Any, final +from app.adminapi import presentation as adminapi from app.data import model, repositories -from app.presentation import adminapi _INTERNAL_COLUMNS = frozenset({"record_id", "object_id", "id", "modification_time"}) diff --git a/app/domain/adminapi/crossmatch.py b/app/adminapi/domain/crossmatch.py similarity index 99% rename from app/domain/adminapi/crossmatch.py rename to app/adminapi/domain/crossmatch.py index f31aceb2..356c28e9 100644 --- a/app/domain/adminapi/crossmatch.py +++ b/app/adminapi/domain/crossmatch.py @@ -4,13 +4,13 @@ from astropy import coordinates from astropy import units as u +from app.adminapi import presentation as adminapi from app.data import model from app.data.repositories import layer0, layer1, layer2 from app.data.repositories.layer0.records import AssignRecordPgcsPreconditionError from app.lib import astronomy from app.lib.storage import enums from app.lib.web.errors import ConflictError, NotFoundError -from app.presentation import adminapi logger = structlog.stdlib.get_logger() diff --git a/app/domain/adminapi/layer1_write.py b/app/adminapi/domain/layer1_write.py similarity index 97% rename from app/domain/adminapi/layer1_write.py rename to app/adminapi/domain/layer1_write.py index 166004b7..5cec9c66 100644 --- a/app/domain/adminapi/layer1_write.py +++ b/app/adminapi/domain/layer1_write.py @@ -1,10 +1,10 @@ import psycopg from astropy import units as u +from app.adminapi import presentation as adminapi from app.data import model from app.data.repositories import layer1 from app.lib.web.errors import RuleValidationError -from app.presentation import adminapi class Layer1Writer: diff --git a/app/domain/adminapi/login.py b/app/adminapi/domain/login.py similarity index 92% rename from app/domain/adminapi/login.py rename to app/adminapi/domain/login.py index 35a77fd2..aa7a3c73 100644 --- a/app/domain/adminapi/login.py +++ b/app/adminapi/domain/login.py @@ -1,6 +1,6 @@ +from app.adminapi import presentation as adminapi from app.lib import auth from app.lib.web.errors import UnauthorizedError -from app.presentation import adminapi class LoginManager: diff --git a/app/domain/adminapi/mock.py b/app/adminapi/domain/mock.py similarity index 89% rename from app/domain/adminapi/mock.py rename to app/adminapi/domain/mock.py index b7fede53..d4ae74d0 100644 --- a/app/domain/adminapi/mock.py +++ b/app/adminapi/domain/mock.py @@ -1,9 +1,9 @@ import datetime from unittest import mock -from app.domain.adminapi import actions +from app.adminapi import presentation as adminapi +from app.adminapi.domain import actions from app.lib import auth, clients -from app.presentation import adminapi def get_mock_table_stats_cache() -> mock.MagicMock: diff --git a/app/domain/adminapi/pgc.py b/app/adminapi/domain/pgc.py similarity index 95% rename from app/domain/adminapi/pgc.py rename to app/adminapi/domain/pgc.py index 3d533dc5..0950fff3 100644 --- a/app/domain/adminapi/pgc.py +++ b/app/adminapi/domain/pgc.py @@ -1,8 +1,8 @@ from typing import final +from app.adminapi import presentation as adminapi from app.data import repositories from app.lib.web.errors import NotFoundError -from app.presentation import adminapi @final diff --git a/app/domain/adminapi/sources.py b/app/adminapi/domain/sources.py similarity index 94% rename from app/domain/adminapi/sources.py rename to app/adminapi/domain/sources.py index cbc07a62..10f77105 100644 --- a/app/domain/adminapi/sources.py +++ b/app/adminapi/domain/sources.py @@ -1,5 +1,5 @@ +from app.adminapi import presentation as adminapi from app.data import repositories -from app.presentation import adminapi ALLOWED_CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_" diff --git a/app/domain/adminapi/table_stats.py b/app/adminapi/domain/table_stats.py similarity index 96% rename from app/domain/adminapi/table_stats.py rename to app/adminapi/domain/table_stats.py index 62993e41..b74b6ca8 100644 --- a/app/domain/adminapi/table_stats.py +++ b/app/adminapi/domain/table_stats.py @@ -1,8 +1,8 @@ import datetime from collections.abc import Callable +from app.adminapi import presentation as adminapi from app.data import model, repositories -from app.presentation import adminapi def table_progress_to_presentation(progress: model.TableProgress) -> adminapi.TableProgress: diff --git a/app/domain/adminapi/table_upload.py b/app/adminapi/domain/table_upload.py similarity index 99% rename from app/domain/adminapi/table_upload.py rename to app/adminapi/domain/table_upload.py index a87e1793..c871f5b1 100644 --- a/app/domain/adminapi/table_upload.py +++ b/app/adminapi/domain/table_upload.py @@ -11,14 +11,14 @@ from astropy import units as u from astroquery import nasa_ads as ads +from app.adminapi import presentation as adminapi +from app.adminapi.domain import table_stats from app.data import model, repositories from app.data.repositories.common import ColumnSchemaInfo, TableSchemaInfo from app.data.repositories.layer0.common import RAWDATA_SCHEMA -from app.domain.adminapi import table_stats from app.lib import astronomy, cache, clients, concurrency from app.lib.storage import enums, mapping from app.lib.web.errors import NotFoundError, RuleValidationError -from app.presentation import adminapi BIBCODE_REGEX = "^([0-9]{4}[A-Za-z.&]{5}[A-Za-z0-9.]{4}[AELPQ-Z0-9.][0-9.]{4}[A-Z])$" diff --git a/app/adminapi/presentation/__init__.py b/app/adminapi/presentation/__init__.py new file mode 100644 index 00000000..a638e1be --- /dev/null +++ b/app/adminapi/presentation/__init__.py @@ -0,0 +1,5 @@ +from app.adminapi.presentation.interface import * +from app.adminapi.presentation.records import * +from app.adminapi.presentation.server import Server + +__all__ = ["Server"] diff --git a/app/presentation/adminapi/interface.py b/app/adminapi/presentation/interface.py similarity index 99% rename from app/presentation/adminapi/interface.py rename to app/adminapi/presentation/interface.py index 52f3369e..3dc5823f 100644 --- a/app/presentation/adminapi/interface.py +++ b/app/adminapi/presentation/interface.py @@ -5,8 +5,8 @@ import pydantic +from app.adminapi.presentation.records import GetRecordsRequest, GetRecordsResponse from app.lib.storage import enums, mapping -from app.presentation.adminapi.records import GetRecordsRequest, GetRecordsResponse DatatypeEnum = enum.StrEnum( "DatatypeEnum", diff --git a/app/presentation/adminapi/records.py b/app/adminapi/presentation/records.py similarity index 100% rename from app/presentation/adminapi/records.py rename to app/adminapi/presentation/records.py diff --git a/app/presentation/adminapi/server.py b/app/adminapi/presentation/server.py similarity index 99% rename from app/presentation/adminapi/server.py rename to app/adminapi/presentation/server.py index 6c1588e6..8fdc0729 100644 --- a/app/presentation/adminapi/server.py +++ b/app/adminapi/presentation/server.py @@ -4,10 +4,10 @@ import fastapi import structlog +from app.adminapi.presentation import interface from app.lib import audit, auth from app.lib.web import server from app.lib.web.middlewares import identity_from_request -from app.presentation.adminapi import interface def _logout_token(request: fastapi.Request) -> str: diff --git a/app/cli.py b/app/cli.py index ae67b7b2..642defe4 100644 --- a/app/cli.py +++ b/app/cli.py @@ -1,10 +1,8 @@ -import os - import click -from app.commands.adminapi import AdminAPICommand -from app.commands.dataapi import DataAPICommand -from app.lib import commands +from app.adminapi.cli import adminapi +from app.dataapi.cli import dataapi +from app.tasks.cli import serve_tasks @click.group() @@ -12,36 +10,6 @@ def cli() -> None: pass -@cli.command(short_help=AdminAPICommand.help()) -@click.option( - "-c", - "--config", - type=str, - default=lambda: os.environ.get("CONFIG", ""), - help="Path to configuration file", -) -def adminapi(config: str): - commands.run(AdminAPICommand(config)) - - -@cli.command(short_help=DataAPICommand.help()) -@click.option( - "-c", - "--config", - type=str, - default=lambda: os.environ.get("CONFIG", ""), - help="Path to configuration file", -) -def dataapi(config: str): - commands.run(DataAPICommand(config)) - - -@cli.command( - "serve-tasks", - short_help="Registers layer2 import Prefect deployments and serves them.", -) -def serve_tasks() -> None: - # Lazy-load: ServeTasksCommand pulls in Prefect, which slows other commands - from app.commands.serve_tasks import ServeTasksCommand # noqa: PLC0415 - - commands.run(ServeTasksCommand()) +cli.add_command(adminapi) +cli.add_command(dataapi) +cli.add_command(serve_tasks) diff --git a/app/commands/__init__.py b/app/commands/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/app/commands/adminapi/__init__.py b/app/commands/adminapi/__init__.py deleted file mode 100644 index 93b2cf8a..00000000 --- a/app/commands/adminapi/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from app.commands.adminapi.command import AdminAPICommand, Config - -__all__ = [ - "AdminAPICommand", - "Config", -] diff --git a/app/commands/dataapi/__init__.py b/app/commands/dataapi/__init__.py deleted file mode 100644 index 0c7bebbb..00000000 --- a/app/commands/dataapi/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from app.commands.dataapi.command import Config, DataAPICommand - -__all__ = ["DataAPICommand", "Config"] diff --git a/app/commands/serve_tasks/__init__.py b/app/commands/serve_tasks/__init__.py deleted file mode 100644 index 386467cd..00000000 --- a/app/commands/serve_tasks/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from app.commands.serve_tasks.command import ServeTasksCommand - -__all__ = ["ServeTasksCommand"] diff --git a/app/dataapi/cli.py b/app/dataapi/cli.py new file mode 100644 index 00000000..f384b9c4 --- /dev/null +++ b/app/dataapi/cli.py @@ -0,0 +1,18 @@ +import os + +import click + +from app.dataapi.command import DataAPICommand +from app.lib import commands + + +@click.command(short_help=DataAPICommand.help()) +@click.option( + "-c", + "--config", + type=str, + default=lambda: os.environ.get("CONFIG", ""), + help="Path to configuration file", +) +def dataapi(config: str) -> None: + commands.run(DataAPICommand(config)) diff --git a/app/commands/dataapi/command.py b/app/dataapi/command.py similarity index 95% rename from app/commands/dataapi/command.py rename to app/dataapi/command.py index ac31335d..e657ce69 100644 --- a/app/commands/dataapi/command.py +++ b/app/dataapi/command.py @@ -7,13 +7,11 @@ import yaml from app.data import repositories -from app.domain import dataapi as domain -from app.domain import responders +from app.dataapi import domain, presentation, responders from app.lib import auth, commands, config, tracing from app.lib.storage import postgres from app.lib.tracing import TracingConfig from app.lib.web import server -from app.presentation import dataapi as presentation log: structlog.stdlib.BoundLogger = structlog.get_logger() diff --git a/app/dataapi/domain/__init__.py b/app/dataapi/domain/__init__.py new file mode 100644 index 00000000..ee480c48 --- /dev/null +++ b/app/dataapi/domain/__init__.py @@ -0,0 +1,5 @@ +from app.dataapi.domain.actions import Actions + +__all__ = [ + "Actions", +] diff --git a/app/domain/dataapi/actions.py b/app/dataapi/domain/actions.py similarity index 96% rename from app/domain/dataapi/actions.py rename to app/dataapi/domain/actions.py index 9d4b6a7e..c75ba8c5 100644 --- a/app/domain/dataapi/actions.py +++ b/app/dataapi/domain/actions.py @@ -1,9 +1,9 @@ from typing import final from app.data import model, repositories -from app.domain import responders -from app.domain.dataapi import parameterized_query, tap_types -from app.presentation import dataapi +from app.dataapi import presentation as dataapi +from app.dataapi import responders +from app.dataapi.domain import parameterized_query, tap_types ENABLED_CATALOGS = [ model.RawCatalog.DESIGNATION, diff --git a/app/domain/dataapi/parameterized_query.py b/app/dataapi/domain/parameterized_query.py similarity index 97% rename from app/domain/dataapi/parameterized_query.py rename to app/dataapi/domain/parameterized_query.py index f93a9a04..1225c85f 100644 --- a/app/domain/dataapi/parameterized_query.py +++ b/app/dataapi/domain/parameterized_query.py @@ -1,7 +1,7 @@ from app.data import model, repositories from app.data.repositories import layer2 -from app.domain import responders -from app.presentation import dataapi +from app.dataapi import presentation as dataapi +from app.dataapi import responders CATALOGS_FOR_PGC_QUERY = [ model.RawCatalog.DESIGNATION, diff --git a/app/domain/dataapi/tap_types.py b/app/dataapi/domain/tap_types.py similarity index 100% rename from app/domain/dataapi/tap_types.py rename to app/dataapi/domain/tap_types.py diff --git a/app/dataapi/presentation/__init__.py b/app/dataapi/presentation/__init__.py new file mode 100644 index 00000000..90df24e4 --- /dev/null +++ b/app/dataapi/presentation/__init__.py @@ -0,0 +1,5 @@ +from app.dataapi.presentation.interface import * +from app.dataapi.presentation.server import Server +from app.dataapi.presentation.tap import * + +__all__ = ["Server"] diff --git a/app/presentation/dataapi/interface.py b/app/dataapi/presentation/interface.py similarity index 99% rename from app/presentation/dataapi/interface.py rename to app/dataapi/presentation/interface.py index 7cf67170..f86216b3 100644 --- a/app/presentation/dataapi/interface.py +++ b/app/dataapi/presentation/interface.py @@ -2,7 +2,7 @@ import pydantic -from app.presentation.dataapi import tap +from app.dataapi.presentation import tap class EquatorialCoordinates(pydantic.BaseModel): diff --git a/app/presentation/dataapi/server.py b/app/dataapi/presentation/server.py similarity index 98% rename from app/presentation/dataapi/server.py rename to app/dataapi/presentation/server.py index 6388f071..5453556b 100644 --- a/app/presentation/dataapi/server.py +++ b/app/dataapi/presentation/server.py @@ -4,9 +4,9 @@ import fastapi import structlog +from app.dataapi.presentation import interface, tap from app.lib import auth from app.lib.web import server -from app.presentation.dataapi import interface, tap logger = structlog.stdlib.get_logger() diff --git a/app/presentation/dataapi/tap.py b/app/dataapi/presentation/tap.py similarity index 100% rename from app/presentation/dataapi/tap.py rename to app/dataapi/presentation/tap.py diff --git a/app/dataapi/responders/__init__.py b/app/dataapi/responders/__init__.py new file mode 100644 index 00000000..3a581002 --- /dev/null +++ b/app/dataapi/responders/__init__.py @@ -0,0 +1,3 @@ +from app.dataapi.responders.structured_responder import CatalogConfig, StructuredResponder + +__all__ = ["StructuredResponder", "CatalogConfig"] diff --git a/app/domain/responders/interface.py b/app/dataapi/responders/interface.py similarity index 100% rename from app/domain/responders/interface.py rename to app/dataapi/responders/interface.py diff --git a/app/domain/responders/structured_responder.py b/app/dataapi/responders/structured_responder.py similarity index 99% rename from app/domain/responders/structured_responder.py rename to app/dataapi/responders/structured_responder.py index 4d3804c1..b45a4efc 100644 --- a/app/domain/responders/structured_responder.py +++ b/app/dataapi/responders/structured_responder.py @@ -5,9 +5,9 @@ from app.data import model from app.data.model import layer2 -from app.domain.responders import interface +from app.dataapi import presentation as dataapi +from app.dataapi.responders import interface from app.lib import astronomy, config -from app.presentation import dataapi DATA_SCHEMA = dataapi.Schema( units=dataapi.Units( diff --git a/app/domain/__init__.py b/app/domain/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/app/domain/adminapi/__init__.py b/app/domain/adminapi/__init__.py deleted file mode 100644 index fbc924c2..00000000 --- a/app/domain/adminapi/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -from app.domain.adminapi.actions import Actions -from app.domain.adminapi.crossmatch import CrossmatchManager -from app.domain.adminapi.login import LoginManager -from app.domain.adminapi.mock import get_mock_actions -from app.domain.adminapi.sources import SourceManager -from app.domain.adminapi.table_upload import TableUploadManager - -__all__ = [ - "Actions", - "CrossmatchManager", - "get_mock_actions", - "LoginManager", - "TableUploadManager", - "SourceManager", -] diff --git a/app/domain/dataapi/__init__.py b/app/domain/dataapi/__init__.py deleted file mode 100644 index 53ff2b5d..00000000 --- a/app/domain/dataapi/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from app.domain.dataapi.actions import Actions - -__all__ = [ - "Actions", -] diff --git a/app/domain/responders/__init__.py b/app/domain/responders/__init__.py deleted file mode 100644 index a655b93c..00000000 --- a/app/domain/responders/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from app.domain.responders.structured_responder import CatalogConfig, StructuredResponder - -__all__ = ["StructuredResponder", "CatalogConfig"] diff --git a/app/presentation/__init__.py b/app/presentation/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/app/presentation/adminapi/__init__.py b/app/presentation/adminapi/__init__.py deleted file mode 100644 index 9a79c65e..00000000 --- a/app/presentation/adminapi/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from app.presentation.adminapi.interface import * -from app.presentation.adminapi.records import * -from app.presentation.adminapi.server import Server - -__all__ = ["Server"] diff --git a/app/presentation/dataapi/__init__.py b/app/presentation/dataapi/__init__.py deleted file mode 100644 index db77530e..00000000 --- a/app/presentation/dataapi/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from app.presentation.dataapi.interface import * -from app.presentation.dataapi.server import Server -from app.presentation.dataapi.tap import * - -__all__ = ["Server"] diff --git a/app/tasks/cli.py b/app/tasks/cli.py new file mode 100644 index 00000000..fe0f9afe --- /dev/null +++ b/app/tasks/cli.py @@ -0,0 +1,14 @@ +import click + +from app.lib import commands + + +@click.command( + "serve-tasks", + short_help="Registers layer2 import Prefect deployments and serves them.", +) +def serve_tasks() -> None: + # Lazy-load: ServeTasksCommand pulls in Prefect, which slows other commands + from app.tasks.command import ServeTasksCommand # noqa: PLC0415 + + commands.run(ServeTasksCommand()) diff --git a/app/commands/serve_tasks/command.py b/app/tasks/command.py similarity index 93% rename from app/commands/serve_tasks/command.py rename to app/tasks/command.py index 112683ab..858a87d4 100644 --- a/app/commands/serve_tasks/command.py +++ b/app/tasks/command.py @@ -2,8 +2,8 @@ from prefect import serve -from app.commands.serve_tasks import flows from app.lib import commands +from app.tasks import flows @final diff --git a/app/commands/serve_tasks/flows.py b/app/tasks/flows.py similarity index 100% rename from app/commands/serve_tasks/flows.py rename to app/tasks/flows.py diff --git a/tests/env_test.py b/tests/env_test.py index 1e67ef58..ca9b9966 100644 --- a/tests/env_test.py +++ b/tests/env_test.py @@ -2,8 +2,8 @@ from parameterized import parameterized -import app.commands.adminapi.command as adminapi -import app.commands.dataapi.command as dataapi +import app.adminapi.command as adminapi +import app.dataapi.command as dataapi class TestEnvironment(unittest.TestCase): diff --git a/tests/integration/assign_record_pgcs_test.py b/tests/integration/assign_record_pgcs_test.py index 5debc151..b0c86376 100644 --- a/tests/integration/assign_record_pgcs_test.py +++ b/tests/integration/assign_record_pgcs_test.py @@ -3,11 +3,11 @@ import structlog +from app.adminapi import presentation as adminapi +from app.adminapi.domain import crossmatch from app.data import model, repositories -from app.domain.adminapi import crossmatch from app.lib.storage import enums from app.lib.web import errors -from app.presentation import adminapi from tests import lib diff --git a/tests/integration/catalogs_api_test.py b/tests/integration/catalogs_api_test.py index f9c28dbd..e21543f6 100644 --- a/tests/integration/catalogs_api_test.py +++ b/tests/integration/catalogs_api_test.py @@ -3,12 +3,12 @@ import structlog from starlette import testclient +from app.adminapi import domain +from app.adminapi.domain.mock import get_mock_table_stats_cache +from app.adminapi.presentation.server import Server from app.data import repositories -from app.domain import adminapi as domain -from app.domain.adminapi.mock import get_mock_table_stats_cache from app.lib import audit, auth, clients from app.lib.web import server -from app.presentation.adminapi.server import Server from tests import lib diff --git a/tests/integration/create_table_test.py b/tests/integration/create_table_test.py index 7ac2d2b3..226293e1 100644 --- a/tests/integration/create_table_test.py +++ b/tests/integration/create_table_test.py @@ -2,12 +2,11 @@ import structlog +from app.adminapi import domain, presentation +from app.adminapi.domain.mock import get_mock_table_stats_cache from app.data import repositories -from app.domain import adminapi as domain -from app.domain.adminapi.mock import get_mock_table_stats_cache from app.lib import clients from app.lib.storage import enums -from app.presentation import adminapi as presentation from tests import lib diff --git a/tests/integration/merge_pgcs_test.py b/tests/integration/merge_pgcs_test.py index 17bf3252..5d7d654e 100644 --- a/tests/integration/merge_pgcs_test.py +++ b/tests/integration/merge_pgcs_test.py @@ -4,10 +4,10 @@ import pydantic import structlog +from app.adminapi import presentation as adminapi +from app.adminapi.domain import pgc from app.data import model, repositories -from app.domain.adminapi import pgc from app.lib.web import errors -from app.presentation import adminapi from tests import lib diff --git a/tests/integration/metadata_api_test.py b/tests/integration/metadata_api_test.py index a75ccdc6..11604fa1 100644 --- a/tests/integration/metadata_api_test.py +++ b/tests/integration/metadata_api_test.py @@ -5,12 +5,12 @@ import structlog from starlette import testclient -from app.commands.dataapi import command as dataapi_command +import app.dataapi.command as dataapi_command from app.data import repositories -from app.domain import dataapi as domain -from app.domain.dataapi import actions as dataapi_actions +from app.dataapi import domain +from app.dataapi.domain import actions as dataapi_actions +from app.dataapi.presentation.server import Server from app.lib import auth -from app.presentation.dataapi.server import Server from tests import lib diff --git a/tests/integration/rawdata_table_test.py b/tests/integration/rawdata_table_test.py index a65ff10d..5fb58f68 100644 --- a/tests/integration/rawdata_table_test.py +++ b/tests/integration/rawdata_table_test.py @@ -5,13 +5,12 @@ import structlog from pandas import DataFrame +from app.adminapi import domain, presentation +from app.adminapi.domain.mock import get_mock_table_stats_cache from app.data import model, repositories -from app.domain import adminapi as domain -from app.domain.adminapi.mock import get_mock_table_stats_cache from app.lib import clients from app.lib.storage import enums from app.lib.storage.mapping import TYPE_INTEGER, TYPE_TEXT -from app.presentation import adminapi as presentation from tests import lib diff --git a/tests/regression/patch_table_metadata.py b/tests/regression/patch_table_metadata.py index 3bb16b7d..f7f50a7f 100644 --- a/tests/regression/patch_table_metadata.py +++ b/tests/regression/patch_table_metadata.py @@ -3,8 +3,8 @@ import requests +from app.adminapi import presentation as adminapi from app.lib.storage import enums -from app.presentation import adminapi from tests import lib TABLE_DESCRIPTION_NEW = "Regression: patched table description" diff --git a/tests/regression/upload_simple_table.py b/tests/regression/upload_simple_table.py index 89b0b6df..3b29b6f3 100644 --- a/tests/regression/upload_simple_table.py +++ b/tests/regression/upload_simple_table.py @@ -9,8 +9,8 @@ import structlog from app import tasks +from app.adminapi import presentation as adminapi from app.lib.storage import enums -from app.presentation import adminapi from tests import lib random.seed(time.time()) diff --git a/tests/unit/commands/serve_tasks_test.py b/tests/unit/commands/serve_tasks_test.py index d901dd81..6f051297 100644 --- a/tests/unit/commands/serve_tasks_test.py +++ b/tests/unit/commands/serve_tasks_test.py @@ -1,6 +1,6 @@ import unittest -from app.commands.serve_tasks import flows +from app.tasks import flows class ServeTasksTest(unittest.TestCase): diff --git a/tests/unit/domain/create_source_test.py b/tests/unit/domain/create_source_test.py index 5aeb8498..020f0676 100644 --- a/tests/unit/domain/create_source_test.py +++ b/tests/unit/domain/create_source_test.py @@ -1,6 +1,6 @@ import unittest -from app.domain.adminapi.sources import construct_code +from app.adminapi.domain.sources import construct_code class ConstructCodeTest(unittest.TestCase): diff --git a/tests/unit/domain/parameterized_query_test.py b/tests/unit/domain/parameterized_query_test.py index 377f2dc0..d59ce28d 100644 --- a/tests/unit/domain/parameterized_query_test.py +++ b/tests/unit/domain/parameterized_query_test.py @@ -1,7 +1,7 @@ import unittest from app.data import model -from app.domain.dataapi import parameterized_query +from app.dataapi.domain import parameterized_query DEFAULT = [ model.RawCatalog.DESIGNATION, diff --git a/tests/unit/domain/table_stats_test.py b/tests/unit/domain/table_stats_test.py index e8b769c7..24432ba0 100644 --- a/tests/unit/domain/table_stats_test.py +++ b/tests/unit/domain/table_stats_test.py @@ -1,8 +1,8 @@ import unittest from unittest import mock +from app.adminapi.domain import table_stats from app.data import model -from app.domain.adminapi import table_stats class TableStatsTest(unittest.TestCase): diff --git a/tests/unit/domain/table_upload_test.py b/tests/unit/domain/table_upload_test.py index 8798c508..c01a4c64 100644 --- a/tests/unit/domain/table_upload_test.py +++ b/tests/unit/domain/table_upload_test.py @@ -5,14 +5,13 @@ from astropy import units from parameterized import param, parameterized +from app.adminapi import domain, presentation +from app.adminapi.domain.mock import get_mock_table_stats_cache +from app.adminapi.domain.table_upload import domain_descriptions_to_data, get_source_id from app.data import model, repositories -from app.domain import adminapi as domain -from app.domain.adminapi.mock import get_mock_table_stats_cache -from app.domain.adminapi.table_upload import domain_descriptions_to_data, get_source_id from app.lib import clients from app.lib.storage import enums, mapping from app.lib.web import errors -from app.presentation import adminapi as presentation from tests import lib From eac76dff5752e85c81d4c17578e7990eb0a9b77f Mon Sep 17 00:00:00 2001 From: kraysent Date: Mon, 27 Jul 2026 00:17:35 +0100 Subject: [PATCH 2/9] fix entrypoints --- .github/workflows/ci.yaml | 6 ++++- app/adminapi/cli.py | 2 +- app/cli.py | 12 +++++----- app/dataapi/cli.py | 2 +- app/tasks/cli.py | 11 +++------ makefile | 6 ++--- pyproject.toml | 6 +++-- tests/integration/adminapi_auth_test.py | 1 - tests/integration/adminapi_server_test.py | 1 - tests/integration/dataapi_server_test.py | 1 - uv.lock | 28 ----------------------- 11 files changed, 23 insertions(+), 53 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 99cf9bed..b4a01f71 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -41,7 +41,11 @@ jobs: tags: local/app:latest - name: Run image help - run: docker run --rm --entrypoint "app" local/app:latest --help + run: | + docker run --rm --entrypoint "app" local/app:latest --help + docker run --rm --entrypoint "adminapi" local/app:latest --help + docker run --rm --entrypoint "dataapi" local/app:latest --help + docker run --rm --entrypoint "tasks" local/app:latest --help - name: Save image run: docker save local/app:latest -o image.tar diff --git a/app/adminapi/cli.py b/app/adminapi/cli.py index cb8cd092..9ac8c384 100644 --- a/app/adminapi/cli.py +++ b/app/adminapi/cli.py @@ -14,5 +14,5 @@ default=lambda: os.environ.get("CONFIG", ""), help="Path to configuration file", ) -def adminapi(config: str) -> None: +def main(config: str) -> None: commands.run(AdminAPICommand(config)) diff --git a/app/cli.py b/app/cli.py index 642defe4..4d849cc7 100644 --- a/app/cli.py +++ b/app/cli.py @@ -1,8 +1,8 @@ import click -from app.adminapi.cli import adminapi -from app.dataapi.cli import dataapi -from app.tasks.cli import serve_tasks +from app.adminapi.cli import main as adminapi +from app.dataapi.cli import main as dataapi +from app.tasks.cli import main as serve_tasks @click.group() @@ -10,6 +10,6 @@ def cli() -> None: pass -cli.add_command(adminapi) -cli.add_command(dataapi) -cli.add_command(serve_tasks) +cli.add_command(adminapi, name="adminapi") +cli.add_command(dataapi, name="dataapi") +cli.add_command(serve_tasks, name="serve-tasks") diff --git a/app/dataapi/cli.py b/app/dataapi/cli.py index f384b9c4..b0ceee28 100644 --- a/app/dataapi/cli.py +++ b/app/dataapi/cli.py @@ -14,5 +14,5 @@ default=lambda: os.environ.get("CONFIG", ""), help="Path to configuration file", ) -def dataapi(config: str) -> None: +def main(config: str) -> None: commands.run(DataAPICommand(config)) diff --git a/app/tasks/cli.py b/app/tasks/cli.py index fe0f9afe..0bce3f2a 100644 --- a/app/tasks/cli.py +++ b/app/tasks/cli.py @@ -1,14 +1,9 @@ import click from app.lib import commands +from app.tasks.command import ServeTasksCommand -@click.command( - "serve-tasks", - short_help="Registers layer2 import Prefect deployments and serves them.", -) -def serve_tasks() -> None: - # Lazy-load: ServeTasksCommand pulls in Prefect, which slows other commands - from app.tasks.command import ServeTasksCommand # noqa: PLC0415 - +@click.command(short_help="Registers layer2 import Prefect deployments and serves them.") +def main() -> None: commands.run(ServeTasksCommand()) diff --git a/makefile b/makefile index 85c73db9..d595121c 100644 --- a/makefile +++ b/makefile @@ -116,13 +116,13 @@ update-template: ## General targets adminapi: - uv run app adminapi -c configs/dev/adminapi.yaml + uv run adminapi -c configs/dev/adminapi.yaml adminapi-dev: set -a && source .env.local && set +a && make adminapi dataapi: - uv run app dataapi -c configs/dev/dataapi.yaml + uv run dataapi -c configs/dev/dataapi.yaml start-db: docker-compose up -d @@ -138,7 +138,7 @@ start-prefect: uv run prefect server start start-tasks: - uv run app serve-tasks + uv run tasks docs: uvx \ diff --git a/pyproject.toml b/pyproject.toml index a99fc2e6..a02db8c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,6 @@ dependencies = [ "pyyaml>=6.0.2", "requests>=2.32.3", "regex>=2024.11.6", - "rq>=2.1.0", "structlog>=25.1.0", "swagger-ui-py>=23.9.23", "fastapi>=0.116.1", @@ -35,7 +34,10 @@ dependencies = [ ] [project.scripts] -"app" = "app.cli:cli" +app = "app.cli:cli" +adminapi = "app.adminapi.cli:main" +dataapi = "app.dataapi.cli:main" +tasks = "app.tasks.cli:main" [build-system] requires = ["hatchling"] diff --git a/tests/integration/adminapi_auth_test.py b/tests/integration/adminapi_auth_test.py index 4e594eda..8f49fb31 100644 --- a/tests/integration/adminapi_auth_test.py +++ b/tests/integration/adminapi_auth_test.py @@ -41,7 +41,6 @@ def setUpClass(cls) -> None: [ "uv", "run", - "app", "adminapi", "-c", "configs/dev/adminapi.yaml", diff --git a/tests/integration/adminapi_server_test.py b/tests/integration/adminapi_server_test.py index 50e58e28..bf6beac8 100644 --- a/tests/integration/adminapi_server_test.py +++ b/tests/integration/adminapi_server_test.py @@ -45,7 +45,6 @@ def setUpClass(cls) -> None: [ "uv", "run", - "app", "adminapi", "-c", "configs/dev/adminapi.yaml", diff --git a/tests/integration/dataapi_server_test.py b/tests/integration/dataapi_server_test.py index ebafe27c..a2d7911a 100644 --- a/tests/integration/dataapi_server_test.py +++ b/tests/integration/dataapi_server_test.py @@ -51,7 +51,6 @@ def setUpClass(cls) -> None: [ "uv", "run", - "app", "dataapi", "-c", "configs/dev/dataapi.yaml", diff --git a/uv.lock b/uv.lock index 856b8e80..73a15220 100644 --- a/uv.lock +++ b/uv.lock @@ -94,7 +94,6 @@ dependencies = [ { name = "pyyaml" }, { name = "regex" }, { name = "requests" }, - { name = "rq" }, { name = "slowapi" }, { name = "starlette" }, { name = "structlog" }, @@ -143,7 +142,6 @@ requires-dist = [ { name = "pyyaml", specifier = ">=6.0.2" }, { name = "regex", specifier = ">=2024.11.6" }, { name = "requests", specifier = ">=2.32.3" }, - { name = "rq", specifier = ">=2.1.0" }, { name = "slowapi", specifier = ">=0.1.9" }, { name = "starlette", specifier = ">=0.47.2" }, { name = "structlog", specifier = ">=25.1.0" }, @@ -522,18 +520,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/97/35/cf824e535233c432048bd7455b28808776471ab5861375772b2c98ea4cbd/coolname-5.0.0-py3-none-any.whl", hash = "sha256:b86eea9670aa0620965167d1bfadfe654fabec839a5b51e8da611c4a64f86192", size = 47368, upload-time = "2026-04-23T06:04:49.374Z" }, ] -[[package]] -name = "croniter" -version = "6.2.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "python-dateutil" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/df/de/5832661ed55107b8a09af3f0a2e71e0957226a59eb1dcf0a445cce6daf20/croniter-6.2.2.tar.gz", hash = "sha256:ba60832a5ec8e12e51b8691c3309a113d1cf6526bdf1a48150ce8ec7a532d0ab", size = 113762, upload-time = "2026-03-15T08:43:48.112Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d0/39/783980e78cb92c2d7bdb1fc7dbc86e94ccc6d58224d76a7f1f51b6c51e30/croniter-6.2.2-py3-none-any.whl", hash = "sha256:a5d17b1060974d36251ea4faf388233eca8acf0d09cbd92d35f4c4ac8f279960", size = 45422, upload-time = "2026-03-15T08:43:46.626Z" }, -] - [[package]] name = "cronsim" version = "2.7" @@ -2217,20 +2203,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/27/17/b8642c12930b71bc2b25831f6708ccf0f75abcd11883932ec9ce54ba3a78/rpds_py-2026.6.3-cp313-cp313-win_arm64.whl", hash = "sha256:900a67df3fd1660b035a4761c4ce73c382ea6b35f90f9863c36c6fd8bf8b09bb", size = 215063, upload-time = "2026-06-30T07:15:56.573Z" }, ] -[[package]] -name = "rq" -version = "2.9.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "croniter" }, - { name = "redis" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/15/5e/43a7a61f3ebfa79789c72bf442a47fd80bb1a743caeea47b2b833001f388/rq-2.9.0.tar.gz", hash = "sha256:db5dfc1e1fe80ef977fd557d4305107dcf99a80b33d381c9a06e8a2bb11730e5", size = 744959, upload-time = "2026-05-19T15:00:29.894Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4d/a9/aa38ae2505e5dcb1e898f83a263e703b05829580f75ee86c5e480760ae1a/rq-2.9.0-py3-none-any.whl", hash = "sha256:665b9ad34e36ea15913e60d2a32e1775fef1e9aff907bcae297d6f70dccffed2", size = 120113, upload-time = "2026-05-19T15:00:27.511Z" }, -] - [[package]] name = "ruamel-yaml" version = "0.19.1" From 739f2f90e43d3f3e0d457cc30200d68c2058d8da Mon Sep 17 00:00:00 2001 From: kraysent Date: Mon, 27 Jul 2026 00:21:42 +0100 Subject: [PATCH 3/9] add import-linter --- makefile | 3 +++ pyproject.toml | 24 ++++++++++++++++++++++++ uv.lock | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+) diff --git a/makefile b/makefile index d595121c..30861906 100644 --- a/makefile +++ b/makefile @@ -63,6 +63,9 @@ check: --config=pyproject.toml @echo "Linter ok." + @uv run lint-imports + @echo "Import contracts ok." + @output=$$(uv run basedpyright 2>&1); exit_code=$$?; \ if [ $$exit_code -ne 0 ]; then echo "$$output"; fi; \ exit $$exit_code diff --git a/pyproject.toml b/pyproject.toml index a02db8c0..00ac521d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,6 +64,7 @@ dev = [ "pandas-stubs>=2.3.2.250827", "parameterized>=0.9.0", "testcontainers-postgres>=0.0.1rc1", + "import-linter>=2.0", ] [tool.pytest.ini_options] @@ -188,3 +189,26 @@ ignore_names = [ "dotenv_settings", "file_secret_settings", ] + +[tool.importlinter] +root_package = "app" + +[[tool.importlinter.contracts]] +id = "services-independent" +name = "Services must not import each other" +type = "independence" +modules = [ + "app.adminapi", + "app.dataapi", + "app.tasks", +] + +[[tool.importlinter.contracts]] +id = "layers" +name = "Layered architecture" +type = "layers" +layers = [ + "app.adminapi | app.dataapi | app.tasks", + "app.data", + "app.lib", +] diff --git a/uv.lock b/uv.lock index 73a15220..f0fdb5ce 100644 --- a/uv.lock +++ b/uv.lock @@ -107,6 +107,7 @@ dev = [ { name = "basedpyright" }, { name = "datamodel-code-generator" }, { name = "fabric" }, + { name = "import-linter" }, { name = "pandas-stubs" }, { name = "parameterized" }, { name = "pymdown-extensions" }, @@ -155,6 +156,7 @@ dev = [ { name = "basedpyright", specifier = "~=1.38.3" }, { name = "datamodel-code-generator", specifier = ">=0.26.5" }, { name = "fabric", specifier = ">=3.2.2" }, + { name = "import-linter", specifier = ">=2.0" }, { name = "pandas-stubs", specifier = ">=2.3.2.250827" }, { name = "parameterized", specifier = ">=0.9.0" }, { name = "pymdown-extensions", specifier = ">=10.14.1" }, @@ -792,6 +794,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e4/d3/5268aeabf2ad82658c4e2ff3a060648d0f02f3926cb53247c0e4d0dab49e/griffelib-2.1.0-py3-none-any.whl", hash = "sha256:cc7b3d2d2865ad0b909fcc38086e3f554b5ea7acbaa7bbb7ecaa3f5dfb7d9f00", size = 142560, upload-time = "2026-06-19T12:05:38.742Z" }, ] +[[package]] +name = "grimp" +version = "3.15" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b2/73/ce58881177b003def779c87b5e10f396deef068933c97d6d206bd46d4cb7/grimp-3.15.tar.gz", hash = "sha256:91b57d4d801dc107ebfb5a7040d4777a152c579b5dc202426e1185e50931fe1e", size = 831734, upload-time = "2026-07-03T12:09:36.244Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/6a/3e0a0760cc509cd09764d128de78a1f329740306c74e42dba82c58a99118/grimp-3.15-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:f19b957053d1c736aaa0015eed2d4855bb2511637c5360d32b3c5ea045904e7a", size = 2143197, upload-time = "2026-07-03T12:08:51.685Z" }, + { url = "https://files.pythonhosted.org/packages/c8/2e/127cbce04a5603d382c6a2bbc1a19b6889be49d60b88864bcdb174c8926d/grimp-3.15-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a6480472c2d1f7f6a903906c92e9897d4e3dee5d69ce3881f04368d4ec6d2dde", size = 2088342, upload-time = "2026-07-03T12:08:44.227Z" }, + { url = "https://files.pythonhosted.org/packages/b0/36/af9df683bf6c8711e0e9136876ca130f9971102d945ff3a36d0c45dae2ec/grimp-3.15-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c14b64dbf2e4397df2e35fa8aa7028533621ecf1f8ea7ec24bc296a9c695ea4", size = 2254509, upload-time = "2026-07-03T12:07:35.809Z" }, + { url = "https://files.pythonhosted.org/packages/02/f5/e712633b68ea14d04e7de84ede4f8ddbba763d5f2d29ae8d6af721f84870/grimp-3.15-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:26364c2c9f7db88243365299b4d1c4d948b74304ff03c8a6e4f028147fed3b22", size = 2193831, upload-time = "2026-07-03T12:07:46.309Z" }, + { url = "https://files.pythonhosted.org/packages/13/74/151bdd73d6a60bd77d4b956f36d03dd558dd46a9b5ec8f5ad15921b503d7/grimp-3.15-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:69331e1be693415596c6084342059b9ba3ecf1cfe2e3b1c761598dd2fe14d522", size = 2345289, upload-time = "2026-07-03T12:08:19.458Z" }, + { url = "https://files.pythonhosted.org/packages/a1/b8/3fc950fa73b757cbc35f77542bd662f431b9a8f360e63196ded640771a33/grimp-3.15-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fe397991c269868c2fb08114099b2aa4f1bc803d03fadaaf97e006019f9e5da2", size = 2604407, upload-time = "2026-07-03T12:07:57.217Z" }, + { url = "https://files.pythonhosted.org/packages/0a/d8/98916b9dc0b89a3e89e0d714ce0872be859fff40ceee3e2cb6886b106eb4/grimp-3.15-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1d556bd62664ee044c47cff64d737be132408064d4ba68ca9f756cb29b41cc2d", size = 2326769, upload-time = "2026-07-03T12:08:08.773Z" }, + { url = "https://files.pythonhosted.org/packages/63/51/39595c5857f609e976e0bba1f19c1f45182ee4d8d2ea5cdfab72841eafbc/grimp-3.15-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9061c5b6f01130ff8639c49c80176d29d921acc36741b2ae0b763a7668106082", size = 2272498, upload-time = "2026-07-03T12:08:32.03Z" }, + { url = "https://files.pythonhosted.org/packages/7d/d8/a44cc9db500ae80c45425238ee44d9556796aa88b8c0649cfa613fb88d14/grimp-3.15-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a9d04c195f0c6da4476361560d3d206a6a784b9eb0da7156fc6974511337e2e3", size = 2431075, upload-time = "2026-07-03T12:08:58.935Z" }, + { url = "https://files.pythonhosted.org/packages/a9/41/544f197ddb44990789683c25740ffa72474a57f0b16bc6b4a544edf9a2a9/grimp-3.15-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:01fd68c74cfd08b110bfd338d77efaa470670530f7179e6977764f44cd74d5cc", size = 2467361, upload-time = "2026-07-03T12:09:09.275Z" }, + { url = "https://files.pythonhosted.org/packages/3b/b1/8261018b9f1ab47fffbb7739d7af3f04c8b529555b7fb2ae8b742d42d3db/grimp-3.15-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:bd1ddd428a124d6730bed49ea60fb2ca6c8e0640c8f5abe1d0f6fc27a92fd8bc", size = 2503500, upload-time = "2026-07-03T12:09:19.585Z" }, + { url = "https://files.pythonhosted.org/packages/db/d6/99a2421c4c0de9f203a7e246030b13b676766e62e48504883863942646fb/grimp-3.15-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:07e645e9d45ed43bb8ea8da5982c19eacf13ee685d8440e3dc280064c005599c", size = 2513834, upload-time = "2026-07-03T12:09:29.758Z" }, + { url = "https://files.pythonhosted.org/packages/62/a5/263729e23d64541cd99d36dbb592e29c1ecea803deb3bb5c0c463b43c2ec/grimp-3.15-cp313-cp313-win32.whl", hash = "sha256:473646e0a74a554b4ab071d7fcbf5f442eb8cf87561770dae269818636b8edf2", size = 1855743, upload-time = "2026-07-03T12:09:49.789Z" }, + { url = "https://files.pythonhosted.org/packages/1c/8c/a072bbea2e2e94da38f90bd8794037c90fb4eba389b49d01cdd2bb85e13c/grimp-3.15-cp313-cp313-win_amd64.whl", hash = "sha256:dbc2c15a1fbca2ff358f86cc90067176096dd73bec27d002515521b3125ba507", size = 1984546, upload-time = "2026-07-03T12:09:42.543Z" }, +] + [[package]] name = "grpcio" version = "1.80.0" @@ -917,6 +941,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/94/16/70255075a9859a0e3adb789b68ceb0e210dec03934245fd98d248226572f/idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5", size = 74165, upload-time = "2026-05-22T00:16:16.698Z" }, ] +[[package]] +name = "import-linter" +version = "2.13" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "grimp" }, + { name = "rich" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/97/c6/42962eb043df4d6984c1540220735b20442572fe37dab5e65ac807c939b9/import_linter-2.13.tar.gz", hash = "sha256:13af4a1d6b06044c58ea784e8732fd7fe48eec821a75feb4d6a1a2de36dd5c27", size = 1279761, upload-time = "2026-07-03T14:00:31.285Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/13/e7725e6eb32607fd4af51ccf3edfe835826e5cdf783b4d7fdc8f459196ae/import_linter-2.13-py3-none-any.whl", hash = "sha256:c0372e7ee5e15657bc06a8e841445e13237afd738a672d26863dc927af9f0bf5", size = 638185, upload-time = "2026-07-03T14:00:29.676Z" }, +] + [[package]] name = "inflect" version = "7.5.0" From be0f6879ae1038f116502c2062c2806d8923d926 Mon Sep 17 00:00:00 2001 From: kraysent Date: Mon, 27 Jul 2026 00:25:17 +0100 Subject: [PATCH 4/9] move clients to adminapi --- app/{lib => adminapi}/clients/__init__.py | 0 app/adminapi/command.py | 4 ++-- app/adminapi/domain/actions.py | 3 ++- app/adminapi/domain/mock.py | 3 ++- app/adminapi/domain/table_upload.py | 3 ++- tests/integration/catalogs_api_test.py | 4 ++-- tests/integration/create_table_test.py | 3 +-- tests/integration/rawdata_table_test.py | 3 +-- tests/unit/domain/table_upload_test.py | 3 +-- 9 files changed, 13 insertions(+), 13 deletions(-) rename app/{lib => adminapi}/clients/__init__.py (100%) diff --git a/app/lib/clients/__init__.py b/app/adminapi/clients/__init__.py similarity index 100% rename from app/lib/clients/__init__.py rename to app/adminapi/clients/__init__.py diff --git a/app/adminapi/command.py b/app/adminapi/command.py index 1ecceaf4..e29c2217 100644 --- a/app/adminapi/command.py +++ b/app/adminapi/command.py @@ -8,10 +8,10 @@ import structlog import yaml -from app.adminapi import domain, presentation +from app.adminapi import clients, domain, presentation from app.adminapi.domain import table_stats from app.data import repositories -from app.lib import audit, auth, cache, clients, commands, config, tracing +from app.lib import audit, auth, cache, commands, config, tracing from app.lib.storage import postgres from app.lib.tracing import TracingConfig from app.lib.web import server diff --git a/app/adminapi/domain/actions.py b/app/adminapi/domain/actions.py index 6e06706e..99405a22 100644 --- a/app/adminapi/domain/actions.py +++ b/app/adminapi/domain/actions.py @@ -1,9 +1,10 @@ from typing import final +from app.adminapi import clients from app.adminapi import presentation as adminapi from app.adminapi.domain import catalogs, crossmatch, layer1_write, login, pgc, sources, table_upload from app.data import repositories -from app.lib import auth, cache, clients +from app.lib import auth, cache @final diff --git a/app/adminapi/domain/mock.py b/app/adminapi/domain/mock.py index d4ae74d0..4827251e 100644 --- a/app/adminapi/domain/mock.py +++ b/app/adminapi/domain/mock.py @@ -1,9 +1,10 @@ import datetime from unittest import mock +from app.adminapi import clients from app.adminapi import presentation as adminapi from app.adminapi.domain import actions -from app.lib import auth, clients +from app.lib import auth def get_mock_table_stats_cache() -> mock.MagicMock: diff --git a/app/adminapi/domain/table_upload.py b/app/adminapi/domain/table_upload.py index c871f5b1..e315647f 100644 --- a/app/adminapi/domain/table_upload.py +++ b/app/adminapi/domain/table_upload.py @@ -11,12 +11,13 @@ from astropy import units as u from astroquery import nasa_ads as ads +from app.adminapi import clients from app.adminapi import presentation as adminapi from app.adminapi.domain import table_stats from app.data import model, repositories from app.data.repositories.common import ColumnSchemaInfo, TableSchemaInfo from app.data.repositories.layer0.common import RAWDATA_SCHEMA -from app.lib import astronomy, cache, clients, concurrency +from app.lib import astronomy, cache, concurrency from app.lib.storage import enums, mapping from app.lib.web.errors import NotFoundError, RuleValidationError diff --git a/tests/integration/catalogs_api_test.py b/tests/integration/catalogs_api_test.py index e21543f6..a3115d5b 100644 --- a/tests/integration/catalogs_api_test.py +++ b/tests/integration/catalogs_api_test.py @@ -3,11 +3,11 @@ import structlog from starlette import testclient -from app.adminapi import domain +from app.adminapi import clients, domain from app.adminapi.domain.mock import get_mock_table_stats_cache from app.adminapi.presentation.server import Server from app.data import repositories -from app.lib import audit, auth, clients +from app.lib import audit, auth from app.lib.web import server from tests import lib diff --git a/tests/integration/create_table_test.py b/tests/integration/create_table_test.py index 226293e1..36bbe4a7 100644 --- a/tests/integration/create_table_test.py +++ b/tests/integration/create_table_test.py @@ -2,10 +2,9 @@ import structlog -from app.adminapi import domain, presentation +from app.adminapi import clients, domain, presentation from app.adminapi.domain.mock import get_mock_table_stats_cache from app.data import repositories -from app.lib import clients from app.lib.storage import enums from tests import lib diff --git a/tests/integration/rawdata_table_test.py b/tests/integration/rawdata_table_test.py index 5fb58f68..6fe0819d 100644 --- a/tests/integration/rawdata_table_test.py +++ b/tests/integration/rawdata_table_test.py @@ -5,10 +5,9 @@ import structlog from pandas import DataFrame -from app.adminapi import domain, presentation +from app.adminapi import clients, domain, presentation from app.adminapi.domain.mock import get_mock_table_stats_cache from app.data import model, repositories -from app.lib import clients from app.lib.storage import enums from app.lib.storage.mapping import TYPE_INTEGER, TYPE_TEXT from tests import lib diff --git a/tests/unit/domain/table_upload_test.py b/tests/unit/domain/table_upload_test.py index c01a4c64..76821c3e 100644 --- a/tests/unit/domain/table_upload_test.py +++ b/tests/unit/domain/table_upload_test.py @@ -5,11 +5,10 @@ from astropy import units from parameterized import param, parameterized -from app.adminapi import domain, presentation +from app.adminapi import clients, domain, presentation from app.adminapi.domain.mock import get_mock_table_stats_cache from app.adminapi.domain.table_upload import domain_descriptions_to_data, get_source_id from app.data import model, repositories -from app.lib import clients from app.lib.storage import enums, mapping from app.lib.web import errors from tests import lib From 78a6ea366555ad6218403ab8457424d24c649c21 Mon Sep 17 00:00:00 2001 From: kraysent Date: Mon, 27 Jul 2026 00:26:04 +0100 Subject: [PATCH 5/9] better output --- makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/makefile b/makefile index 30861906..17380482 100644 --- a/makefile +++ b/makefile @@ -63,7 +63,9 @@ check: --config=pyproject.toml @echo "Linter ok." - @uv run lint-imports + @output=$$(uv run lint-imports 2>&1); exit_code=$$?; \ + if [ $$exit_code -ne 0 ]; then echo "$$output"; fi; \ + exit $$exit_code @echo "Import contracts ok." @output=$$(uv run basedpyright 2>&1); exit_code=$$?; \ From 28a03aad5214d7f25e421148517d361f2c9779e2 Mon Sep 17 00:00:00 2001 From: kraysent Date: Mon, 27 Jul 2026 00:29:17 +0100 Subject: [PATCH 6/9] logging module to tasks --- app/lib/logging/__init__.py | 3 --- app/tasks/layer2_import_designation.py | 4 ++-- app/tasks/layer2_import_icrs.py | 4 ++-- app/tasks/layer2_import_nature.py | 4 ++-- app/tasks/layer2_import_redshift.py | 4 ++-- app/tasks/logging/__init__.py | 3 +++ app/{lib => tasks}/logging/table.py | 0 7 files changed, 11 insertions(+), 11 deletions(-) delete mode 100644 app/lib/logging/__init__.py create mode 100644 app/tasks/logging/__init__.py rename app/{lib => tasks}/logging/table.py (100%) diff --git a/app/lib/logging/__init__.py b/app/lib/logging/__init__.py deleted file mode 100644 index 9b7951f5..00000000 --- a/app/lib/logging/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from app.lib.logging.table import print_table - -__all__ = ["print_table"] diff --git a/app/tasks/layer2_import_designation.py b/app/tasks/layer2_import_designation.py index 61e4f9b6..14634338 100644 --- a/app/tasks/layer2_import_designation.py +++ b/app/tasks/layer2_import_designation.py @@ -4,9 +4,9 @@ import structlog from app.data import model, repositories -from app.lib import containers, logging +from app.lib import containers from app.lib.storage import postgres -from app.tasks import interface +from app.tasks import interface, logging DESIGNATION_COLUMNS = ["design"] diff --git a/app/tasks/layer2_import_icrs.py b/app/tasks/layer2_import_icrs.py index 9124eb4e..772f6f5c 100644 --- a/app/tasks/layer2_import_icrs.py +++ b/app/tasks/layer2_import_icrs.py @@ -7,9 +7,9 @@ from astropy import units as u from app.data import model, repositories -from app.lib import containers, logging +from app.lib import containers from app.lib.storage import postgres -from app.tasks import interface +from app.tasks import interface, logging ICRS_COLUMNS = ["ra", "e_ra", "dec", "e_dec"] DEG = "deg" diff --git a/app/tasks/layer2_import_nature.py b/app/tasks/layer2_import_nature.py index d23ca1aa..79d2c36f 100644 --- a/app/tasks/layer2_import_nature.py +++ b/app/tasks/layer2_import_nature.py @@ -4,9 +4,9 @@ import structlog from app.data import model, repositories -from app.lib import containers, logging +from app.lib import containers from app.lib.storage import postgres -from app.tasks import interface +from app.tasks import interface, logging @final diff --git a/app/tasks/layer2_import_redshift.py b/app/tasks/layer2_import_redshift.py index c63ea05e..fe4056ce 100644 --- a/app/tasks/layer2_import_redshift.py +++ b/app/tasks/layer2_import_redshift.py @@ -7,9 +7,9 @@ from astropy import units as u from app.data import model, repositories -from app.lib import containers, logging +from app.lib import containers from app.lib.storage import postgres -from app.tasks import interface +from app.tasks import interface, logging REDSHIFT_COLUMNS = ["cz", "e_cz"] VELOCITY_UNIT = "km/s" diff --git a/app/tasks/logging/__init__.py b/app/tasks/logging/__init__.py new file mode 100644 index 00000000..1ac622c5 --- /dev/null +++ b/app/tasks/logging/__init__.py @@ -0,0 +1,3 @@ +from app.tasks.logging.table import print_table + +__all__ = ["print_table"] diff --git a/app/lib/logging/table.py b/app/tasks/logging/table.py similarity index 100% rename from app/lib/logging/table.py rename to app/tasks/logging/table.py From 95eb549155f62b9064cab96c4ef40d69da666634 Mon Sep 17 00:00:00 2001 From: kraysent Date: Mon, 27 Jul 2026 00:34:02 +0100 Subject: [PATCH 7/9] cache to adminapi and stop dependence between postgres and web --- app/adminapi/cache/__init__.py | 5 +++++ app/{lib => adminapi}/cache/background_cache.py | 0 app/adminapi/command.py | 4 ++-- app/adminapi/domain/actions.py | 4 ++-- app/adminapi/domain/table_upload.py | 4 ++-- app/lib/cache/__init__.py | 5 ----- app/lib/storage/postgres/postgres_storage.py | 5 ++--- tests/unit/adminapi/__init__.py | 0 tests/unit/{lib => adminapi}/cache_test.py | 4 ++-- 9 files changed, 15 insertions(+), 16 deletions(-) create mode 100644 app/adminapi/cache/__init__.py rename app/{lib => adminapi}/cache/background_cache.py (100%) delete mode 100644 app/lib/cache/__init__.py create mode 100644 tests/unit/adminapi/__init__.py rename tests/unit/{lib => adminapi}/cache_test.py (95%) diff --git a/app/adminapi/cache/__init__.py b/app/adminapi/cache/__init__.py new file mode 100644 index 00000000..7c6f9862 --- /dev/null +++ b/app/adminapi/cache/__init__.py @@ -0,0 +1,5 @@ +from app.adminapi.cache.background_cache import BackgroundCache + +__all__ = [ + "BackgroundCache", +] diff --git a/app/lib/cache/background_cache.py b/app/adminapi/cache/background_cache.py similarity index 100% rename from app/lib/cache/background_cache.py rename to app/adminapi/cache/background_cache.py diff --git a/app/adminapi/command.py b/app/adminapi/command.py index e29c2217..ed29d621 100644 --- a/app/adminapi/command.py +++ b/app/adminapi/command.py @@ -8,10 +8,10 @@ import structlog import yaml -from app.adminapi import clients, domain, presentation +from app.adminapi import cache, clients, domain, presentation from app.adminapi.domain import table_stats from app.data import repositories -from app.lib import audit, auth, cache, commands, config, tracing +from app.lib import audit, auth, commands, config, tracing from app.lib.storage import postgres from app.lib.tracing import TracingConfig from app.lib.web import server diff --git a/app/adminapi/domain/actions.py b/app/adminapi/domain/actions.py index 99405a22..0be06d72 100644 --- a/app/adminapi/domain/actions.py +++ b/app/adminapi/domain/actions.py @@ -1,10 +1,10 @@ from typing import final -from app.adminapi import clients +from app.adminapi import cache, clients from app.adminapi import presentation as adminapi from app.adminapi.domain import catalogs, crossmatch, layer1_write, login, pgc, sources, table_upload from app.data import repositories -from app.lib import auth, cache +from app.lib import auth @final diff --git a/app/adminapi/domain/table_upload.py b/app/adminapi/domain/table_upload.py index e315647f..ad7e7d9d 100644 --- a/app/adminapi/domain/table_upload.py +++ b/app/adminapi/domain/table_upload.py @@ -11,13 +11,13 @@ from astropy import units as u from astroquery import nasa_ads as ads -from app.adminapi import clients +from app.adminapi import cache, clients from app.adminapi import presentation as adminapi from app.adminapi.domain import table_stats from app.data import model, repositories from app.data.repositories.common import ColumnSchemaInfo, TableSchemaInfo from app.data.repositories.layer0.common import RAWDATA_SCHEMA -from app.lib import astronomy, cache, concurrency +from app.lib import astronomy, concurrency from app.lib.storage import enums, mapping from app.lib.web.errors import NotFoundError, RuleValidationError diff --git a/app/lib/cache/__init__.py b/app/lib/cache/__init__.py deleted file mode 100644 index 13f5d8ad..00000000 --- a/app/lib/cache/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from app.lib.cache.background_cache import BackgroundCache - -__all__ = [ - "BackgroundCache", -] diff --git a/app/lib/storage/postgres/postgres_storage.py b/app/lib/storage/postgres/postgres_storage.py index 81d4765e..25e2f259 100644 --- a/app/lib/storage/postgres/postgres_storage.py +++ b/app/lib/storage/postgres/postgres_storage.py @@ -12,7 +12,6 @@ from app.lib.storage import enums from app.lib.storage.postgres import config -from app.lib.web.errors import InternalError log: structlog.stdlib.BoundLogger = structlog.get_logger() @@ -85,14 +84,14 @@ def set_thread_conn(self, conn: psycopg.Connection | None) -> None: def get_pool(self) -> ConnectionPool: if self._pool is None: - raise InternalError("connection pool is not initialized") + raise RuntimeError("connection pool is not initialized") return self._pool def get_connection(self) -> psycopg.Connection: conn = self.get_thread_conn() if conn is not None: return conn - raise InternalError("no active transaction connection on this thread") + raise RuntimeError("no active transaction connection on this thread") def disconnect(self) -> None: if self._pool is not None: diff --git a/tests/unit/adminapi/__init__.py b/tests/unit/adminapi/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/lib/cache_test.py b/tests/unit/adminapi/cache_test.py similarity index 95% rename from tests/unit/lib/cache_test.py rename to tests/unit/adminapi/cache_test.py index 08fb513e..b52c34e6 100644 --- a/tests/unit/lib/cache_test.py +++ b/tests/unit/adminapi/cache_test.py @@ -7,8 +7,8 @@ import pydantic -from app.lib.cache import BackgroundCache -from app.lib.cache.background_cache import _run_in_context +from app.adminapi.cache import BackgroundCache +from app.adminapi.cache.background_cache import _run_in_context class SampleSnapshot(pydantic.BaseModel): From a48ed4665ee9225a63106f30ffa0ab850c85ae8a Mon Sep 17 00:00:00 2001 From: kraysent Date: Mon, 27 Jul 2026 00:45:05 +0100 Subject: [PATCH 8/9] move tests as well --- makefile | 2 +- tests/{integration => adminapi}/__init__.py | 0 tests/{unit => adminapi/integration}/__init__.py | 0 .../integration/adminapi_auth_test.py | 0 .../integration/adminapi_server_test.py | 0 .../integration/assign_record_pgcs_test.py | 0 .../integration/catalogs_api_test.py | 0 .../integration/create_table_test.py | 0 .../integration/merge_pgcs_test.py | 0 .../integration/rawdata_table_test.py | 0 .../{unit/adminapi => adminapi/unit}/__init__.py | 0 .../adminapi => adminapi/unit}/cache_test.py | 0 .../unit}/create_source_test.py | 0 .../domain => adminapi/unit}/table_stats_test.py | 0 .../unit}/table_upload_test.py | 0 tests/assets/default_rules.csv | 16 ---------------- tests/{unit => }/data/__init__.py | 0 .../domain => data/integration}/__init__.py | 0 .../integration/layer0_tables_repository_test.py | 0 .../integration/layer1_repository_test.py | 0 .../integration/layer2_repository_test.py | 0 tests/{unit/lib => data/unit}/__init__.py | 0 .../data => data/unit}/layer0_repository_test.py | 0 tests/dataapi/__init__.py | 0 tests/dataapi/integration/__init__.py | 0 .../integration/dataapi_server_test.py | 0 .../integration/metadata_api_test.py | 2 +- tests/dataapi/unit/__init__.py | 0 .../unit}/parameterized_query_test.py | 0 tests/lib/integration/__init__.py | 0 .../integration/transactional_storage_test.py | 0 tests/lib/unit/__init__.py | 0 tests/{unit/lib => lib/unit}/astronomy_test.py | 0 .../lib => lib/unit}/audit_truncate_test.py | 0 tests/{unit/lib => lib/unit}/auth_test.py | 0 tests/{unit/lib => lib/unit}/commands_test.py | 0 tests/{unit/lib => lib/unit}/concurrency_test.py | 0 tests/{unit/lib => lib/unit}/containers_test.py | 0 .../lib => lib/unit}/log_middleware_test.py | 0 tests/{unit/lib => lib/unit}/server_test.py | 0 .../lib => lib/unit}/tracing_middleware_test.py | 0 tests/tasks/__init__.py | 0 tests/tasks/integration/__init__.py | 0 .../integration/layer2_import_test.py | 0 tests/tasks/unit/__init__.py | 0 .../tasks => tasks/unit}/layer2_import_test.py | 0 .../commands => tasks/unit}/serve_tasks_test.py | 0 47 files changed, 2 insertions(+), 18 deletions(-) rename tests/{integration => adminapi}/__init__.py (100%) rename tests/{unit => adminapi/integration}/__init__.py (100%) rename tests/{ => adminapi}/integration/adminapi_auth_test.py (100%) rename tests/{ => adminapi}/integration/adminapi_server_test.py (100%) rename tests/{ => adminapi}/integration/assign_record_pgcs_test.py (100%) rename tests/{ => adminapi}/integration/catalogs_api_test.py (100%) rename tests/{ => adminapi}/integration/create_table_test.py (100%) rename tests/{ => adminapi}/integration/merge_pgcs_test.py (100%) rename tests/{ => adminapi}/integration/rawdata_table_test.py (100%) rename tests/{unit/adminapi => adminapi/unit}/__init__.py (100%) rename tests/{unit/adminapi => adminapi/unit}/cache_test.py (100%) rename tests/{unit/domain => adminapi/unit}/create_source_test.py (100%) rename tests/{unit/domain => adminapi/unit}/table_stats_test.py (100%) rename tests/{unit/domain => adminapi/unit}/table_upload_test.py (100%) delete mode 100644 tests/assets/default_rules.csv rename tests/{unit => }/data/__init__.py (100%) rename tests/{unit/domain => data/integration}/__init__.py (100%) rename tests/{ => data}/integration/layer0_tables_repository_test.py (100%) rename tests/{ => data}/integration/layer1_repository_test.py (100%) rename tests/{ => data}/integration/layer2_repository_test.py (100%) rename tests/{unit/lib => data/unit}/__init__.py (100%) rename tests/{unit/data => data/unit}/layer0_repository_test.py (100%) create mode 100644 tests/dataapi/__init__.py create mode 100644 tests/dataapi/integration/__init__.py rename tests/{ => dataapi}/integration/dataapi_server_test.py (100%) rename tests/{ => dataapi}/integration/metadata_api_test.py (98%) create mode 100644 tests/dataapi/unit/__init__.py rename tests/{unit/domain => dataapi/unit}/parameterized_query_test.py (100%) create mode 100644 tests/lib/integration/__init__.py rename tests/{ => lib}/integration/transactional_storage_test.py (100%) create mode 100644 tests/lib/unit/__init__.py rename tests/{unit/lib => lib/unit}/astronomy_test.py (100%) rename tests/{unit/lib => lib/unit}/audit_truncate_test.py (100%) rename tests/{unit/lib => lib/unit}/auth_test.py (100%) rename tests/{unit/lib => lib/unit}/commands_test.py (100%) rename tests/{unit/lib => lib/unit}/concurrency_test.py (100%) rename tests/{unit/lib => lib/unit}/containers_test.py (100%) rename tests/{unit/lib => lib/unit}/log_middleware_test.py (100%) rename tests/{unit/lib => lib/unit}/server_test.py (100%) rename tests/{unit/lib => lib/unit}/tracing_middleware_test.py (100%) create mode 100644 tests/tasks/__init__.py create mode 100644 tests/tasks/integration/__init__.py rename tests/{ => tasks}/integration/layer2_import_test.py (100%) create mode 100644 tests/tasks/unit/__init__.py rename tests/{unit/tasks => tasks/unit}/layer2_import_test.py (100%) rename tests/{unit/commands => tasks/unit}/serve_tasks_test.py (100%) diff --git a/makefile b/makefile index 17380482..02a26d59 100644 --- a/makefile +++ b/makefile @@ -79,7 +79,7 @@ check: @uv run pytest \ --quiet \ --config-file=pyproject.toml \ - tests/env_test.py tests/unit + tests/env_test.py tests/*/unit @echo "Testing ok." fix: diff --git a/tests/integration/__init__.py b/tests/adminapi/__init__.py similarity index 100% rename from tests/integration/__init__.py rename to tests/adminapi/__init__.py diff --git a/tests/unit/__init__.py b/tests/adminapi/integration/__init__.py similarity index 100% rename from tests/unit/__init__.py rename to tests/adminapi/integration/__init__.py diff --git a/tests/integration/adminapi_auth_test.py b/tests/adminapi/integration/adminapi_auth_test.py similarity index 100% rename from tests/integration/adminapi_auth_test.py rename to tests/adminapi/integration/adminapi_auth_test.py diff --git a/tests/integration/adminapi_server_test.py b/tests/adminapi/integration/adminapi_server_test.py similarity index 100% rename from tests/integration/adminapi_server_test.py rename to tests/adminapi/integration/adminapi_server_test.py diff --git a/tests/integration/assign_record_pgcs_test.py b/tests/adminapi/integration/assign_record_pgcs_test.py similarity index 100% rename from tests/integration/assign_record_pgcs_test.py rename to tests/adminapi/integration/assign_record_pgcs_test.py diff --git a/tests/integration/catalogs_api_test.py b/tests/adminapi/integration/catalogs_api_test.py similarity index 100% rename from tests/integration/catalogs_api_test.py rename to tests/adminapi/integration/catalogs_api_test.py diff --git a/tests/integration/create_table_test.py b/tests/adminapi/integration/create_table_test.py similarity index 100% rename from tests/integration/create_table_test.py rename to tests/adminapi/integration/create_table_test.py diff --git a/tests/integration/merge_pgcs_test.py b/tests/adminapi/integration/merge_pgcs_test.py similarity index 100% rename from tests/integration/merge_pgcs_test.py rename to tests/adminapi/integration/merge_pgcs_test.py diff --git a/tests/integration/rawdata_table_test.py b/tests/adminapi/integration/rawdata_table_test.py similarity index 100% rename from tests/integration/rawdata_table_test.py rename to tests/adminapi/integration/rawdata_table_test.py diff --git a/tests/unit/adminapi/__init__.py b/tests/adminapi/unit/__init__.py similarity index 100% rename from tests/unit/adminapi/__init__.py rename to tests/adminapi/unit/__init__.py diff --git a/tests/unit/adminapi/cache_test.py b/tests/adminapi/unit/cache_test.py similarity index 100% rename from tests/unit/adminapi/cache_test.py rename to tests/adminapi/unit/cache_test.py diff --git a/tests/unit/domain/create_source_test.py b/tests/adminapi/unit/create_source_test.py similarity index 100% rename from tests/unit/domain/create_source_test.py rename to tests/adminapi/unit/create_source_test.py diff --git a/tests/unit/domain/table_stats_test.py b/tests/adminapi/unit/table_stats_test.py similarity index 100% rename from tests/unit/domain/table_stats_test.py rename to tests/adminapi/unit/table_stats_test.py diff --git a/tests/unit/domain/table_upload_test.py b/tests/adminapi/unit/table_upload_test.py similarity index 100% rename from tests/unit/domain/table_upload_test.py rename to tests/adminapi/unit/table_upload_test.py diff --git a/tests/assets/default_rules.csv b/tests/assets/default_rules.csv deleted file mode 100644 index 0b72cbc3..00000000 --- a/tests/assets/default_rules.csv +++ /dev/null @@ -1,16 +0,0 @@ -"catalog","parameter","key","filters","priority" -icrs,ra,"","{""ucd"": ""pos.eq.ra""}",1 -icrs,dec,"","{""ucd"": ""pos.eq.dec""}",1 -icrs,ra,"","{""ucd"": ""pos.eq.ra;meta.main""}",2 -icrs,dec,"","{""ucd"": ""pos.eq.dec;meta.main""}",2 -icrs,dec,"","{""ucd"": ""pos.eq.dec;meta.main;stat.mean""}",3 -icrs,ra,"","{""ucd"": ""pos.eq.ra;meta.main;stat.mean""}",3 -designation,design,"","{""ucd"": ""meta.id""}",1 -designation,design,"","{""ucd"": ""meta.id"", ""datatype_oneof"": [""text""]}",2 -designation,design,"","{""ucd"": ""meta.id;meta.main""}",3 -designation,design,"","{""ucd"": ""meta.id;meta.main"", ""datatype_oneof"": [""text""]}",4 -redshift,z,"","{""ucd"": ""src.redshift""}",1 -redshift,cz,"","{""ucd"": ""phys.veloc""}",1 -redshift,cz,"","{""ucd"": ""phys.veloc;meta.main""}",2 -redshift,cz,"","{""ucd"": ""phys.veloc;meta.main;stat.mean""}",3 -redshift,e_cz,"","{""ucd"": ""stat.error;phys.veloc""}",1 \ No newline at end of file diff --git a/tests/unit/data/__init__.py b/tests/data/__init__.py similarity index 100% rename from tests/unit/data/__init__.py rename to tests/data/__init__.py diff --git a/tests/unit/domain/__init__.py b/tests/data/integration/__init__.py similarity index 100% rename from tests/unit/domain/__init__.py rename to tests/data/integration/__init__.py diff --git a/tests/integration/layer0_tables_repository_test.py b/tests/data/integration/layer0_tables_repository_test.py similarity index 100% rename from tests/integration/layer0_tables_repository_test.py rename to tests/data/integration/layer0_tables_repository_test.py diff --git a/tests/integration/layer1_repository_test.py b/tests/data/integration/layer1_repository_test.py similarity index 100% rename from tests/integration/layer1_repository_test.py rename to tests/data/integration/layer1_repository_test.py diff --git a/tests/integration/layer2_repository_test.py b/tests/data/integration/layer2_repository_test.py similarity index 100% rename from tests/integration/layer2_repository_test.py rename to tests/data/integration/layer2_repository_test.py diff --git a/tests/unit/lib/__init__.py b/tests/data/unit/__init__.py similarity index 100% rename from tests/unit/lib/__init__.py rename to tests/data/unit/__init__.py diff --git a/tests/unit/data/layer0_repository_test.py b/tests/data/unit/layer0_repository_test.py similarity index 100% rename from tests/unit/data/layer0_repository_test.py rename to tests/data/unit/layer0_repository_test.py diff --git a/tests/dataapi/__init__.py b/tests/dataapi/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/dataapi/integration/__init__.py b/tests/dataapi/integration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/dataapi_server_test.py b/tests/dataapi/integration/dataapi_server_test.py similarity index 100% rename from tests/integration/dataapi_server_test.py rename to tests/dataapi/integration/dataapi_server_test.py diff --git a/tests/integration/metadata_api_test.py b/tests/dataapi/integration/metadata_api_test.py similarity index 98% rename from tests/integration/metadata_api_test.py rename to tests/dataapi/integration/metadata_api_test.py index 11604fa1..6b9173f0 100644 --- a/tests/integration/metadata_api_test.py +++ b/tests/dataapi/integration/metadata_api_test.py @@ -18,7 +18,7 @@ class MetadataAPITest(unittest.TestCase): @classmethod def setUpClass(cls) -> None: cls.pg_storage = lib.TestPostgresStorage.get() - cfg_path = pathlib.Path(__file__).resolve().parents[2] / "configs" / "dev" / "dataapi.yaml" + cfg_path = pathlib.Path(__file__).resolve().parents[3] / "configs" / "dev" / "dataapi.yaml" cls.cfg = dataapi_command.parse_config(str(cfg_path)) cls.log = structlog.get_logger() diff --git a/tests/dataapi/unit/__init__.py b/tests/dataapi/unit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/domain/parameterized_query_test.py b/tests/dataapi/unit/parameterized_query_test.py similarity index 100% rename from tests/unit/domain/parameterized_query_test.py rename to tests/dataapi/unit/parameterized_query_test.py diff --git a/tests/lib/integration/__init__.py b/tests/lib/integration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/transactional_storage_test.py b/tests/lib/integration/transactional_storage_test.py similarity index 100% rename from tests/integration/transactional_storage_test.py rename to tests/lib/integration/transactional_storage_test.py diff --git a/tests/lib/unit/__init__.py b/tests/lib/unit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/lib/astronomy_test.py b/tests/lib/unit/astronomy_test.py similarity index 100% rename from tests/unit/lib/astronomy_test.py rename to tests/lib/unit/astronomy_test.py diff --git a/tests/unit/lib/audit_truncate_test.py b/tests/lib/unit/audit_truncate_test.py similarity index 100% rename from tests/unit/lib/audit_truncate_test.py rename to tests/lib/unit/audit_truncate_test.py diff --git a/tests/unit/lib/auth_test.py b/tests/lib/unit/auth_test.py similarity index 100% rename from tests/unit/lib/auth_test.py rename to tests/lib/unit/auth_test.py diff --git a/tests/unit/lib/commands_test.py b/tests/lib/unit/commands_test.py similarity index 100% rename from tests/unit/lib/commands_test.py rename to tests/lib/unit/commands_test.py diff --git a/tests/unit/lib/concurrency_test.py b/tests/lib/unit/concurrency_test.py similarity index 100% rename from tests/unit/lib/concurrency_test.py rename to tests/lib/unit/concurrency_test.py diff --git a/tests/unit/lib/containers_test.py b/tests/lib/unit/containers_test.py similarity index 100% rename from tests/unit/lib/containers_test.py rename to tests/lib/unit/containers_test.py diff --git a/tests/unit/lib/log_middleware_test.py b/tests/lib/unit/log_middleware_test.py similarity index 100% rename from tests/unit/lib/log_middleware_test.py rename to tests/lib/unit/log_middleware_test.py diff --git a/tests/unit/lib/server_test.py b/tests/lib/unit/server_test.py similarity index 100% rename from tests/unit/lib/server_test.py rename to tests/lib/unit/server_test.py diff --git a/tests/unit/lib/tracing_middleware_test.py b/tests/lib/unit/tracing_middleware_test.py similarity index 100% rename from tests/unit/lib/tracing_middleware_test.py rename to tests/lib/unit/tracing_middleware_test.py diff --git a/tests/tasks/__init__.py b/tests/tasks/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/tasks/integration/__init__.py b/tests/tasks/integration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/layer2_import_test.py b/tests/tasks/integration/layer2_import_test.py similarity index 100% rename from tests/integration/layer2_import_test.py rename to tests/tasks/integration/layer2_import_test.py diff --git a/tests/tasks/unit/__init__.py b/tests/tasks/unit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/tasks/layer2_import_test.py b/tests/tasks/unit/layer2_import_test.py similarity index 100% rename from tests/unit/tasks/layer2_import_test.py rename to tests/tasks/unit/layer2_import_test.py diff --git a/tests/unit/commands/serve_tasks_test.py b/tests/tasks/unit/serve_tasks_test.py similarity index 100% rename from tests/unit/commands/serve_tasks_test.py rename to tests/tasks/unit/serve_tasks_test.py From 3b05fc5e5301b476dafe29d1c4d4b5c501a63fa6 Mon Sep 17 00:00:00 2001 From: kraysent Date: Mon, 27 Jul 2026 00:49:09 +0100 Subject: [PATCH 9/9] move enums out of the lib --- app/adminapi/command.py | 3 ++- app/data/enums.py | 8 ++++++++ app/dataapi/command.py | 3 ++- app/lib/storage/postgres/postgres_storage.py | 16 ++++++++-------- app/tasks/layer2_import.py | 3 ++- app/tasks/layer2_import_designation.py | 3 ++- app/tasks/layer2_import_icrs.py | 3 ++- app/tasks/layer2_import_nature.py | 3 ++- app/tasks/layer2_import_redshift.py | 3 ++- app/tasks/layer2_orphan_cleanup.py | 3 ++- tests/lib/postgres.py | 3 ++- 11 files changed, 34 insertions(+), 17 deletions(-) create mode 100644 app/data/enums.py diff --git a/app/adminapi/command.py b/app/adminapi/command.py index ed29d621..9a2cec16 100644 --- a/app/adminapi/command.py +++ b/app/adminapi/command.py @@ -10,6 +10,7 @@ from app.adminapi import cache, clients, domain, presentation from app.adminapi.domain import table_stats +from app.data import enums as data_enums from app.data import repositories from app.lib import audit, auth, commands, config, tracing from app.lib.storage import postgres @@ -37,7 +38,7 @@ def prepare(self): tracing.setup_tracing("adminapi", cfg.tracing) - self.pg_storage = postgres.PgStorage(cfg.storage, log) + self.pg_storage = postgres.PgStorage(cfg.storage, log, data_enums.PG_ENUM_REGISTRY) self.pg_storage.connect() authenticator: auth.Authenticator = ( diff --git a/app/data/enums.py b/app/data/enums.py new file mode 100644 index 00000000..f17a0738 --- /dev/null +++ b/app/data/enums.py @@ -0,0 +1,8 @@ +from psycopg.types import enum + +from app.lib.storage import enums + +PG_ENUM_REGISTRY: list[tuple[type[enum.Enum], str]] = [ + (enums.DataType, "common.datatype"), + (enums.RecordTriageStatus, "layer0.triage_status"), +] diff --git a/app/dataapi/command.py b/app/dataapi/command.py index e657ce69..ad7413ff 100644 --- a/app/dataapi/command.py +++ b/app/dataapi/command.py @@ -6,6 +6,7 @@ import structlog import yaml +from app.data import enums as data_enums from app.data import repositories from app.dataapi import domain, presentation, responders from app.lib import auth, commands, config, tracing @@ -32,7 +33,7 @@ def prepare(self): tracing.setup_tracing("dataapi", self.config.tracing) self.pg_auth = postgres.PgStorage(self.config.storage.auth, log) - self.pg_main = postgres.PgStorage(self.config.storage.main, log) + self.pg_main = postgres.PgStorage(self.config.storage.main, log, data_enums.PG_ENUM_REGISTRY) authenticator: auth.Authenticator = ( auth.PostgresAuthenticator(self.pg_auth) if self.config.auth_enabled else auth.NoopAuthenticator() diff --git a/app/lib/storage/postgres/postgres_storage.py b/app/lib/storage/postgres/postgres_storage.py index 25e2f259..547d3a7e 100644 --- a/app/lib/storage/postgres/postgres_storage.py +++ b/app/lib/storage/postgres/postgres_storage.py @@ -10,7 +10,6 @@ from psycopg.types import enum, numeric from psycopg_pool import ConnectionPool -from app.lib.storage import enums from app.lib.storage.postgres import config log: structlog.stdlib.BoundLogger = structlog.get_logger() @@ -35,24 +34,25 @@ def dump(self, obj: Any) -> bytes | bytearray | memoryview: (np.int64, NumpyIntDumper), ] -DEFAULT_ENUMS: list[tuple[type[enum.Enum], str]] = [ - (enums.DataType, "common.datatype"), - (enums.RecordTriageStatus, "layer0.triage_status"), -] - class PgStorage: - def __init__(self, cfg: config.PgStorageConfig, logger: structlog.stdlib.BoundLogger) -> None: + def __init__( + self, + cfg: config.PgStorageConfig, + logger: structlog.stdlib.BoundLogger, + enum_registry: Sequence[tuple[type[enum.Enum], str]] = (), + ) -> None: self._config = cfg self._pool: ConnectionPool | None = None self._logger = logger self._local = threading.local() + self._enum_registry: list[tuple[type[enum.Enum], str]] = list(enum_registry) self._extra_enums: list[tuple[type[enum.Enum], str]] = [] def _configure_connection(self, conn: psycopg.Connection) -> None: for python_type, dumper in DEFAULT_DUMPERS: conn.adapters.register_dumper(python_type, dumper) - for enum_type, pg_type in DEFAULT_ENUMS + self._extra_enums: + for enum_type, pg_type in self._enum_registry + self._extra_enums: type_info = enum.EnumInfo.fetch(conn, pg_type) if type_info is None: raise RuntimeError(f"Unable to find enum {pg_type} in DB") diff --git a/app/tasks/layer2_import.py b/app/tasks/layer2_import.py index 88630429..436bdace 100644 --- a/app/tasks/layer2_import.py +++ b/app/tasks/layer2_import.py @@ -4,6 +4,7 @@ import structlog +from app.data import enums as data_enums from app.data import repositories from app.lib.storage import postgres from app.tasks import ( @@ -55,7 +56,7 @@ def name(cls) -> str: return "layer2-import" def prepare(self, config: interface.Config) -> None: - self.pg_storage = postgres.PgStorage(config.storage, self.log) + self.pg_storage = postgres.PgStorage(config.storage, self.log, data_enums.PG_ENUM_REGISTRY) self.pg_storage.connect() self.layer1_repository = repositories.Layer1Repository(self.pg_storage, self.log) self.layer2_repository = repositories.Layer2Repository(self.pg_storage, self.log) diff --git a/app/tasks/layer2_import_designation.py b/app/tasks/layer2_import_designation.py index 14634338..3dd35328 100644 --- a/app/tasks/layer2_import_designation.py +++ b/app/tasks/layer2_import_designation.py @@ -3,6 +3,7 @@ import structlog +from app.data import enums as data_enums from app.data import model, repositories from app.lib import containers from app.lib.storage import postgres @@ -34,7 +35,7 @@ def name(cls) -> str: return "layer2-import-designation" def prepare(self, config: interface.Config) -> None: - self.pg_storage = postgres.PgStorage(config.storage, self.log) + self.pg_storage = postgres.PgStorage(config.storage, self.log, data_enums.PG_ENUM_REGISTRY) self.pg_storage.connect() self.layer1_repository = repositories.Layer1Repository(self.pg_storage, self.log) self.layer2_repository = repositories.Layer2Repository(self.pg_storage, self.log) diff --git a/app/tasks/layer2_import_icrs.py b/app/tasks/layer2_import_icrs.py index 772f6f5c..dd13d29a 100644 --- a/app/tasks/layer2_import_icrs.py +++ b/app/tasks/layer2_import_icrs.py @@ -6,6 +6,7 @@ import structlog from astropy import units as u +from app.data import enums as data_enums from app.data import model, repositories from app.lib import containers from app.lib.storage import postgres @@ -46,7 +47,7 @@ def name(cls) -> str: return "layer2-import-icrs" def prepare(self, config: interface.Config) -> None: - self.pg_storage = postgres.PgStorage(config.storage, self.log) + self.pg_storage = postgres.PgStorage(config.storage, self.log, data_enums.PG_ENUM_REGISTRY) self.pg_storage.connect() self.layer1_repository = repositories.Layer1Repository(self.pg_storage, self.log) self.layer2_repository = repositories.Layer2Repository(self.pg_storage, self.log) diff --git a/app/tasks/layer2_import_nature.py b/app/tasks/layer2_import_nature.py index 79d2c36f..3d3e929c 100644 --- a/app/tasks/layer2_import_nature.py +++ b/app/tasks/layer2_import_nature.py @@ -3,6 +3,7 @@ import structlog +from app.data import enums as data_enums from app.data import model, repositories from app.lib import containers from app.lib.storage import postgres @@ -32,7 +33,7 @@ def name(cls) -> str: return "layer2-import-nature" def prepare(self, config: interface.Config) -> None: - self.pg_storage = postgres.PgStorage(config.storage, self.log) + self.pg_storage = postgres.PgStorage(config.storage, self.log, data_enums.PG_ENUM_REGISTRY) self.pg_storage.connect() self.layer1_repository = repositories.Layer1Repository(self.pg_storage, self.log) self.layer2_repository = repositories.Layer2Repository(self.pg_storage, self.log) diff --git a/app/tasks/layer2_import_redshift.py b/app/tasks/layer2_import_redshift.py index fe4056ce..bad24902 100644 --- a/app/tasks/layer2_import_redshift.py +++ b/app/tasks/layer2_import_redshift.py @@ -6,6 +6,7 @@ import structlog from astropy import units as u +from app.data import enums as data_enums from app.data import model, repositories from app.lib import containers from app.lib.storage import postgres @@ -46,7 +47,7 @@ def name(cls) -> str: return "layer2-import-redshift" def prepare(self, config: interface.Config) -> None: - self.pg_storage = postgres.PgStorage(config.storage, self.log) + self.pg_storage = postgres.PgStorage(config.storage, self.log, data_enums.PG_ENUM_REGISTRY) self.pg_storage.connect() self.layer1_repository = repositories.Layer1Repository(self.pg_storage, self.log) self.layer2_repository = repositories.Layer2Repository(self.pg_storage, self.log) diff --git a/app/tasks/layer2_orphan_cleanup.py b/app/tasks/layer2_orphan_cleanup.py index 401ca260..cdfa8719 100644 --- a/app/tasks/layer2_orphan_cleanup.py +++ b/app/tasks/layer2_orphan_cleanup.py @@ -2,6 +2,7 @@ import structlog +from app.data import enums as data_enums from app.data import model, repositories from app.lib.storage import postgres from app.tasks import interface @@ -28,7 +29,7 @@ def name(cls) -> str: return "layer2-orphan-cleanup" def prepare(self, config: interface.Config) -> None: - self.pg_storage = postgres.PgStorage(config.storage, self.log) + self.pg_storage = postgres.PgStorage(config.storage, self.log, data_enums.PG_ENUM_REGISTRY) self.pg_storage.connect() self.layer2_repository = repositories.Layer2Repository(self.pg_storage, self.log) diff --git a/tests/lib/postgres.py b/tests/lib/postgres.py index 4e9ea222..4920cf54 100644 --- a/tests/lib/postgres.py +++ b/tests/lib/postgres.py @@ -6,6 +6,7 @@ import structlog from testcontainers import postgres as pgcontainer +from app.data import enums as data_enums from app.lib.storage import postgres from tests.lib import web @@ -52,7 +53,7 @@ def __init__(self, migrations_dir: str) -> None: dbname="hyperleda", ) - self.storage = postgres.PgStorage(self.config, logger) + self.storage = postgres.PgStorage(self.config, logger, data_enums.PG_ENUM_REGISTRY) self.migrations_dir = migrations_dir