diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 99cf9bed..b4a01f71 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -41,7 +41,11 @@ jobs: tags: local/app:latest - name: Run image help - run: docker run --rm --entrypoint "app" local/app:latest --help + run: | + docker run --rm --entrypoint "app" local/app:latest --help + docker run --rm --entrypoint "adminapi" local/app:latest --help + docker run --rm --entrypoint "dataapi" local/app:latest --help + docker run --rm --entrypoint "tasks" local/app:latest --help - name: Save image run: docker save local/app:latest -o image.tar diff --git a/app/adminapi/cache/__init__.py b/app/adminapi/cache/__init__.py new file mode 100644 index 00000000..7c6f9862 --- /dev/null +++ b/app/adminapi/cache/__init__.py @@ -0,0 +1,5 @@ +from app.adminapi.cache.background_cache import BackgroundCache + +__all__ = [ + "BackgroundCache", +] diff --git a/app/lib/cache/background_cache.py b/app/adminapi/cache/background_cache.py similarity index 100% rename from app/lib/cache/background_cache.py rename to app/adminapi/cache/background_cache.py diff --git a/app/adminapi/cli.py b/app/adminapi/cli.py new file mode 100644 index 00000000..9ac8c384 --- /dev/null +++ b/app/adminapi/cli.py @@ -0,0 +1,18 @@ +import os + +import click + +from app.adminapi.command import AdminAPICommand +from app.lib import commands + + +@click.command(short_help=AdminAPICommand.help()) +@click.option( + "-c", + "--config", + type=str, + default=lambda: os.environ.get("CONFIG", ""), + help="Path to configuration file", +) +def main(config: str) -> None: + commands.run(AdminAPICommand(config)) diff --git a/app/lib/clients/__init__.py b/app/adminapi/clients/__init__.py similarity index 100% rename from app/lib/clients/__init__.py rename to app/adminapi/clients/__init__.py diff --git a/app/commands/adminapi/command.py b/app/adminapi/command.py similarity index 91% rename from app/commands/adminapi/command.py rename to app/adminapi/command.py index dbf19137..9a2cec16 100644 --- a/app/commands/adminapi/command.py +++ b/app/adminapi/command.py @@ -8,14 +8,14 @@ import structlog import yaml +from app.adminapi import cache, clients, domain, presentation +from app.adminapi.domain import table_stats +from app.data import enums as data_enums from app.data import repositories -from app.domain import adminapi as domain -from app.domain.adminapi import table_stats -from app.lib import audit, auth, cache, clients, commands, config, tracing +from app.lib import audit, auth, commands, config, tracing from app.lib.storage import postgres from app.lib.tracing import TracingConfig from app.lib.web import server -from app.presentation import adminapi as presentation log: structlog.stdlib.BoundLogger = structlog.get_logger() @@ -38,7 +38,7 @@ def prepare(self): tracing.setup_tracing("adminapi", cfg.tracing) - self.pg_storage = postgres.PgStorage(cfg.storage, log) + self.pg_storage = postgres.PgStorage(cfg.storage, log, data_enums.PG_ENUM_REGISTRY) self.pg_storage.connect() authenticator: auth.Authenticator = ( diff --git a/app/adminapi/domain/__init__.py b/app/adminapi/domain/__init__.py new file mode 100644 index 00000000..d63e8661 --- /dev/null +++ b/app/adminapi/domain/__init__.py @@ -0,0 +1,15 @@ +from app.adminapi.domain.actions import Actions +from app.adminapi.domain.crossmatch import CrossmatchManager +from app.adminapi.domain.login import LoginManager +from app.adminapi.domain.mock import get_mock_actions +from app.adminapi.domain.sources import SourceManager +from app.adminapi.domain.table_upload import TableUploadManager + +__all__ = [ + "Actions", + "CrossmatchManager", + "get_mock_actions", + "LoginManager", + "TableUploadManager", + "SourceManager", +] diff --git a/app/domain/adminapi/actions.py b/app/adminapi/domain/actions.py similarity index 94% rename from app/domain/adminapi/actions.py rename to app/adminapi/domain/actions.py index 5b7548be..0be06d72 100644 --- a/app/domain/adminapi/actions.py +++ b/app/adminapi/domain/actions.py @@ -1,9 +1,10 @@ from typing import final +from app.adminapi import cache, clients +from app.adminapi import presentation as adminapi +from app.adminapi.domain import catalogs, crossmatch, layer1_write, login, pgc, sources, table_upload from app.data import repositories -from app.domain.adminapi import catalogs, crossmatch, layer1_write, login, pgc, sources, table_upload -from app.lib import auth, cache, clients -from app.presentation import adminapi +from app.lib import auth @final diff --git a/app/domain/adminapi/catalogs.py b/app/adminapi/domain/catalogs.py similarity index 98% rename from app/domain/adminapi/catalogs.py rename to app/adminapi/domain/catalogs.py index 672b6ede..5f5bfd57 100644 --- a/app/domain/adminapi/catalogs.py +++ b/app/adminapi/domain/catalogs.py @@ -1,7 +1,7 @@ from typing import Any, final +from app.adminapi import presentation as adminapi from app.data import model, repositories -from app.presentation import adminapi _INTERNAL_COLUMNS = frozenset({"record_id", "object_id", "id", "modification_time"}) diff --git a/app/domain/adminapi/crossmatch.py b/app/adminapi/domain/crossmatch.py similarity index 99% rename from app/domain/adminapi/crossmatch.py rename to app/adminapi/domain/crossmatch.py index f31aceb2..356c28e9 100644 --- a/app/domain/adminapi/crossmatch.py +++ b/app/adminapi/domain/crossmatch.py @@ -4,13 +4,13 @@ from astropy import coordinates from astropy import units as u +from app.adminapi import presentation as adminapi from app.data import model from app.data.repositories import layer0, layer1, layer2 from app.data.repositories.layer0.records import AssignRecordPgcsPreconditionError from app.lib import astronomy from app.lib.storage import enums from app.lib.web.errors import ConflictError, NotFoundError -from app.presentation import adminapi logger = structlog.stdlib.get_logger() diff --git a/app/domain/adminapi/layer1_write.py b/app/adminapi/domain/layer1_write.py similarity index 97% rename from app/domain/adminapi/layer1_write.py rename to app/adminapi/domain/layer1_write.py index 166004b7..5cec9c66 100644 --- a/app/domain/adminapi/layer1_write.py +++ b/app/adminapi/domain/layer1_write.py @@ -1,10 +1,10 @@ import psycopg from astropy import units as u +from app.adminapi import presentation as adminapi from app.data import model from app.data.repositories import layer1 from app.lib.web.errors import RuleValidationError -from app.presentation import adminapi class Layer1Writer: diff --git a/app/domain/adminapi/login.py b/app/adminapi/domain/login.py similarity index 92% rename from app/domain/adminapi/login.py rename to app/adminapi/domain/login.py index 35a77fd2..aa7a3c73 100644 --- a/app/domain/adminapi/login.py +++ b/app/adminapi/domain/login.py @@ -1,6 +1,6 @@ +from app.adminapi import presentation as adminapi from app.lib import auth from app.lib.web.errors import UnauthorizedError -from app.presentation import adminapi class LoginManager: diff --git a/app/domain/adminapi/mock.py b/app/adminapi/domain/mock.py similarity index 83% rename from app/domain/adminapi/mock.py rename to app/adminapi/domain/mock.py index b7fede53..4827251e 100644 --- a/app/domain/adminapi/mock.py +++ b/app/adminapi/domain/mock.py @@ -1,9 +1,10 @@ import datetime from unittest import mock -from app.domain.adminapi import actions -from app.lib import auth, clients -from app.presentation import adminapi +from app.adminapi import clients +from app.adminapi import presentation as adminapi +from app.adminapi.domain import actions +from app.lib import auth def get_mock_table_stats_cache() -> mock.MagicMock: diff --git a/app/domain/adminapi/pgc.py b/app/adminapi/domain/pgc.py similarity index 95% rename from app/domain/adminapi/pgc.py rename to app/adminapi/domain/pgc.py index 3d533dc5..0950fff3 100644 --- a/app/domain/adminapi/pgc.py +++ b/app/adminapi/domain/pgc.py @@ -1,8 +1,8 @@ from typing import final +from app.adminapi import presentation as adminapi from app.data import repositories from app.lib.web.errors import NotFoundError -from app.presentation import adminapi @final diff --git a/app/domain/adminapi/sources.py b/app/adminapi/domain/sources.py similarity index 94% rename from app/domain/adminapi/sources.py rename to app/adminapi/domain/sources.py index cbc07a62..10f77105 100644 --- a/app/domain/adminapi/sources.py +++ b/app/adminapi/domain/sources.py @@ -1,5 +1,5 @@ +from app.adminapi import presentation as adminapi from app.data import repositories -from app.presentation import adminapi ALLOWED_CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_" diff --git a/app/domain/adminapi/table_stats.py b/app/adminapi/domain/table_stats.py similarity index 96% rename from app/domain/adminapi/table_stats.py rename to app/adminapi/domain/table_stats.py index 62993e41..b74b6ca8 100644 --- a/app/domain/adminapi/table_stats.py +++ b/app/adminapi/domain/table_stats.py @@ -1,8 +1,8 @@ import datetime from collections.abc import Callable +from app.adminapi import presentation as adminapi from app.data import model, repositories -from app.presentation import adminapi def table_progress_to_presentation(progress: model.TableProgress) -> adminapi.TableProgress: diff --git a/app/domain/adminapi/table_upload.py b/app/adminapi/domain/table_upload.py similarity index 99% rename from app/domain/adminapi/table_upload.py rename to app/adminapi/domain/table_upload.py index a87e1793..ad7e7d9d 100644 --- a/app/domain/adminapi/table_upload.py +++ b/app/adminapi/domain/table_upload.py @@ -11,14 +11,15 @@ from astropy import units as u from astroquery import nasa_ads as ads +from app.adminapi import cache, clients +from app.adminapi import presentation as adminapi +from app.adminapi.domain import table_stats from app.data import model, repositories from app.data.repositories.common import ColumnSchemaInfo, TableSchemaInfo from app.data.repositories.layer0.common import RAWDATA_SCHEMA -from app.domain.adminapi import table_stats -from app.lib import astronomy, cache, clients, concurrency +from app.lib import astronomy, concurrency from app.lib.storage import enums, mapping from app.lib.web.errors import NotFoundError, RuleValidationError -from app.presentation import adminapi BIBCODE_REGEX = "^([0-9]{4}[A-Za-z.&]{5}[A-Za-z0-9.]{4}[AELPQ-Z0-9.][0-9.]{4}[A-Z])$" diff --git a/app/adminapi/presentation/__init__.py b/app/adminapi/presentation/__init__.py new file mode 100644 index 00000000..a638e1be --- /dev/null +++ b/app/adminapi/presentation/__init__.py @@ -0,0 +1,5 @@ +from app.adminapi.presentation.interface import * +from app.adminapi.presentation.records import * +from app.adminapi.presentation.server import Server + +__all__ = ["Server"] diff --git a/app/presentation/adminapi/interface.py b/app/adminapi/presentation/interface.py similarity index 99% rename from app/presentation/adminapi/interface.py rename to app/adminapi/presentation/interface.py index 52f3369e..3dc5823f 100644 --- a/app/presentation/adminapi/interface.py +++ b/app/adminapi/presentation/interface.py @@ -5,8 +5,8 @@ import pydantic +from app.adminapi.presentation.records import GetRecordsRequest, GetRecordsResponse from app.lib.storage import enums, mapping -from app.presentation.adminapi.records import GetRecordsRequest, GetRecordsResponse DatatypeEnum = enum.StrEnum( "DatatypeEnum", diff --git a/app/presentation/adminapi/records.py b/app/adminapi/presentation/records.py similarity index 100% rename from app/presentation/adminapi/records.py rename to app/adminapi/presentation/records.py diff --git a/app/presentation/adminapi/server.py b/app/adminapi/presentation/server.py similarity index 99% rename from app/presentation/adminapi/server.py rename to app/adminapi/presentation/server.py index 6c1588e6..8fdc0729 100644 --- a/app/presentation/adminapi/server.py +++ b/app/adminapi/presentation/server.py @@ -4,10 +4,10 @@ import fastapi import structlog +from app.adminapi.presentation import interface from app.lib import audit, auth from app.lib.web import server from app.lib.web.middlewares import identity_from_request -from app.presentation.adminapi import interface def _logout_token(request: fastapi.Request) -> str: diff --git a/app/cli.py b/app/cli.py index ae67b7b2..4d849cc7 100644 --- a/app/cli.py +++ b/app/cli.py @@ -1,10 +1,8 @@ -import os - import click -from app.commands.adminapi import AdminAPICommand -from app.commands.dataapi import DataAPICommand -from app.lib import commands +from app.adminapi.cli import main as adminapi +from app.dataapi.cli import main as dataapi +from app.tasks.cli import main as serve_tasks @click.group() @@ -12,36 +10,6 @@ def cli() -> None: pass -@cli.command(short_help=AdminAPICommand.help()) -@click.option( - "-c", - "--config", - type=str, - default=lambda: os.environ.get("CONFIG", ""), - help="Path to configuration file", -) -def adminapi(config: str): - commands.run(AdminAPICommand(config)) - - -@cli.command(short_help=DataAPICommand.help()) -@click.option( - "-c", - "--config", - type=str, - default=lambda: os.environ.get("CONFIG", ""), - help="Path to configuration file", -) -def dataapi(config: str): - commands.run(DataAPICommand(config)) - - -@cli.command( - "serve-tasks", - short_help="Registers layer2 import Prefect deployments and serves them.", -) -def serve_tasks() -> None: - # Lazy-load: ServeTasksCommand pulls in Prefect, which slows other commands - from app.commands.serve_tasks import ServeTasksCommand # noqa: PLC0415 - - commands.run(ServeTasksCommand()) +cli.add_command(adminapi, name="adminapi") +cli.add_command(dataapi, name="dataapi") +cli.add_command(serve_tasks, name="serve-tasks") diff --git a/app/commands/adminapi/__init__.py b/app/commands/adminapi/__init__.py deleted file mode 100644 index 93b2cf8a..00000000 --- a/app/commands/adminapi/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from app.commands.adminapi.command import AdminAPICommand, Config - -__all__ = [ - "AdminAPICommand", - "Config", -] diff --git a/app/commands/dataapi/__init__.py b/app/commands/dataapi/__init__.py deleted file mode 100644 index 0c7bebbb..00000000 --- a/app/commands/dataapi/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from app.commands.dataapi.command import Config, DataAPICommand - -__all__ = ["DataAPICommand", "Config"] diff --git a/app/commands/serve_tasks/__init__.py b/app/commands/serve_tasks/__init__.py deleted file mode 100644 index 386467cd..00000000 --- a/app/commands/serve_tasks/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from app.commands.serve_tasks.command import ServeTasksCommand - -__all__ = ["ServeTasksCommand"] diff --git a/app/data/enums.py b/app/data/enums.py new file mode 100644 index 00000000..f17a0738 --- /dev/null +++ b/app/data/enums.py @@ -0,0 +1,8 @@ +from psycopg.types import enum + +from app.lib.storage import enums + +PG_ENUM_REGISTRY: list[tuple[type[enum.Enum], str]] = [ + (enums.DataType, "common.datatype"), + (enums.RecordTriageStatus, "layer0.triage_status"), +] diff --git a/app/dataapi/cli.py b/app/dataapi/cli.py new file mode 100644 index 00000000..b0ceee28 --- /dev/null +++ b/app/dataapi/cli.py @@ -0,0 +1,18 @@ +import os + +import click + +from app.dataapi.command import DataAPICommand +from app.lib import commands + + +@click.command(short_help=DataAPICommand.help()) +@click.option( + "-c", + "--config", + type=str, + default=lambda: os.environ.get("CONFIG", ""), + help="Path to configuration file", +) +def main(config: str) -> None: + commands.run(DataAPICommand(config)) diff --git a/app/commands/dataapi/command.py b/app/dataapi/command.py similarity index 95% rename from app/commands/dataapi/command.py rename to app/dataapi/command.py index ac31335d..ad7413ff 100644 --- a/app/commands/dataapi/command.py +++ b/app/dataapi/command.py @@ -6,14 +6,13 @@ import structlog import yaml +from app.data import enums as data_enums from app.data import repositories -from app.domain import dataapi as domain -from app.domain import responders +from app.dataapi import domain, presentation, responders from app.lib import auth, commands, config, tracing from app.lib.storage import postgres from app.lib.tracing import TracingConfig from app.lib.web import server -from app.presentation import dataapi as presentation log: structlog.stdlib.BoundLogger = structlog.get_logger() @@ -34,7 +33,7 @@ def prepare(self): tracing.setup_tracing("dataapi", self.config.tracing) self.pg_auth = postgres.PgStorage(self.config.storage.auth, log) - self.pg_main = postgres.PgStorage(self.config.storage.main, log) + self.pg_main = postgres.PgStorage(self.config.storage.main, log, data_enums.PG_ENUM_REGISTRY) authenticator: auth.Authenticator = ( auth.PostgresAuthenticator(self.pg_auth) if self.config.auth_enabled else auth.NoopAuthenticator() diff --git a/app/dataapi/domain/__init__.py b/app/dataapi/domain/__init__.py new file mode 100644 index 00000000..ee480c48 --- /dev/null +++ b/app/dataapi/domain/__init__.py @@ -0,0 +1,5 @@ +from app.dataapi.domain.actions import Actions + +__all__ = [ + "Actions", +] diff --git a/app/domain/dataapi/actions.py b/app/dataapi/domain/actions.py similarity index 96% rename from app/domain/dataapi/actions.py rename to app/dataapi/domain/actions.py index 9d4b6a7e..c75ba8c5 100644 --- a/app/domain/dataapi/actions.py +++ b/app/dataapi/domain/actions.py @@ -1,9 +1,9 @@ from typing import final from app.data import model, repositories -from app.domain import responders -from app.domain.dataapi import parameterized_query, tap_types -from app.presentation import dataapi +from app.dataapi import presentation as dataapi +from app.dataapi import responders +from app.dataapi.domain import parameterized_query, tap_types ENABLED_CATALOGS = [ model.RawCatalog.DESIGNATION, diff --git a/app/domain/dataapi/parameterized_query.py b/app/dataapi/domain/parameterized_query.py similarity index 97% rename from app/domain/dataapi/parameterized_query.py rename to app/dataapi/domain/parameterized_query.py index f93a9a04..1225c85f 100644 --- a/app/domain/dataapi/parameterized_query.py +++ b/app/dataapi/domain/parameterized_query.py @@ -1,7 +1,7 @@ from app.data import model, repositories from app.data.repositories import layer2 -from app.domain import responders -from app.presentation import dataapi +from app.dataapi import presentation as dataapi +from app.dataapi import responders CATALOGS_FOR_PGC_QUERY = [ model.RawCatalog.DESIGNATION, diff --git a/app/domain/dataapi/tap_types.py b/app/dataapi/domain/tap_types.py similarity index 100% rename from app/domain/dataapi/tap_types.py rename to app/dataapi/domain/tap_types.py diff --git a/app/dataapi/presentation/__init__.py b/app/dataapi/presentation/__init__.py new file mode 100644 index 00000000..90df24e4 --- /dev/null +++ b/app/dataapi/presentation/__init__.py @@ -0,0 +1,5 @@ +from app.dataapi.presentation.interface import * +from app.dataapi.presentation.server import Server +from app.dataapi.presentation.tap import * + +__all__ = ["Server"] diff --git a/app/presentation/dataapi/interface.py b/app/dataapi/presentation/interface.py similarity index 99% rename from app/presentation/dataapi/interface.py rename to app/dataapi/presentation/interface.py index 7cf67170..f86216b3 100644 --- a/app/presentation/dataapi/interface.py +++ b/app/dataapi/presentation/interface.py @@ -2,7 +2,7 @@ import pydantic -from app.presentation.dataapi import tap +from app.dataapi.presentation import tap class EquatorialCoordinates(pydantic.BaseModel): diff --git a/app/presentation/dataapi/server.py b/app/dataapi/presentation/server.py similarity index 98% rename from app/presentation/dataapi/server.py rename to app/dataapi/presentation/server.py index 6388f071..5453556b 100644 --- a/app/presentation/dataapi/server.py +++ b/app/dataapi/presentation/server.py @@ -4,9 +4,9 @@ import fastapi import structlog +from app.dataapi.presentation import interface, tap from app.lib import auth from app.lib.web import server -from app.presentation.dataapi import interface, tap logger = structlog.stdlib.get_logger() diff --git a/app/presentation/dataapi/tap.py b/app/dataapi/presentation/tap.py similarity index 100% rename from app/presentation/dataapi/tap.py rename to app/dataapi/presentation/tap.py diff --git a/app/dataapi/responders/__init__.py b/app/dataapi/responders/__init__.py new file mode 100644 index 00000000..3a581002 --- /dev/null +++ b/app/dataapi/responders/__init__.py @@ -0,0 +1,3 @@ +from app.dataapi.responders.structured_responder import CatalogConfig, StructuredResponder + +__all__ = ["StructuredResponder", "CatalogConfig"] diff --git a/app/domain/responders/interface.py b/app/dataapi/responders/interface.py similarity index 100% rename from app/domain/responders/interface.py rename to app/dataapi/responders/interface.py diff --git a/app/domain/responders/structured_responder.py b/app/dataapi/responders/structured_responder.py similarity index 99% rename from app/domain/responders/structured_responder.py rename to app/dataapi/responders/structured_responder.py index 4d3804c1..b45a4efc 100644 --- a/app/domain/responders/structured_responder.py +++ b/app/dataapi/responders/structured_responder.py @@ -5,9 +5,9 @@ from app.data import model from app.data.model import layer2 -from app.domain.responders import interface +from app.dataapi import presentation as dataapi +from app.dataapi.responders import interface from app.lib import astronomy, config -from app.presentation import dataapi DATA_SCHEMA = dataapi.Schema( units=dataapi.Units( diff --git a/app/domain/adminapi/__init__.py b/app/domain/adminapi/__init__.py deleted file mode 100644 index fbc924c2..00000000 --- a/app/domain/adminapi/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -from app.domain.adminapi.actions import Actions -from app.domain.adminapi.crossmatch import CrossmatchManager -from app.domain.adminapi.login import LoginManager -from app.domain.adminapi.mock import get_mock_actions -from app.domain.adminapi.sources import SourceManager -from app.domain.adminapi.table_upload import TableUploadManager - -__all__ = [ - "Actions", - "CrossmatchManager", - "get_mock_actions", - "LoginManager", - "TableUploadManager", - "SourceManager", -] diff --git a/app/domain/dataapi/__init__.py b/app/domain/dataapi/__init__.py deleted file mode 100644 index 53ff2b5d..00000000 --- a/app/domain/dataapi/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from app.domain.dataapi.actions import Actions - -__all__ = [ - "Actions", -] diff --git a/app/domain/responders/__init__.py b/app/domain/responders/__init__.py deleted file mode 100644 index a655b93c..00000000 --- a/app/domain/responders/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from app.domain.responders.structured_responder import CatalogConfig, StructuredResponder - -__all__ = ["StructuredResponder", "CatalogConfig"] diff --git a/app/lib/cache/__init__.py b/app/lib/cache/__init__.py deleted file mode 100644 index 13f5d8ad..00000000 --- a/app/lib/cache/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from app.lib.cache.background_cache import BackgroundCache - -__all__ = [ - "BackgroundCache", -] diff --git a/app/lib/logging/__init__.py b/app/lib/logging/__init__.py deleted file mode 100644 index 9b7951f5..00000000 --- a/app/lib/logging/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from app.lib.logging.table import print_table - -__all__ = ["print_table"] diff --git a/app/lib/storage/postgres/postgres_storage.py b/app/lib/storage/postgres/postgres_storage.py index 81d4765e..547d3a7e 100644 --- a/app/lib/storage/postgres/postgres_storage.py +++ b/app/lib/storage/postgres/postgres_storage.py @@ -10,9 +10,7 @@ from psycopg.types import enum, numeric from psycopg_pool import ConnectionPool -from app.lib.storage import enums from app.lib.storage.postgres import config -from app.lib.web.errors import InternalError log: structlog.stdlib.BoundLogger = structlog.get_logger() @@ -36,24 +34,25 @@ def dump(self, obj: Any) -> bytes | bytearray | memoryview: (np.int64, NumpyIntDumper), ] -DEFAULT_ENUMS: list[tuple[type[enum.Enum], str]] = [ - (enums.DataType, "common.datatype"), - (enums.RecordTriageStatus, "layer0.triage_status"), -] - class PgStorage: - def __init__(self, cfg: config.PgStorageConfig, logger: structlog.stdlib.BoundLogger) -> None: + def __init__( + self, + cfg: config.PgStorageConfig, + logger: structlog.stdlib.BoundLogger, + enum_registry: Sequence[tuple[type[enum.Enum], str]] = (), + ) -> None: self._config = cfg self._pool: ConnectionPool | None = None self._logger = logger self._local = threading.local() + self._enum_registry: list[tuple[type[enum.Enum], str]] = list(enum_registry) self._extra_enums: list[tuple[type[enum.Enum], str]] = [] def _configure_connection(self, conn: psycopg.Connection) -> None: for python_type, dumper in DEFAULT_DUMPERS: conn.adapters.register_dumper(python_type, dumper) - for enum_type, pg_type in DEFAULT_ENUMS + self._extra_enums: + for enum_type, pg_type in self._enum_registry + self._extra_enums: type_info = enum.EnumInfo.fetch(conn, pg_type) if type_info is None: raise RuntimeError(f"Unable to find enum {pg_type} in DB") @@ -85,14 +84,14 @@ def set_thread_conn(self, conn: psycopg.Connection | None) -> None: def get_pool(self) -> ConnectionPool: if self._pool is None: - raise InternalError("connection pool is not initialized") + raise RuntimeError("connection pool is not initialized") return self._pool def get_connection(self) -> psycopg.Connection: conn = self.get_thread_conn() if conn is not None: return conn - raise InternalError("no active transaction connection on this thread") + raise RuntimeError("no active transaction connection on this thread") def disconnect(self) -> None: if self._pool is not None: diff --git a/app/presentation/adminapi/__init__.py b/app/presentation/adminapi/__init__.py deleted file mode 100644 index 9a79c65e..00000000 --- a/app/presentation/adminapi/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from app.presentation.adminapi.interface import * -from app.presentation.adminapi.records import * -from app.presentation.adminapi.server import Server - -__all__ = ["Server"] diff --git a/app/presentation/dataapi/__init__.py b/app/presentation/dataapi/__init__.py deleted file mode 100644 index db77530e..00000000 --- a/app/presentation/dataapi/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from app.presentation.dataapi.interface import * -from app.presentation.dataapi.server import Server -from app.presentation.dataapi.tap import * - -__all__ = ["Server"] diff --git a/app/tasks/cli.py b/app/tasks/cli.py new file mode 100644 index 00000000..0bce3f2a --- /dev/null +++ b/app/tasks/cli.py @@ -0,0 +1,9 @@ +import click + +from app.lib import commands +from app.tasks.command import ServeTasksCommand + + +@click.command(short_help="Registers layer2 import Prefect deployments and serves them.") +def main() -> None: + commands.run(ServeTasksCommand()) diff --git a/app/commands/serve_tasks/command.py b/app/tasks/command.py similarity index 93% rename from app/commands/serve_tasks/command.py rename to app/tasks/command.py index 112683ab..858a87d4 100644 --- a/app/commands/serve_tasks/command.py +++ b/app/tasks/command.py @@ -2,8 +2,8 @@ from prefect import serve -from app.commands.serve_tasks import flows from app.lib import commands +from app.tasks import flows @final diff --git a/app/commands/serve_tasks/flows.py b/app/tasks/flows.py similarity index 100% rename from app/commands/serve_tasks/flows.py rename to app/tasks/flows.py diff --git a/app/tasks/layer2_import.py b/app/tasks/layer2_import.py index 88630429..436bdace 100644 --- a/app/tasks/layer2_import.py +++ b/app/tasks/layer2_import.py @@ -4,6 +4,7 @@ import structlog +from app.data import enums as data_enums from app.data import repositories from app.lib.storage import postgres from app.tasks import ( @@ -55,7 +56,7 @@ def name(cls) -> str: return "layer2-import" def prepare(self, config: interface.Config) -> None: - self.pg_storage = postgres.PgStorage(config.storage, self.log) + self.pg_storage = postgres.PgStorage(config.storage, self.log, data_enums.PG_ENUM_REGISTRY) self.pg_storage.connect() self.layer1_repository = repositories.Layer1Repository(self.pg_storage, self.log) self.layer2_repository = repositories.Layer2Repository(self.pg_storage, self.log) diff --git a/app/tasks/layer2_import_designation.py b/app/tasks/layer2_import_designation.py index 61e4f9b6..3dd35328 100644 --- a/app/tasks/layer2_import_designation.py +++ b/app/tasks/layer2_import_designation.py @@ -3,10 +3,11 @@ import structlog +from app.data import enums as data_enums from app.data import model, repositories -from app.lib import containers, logging +from app.lib import containers from app.lib.storage import postgres -from app.tasks import interface +from app.tasks import interface, logging DESIGNATION_COLUMNS = ["design"] @@ -34,7 +35,7 @@ def name(cls) -> str: return "layer2-import-designation" def prepare(self, config: interface.Config) -> None: - self.pg_storage = postgres.PgStorage(config.storage, self.log) + self.pg_storage = postgres.PgStorage(config.storage, self.log, data_enums.PG_ENUM_REGISTRY) self.pg_storage.connect() self.layer1_repository = repositories.Layer1Repository(self.pg_storage, self.log) self.layer2_repository = repositories.Layer2Repository(self.pg_storage, self.log) diff --git a/app/tasks/layer2_import_icrs.py b/app/tasks/layer2_import_icrs.py index 9124eb4e..dd13d29a 100644 --- a/app/tasks/layer2_import_icrs.py +++ b/app/tasks/layer2_import_icrs.py @@ -6,10 +6,11 @@ import structlog from astropy import units as u +from app.data import enums as data_enums from app.data import model, repositories -from app.lib import containers, logging +from app.lib import containers from app.lib.storage import postgres -from app.tasks import interface +from app.tasks import interface, logging ICRS_COLUMNS = ["ra", "e_ra", "dec", "e_dec"] DEG = "deg" @@ -46,7 +47,7 @@ def name(cls) -> str: return "layer2-import-icrs" def prepare(self, config: interface.Config) -> None: - self.pg_storage = postgres.PgStorage(config.storage, self.log) + self.pg_storage = postgres.PgStorage(config.storage, self.log, data_enums.PG_ENUM_REGISTRY) self.pg_storage.connect() self.layer1_repository = repositories.Layer1Repository(self.pg_storage, self.log) self.layer2_repository = repositories.Layer2Repository(self.pg_storage, self.log) diff --git a/app/tasks/layer2_import_nature.py b/app/tasks/layer2_import_nature.py index d23ca1aa..3d3e929c 100644 --- a/app/tasks/layer2_import_nature.py +++ b/app/tasks/layer2_import_nature.py @@ -3,10 +3,11 @@ import structlog +from app.data import enums as data_enums from app.data import model, repositories -from app.lib import containers, logging +from app.lib import containers from app.lib.storage import postgres -from app.tasks import interface +from app.tasks import interface, logging @final @@ -32,7 +33,7 @@ def name(cls) -> str: return "layer2-import-nature" def prepare(self, config: interface.Config) -> None: - self.pg_storage = postgres.PgStorage(config.storage, self.log) + self.pg_storage = postgres.PgStorage(config.storage, self.log, data_enums.PG_ENUM_REGISTRY) self.pg_storage.connect() self.layer1_repository = repositories.Layer1Repository(self.pg_storage, self.log) self.layer2_repository = repositories.Layer2Repository(self.pg_storage, self.log) diff --git a/app/tasks/layer2_import_redshift.py b/app/tasks/layer2_import_redshift.py index c63ea05e..bad24902 100644 --- a/app/tasks/layer2_import_redshift.py +++ b/app/tasks/layer2_import_redshift.py @@ -6,10 +6,11 @@ import structlog from astropy import units as u +from app.data import enums as data_enums from app.data import model, repositories -from app.lib import containers, logging +from app.lib import containers from app.lib.storage import postgres -from app.tasks import interface +from app.tasks import interface, logging REDSHIFT_COLUMNS = ["cz", "e_cz"] VELOCITY_UNIT = "km/s" @@ -46,7 +47,7 @@ def name(cls) -> str: return "layer2-import-redshift" def prepare(self, config: interface.Config) -> None: - self.pg_storage = postgres.PgStorage(config.storage, self.log) + self.pg_storage = postgres.PgStorage(config.storage, self.log, data_enums.PG_ENUM_REGISTRY) self.pg_storage.connect() self.layer1_repository = repositories.Layer1Repository(self.pg_storage, self.log) self.layer2_repository = repositories.Layer2Repository(self.pg_storage, self.log) diff --git a/app/tasks/layer2_orphan_cleanup.py b/app/tasks/layer2_orphan_cleanup.py index 401ca260..cdfa8719 100644 --- a/app/tasks/layer2_orphan_cleanup.py +++ b/app/tasks/layer2_orphan_cleanup.py @@ -2,6 +2,7 @@ import structlog +from app.data import enums as data_enums from app.data import model, repositories from app.lib.storage import postgres from app.tasks import interface @@ -28,7 +29,7 @@ def name(cls) -> str: return "layer2-orphan-cleanup" def prepare(self, config: interface.Config) -> None: - self.pg_storage = postgres.PgStorage(config.storage, self.log) + self.pg_storage = postgres.PgStorage(config.storage, self.log, data_enums.PG_ENUM_REGISTRY) self.pg_storage.connect() self.layer2_repository = repositories.Layer2Repository(self.pg_storage, self.log) diff --git a/app/tasks/logging/__init__.py b/app/tasks/logging/__init__.py new file mode 100644 index 00000000..1ac622c5 --- /dev/null +++ b/app/tasks/logging/__init__.py @@ -0,0 +1,3 @@ +from app.tasks.logging.table import print_table + +__all__ = ["print_table"] diff --git a/app/lib/logging/table.py b/app/tasks/logging/table.py similarity index 100% rename from app/lib/logging/table.py rename to app/tasks/logging/table.py diff --git a/makefile b/makefile index 85c73db9..02a26d59 100644 --- a/makefile +++ b/makefile @@ -63,6 +63,11 @@ check: --config=pyproject.toml @echo "Linter ok." + @output=$$(uv run lint-imports 2>&1); exit_code=$$?; \ + if [ $$exit_code -ne 0 ]; then echo "$$output"; fi; \ + exit $$exit_code + @echo "Import contracts ok." + @output=$$(uv run basedpyright 2>&1); exit_code=$$?; \ if [ $$exit_code -ne 0 ]; then echo "$$output"; fi; \ exit $$exit_code @@ -74,7 +79,7 @@ check: @uv run pytest \ --quiet \ --config-file=pyproject.toml \ - tests/env_test.py tests/unit + tests/env_test.py tests/*/unit @echo "Testing ok." fix: @@ -116,13 +121,13 @@ update-template: ## General targets adminapi: - uv run app adminapi -c configs/dev/adminapi.yaml + uv run adminapi -c configs/dev/adminapi.yaml adminapi-dev: set -a && source .env.local && set +a && make adminapi dataapi: - uv run app dataapi -c configs/dev/dataapi.yaml + uv run dataapi -c configs/dev/dataapi.yaml start-db: docker-compose up -d @@ -138,7 +143,7 @@ start-prefect: uv run prefect server start start-tasks: - uv run app serve-tasks + uv run tasks docs: uvx \ diff --git a/pyproject.toml b/pyproject.toml index a99fc2e6..00ac521d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,6 @@ dependencies = [ "pyyaml>=6.0.2", "requests>=2.32.3", "regex>=2024.11.6", - "rq>=2.1.0", "structlog>=25.1.0", "swagger-ui-py>=23.9.23", "fastapi>=0.116.1", @@ -35,7 +34,10 @@ dependencies = [ ] [project.scripts] -"app" = "app.cli:cli" +app = "app.cli:cli" +adminapi = "app.adminapi.cli:main" +dataapi = "app.dataapi.cli:main" +tasks = "app.tasks.cli:main" [build-system] requires = ["hatchling"] @@ -62,6 +64,7 @@ dev = [ "pandas-stubs>=2.3.2.250827", "parameterized>=0.9.0", "testcontainers-postgres>=0.0.1rc1", + "import-linter>=2.0", ] [tool.pytest.ini_options] @@ -186,3 +189,26 @@ ignore_names = [ "dotenv_settings", "file_secret_settings", ] + +[tool.importlinter] +root_package = "app" + +[[tool.importlinter.contracts]] +id = "services-independent" +name = "Services must not import each other" +type = "independence" +modules = [ + "app.adminapi", + "app.dataapi", + "app.tasks", +] + +[[tool.importlinter.contracts]] +id = "layers" +name = "Layered architecture" +type = "layers" +layers = [ + "app.adminapi | app.dataapi | app.tasks", + "app.data", + "app.lib", +] diff --git a/app/commands/__init__.py b/tests/adminapi/__init__.py similarity index 100% rename from app/commands/__init__.py rename to tests/adminapi/__init__.py diff --git a/app/domain/__init__.py b/tests/adminapi/integration/__init__.py similarity index 100% rename from app/domain/__init__.py rename to tests/adminapi/integration/__init__.py diff --git a/tests/integration/adminapi_auth_test.py b/tests/adminapi/integration/adminapi_auth_test.py similarity index 99% rename from tests/integration/adminapi_auth_test.py rename to tests/adminapi/integration/adminapi_auth_test.py index 4e594eda..8f49fb31 100644 --- a/tests/integration/adminapi_auth_test.py +++ b/tests/adminapi/integration/adminapi_auth_test.py @@ -41,7 +41,6 @@ def setUpClass(cls) -> None: [ "uv", "run", - "app", "adminapi", "-c", "configs/dev/adminapi.yaml", diff --git a/tests/integration/adminapi_server_test.py b/tests/adminapi/integration/adminapi_server_test.py similarity index 98% rename from tests/integration/adminapi_server_test.py rename to tests/adminapi/integration/adminapi_server_test.py index 50e58e28..bf6beac8 100644 --- a/tests/integration/adminapi_server_test.py +++ b/tests/adminapi/integration/adminapi_server_test.py @@ -45,7 +45,6 @@ def setUpClass(cls) -> None: [ "uv", "run", - "app", "adminapi", "-c", "configs/dev/adminapi.yaml", diff --git a/tests/integration/assign_record_pgcs_test.py b/tests/adminapi/integration/assign_record_pgcs_test.py similarity index 98% rename from tests/integration/assign_record_pgcs_test.py rename to tests/adminapi/integration/assign_record_pgcs_test.py index 5debc151..b0c86376 100644 --- a/tests/integration/assign_record_pgcs_test.py +++ b/tests/adminapi/integration/assign_record_pgcs_test.py @@ -3,11 +3,11 @@ import structlog +from app.adminapi import presentation as adminapi +from app.adminapi.domain import crossmatch from app.data import model, repositories -from app.domain.adminapi import crossmatch from app.lib.storage import enums from app.lib.web import errors -from app.presentation import adminapi from tests import lib diff --git a/tests/integration/catalogs_api_test.py b/tests/adminapi/integration/catalogs_api_test.py similarity index 94% rename from tests/integration/catalogs_api_test.py rename to tests/adminapi/integration/catalogs_api_test.py index f9c28dbd..a3115d5b 100644 --- a/tests/integration/catalogs_api_test.py +++ b/tests/adminapi/integration/catalogs_api_test.py @@ -3,12 +3,12 @@ import structlog from starlette import testclient +from app.adminapi import clients, domain +from app.adminapi.domain.mock import get_mock_table_stats_cache +from app.adminapi.presentation.server import Server from app.data import repositories -from app.domain import adminapi as domain -from app.domain.adminapi.mock import get_mock_table_stats_cache -from app.lib import audit, auth, clients +from app.lib import audit, auth from app.lib.web import server -from app.presentation.adminapi.server import Server from tests import lib diff --git a/tests/integration/create_table_test.py b/tests/adminapi/integration/create_table_test.py similarity index 95% rename from tests/integration/create_table_test.py rename to tests/adminapi/integration/create_table_test.py index 7ac2d2b3..36bbe4a7 100644 --- a/tests/integration/create_table_test.py +++ b/tests/adminapi/integration/create_table_test.py @@ -2,12 +2,10 @@ import structlog +from app.adminapi import clients, domain, presentation +from app.adminapi.domain.mock import get_mock_table_stats_cache from app.data import repositories -from app.domain import adminapi as domain -from app.domain.adminapi.mock import get_mock_table_stats_cache -from app.lib import clients from app.lib.storage import enums -from app.presentation import adminapi as presentation from tests import lib diff --git a/tests/integration/merge_pgcs_test.py b/tests/adminapi/integration/merge_pgcs_test.py similarity index 97% rename from tests/integration/merge_pgcs_test.py rename to tests/adminapi/integration/merge_pgcs_test.py index 17bf3252..5d7d654e 100644 --- a/tests/integration/merge_pgcs_test.py +++ b/tests/adminapi/integration/merge_pgcs_test.py @@ -4,10 +4,10 @@ import pydantic import structlog +from app.adminapi import presentation as adminapi +from app.adminapi.domain import pgc from app.data import model, repositories -from app.domain.adminapi import pgc from app.lib.web import errors -from app.presentation import adminapi from tests import lib diff --git a/tests/integration/rawdata_table_test.py b/tests/adminapi/integration/rawdata_table_test.py similarity index 98% rename from tests/integration/rawdata_table_test.py rename to tests/adminapi/integration/rawdata_table_test.py index a65ff10d..6fe0819d 100644 --- a/tests/integration/rawdata_table_test.py +++ b/tests/adminapi/integration/rawdata_table_test.py @@ -5,13 +5,11 @@ import structlog from pandas import DataFrame +from app.adminapi import clients, domain, presentation +from app.adminapi.domain.mock import get_mock_table_stats_cache from app.data import model, repositories -from app.domain import adminapi as domain -from app.domain.adminapi.mock import get_mock_table_stats_cache -from app.lib import clients from app.lib.storage import enums from app.lib.storage.mapping import TYPE_INTEGER, TYPE_TEXT -from app.presentation import adminapi as presentation from tests import lib diff --git a/app/presentation/__init__.py b/tests/adminapi/unit/__init__.py similarity index 100% rename from app/presentation/__init__.py rename to tests/adminapi/unit/__init__.py diff --git a/tests/unit/lib/cache_test.py b/tests/adminapi/unit/cache_test.py similarity index 95% rename from tests/unit/lib/cache_test.py rename to tests/adminapi/unit/cache_test.py index 08fb513e..b52c34e6 100644 --- a/tests/unit/lib/cache_test.py +++ b/tests/adminapi/unit/cache_test.py @@ -7,8 +7,8 @@ import pydantic -from app.lib.cache import BackgroundCache -from app.lib.cache.background_cache import _run_in_context +from app.adminapi.cache import BackgroundCache +from app.adminapi.cache.background_cache import _run_in_context class SampleSnapshot(pydantic.BaseModel): diff --git a/tests/unit/domain/create_source_test.py b/tests/adminapi/unit/create_source_test.py similarity index 91% rename from tests/unit/domain/create_source_test.py rename to tests/adminapi/unit/create_source_test.py index 5aeb8498..020f0676 100644 --- a/tests/unit/domain/create_source_test.py +++ b/tests/adminapi/unit/create_source_test.py @@ -1,6 +1,6 @@ import unittest -from app.domain.adminapi.sources import construct_code +from app.adminapi.domain.sources import construct_code class ConstructCodeTest(unittest.TestCase): diff --git a/tests/unit/domain/table_stats_test.py b/tests/adminapi/unit/table_stats_test.py similarity index 97% rename from tests/unit/domain/table_stats_test.py rename to tests/adminapi/unit/table_stats_test.py index e8b769c7..24432ba0 100644 --- a/tests/unit/domain/table_stats_test.py +++ b/tests/adminapi/unit/table_stats_test.py @@ -1,8 +1,8 @@ import unittest from unittest import mock +from app.adminapi.domain import table_stats from app.data import model -from app.domain.adminapi import table_stats class TableStatsTest(unittest.TestCase): diff --git a/tests/unit/domain/table_upload_test.py b/tests/adminapi/unit/table_upload_test.py similarity index 98% rename from tests/unit/domain/table_upload_test.py rename to tests/adminapi/unit/table_upload_test.py index 8798c508..76821c3e 100644 --- a/tests/unit/domain/table_upload_test.py +++ b/tests/adminapi/unit/table_upload_test.py @@ -5,14 +5,12 @@ from astropy import units from parameterized import param, parameterized +from app.adminapi import clients, domain, presentation +from app.adminapi.domain.mock import get_mock_table_stats_cache +from app.adminapi.domain.table_upload import domain_descriptions_to_data, get_source_id from app.data import model, repositories -from app.domain import adminapi as domain -from app.domain.adminapi.mock import get_mock_table_stats_cache -from app.domain.adminapi.table_upload import domain_descriptions_to_data, get_source_id -from app.lib import clients from app.lib.storage import enums, mapping from app.lib.web import errors -from app.presentation import adminapi as presentation from tests import lib diff --git a/tests/assets/default_rules.csv b/tests/assets/default_rules.csv deleted file mode 100644 index 0b72cbc3..00000000 --- a/tests/assets/default_rules.csv +++ /dev/null @@ -1,16 +0,0 @@ -"catalog","parameter","key","filters","priority" -icrs,ra,"","{""ucd"": ""pos.eq.ra""}",1 -icrs,dec,"","{""ucd"": ""pos.eq.dec""}",1 -icrs,ra,"","{""ucd"": ""pos.eq.ra;meta.main""}",2 -icrs,dec,"","{""ucd"": ""pos.eq.dec;meta.main""}",2 -icrs,dec,"","{""ucd"": ""pos.eq.dec;meta.main;stat.mean""}",3 -icrs,ra,"","{""ucd"": ""pos.eq.ra;meta.main;stat.mean""}",3 -designation,design,"","{""ucd"": ""meta.id""}",1 -designation,design,"","{""ucd"": ""meta.id"", ""datatype_oneof"": [""text""]}",2 -designation,design,"","{""ucd"": ""meta.id;meta.main""}",3 -designation,design,"","{""ucd"": ""meta.id;meta.main"", ""datatype_oneof"": [""text""]}",4 -redshift,z,"","{""ucd"": ""src.redshift""}",1 -redshift,cz,"","{""ucd"": ""phys.veloc""}",1 -redshift,cz,"","{""ucd"": ""phys.veloc;meta.main""}",2 -redshift,cz,"","{""ucd"": ""phys.veloc;meta.main;stat.mean""}",3 -redshift,e_cz,"","{""ucd"": ""stat.error;phys.veloc""}",1 \ No newline at end of file diff --git a/tests/integration/__init__.py b/tests/data/__init__.py similarity index 100% rename from tests/integration/__init__.py rename to tests/data/__init__.py diff --git a/tests/unit/__init__.py b/tests/data/integration/__init__.py similarity index 100% rename from tests/unit/__init__.py rename to tests/data/integration/__init__.py diff --git a/tests/integration/layer0_tables_repository_test.py b/tests/data/integration/layer0_tables_repository_test.py similarity index 100% rename from tests/integration/layer0_tables_repository_test.py rename to tests/data/integration/layer0_tables_repository_test.py diff --git a/tests/integration/layer1_repository_test.py b/tests/data/integration/layer1_repository_test.py similarity index 100% rename from tests/integration/layer1_repository_test.py rename to tests/data/integration/layer1_repository_test.py diff --git a/tests/integration/layer2_repository_test.py b/tests/data/integration/layer2_repository_test.py similarity index 100% rename from tests/integration/layer2_repository_test.py rename to tests/data/integration/layer2_repository_test.py diff --git a/tests/unit/data/__init__.py b/tests/data/unit/__init__.py similarity index 100% rename from tests/unit/data/__init__.py rename to tests/data/unit/__init__.py diff --git a/tests/unit/data/layer0_repository_test.py b/tests/data/unit/layer0_repository_test.py similarity index 100% rename from tests/unit/data/layer0_repository_test.py rename to tests/data/unit/layer0_repository_test.py diff --git a/tests/unit/domain/__init__.py b/tests/dataapi/__init__.py similarity index 100% rename from tests/unit/domain/__init__.py rename to tests/dataapi/__init__.py diff --git a/tests/unit/lib/__init__.py b/tests/dataapi/integration/__init__.py similarity index 100% rename from tests/unit/lib/__init__.py rename to tests/dataapi/integration/__init__.py diff --git a/tests/integration/dataapi_server_test.py b/tests/dataapi/integration/dataapi_server_test.py similarity index 99% rename from tests/integration/dataapi_server_test.py rename to tests/dataapi/integration/dataapi_server_test.py index ebafe27c..a2d7911a 100644 --- a/tests/integration/dataapi_server_test.py +++ b/tests/dataapi/integration/dataapi_server_test.py @@ -51,7 +51,6 @@ def setUpClass(cls) -> None: [ "uv", "run", - "app", "dataapi", "-c", "configs/dev/dataapi.yaml", diff --git a/tests/integration/metadata_api_test.py b/tests/dataapi/integration/metadata_api_test.py similarity index 95% rename from tests/integration/metadata_api_test.py rename to tests/dataapi/integration/metadata_api_test.py index a75ccdc6..6b9173f0 100644 --- a/tests/integration/metadata_api_test.py +++ b/tests/dataapi/integration/metadata_api_test.py @@ -5,12 +5,12 @@ import structlog from starlette import testclient -from app.commands.dataapi import command as dataapi_command +import app.dataapi.command as dataapi_command from app.data import repositories -from app.domain import dataapi as domain -from app.domain.dataapi import actions as dataapi_actions +from app.dataapi import domain +from app.dataapi.domain import actions as dataapi_actions +from app.dataapi.presentation.server import Server from app.lib import auth -from app.presentation.dataapi.server import Server from tests import lib @@ -18,7 +18,7 @@ class MetadataAPITest(unittest.TestCase): @classmethod def setUpClass(cls) -> None: cls.pg_storage = lib.TestPostgresStorage.get() - cfg_path = pathlib.Path(__file__).resolve().parents[2] / "configs" / "dev" / "dataapi.yaml" + cfg_path = pathlib.Path(__file__).resolve().parents[3] / "configs" / "dev" / "dataapi.yaml" cls.cfg = dataapi_command.parse_config(str(cfg_path)) cls.log = structlog.get_logger() diff --git a/tests/dataapi/unit/__init__.py b/tests/dataapi/unit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/domain/parameterized_query_test.py b/tests/dataapi/unit/parameterized_query_test.py similarity index 96% rename from tests/unit/domain/parameterized_query_test.py rename to tests/dataapi/unit/parameterized_query_test.py index 377f2dc0..d59ce28d 100644 --- a/tests/unit/domain/parameterized_query_test.py +++ b/tests/dataapi/unit/parameterized_query_test.py @@ -1,7 +1,7 @@ import unittest from app.data import model -from app.domain.dataapi import parameterized_query +from app.dataapi.domain import parameterized_query DEFAULT = [ model.RawCatalog.DESIGNATION, diff --git a/tests/env_test.py b/tests/env_test.py index 1e67ef58..ca9b9966 100644 --- a/tests/env_test.py +++ b/tests/env_test.py @@ -2,8 +2,8 @@ from parameterized import parameterized -import app.commands.adminapi.command as adminapi -import app.commands.dataapi.command as dataapi +import app.adminapi.command as adminapi +import app.dataapi.command as dataapi class TestEnvironment(unittest.TestCase): diff --git a/tests/lib/integration/__init__.py b/tests/lib/integration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/transactional_storage_test.py b/tests/lib/integration/transactional_storage_test.py similarity index 100% rename from tests/integration/transactional_storage_test.py rename to tests/lib/integration/transactional_storage_test.py diff --git a/tests/lib/postgres.py b/tests/lib/postgres.py index 4e9ea222..4920cf54 100644 --- a/tests/lib/postgres.py +++ b/tests/lib/postgres.py @@ -6,6 +6,7 @@ import structlog from testcontainers import postgres as pgcontainer +from app.data import enums as data_enums from app.lib.storage import postgres from tests.lib import web @@ -52,7 +53,7 @@ def __init__(self, migrations_dir: str) -> None: dbname="hyperleda", ) - self.storage = postgres.PgStorage(self.config, logger) + self.storage = postgres.PgStorage(self.config, logger, data_enums.PG_ENUM_REGISTRY) self.migrations_dir = migrations_dir diff --git a/tests/lib/unit/__init__.py b/tests/lib/unit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/lib/astronomy_test.py b/tests/lib/unit/astronomy_test.py similarity index 100% rename from tests/unit/lib/astronomy_test.py rename to tests/lib/unit/astronomy_test.py diff --git a/tests/unit/lib/audit_truncate_test.py b/tests/lib/unit/audit_truncate_test.py similarity index 100% rename from tests/unit/lib/audit_truncate_test.py rename to tests/lib/unit/audit_truncate_test.py diff --git a/tests/unit/lib/auth_test.py b/tests/lib/unit/auth_test.py similarity index 100% rename from tests/unit/lib/auth_test.py rename to tests/lib/unit/auth_test.py diff --git a/tests/unit/lib/commands_test.py b/tests/lib/unit/commands_test.py similarity index 100% rename from tests/unit/lib/commands_test.py rename to tests/lib/unit/commands_test.py diff --git a/tests/unit/lib/concurrency_test.py b/tests/lib/unit/concurrency_test.py similarity index 100% rename from tests/unit/lib/concurrency_test.py rename to tests/lib/unit/concurrency_test.py diff --git a/tests/unit/lib/containers_test.py b/tests/lib/unit/containers_test.py similarity index 100% rename from tests/unit/lib/containers_test.py rename to tests/lib/unit/containers_test.py diff --git a/tests/unit/lib/log_middleware_test.py b/tests/lib/unit/log_middleware_test.py similarity index 100% rename from tests/unit/lib/log_middleware_test.py rename to tests/lib/unit/log_middleware_test.py diff --git a/tests/unit/lib/server_test.py b/tests/lib/unit/server_test.py similarity index 100% rename from tests/unit/lib/server_test.py rename to tests/lib/unit/server_test.py diff --git a/tests/unit/lib/tracing_middleware_test.py b/tests/lib/unit/tracing_middleware_test.py similarity index 100% rename from tests/unit/lib/tracing_middleware_test.py rename to tests/lib/unit/tracing_middleware_test.py diff --git a/tests/regression/patch_table_metadata.py b/tests/regression/patch_table_metadata.py index 3bb16b7d..f7f50a7f 100644 --- a/tests/regression/patch_table_metadata.py +++ b/tests/regression/patch_table_metadata.py @@ -3,8 +3,8 @@ import requests +from app.adminapi import presentation as adminapi from app.lib.storage import enums -from app.presentation import adminapi from tests import lib TABLE_DESCRIPTION_NEW = "Regression: patched table description" diff --git a/tests/regression/upload_simple_table.py b/tests/regression/upload_simple_table.py index 89b0b6df..3b29b6f3 100644 --- a/tests/regression/upload_simple_table.py +++ b/tests/regression/upload_simple_table.py @@ -9,8 +9,8 @@ import structlog from app import tasks +from app.adminapi import presentation as adminapi from app.lib.storage import enums -from app.presentation import adminapi from tests import lib random.seed(time.time()) diff --git a/tests/tasks/__init__.py b/tests/tasks/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/tasks/integration/__init__.py b/tests/tasks/integration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/layer2_import_test.py b/tests/tasks/integration/layer2_import_test.py similarity index 100% rename from tests/integration/layer2_import_test.py rename to tests/tasks/integration/layer2_import_test.py diff --git a/tests/tasks/unit/__init__.py b/tests/tasks/unit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/tasks/layer2_import_test.py b/tests/tasks/unit/layer2_import_test.py similarity index 100% rename from tests/unit/tasks/layer2_import_test.py rename to tests/tasks/unit/layer2_import_test.py diff --git a/tests/unit/commands/serve_tasks_test.py b/tests/tasks/unit/serve_tasks_test.py similarity index 97% rename from tests/unit/commands/serve_tasks_test.py rename to tests/tasks/unit/serve_tasks_test.py index d901dd81..6f051297 100644 --- a/tests/unit/commands/serve_tasks_test.py +++ b/tests/tasks/unit/serve_tasks_test.py @@ -1,6 +1,6 @@ import unittest -from app.commands.serve_tasks import flows +from app.tasks import flows class ServeTasksTest(unittest.TestCase): diff --git a/uv.lock b/uv.lock index 856b8e80..f0fdb5ce 100644 --- a/uv.lock +++ b/uv.lock @@ -94,7 +94,6 @@ dependencies = [ { name = "pyyaml" }, { name = "regex" }, { name = "requests" }, - { name = "rq" }, { name = "slowapi" }, { name = "starlette" }, { name = "structlog" }, @@ -108,6 +107,7 @@ dev = [ { name = "basedpyright" }, { name = "datamodel-code-generator" }, { name = "fabric" }, + { name = "import-linter" }, { name = "pandas-stubs" }, { name = "parameterized" }, { name = "pymdown-extensions" }, @@ -143,7 +143,6 @@ requires-dist = [ { name = "pyyaml", specifier = ">=6.0.2" }, { name = "regex", specifier = ">=2024.11.6" }, { name = "requests", specifier = ">=2.32.3" }, - { name = "rq", specifier = ">=2.1.0" }, { name = "slowapi", specifier = ">=0.1.9" }, { name = "starlette", specifier = ">=0.47.2" }, { name = "structlog", specifier = ">=25.1.0" }, @@ -157,6 +156,7 @@ dev = [ { name = "basedpyright", specifier = "~=1.38.3" }, { name = "datamodel-code-generator", specifier = ">=0.26.5" }, { name = "fabric", specifier = ">=3.2.2" }, + { name = "import-linter", specifier = ">=2.0" }, { name = "pandas-stubs", specifier = ">=2.3.2.250827" }, { name = "parameterized", specifier = ">=0.9.0" }, { name = "pymdown-extensions", specifier = ">=10.14.1" }, @@ -522,18 +522,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/97/35/cf824e535233c432048bd7455b28808776471ab5861375772b2c98ea4cbd/coolname-5.0.0-py3-none-any.whl", hash = "sha256:b86eea9670aa0620965167d1bfadfe654fabec839a5b51e8da611c4a64f86192", size = 47368, upload-time = "2026-04-23T06:04:49.374Z" }, ] -[[package]] -name = "croniter" -version = "6.2.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "python-dateutil" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/df/de/5832661ed55107b8a09af3f0a2e71e0957226a59eb1dcf0a445cce6daf20/croniter-6.2.2.tar.gz", hash = "sha256:ba60832a5ec8e12e51b8691c3309a113d1cf6526bdf1a48150ce8ec7a532d0ab", size = 113762, upload-time = "2026-03-15T08:43:48.112Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d0/39/783980e78cb92c2d7bdb1fc7dbc86e94ccc6d58224d76a7f1f51b6c51e30/croniter-6.2.2-py3-none-any.whl", hash = "sha256:a5d17b1060974d36251ea4faf388233eca8acf0d09cbd92d35f4c4ac8f279960", size = 45422, upload-time = "2026-03-15T08:43:46.626Z" }, -] - [[package]] name = "cronsim" version = "2.7" @@ -806,6 +794,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e4/d3/5268aeabf2ad82658c4e2ff3a060648d0f02f3926cb53247c0e4d0dab49e/griffelib-2.1.0-py3-none-any.whl", hash = "sha256:cc7b3d2d2865ad0b909fcc38086e3f554b5ea7acbaa7bbb7ecaa3f5dfb7d9f00", size = 142560, upload-time = "2026-06-19T12:05:38.742Z" }, ] +[[package]] +name = "grimp" +version = "3.15" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b2/73/ce58881177b003def779c87b5e10f396deef068933c97d6d206bd46d4cb7/grimp-3.15.tar.gz", hash = "sha256:91b57d4d801dc107ebfb5a7040d4777a152c579b5dc202426e1185e50931fe1e", size = 831734, upload-time = "2026-07-03T12:09:36.244Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/6a/3e0a0760cc509cd09764d128de78a1f329740306c74e42dba82c58a99118/grimp-3.15-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:f19b957053d1c736aaa0015eed2d4855bb2511637c5360d32b3c5ea045904e7a", size = 2143197, upload-time = "2026-07-03T12:08:51.685Z" }, + { url = "https://files.pythonhosted.org/packages/c8/2e/127cbce04a5603d382c6a2bbc1a19b6889be49d60b88864bcdb174c8926d/grimp-3.15-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a6480472c2d1f7f6a903906c92e9897d4e3dee5d69ce3881f04368d4ec6d2dde", size = 2088342, upload-time = "2026-07-03T12:08:44.227Z" }, + { url = "https://files.pythonhosted.org/packages/b0/36/af9df683bf6c8711e0e9136876ca130f9971102d945ff3a36d0c45dae2ec/grimp-3.15-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c14b64dbf2e4397df2e35fa8aa7028533621ecf1f8ea7ec24bc296a9c695ea4", size = 2254509, upload-time = "2026-07-03T12:07:35.809Z" }, + { url = "https://files.pythonhosted.org/packages/02/f5/e712633b68ea14d04e7de84ede4f8ddbba763d5f2d29ae8d6af721f84870/grimp-3.15-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:26364c2c9f7db88243365299b4d1c4d948b74304ff03c8a6e4f028147fed3b22", size = 2193831, upload-time = "2026-07-03T12:07:46.309Z" }, + { url = "https://files.pythonhosted.org/packages/13/74/151bdd73d6a60bd77d4b956f36d03dd558dd46a9b5ec8f5ad15921b503d7/grimp-3.15-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:69331e1be693415596c6084342059b9ba3ecf1cfe2e3b1c761598dd2fe14d522", size = 2345289, upload-time = "2026-07-03T12:08:19.458Z" }, + { url = "https://files.pythonhosted.org/packages/a1/b8/3fc950fa73b757cbc35f77542bd662f431b9a8f360e63196ded640771a33/grimp-3.15-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fe397991c269868c2fb08114099b2aa4f1bc803d03fadaaf97e006019f9e5da2", size = 2604407, upload-time = "2026-07-03T12:07:57.217Z" }, + { url = "https://files.pythonhosted.org/packages/0a/d8/98916b9dc0b89a3e89e0d714ce0872be859fff40ceee3e2cb6886b106eb4/grimp-3.15-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1d556bd62664ee044c47cff64d737be132408064d4ba68ca9f756cb29b41cc2d", size = 2326769, upload-time = "2026-07-03T12:08:08.773Z" }, + { url = "https://files.pythonhosted.org/packages/63/51/39595c5857f609e976e0bba1f19c1f45182ee4d8d2ea5cdfab72841eafbc/grimp-3.15-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9061c5b6f01130ff8639c49c80176d29d921acc36741b2ae0b763a7668106082", size = 2272498, upload-time = "2026-07-03T12:08:32.03Z" }, + { url = "https://files.pythonhosted.org/packages/7d/d8/a44cc9db500ae80c45425238ee44d9556796aa88b8c0649cfa613fb88d14/grimp-3.15-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a9d04c195f0c6da4476361560d3d206a6a784b9eb0da7156fc6974511337e2e3", size = 2431075, upload-time = "2026-07-03T12:08:58.935Z" }, + { url = "https://files.pythonhosted.org/packages/a9/41/544f197ddb44990789683c25740ffa72474a57f0b16bc6b4a544edf9a2a9/grimp-3.15-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:01fd68c74cfd08b110bfd338d77efaa470670530f7179e6977764f44cd74d5cc", size = 2467361, upload-time = "2026-07-03T12:09:09.275Z" }, + { url = "https://files.pythonhosted.org/packages/3b/b1/8261018b9f1ab47fffbb7739d7af3f04c8b529555b7fb2ae8b742d42d3db/grimp-3.15-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:bd1ddd428a124d6730bed49ea60fb2ca6c8e0640c8f5abe1d0f6fc27a92fd8bc", size = 2503500, upload-time = "2026-07-03T12:09:19.585Z" }, + { url = "https://files.pythonhosted.org/packages/db/d6/99a2421c4c0de9f203a7e246030b13b676766e62e48504883863942646fb/grimp-3.15-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:07e645e9d45ed43bb8ea8da5982c19eacf13ee685d8440e3dc280064c005599c", size = 2513834, upload-time = "2026-07-03T12:09:29.758Z" }, + { url = "https://files.pythonhosted.org/packages/62/a5/263729e23d64541cd99d36dbb592e29c1ecea803deb3bb5c0c463b43c2ec/grimp-3.15-cp313-cp313-win32.whl", hash = "sha256:473646e0a74a554b4ab071d7fcbf5f442eb8cf87561770dae269818636b8edf2", size = 1855743, upload-time = "2026-07-03T12:09:49.789Z" }, + { url = "https://files.pythonhosted.org/packages/1c/8c/a072bbea2e2e94da38f90bd8794037c90fb4eba389b49d01cdd2bb85e13c/grimp-3.15-cp313-cp313-win_amd64.whl", hash = "sha256:dbc2c15a1fbca2ff358f86cc90067176096dd73bec27d002515521b3125ba507", size = 1984546, upload-time = "2026-07-03T12:09:42.543Z" }, +] + [[package]] name = "grpcio" version = "1.80.0" @@ -931,6 +941,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/94/16/70255075a9859a0e3adb789b68ceb0e210dec03934245fd98d248226572f/idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5", size = 74165, upload-time = "2026-05-22T00:16:16.698Z" }, ] +[[package]] +name = "import-linter" +version = "2.13" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "grimp" }, + { name = "rich" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/97/c6/42962eb043df4d6984c1540220735b20442572fe37dab5e65ac807c939b9/import_linter-2.13.tar.gz", hash = "sha256:13af4a1d6b06044c58ea784e8732fd7fe48eec821a75feb4d6a1a2de36dd5c27", size = 1279761, upload-time = "2026-07-03T14:00:31.285Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/13/e7725e6eb32607fd4af51ccf3edfe835826e5cdf783b4d7fdc8f459196ae/import_linter-2.13-py3-none-any.whl", hash = "sha256:c0372e7ee5e15657bc06a8e841445e13237afd738a672d26863dc927af9f0bf5", size = 638185, upload-time = "2026-07-03T14:00:29.676Z" }, +] + [[package]] name = "inflect" version = "7.5.0" @@ -2217,20 +2242,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/27/17/b8642c12930b71bc2b25831f6708ccf0f75abcd11883932ec9ce54ba3a78/rpds_py-2026.6.3-cp313-cp313-win_arm64.whl", hash = "sha256:900a67df3fd1660b035a4761c4ce73c382ea6b35f90f9863c36c6fd8bf8b09bb", size = 215063, upload-time = "2026-06-30T07:15:56.573Z" }, ] -[[package]] -name = "rq" -version = "2.9.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "croniter" }, - { name = "redis" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/15/5e/43a7a61f3ebfa79789c72bf442a47fd80bb1a743caeea47b2b833001f388/rq-2.9.0.tar.gz", hash = "sha256:db5dfc1e1fe80ef977fd557d4305107dcf99a80b33d381c9a06e8a2bb11730e5", size = 744959, upload-time = "2026-05-19T15:00:29.894Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4d/a9/aa38ae2505e5dcb1e898f83a263e703b05829580f75ee86c5e480760ae1a/rq-2.9.0-py3-none-any.whl", hash = "sha256:665b9ad34e36ea15913e60d2a32e1775fef1e9aff907bcae297d6f70dccffed2", size = 120113, upload-time = "2026-05-19T15:00:27.511Z" }, -] - [[package]] name = "ruamel-yaml" version = "0.19.1"