re-slice internal structure of the project

.github/workflows/ci.yaml

@@ -41,7 +41,11 @@ jobs:
           tags: local/app:latest
 
       - name: Run image help
-        run: docker run --rm --entrypoint "app" local/app:latest --help
+        run: |
+          docker run --rm --entrypoint "app" local/app:latest --help
+          docker run --rm --entrypoint "adminapi" local/app:latest --help
+          docker run --rm --entrypoint "dataapi" local/app:latest --help
+          docker run --rm --entrypoint "tasks" local/app:latest --help
 
       - name: Save image
         run: docker save local/app:latest -o image.tar

app/adminapi/cache/__init__.py

@@ -0,0 +1,5 @@
+from app.adminapi.cache.background_cache import BackgroundCache
+
+__all__ = [
+    "BackgroundCache",
+]

app/adminapi/cli.py

@@ -0,0 +1,18 @@
+import os
+
+import click
+
+from app.adminapi.command import AdminAPICommand
+from app.lib import commands
+
+
+@click.command(short_help=AdminAPICommand.help())
+@click.option(
+    "-c",
+    "--config",
+    type=str,
+    default=lambda: os.environ.get("CONFIG", ""),
+    help="Path to configuration file",
+)
+def main(config: str) -> None:
+    commands.run(AdminAPICommand(config))

app/commands/adminapi/command.py → app/adminapi/command.py

@@ -8,14 +8,14 @@
 import structlog
 import yaml
 
+from app.adminapi import cache, clients, domain, presentation
+from app.adminapi.domain import table_stats
+from app.data import enums as data_enums
 from app.data import repositories
-from app.domain import adminapi as domain
-from app.domain.adminapi import table_stats
-from app.lib import audit, auth, cache, clients, commands, config, tracing
+from app.lib import audit, auth, commands, config, tracing
 from app.lib.storage import postgres
 from app.lib.tracing import TracingConfig
 from app.lib.web import server
-from app.presentation import adminapi as presentation
 
 log: structlog.stdlib.BoundLogger = structlog.get_logger()
 
@@ -38,7 +38,7 @@ def prepare(self):
 
         tracing.setup_tracing("adminapi", cfg.tracing)
 
-        self.pg_storage = postgres.PgStorage(cfg.storage, log)
+        self.pg_storage = postgres.PgStorage(cfg.storage, log, data_enums.PG_ENUM_REGISTRY)
         self.pg_storage.connect()
 
         authenticator: auth.Authenticator = (

app/adminapi/domain/__init__.py

@@ -0,0 +1,15 @@
+from app.adminapi.domain.actions import Actions
+from app.adminapi.domain.crossmatch import CrossmatchManager
+from app.adminapi.domain.login import LoginManager
+from app.adminapi.domain.mock import get_mock_actions
+from app.adminapi.domain.sources import SourceManager
+from app.adminapi.domain.table_upload import TableUploadManager
+
+__all__ = [
+    "Actions",
+    "CrossmatchManager",
+    "get_mock_actions",
+    "LoginManager",
+    "TableUploadManager",
+    "SourceManager",
+]

app/domain/adminapi/actions.py → app/adminapi/domain/actions.py

@@ -1,9 +1,10 @@
 from typing import final
 
+from app.adminapi import cache, clients
+from app.adminapi import presentation as adminapi
+from app.adminapi.domain import catalogs, crossmatch, layer1_write, login, pgc, sources, table_upload
 from app.data import repositories
-from app.domain.adminapi import catalogs, crossmatch, layer1_write, login, pgc, sources, table_upload
-from app.lib import auth, cache, clients
-from app.presentation import adminapi
+from app.lib import auth
 
 
 @final

app/domain/adminapi/catalogs.py → app/adminapi/domain/catalogs.py

@@ -1,7 +1,7 @@
 from typing import Any, final
 
+from app.adminapi import presentation as adminapi
 from app.data import model, repositories
-from app.presentation import adminapi
 
 _INTERNAL_COLUMNS = frozenset({"record_id", "object_id", "id", "modification_time"})
 

app/domain/adminapi/crossmatch.py → app/adminapi/domain/crossmatch.py

@@ -4,13 +4,13 @@
 from astropy import coordinates
 from astropy import units as u
 
+from app.adminapi import presentation as adminapi
 from app.data import model
 from app.data.repositories import layer0, layer1, layer2
 from app.data.repositories.layer0.records import AssignRecordPgcsPreconditionError
 from app.lib import astronomy
 from app.lib.storage import enums
 from app.lib.web.errors import ConflictError, NotFoundError
-from app.presentation import adminapi
 
 logger = structlog.stdlib.get_logger()
 

app/domain/adminapi/layer1_write.py → app/adminapi/domain/layer1_write.py

@@ -1,10 +1,10 @@
 import psycopg
 from astropy import units as u
 
+from app.adminapi import presentation as adminapi
 from app.data import model
 from app.data.repositories import layer1
 from app.lib.web.errors import RuleValidationError
-from app.presentation import adminapi
 
 
 class Layer1Writer:

app/domain/adminapi/login.py → app/adminapi/domain/login.py

@@ -1,6 +1,6 @@
+from app.adminapi import presentation as adminapi
 from app.lib import auth
 from app.lib.web.errors import UnauthorizedError
-from app.presentation import adminapi
 
 
 class LoginManager:

app/domain/adminapi/mock.py → app/adminapi/domain/mock.py

@@ -1,9 +1,10 @@
 import datetime
 from unittest import mock
 
-from app.domain.adminapi import actions
-from app.lib import auth, clients
-from app.presentation import adminapi
+from app.adminapi import clients
+from app.adminapi import presentation as adminapi
+from app.adminapi.domain import actions
+from app.lib import auth
 
 
 def get_mock_table_stats_cache() -> mock.MagicMock:

app/domain/adminapi/pgc.py → app/adminapi/domain/pgc.py

@@ -1,8 +1,8 @@
 from typing import final
 
+from app.adminapi import presentation as adminapi
 from app.data import repositories
 from app.lib.web.errors import NotFoundError
-from app.presentation import adminapi
 
 
 @final

app/domain/adminapi/sources.py → app/adminapi/domain/sources.py

@@ -1,5 +1,5 @@
+from app.adminapi import presentation as adminapi
 from app.data import repositories
-from app.presentation import adminapi
 
 ALLOWED_CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_"
 

app/domain/adminapi/table_stats.py → app/adminapi/domain/table_stats.py

@@ -1,8 +1,8 @@
 import datetime
 from collections.abc import Callable
 
+from app.adminapi import presentation as adminapi
 from app.data import model, repositories
-from app.presentation import adminapi
 
 
 def table_progress_to_presentation(progress: model.TableProgress) -> adminapi.TableProgress:

app/domain/adminapi/table_upload.py → app/adminapi/domain/table_upload.py

@@ -11,14 +11,15 @@
 from astropy import units as u
 from astroquery import nasa_ads as ads
 
+from app.adminapi import cache, clients
+from app.adminapi import presentation as adminapi
+from app.adminapi.domain import table_stats
 from app.data import model, repositories
 from app.data.repositories.common import ColumnSchemaInfo, TableSchemaInfo
 from app.data.repositories.layer0.common import RAWDATA_SCHEMA
-from app.domain.adminapi import table_stats
-from app.lib import astronomy, cache, clients, concurrency
+from app.lib import astronomy, concurrency
 from app.lib.storage import enums, mapping
 from app.lib.web.errors import NotFoundError, RuleValidationError
-from app.presentation import adminapi
 
 BIBCODE_REGEX = "^([0-9]{4}[A-Za-z.&]{5}[A-Za-z0-9.]{4}[AELPQ-Z0-9.][0-9.]{4}[A-Z])$"
 

app/adminapi/presentation/__init__.py

@@ -0,0 +1,5 @@
+from app.adminapi.presentation.interface import *
+from app.adminapi.presentation.records import *
+from app.adminapi.presentation.server import Server
+
+__all__ = ["Server"]

app/presentation/adminapi/interface.py → app/adminapi/presentation/interface.py

@@ -5,8 +5,8 @@
 
 import pydantic
 
+from app.adminapi.presentation.records import GetRecordsRequest, GetRecordsResponse
 from app.lib.storage import enums, mapping
-from app.presentation.adminapi.records import GetRecordsRequest, GetRecordsResponse
 
 DatatypeEnum = enum.StrEnum(
     "DatatypeEnum",

app/presentation/adminapi/server.py → app/adminapi/presentation/server.py

@@ -4,10 +4,10 @@
 import fastapi
 import structlog
 
+from app.adminapi.presentation import interface
 from app.lib import audit, auth
 from app.lib.web import server
 from app.lib.web.middlewares import identity_from_request
-from app.presentation.adminapi import interface
 
 
 def _logout_token(request: fastapi.Request) -> str:

app/cli.py

@@ -1,47 +1,15 @@
-import os
-
 import click
 
-from app.commands.adminapi import AdminAPICommand
-from app.commands.dataapi import DataAPICommand
-from app.lib import commands
+from app.adminapi.cli import main as adminapi
+from app.dataapi.cli import main as dataapi
+from app.tasks.cli import main as serve_tasks
 
 
 @click.group()
 def cli() -> None:
     pass
 
 
-@cli.command(short_help=AdminAPICommand.help())
-@click.option(
-    "-c",
-    "--config",
-    type=str,
-    default=lambda: os.environ.get("CONFIG", ""),
-    help="Path to configuration file",
-)
-def adminapi(config: str):
-    commands.run(AdminAPICommand(config))
-
-
-@cli.command(short_help=DataAPICommand.help())
-@click.option(
-    "-c",
-    "--config",
-    type=str,
-    default=lambda: os.environ.get("CONFIG", ""),
-    help="Path to configuration file",
-)
-def dataapi(config: str):
-    commands.run(DataAPICommand(config))
-
-
-@cli.command(
-    "serve-tasks",
-    short_help="Registers layer2 import Prefect deployments and serves them.",
-)
-def serve_tasks() -> None:
-    # Lazy-load: ServeTasksCommand pulls in Prefect, which slows other commands
-    from app.commands.serve_tasks import ServeTasksCommand  # noqa: PLC0415
-
-    commands.run(ServeTasksCommand())
+cli.add_command(adminapi, name="adminapi")
+cli.add_command(dataapi, name="dataapi")
+cli.add_command(serve_tasks, name="serve-tasks")

app/data/enums.py

@@ -0,0 +1,8 @@
+from psycopg.types import enum
+
+from app.lib.storage import enums
+
+PG_ENUM_REGISTRY: list[tuple[type[enum.Enum], str]] = [
+    (enums.DataType, "common.datatype"),
+    (enums.RecordTriageStatus, "layer0.triage_status"),
+]

app/dataapi/cli.py

@@ -0,0 +1,18 @@
+import os
+
+import click
+
+from app.dataapi.command import DataAPICommand
+from app.lib import commands
+
+
+@click.command(short_help=DataAPICommand.help())
+@click.option(
+    "-c",
+    "--config",
+    type=str,
+    default=lambda: os.environ.get("CONFIG", ""),
+    help="Path to configuration file",
+)
+def main(config: str) -> None:
+    commands.run(DataAPICommand(config))

app/commands/dataapi/command.py → app/dataapi/command.py

@@ -6,14 +6,13 @@
 import structlog
 import yaml
 
+from app.data import enums as data_enums
 from app.data import repositories
-from app.domain import dataapi as domain
-from app.domain import responders
+from app.dataapi import domain, presentation, responders
 from app.lib import auth, commands, config, tracing
 from app.lib.storage import postgres
 from app.lib.tracing import TracingConfig
 from app.lib.web import server
-from app.presentation import dataapi as presentation
 
 log: structlog.stdlib.BoundLogger = structlog.get_logger()
 
@@ -34,7 +33,7 @@ def prepare(self):
         tracing.setup_tracing("dataapi", self.config.tracing)
 
         self.pg_auth = postgres.PgStorage(self.config.storage.auth, log)
-        self.pg_main = postgres.PgStorage(self.config.storage.main, log)
+        self.pg_main = postgres.PgStorage(self.config.storage.main, log, data_enums.PG_ENUM_REGISTRY)
 
         authenticator: auth.Authenticator = (
             auth.PostgresAuthenticator(self.pg_auth) if self.config.auth_enabled else auth.NoopAuthenticator()

app/dataapi/domain/__init__.py

@@ -0,0 +1,5 @@
+from app.dataapi.domain.actions import Actions
+
+__all__ = [
+    "Actions",
+]

app/domain/dataapi/actions.py → app/dataapi/domain/actions.py

@@ -1,9 +1,9 @@
 from typing import final
 
 from app.data import model, repositories
-from app.domain import responders
-from app.domain.dataapi import parameterized_query, tap_types
-from app.presentation import dataapi
+from app.dataapi import presentation as dataapi
+from app.dataapi import responders
+from app.dataapi.domain import parameterized_query, tap_types
 
 ENABLED_CATALOGS = [
     model.RawCatalog.DESIGNATION,

app/domain/dataapi/parameterized_query.py → app/dataapi/domain/parameterized_query.py

@@ -1,7 +1,7 @@
 from app.data import model, repositories
 from app.data.repositories import layer2
-from app.domain import responders
-from app.presentation import dataapi
+from app.dataapi import presentation as dataapi
+from app.dataapi import responders
 
 CATALOGS_FOR_PGC_QUERY = [
     model.RawCatalog.DESIGNATION,

app/dataapi/presentation/__init__.py

@@ -0,0 +1,5 @@
+from app.dataapi.presentation.interface import *
+from app.dataapi.presentation.server import Server
+from app.dataapi.presentation.tap import *
+
+__all__ = ["Server"]

app/presentation/dataapi/interface.py → app/dataapi/presentation/interface.py

@@ -2,7 +2,7 @@
 
 import pydantic
 
-from app.presentation.dataapi import tap
+from app.dataapi.presentation import tap
 
 
 class EquatorialCoordinates(pydantic.BaseModel):