diff --git a/.agents/AGENTS.md b/.agents/AGENTS.md index dffce8c..f6dc7ce 100644 --- a/.agents/AGENTS.md +++ b/.agents/AGENTS.md @@ -57,6 +57,10 @@ Roadmap.md always has these sections in this order: Every milestone must have at minimum: **Goal**, **What was done**, and **Key decisions**. Optional subsections: **Known issues / follow-ups**, **Tests**. -## Documentation-First Rule +## Product Development Workflow -Read `PRD.md`, `Architecture.md`, `Rules.md`, and `Phases.md` before implementing any feature. If a proposed change adds scope, update documentation first — code changes follow doc changes, not the other way around. +Read the relevant current documents in `Documentation/`: `PRD.md`, `Architecture.md`, `Phases.md`, and `Development.md`. Update product/design documentation before or alongside meaningful scope changes. Routine implementation choices and fixes do not require a new approval or planning ceremony. + +The deleted rules file is retired. Its fixed library allowlist, MVP-only scope restrictions, exact folder prescriptions, and one-phase-at-a-time execution rule no longer apply. Phases define dependencies and release gates; independent work may proceed concurrently once contracts are settled. Choose tools and libraries using maintainability, compatibility, security, and measured need. + +Keep current implementation, planned capabilities, and measured release evidence distinct. Preserve the historical milestones and maintain Roadmap as described above. Actual environment permissions and user authorization still govern production, destructive, and externally charged actions. diff --git a/.dockerignore b/.dockerignore index 9403dfc..9993344 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,10 @@ -target/ +target/* +!target/ai-inference-gateway-0.0.1-SNAPSHOT.jar +.build-cache/ +*.log +.agents/ +.codex/ +.github/ .git/ .gitignore .idea/ diff --git a/.env.example b/.env.example index 4511b18..45f76bb 100644 --- a/.env.example +++ b/.env.example @@ -1,24 +1,16 @@ -# ── PostgreSQL ──────────────────────────────────────── +# Local Compose bootstrap only; do not use these demo credentials in production. POSTGRES_USER=gateway POSTGRES_PASSWORD=gateway POSTGRES_DB=gateway_db +LOCAL_API_KEY=test-api-key-1 -# ── Gateway Database Connection ────────────────────── -DB_HOST=postgres -DB_PORT=5432 -DB_NAME=gateway_db -DB_USERNAME=gateway -DB_PASSWORD=gateway - -# ── Ollama ─────────────────────────────────────────── -OLLAMA_BASE_URL=http://ollama:11434 - -# ── Test API Key (plaintext) ───────────────────────── -# The seeded API key hash in the database corresponds to this plaintext value. -# Use this in the X-API-Key header when testing locally. -TEST_API_KEY=test-api-key-1 - -# ── Notes ──────────────────────────────────────────── -# 1. Copy this file to .env before running: cp .env.example .env -# 2. After first `docker compose up`, pull the Ollama model: -# docker exec ollama ollama pull qwen3 +# Running the JAR directly: explicitly use SPRING_PROFILES_ACTIVE=local for demos. +# Production is the default profile and requires DB_PASSWORD and existing non-demo keys. +# Direct-JAR infrastructure properties: +# DB_HOST=localhost +# DB_PORT=5432 +# DB_NAME=gateway_db +# DB_USERNAME=gateway +# DB_PASSWORD= +# Local profile updates the seeded provider URL from this property: +# OLLAMA_BASE_URL=http://localhost:11434 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..aec7a13 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +* text=auto +mvnw text eol=lf +*.sh text eol=lf +*.cmd text eol=crlf +*.yml text eol=lf +*.sql text eol=lf diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..11505cf --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,87 @@ +name: Foundation verification +on: + push: + pull_request: + workflow_dispatch: +permissions: + contents: read +concurrency: + group: foundation-${{ github.ref }} + cancel-in-progress: true +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 25 + services: + postgres: + image: postgres:16-alpine@sha256:721873c34ceb9f8d8fc265984940dc982404c105f19ad51be9fdc5970a6080ea + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: ci-disposable-only + POSTGRES_DB: gateway_test + ports: ["5432:5432"] + options: >- + --health-cmd pg_isready + --health-interval 5s + --health-timeout 5s + --health-retries 10 + env: + TEST_DATABASE_URL: jdbc:postgresql://localhost:5432/gateway_test + TEST_DATABASE_USER: postgres + TEST_DATABASE_PASSWORD: ci-disposable-only + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + persist-credentials: false + - uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5 + with: + distribution: temurin + java-version: '21' + cache: maven + - name: Build and test including PostgreSQL migrations + run: sh mvnw -B -Ppostgres-it verify + - name: Artifact manifest + run: python3 scripts/release_manifest.py target/ai-inference-gateway-0.0.1-SNAPSHOT.jar + - name: Build container + run: docker build --target packaged -t gateway:ci . + - name: Dependency secret and configuration scan + uses: aquasecurity/trivy-action@d2a0b60797ff03db6132bd4e2b293f9b37081297 + with: + scan-type: fs + version: v0.74.0 + scan-ref: . + scanners: vuln,secret,misconfig + severity: HIGH,CRITICAL + exit-code: '1' + skip-dirs: .git,.build-cache,.cache + format: json + output: target/source-scan.json + - name: Container vulnerability scan + uses: aquasecurity/trivy-action@d2a0b60797ff03db6132bd4e2b293f9b37081297 + with: + image-ref: gateway:ci + version: v0.74.0 + severity: HIGH,CRITICAL + exit-code: '1' + format: json + output: target/image-scan.json + - name: Container SBOM + uses: aquasecurity/trivy-action@d2a0b60797ff03db6132bd4e2b293f9b37081297 + with: + image-ref: gateway:ci + version: v0.74.0 + format: cyclonedx + output: target/container-sbom.json + exit-code: '0' + - name: Preserve verification evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: foundation-evidence + path: | + target/surefire-reports/ + target/failsafe-reports/ + target/release-manifest.json + target/container-sbom.json + target/source-scan.json + target/image-scan.json diff --git a/.gitignore b/.gitignore index 8f056fa..b0cf2bd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ # Maven +.build-cache/ target/ -!.mvn/wrapper/maven-wrapper.jar !**/src/main/**/target/ !**/src/test/**/target/ @@ -9,6 +9,15 @@ target/ *.jar *.war *.ear +# Keep the wrapper binary versionable if the wrapper distribution type changes. +!.mvn/wrapper/maven-wrapper.jar +*.hprof + +# Local tooling and CI scanner caches +/.cache/ +__pycache__/ +*.pyc +*.pyo # IDE - IntelliJ .idea/ @@ -30,6 +39,8 @@ Thumbs.db # Environment .env +.env.* +!.env.example # Logs *.log diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 0000000..222cba3 --- /dev/null +++ b/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1,4 @@ +wrapperVersion=3.3.4 +distributionType=only-script +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.11/apache-maven-3.9.11-bin.zip +distributionSha256Sum=0d7125e8c91097b36edb990ea5934e6c68b4440eef4ea96510a0f6815e7eeadb diff --git a/Dockerfile b/Dockerfile index 12d6fcb..c2e150d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,24 @@ -# ── Stage 1: Build ─────────────────────────────────── -FROM maven:3.9-eclipse-temurin-21 AS build +# Pinned multi-architecture base manifests; update through reviewed dependency maintenance. +FROM eclipse-temurin:21-jdk-alpine@sha256:cd87715a8d45cfaa42419207c64680234f62785c49055cccd20437b5c9018380 AS build WORKDIR /app - -# Cache dependencies first (layer caching optimization) -COPY pom.xml . -RUN mvn dependency:go-offline -B - -# Copy source and build +COPY .mvn .mvn +COPY mvnw pom.xml ./ +RUN chmod +x mvnw COPY src ./src -RUN mvn package -DskipTests -B +# CI runs the full suite (including PostgreSQL) before building the image. +RUN --mount=type=cache,target=/root/.m2 ./mvnw -B -DskipTests package -# ── Stage 2: Run ───────────────────────────────────── -FROM eclipse-temurin:21-jre-alpine +FROM eclipse-temurin:21-jre-alpine@sha256:1a29e1fe337eb28b5bec30f0ee8ed29f0ff80ab6f75dcf9313efe82911065a52 AS runtime WORKDIR /app +RUN addgroup -g 10001 gateway && adduser -D -H -u 10001 -G gateway gateway +USER 10001:10001 +EXPOSE 8080 9090 +ENTRYPOINT ["java", "-jar", "app.jar"] -COPY --from=build /app/target/*.jar app.jar - -EXPOSE 8080 +# CI packages the exact tested/manifested artifact, avoiding a second compilation. +FROM runtime AS packaged +COPY --chown=10001:10001 target/ai-inference-gateway-0.0.1-SNAPSHOT.jar app.jar -ENTRYPOINT ["java", "-jar", "app.jar"] +# Default local Compose build remains self-contained from source. +FROM runtime AS final +COPY --from=build --chown=10001:10001 /app/target/ai-inference-gateway-0.0.1-SNAPSHOT.jar app.jar diff --git a/Documentation/API.md b/Documentation/API.md new file mode 100644 index 0000000..aa5f289 --- /dev/null +++ b/Documentation/API.md @@ -0,0 +1,105 @@ +# API contracts and compatibility plan + +Updated 2026-09-21. The first section describes current Phase 6 behavior. Sections marked planned describe future contracts. This is a design contract, not a generated OpenAPI specification. Phase 7 produces machine-readable schemas and wire fixtures; Phase 12 completes admin schemas. + +Legacy preservation below is the default transition design. Under the product owner's redesign authorization, a documented replacement can revise it when preserving demo behavior would perpetuate defects or unnecessary complexity. Assess actual consumers, define the replacement/migration, update examples and tests, and revise the relevant release gates before shipping. Internal implementation changes alone do not require wire compatibility work. See [Development.md](Development.md). + +## Current endpoints + +| Method and path | Authentication | Current contract | +|---|---|---| +| `POST /v1/inference` | `X-API-Key` | Required nonblank `model` and `prompt`; optional `provider`; synchronous generation | +| `GET /v1/providers` | `X-API-Key` | Array of `name`, `type`, `models`; no provider base URL | +| `GET /v1/logs` | `X-API-Key` | Pagination envelope: `content`, `page`, `size`, `totalElements`, `totalPages` | +| `GET :9090/actuator/health` | None; private listener | Minimal status only; liveness/readiness subpaths supported | +| `GET :9090/actuator/prometheus` | None; private listener | Operational metrics; blocked on the public listener | + +Current log filters are `provider`, `status`, `from`, `to`, `page` (default 0), and `size` (default 20, upper cap 100). Times are ISO local date-times without an offset. Negative page, nonpositive size, status outside `SUCCESS`/`FAILURE`, malformed timestamps and reversed ranges return 400. Inference requires JSON, nonblank model/prompt, maximum 200-character model/provider and 262,144-character prompt, within the 1 MiB body limit. + +Phase 6 deliberately corrects provider-unavailable from 502 to 503 and returns fixed safe error messages rather than raw parser/provider text. Malformed input returns 400; missing/invalid key 401; forbidden access 403; oversized body 413; unsupported media 415; unexpected failure 500; local saturation/upstream unavailability 503; provider/deadline timeout 504. Stable error codes distinguish `OVERLOADED` and `PROVIDER_UNAVAILABLE`. Requests rejected before inference use metrics, not database diagnostic rows. No automatic retries or rate-limit accounting is implied by these statuses. + +Legacy inference example: + +```json +{"model":"qwen3","prompt":"What is 2+2?","provider":"ollama-local"} +``` + +Legacy success fields are `requestId`, `text`, `model`, `provider`, `latencyMs`. Legacy error fields are `timestamp`, `status`, `error`, `message`, `path`, `requestId`. Every request handled by the ID filter gets `X-Request-Id`; not every success envelope embeds it. README and `http/requests.http` describe the runnable MVP. + +## Planned inference endpoints + +| Endpoint | Delivery | Contract scope | +|---|---|---| +| `POST /v1/chat/completions` | Phase 7 | Chat messages, text, SSE, tool declarations/results and structured output where deployment supports them | +| `POST /v1/embeddings` | Phase 7 | Text/string-array inputs, vectors, model and usage; no claim of universal dimensions/encoding support | +| `GET /v1/models` | Phase 7; tenant-scoped in Phase 8 | Discover only authorized virtual model aliases | +| `POST /v1/inference` | Retained | Existing shape through first GA series; internal legacy facade | +| `POST /v1/responses` | Phase 14 | Native Responses contract after state/tool/storage semantics are designed | +| `POST /v1/messages` | Phase 14 | Native Anthropic Messages contract; separate from Phase 7 upstream adapter | + +New SDK-compatible routes use `Authorization: Bearer `. The bearer value is a gateway credential, never an upstream provider key. Legacy routes continue accepting `X-API-Key` during migration. When both accepted headers are present, require the same credential or reject the request; do not silently choose a more privileged one. Reject oversized/malformed keys before expensive verification. + +Planned Chat Completions request (illustrative alias, not seeded today): + +```json +{ + "model": "support-chat", + "messages": [{"role": "user", "content": "Summarize this incident."}], + "stream": true +} +``` + +Keep the selected protocol's normal response shape. Return gateway correlation in `X-Request-Id`; do not wrap compatible responses in the legacy envelope. Gateway-specific metadata belongs in documented headers or a namespaced extension, not arbitrary schema changes. + +### Compatibility rules + +- Publish a versioned matrix for role/content types, generation controls, output constraints, tools, usage and streaming. A provider adapter may support a smaller set than another. +- Preserve tool IDs, argument fragments, finish reasons and usage semantics. The GA gateway passes tool requests to the application; it does not execute tools. +- Recognized but unsupported parameters return a structured 400 before dispatch. Define an explicit unknown-field policy in schema fixtures; no silent dropping of safety or semantic fields. +- Fallback candidates must support the requested operation and options. A text-only candidate cannot receive a vision request. +- Provider passthrough, if introduced later, is an explicitly authorized endpoint with an allowlist and size limits; it cannot bypass governance. +- Publish tested client SDK versions, provider/model/API versions and unsupported cases. SDK compatibility is scoped, not universal. + +### Streaming + +Use `text/event-stream`, disable intermediary response buffering, and preserve ordered events. Phase 7 fixtures define the Chat Completions chunk shape, optional final usage chunk and terminal marker. A disconnect may prevent final usage delivery; persist unknown/estimated usage internally when provider totals are absent. + +Before response commitment, errors use the appropriate HTTP status and protocol error envelope. After response bytes are sent, emit a safe stream error where supported and close; an already-sent 200 cannot become 503. Do not fabricate normal completion or replay another provider after any downstream bytes. Native protocols introduced later need their own terminal/error fixtures. + +### Error taxonomy + +| Category | Target HTTP status before commitment | Expected behavior | +|---|---|---| +| Malformed/unsupported input | 400 | Safe field-level message, no upstream call | +| Invalid/expired credential | 401 | No resource disclosure | +| Forbidden model/scope/policy | 403 | Terminal; no fallback to evade policy | +| Unknown visible model/resource | 404 | Do not reveal another tenant's object existence | +| Concurrent admin update/idempotency mismatch | 409 or 412 for failed `If-Match` | Caller refreshes; no blind overwrite | +| Too large / unsupported media | 413 / 415 | Reject before expensive work | +| Request/token/budget limit exceeded | 429 with distinct stable code | `Retry-After` only when a meaningful retry time is known | +| No eligible healthy target/dependency unavailable | 503 | Bounded response; safe retry guidance | +| Total/provider timeout | 504 | No unbounded attempts | +| Unexpected internal failure | 500 | Sanitized message and correlation ID | + +Internal categories map to protocol-native error bodies. Legacy routes retain their envelope. Add mappings in filters, async handlers and the global error handler; controller advice alone cannot cover every failure. Upstream credential errors are gateway deployment failures, not a 401 blaming a correctly authenticated client. + +## Planned administration and query API + +Use `/admin/v1` with OIDC and explicit role/scope checks. Proposed resources are `/tenants`, `/applications`, `/service-accounts`, `/keys`, `/providers`, `/deployments`, `/virtual-models`, `/route-policies`, `/budgets`, `/usage`, `/requests`, `/audit-events`, and `/config-versions`. These paths are not implemented yet. + +Tenant-scoped endpoints derive authorization from the principal and validate any selected tenant. Platform-wide operations require platform role and audit. API keys are shown once at creation; subsequent representations expose metadata only. Provider credentials return masked metadata/reference IDs, never secret values. + +Configuration actions on `/config-versions/{id}` include validation, simulation, staging and activation. Rollback creates an audited activation of an earlier revision. Return an operation identifier and convergence state, rather than claiming all replicas switched atomically. + +New admin/event APIs use stable cursor pagination, bounded filters and UTC offset timestamps. Mutations support `ETag/If-Match` and scoped idempotency keys. Reusing an idempotency key with a different body is an error. A repeated admin request can be deduplicated; repeated inference is not guaranteed exactly once unless the operation and provider explicitly support it. + +## Migration policy + +1. Capture existing API fixtures before implementation and preserve legacy bodies through GA. +2. Add new protocols without redirects from `/v1/inference`. +3. Rotate legacy keys because their hashes cannot yield lookup prefixes. Publish the legacy acceptance window and disable it by default for new deployments. +4. Restrict old logs/providers endpoints by tenant and read scope. Existing globally visible records become restricted legacy history, not guessed ownership. +5. New clients use model aliases. Existing model names map explicitly when unambiguous; provider override still passes access/residency/capability checks. +6. Publish security changes and deprecation timelines before removal. Provider API changes and unsupported fields require a compatibility-matrix update and regressions. + +See [Architecture.md](Architecture.md) for lifecycle, [Security.md](Security.md) for roles and [Phases.md](Phases.md) for release gates. diff --git a/Documentation/Architecture.md b/Documentation/Architecture.md index 260ac82..3f84393 100644 --- a/Documentation/Architecture.md +++ b/Documentation/Architecture.md @@ -1,241 +1,243 @@ -# Architecture.md — AI Inference Gateway +# AI Inference Gateway architecture + +Updated 2026-09-21. Sections explicitly marked **current** describe implemented Phase 6 code. All other sections describe the **target design**, to be implemented through [Phases.md](Phases.md). [Phase6.md](Phase6.md) records verification and open gates. [PRD.md](PRD.md) owns product scope; [API.md](API.md), [Security.md](Security.md), and [Operations.md](Operations.md) expand the contracts. + +The product owner authorizes revising existing architecture and implementation when correctness, maintainability, sound design or future complexity warrants it. Decisions below are current design choices, not immutable constraints. Document significant replacements and their evidence using [Development.md](Development.md); keep dependent contracts, migrations and release gates aligned. + +## 1. Current implementation and gaps + +The current application is a Java 21 / Spring Boot 4.1.1 modular backend using Spring MVC, JPA, Flyway and PostgreSQL, with a Tomcat 11.0.25 security patch override. Its Ollama adapter uses a shared Apache HttpClient 5 transport and Jackson 3 with `/api/generate`, streaming disabled. Redis 7 remains an unused Compose service. PostgreSQL 16 is the local database image. Maven 3.9.11 is checksum-pinned through the wrapper; image base manifests and CI actions are pinned. These choices establish a tested foundation, not production qualification. + +Current request flow: `RequestIdFilter → AdmissionFilter → ApiKeyFilter → InferenceController → InferenceService → RoutingEngine → OllamaProvider → ProviderTransport`, followed by one best-effort terminal `RequestLogService` call. Admission bounds body bytes and concurrent work before BCrypt; the request deadline includes admission/authentication time before upstream dispatch. IDs are assigned before authentication and returned in `X-Request-Id`. Inference/error bodies contain an ID; the providers array and pagination envelope do not universally contain a top-level ID. + +| Area | Evidence in current code | Required change | +|---|---|---| +| Authentication | `ApiKeyService` scans all active BCrypt hashes; filter creates generic `api-key-user` | Indexed key lookup, immutable principal, tenant/scopes, lifecycle | +| Authorization | Any valid key can query global providers/logs | Tenant-scoped queries and separate admin roles before shared use | +| Routing/config | Registry builds adapters at startup; routing/model validation also query DB per request | Consistent immutable config snapshots; explicit refresh/versioning | +| Provider abstraction | `AIProvider.infer/info/health`; registry has an Ollama switch | Capability-aware operations and adapter factories; abstraction changes are expected | +| Timeouts/errors | Explicit connect/read/pool timeouts, deadline cancellation, safe fixed error messages; no retry/redirect | Streaming cancellation and per-deployment pool isolation in Phases 7/9 | +| Logging | Routing and unexpected failures tracked; diagnostic save failure cannot replace original outcome | Durable usage/audit/outbox remains later work; diagnostic writes are synchronous and best effort | +| Privacy | No content in diagnostic records; provider exception text/bodies excluded from gateway error logs | Complete privacy/export/retention and dependency-log review in Phase 11 | +| Validation | Parser, query, media and body limits enforced; 400/413/415 mapped explicitly | New protocol feature/capability validation in Phase 7 | +| Database | V6 deactivates seed credential; nonlocal startup rejects active demo credentials; V1–V5 unchanged | Tenant identifiers, event identities, scoped data migration in Phase 8 | +| Testing | Fast H2 suite plus real PostgreSQL fresh/upgrade/startup tests and controllable HTTP upstream | Redis, multi-replica and SDK tests as those features arrive | +| Operations | Explicit local bootstrap; production default; private management with minimal health; non-root image; CI verification/scans | Remote CI evidence, deployment packages, HA/recovery and qualification in Phase 13 | + +Existing packages are `api`, `auth`, `routing`, `provider`, `inference`, `logging`, `config`, `error`, and `common` under `src/main/java/com/gateway`. Existing tables are `providers`, `models`, `api_keys`, and `request_logs`. Reuse components that fit the product and refactor or replace those that do not; there is no requirement to preserve the existing structure or restructure every file first. + +## 2. Architectural decisions + +| ID | Decision | Reason and reconsideration trigger | +|---|---|---| +| A01 | One deployable modular monolith first | Fast delivery and simple operation. Split control/data processes only after measured contention, independent scaling, or isolation needs | +| A02 | Retain Java 21, MVC, Spring Security, JPA and Flyway; upgrade to Boot 4.1.1 and Jackson 3 in Phase 6 | Tests and real PostgreSQL startup validate the upgrade; no Jackson compatibility layer. Reassess if measured streaming needs warrant a different edge | +| A03 | Start with MVC asynchronous SSE and a cancellable streaming HTTP transport | Avoid a mandatory whole-stack rewrite. Phase 7 compares bounded MVC streaming with a reactive edge under slow-client load; choose based on evidence | +| A04 | PostgreSQL owns config, identities, audit, metering and hard-budget reservations; Redis handles fast ephemeral state | Durable correctness stays recoverable. Redis loss must not reset authoritative spending | +| A05 | Typed declarative policy and immutable snapshots | Deterministic evaluation, simulation and rollback; no arbitrary scripts on request threads. CEL/OPA is an optional later adapter | +| A06 | Generic HTTP protocol adapters plus provider-specific translation at the edge | Reuse compatible providers without pretending their features are identical; narrow SDK use is allowed for signing/identity | +| A07 | Metadata-only observability; exact caching opt-in | Preserve privacy while adding useful diagnosis; semantic cache requires a later quality/privacy design | +| A08 | Durable outbox and bounded exporters before a broker | No mandatory Kafka. Add a broker only when throughput, consumer isolation or delivery needs exceed measured PostgreSQL capacity | +| A09 | Single-region HA first; equivalent policy semantics across deployment packages | Avoid global budget/consistency promises before a regional system is proven | +| A10 | Phase 6 uses Apache HttpClient 5 behind `ProviderTransport`, shared strict pool, explicit abort and no retries | Replaces implicit RestClient lifecycle with testable bounds. Keep native wire encoding in OllamaProvider; revisit transport in A03 and isolate deployments in Phase 9 | + +### Current Phase 6 bounds + +Defaults: 1 MiB request body; 262,144 prompt characters; 200-character model/provider names; 64 admitted `/v1/` requests; 100 servlet threads; 32 total / 8 per-route upstream connections; 250 ms pool acquisition; 2 s connect; 30 s response/idle-read; 120 s admission-to-upstream deadline; 2 MiB upstream body. HTTP retries and redirects are disabled. Oversized input returns 413 and saturation 503. Low-cardinality admission/outcome/latency metrics exclude client-supplied identifiers. + +The upstream deadline actively cancels the socket, including a trickling response. Incoming body reads check the deadline between reads and use Tomcat's 5 s connection/read idle bound. Authentication and synchronous diagnostics use separately bounded database operations (2 s pool/query, 5 s socket/transaction); the deadline is **not** a hard bound on all servlet processing or response delivery. A downstream disconnect does not yet cancel non-streaming generation immediately. Phase 7 owns async context, stream backpressure and disconnect propagation. The shared pool does not yet provide deployment bulkheads. + +Spring documents SSE and reactive-client integration in MVC, while response writes remain blocking. This makes executor sizing and slow-client tests mandatory for A03. The final transport decision must use documentation for the chosen Spring version. [Spring asynchronous requests](https://docs.spring.io/spring-framework/reference/web/webmvc/mvc-ann-async.html) + +## 3. Target topology and module boundaries + +```mermaid +flowchart LR + C[Applications and SDKs] --> E[TLS ingress and request bounds] + E --> D[Data plane] + D --> I[Identity and admission] + I --> P[Policy and virtual model routing] + P --> A[Provider adapters] + A --> U[Cloud or self hosted models] + P --> R[(Redis limits and optional cache)] + D --> M[Usage and decision recording] + M --> DB[(PostgreSQL)] + O[Operators and console] --> CP[Admin API and config lifecycle] + CP --> DB + DB --> S[Versioned config snapshots] + S --> D + DB --> X[Outbox exporters] + X --> T[Telemetry and audit destinations] +``` -> This file defines **how** the project is built: tech stack, folder structure, system components, and request flow. Read `PRD.md` first for product context, then this file for technical context before writing code. +Data and control planes are boundaries inside one application initially. The inference path consumes published snapshots; it never calls the admin API. The console has no direct database access. ---- +| Module | Responsibility | Boundary | +|---|---|---| +| `protocol` / existing `api` | Legacy, Chat Completions, embeddings and later native protocol mapping | No provider credentials or repository orchestration in controllers | +| `identity` / existing `auth` | Credentials, principal resolution, scopes and tenant context | Downstream code consumes authenticated context, not untrusted identity headers | +| `inference` | Lifecycle, deadline, cancellation, admission and finalization | No provider-specific wire DTOs | +| `policy` | Permissions, residency, privacy, limits, guardrail and route decisions | Pure evaluation over typed input and pinned snapshot | +| `routing` | Eligible candidates, pools, health, attempts and fallback | All attempts honor the same security constraints | +| `provider` | Adapter factories, capability descriptors, transport and error translation | Credentials resolved just in time; isolated connection pools | +| `governance` | Request/token quotas, reservations, normalized usage and pricing | Atomic admission; recoverable settlement | +| `telemetry` / existing `logging` | Request summaries, attempt events, metrics and spans | Bounded queues, no implicit content logging | +| `control` / existing `config` | Admin services, revisions, publication, simulation and audit | Transactional mutation plus audit/outbox | +| `cache`, `guardrail` | Optional response reuse and explicit content policy | Tenant-aware; policy governs external destinations | +| `operations` | Readiness, draining, reconciliation and export workers | Lifecycle and bounded recovery jobs | -## 1. Tech Stack +These are ownership boundaries, not a prescribed directory migration or a requirement to create empty packages. Use ordinary module tests; introduce build modules only when isolation helps. -| Layer | Technology | Purpose | -|-------|-----------|---------| -| Backend / Gateway | **Spring Boot** (Java) | Core gateway service — API, routing, orchestration | -| Local AI Provider | **Ollama** | Local model provider for dev/test, no paid API needed | -| Database | **PostgreSQL** | Providers, models, API keys, request logs | -| Cache / Future Infra | **Redis** | Provisioned now, used later for rate limiting/caching | -| Containerization | **Docker / Docker Compose** | Runs gateway + Postgres + Redis + Ollama together locally | -| Security | Spring Security (API key filter only) | Validates `X-API-Key` on every request | -| Future (post-MVP) | Kafka, Prometheus, Grafana, Kubernetes | Event-driven processing, observability, scaling | +## 4. Execution contracts and request lifecycle ---- +### Internal contracts -## 2. High-Level Components +Define immutable domain records before changing adapters: -``` -┌─────────────┐ ┌──────────────────────────────────────┐ ┌───────────────┐ -│ Client │──────▶│ Gateway Service │──────▶│ Providers │ -│ Application │ API │ (Spring Boot) │ infer │ (Ollama, ...) │ -└─────────────┘ Key │ │ └───────────────┘ - │ ┌────────────┐ ┌──────────────────┐ │ - │ │ Auth │ │ Routing Engine │ │ - │ │ Filter │ │ (explicit/default)│ - │ └────────────┘ └──────────────────┘ │ - │ ┌────────────┐ ┌──────────────────┐ │ - │ │ Provider │ │ Response/Error │ │ - │ │ Registry │ │ Normalizer │ │ - │ └────────────┘ └──────────────────┘ │ - └───────────────┬────────────────────────┘ - │ - ┌───────────────▼────────────────┐ - │ PostgreSQL │ - │ providers | models | api_keys │ - │ request_logs │ - └───────────────────────────────────┘ - ┌───────────────────────────────────┐ - │ Redis │ - │ (reserved, not used yet) │ - └───────────────────────────────────┘ -``` +- `GatewayContext`: server request ID, trace context, tenant/application/service-account/key IDs, granted scopes, config revision, receipt time, absolute deadline and cancellation handle. No raw credential or content in `toString`. +- `InferenceCommand`: operation, client model alias, typed messages/content or embedding inputs, generation options, tool declarations, response constraints, stream flag and allowlisted metadata. +- `DeploymentCapabilities`: supported operations, streaming, modalities, tools, structured output, usage reporting, context/output bounds and API version. Capabilities belong to deployment/model combinations, not only provider brands. +- `InferenceResult`: typed output, finish reason, normalized usage, upstream correlation ID, deployment ID and safe extensions. +- `StreamEvent`: start, text/content delta, tool delta, usage, finish, or error; ordered sequence within one attempt. Terminal completion is exclusive and idempotent. +- `ProviderFailure`: category, safe message, upstream status, retry-after, dispatch certainty, usage certainty and retry eligibility. Raw upstream bodies are excluded from public errors. +- `DecisionRecord`: revision/rule IDs, eligible/rejected deployment IDs with safe reasons, selection strategy, attempt outcomes, limit/guardrail results and final status. ---- +Evolve `AIProvider` into an internal v2 SPI with separate execute and stream operations, capabilities and probes. The stream contract must expose cancellation and bounded consumption; choose its Java type during A03's spike. Do not freeze a public SDK/SPI before adapter conformance stabilizes. Preserve the legacy DTO facade while introducing these records. -## 3. Request Flow +### Target execution order -1. Client sends `POST /v1/inference` with `X-API-Key` header and JSON body (`prompt`, optional `provider`, optional `model`, etc.) -2. **Auth Filter** validates the API key against PostgreSQL. Invalid/missing key → `401` using the standard error schema. -3. Gateway generates a unique `requestId`. -4. **Routing Engine** resolves the target provider: - - If `provider` is present in the request → use it. - - Else → use the active default provider from config. - - If neither resolves → `BAD_CONFIGURATION` error. -5. Gateway calls `infer()` on the resolved provider's adapter (implements the shared `AIProvider` interface). -6. Provider adapter calls the actual backend (e.g., Ollama's local API) and returns a result, or throws/times out. -7. **Response/Error Normalizer** converts the result into the standard success or error schema, tagged with `requestId`. -8. Gateway writes a row to `request_logs` (provider, model, status, latency, error code if any). -9. Gateway returns the normalized response to the client. +1. Assign request/trace correlation and enforce header/body/connection bounds before expensive authentication. +2. Authenticate; resolve immutable principal and scope. Pin a valid config snapshot. Reject stale security state. +3. Parse protocol into a typed command and validate operation, alias and requested features. +4. Evaluate access, residency and input guardrails; record matched rules. User metadata may inform explicitly allowed routing conditions, never establish identity. +5. Apply request and concurrency limits, then check an eligible exact-cache entry. Authorization and current output policy apply even to cache hits. +6. Resolve eligible deployments, excluding unsupported capabilities, forbidden regions, disabled targets and open circuits. +7. Reserve estimated tokens and the conservative cost of the next attempt. Record a durable attempt intent before paid dispatch. Do not hold a SQL transaction open during upstream generation. +8. Invoke provider with remaining deadline, bounded buffers and cancellation. Retry/fallback only under the matrix below; each attempt obtains its own cost reservation. +9. Apply output policy and encode the chosen wire protocol. Record whether response bytes have been committed. +10. Finalize on success, denial, failure, timeout or cancellation. Persist usage/decision outcome, settle or retain reservations, and publish export work through the outbox. Release concurrency permits idempotently. ---- +A cache hit consumes request/concurrency policy, records gateway-served usage and zero upstream invocation cost, and does not consume an upstream token reservation. Separate served-token quotas may be configured. Failed authentication produces bounded security telemetry without inventing a tenant attribution. -## 4. Provider Abstraction +## 5. Streaming and reliability -All providers implement one shared interface. The gateway core never talks to a provider-specific SDK/API directly — only through this interface. +Maintain one absolute deadline across queueing, policy, retries and upstream reads. Separate connect, first-byte, idle-read and total timeouts are capped by remaining time. The target streaming policy starts with 2 seconds connect, 30 seconds first-byte, 15 seconds idle, and 120 seconds total; local slow models require explicit longer profiles. Phase 6 currently enforces 2 seconds connect, a combined 30-second response/idle-read timeout and a 120-second upstream deadline; distinct streaming first-byte/idle controls remain Phase 7 work. -```java -public interface AIProvider { - InferenceResponse infer(InferenceRequest request); - ProviderInfo info(); - HealthStatus health(); -} -``` +Default automatic retry is off until Phase 9. Initial resilient policy permits at most three upstream attempts total, including fallbacks. Use jittered backoff and honor valid `Retry-After` only if the remaining deadline allows it. Disable hidden HTTP-client/SDK retries or charge them to the same budget. -- `OllamaProvider` — MVP's only concrete implementation. -- Future: `OpenAIProvider`, `ClaudeProvider`, `BedrockProvider`, etc. — each is a new adapter class + a config/registration entry. **No changes to gateway core logic.** +| Failure or event | Action before downstream bytes | After downstream bytes | +|---|---|---| +| Invalid input, forbidden model/region, guardrail block | Terminal client/policy error; no fallback | Cancel and terminate if detected during output enforcement | +| Upstream credential failure | Disable/alert on affected deployment; no same-target retry; fallback only to separately authorized target by explicit policy | Terminate; no replay | +| Connect failure known before dispatch | Eligible for bounded retry/fallback | Terminate; no replay | +| Upstream 429 | Classify rate limit versus exhausted quota; honor cooldown; eligible alternatives still obey tenant limits | Terminate; no replay | +| Transient 502/503/504 or reset | Retry only if attempt semantics permit; uncertain dispatch means possible duplicate charge | Terminate; retain uncertain usage | +| Read timeout after upstream accepted request | No automatic retry by default; opt-in safe generation policy may accept duplicate computation/cost | Terminate; retain uncertain usage | +| Malformed response/stream | Mark provider failure; pre-output fallback only when safe and explicitly classified | Emit safe protocol error if possible, close | +| Client disconnect or total deadline | Cancel outstanding call; finalize once; no further attempt | Same | ---- +For SSE, withhold downstream headers/body until a valid first event or a terminal pre-stream error where transport permits. Any downstream byte, including a heartbeat, closes the retry window. Partial tool arguments must never be represented as a completed tool call after failure. A stream error cannot replace an already-sent HTTP 200 status; document wire behavior in [API.md](API.md). -## 5. Folder Structure +Bound decoded event size, per-stream buffered bytes, output size and write duration. A slow client must backpressure the upstream or trigger cancellation, never accumulate an unbounded queue. Disconnect detection is not instantaneous in Servlet APIs; use periodic safe heartbeat/write checks and idle timeouts, measuring cleanup from detection separately from detection latency. Carry context across async callbacks and clear it after completion. -``` -ai-inference-gateway/ -├── docker-compose.yml -├── Dockerfile -├── pom.xml -├── README.md -├── .env.example -├── Documentation/ -│ ├── PRD.md -│ ├── Architecture.md -│ ├── Rules.md -│ ├── Phases.md -│ └── Roadmap.md -├── src/ -│ ├── main/ -│ │ ├── java/com/gateway/ -│ │ │ ├── GatewayApplication.java -│ │ │ │ -│ │ │ ├── api/ # Controllers (HTTP layer only) -│ │ │ │ ├── InferenceController.java -│ │ │ │ ├── ProviderController.java -│ │ │ │ └── LogController.java -│ │ │ │ -│ │ │ ├── auth/ # API key authentication -│ │ │ │ ├── ApiKey.java # entity -│ │ │ │ ├── ApiKeyRepository.java -│ │ │ │ ├── ApiKeyFilter.java -│ │ │ │ ├── ApiKeyService.java -│ │ │ │ └── SecurityConfig.java # Spring Security configuration -│ │ │ │ -│ │ │ ├── routing/ # Provider resolution logic -│ │ │ │ └── RoutingEngine.java -│ │ │ │ -│ │ │ ├── provider/ # Provider abstraction + adapters -│ │ │ │ ├── AIProvider.java # interface -│ │ │ │ ├── ProviderRegistry.java -│ │ │ │ └── ollama/ -│ │ │ │ └── OllamaProvider.java -│ │ │ │ -│ │ │ ├── inference/ # Core orchestration service -│ │ │ │ ├── InferenceService.java -│ │ │ │ ├── InferenceRequest.java -│ │ │ │ └── InferenceResponse.java -│ │ │ │ -│ │ │ ├── logging/ # Request logging -│ │ │ │ ├── RequestLog.java # entity -│ │ │ │ ├── RequestLogRepository.java -│ │ │ │ └── RequestLogService.java -│ │ │ │ -│ │ │ ├── config/ # Provider/model config management -│ │ │ │ ├── ProviderConfig.java # entity -│ │ │ │ ├── ProviderConfigRepository.java -│ │ │ │ ├── ModelConfig.java # entity -│ │ │ │ └── ModelConfigRepository.java -│ │ │ │ -│ │ │ ├── error/ # Standard error schema + handling -│ │ │ │ ├── ApiError.java -│ │ │ │ ├── ErrorCode.java # enum: INVALID_REQUEST, PROVIDER_NOT_FOUND, etc. -│ │ │ │ └── GlobalExceptionHandler.java -│ │ │ │ -│ │ │ └── common/ -│ │ │ └── RequestIdGenerator.java -│ │ │ -│ │ └── resources/ -│ │ ├── application.yml -│ │ └── db/migration/ # Flyway SQL migrations -│ │ ├── V1__create_providers.sql -│ │ ├── V2__create_models.sql -│ │ ├── V3__create_api_keys.sql -│ │ ├── V4__create_request_logs.sql -│ │ └── V5__seed_data.sql -│ │ -│ └── test/ -│ └── java/com/gateway/ -│ ├── GatewayApplicationTests.java -│ ├── routing/RoutingEngineTest.java -│ ├── provider/OllamaProviderTest.java -│ └── api/InferenceControllerTest.java -│ -└── postman/ or http/ # Sample requests for manual testing - └── inference-requests.http -``` +Active probes assess reachability/model readiness without repeatedly generating billable text. Passive health tracks real attempts. Circuit breakers are per deployment locally; Redis may share cooldown hints but is not an authoritative global circuit. Use limited half-open probes and recovery jitter. Round-robin and least-latency policies follow priority/weighted pools once sample freshness and starvation controls are tested. Record the runtime health inputs needed for explainability; simulation cannot predict future availability or quality. + +## 6. Identity and tenant isolation + +Keys use a versioned format such as `gw__`. Look up one indexed public ID and verify only its secret hash. Never derive the new prefix from an existing BCrypt hash. Old keys require explicit rotation or a time-bounded legacy path; invalid new-format keys never fall back to an all-key scan. + +Principals carry tenant, application, service account and scope. Separate inference keys from admin OIDC access tokens. Validate issuer, audience, signature, expiry and tenant membership. Minimal beta roles are platform administrator, tenant administrator, developer and auditor; detailed permissions are in [Security.md](Security.md). + +Tenant-owned tables include non-null tenant IDs, composite tenant/resource constraints and tenant-scoped repository access. Add PostgreSQL row-level security as defense in depth using transaction-local tenant context and a non-owner runtime role; test pooling and worker context cleanup. Dedicated deployment is the initial option for customers requiring stronger isolation than shared tables. + +Revocation and tenant suspension use a security epoch checked independently of a pinned routing snapshot. Healthy replicas should observe changes in 5 seconds; security state older than 30 seconds rejects new governed calls. Recheck before every fallback; emergency tenant/deployment disable can cancel in-flight work. Ordinary route publication does not alter an in-flight request. + +## 7. Logical data model + +Target entity names guide migrations; they are not tables already present. + +| Entity | Essential fields and constraints | +|---|---| +| `tenants` | UUID, unique name/slug, status, security epoch, default privacy/residency policy | +| `applications`, `service_accounts` | Tenant ID, owner/application linkage, environment, status; tenant-safe foreign keys | +| `memberships`, `role_bindings` | Tenant, external issuer/subject or service-account ID, role, delegated resource scope and status; no cross-tenant grant by implicit inheritance | +| `gateway_keys` | Unique public ID, secret hash, tenant/application/principal, scopes, expiry, revoked time, rotation group, coarse last-used time | +| `providers` | Adapter type, ownership scope, approved endpoint and credential reference; evolves current provider config | +| `deployments` | Tenant, provider, upstream model, region, capabilities, connection profile and credential override reference | +| `virtual_models`, `route_policies` | Unique tenant/alias, operation set, revisioned candidate pools and constraints | +| `config_versions`, `config_activations` | Schema version, immutable payload/hash, author, parent revision, lifecycle and active revision per scope | +| `replica_config_status` | Replica, revision, activation acknowledgement, heartbeat, validation error | +| `request_events`, `attempt_events` | Tenant/request/attempt IDs, revision, route and outcome, timings; unique event identity; no default content | +| `usage_events` | Append-only event ID, attempt, token categories, usage source, price version, currency, amount, correction linkage | +| `price_versions` | Deployment/model, effective interval, explicit category rates and currency; never overwrite past rates | +| `budgets`, `budget_reservations` | Tenant/application/key scope, UTC period, limit, spent/reserved, attempt ID, state/lease; unique reservation | +| `audit_events`, `outbox_events` | Actor, action, scope, resource/revision hashes, event identity and timestamp; exporter state separate | +| Optional `content_objects` | Tenant, encrypted object reference, key reference, retention expiry and access policy; never inline unrestricted payloads | + +Use UUIDs and UTC `timestamptz` for new event data. Use integer token counts and fixed-precision decimal money with currency and price version, never floating point. Reasoning and cached tokens may be subsets of provider totals; keep original category semantics to prevent double counting. Provider usage and local estimates are distinguishable. + +Index tenant/time and request/attempt lookups. Introduce time partitions when measured retention/query volume justifies them. Use stable cursor pagination for new event APIs. Retention removes partitions/events through an authorized process; append-only does not mean legally or operationally undeletable. Audit integrity needs restricted roles and optional immutable external storage, not a claim that SQL alone is tamper-proof. + +## 8. Limits and durable accounting + +Redis atomic operations enforce request/token windows and bounded concurrency leases across replicas. All applicable tenant/application/key/model constraints are checked; a request cannot pass one scope while bypassing another. Document window boundaries, clock source, reservation release and maximum drift. Start with one region; partition keys consistently if later introducing Redis Cluster. + +PostgreSQL transactions reserve hard-budget cost at every applicable scope using consistent lock order and an idempotent attempt ID. Reservation plus durable dispatch intent precedes upstream work. Record actual usage and settle reservations in one transaction; adjustments append correction events. Successful output does not imply provider billing certainty. + +A crashed/expired worker does not prove a request was unbilled. Recovery marks the attempt uncertain, retains its reservation and reconciles from provider data or an audited conservative settlement policy. TTL may release concurrency leases, not erase financial liability. A price change cannot retroactively change a reservation's price version. + +No distributed transaction spans PostgreSQL, Redis and a model provider. Partial admission failures release safe Redis reservations idempotently; outstanding database reservations are reconciled. Durable usage means recoverable intent and explicit uncertainty, not exactly-once generation. If a final write fails after streaming has started, stop further dispatch, retain the durable intent, record bounded recovery state and reconcile; do not claim a missing final row means zero cost. + +## 9. Configuration lifecycle and policy simulation + +One configuration version contains internally consistent aliases, deployments and policy references. The lifecycle is `draft → validated → staged → active → superseded`; validation failures never publish. Admin mutations use optimistic locking (`ETag/If-Match`) and idempotency keys. A publish transaction writes activation, audit and outbox together. + +Workers validate and construct a new immutable snapshot, then atomically swap references. Notifications accelerate refresh; periodic polling catches missed messages. Pin old adapters while their requests drain, then close them. Reject unknown schema versions and invalid secret/capability references. Report activation as converging until replicas acknowledge; remove stale replicas from readiness after the security freshness bound. + +Rollback activates a previously validated immutable revision through a new audited activation. It does not roll back schema migrations, revoke usage events, undo spending, or resurrect revoked credentials. Ordinary revision rollback cannot override the latest security epoch. + +The simulator shares the evaluator, takes synthetic/redacted inputs, supplied time/health/counter snapshots and a deterministic random seed, and returns matched rules and candidate exclusions. It performs no external calls, budget mutation or side effects. Metadata-only replay cannot evaluate unseen prompt-dependent rules: report those checks as unevaluated. Never label a synthetic result a prediction of model quality. + +## 10. Cache, guardrails and observability + +Exact cache is off by default. Authorization precedes every lookup. Keys include tenant, application/access partition, operation, virtual model, effective deployment identity, model/config/prompt/guardrail revisions, normalized parameters and a tenant-keyed digest of inputs. Store content only in approved encrypted infrastructure with TTL, size bounds and purge controls. Arbitrary user IDs cannot expand cache sharing. + +GA cache supports explicitly opted-in non-streaming text/embedding results with reproducible semantics. Bypass streams, tool calls, secrets, personalized requests without safe partitioning, and nondeterministic outputs unless policy explicitly allows reuse. Never cache errors or policy blocks. Reapply applicable output policy on hits. + +Guardrail interfaces support pre-request, post-response and streaming checks with allow, audit, block and redact results. Mandatory security checks fail closed on timeout/unavailability; advisory checks may fail open only by policy and with a degraded event. Regex/secret detection is a bounded first implementation, not a guarantee of PII detection. External guardrails are data egress destinations subject to residency and credentials policy. + +Whole-output checks require buffering within a configured cap before release or disabling streaming for that policy. Chunk checks cannot retract emitted text and can miss cross-boundary semantics; bounded look-behind supports specific detectors only. The capability/policy validator must reject combinations it cannot enforce. + +Publish Micrometer/Prometheus counters/histograms and OpenTelemetry spans for admission, policy, attempts, cache and finalization. Record TTFT, total latency, upstream latency, overhead, token throughput, retries, fallbacks, cancellations, exporter lag and reservation age. Use low-cardinality labels; key/request IDs belong in authorized traces/events, not metric labels. Pin telemetry schema versions because GenAI conventions evolve. [OpenTelemetry GenAI conventions notice](https://opentelemetry.io/docs/specs/semconv/gen-ai/) + +## 11. Dependency outages and operating behavior + +| Dependency/event | Target behavior | +|---|---| +| PostgreSQL unavailable | Reject new governed admission and admin mutations with sanitized 503; do not restart solely for DB loss; in-flight outcomes may require reconciliation | +| Redis unavailable | Hard distributed limits fail closed with 503; explicitly best-effort routes may use conservative local caps and degraded telemetry; cache bypasses | +| Telemetry collector unavailable | Inference continues with bounded queues, dropped-diagnostic counters and durable outbox where required | +| Audit persistence unavailable | Reject admin mutation before activation; audit cannot be silently discarded | +| Required secret resolver unavailable | Use only an unexpired permitted cached credential, otherwise fail deployment admission; never reuse arbitrary stale secrets | +| Mandatory guardrail unavailable | Fail closed; advisory mode emits explicit degradation | +| All providers unavailable | Return bounded 503/504 with request ID; probes/recovery continue; avoid restart loops | +| Stale config/revocation view | Reject new governed traffic after freshness threshold; report unready | +| Rolling shutdown | Become unready, stop admission, drain/cancel within configured grace, finalize attempts and release leases | + +Reference topology is two or more stateless replicas behind TLS ingress in one region, managed/operated HA PostgreSQL and Redis, optional object storage, and an external OIDC issuer. No sticky sessions are required for stateless inference; an open stream stays on its current connection. [Operations.md](Operations.md) defines deployment and release evidence. + +## 12. Migration from the MVP + +The following is the default continuity path, conditional on the Phase 6 assessment of consumers and stored data. A simpler replacement or revised contract is allowed where justified and documented; update this path and its tests accordingly. Preserve applied migration checksums for databases being upgraded. A fresh baseline can target new or explicitly disposable environments without assuming existing data may be deleted. + +1. Capture baseline API fixtures and run the existing suite plus real PostgreSQL V1–V5 tests. Back up any non-disposable database. +2. Preserve applied V1–V5 checksums. Add forward migrations that deactivate the known demo key and separate subsequent dev bootstrap from production startup. Production must not become ready while known demo credentials remain valid. +3. Add tenant/application/service-account tables and nullable ownership fields. Create an explicit legacy tenant and application; map existing providers, models and keys through a reviewed mapping. +4. Backfill aliases/deployments and verify counts and referential integrity. New legacy aliases keep existing model names when unambiguous. Reject ambiguous defaults instead of selecting one arbitrarily. +5. Historical logs lack identity. Mark them legacy/unattributed and restrict access to migration/platform administrators; never guess the tenant from a provider name. +6. Issue new indexed keys through secure bootstrap; keep legacy hashes only for a configured migration window. New deployments disable the scan path. Do not attempt to recover original secrets from hashes. +7. Switch readers and writers behind feature flags; compare shadow decisions using metadata without duplicate upstream calls. Test both fresh installation and upgrade from an MVP snapshot. +8. Enforce non-null ownership, composite constraints and RLS once backfill is validated. Move visibility endpoints to tenant-scoped services. +9. Remove obsolete schema/legacy key paths only in a later migration after the published compatibility window. Binary rollback works only across tested compatible schemas; prefer roll-forward after irreversible data changes. -**Layering rule:** `api` → `inference`/`routing` → `provider` → external service. Controllers never talk to repositories or providers directly; they go through service classes. - ---- - -## 6. Data Model (PostgreSQL) - -### `providers` -| Column | Type | Notes | -|--------|------|-------| -| id | UUID | PK | -| name | text | e.g. `ollama-local` | -| type | text | e.g. `ollama`, `openai` | -| base_url | text | connection info | -| is_default | boolean | used by routing default rule | -| active | boolean | | - -### `models` -| Column | Type | Notes | -|--------|------|-------| -| id | UUID | PK | -| provider_id | UUID | FK → providers.id | -| name | text | e.g. `qwen3` | -| active | boolean | | - -### `api_keys` -| Column | Type | Notes | -|--------|------|-------| -| id | UUID | PK | -| key_hash | text | hashed at rest, never stored plaintext | -| label | text | which app/owner the key belongs to | -| active | boolean | | -| created_at | timestamp | | - -### `request_logs` -| Column | Type | Notes | -|--------|------|-------| -| id | UUID | PK | -| request_id | text | `req_xxxxxxxxx`, indexed | -| timestamp | timestamp | | -| provider | text | | -| model | text | | -| status | text | `SUCCESS` / `FAILURE` | -| error_code | text | nullable | -| latency_ms | integer | | - ---- - -## 7. API Surface (MVP) - -| Endpoint | Method | Auth | Notes | -|----------|--------|------|-------| -| `/v1/inference` | POST | required | Core inference call | -| `/v1/providers` | GET | required | List configured providers | -| `/v1/logs` | GET | required | Paginated/filterable logs (`page`, `size`, `provider`, `status`, `from`, `to`) | - -All responses (success and error) include `requestId`. All errors use the standard schema defined in `Rules.md` / `PRD.md`. - ---- - -## 8. Local Development - -- `docker compose up` brings up: gateway, PostgreSQL, Redis, Ollama (with a pulled model). -- Flyway/Liquibase migrations run automatically on gateway startup. -- Default provider + a seeded API key should be inserted via migration/seed script for local testing out of the box. - ---- - -## 9. Related Docs - -- `PRD.md` — product scope, features, target users -- `Rules.md` — boundaries for AI-assisted development -- `Phases.md` — build order broken into stages \ No newline at end of file +Each migration includes validation, rollback/restore limits and expected lock duration. Configuration rollback is separate from binary/database recovery. No implementation or infrastructure deployment is performed by this documentation revision. diff --git a/Documentation/Development.md b/Documentation/Development.md new file mode 100644 index 0000000..c951601 --- /dev/null +++ b/Documentation/Development.md @@ -0,0 +1,53 @@ +# Development workflow + +Updated 2026-09-15. The project is moving from a constrained MVP to a complete product. The deleted rules file is retired. Its historical library prohibitions, fixed folder prescriptions and one-phase-at-a-time workflow are not current requirements. + +## Authority to improve the MVP + +The product owner explicitly authorized changes to existing architecture and implementation when they correct defects, improve maintainability or design, or prevent unnecessary future complexity. Developers and agents may refactor, replace or remove unsuitable MVP components as part of implementation without seeking renewed design permission for each change. Existing code and the proposed architecture are starting points that can be revised on evidence. + +Assess correctness, responsibility boundaries, coupling, testability, operational cost and the needs of planned features. Choose the smallest coherent change that resolves the problem; a broader redesign is appropriate when incremental fixes would preserve the underlying problem. Retaining the current stack or structure is not an objective in itself. + +For a significant change, document the concrete problem, alternatives considered, chosen design, expected benefit, affected components/contracts/data, migration or compatibility impact, validation and recovery limits. Update Architecture and affected product/API/phase documents before or alongside implementation; append the actual change and results to Roadmap when complete. Do not require a separate ADR for every refactor. + +Internal implementation compatibility is not required. Legacy API/schema preservation in the plan is a default transition strategy, not a veto on necessary redesign: document and test any deliberate contract change and update affected clients/examples and release gates. Inspect actual consumers and data before choosing a transition. The MVP's demo origin alone does not establish that stored data is disposable or authorize deleting it. Existing environment permissions and authorization for production actions still apply. + +## Working documents + +| Document | Responsibility | +|---|---| +| [PRD.md](PRD.md) | Product outcomes, scope, requirement IDs and release targets | +| [Architecture.md](Architecture.md) | Inspected baseline, target design, contracts, tradeoffs and migration | +| [Phases.md](Phases.md) | Dependencies, work packages, acceptance evidence and next backlog | +| [API.md](API.md) | Current versus proposed endpoints and compatibility | +| [Security.md](Security.md) | Threat model, identity, privacy and security acceptance | +| [Operations.md](Operations.md) | Deployment contract, recovery and production evidence | +| [Research.md](Research.md) | Research-to-plan traceability and sources | +| [Roadmap.md](Roadmap.md) | Append-only development history plus accurate current-state summary | +| [README.md](../README.md) | Runnable setup and implemented capabilities | + +Read the relevant context before a meaningful change; do not turn every small edit into a documentation ceremony. Product/design changes update the owning document before or alongside implementation. Routine fixes need appropriate tests and a clear explanation, not a new planning phase. + +## How to deliver + +1. Select a bounded outcome from the phase backlog and inspect actual code. Record prerequisites and any assumptions that affect correctness. +2. Choose libraries and implementation details using maintainability, compatibility, licensing, security and measured need. The old allowlist no longer applies. Prefer existing stack components when suitable, without treating that preference as a ban on alternatives. +3. Implement a vertical slice with relevant tests and migration/rollback notes. Independent work may proceed concurrently when contracts and file ownership are clear; phases are not a global execution lock. +4. Validate the behavior changed. Use fast unit tests for logic, real PostgreSQL for persistence/migrations, Redis/multiple replicas for distributed enforcement, wire/SDK tests for protocols and focused load/chaos tests for concurrency/failure behavior. +5. Explain the outcome, validation and limitations. Update the historical log for meaningful shipped work. Mark a phase complete only when its exit evidence exists. + +Agentic implementation may make routine reversible choices within the accepted scope without repeated approval requests. Customer-facing publication, production changes, destructive migration, credential use or externally charged testing still follow the user's authorization and environment permissions. This workflow does not grant permission to deploy or spend merely because a feature appears in the plan. + +## Engineering expectations + +Keep provider wire details behind adapters, policy evaluation deterministic, credentials out of content/logs and ownership explicit. Bound time, memory, concurrency and retries. Preserve supported API behavior unless a deliberate contract change is documented under the policy above; evolve persisted data through a tested migration strategy. These are product correctness needs, not a requirement for a particular package layout or exhaustive tests that duplicate implementation. + +Start with a modular monolith. A broker, microservice split, UI framework, SDK or reactive component is a design choice justified by concrete needs. Record substantial decisions with problem, chosen option, alternatives, consequences and a reconsideration trigger; the architecture decision table is sufficient until a separate ADR would help. + +For each issue/PR, include the trigger and resulting behavior, requirement/phase reference, validation, migration impact and material limitations. Never describe mock-only tests as real provider validation. Never infer production readiness from the historical MVP suite. + +## Documentation maintenance + +Keep planned and implemented features visibly separate. README examples must run on the current checkout; proposed APIs belong in API/Architecture until implemented. Do not recreate the deleted rules file or restore its constraints through another filename. + +Preserve Roadmap milestone text. Corrections are new entries, not historical rewrites. Its current architecture/stack summaries and next-step links can be updated in place. Future features move into the current summary only after implementation and evidence. diff --git a/Documentation/Operations.md b/Documentation/Operations.md new file mode 100644 index 0000000..d53204a --- /dev/null +++ b/Documentation/Operations.md @@ -0,0 +1,128 @@ +# Deployment and production readiness plan + +Updated 2026-09-21. Local Compose, a non-root Dockerfile, Maven wrapper and CI build/test/scan workflow are implemented. [Phase6.md](Phase6.md) records observed verification and open gates. Helm, systemd, Podman, signed releases, HA and recovery procedures below remain **planned deliverables**. Use [README.md](../README.md) for current setup. + +The MVP upgrade path below is the current continuity strategy. Phase 6 may replace it with a documented transition if design/consumer/data assessment justifies a simpler architecture. Adjust the release evidence accordingly; preserve real data and supported release recovery requirements. A demo origin is not permission to reset an existing environment. + +## Deployment contract + +First GA supports a single region, customer-owned providers/credentials, and customer-operated stateful services. A reference HA deployment has at least two gateway replicas behind TLS ingress, HA PostgreSQL and Redis, an OIDC issuer for administration, and optional encrypted object storage/telemetry destinations. Managed database/cache services are acceptable; Kubernetes is not required. + +| Package | Purpose | Qualification | +|---|---|---| +| Docker Compose | Local development and demonstrations | Fresh clone, explicit local bootstrap, model pull, API examples | +| OCI image and Helm | Kubernetes customer deployment | Probes, resources, rolling drain, secret references, network policy, external DB/Redis, HPA and disruption budget | +| JAR with systemd unit | Linux VM/bare-metal deployment | Non-root identity, restart/drain, mounted config/secrets, log rotation and external state | +| Podman Quadlet | Linux container deployment | Equivalent security/configuration, service dependencies, volumes, upgrades and drain | +| Offline bundle | Disconnected environment with local provider | Verified images/JAR, schemas, catalogs, migrations, checksums/signatures and no mandatory remote callback | + +Phase 13 records exact tested OS/runtime/package versions. Do not promise compatibility with all distributions. Local Ollama model weights remain separately installed/licensed prerequisites; an offline install must supply them before disconnecting. Cloud providers and external identity/guardrail services require connectivity if selected. + +## Configuration and secrets + +Keep one documented property schema across packages with precedence: built-in safe defaults → mounted configuration → explicit environment overrides. Published tenant routing/policy revisions live in PostgreSQL; environment variables configure bootstrap/infrastructure and cannot silently override tenant authorization. + +Document all planned properties when implemented: environment/profile, public and management listeners, database/migration credentials, Redis TLS/auth/topology, OIDC issuer/audience, secret resolver, provider connection profiles, request/stream size and timeout limits, concurrency, shutdown grace, telemetry destinations, retention, feature flags and security freshness. + +Current direct JAR configuration uses `DB_*`; Compose interpolates `POSTGRES_*` and supplies the corresponding runtime properties. The adapter uses the database provider `base_url`. Explicit local bootstrap updates the seeded Ollama row from `OLLAMA_BASE_URL`; other profiles do not rewrite provider configuration from that variable. These infrastructure properties are not yet a complete production configuration contract. + +Production startup must reject known demo credentials, missing required secret references and invalid security settings. Keep secrets out of Git, image layers, process arguments, API exports and logs. Use mounted secret files or a supported resolver. Restrict local exposed infrastructure ports; reference production packages expose only intended ingress and private management. + +## Lifecycle and dependency readiness + +### Implemented Phase 6 behavior + +Production is the default profile; `DB_PASSWORD` must be nonblank and different from `gateway`. V6 disables the historical fixed seed key without deleting customer rows. Startup rejects active keys matching the original seed ID or known demo plaintext, including rehashed copies. Local bootstrap requires `local` without `production`, is enabled in `application-local.yml`, and hashes `LOCAL_API_KEY` before saving it. Never start the local profile against customer data: it intentionally rewrites the demo key and seeded provider URL. + +Before upgrading an MVP database, back it up and arrange a non-demo key through the existing operator-controlled database process; there is no administration/provisioning API yet. Apply V6 through normal Flyway startup and verify existing non-demo key access. If a previous local bootstrap reactivated the demo row, deactivate that row before production startup. Do not edit V1–V5, run Flyway repair to hide checksum changes, or reactivate the demo credential to recover access. Roll forward for defects; reverting to the MVP binary would remove these runtime safeguards. + +Management defaults to loopback port 9090 with health/prometheus only and no health components/details. Prometheus access requires a distinct management listener; using the same public port does not grant access. Compose binds management to the container network but publishes it on host loopback only. Place authenticated network access in front of management if remote scraping is needed; there is no management-user role yet. Readiness includes the database and readiness state; it does not certify provider/model availability. Flyway and credential validation complete before application readiness. + +The runtime runs as UID/GID 10001. `docker build --target packaged -t gateway:local .` packages the exact JAR already tested by the wrapper; CI uses this target and generates its SHA-256 manifest. Default `docker build`/Compose builds source inside the JDK stage and needs Maven Central access. Neither path implies that live-provider or production qualification passed. + +| Property (Spring relaxed environment binding supported) | Default | Purpose | +|---|---|---| +| `gateway.max-request-bytes` | 1048576 | Bound request buffering, including unknown-length bodies | +| `gateway.max-concurrent-requests` | 64 | Fail-fast `/v1/` admission before authentication | +| `gateway.total-timeout-ms` | 120000 | Remaining admission-to-upstream deadline; actively abort provider I/O | +| `gateway.transport.connect-timeout-ms` | 2000 | TCP connection bound | +| `gateway.transport.read-timeout-ms` | 30000 | Response and socket-idle bound | +| `gateway.transport.pool-acquire-timeout-ms` | 250 | Maximum shared-pool queue wait | +| `gateway.transport.max-connections` / `max-connections-per-route` | 32 / 8 | Strict shared HTTP pool | +| `gateway.transport.max-response-bytes` | 2097152 | Maximum buffered upstream success body | + +The synchronous request deadline does not guarantee total servlet completion: body reads check between reads with a 5 s idle bound, while SQL and response writes have separate bounds/limitations. Database defaults are 10 Hikari connections, 2 s connection/query timeout and 5 s JDBC socket/transaction timeout. Measure before raising concurrency; Phase 7 must add stream/write/disconnect controls. Spring graceful shutdown currently uses its framework grace default; the longer stream drain procedure below is a future qualification requirement. + +Metrics include `gateway.requests` by HTTP status, `gateway.requests.active`, `gateway.admission.rejected`, `gateway.inference.outcomes`, `gateway.inference.duration`, `gateway.provider.duration`, and `gateway.diagnostics.dropped` (Prometheus normalizes dots to underscores). Validation/authentication denials do not create database rows. Diagnostic writes occur once per inference execution and are best effort; a failure increments the drop counter without replacing the inference result. These are not audit or financial records. + +### Target lifecycle for subsequent phases + +- Liveness reflects whether the process can make progress, not whether every provider is healthy. Avoid outage-driven restart storms. +- Readiness requires compatible schema, usable published configuration, fresh security state, and dependencies needed for governed admission. Failed providers affect route availability, not automatically process liveness. +- Startup probes allow migrations/bootstrap to complete. Run schema migration as an explicit deployment job with a privileged migration role; runtime role is restricted. Test lock contention and restart behavior. +- On shutdown, mark unready, stop new admission, drain streams, cancel remaining attempts after the grace deadline, settle or retain reservations and close pools. Reference drain grace is 150 seconds for a 120-second request deadline; align ingress/load-balancer and termination timing. +- Bound concurrency and queues. Scale with active requests/streams, queue time and CPU/memory, not CPU alone. Record DB connections per replica so autoscaling cannot exhaust the database. + +The dependency outage matrix in [Architecture.md](Architecture.md) is authoritative. Strict limits never silently become best-effort when Redis fails. PostgreSQL loss blocks new governed admission; stale revocations block new traffic after the security freshness bound. + +## Monitoring and capacity evidence + +Provide dashboards and alerts for gateway-controlled errors, client-visible upstream errors, request/stream saturation, p95/p99 latency, TTFT, deadline/cancellation rates, retries/fallback, config convergence, circuit status, Redis/DB availability, exporter backlog, dropped diagnostic events and old budget reservations. + +Define the availability SLI from valid admitted requests. Policy/limit denials are counted separately; upstream failures remain visible in a separate end-to-end SLI rather than disappearing from operational reports. A stream that starts successfully and later fails is a failed completion. Record error-budget burn alerts and incident thresholds with the selected SLO window. + +The initial performance target from PRD uses a controlled upstream, 100 requests/s and 200 concurrent streams. Phase 6 records payload sizes, stream length/token cadence, operation mix, hardware, JVM/GC, DB/Redis topology, network and test duration. Measure non-streaming overhead against direct-upstream latency, and added TTFT separately from generation time. Publish saturation and slow-consumer behavior. Vendor benchmarks are not evidence for this application. + +Qualification includes at least a one-hour steady-state test and a longer soak sized to expose resource leaks, plus overload, provider outages, dependency recovery and rolling upgrades. Assign the final soak duration and workload after the baseline; do not claim these runs have happened. + +## Backup, restore and upgrade + +PostgreSQL backups and WAL/PITR protect configuration, identities, audit, usage and reservations. Back up encryption-key references/configuration and required object data separately; storing ciphertext without recoverable keys is not a restore plan. Redis is rebuildable cache/limiter state, not authoritative spend. + +Reference recovery targets are RPO ≤15 minutes and RTO ≤60 minutes, to be proven on the declared topology. Database recovery can lose recent usage records within RPO: retain/recover durable dispatch evidence and reconcile provider activity before reopening strict-budget paid traffic. Never reset spending to the last backup and assume no later cost occurred. + +Restore procedure to implement and rehearse: + +1. Isolate ingress and restore database/object data into a clean environment using verified backups. +2. Validate schema, configuration hashes, tenant ownership and integrity; restore secret access separately. +3. Reapply current revocations, retention/deletion tombstones and security state from authorized recovery records. +4. Reconcile unsettled attempts and any provider activity beyond the restored point. Keep affected paid routes closed or conservatively reserved while uncertain. +5. Rebuild Redis state conservatively, warm config snapshots and run synthetic tenant/permission/inference checks. +6. Reopen a canary, monitor errors/spend/config convergence, then expand. Record actual RPO/RTO and gaps. + +Upgrade path: back up → verify source/target compatibility → apply additive schema → stage new binary/config → canary → monitor → promote. Keep old instances only while both binaries support the schema and security contract. Destructive schema contraction happens after a release compatibility window. Database down-migration is not the default rollback; use roll-forward or isolated restore when appropriate. Configuration rollback never reverses migrations, charges or revocations. + +## Runbooks required before GA + +| Incident | Required operator action and evidence | +|---|---| +| Provider timeout/429/outage | Inspect sanitized attempt reasons, deadlines/circuits and eligibility; disable target or activate validated policy; no guardrail/residency bypass | +| Database failure | Stop governed admission; restore/fail over; reconcile in-flight uncertainty before reopening | +| Redis failure | Observe fail-closed strict routes, bypass cache, restore counters conservatively and verify multi-replica limits | +| Bad configuration | Inspect replica acknowledgements, activate prior validated revision, keep current security epoch | +| Key or provider-secret compromise | Revoke/disable, propagate epoch, rotate, inspect audit/usage, verify convergence | +| Budget discrepancy | Inspect price/usage source, retries and uncertain reservations; append corrections rather than overwrite ledger | +| Export backlog | Inspect bounded queue/outbox age, destination health and poison events; retry idempotently, alert on retention pressure | +| Capacity exhaustion/slow clients | Shed load within bounds, inspect active streams/pools/DB connections, scale within downstream capacity | + +## Release evidence checklist + +These are intentionally unchecked until implementation and verification occur. A release owner attaches revision, environment, command/report and responsible owner to each item. + +- [ ] Supported dependency/runtime versions, release artifact manifest, SBOM, scans and signatures verified. +- [ ] Fresh install, populated MVP upgrade and previous-supported-version upgrade passed on PostgreSQL. +- [ ] Demo credential rejection, restricted management and non-root runtime verified. +- [ ] SDK/protocol/provider conformance matrix completed, including controlled live-provider smoke gates. +- [ ] Two-tenant API/database/cache/async authorization tests passed. +- [ ] Cross-replica quotas, budget reservation, crash/reconciliation and Redis/DB failure tests passed. +- [ ] Stream cancellation, slow consumers, partial failure and no-post-output-retry tests passed. +- [ ] Default content/secret non-disclosure, guardrail and retention/delete/export tests passed. +- [ ] Config activation, missed-notification recovery, stale replica removal, revocation and rollback tested. +- [ ] Load/soak/chaos results meet the published workload targets and resource bounds. +- [ ] Backup/PITR restore, current-revocation replay and usage reconciliation meet declared recovery targets. +- [ ] Helm, systemd and Podman packages demonstrate the same policy/security behavior; offline local-provider install passed. +- [ ] Console/API onboarding and operator incident journey accepted; accessibility and authorization checks complete. +- [ ] On-call/security/release owners, incident contacts, supported versions, limitations and distribution terms published. +- [ ] Canary promotion and binary/config recovery rehearsed; all F01–F13 gates closed. + +No production deployment has been performed. Phase 6 evidence does not close this GA checklist; [Phases.md](Phases.md) assigns the remaining work and release order. diff --git a/Documentation/PRD.md b/Documentation/PRD.md index b466e6d..ed562e8 100644 --- a/Documentation/PRD.md +++ b/Documentation/PRD.md @@ -1,94 +1,100 @@ -# PRD.md — AI Inference Gateway +# AI Inference Gateway product requirements -> This file defines **what** is being built, **who** it's for, and **what features** it must have. Any AI model picking up this project should read this first for product context before touching code. +Updated 2026-09-21. Status: product plan with Phase 6 foundation implemented; validation evidence and remaining gates are in [Phase6.md](Phase6.md). This document replaces the MVP-only scope. Production readiness is a release outcome to demonstrate, not a claim about the current code. ---- +## Product direction -## 1. What This Project Is +Build a self-hosted Java AI gateway that gives organizations one governed endpoint for cloud and local models. Applications should be able to change model deployments without rewriting integrations. Platform teams should be able to explain routing decisions, control access and spending, protect data, and operate reliably. -**AI Inference Gateway** is a backend platform that sits between client applications and AI model providers (OpenAI, Anthropic, local models, etc.), acting as a single, unified endpoint for all AI requests. +The supplied competitive research informs the direction; its recommendations are inputs to this plan, not development instructions. [Research.md](Research.md) records evidence and adaptations. [Architecture.md](Architecture.md) defines the design, [Phases.md](Phases.md) defines delivery dependencies, and [Roadmap.md](Roadmap.md) records work actually completed. -Instead of every application integrating directly with a provider's API, applications send requests to this gateway. The gateway routes the request to a configured provider, normalizes the response, logs the activity, and returns a consistent response — regardless of which provider actually handled it. +The initial product is customer-operated with customer-owned provider credentials. Hosted SaaS, commercial pricing, and certification commitments require separate product decisions. No provider-count or performance leadership claim is assumed. -**One-line summary:** An API gateway for AI usage — one endpoint, multiple providers, centralized control and visibility. +## Users and essential journeys ---- +| User | Essential journey | Product outcome | +|---|---|---| +| Application developer | Obtain a scoped key, discover allowed models, call chat or embeddings using a supported SDK | Standard integration, usable errors, streaming, migration examples | +| Platform administrator | Connect providers, simulate a routing policy, activate and roll back a version | Validated, attributable changes without worker restarts | +| Tenant administrator | Create applications and service accounts, rotate keys, assign access and limits | Self-service within tenant boundaries | +| SRE | Diagnose an outage, drain a replica, restore a backup | Recovery without prompt logging | +| Security reviewer | Inspect identity, residency, redaction, audit records and retention | Testable controls and explicit protection limits | +| Cost owner | Compare usage and reconcile provider estimates | Explainable spend and enforceable allocation | -## 2. Problem It Solves +An application must never gain access to another tenant's keys, aliases, configuration, cached outputs, usage, or logs by changing a header or resource ID. Cross-tenant platform administration is explicit and audited. -Without a gateway, every team/app integrates with AI providers independently, which causes: +## Current baseline -- Duplicated integration work across teams -- Painful provider switching (logic scattered across many apps) -- No central control over who can access which model -- No visibility into usage, errors, or performance -- Inconsistent AI access patterns across the org +The code exposes `POST /v1/inference`, `GET /v1/providers`, `GET /v1/logs`, and unauthenticated health. It supports synchronous Ollama generation, explicit/default provider selection, model validation, BCrypt API keys, request IDs, and PostgreSQL metadata logs. Redis is provisioned but unused. There is no control-plane API or UI. -This project centralizes AI access behind one platform layer. +MVP Phases 0–5 were recorded complete, and the original 62 tests were reproduced before Phase 6 changes. Phase 6 adds bounded transport/admission, safe errors, terminal diagnostic handling, private management, seed safeguards, wrapper/CI and real PostgreSQL migration tests. Identity is still a generic principal and logs are globally visible to valid keys; tenant isolation remains Phase 8. See [Architecture.md](Architecture.md) and [Phase6.md](Phase6.md) for current limits and measured evidence. ---- +## Requirements and release scope -## 3. Target Users +Requirements below remain **planned**, except the baseline above and implemented F01/basic F07 controls documented in Phase6. F01's remote CI gate remains open until a clean-checkout workflow passes. IDs link requirements to phases and release evidence. -| User | What they need from this project | -|------|-----------------------------------| -| **Application developers** | One stable endpoint to call instead of integrating with multiple providers directly | -| **Platform / infra teams** | Central control over configured providers, routing, and traffic visibility | -| **Engineering teams evaluating providers** | Ability to test locally (Ollama) now and swap in real providers later without changing client code | +| ID | Requirement | Observable acceptance | Delivery | +|---|---|---|---| +| F01 | Safe foundation | Sanitized errors, deadlines, bounded input/concurrency, seed safeguards, PostgreSQL migration tests, repeatable CI | Phase 6 | +| F02 | Protocol compatibility | Chat Completions text, tools and structured output where supported; SSE; scoped model discovery; published compatibility subset | Phase 7; GA | +| F03 | Provider runtime | Generic OpenAI-compatible HTTP adapter, Ollama, validated OpenAI and Anthropic adapters, deployment capabilities and credential references | Phase 7; GA | +| F04 | Identity and isolation | Tenants, applications, service accounts, indexed keys, scopes, expiry, rotation, revocation, admin OIDC and roles | Phase 8; GA | +| F05 | Reliable routing | Virtual models, pools, ordered fallback, priority/weighted selection, bounded retries, health and circuits; recorded reasons | Phase 9; GA | +| F06 | Consumption governance | Distributed request/token admission, concurrency controls, durable usage and price versions, alerts, hard budget reservations and reconciliation | Phase 10; GA | +| F07 | Operational visibility | Metrics, traces, request/attempt correlation, TTFT, cancellation, usage, routing reasons, dashboards and alerts | Basic Phase 6; complete Phase 11; GA | +| F08 | Privacy and guardrails | Content off by default, scoped retention/export, built-in secret/regex policies, pre/post hooks, streaming and failure semantics | Phase 11; GA | +| F09 | Configuration lifecycle | Admin APIs and declarative import/export; validate, simulate, stage, activate, monitor convergence, roll back; immutable revisions and audit | Phase 12; GA | +| F10 | Operator experience | API-backed console for onboarding, access, routes, decisions and usage; onboarding and SDK examples | Phase 12; GA | +| F11 | Exact caching | Opt-in, tenant-scoped, policy/version-aware keys, TTL and size bounds, purge and hit accounting; sensitive requests bypass | Phase 11; GA | +| F12 | Deployable product | Hardened image, Helm and Linux systemd/Podman packages, local Compose, backup/restore, upgrades, rollback and offline verification | Phase 13; GA | +| F13 | Embeddings | Text embeddings and normalized usage for supported deployments | Phase 7; GA | +| F14 | Enterprise extensions | Bedrock/Azure adapters, native Responses and Anthropic Messages endpoints, SCIM, richer multimodal/batch workloads, secret-manager integrations | Phase 14; post-GA | +| F15 | Agent and serving extensions | MCP registry/proxy with per-tool authorization; inference endpoint-picker integration; prompt/evaluation modules | Phase 15; post-GA | ---- +GA means F01–F13 pass release gates. F14–F15 are product expansion, not hidden blockers for the first production deployment. A private preview after Phases 6–7 is restricted to trusted isolated evaluation. Enterprise beta requires Phases 6–10 and carries no production availability promise. -## 4. Core Goal (MVP) +## Required behavior -> Prove that a single gateway endpoint can route AI requests to multiple pluggable providers, while centralizing logging, visibility, and provider abstraction — without needing to solve every enterprise problem on day one. +### Client compatibility ---- +The product owner permits redesigning unsuitable MVP architecture and implementation. The legacy continuity commitments below are the initial transition plan and may be revised where they preserve defects or create unjustified complexity. Record the reason, actual consumer/data impact and replacement contract, and update API documentation, examples and acceptance tests together. This permits deliberate design changes without assuming that existing data can be discarded. See [Development.md](Development.md). -## 5. Features — In Scope (MVP) +Keep `/v1/inference` and its existing JSON shape through the first GA series. It is a legacy facade, not a redirect or a wire-compatible alias of Chat Completions. New `/v1/chat/completions`, `/v1/embeddings`, and `/v1/models` endpoints expose a documented subset. Changing base URL, gateway credential, and model alias should suffice for that subset; arbitrary provider features are not guaranteed. -- Single unified API endpoint for AI requests: `POST /v1/inference` -- Provider abstraction via a common interface (`infer()`, `info()`, `health()`) — gateway core has zero provider-specific knowledge -- One concrete provider integration for MVP: local model via **Ollama** -- Deterministic rule-based routing: explicit `provider` field → else configured default provider -- Standardized success response schema, and a single standardized error schema across all endpoints -- Every request/response tagged with a unique `requestId` -- Request logging to PostgreSQL (timestamp, provider, model, status, latency, error code) -- Paginated + filterable log retrieval: `GET /v1/logs` (`page`, `size`, `provider`, `status`, `from`, `to`) -- Provider configuration management, stored in PostgreSQL, viewable via `GET /v1/providers` -- Supported models stored as config data (not hardcoded) -- Simple API key authentication (`X-API-Key` header) on all endpoints -- Fully containerized local dev environment (Docker Compose: gateway + Postgres + Redis + Ollama) +Reject unsupported features before upstream invocation. Never silently drop tools, structured-output constraints, safety settings, or modalities. New policies apply to legacy routes too. Tenant restrictions on legacy visibility endpoints and key rotation are intentional security migration changes documented before rollout. [API.md](API.md) defines the transition. -## 6. Features — Explicitly Out of Scope (MVP) +### Policy and reliability -- Rate limiting, quotas, caching (Redis is provisioned for this later, not used for logic yet) -- Provider failover / load balancing -- Active/background provider health monitoring (failures are surfaced per-request only) -- Streaming responses (token-by-token/SSE) -- JWT, OAuth2, RBAC, user accounts (API keys only) -- Multi-tenant governance, advanced policy management -- Cost tracking, billing, deep analytics/dashboards -- Multi-region deployment, compliance certifications (SOC2, HIPAA, etc.) -- Audit logs (request logs only, for now) +A virtual model represents an operator-approved set of interchangeable deployments. Fallback cannot bypass model permissions, residency, guardrails, context limits, or price constraints. Policy rejection is terminal. Retries and fallbacks consume one attempt budget and total deadline. Once downstream response bytes have been sent, generation cannot restart transparently. ---- +Every admitted inference request receives a decision summary with configuration version, selected deployment, relevant rule IDs, and outcome. Denials record safe reasons without exposing other tenants' resources. Simulation uses the live policy evaluator and cannot invoke a model, consume budget, or execute a tool. -## 7. Success Criteria +### Spending and privacy -The MVP is complete when: +Distinguish estimates from provider-reported tokens and reconciled cost. Account for charged failed attempts, retries and cancellations. Missing usage is unknown, never automatically zero. Hard-budget routes require a conservative price/usage bound before dispatch; unsupported bounds reject or use explicitly designated soft-budget mode. -- [x] An app can send a request to one stable gateway endpoint and get a provider response back -- [x] Requests route correctly per the explicit-provider / default-provider rule -- [x] Every response (success or error) carries a `requestId` -- [x] Requests without a valid API key are rejected with the standard error schema -- [x] Every request is logged with full metadata, retrievable via paginated/filtered `/v1/logs` -- [x] Configured providers are viewable via `/v1/providers` -- [x] The whole system runs locally via `docker compose up` with no external paid API dependency +Store no prompt, completion, tool argument, or secret by default in logs, traces, audit records, or exports. Content capture and caching require separate explicit policy. Redacted sampling and encrypted capture are tenant-controlled. A logging setting does not authorize response reuse across users. ---- +## Proposed quality targets -## 8. Related Docs +These are initial engineering targets, not measured capabilities or contractual SLAs. Phase 6 establishes a reproducible workload; Phase 13 validates them or records a product-approved revision with evidence. -- `Architecture.md` — tech stack, folder structure, system flow -- `Rules.md` — boundaries for AI-assisted development (libraries, error handling, do's/don'ts) -- `Phases.md` — build order broken into stages \ No newline at end of file +| Dimension | Target and measurement | +|---|---| +| Availability | 99.9% monthly gateway-controlled success for valid, admitted calls; exclude intentional policy/limit denials, separately report upstream failures and total client-visible success | +| Overhead | At 100 requests/s and 200 concurrent streams on the declared reference environment: p95 added non-streaming latency ≤50 ms and p95 added TTFT ≤50 ms against a controlled upstream; publish p50/p95/p99 and saturation | +| Streaming | Cancel upstream and release resources within 2 seconds of detected disconnect; bounded detection through writes/heartbeat or timeout; no post-output replay | +| Limits | Zero unauthorized admissions in two-replica atomic request-limit tests; token-estimate drift measured and reconciled | +| Budget | Concurrent reservations cannot exceed configured admission budget; uncertain/charged attempts remain attributable until settlement | +| Config and revocation | Healthy replicas converge within 5 seconds; replicas with security state older than 30 seconds reject new governed traffic | +| Recovery | Reference RPO ≤15 minutes and RTO ≤60 minutes, proven by restore drill; reconcile strict budget state before reopening paid traffic | +| Security | No unresolved critical/high exploitable release findings; cross-tenant denial and secret/content non-disclosure tests pass | +| Compatibility | Version-pinned official Java, Python and Node SDK fixtures pass the advertised subset; legacy regressions pass | + +## Boundaries and open product decisions + +The gateway does not train or host model weights, execute application tool calls in the GA core, guarantee model correctness, or neutralize all prompt injection. Applications remain responsible for validating outputs and authorizing business actions. + +Semantic caching, quality-based automatic routing, mandatory Kafka, independent microservices, active-active multi-region accounting, hosted billing, and compliance certifications are outside first GA. They require measured need or a separate product case. Offline means no mandatory vendor callback; selected cloud providers still require network access. + +Before beta, the product owner selects pilot workloads, deployment environment, identity provider, residency regions and retention settings. Before GA, the release owner names operational owners, supported versions, support policy and license/distribution terms. Missing dates or staffing do not justify invented delivery commitments. diff --git a/Documentation/Phase6.md b/Documentation/Phase6.md new file mode 100644 index 0000000..98d9af4 --- /dev/null +++ b/Documentation/Phase6.md @@ -0,0 +1,79 @@ +# Phase 6 implementation evidence + +Updated 2026-09-21. **Implementation delivered; acceptance remains open for the full image dependency scan/SBOM and a clean-checkout remote CI run.** Local regression, PostgreSQL, source scan, container OS scan, packaging and startup checks passed. This is foundation evidence, not production readiness. + +## Baseline and design assessment + +The unmodified application passed 62 tests on Java 21.0.7 and Maven 3.9.9 (Windows 11). H2 tests did not exercise Flyway. Maven's sandbox default repository was unwritable; a project-local ignored cache was used with filesystem/network approval. Docker Desktop was installed under the user's local Programs directory rather than on PATH. + +Retain the modular monolith, controller/service separation, Java 21, JPA, SQL migrations and legacy API shape. Replace the implicit provider HTTP transport with an explicitly pooled, deadline-cancellable transport with bounded response bodies and no automatic retries. Repair orchestration to record every admitted inference outcome exactly once on a best-effort diagnostic basis. Diagnostic database failures must not replace a successful provider response or its original failure; financial accounting remains later work. + +Upgraded the framework baseline to Spring Boot 4.1.1 using the official [Spring Boot migration guide](https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-4.0-Migration-Guide) and [release information](https://spring.io/projects/spring-boot/). Adopted Jackson 3 and the new MVC test/Flyway modules rather than introducing a permanent compatibility layer. Overrode managed Tomcat 11.0.24 with 11.0.25 after scanning found CVE-2026-65182, CVE-2026-65905 and CVE-2026-68525. The [Tomcat security advisory](https://tomcat.apache.org/security-11.html#Fixed_in_Apache_Tomcat_11.0.25) identifies the fixed patch; remove the override when Boot manages it or a newer version. Trivy's critical severity classifications differ from the vendor's contextual ratings; no finding was suppressed on that basis. + +No external consumer or production deployment is documented in this repository. Preserve the public inference shape while deliberately correcting erroneous status codes and making demo credentials explicit local bootstrap only. Preserve V1–V5 checksums; add forward migration and startup safeguards. Do not delete or reset existing databases. + +## Implemented scope + +| Work package | Delivered | +|---|---| +| G001 baseline/build | Wrapper 3.9.11 with verified SHA-256; Boot/Jackson upgrade; patched Tomcat; deterministic JAR timestamp; pinned image bases/actions; CI tests/scans/SBOM; release manifest | +| G002 database | Four real PostgreSQL integration checks covering V1–V6 fresh install, populated V5 upgrade, real application startup/private management, and rehashed demo rejection | +| G003 bootstrap | V6 deactivates demo key; startup guard; explicit local key/provider bootstrap; loopback Compose publication; private management; non-root image | +| G004 lifecycle/errors | Input/query/media/status repair; safe fixed responses and operational provider errors; one best-effort diagnostic call per inference outcome; diagnostic failures preserve original result | +| G005 bounds | Pre-authentication body/concurrency admission, pooled HTTP limits, typed connect/read/acquisition timeout handling, cancellable upstream deadline, no retries or redirects | +| G006 evidence | Bounded metric tags, controllable HTTP upstream, slow/error/saturation/privacy tests, opt-in benchmark and Phase 7 streaming experiment | + +The implementation keeps the legacy request/success/error shapes; deliberate compatibility changes are safe fixed messages, provider-unavailable 503, validated bad queries, enforced body/media limits, disabled nonlocal demo authentication and management on port 9090. [Operations.md](Operations.md) covers upgrade and rollback limitations. Existing customer databases were not accessed; upgrade evidence used disposable schemas with representative rows. + +## Observed verification + +Source base revision: `cb95950a2d78431656cb6ca4756a0b056c20bdc0`, **dirty working tree with these Phase 6 changes**. No new commit or remote workflow run was created. Logs and detailed Maven reports are local ignored build outputs; durable benchmark data is in [evidence/phase6-baseline.json](evidence/phase6-baseline.json). + +Environment: Windows 11, Java 21.0.7, Maven wrapper 3.9.11, Docker Desktop 4.90.0 / engine 29.7.2, PostgreSQL 16.15 (`postgres:16-alpine` pinned digest in CI). Runtime image: Temurin 21.0.12+8, Alpine 3.24.2, UID/GID 10001. Tests used a disposable PostgreSQL container on loopback port 15432 and random `p6_…` schemas; fixtures dropped only their own schemas. No existing database was reset. + +| Check | Observed result | +|---|---| +| Original `mvn -B test` | 62 passed before implementation | +| Final `mvnw.cmd -B -Ppostgres-it clean verify` | 88 unit/MVC/transport tests passed; one opt-in benchmark skipped; four PostgreSQL integration tests passed; zero failures/errors | +| `mvnw.cmd -B -Dgateway.benchmark=true -Dtest=BaselineBenchmarkTest test` | One benchmark test passed separately; 320 successful mock HTTP calls including warm-up | +| Repeat `mvnw.cmd -B -o -DskipTests package` after clean build | Identical SHA-256 on both JARs | +| Packaged image | Built from exact tested JAR; container JAR SHA-256 matched; UID/GID 10001; real PostgreSQL readiness returned `{"status":"UP"}` | +| Source image | Built successfully with temporary `--add-host repo.maven.apache.org:104.18.19.12` for this host's Docker DNS issue; no IP override committed | +| Source security scan | Trivy 0.74.0: zero high/critical dependency, secret or configuration findings after Tomcat patch | +| Runtime OS scan | Trivy 0.74.0: 73 Alpine packages checked; zero high/critical findings | +| Full runtime dependency scan / SBOM | **Not verified locally**: Java index downloads from both official registries ended with unexpected EOF | +| GitHub Actions clean-checkout verification | **Not run remotely**; workflow configured to fail on test failures or high/critical findings and retain reports | + +Final JAR SHA-256: `ae60015186ed6b34b557ca1d0d23fc05567293efc1d719aad26459f399505aa8` (64,614,319 bytes). Final packaged image ID: `sha256:c0017a487d066808cb90721745876856d319d40ce857aa7156ac01163c6ed1fd`. These identify a local uncommitted build, not a signed release. `scripts/release_manifest.py` writes source revision/dirty state, artifact hash/size and toolchain target to `target/release-manifest.json`. + +The source scan used a complete local Maven repository mounted at `/root/.m2/repository`, `--offline-scan`, a vulnerability DB updated 2026-09-21T07:13Z, and Trivy's embedded configuration checks after a policy download failed. It excluded `.git`, `.build-cache`, and generated `target` output. The independent OS scan explicitly excluded JARs; it is not a substitute for the pending full image scan. CI retains the full source/JAR/image scans and has no vulnerability suppression or `continue-on-error` bypass. + +### Reproduction + +Use the commands in README with Java 21 and a disposable PostgreSQL database. This Windows run additionally set `MAVEN_USER_HOME` to the ignored `.build-cache/wrapper` and passed `"-Dmaven.repo.local=E:\ashish\Heisenberg\AI-Inference-Gateway\.build-cache\m2"`. Set `TEST_DATABASE_URL`, `TEST_DATABASE_USER`, and `TEST_DATABASE_PASSWORD` for `-Ppostgres-it`; missing configuration fails rather than skips the gate. The test user needs permission to create/drop schemas in the designated test database. + +The slow-response fixture trickles bytes every 20 ms, configures a 150 ms total deadline with a longer read timeout, checks bounded cancellation, then successfully reuses the single-connection pool. Other fixtures check pool-acquisition saturation without extra dispatch, expired deadline/no dispatch, 429/500/redirect/oversized provider responses without retry or raw text, request-body limits without Content-Length, permit release, parser/query errors, safe 403, diagnostic failure after success and failure, and management isolation on real listeners. + +### Non-streaming baseline + +The benchmark used loopback HTTP, synthetic Ollama output, a 1,024-character prompt, one BCrypt key, H2 in-process diagnostic writes, 20 gateway warm-up requests and 100 samples per measured case. Java saw 24 logical processors and a 4,215,275,520-byte maximum heap. It used the upgraded Boot baseline before the Tomcat security patch. No Redis, external provider latency or real model inference was involved. + +| Case | Concurrency | p50 | p95 | p99 | Observed requests/s | +|---|---:|---:|---:|---:|---:| +| Direct mock | 1 | 0.55 ms | 1.13 ms | 1.97 ms | 1479.3 | +| Gateway | 1 | 51.79 ms | 60.24 ms | 64.85 ms | 18.8 | +| Gateway | 10 | 54.56 ms | 59.13 ms | 62.02 ms | 177.1 | + +End-of-run used Java heap was 72,000,456 bytes, a snapshot rather than a peak-memory or leak measurement. Sequential gateway/direct p95 difference was approximately 59.11 ms; subtracting percentiles is only a comparison, not a paired per-request overhead distribution. The sequential gateway sample lasted approximately 5.3 seconds. This short closed workload does not validate the PRD's 100 requests/s, 200-stream, one-hour target or its ≤50 ms overhead goal. BCrypt verification and synchronous diagnostics are included; this experiment did not profile their individual contribution. CPU utilization, RSS, peak buffers and production PostgreSQL latency were not measured here. + +## Remaining acceptance and operating limits + +1. Run the committed workflow from a clean checkout; retain its exact revision, test reports and scan artifacts. No remote CI result is implied by local success. +2. Complete the full image dependency scan and container SBOM once the Java advisory database can be downloaded. Do not accept a partial scan or disable that gate to publish a release. +3. Recheck dependency/base-image advisories at release time. Pins provide repeatability, not permanent security; maintain them. + +Immediate downstream disconnect cancellation, bounded stream writes, deployment-isolated pools, indexed identities, tenancy, durable usage/budgets, configuration refresh, recovery drills and production load/soak tests remain future phases. The current total deadline actively aborts upstream I/O but does not hard-limit all synchronous database work or downstream delivery. Local diagnostics can be dropped; they are not audit or billing records. No real Ollama/cloud-provider smoke test, full Compose model download, external consumer migration or production deployment was performed. + +## Streaming experiment for Phase 7 + +Compare async MVC with a reactive edge using the same mock upstream: 100 requests/s, up to 200 simultaneous streams, 1 KiB input, 256 chunks of 64 bytes at 20 ms intervals, a 1% mid-stream failure mix, and 10% slow/disconnecting clients. Publish JVM/hardware/network, executor/pool sizes, CPU/RSS, added TTFT, cleanup latency, error rate and peak buffered bytes. Run a one-hour steady load after a warm-up and a separate saturation ramp. The Phase 6 non-streaming measurements are not evidence for this streaming target. diff --git a/Documentation/Phases.md b/Documentation/Phases.md index 024b28e..f5a787e 100644 --- a/Documentation/Phases.md +++ b/Documentation/Phases.md @@ -1,152 +1,301 @@ -# Phases.md — AI Inference Gateway +# AI Inference Gateway delivery phases -> This file breaks the MVP into sequential, buildable phases. Build **one phase at a time, in order** — do not skip ahead or combine phases. Each phase should be completed, tested, and working before moving to the next. Read `PRD.md`, `Architecture.md`, and `Rules.md` first. +Updated 2026-09-21. This is the forward implementation plan, not a completion log. **Phase 6 implementation is present with local test evidence; its remaining build/security/remote-CI gates are tracked in [Phase6.md](Phase6.md). Phases 7–15 have not started.** [Roadmap.md](Roadmap.md) preserves actual history. Requirement IDs refer to [PRD.md](PRD.md); technical decisions refer to [Architecture.md](Architecture.md). ---- +## Working approach -## Phase 0 — Project Scaffolding & Data Model +The MVP's one-phase-at-a-time rule and fixed library restrictions are retired. Phases now describe dependencies and release gates. Independent work may proceed concurrently once shared contracts are settled; related changes may span phases. There is no requirement to request approval for every routine implementation choice or wait for an entire phase to finish before starting independent work. -**Goal:** A running Spring Boot service with all infrastructure wired up, database tables created, and seed data loaded — no business logic yet. +Existing architecture and implementation may be changed wherever they are incorrect, difficult to maintain, poorly structured or likely to create unnecessary future complexity. Phase 6 explicitly assesses these issues; later phases can revisit decisions as evidence emerges. Update the design, affected work packages and validation together. Legacy preservation and migration tasks below follow the current transition design and can be revised under the documented redesign policy in Development; tests must validate the accepted replacement behavior rather than enshrine known defects. -### Scaffolding -- Initialize Spring Boot project (Web, Data JPA, Validation, PostgreSQL driver). -- Set up `docker-compose.yml`: gateway service, PostgreSQL, Redis, Ollama. -- Set up Flyway migration tooling (empty baseline). -- Set up `application.yml` with env-var-based config (DB connection, Ollama base URL, Redis). -- Create the folder structure exactly as defined in `Architecture.md §5` (empty packages are fine). -- Add a basic health endpoint (`GET /actuator/health` or a simple `GET /ping`) to confirm the service boots inside Docker. +Use small vertical slices with an owner, acceptance evidence, compatibility/migration impact and rollback notes. Update relevant docs with meaningful scope/design changes and append a historical milestone only after work is done. No arbitrary dates or delivery durations are assigned without team capacity and pilot workload information. [Development.md](Development.md) describes the lightweight workflow. -### Data Model & Migrations -- Write Flyway migrations for `providers`, `models`, `api_keys`, `request_logs` (schemas in `Architecture.md §6`). -- Create corresponding JPA entities + Spring Data repositories. -- Write a seed migration: one `ollama-local` provider row (`is_default = true`), one model row (e.g. `qwen3`), one API key (hashed) for local testing. +## Historical baseline -**Done when:** `docker compose up` starts all four containers, migrations run automatically on startup, the gateway responds to a health check, and seed data is queryable directly via the DB. +| Original phase | Recorded result | Status | +|---|---|---| +| 0 | Spring Boot, Compose, entities and V1–V5 migrations | Recorded complete | +| 1 | Request IDs, error model, provider SPI/registry, Ollama and routing | Recorded complete | +| 2 | BCrypt API-key authentication | Recorded complete | +| 3 | Inference endpoint and metadata logging | Recorded complete | +| 4 | Provider/log visibility endpoints | Recorded complete | +| 5 | Log filters, expanded tests and onboarding docs | Recorded complete | ---- +The original 62-test baseline was reproduced before Phase 6 changes. The expanded suite and PostgreSQL checks now pass; [Phase6.md](Phase6.md) records counts, environment, benchmark and remaining gates. None is production certification. -## Phase 1 — Core Gateway Internals +## Dependencies and releases -**Goal:** All internal machinery — error handling, request ID generation, provider abstraction, routing — is built and unit-tested. Nothing is exposed via HTTP yet (beyond the health check), but every piece works in isolation. +```mermaid +flowchart LR + P6[6 Foundation] --> P7[7 Protocols and providers] + P6 --> P8[8 Identity and tenancy] + P7 --> P9[9 Reliable routing] + P8 --> P9 + P8 --> P10[10 Usage and budgets] + P9 --> P10 + P10 --> P11[11 Privacy and visibility] + P9 --> P12[12 Control plane and console] + P10 --> P12 + P11 --> P12 + P11 --> P13[13 Production qualification] + P12 --> P13 + P13 --> P14[14 Enterprise breadth] + P13 --> P15[15 Agent and serving extensions] +``` -### Error Schema & Request ID -- Implement `ErrorCode` enum (`INVALID_REQUEST`, `PROVIDER_NOT_FOUND`, `PROVIDER_UNAVAILABLE`, `MODEL_NOT_FOUND`, `PROVIDER_TIMEOUT`, `INTERNAL_ERROR`, `BAD_CONFIGURATION`, `UNAUTHORIZED`). -- Implement `ApiError` response shape per `Rules.md §4`. -- Implement `GlobalExceptionHandler` — central catch-all that converts exceptions into standardized `ApiError` responses. -- Implement `RequestIdGenerator` (`req_xxxxxxxxx` format). -- Wire request ID generation into a request-scoped context (filter or interceptor) so it's available to controllers, services, and the error handler. +The graph describes **completion dependencies**. Observability, CI, operations, threat modeling and control-plane schemas start early; they do not wait until their final phase. Phase 10's usage schema can begin after Phase 7's contracts; it cannot close until Phase 9 attempt accounting is integrated. Phase 12's API/schema design can begin in Phase 8, but publication requires privacy and guardrail revisions from Phase 11. -### Provider Abstraction & Ollama Adapter -- Define the `AIProvider` interface (`infer()`, `info()`, `health()`) per `Architecture.md §4`. -- Implement `OllamaProvider` — calls the local Ollama HTTP API via Spring `RestClient` and maps the response into the internal `InferenceResponse` shape. -- Implement `ProviderRegistry` — loads active providers from the DB at startup and exposes lookup by name. +| Release | Required evidence | Exposure | +|---|---|---| +| Private preview | Phases 6–7; legacy tests; streaming/provider contracts | Isolated trusted pilot, no shared tenant service | +| Enterprise beta | Phases 6–10; tenant isolation; real distributed limits and accounting | Controlled pilot with explicit operating limitations | +| GA release candidate | Phases 6–12 plus completed Phase 13 qualification | Staging and customer acceptance | +| Production GA | All F01–F13 gates and Phase 13 evidence signed off | Supported single-region customer-operated product | +| Expansion | Phases 14–15 independently qualified | Only advertised capabilities carry support | -### Routing Engine -- Implement `RoutingEngine`: explicit `provider` field in request → else active default provider from DB → else `BAD_CONFIGURATION` error. +## Phase 6 Foundation and baseline repair -### Tests -- Unit test `OllamaProvider.infer()` with a mocked HTTP call — returns a normalized response. -- Unit test `RoutingEngine`: explicit provider resolves correctly; omitted provider falls back to default; no default configured produces the correct error. -- Unit test `GlobalExceptionHandler` — each defined exception type produces the correct standardized JSON shape with `requestId`. +**Status:** Implementation delivered and locally tested; acceptance remains open until every recorded build/scan gate and the clean-checkout CI workflow pass. This is not production readiness. See [Phase6.md](Phase6.md) for reproducible evidence and limitations. -**Done when:** All internal components are built and pass their unit tests. A manual integration test can call `OllamaProvider.infer()` directly and get back a real model response from the local Ollama container. +**Goal:** Establish safe, reproducible development and remove known MVP hazards. **Requirements:** F01, foundation of F07/F12. **Dependencies:** existing MVP. ---- +**Work packages** -## Phase 2 — Authentication +- Review the MVP's correctness, coupling, responsibility boundaries, testability and operational complexity against upcoming product needs. Record which components to retain, refactor or replace, along with actual API consumers/data dependencies, design rationale and revised acceptance/migration needs. Implement prerequisite improvements within the relevant slices instead of carrying known design problems into new features. +- Reproduce build/test baseline; add Maven wrapper, CI test/build checks, dependency/container/secret scanning and a reproducible release manifest. Inventory versions and upgrade Spring/dependencies to a supported tested baseline. +- Add PostgreSQL-backed tests that execute Flyway; retain fast unit tests. Test fresh V1–V5 install and the next additive migration against a populated MVP database. +- Correct parser/query validation and status mapping: malformed JSON, type mismatch, negative page, zero size, invalid status, reversed time range, oversized body and unsupported media. +- Sanitize public and operational provider errors. Cover routing failures, unexpected transport errors and diagnostic persistence failures in terminal request accounting; define metrics for unauthenticated/validation rejections without unbounded DB writes. +- Add explicit transport and total timeouts, connection-pool/concurrency bounds and cancellation hooks, with retries disabled. +- Deactivate known demo credentials through additive migration/production startup validation; introduce explicit local bootstrap and private management defaults. Do not edit applied V1–V5 checksums. +- Add basic request/outcome/latency metrics and a controllable mock upstream, including delayed and failed responses. +- Record deployment/workload assumptions and the A03 streaming experiment design. Start operational and security checklists. -**Goal:** API key auth is in place *before* any business endpoint goes live, so every endpoint is born protected. +**Exit evidence** -- Implement `ApiKeyService` — validates a plaintext key against stored hashes using Spring Security's built-in hashing utilities. -- Implement `ApiKeyFilter` — Spring Security filter that reads the `X-API-Key` header on every request and delegates to `ApiKeyService`. -- Configure Spring Security to apply the filter to all `/v1/*` routes. Health/actuator endpoints remain unprotected. -- Missing or invalid key → `401 UNAUTHORIZED` using the standard `ApiError` schema (with `requestId`). +- CI reproduces a clean checkout build and tests; actual results recorded with command, environment and revision. +- PostgreSQL fresh-install and MVP-upgrade tests pass; production cannot become ready with the known test key. +- Error fixtures show safe 400/401/403/413/415/500/503/504 behavior as appropriate; request IDs survive failures. +- Slow/unreachable upstream tests return within configured bounds; logging failure does not erase the original diagnostic outcome. +- Baseline latency/resource report exists. No new production-readiness claim. -### Tests -- Unit test `ApiKeyService` — valid key passes, invalid/missing key is rejected. -- Integration test — request to a protected path without a key returns `401`; request with the seeded test key passes the filter. +**Migration/rollout:** Additive changes, local profile retains a documented demo path; production startup safeguards intentionally reject unsafe defaults. -**Done when:** All `/v1/*` routes reject requests without a valid `X-API-Key`. Requests with the seeded test key pass through. Health/actuator endpoints remain accessible without a key. +## Phase 7 Protocol compatibility and provider foundation ---- +**Goal:** Make the gateway usable with standard client integrations. **Requirements:** F02, F03, F13. **Dependencies:** Phase 6; coordinate principal/tenant contracts with Phase 8. -## Phase 3 — `/v1/inference` Endpoint & Request Logging +**Work packages** -**Goal:** The primary use case works end-to-end — a client can send a prompt, get an AI response, and the request is logged. This is the core value delivery of the MVP. +- Define immutable command/result/usage/error/context contracts and adapter capabilities. +- Add Chat Completions non-streaming/SSE, text embeddings and model discovery. Keep the legacy inference facade and its response body stable. +- Implement the generic OpenAI-compatible HTTP adapter and adapter factories. Keep native Ollama generation for the legacy path until equivalence is tested; validate Ollama chat/embedding capabilities independently. +- Add direct OpenAI and Anthropic adapters for advertised operations. Anthropic upstream translation does not imply a native inbound Messages endpoint. Never fake unsupported embeddings/tools. +- Add credential references, endpoint allowlists and capability checks before dispatch. Preview may use one explicitly configured isolated tenant context; this is not multi-tenancy. +- Execute A03 spike: bounded async MVC SSE versus a reactive edge, slow clients, cancellation and long streams. Document the chosen transport and executor/connection limits. +- Test tools/structured output only on deployments advertising support; publish a field/feature matrix, SDK versions and error/stream fixtures. +- Add adapter contract tests, mock-only CI, and credential-gated real-provider smoke tests with explicit spend caps and no content artifacts. -### Inference Endpoint -- Implement `InferenceController` → delegates to `InferenceService`. -- `InferenceService` orchestrates: request validation → routing engine → provider adapter → response normalization. -- Define request/response DTOs with `jakarta.validation` annotations (`@Valid`, `@NotNull`, etc.). -- Return the normalized success response (with `requestId`), or a standardized `ApiError` if provider/routing fails. +**Exit evidence** -### Request Logging -- Implement `RequestLogService` — writes a row to `request_logs` after every inference call (success or failure), including `requestId`, provider, model, status, latency_ms, error code. -- Hook logging into `InferenceService` so it happens regardless of outcome (success or caught exception). +- Official Java, Python and Node SDK fixtures pass the advertised subset using gateway URL/key/alias changes. +- Text, tools, usage finalization, fragmented UTF-8/SSE, errors before and after first output, cancellation and slow consumers pass. +- Cancellation releases resources within the proposed measured bound; no duplicate terminal event; unknown usage stays unknown. +- Legacy regression suite passes; unsupported features fail explicitly before upstream dispatch. +- Required adapters each have documented live smoke evidence for advertised capabilities; absent credentials are reported as an open release gate, never marked passed. -### Tests -- Integration test: `POST /v1/inference` with a valid key and prompt → returns a real Ollama-generated response in the standard schema. -- Integration test: `POST /v1/inference` with explicit `provider` field → routes to that provider. -- Integration test: `POST /v1/inference` with missing/invalid fields → returns standardized `ApiError`. -- Verify that every inference call (success or failure) produces exactly one row in `request_logs`. +**Migration/rollout:** New APIs behind capability flags; preserve legacy routes and default names. Deliver isolated private preview only. -**Done when:** A `POST /v1/inference` call (with or without an explicit `provider`) returns a real Ollama-generated response in the standard schema, failures return standardized errors, and every call is logged to `request_logs`. +## Phase 8 Identity, administration foundation and tenant isolation ---- +**Goal:** Safely host multiple applications/tenants before exposing shared configuration or spend. **Requirements:** F04; foundation of F09/F10. **Dependencies:** Phase 6, agreed Phase 7 context contract; can progress alongside Phase 7. -## Phase 4 — Visibility Endpoints +**Work packages** -**Goal:** Platform/infra teams can inspect configured providers and query request history. +- Add tenants, applications, service accounts, gateway keys, memberships and role bindings. Carry immutable authenticated identity through async work. +- Replace all-key scanning for new keys with indexed public-ID lookup; add one-time reveal, expiry, rotation, revocation and coarse last-used updates. +- Implement admin OIDC validation and scoped permissions; separate admin APIs from inference keys. Secure one-time bootstrap with no fixed admin password. +- Create the minimal audited admin APIs needed for tenant/key/provider lifecycle; all mutations and their audit events commit together. +- Enforce tenant predicates, composite tenant foreign keys and PostgreSQL RLS with a restricted runtime role. Test pooled connections and background workers. +- Migrate legacy records through an explicit legacy tenant mapping; isolate unattributed historical logs. Add key-rotation guide and time-bounded compatibility switch. +- Restrict old logs/providers endpoints by scope and tenant. Add independent revocation/security-epoch propagation and stale-state rejection. -### `/v1/providers` -- Implement `ProviderController` → `GET /v1/providers`. -- Returns the list of configured providers and their associated models. +**Exit evidence** -### `/v1/logs` -- Implement `LogController` → `GET /v1/logs`. -- Supports query parameters: `page`, `size`, `provider`, `status`, `from`, `to`. -- Results sorted newest-first. -- Invalid filter values return a standardized `ApiError`. +- Two-tenant integration tests cover inference, discovery, log queries, admin resource IDs, service accounts and async context. +- Forged tenant headers, cross-tenant IDs, expired/revoked keys and wrong OIDC issuer/audience cannot bypass checks. +- Inference keys cannot mutate administration; unauthorized mutations do not create changes; audit writes accompany authorized ones. +- Key lookup cost is independent of total key count for new-format keys. +- Fresh/migrated installs pass ownership constraints; security freshness and revocation targets pass across two replicas. -### Tests -- Integration test: `GET /v1/providers` returns the seeded provider and model data. -- Integration test: `GET /v1/logs` returns paginated results; filtering by provider, status, and date range works correctly. -- Both endpoints require a valid API key (verified by existing auth filter from Phase 2). +**Migration/rollout:** Backfill before enforcing non-null constraints. Deliberate visibility restrictions and rotation requirements are announced; do not preserve insecure global access for compatibility. -**Done when:** Both endpoints return correct, paginated, filterable data. Auth is enforced. Invalid inputs produce standardized errors. +## Phase 9 Virtual models and reliable routing ---- +**Goal:** Route across eligible deployments predictably during normal operation and failure. **Requirements:** F05. **Dependencies:** Phases 7–8. -## Phase 5 — Hardening, Test Coverage & Documentation +**Work packages** -**Goal:** The MVP is demo-ready, stable, and welcoming to new contributors. +- Add deployments, tenant virtual models, capability descriptors, route policies and an initial immutable snapshot loader. +- Implement priority, weighted and ordered fallback pools; add round-robin/least-latency after deterministic selection and stale-sample behavior are tested. +- Implement typed failure classification, total deadlines, jittered retries and Retry-After support. Bound all attempts together; disable hidden client retries. +- Add per-deployment connection isolation, local circuit breakers, bounded probes, recovery jitter and optional shared cooldown hints. +- Reevaluate security eligibility before every attempt; preserve residency, tools, output schema and budgets during fallback. +- Emit request/attempt decision events with configuration version, rejected candidates and selection reasons. Add shadow policy evaluation without upstream duplication. -### Test Coverage -- Fill in any remaining unit tests (controllers, services, edge cases) per `Rules.md §6`. -- Add integration tests covering the full end-to-end happy path and at least 2–3 error paths (invalid key, unknown provider, Ollama down). -- Verify no stack traces, internal exception messages, or raw provider payloads leak to the client. +**Exit evidence** -### Review & Polish -- Review all responses against `Rules.md` — standard error schema everywhere, no hardcoded provider/model names in Java code, no scope creep. -- Verify every response (success and error) includes `requestId`. -- Verify all MVP success criteria in `PRD.md §7` are checked off. +- Failure matrix in Architecture section 5 passes for connect failure, 429/quota, 5xx, malformed stream, partial output, cancellation and recovery. +- No attempt starts after deadline, cancellation or downstream output; fallback never bypasses tenant/residency/capability policy. +- All providers down yields bounded errors; one broken deployment does not exhaust unrelated connection pools. +- Attempt IDs and potentially charged failures are available to Phase 10; weighted selection and least-latency behavior have meaningful statistical/freshness checks. -### Documentation -- Finalize `README.md`: project overview, prerequisites, `docker compose up` setup, example requests (curl or `.http` file), environment variables reference. -- Add sample request files in `postman/` or `http/` directory for manual testing. +**Migration/rollout:** Map old provider/model records to explicit aliases. Canary revised policies by authorized tenant/application, compare outcomes, and retain the previous snapshot. -**Done when:** All `PRD.md §7` success criteria are met. A fresh `git clone` → `docker compose up` gets a new developer to a working demo with no manual steps beyond providing env vars. Test suite passes green. +## Phase 10 Distributed consumption and cost governance ---- +**Goal:** Enforce shared limits and make consumption recoverable and explainable. **Requirements:** F06. **Dependencies:** Phases 8–9 and Phase 7 usage contract. -## Explicitly Deferred (Do Not Build in These Phases) +**Work packages** -Anything listed in `PRD.md §6` as out of scope — rate limiting, caching logic, failover, streaming, JWT/OAuth/RBAC, multi-tenancy, audit logs, active health monitoring, cost tracking, analytics dashboards — is **not** a phase here. These become Phase 6+ only after the MVP phases above are complete and the docs are explicitly updated to bring them into scope. +- Integrate Redis atomic request/token admission and concurrency leases across applicable tenant/application/key/model scopes. +- Add normalized usage categories, estimate/source markers and immutable price versions; account for cached/reasoning categories without double counting. +- Implement PostgreSQL hard-budget reservations and durable attempt intents; settle actual usage and append corrections. Reserve again before each retry/fallback. +- Add outbox delivery, idempotent workers, alerts/webhook configuration, usage query/export APIs and reconciliation jobs. Test webhook signatures and egress restrictions. +- Keep uncertain crashed/cancelled attempts reserved until audited reconciliation; distinguish cost reporting from external invoice billing. +- Define UTC budget windows, reset boundaries, missing price/usage behavior and Redis/PostgreSQL failure modes. ---- +**Exit evidence** -## Related Docs +- At least two replicas enforce the same request limits; token estimation drift is quantified. +- Concurrent reservations cannot oversubscribe any applicable hard budget; failover and process kill do not create free capacity from uncertain spend. +- Duplicate settlement/outbox delivery is idempotent; failed charged attempts appear in spend. +- Unknown price or an unbounded usage model cannot use hard-budget mode silently. +- Export retry/backpressure and dependency-outage tests pass; sample provider usage reconciliation records differences. -- `PRD.md` — product scope, features, target users -- `Architecture.md` — tech stack, folder structure, system flow -- `Rules.md` — boundaries for AI-assisted development \ No newline at end of file +**Migration/rollout:** Begin with metering/shadow limits; compare outcomes, then enable enforced policies per tenant. This closes enterprise beta only with all earlier gates satisfied. + +## Phase 11 Observability, privacy, guardrails and exact caching + +**Goal:** Make the governed gateway diagnosable and safe for sensitive workloads. **Requirements:** F07, F08, F11. **Dependencies:** Phases 8–10; baseline telemetry starts in Phase 6. + +**Work packages** + +- Complete Micrometer metrics, OpenTelemetry spans, request/attempt search, routing decision views, dashboards, alerts and cardinality limits. +- Implement policy-controlled retention, redacted sampling, optional encrypted content storage and export destinations. Default every sink to content off. +- Add bounded regex/secret detectors, pre/post/stream hooks and adapters for optional external checks. Validate policy capability and fail-open/fail-closed choices. +- Implement opt-in exact caching for eligible non-streaming text/embedding requests with tenant/access partition, versioned keys, TTL/size caps and purge. +- Separate request events, usage events and administrative audit; test access/deletion policies for each. +- Add privacy regression fixtures containing secrets, tool arguments, prompt injection strings and sensitive synthetic data. + +**Exit evidence** + +- A failed request can be traced from admission through attempts without recording content. +- Synthetic sensitive values do not appear in default DB/log/trace/export artifacts. +- Whole-output policies buffer within caps or reject streaming; chunked checks document limitations and detect their tested cross-chunk patterns. +- Cache entries cannot cross tenant/access partitions; revoked access and changed policies invalidate eligibility. +- Export outages and full queues do not grow memory without bounds; required metering/audit state remains durable. + +**Migration/rollout:** Content capture and caching remain disabled until tenant policy opts in. Retention changes include a deletion rehearsal and backups/export treatment. + +## Phase 12 Versioned control plane, simulation and console + +**Goal:** Let operators administer and understand the product without direct database edits. **Requirements:** F09, F10. **Dependencies:** Phases 8–11; API/schema work can start earlier. + +**Work packages** + +- Complete admin CRUD for providers/deployments/aliases, policies, budgets, retention and guardrails, using role-scoped services. +- Add schema-versioned declarative import/export, optimistic locking, idempotent mutations and validation. +- Implement draft/validate/stage/activate/rollback, atomic snapshot swap, outbox notifications, polling reconciliation and per-replica acknowledgement. +- Add policy simulation using the live evaluator, synthetic time/health inputs, deterministic seeds and golden fixtures. Mark content-dependent replay checks unevaluated when content is unavailable. +- Build an API-backed console: identity onboarding, provider test connection, model aliases, keys (one-time secret display), budgets/usage, request decisions and configuration diff/rollback. +- Add Java/Python/Node onboarding examples and a Java extension example. Select console framework by team fit and dependency support, not an inherited MVP restriction. +- Complete user-facing errors, accessible navigation, loading/empty/error states, authorization tests and console security. + +**Exit evidence** + +- Simulate → stage → activate → inspect decision → rollback works without direct DB edits. +- Invalid revisions never reach workers; missed notifications recover via polling; stale workers become unready. +- Concurrent edits return a conflict instead of silently overwriting. +- Console actions match API permissions and audit records; full keys/secrets cannot be recovered after one-time display. +- Rollback retains current revocations and spending. In-flight requests keep their routing revision; emergency disable is independently effective. + +**Migration/rollout:** Keep old snapshots until requests drain. Configuration schema changes include compatibility tests; binary rollback is never equated with config rollback. + +## Phase 13 Production packaging and release qualification + +**Goal:** Demonstrate first GA on a declared operating environment. **Requirements:** F12 and all F01–F13 gates. **Dependencies:** Phases 6–12. Packaging/CI work starts in Phase 6. + +**Work packages** + +- Produce non-root, pinned/scanned/signed OCI images, SBOMs, checksums and reproducible versioned artifacts. +- Deliver development Compose, production Helm, Linux systemd and Podman Quadlet examples using one documented configuration contract. Package differences and supported OS/runtime versions are explicit. +- Exercise single-region HA, management isolation, TLS/secrets, readiness/liveness, startup migrations, rolling drains, resource requests/limits and capacity planning. +- Build offline installation/upgrade bundles with images/JAR, schemas, migrations, catalogs, signatures and local-model prerequisites. +- Complete on-call runbooks, backup/PITR restore, reservation reconciliation, incident response, key compromise, upgrade and rollback drills. +- Run controlled load/soak, chaos, tenant security, compatibility, config convergence and recovery qualification. Close release findings and publish known limitations. +- Name release, platform, security and operational owners. Set supported version/upgrade policy and distribution/license terms before public artifacts. + +**Exit evidence** + +- Every item in [Operations.md](Operations.md)'s release evidence checklist has an artifact, tested revision and owner. +- PRD targets are met on the declared reference environment or explicitly revised with evidence before release. +- MVP-to-GA and previous-supported-version upgrade paths pass; backup restore meets the declared RPO/RTO and reconciles uncertain spend. +- A disconnected install with a local provider succeeds without vendor callbacks. Kubernetes and non-Kubernetes packages demonstrate equivalent auth/routing/privacy behavior. +- Staged canary and rollback rehearsals pass; no unresolved critical/high exploitable finding. +- Customer/operator acceptance covers a complete onboarding-to-incident-recovery journey. + +**Rollout:** Internal staging → limited pilot/canary → monitored expansion → GA. Promotion is a release decision with concrete evidence; passing unit tests alone is insufficient. + +## Phase 14 Enterprise breadth + +**Goal:** Expand supported workloads and organization integration without weakening GA semantics. **Requirements:** F14. **Dependencies:** GA contracts; individual spikes may start earlier. + +Work includes Bedrock with signed/workload identity access, Azure deployment/auth semantics, native Responses and Anthropic Messages APIs, vision/audio/images/rerank/batch operations by capability, SAML federation through the identity provider, SCIM lifecycle, Vault/cloud secret resolvers, richer audit export and dedicated-tenant deployment options. + +Each capability ships independently with wire contracts, privacy/retention implications, charging bounds, adapter conformance, lifecycle/cancellation tests and upgrade documentation. Stateful Responses, files and batch jobs need explicit storage, tenant ownership, idempotency and cleanup design; they are not automatic extensions of synchronous chat. Regional resilience and formal compliance work require separate customer-backed scope. + +## Phase 15 Agent governance and inference-aware integrations + +**Goal:** Extend a reliable model gateway into governed tool and serving traffic. **Requirements:** F15. **Dependencies:** GA identity, policy, audit and operations. + +Work includes an MCP server registry/proxy, tool discovery filtering, principal/tool/resource authorization, credential isolation and approval integration points. Tool invocation is a separate operation family with side effects, not a model retry. Add per-call audit and explicit no-blind-retry defaults. Publish protocol-version conformance and test cancellation/session ownership. + +Explore Kubernetes InferencePool/endpoint-picker interoperability through an integration boundary; do not embed a Kubernetes requirement in core inference. Optional prompt registry and evaluation modules use explicit version references and customer-controlled content storage. Semantic caching or quality-aware routing proceeds only after approved privacy rules and a representative evaluation dataset. + +Exit evidence includes attempts to bypass tool authorization, SSRF, credential forwarding, cross-tenant sessions and approval/replay controls; serving integration proves policy remains enforced. Optional modules must be disableable without affecting the GA core. + +## First implementation backlog + +G001–G006 are implemented. G002–G006 have passing local tests; G001's full build/security/remote-CI acceptance follows the open gates in Phase6. G007–G012 are not started. These are issue-sized starting points, not an additional rigid sequence. + +| ID | Deliverable | Dependency | Acceptance | +|---|---|---|---| +| G001 | Assess MVP design, reproduce baseline and add build/CI wrapper | MVP | Retain/refactor/replace decisions with rationale and consumer/data impact; clean checkout test/build report; observed failures tracked | +| G002 | PostgreSQL fresh/upgrade migration harness | G001 | V1–V5 plus additive test migration pass on real PostgreSQL | +| G003 | Production bootstrap and seed safeguards | G002 | Demo key rejected before production readiness; local setup documented | +| G004 | Error/validation and lifecycle repair | G001 | Routing, transport, malformed input and log-failure regression fixtures pass | +| G005 | Explicit timeout/concurrency profile | G004 | Slow upstream bounded; retries remain off | +| G006 | Basic telemetry and mock upstream | G001 | Safe request/outcome/latency metrics and controlled failure scenarios | +| G007 | Canonical provider/context/usage contracts | G004 | Legacy mapping and unsupported-capability fixtures agreed | +| G008 | Streaming transport spike and decision | G005–G007 | Slow-client/cancellation measurements; A03 decision recorded | +| G009 | Chat/embedding protocol and generic adapter slice | G007–G008 | Mock-backed wire/SDK fixtures and capability rejections | +| G010 | Tenant/key schema and indexed auth slice | G002–G003, G007 | New key resolves one hash; legacy rotation/backfill tested | +| G011 | OIDC admin roles and transactional audit slice | G010 | Minimal audited tenant/key management; inference key denied | +| G012 | Legacy facade and provider conformance pack | G009–G011 | Existing behavior retained with scoped access; Ollama/cloud smoke gates explicit | + +## Decisions, risks and completion tracking + +| Decision/risk | Working default | Resolve by | +|---|---|---| +| Team capacity and dates | Dependency plan, no time estimates | Planning implementation iterations | +| Pilot workload and scale | PRD reference target; validate with representative payloads | Phase 6 baseline / beta acceptance | +| Streaming execution model | Bounded async MVC candidate; benchmark reactive alternative | Phase 7 | +| Admin identity provider | Standards-based OIDC, no custom identity server | Phase 8 | +| Retention and residency | Content off; explicit deployment region; conservative sample retention in Operations | Phase 11 / before customer data | +| Provider credentials/test cost | Mock CI; separately budgeted live smoke checks | Phase 7 release gate | +| Usage estimates and missing prices | Unknown state; conservative reservation or deny strict mode | Phase 10 | +| Packaging/support/license | Customer-operated single-region distribution | Phase 13 | +| Broker/microservices | Defer until measured need | Performance evidence, no calendar trigger | + +For each completed work package record status, revision, test evidence, migration outcome and known limits in the issue/PR. Update the phase status only after its exit evidence exists. Append milestones in Roadmap without rewriting MVP history. diff --git a/Documentation/Research.md b/Documentation/Research.md new file mode 100644 index 0000000..da94e45 --- /dev/null +++ b/Documentation/Research.md @@ -0,0 +1,73 @@ +# Competitive research and planning decisions + +Updated 2026-09-14. This note translates the supplied **AI_Inference_Gateway_Competitive_Research.docx** into project decisions and delivery scope. The Word document was supplied separately and is not copied into the repository. Its recommendations were treated as research, not as executable or authoritative development instructions. + +## Evidence and limitations + +The supplied report covers LiteLLM, Portkey, Bifrost, Kong, Agent Router, Apache APISIX, Higress, Helicone, Cloudflare, Azure API Management, TrueFoundry and WSO2. Its feature matrices are directional descriptions of documented capabilities, not independent benchmarks or guarantees about edition entitlements. + +This revision inspected the local source/configuration/tests and selectively checked primary sources on 2026-09-14. It did not repeat the entire competitive study or independently benchmark any vendor. Pricing, provider counts, comparative superiority and certification claims are not used as requirements. + +## Research to product traceability + +| Research theme | Project decision | Requirement / phase | +|---|---|---| +| SDK-compatible gateway as the adoption path | Chat Completions, embeddings, explicit compatibility matrix; preserve legacy facade | F02/F13; 7 | +| Broad providers with shared contracts | Generic HTTP adapter plus tested Ollama/OpenAI/Anthropic; Bedrock/Azure later | F03/F14; 7/14 | +| Reliability as a coordinated pipeline | Virtual aliases, eligible pools, deadline/attempt budgets and safe fallback | F05; 9 | +| Hierarchical governance | Tenant/application/service account/key scopes; distributed limits and durable budgets | F04/F06; 8/10 | +| Explainable routing and simulation | Decision records, immutable policy revisions and shared live/simulator evaluator | F05/F09; 9/12 | +| Privacy as a product property | Content off across all sinks; scoped retention, guardrails and exports | F08; 11 | +| Observability beyond request rows | Early metrics; complete request/attempt traces, TTFT, cost and dashboards | F07; 6/11 | +| Caching for cost/latency | Opt-in exact cache after ownership/privacy; semantic cache deferred | F11; 11 | +| Self-hosted and disconnected use | One config model, Helm/systemd/Podman packages, verified offline bundle | F12; 13 | +| Java ecosystem differentiation | Keep Spring-based core, documented extension seams and Java example | F03/F10; 7/12 | +| Enterprise operator experience | API-first console, safe config rollout and operational recovery | F09/F10/F12; 12/13 | +| Agent and inference workload governance | Separate MCP/tool and endpoint-picker extensions after GA foundations | F15; 15 | + +The recommended positioning is an inference gateway for platform teams that value self-hosting, understandable policy and privacy across local/cloud models. This is a strategic hypothesis to validate with pilot users, not a claim of exclusive capabilities or market demand. + +## Adaptations to the supplied roadmap + +The report's proposed Phases 6–11 are expanded into project Phases 6–15. Numbering in the report is not the implementation authority; Phases.md is. + +- Added a foundation phase before provider expansion because source inspection exposed timeout, validation, logging, seed and migration-test gaps. +- Moved identity, minimal admin authorization and audit before shared governance. Observability and operational design start early rather than waiting for a late hardening phase. +- Separated full control-plane/console delivery from production qualification, with measurable deployment and recovery gates. +- Preserved `/v1/inference` as a facade; the report's “compatibility alias” wording does not mean identical wire formats or an HTTP redirect. +- Kept PostgreSQL authoritative for hard-budget reservations; Redis atomic limits do not imply durable financial accounting or globally consistent circuits. +- Added explicit uncertain usage, crash recovery and no-post-output-retry behavior. “No duplicate accounting” does not mean exactly-once provider execution. +- Limited exact caching to explicitly eligible workloads and recognized the limits of streaming redaction and metadata-only replay. +- Retained MVC as a measured candidate; streaming is not by itself a reason to rewrite all persistence/control-plane code. +- Deferred native Responses/Messages APIs and broader modalities to explicit operation designs; upstream Anthropic support does not imply inbound protocol parity. + +## Primary sources selectively checked + +| Source | Observation used in planning | +|---|---| +| [LiteLLM reliability](https://docs.litellm.ai/docs/proxy/reliability) | Documents ordered model-group fallbacks and their interaction with retries; used as a reliability design reference, not copied failure policy | +| [Cloudflare dynamic routing](https://developers.cloudflare.com/ai-gateway/features/dynamic-routing/) | Documents named routes, conditional/percentage decisions, versions and rollback; supports treating routing changes as versioned configuration | +| [Bifrost overview](https://docs.getbifrost.ai/overview) | Reference for a gateway combining provider access and operational/governance features; no performance claim adopted | +| [Ollama OpenAI compatibility](https://docs.ollama.com/api/openai-compatibility) | Documents compatibility with parts of the OpenAI API; supports testing a generic adapter rather than assuming full equivalence | +| [Spring asynchronous requests](https://docs.spring.io/spring-framework/reference/web/webmvc/mvc-ann-async.html) | MVC offers SSE/async integration; bounded execution and slow-client measurement remain design requirements | +| [OpenTelemetry GenAI notice](https://opentelemetry.io/docs/specs/semconv/gen-ai/) | Current page directs readers to a dedicated conventions repository; pin a reviewed schema at implementation time | +| [OWASP GenAI risks](https://genai.owasp.org/llm-top-10/) | Input to threat coverage for injection, sensitive content and consumption risks; not a certification standard | +| [Kubernetes Inference Extension](https://gateway-api-inference-extension.sigs.k8s.io/) | Defines inference routing and endpoint-picker concepts; suitable later integration boundary for self-hosted serving | + +## Additional sources in the supplied report + +These links preserve the report's wider reference trail; their full capability matrices were not independently revalidated in this revision. + +- [Portkey AI Gateway](https://portkey.ai/docs/product/ai-gateway) +- [Kong AI Gateway](https://developer.konghq.com/ai-gateway/get-started/) +- [Agent Router](https://theagentrouter.ai/) +- [Apache APISIX AI Gateway](https://apisix.apache.org/ai-gateway/) +- [Higress](https://higress.ai/) +- [Helicone repository](https://github.com/Helicone/helicone) +- [Azure API Management AI gateway capabilities](https://learn.microsoft.com/en-us/azure/api-management/genai-gateway-capabilities) +- [TrueFoundry AI Gateway](https://www.truefoundry.com/docs/ai-gateway/intro-to-llm-gateway) +- [WSO2 AI Gateway](https://apim.docs.wso2.com/en/latest/ai-gateway/ai-gateway-overview/) +- [LiteLLM access control](https://docs.litellm.ai/docs/proxy/access_control) +- [Cloudflare AI Gateway](https://developers.cloudflare.com/ai-gateway/) + +The report also cited this project's README, PRD and Roadmap. Local source inspection takes precedence over old documentation where implementation differs. Before implementing a protocol, provider or deployment integration, verify its current primary specification and record supported versions in the conformance matrix. diff --git a/Documentation/Roadmap.md b/Documentation/Roadmap.md index c4d325f..c5eb65f 100644 --- a/Documentation/Roadmap.md +++ b/Documentation/Roadmap.md @@ -2,20 +2,25 @@ ## Architecture & System Flow -AI Inference Gateway is a Spring Boot (Java) backend that sits between client applications and AI model providers, exposing a single unified endpoint (`POST /v1/inference`) for all AI requests. A client sends a request with an `X-API-Key` header and a JSON body; the gateway's auth filter validates the key against hashed keys in PostgreSQL, a routing engine resolves the target provider (explicit `provider` field → else active default from DB), and the resolved provider's `AIProvider` adapter calls the actual backend (Ollama for MVP). The response is normalized into a standard schema tagged with a unique `requestId`, a metadata row is written to `request_logs`, and the response is returned to the client. All errors follow a single standardized `ApiError` shape. Two additional visibility endpoints (`GET /v1/providers`, `GET /v1/logs`) expose configuration and request history. +The current implementation includes the Phase 6 foundation: `POST /v1/inference`, `GET /v1/providers`, `GET /v1/logs`, and private health/metrics. `RequestIdFilter` assigns correlation before bounded `AdmissionFilter` and API-key authentication. `RoutingEngine` selects an explicit/default provider and validates the model. `OllamaProvider` encodes synchronous generation through the pooled, deadline-cancellable `ProviderTransport`. `InferenceService` records one best-effort terminal diagnostic outcome, including routing/unexpected failures; diagnostic persistence cannot replace the provider result. + +Keys still resolve to a generic principal; providers/logs are not tenant-scoped. Registry adapters load at startup while routing also queries the database. Provider/log success bodies do not all embed a request ID, though the response header does. Redis is provisioned only. Fast tests use H2; the separate required CI profile runs PostgreSQL/Flyway fresh-install, upgrade and production-startup tests. V6 disables the public demo key, local bootstrap is explicit, and production startup rejects active demo credentials. Streaming, deployment bulkheads, durable accounting and enterprise authorization remain unimplemented. + +The production target is documented separately in [Architecture.md](Architecture.md) and [Phases.md](Phases.md). A documentation update does not mean the target modules or release controls exist. ## Tech Stack -- **Backend / Gateway:** Spring Boot (Java), Spring MVC (synchronous) +- **Backend / Gateway:** Spring Boot 4.1.1, Java 21, Spring MVC (synchronous), Jackson 3; Tomcat 11.0.25 security override - **Local AI Provider:** Ollama (no paid API dependency for dev/test) - **Database:** PostgreSQL (providers, models, API keys, request logs) - **Cache / Future Infra:** Redis (provisioned, not used in MVP logic) - **Containerization:** Docker / Docker Compose (gateway + Postgres + Redis + Ollama) - **Migrations:** Flyway - **Security:** Spring Security — API key filter only -- **HTTP Client:** Spring `RestClient` (provider adapters only) +- **HTTP Client:** Apache HttpClient 5 in shared bounded ProviderTransport; no retries/redirects - **Validation:** `jakarta.validation` -- **Testing:** JUnit 5 + Mockito +- **Testing:** JUnit/Jupiter + Mockito, real PostgreSQL integration profile, controllable loopback HTTP upstream +- **Build / Operations:** Maven wrapper 3.9.11 with SHA-256 verification, pinned non-root runtime image, Micrometer/Prometheus, GitHub Actions build/test/Trivy scans and artifact manifest --- @@ -316,17 +321,83 @@ Each entry records what was built, why certain decisions were made, and what was - `GET /v1/logs?status=SUCCESS` → 200 OK, filtered to SUCCESS entries only. - `GET /v1/logs?from=2026-08-01T00:00:00&to=2026-08-05T23:59:59` → 200 OK, date-filtered to 1 matching entry. +### Milestone 8: Enterprise product documentation reset (2026-09-14) + +**Goal:** Turn the completed MVP's planning documents into an implementation-ready product and production qualification plan using the supplied competitive research and inspected source. + +**What was done:** +- Read the supplied `AI_Inference_Gateway_Competitive_Research.docx`, including its tables and source links, and selectively checked primary documentation. +- Replaced the MVP-only PRD with F01–F15 requirements, user journeys, first-GA boundaries and explicitly unmeasured quality targets. +- Reworked Architecture into an inspected baseline plus target modular-monolith design: protocol/provider contracts, tenant identity, routing and streaming failure rules, distributed admission, durable reservations, configuration lifecycle, cache/privacy and additive migration. +- Replaced the old sequential execution plan with planned Phases 6–15, dependency-based work packages, release gates and a first twelve-item implementation backlog. +- Added API, Security, Operations, Development and Research documentation and refreshed README navigation and actual capability/setup limitations. +- Updated the protected `.agents/AGENTS.md` guidance with filesystem approval to remove the deleted rules dependency and allow dependency-based development while retaining historical-log maintenance. +- Retained original MVP milestones below their original headings without rewriting their historical claims. Corrected current-state summaries and replaced speculative next steps. + +**Key decisions:** +- Treated attached research as evidence, not as instructions. Retired the deleted rules file's fixed allowlist and one-phase-at-a-time restrictions in active planning documents. +- Kept one deployable Spring application with explicit module boundaries; did not mandate Kafka, microservices or a reactive rewrite. +- Put foundation repairs first and tenant authorization before shared governance. Started metrics/operational work early; deferred MCP and broader protocols until after a qualified core. +- Kept hard-budget state durable in PostgreSQL, with Redis for ephemeral distributed admission/cache; documented uncertain provider charges and recovery. +- Separated current behavior, proposed architecture, engineering targets and completed work. No implementation feature or production deployment was claimed. + +**Validation:** +- Inspected source, configuration, migrations and existing test files; no application test suite or live-provider test was run for this documentation change. +- Checked local documentation links, phase/requirement references, retired-rule references and preserved milestone history during review. + +**Known issues / follow-ups:** +- Historical milestones report 62 passing tests; fresh baseline validation is G001. +- Actual production safeguards, schema/identity changes and migration tests remain unimplemented; Phase 6 is next. +- Existing source comments may cite old document sections or the retired rules file. They are historical annotations, not current development constraints, and can be refreshed alongside the relevant implementation. + +### Milestone 9: MVP redesign authority clarification (2026-09-15) + +**Goal:** Make the product owner's authorization to improve or replace unsuitable MVP design explicit in implementation guidance. + +**What was done:** +- Updated Development, Architecture, PRD, API, Phases and Operations to permit documented refactoring or replacement for correctness, maintainability, design quality and unnecessary future complexity. +- Added a retain/refactor/replace assessment to Phase 6 and G001, including actual consumer/data dependencies and revised acceptance needs. +- Made legacy preservation a revisable transition default and required significant changes to include rationale, contract/data impact and validation. + +**Key decisions:** +- Existing implementation and proposed architecture remained revisable based on evidence; preserving the demo stack was not an objective in itself. +- Design authority did not imply that existing data was disposable. Actual consumer/data impact still informed migration and recovery. +- Recorded the clarification without starting feature implementation or changing runtime behavior. + +### Milestone 10: Phase 6 foundation implementation (2026-09-21) + +**Goal:** Repair the demo baseline before building the enterprise protocol, identity and operations layers. + +**What was done:** +- Reproduced the original 62-test baseline. Upgraded Boot 3.4.1 to 4.1.1, adopted Jackson 3 and the split MVC/Flyway test/runtime modules, and retained Java 21, MVC, JPA and the modular monolith. +- Added a checksum-pinned Maven wrapper, deterministic JAR timestamps, artifact manifest generation, pinned CI actions, PostgreSQL CI service, dependency/secret/configuration/image scans and SBOM output. Added a non-root runtime and an image target that packages the exact tested artifact; retained a source-build target for local Compose. +- Introduced `GatewayProperties`, `AdmissionFilter`, `RequestDeadline` and `ProviderTransport` for body/response bounds, fail-fast concurrency, strict HTTP pool limits, typed timeouts and active deadline cancellation. Disabled client retries and redirects. +- Repaired parser/query/media status mapping, sanitized public/provider error paths, and extended inference terminal logging to routing and unexpected failures. Diagnostic persistence became explicitly best effort and could no longer replace either a successful response or the original provider failure. +- Added Micrometer request/outcome/latency/admission/drop metrics, a controllable HTTP mock, deadline/pool/saturation/privacy regressions and an opt-in full-HTTP mock benchmark. +- Preserved V1–V5 and added V6 to deactivate the known demo key. Added pre-readiness credential validation and explicit local key/provider bootstrap. Made production the default profile, management private, health minimal, and all local Compose port publications loopback-only. +- Added real PostgreSQL tests for fresh migration, populated V5 upgrade, production readiness/management isolation, and rejection of a rehashed demo key. +- Updated README, API, Architecture, PRD, Phases, Security, Operations and HTTP examples; recorded the Phase 7 streaming experiment and evidence in Phase6. + +**Key decisions:** +- Kept the existing public inference shape and durable data, while deliberately correcting provider-unavailable to HTTP 503 and replacing raw error text with stable safe messages. No repository evidence established a deployed external consumer; no external environment or customer database was modified. +- Replaced implicit RestClient transport with explicit pooled cancellation rather than rewriting the application reactively before the streaming experiment. Shared pool isolation and disconnect propagation remained later work. +- Retained fast H2 tests but added actual Flyway/PostgreSQL gates. Diagnostics remained separate from future durable financial accounting. +- Applied Tomcat 11.0.25 after Trivy found three critical advisories in Boot's managed 11.0.24. Used the vendor's fixed patch instead of suppressing findings; remove the override once Boot manages an equivalent or newer patch. + +**Tests and environment:** +- Expanded suite passed 88 tests with one optional benchmark skipped; all four PostgreSQL tests passed. The benchmark ran separately and passed. Full commands, artifact identity, measurements and final build/scan status are in [Phase6.md](Phase6.md). +- Local mock measurements included real BCrypt and H2 diagnostic writes; they were not production or real-provider capacity evidence. +- Maven initially needed an ignored project cache because its sandbox default repository was unwritable. PowerShell Maven properties containing dotted versions required quoted arguments. Docker Desktop was available outside PATH; container DNS and advisory downloads required retries. Remote CI execution was not claimed. + +**Known issues / follow-ups:** +- Phase 6 implementation did not establish production readiness. Remaining acceptance gates are explicitly tracked in Phase6; production qualification remains Phase 13. +- Generic identity, globally visible metadata, all-key BCrypt scans, startup registry refresh, synchronous diagnostics and absent streaming/tenant/budget controls remain visible limitations for Phases 7–12. + ## Next Steps / Future Enhancements -### MVP Status -All 5 MVP phases are complete. All PRD §7 success criteria are met. The gateway is demo-ready. - -### Post-MVP Enhancements -- **Additional providers:** OpenAI, Anthropic, Bedrock adapters (each is a new `AIProvider` implementation). -- **Rate limiting & caching:** Wire Redis into request handling for quota enforcement and response caching. -- **Streaming responses:** SSE/token-by-token streaming via reactive adapter. -- **Observability:** Prometheus metrics, Grafana dashboards, structured logging. -- **Event-driven processing:** Kafka for async request log ingestion and analytics pipelines. -- **Advanced auth:** JWT/OAuth2, RBAC, multi-tenant governance. -- **Deployment:** Kubernetes manifests, multi-region support, CI/CD pipelines. +The original six MVP phases (0–5) are recorded complete. Phase 6 implementation and local verification are recorded in Milestone 10 and [Phase6.md](Phase6.md); acceptance remains open for the unresolved build/security/clean-checkout CI gates listed there. Phases 7–15 have not started. + +Close Phase 6's remaining external verification gates, then start G007 provider/context contracts and G008 streaming transport experiment in [Phases.md](Phases.md). Protocol/provider and identity work can progress concurrently once contracts are agreed. First production GA requires F01–F13 and the evidence checklist in [Operations.md](Operations.md). Broader enterprise protocols and agent/serving extensions are later phases. + +[PRD.md](PRD.md) owns scope, [Architecture.md](Architecture.md) owns design, and [Research.md](Research.md) explains the evidence and adaptations. Record completed work here as it occurs. diff --git a/Documentation/Rules.md b/Documentation/Rules.md deleted file mode 100644 index ec46b03..0000000 --- a/Documentation/Rules.md +++ /dev/null @@ -1,87 +0,0 @@ -# Rules.md — AI Inference Gateway - -> This file sets boundaries for any AI model (or human) contributing code to this project. Read `PRD.md` and `Architecture.md` first. These rules override general habits/defaults — if a suggestion conflicts with this file, this file wins. - ---- - -## 1. Libraries & Tools — Use - -| Concern | Use | Notes | -|---------|-----|-------| -| Web framework | Spring Boot (Spring Web/MVC) | Already the chosen stack — do not propose alternatives | -| Persistence | Spring Data JPA + PostgreSQL | | -| Migrations | Flyway (preferred) or Liquibase | Every schema change is a migration file, never a manual DB edit | -| Security | Spring Security, API-key filter only | No custom crypto — use Spring's built-in password/hash utilities for key hashing | -| HTTP client (to Ollama) | Spring `RestClient` / `WebClient` | Keep it inside the provider adapter only | -| Validation | `jakarta.validation` (`@Valid`, `@NotNull`, etc.) on request DTOs | | -| Testing | JUnit 5 + Mockito | Every service class gets a unit test; every controller gets at least a happy-path + one error-path test | -| Containerization | Docker / Docker Compose | No Kubernetes manifests yet — that's a future phase | - -## 2. Libraries & Tools — Avoid (for MVP) - -- **No JWT / OAuth2 / Spring Security full auth stack** — API key only (see `Architecture.md §6`). -- **No message queues** (Kafka, RabbitMQ) — deferred to a future phase. -- **No caching libraries wired into logic** — Redis is provisioned in `docker-compose.yml` but must not be used for actual caching/rate-limiting logic yet. -- **No ORMs other than JPA/Hibernate** — don't introduce MyBatis, jOOQ, etc. -- **No reactive stack (WebFlux) for the core gateway** — keep it synchronous/blocking Spring MVC. (Streaming/reactive is a future-phase concern.) -- **No lombok-heavy magic that hides business logic** — Lombok for boilerplate (getters/setters/builders) is fine; don't use it to hide validation or business rules. -- **No third-party API gateway frameworks** (e.g., Spring Cloud Gateway, Kong config) — this project *is* the gateway; don't wrap it in another gateway layer. -- **No new AI provider SDKs** unless explicitly requested — MVP only implements the Ollama adapter. - ---- - -## 3. Architectural Boundaries - -- **The gateway core must never import or reference a provider-specific SDK/class directly.** All provider interaction goes through the `AIProvider` interface (`infer()`, `info()`, `health()`). If you're tempted to `import` an OpenAI/Anthropic SDK class into `routing/` or `inference/`, stop — that logic belongs in a new adapter under `provider/`. -- **Controllers do not contain business logic.** They validate input, delegate to a service, and return a response. Routing decisions, provider selection, and normalization belong in service/engine classes. -- **No new REST endpoints outside the three defined in `Architecture.md §7`** (`/v1/inference`, `/v1/providers`, `/v1/logs`) without updating `PRD.md` and `Architecture.md` first. Documentation changes precede code changes for scope additions. -- **No hardcoded provider names or model names in Java code.** Providers and models are config data in PostgreSQL (`providers`, `models` tables). If a model name is being typed into a `.java` file, that's a rule violation. -- **Every request gets a `requestId`.** Do not add an endpoint or log entry that lacks one. - ---- - -## 4. Error Handling Rules - -- **All errors use the single standard schema** (defined in `PRD.md §6` / `Architecture.md §7`): - ```json - { "timestamp": "...", "status": 404, "error": "PROVIDER_NOT_FOUND", "message": "...", "path": "...", "requestId": "..." } - ``` -- **Never leak stack traces, internal exception messages, or raw provider error payloads to the client.** Catch, translate to a known `ErrorCode`, log the internal detail server-side. -- **Use the existing `ErrorCode` enum** (`INVALID_REQUEST`, `PROVIDER_NOT_FOUND`, `PROVIDER_UNAVAILABLE`, `MODEL_NOT_FOUND`, `PROVIDER_TIMEOUT`, `INTERNAL_ERROR`, `BAD_CONFIGURATION`). Adding a new error code requires updating `Architecture.md` too. -- **All exceptions are handled centrally** in `GlobalExceptionHandler` — do not add ad-hoc `try/catch` blocks in controllers that return custom error shapes. -- **Provider failures fail the individual request only.** Do not implement retries, circuit breakers, or background health polling — that's explicitly deferred (see `PRD.md §6`). - ---- - -## 5. Data & Security Rules - -- **API keys are never stored in plaintext.** Hash before persisting; compare hashes on auth. -- **No secrets in code or committed config.** DB credentials, etc. go through environment variables / `.env` (git-ignored), referenced in `docker-compose.yml` and `application.yml`. -- **Every schema change is a migration file**, named and ordered per the convention in `Architecture.md §5` (`V1__...sql`, `V2__...sql`, etc.). Never modify an already-applied migration — add a new one. -- **Logs must not contain full prompts/response bodies** by default (privacy/size) — log metadata only (provider, model, status, latency, error code), per the `request_logs` schema. - ---- - -## 6. What the AI Should Do - -- Always check `PRD.md` and `Architecture.md` before implementing a new feature — confirm it's in scope. -- Follow the existing folder structure (`Architecture.md §5`) — new code goes in the matching package, not a new top-level folder invented on the spot. -- Write a unit test alongside any new service/engine class. -- When a requirement is ambiguous, prefer the simplest MVP-consistent interpretation over a more "complete" enterprise-grade solution. -- Flag (in comments or PR description) any deviation from these rules, with a reason. - -## 7. What the AI Should NOT Do - -- Do not implement anything listed as "Out of Scope" in `PRD.md §6` (rate limiting, caching logic, failover, streaming, JWT/OAuth/RBAC, multi-tenancy, audit logs, analytics dashboards) unless the user explicitly asks and updates the docs first. -- Do not introduce a new provider, library, or framework not listed in `§1` without asking. -- Do not change the API error schema or success schema shape without updating `Architecture.md`. -- Do not bypass the `AIProvider` interface to call a provider directly "just this once." -- Do not silently expand scope while implementing a phase (see `Phases.md`) — if something feels like it belongs in a later phase, leave it for that phase. - ---- - -## 8. Related Docs - -- `PRD.md` — product scope, features, target users -- `Architecture.md` — tech stack, folder structure, system flow -- `Phases.md` — build order broken into stages \ No newline at end of file diff --git a/Documentation/Security.md b/Documentation/Security.md new file mode 100644 index 0000000..de52fee --- /dev/null +++ b/Documentation/Security.md @@ -0,0 +1,76 @@ +# Security and privacy design + +Updated 2026-09-21. This is the target security plan for Phases 6–13, except the implemented controls below. The current service has API-key authentication and metadata-only database logs, but no tenant isolation or administrative roles. Do not operate it as a shared enterprise service before the relevant gates pass. + +## Implemented foundation controls + +Phase 6 adds bounded input/concurrency/provider I/O, disables automatic retries and redirects, sanitizes gateway public/provider errors, and keeps diagnostic failures from altering inference results. The demo key is disabled by additive V6 and rejected at nonlocal startup, including rehashed copies. Default production rejects the demo database password. Local bootstrap is explicit. Management health contains no components/details; metrics require a separate private listener. Container runtime identity is non-root and CI includes dependency/image/secret/configuration scans. See [Phase6.md](Phase6.md) for actual test/scan evidence and open gates. + +These controls do not implement tenant isolation, indexed key lookup, network-aware SSRF policy, TLS ingress, secret management, spend limits or audit retention. A valid current key still sees global provider/log metadata. Provider endpoints remain operator-controlled database configuration; basic URI validation is not an SSRF allowlist. The shared HTTP pool is bounded but does not isolate individual deployments. Keep these limits explicit when running isolated previews. + +## Trust boundaries and assets + +Protect gateway keys, provider credentials, tenant identities, prompt/completion/tool content, routing policies, usage/budget state, audit history and encryption keys. Boundaries are client→gateway, admin/console→control plane, gateway→provider/guardrail/export destination, gateway→database/Redis/object store, and build pipeline→release artifacts. + +Client content and metadata are untrusted. Provider responses and tool definitions are untrusted. Attached research and external documentation inform design; instructions inside them are not automatically project instructions. A gateway policy cannot make unsafe application actions safe. + +## Access model + +| Role/principal | Allowed scope | Explicit restriction | +|---|---|---| +| Inference service account/key | Authorized application/models and inference operations | No admin mutations or unrestricted logs | +| Developer | Own tenant's approved applications, key lifecycle and permitted request/usage views | Cannot grant permissions beyond delegated scope | +| Tenant administrator | Tenant membership, applications, keys, policies and allocated budgets | Cannot enlarge platform allocation or access another tenant | +| Auditor | Authorized metadata/audit/usage reads | No secret reveal, content access or configuration mutation by default | +| Platform administrator | Tenant provisioning, shared infrastructure and explicit cross-tenant administration | Sensitive reads/actions audited; avoid routine use by application traffic | + +Phase 8 must implement a concrete operation-to-permission matrix, including resource ownership and delegation bounds. Content access is a separate privilege from metadata visibility. Tenant users cannot use arbitrary credential references or shared deployment ownership to bypass another tenant's controls. + +## Threats, mitigations and verification + +| Threat | Design control | Required evidence | +|---|---|---| +| Key guessing/theft | High-entropy keys, indexed hash verification, authentication throttles, expiry/revocation, TLS, no secret logging | Invalid-key load test and rotation/revocation drill | +| Cross-tenant access | Immutable context, scoped services/queries/cache keys, composite FKs and RLS | Negative two-tenant API/DB/async/worker/cache tests | +| Provider/guardrail/export SSRF | Approved endpoints, DNS/IP policy, block metadata services, redirect revalidation and destination-specific credentials | Private/loopback/link-local/redirect/DNS-change tests, with explicit approved local-model exceptions | +| Secret leakage | Reference-based storage, mounted secrets/workload identity, redacted errors/telemetry, one-time key display | Synthetic secrets absent from default artifacts | +| Denial of service/wallet | Size/time/concurrency bounds, token admission and durable cost reservations | Saturation, slow-client and concurrent budget tests | +| Prompt injection or malicious output | Explicit guardrails and application/tool authorization boundaries | Synthetic injection tests; documented limits, no universal prevention claim | +| Config privilege escalation | Role checks, validation, immutable revisions, optimistic locking and transactional audit | Unauthorized publish, stale edit and rollback/revocation tests | +| Duplicate/uncertain provider work | Shared retry budget, no replay after downstream bytes, durable attempt IDs | Disconnect, crash, timeout and reconciliation tests | +| Retention or export leak | Content off by default, destination/region controls, encryption, deletion jobs | Retention/delete/export and restore-policy tests | +| Supply-chain compromise | Pinned artifacts, dependency/image scans, SBOM, signing and controlled release credentials | Verifiable provenance and release scan results | + +The OWASP GenAI risk framework is a threat-model input; passing these tests is not a compliance certification. [OWASP GenAI risks](https://genai.owasp.org/llm-top-10/) + +## Credential and administrative security + +Use scoped, short-lived cloud identity where supported; otherwise resolve encrypted/mounted provider credentials by reference. Never forward the incoming gateway key to a provider. Credentials are scoped to an approved deployment/host and cannot follow a redirect to a new origin. Rotate secret versions without tearing down unrelated requests; expired or revoked secrets are never silently reused. + +Administrative tokens must validate issuer, audience, expiry, signature and role/tenant mapping. The console uses OIDC authorization code with PKCE; if server sessions/cookies are introduced, add CSRF defenses, secure HttpOnly cookies and origin checks. Use explicit CORS origins, CSP and dependency review. Never ship the MVP's blanket CSRF disable as an unexamined browser security design. + +Restrict detailed management/metrics to an internal authenticated listener/network. Public liveness/readiness returns minimal status only. The runtime database role cannot own tables or bypass RLS; migrations use a separate privileged role. Tenant context is transaction-local and cleared by pooling/worker lifecycle tests. + +## Content and retention policy + +Default logs, traces, audit and usage records omit prompt/completion/tool bodies. Exception text, URL query strings, HTTP debug logging and client-defined labels also need redaction/size limits. Hash-only mode uses tenant-keyed digests; unsalted hashes of low-entropy prompts are not anonymous. + +Allowed optional modes are redacted sampled content, encrypted full content with restricted access, or customer export to an approved destination. Validate region, encryption key, retention, access and sampling settings before enabling them. Exports and guardrails are separate data transfers; the same residency policy applies to fallback providers, caches, traces, replicas and backups. + +Sample operator defaults are 30 days request metadata, 90 days usage, 365 days administrative audit, content capture disabled, and exact-cache TTL no more than 5 minutes. These are configurable starting points, not legal retention guidance or current settings. Customer requirements must override them before collecting real data. + +Retention/delete workflows cover primary data, object storage and exports. Backups expire on a documented schedule; a restored backup must reapply deletion tombstones and current revocations before exposure. Financial/audit retention exceptions require an explicit customer policy, not silent indefinite retention. + +## Guardrail limits + +Mandatory security checks fail closed; advisory checks can fail open only through explicit policy and degraded telemetry. Cap regex complexity and scanning size/time. External checker failures must not cause unlimited retries or unapproved data export. + +Full-output validation requires bounded buffering or non-streaming mode. A chunk checker cannot retract text already delivered. Validate policy/stream capability at activation and request admission; do not advertise full-response redaction when only per-chunk checks run. + +Tool calls in the GA model API are returned to clients, not executed. Phase 15 introduces a separate tool authorization/approval model and side-effect-aware retry design before enabling an MCP proxy. + +## Release review and incident response + +Security acceptance requires the threat tests above, no unresolved critical/high exploitable release finding, reviewed bootstrap and credential handling, dependency provenance, tenant isolation and retention evidence. Assign a security owner and a private vulnerability reporting channel before public GA; do not invent contact details. + +For compromised credentials: disable the affected key/tenant/deployment, advance the security epoch, cancel work when necessary, rotate upstream secrets, preserve authorized metadata evidence, reconcile uncertain spend and verify revocation convergence. [Operations.md](Operations.md) expands recovery and operational gates. diff --git a/Documentation/evidence/phase6-baseline.json b/Documentation/evidence/phase6-baseline.json new file mode 100644 index 0000000..d138ca6 --- /dev/null +++ b/Documentation/evidence/phase6-baseline.json @@ -0,0 +1,34 @@ +{ + "scope" : "loopback HTTP, real BCrypt, H2 diagnostic writes, synthetic Ollama; not production capacity", + "java" : "21.0.7", + "os" : "Windows 11", + "cpus" : 24, + "inputCharacters" : 1024, + "warmupRequests" : 20, + "directSequential" : { + "p95Ms" : 1.1284, + "requests" : 100, + "p50Ms" : 0.5503, + "p99Ms" : 1.9713, + "requestsPerSecond" : 1479.3118241394104, + "concurrency" : 1 + }, + "gatewaySequential" : { + "p95Ms" : 60.2384, + "requests" : 100, + "p50Ms" : 51.787, + "p99Ms" : 64.8518, + "requestsPerSecond" : 18.80761454109966, + "concurrency" : 1 + }, + "gatewayConcurrent" : { + "p95Ms" : 59.1312, + "requests" : 100, + "p50Ms" : 54.558, + "p99Ms" : 62.0245, + "requestsPerSecond" : 177.08263791505735, + "concurrency" : 10 + }, + "heapUsedBytesAtEnd" : 72000456, + "maxHeapBytes" : 4215275520 +} \ No newline at end of file diff --git a/README.md b/README.md index 17274c5..d80d1b8 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,20 @@ # AI Inference Gateway -A backend API gateway for AI inference — one endpoint, multiple providers, centralized control and visibility. +A Java API gateway for AI inference, with Phase 6 foundation repairs and a plan for a production-ready enterprise product. -Instead of every application integrating directly with a provider's API (OpenAI, Anthropic, Ollama, etc.), applications send requests to this gateway. The gateway routes requests to a configured provider, normalizes responses, logs activity, and returns a consistent response — regardless of which provider handled it. +Applications send requests to the gateway, which selects a configured provider, normalizes the response and records metadata. **The current implementation supports synchronous Ollama generation only.** Multi-provider compatibility, streaming, tenant governance, an operator console and production deployment packages are planned, not implemented. --- -## Quick Start +## Product status and plan + +The MVP covers legacy inference, API-key authentication, provider/model selection and metadata log queries. It is a local demo baseline, not a qualified production deployment. The supplied competitive research has been translated into a dependency-based plan with explicit acceptance and release gates. + +Start with [PRD.md](Documentation/PRD.md) for product scope, [Architecture.md](Documentation/Architecture.md) for current gaps and target design, and [Phases.md](Documentation/Phases.md) for the implementation backlog. **Phase 6 foundation is implemented; [Phase6.md](Documentation/Phase6.md) records validation and open release gates.** The first GA includes compatibility, governance, reliability, privacy, administration and tested operations; broader enterprise and MCP capabilities follow. + +The former MVP rules and rigid phase sequencing are retired. Independent work can proceed once dependencies are satisfied. [Development.md](Documentation/Development.md) explains the workflow; [Roadmap.md](Documentation/Roadmap.md) preserves implementation history. + +## Quick Start (local development) ### Prerequisites @@ -32,7 +40,7 @@ This starts four containers: | Service | Port | Description | |---------|------|-------------| -| `gateway` | 8080 | Spring Boot API gateway | +| `gateway` | 8080 / 9090 | API / private management | | `postgres` | 5432 | PostgreSQL database | | `redis` | 6379 | Redis (provisioned for future use) | | `ollama` | 11434 | Local AI model server | @@ -42,7 +50,7 @@ This starts four containers: After the containers are running, pull the model registered in the seed data: ```bash -docker exec ollama ollama pull qwen3 +docker compose exec ollama ollama pull qwen3 ``` > **Note:** This downloads the model weights into the `ollama-data` Docker volume. You only need to do this once — the model persists across container restarts. @@ -50,18 +58,13 @@ docker exec ollama ollama pull qwen3 ### 4. Verify the gateway is running ```bash -curl http://localhost:8080/actuator/health +curl http://localhost:9090/actuator/health/readiness ``` Expected response: ```json { - "status": "UP", - "components": { - "db": { - "status": "UP" - } - } + "status": "UP" } ``` @@ -69,13 +72,13 @@ Expected response: ## Seed Data -The database is automatically populated with test data on first startup: +V1–V5 preserve the original provider/model seed. V6 disables the demo API key. The explicit `local` profile used by Compose reactivates a BCrypt-hashed `LOCAL_API_KEY` and updates the seeded provider URL. All Compose ports bind to host loopback. Production is the default outside Compose: it requires a non-demo database password and rejects any active known demo key before readiness. Do not use the local Compose stack as a production deployment or start the local profile against production data. | Table | Seed Entry | Notes | |-------|-----------|-------| | `providers` | `ollama-local` | Type `ollama`, default provider, active | | `models` | `qwen3` | Linked to `ollama-local`, active | -| `api_keys` | `local-test-key` | Plaintext: `test-api-key-1` | +| `api_keys` | `local-bootstrap` | Local profile only; default `test-api-key-1`, override `LOCAL_API_KEY` | --- @@ -86,7 +89,8 @@ The database is automatically populated with test data on first startup: | `/v1/inference` | POST | `X-API-Key` | Send a prompt, get an AI response | | `/v1/providers` | GET | `X-API-Key` | List configured providers and models | | `/v1/logs` | GET | `X-API-Key` | Query request history (paginated, filterable) | -| `/actuator/health` | GET | None | Service health check | +| `:9090/actuator/health` | GET | Private listener | Minimal health; `/liveness` and `/readiness` probes | +| `:9090/actuator/prometheus` | GET | Private listener | Metrics; never expose this listener publicly | --- @@ -120,7 +124,7 @@ curl -X POST http://localhost:8080/v1/inference \ "timestamp": "2026-08-05T12:00:00.000", "status": 400, "error": "INVALID_REQUEST", - "message": "prompt: must not be blank", + "message": "Invalid request parameters", "path": "/v1/inference", "requestId": "req_abc123..." } @@ -132,7 +136,7 @@ curl -X POST http://localhost:8080/v1/inference \ "timestamp": "2026-08-05T12:00:00.000", "status": 401, "error": "UNAUTHORIZED", - "message": "Missing API key", + "message": "Missing or invalid API key", "path": "/v1/inference", "requestId": "req_def456..." } @@ -212,10 +216,13 @@ curl "http://localhost:8080/v1/logs?from=2026-08-01T00:00:00&to=2026-08-05T23:59 | `DB_PORT` | `5432` | PostgreSQL port | | `DB_NAME` | `gateway_db` | Database name | | `DB_USERNAME` | `gateway` | Database user | -| `DB_PASSWORD` | `gateway` | Database password | -| `OLLAMA_BASE_URL` | `http://localhost:11434` | Ollama server URL | +| `DB_PASSWORD` | Required | Non-demo database password in production; local profile defaults to `gateway` | +| `SPRING_PROFILES_ACTIVE` | `production` (default profile) | Explicitly choose `local` only for demos | +| `LOCAL_API_KEY` | `test-api-key-1` in local profile | Local bootstrap key, maximum 72 UTF-8 bytes | +| `OLLAMA_BASE_URL` | `http://localhost:11434` in local profile | Local bootstrap updates the seeded provider row; Compose sets `http://ollama:11434` | +| `MANAGEMENT_PORT` / `MANAGEMENT_ADDRESS` | `9090` / `127.0.0.1` | Separate private health/metrics listener | -> When running via Docker Compose, these are set automatically in `docker-compose.yml`. The `.env` file is for customization only. +Compose reads `POSTGRES_USER`, `POSTGRES_PASSWORD`, `POSTGRES_DB` and `LOCAL_API_KEY` from `.env` and passes the corresponding runtime properties. Direct JAR launches use `DB_*`. Outside local bootstrap the database provider URL remains authoritative. [Operations.md](Documentation/Operations.md) lists request/transport limits, management access and upgrade behavior. --- @@ -226,7 +233,7 @@ AI-Inference-Gateway/ ├── docker-compose.yml # Orchestrates all 4 services ├── Dockerfile # Multi-stage build (Maven → JRE Alpine) ├── pom.xml # Maven project descriptor -├── Documentation/ # Project docs (PRD, Architecture, Rules, Phases, Roadmap) +├── Documentation/ # Product, architecture, delivery, API, security and operations docs ├── http/ │ └── requests.http # Sample HTTP requests for manual testing ├── src/ @@ -255,19 +262,43 @@ AI-Inference-Gateway/ ## Documentation -- [PRD.md](Documentation/PRD.md) — product scope, features, target users -- [Architecture.md](Documentation/Architecture.md) — tech stack, system flow, data model -- [Rules.md](Documentation/Rules.md) — development boundaries and constraints -- [Phases.md](Documentation/Phases.md) — build order broken into stages -- [Roadmap.md](Documentation/Roadmap.md) — development history and decision log +- [PRD.md](Documentation/PRD.md) — requirements, scope and release targets +- [Architecture.md](Documentation/Architecture.md) — inspected baseline, target design and migration +- [Phases.md](Documentation/Phases.md) — dependencies, acceptance gates and first implementation issues +- [API.md](Documentation/API.md) — existing and proposed endpoints, compatibility and migration +- [Security.md](Documentation/Security.md) — tenant access, privacy and threat model +- [Operations.md](Documentation/Operations.md) — deployment, recovery and production evidence +- [Development.md](Documentation/Development.md) — flexible development workflow +- [Research.md](Documentation/Research.md) — research decisions and primary sources +- [Roadmap.md](Documentation/Roadmap.md) — preserved history and actual implementation status --- -## Tech Stack +## Build and tests + +With Java 21 installed, use the checksum-pinned Maven wrapper (`mvnw.cmd` on Windows, `sh mvnw` on Linux): + +```bash +sh mvnw -B test +# Use a disposable PostgreSQL database whose user may CREATE/DROP test schemas: +export TEST_DATABASE_URL=jdbc:postgresql://localhost:5432/gateway_test +export TEST_DATABASE_USER=postgres +export TEST_DATABASE_PASSWORD='' +sh mvnw -B -Ppostgres-it verify +# Optional loopback mock benchmark (writes target/phase6-baseline.json): +sh mvnw -B -Dgateway.benchmark=true -Dtest=BaselineBenchmarkTest test +python3 scripts/release_manifest.py target/ai-inference-gateway-0.0.1-SNAPSHOT.jar +``` + +The default suite uses H2 for speed. The required CI profile runs Flyway fresh-install, populated-MVP upgrade and real PostgreSQL production-startup checks; missing database configuration fails this profile. The Docker image build skips tests, while CI runs them first. CI also scans dependencies, secrets, configuration and container vulnerabilities and retains an artifact manifest/SBOM. See [Phase6.md](Documentation/Phase6.md) for actual local results and remote CI status. + +## Current Tech Stack | Layer | Technology | |-------|-----------| -| Backend | Spring Boot 3.4 (Java 21) | +| Backend | Spring Boot 4.1.1 (Java 21), MVC, Jackson 3 | +| Provider HTTP | Apache HttpClient 5, bounded shared pool, no retries | +| Build | Maven wrapper 3.9.11; GitHub Actions verification | | Database | PostgreSQL 16 | | Cache (future) | Redis 7 | | AI Provider (MVP) | Ollama | diff --git a/docker-compose.yml b/docker-compose.yml index cbbd475..a90f247 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,59 +1,48 @@ +# Local development only. No infrastructure port is exposed beyond loopback. services: - # ── Gateway (Spring Boot) ────────────────────────── gateway: build: . - container_name: gateway ports: - - "8080:8080" + - "127.0.0.1:8080:8080" + - "127.0.0.1:9090:9090" environment: + SPRING_PROFILES_ACTIVE: local DB_HOST: postgres DB_PORT: 5432 - DB_NAME: gateway_db - DB_USERNAME: gateway - DB_PASSWORD: gateway + DB_NAME: ${POSTGRES_DB:-gateway_db} + DB_USERNAME: ${POSTGRES_USER:-gateway} + DB_PASSWORD: ${POSTGRES_PASSWORD:-gateway} + LOCAL_API_KEY: ${LOCAL_API_KEY:-test-api-key-1} OLLAMA_BASE_URL: http://ollama:11434 + MANAGEMENT_ADDRESS: 0.0.0.0 depends_on: postgres: condition: service_healthy - restart: unless-stopped - - # ── PostgreSQL ───────────────────────────────────── postgres: image: postgres:16-alpine - container_name: postgres ports: - - "5432:5432" + - "127.0.0.1:5432:5432" environment: - POSTGRES_USER: gateway - POSTGRES_PASSWORD: gateway - POSTGRES_DB: gateway_db + POSTGRES_USER: ${POSTGRES_USER:-gateway} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-gateway} + POSTGRES_DB: ${POSTGRES_DB:-gateway_db} volumes: - postgres-data:/var/lib/postgresql/data healthcheck: - test: ["CMD-SHELL", "pg_isready -U gateway -d gateway_db"] + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] interval: 5s timeout: 5s - retries: 5 - restart: unless-stopped - - # ── Redis (provisioned, not used in MVP logic) ───── + retries: 10 redis: image: redis:7-alpine - container_name: redis ports: - - "6379:6379" - restart: unless-stopped - - # ── Ollama (local AI model server) ───────────────── + - "127.0.0.1:6379:6379" ollama: image: ollama/ollama - container_name: ollama ports: - - "11434:11434" + - "127.0.0.1:11434:11434" volumes: - ollama-data:/root/.ollama - restart: unless-stopped - volumes: postgres-data: ollama-data: diff --git a/http/requests.http b/http/requests.http index 744b3da..ca06d80 100644 --- a/http/requests.http +++ b/http/requests.http @@ -3,7 +3,7 @@ ### ─── Health Check (no auth required) ─── -GET http://localhost:8080/actuator/health +GET http://localhost:9090/actuator/health ### diff --git a/mvnw b/mvnw new file mode 100644 index 0000000..bd8896b --- /dev/null +++ b/mvnw @@ -0,0 +1,295 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Apache Maven Wrapper startup batch script, version 3.3.4 +# +# Optional ENV vars +# ----------------- +# JAVA_HOME - location of a JDK home dir, required when download maven via java source +# MVNW_REPOURL - repo url base for downloading maven distribution +# MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +# MVNW_VERBOSE - true: enable verbose log; debug: trace the mvnw script; others: silence the output +# ---------------------------------------------------------------------------- + +set -euf +[ "${MVNW_VERBOSE-}" != debug ] || set -x + +# OS specific support. +native_path() { printf %s\\n "$1"; } +case "$(uname)" in +CYGWIN* | MINGW*) + [ -z "${JAVA_HOME-}" ] || JAVA_HOME="$(cygpath --unix "$JAVA_HOME")" + native_path() { cygpath --path --windows "$1"; } + ;; +esac + +# set JAVACMD and JAVACCMD +set_java_home() { + # For Cygwin and MinGW, ensure paths are in Unix format before anything is touched + if [ -n "${JAVA_HOME-}" ]; then + if [ -x "$JAVA_HOME/jre/sh/java" ]; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACCMD="$JAVA_HOME/jre/sh/javac" + else + JAVACMD="$JAVA_HOME/bin/java" + JAVACCMD="$JAVA_HOME/bin/javac" + + if [ ! -x "$JAVACMD" ] || [ ! -x "$JAVACCMD" ]; then + echo "The JAVA_HOME environment variable is not defined correctly, so mvnw cannot run." >&2 + echo "JAVA_HOME is set to \"$JAVA_HOME\", but \"\$JAVA_HOME/bin/java\" or \"\$JAVA_HOME/bin/javac\" does not exist." >&2 + return 1 + fi + fi + else + JAVACMD="$( + 'set' +e + 'unset' -f command 2>/dev/null + 'command' -v java + )" || : + JAVACCMD="$( + 'set' +e + 'unset' -f command 2>/dev/null + 'command' -v javac + )" || : + + if [ ! -x "${JAVACMD-}" ] || [ ! -x "${JAVACCMD-}" ]; then + echo "The java/javac command does not exist in PATH nor is JAVA_HOME set, so mvnw cannot run." >&2 + return 1 + fi + fi +} + +# hash string like Java String::hashCode +hash_string() { + str="${1:-}" h=0 + while [ -n "$str" ]; do + char="${str%"${str#?}"}" + h=$(((h * 31 + $(LC_CTYPE=C printf %d "'$char")) % 4294967296)) + str="${str#?}" + done + printf %x\\n $h +} + +verbose() { :; } +[ "${MVNW_VERBOSE-}" != true ] || verbose() { printf %s\\n "${1-}"; } + +die() { + printf %s\\n "$1" >&2 + exit 1 +} + +trim() { + # MWRAPPER-139: + # Trims trailing and leading whitespace, carriage returns, tabs, and linefeeds. + # Needed for removing poorly interpreted newline sequences when running in more + # exotic environments such as mingw bash on Windows. + printf "%s" "${1}" | tr -d '[:space:]' +} + +scriptDir="$(dirname "$0")" +scriptName="$(basename "$0")" + +# parse distributionUrl and optional distributionSha256Sum, requires .mvn/wrapper/maven-wrapper.properties +while IFS="=" read -r key value; do + case "${key-}" in + distributionUrl) distributionUrl=$(trim "${value-}") ;; + distributionSha256Sum) distributionSha256Sum=$(trim "${value-}") ;; + esac +done <"$scriptDir/.mvn/wrapper/maven-wrapper.properties" +[ -n "${distributionUrl-}" ] || die "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties" + +case "${distributionUrl##*/}" in +maven-mvnd-*bin.*) + MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ + case "${PROCESSOR_ARCHITECTURE-}${PROCESSOR_ARCHITEW6432-}:$(uname -a)" in + *AMD64:CYGWIN* | *AMD64:MINGW*) distributionPlatform=windows-amd64 ;; + :Darwin*x86_64) distributionPlatform=darwin-amd64 ;; + :Darwin*arm64) distributionPlatform=darwin-aarch64 ;; + :Linux*x86_64*) distributionPlatform=linux-amd64 ;; + *) + echo "Cannot detect native platform for mvnd on $(uname)-$(uname -m), use pure java version" >&2 + distributionPlatform=linux-amd64 + ;; + esac + distributionUrl="${distributionUrl%-bin.*}-$distributionPlatform.zip" + ;; +maven-mvnd-*) MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ ;; +*) MVN_CMD="mvn${scriptName#mvnw}" _MVNW_REPO_PATTERN=/org/apache/maven/ ;; +esac + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ +[ -z "${MVNW_REPOURL-}" ] || distributionUrl="$MVNW_REPOURL$_MVNW_REPO_PATTERN${distributionUrl#*"$_MVNW_REPO_PATTERN"}" +distributionUrlName="${distributionUrl##*/}" +distributionUrlNameMain="${distributionUrlName%.*}" +distributionUrlNameMain="${distributionUrlNameMain%-bin}" +MAVEN_USER_HOME="${MAVEN_USER_HOME:-${HOME}/.m2}" +MAVEN_HOME="${MAVEN_USER_HOME}/wrapper/dists/${distributionUrlNameMain-}/$(hash_string "$distributionUrl")" + +exec_maven() { + unset MVNW_VERBOSE MVNW_USERNAME MVNW_PASSWORD MVNW_REPOURL || : + exec "$MAVEN_HOME/bin/$MVN_CMD" "$@" || die "cannot exec $MAVEN_HOME/bin/$MVN_CMD" +} + +if [ -d "$MAVEN_HOME" ]; then + verbose "found existing MAVEN_HOME at $MAVEN_HOME" + exec_maven "$@" +fi + +case "${distributionUrl-}" in +*?-bin.zip | *?maven-mvnd-?*-?*.zip) ;; +*) die "distributionUrl is not valid, must match *-bin.zip or maven-mvnd-*.zip, but found '${distributionUrl-}'" ;; +esac + +# prepare tmp dir +if TMP_DOWNLOAD_DIR="$(mktemp -d)" && [ -d "$TMP_DOWNLOAD_DIR" ]; then + clean() { rm -rf -- "$TMP_DOWNLOAD_DIR"; } + trap clean HUP INT TERM EXIT +else + die "cannot create temp dir" +fi + +mkdir -p -- "${MAVEN_HOME%/*}" + +# Download and Install Apache Maven +verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +verbose "Downloading from: $distributionUrl" +verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" + +# select .zip or .tar.gz +if ! command -v unzip >/dev/null; then + distributionUrl="${distributionUrl%.zip}.tar.gz" + distributionUrlName="${distributionUrl##*/}" +fi + +# verbose opt +__MVNW_QUIET_WGET=--quiet __MVNW_QUIET_CURL=--silent __MVNW_QUIET_UNZIP=-q __MVNW_QUIET_TAR='' +[ "${MVNW_VERBOSE-}" != true ] || __MVNW_QUIET_WGET='' __MVNW_QUIET_CURL='' __MVNW_QUIET_UNZIP='' __MVNW_QUIET_TAR=v + +# normalize http auth +case "${MVNW_PASSWORD:+has-password}" in +'') MVNW_USERNAME='' MVNW_PASSWORD='' ;; +has-password) [ -n "${MVNW_USERNAME-}" ] || MVNW_USERNAME='' MVNW_PASSWORD='' ;; +esac + +if [ -z "${MVNW_USERNAME-}" ] && command -v wget >/dev/null; then + verbose "Found wget ... using wget" + wget ${__MVNW_QUIET_WGET:+"$__MVNW_QUIET_WGET"} "$distributionUrl" -O "$TMP_DOWNLOAD_DIR/$distributionUrlName" || die "wget: Failed to fetch $distributionUrl" +elif [ -z "${MVNW_USERNAME-}" ] && command -v curl >/dev/null; then + verbose "Found curl ... using curl" + curl ${__MVNW_QUIET_CURL:+"$__MVNW_QUIET_CURL"} -f -L -o "$TMP_DOWNLOAD_DIR/$distributionUrlName" "$distributionUrl" || die "curl: Failed to fetch $distributionUrl" +elif set_java_home; then + verbose "Falling back to use Java to download" + javaSource="$TMP_DOWNLOAD_DIR/Downloader.java" + targetZip="$TMP_DOWNLOAD_DIR/$distributionUrlName" + cat >"$javaSource" <<-END + public class Downloader extends java.net.Authenticator + { + protected java.net.PasswordAuthentication getPasswordAuthentication() + { + return new java.net.PasswordAuthentication( System.getenv( "MVNW_USERNAME" ), System.getenv( "MVNW_PASSWORD" ).toCharArray() ); + } + public static void main( String[] args ) throws Exception + { + setDefault( new Downloader() ); + java.nio.file.Files.copy( java.net.URI.create( args[0] ).toURL().openStream(), java.nio.file.Paths.get( args[1] ).toAbsolutePath().normalize() ); + } + } + END + # For Cygwin/MinGW, switch paths to Windows format before running javac and java + verbose " - Compiling Downloader.java ..." + "$(native_path "$JAVACCMD")" "$(native_path "$javaSource")" || die "Failed to compile Downloader.java" + verbose " - Running Downloader.java ..." + "$(native_path "$JAVACMD")" -cp "$(native_path "$TMP_DOWNLOAD_DIR")" Downloader "$distributionUrl" "$(native_path "$targetZip")" +fi + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +if [ -n "${distributionSha256Sum-}" ]; then + distributionSha256Result=false + if [ "$MVN_CMD" = mvnd.sh ]; then + echo "Checksum validation is not supported for maven-mvnd." >&2 + echo "Please disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 + exit 1 + elif command -v sha256sum >/dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | sha256sum -c - >/dev/null 2>&1; then + distributionSha256Result=true + fi + elif command -v shasum >/dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | shasum -a 256 -c >/dev/null 2>&1; then + distributionSha256Result=true + fi + else + echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." >&2 + echo "Please install either command, or disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 + exit 1 + fi + if [ $distributionSha256Result = false ]; then + echo "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised." >&2 + echo "If you updated your Maven version, you need to update the specified distributionSha256Sum property." >&2 + exit 1 + fi +fi + +# unzip and move +if command -v unzip >/dev/null; then + unzip ${__MVNW_QUIET_UNZIP:+"$__MVNW_QUIET_UNZIP"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -d "$TMP_DOWNLOAD_DIR" || die "failed to unzip" +else + tar xzf${__MVNW_QUIET_TAR:+"$__MVNW_QUIET_TAR"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -C "$TMP_DOWNLOAD_DIR" || die "failed to untar" +fi + +# Find the actual extracted directory name (handles snapshots where filename != directory name) +actualDistributionDir="" + +# First try the expected directory name (for regular distributions) +if [ -d "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" ]; then + if [ -f "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain/bin/$MVN_CMD" ]; then + actualDistributionDir="$distributionUrlNameMain" + fi +fi + +# If not found, search for any directory with the Maven executable (for snapshots) +if [ -z "$actualDistributionDir" ]; then + # enable globbing to iterate over items + set +f + for dir in "$TMP_DOWNLOAD_DIR"/*; do + if [ -d "$dir" ]; then + if [ -f "$dir/bin/$MVN_CMD" ]; then + actualDistributionDir="$(basename "$dir")" + break + fi + fi + done + set -f +fi + +if [ -z "$actualDistributionDir" ]; then + verbose "Contents of $TMP_DOWNLOAD_DIR:" + verbose "$(ls -la "$TMP_DOWNLOAD_DIR")" + die "Could not find Maven distribution directory in extracted archive" +fi + +verbose "Found extracted Maven distribution directory: $actualDistributionDir" +printf %s\\n "$distributionUrl" >"$TMP_DOWNLOAD_DIR/$actualDistributionDir/mvnw.url" +mv -- "$TMP_DOWNLOAD_DIR/$actualDistributionDir" "$MAVEN_HOME" || [ -d "$MAVEN_HOME" ] || die "fail to move MAVEN_HOME" + +clean || : +exec_maven "$@" diff --git a/mvnw.cmd b/mvnw.cmd new file mode 100644 index 0000000..92450f9 --- /dev/null +++ b/mvnw.cmd @@ -0,0 +1,189 @@ +<# : batch portion +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Apache Maven Wrapper startup batch script, version 3.3.4 +@REM +@REM Optional ENV vars +@REM MVNW_REPOURL - repo url base for downloading maven distribution +@REM MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +@REM MVNW_VERBOSE - true: enable verbose log; others: silence the output +@REM ---------------------------------------------------------------------------- + +@IF "%__MVNW_ARG0_NAME__%"=="" (SET __MVNW_ARG0_NAME__=%~nx0) +@SET __MVNW_CMD__= +@SET __MVNW_ERROR__= +@SET __MVNW_PSMODULEP_SAVE=%PSModulePath% +@SET PSModulePath= +@FOR /F "usebackq tokens=1* delims==" %%A IN (`powershell -noprofile "& {$scriptDir='%~dp0'; $script='%__MVNW_ARG0_NAME__%'; icm -ScriptBlock ([Scriptblock]::Create((Get-Content -Raw '%~f0'))) -NoNewScope}"`) DO @( + IF "%%A"=="MVN_CMD" (set __MVNW_CMD__=%%B) ELSE IF "%%B"=="" (echo %%A) ELSE (echo %%A=%%B) +) +@SET PSModulePath=%__MVNW_PSMODULEP_SAVE% +@SET __MVNW_PSMODULEP_SAVE= +@SET __MVNW_ARG0_NAME__= +@SET MVNW_USERNAME= +@SET MVNW_PASSWORD= +@IF NOT "%__MVNW_CMD__%"=="" ("%__MVNW_CMD__%" %*) +@echo Cannot start maven from wrapper >&2 && exit /b 1 +@GOTO :EOF +: end batch / begin powershell #> + +$ErrorActionPreference = "Stop" +if ($env:MVNW_VERBOSE -eq "true") { + $VerbosePreference = "Continue" +} + +# calculate distributionUrl, requires .mvn/wrapper/maven-wrapper.properties +$distributionUrl = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionUrl +if (!$distributionUrl) { + Write-Error "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties" +} + +switch -wildcard -casesensitive ( $($distributionUrl -replace '^.*/','') ) { + "maven-mvnd-*" { + $USE_MVND = $true + $distributionUrl = $distributionUrl -replace '-bin\.[^.]*$',"-windows-amd64.zip" + $MVN_CMD = "mvnd.cmd" + break + } + default { + $USE_MVND = $false + $MVN_CMD = $script -replace '^mvnw','mvn' + break + } +} + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ +if ($env:MVNW_REPOURL) { + $MVNW_REPO_PATTERN = if ($USE_MVND -eq $False) { "/org/apache/maven/" } else { "/maven/mvnd/" } + $distributionUrl = "$env:MVNW_REPOURL$MVNW_REPO_PATTERN$($distributionUrl -replace "^.*$MVNW_REPO_PATTERN",'')" +} +$distributionUrlName = $distributionUrl -replace '^.*/','' +$distributionUrlNameMain = $distributionUrlName -replace '\.[^.]*$','' -replace '-bin$','' + +$MAVEN_M2_PATH = "$HOME/.m2" +if ($env:MAVEN_USER_HOME) { + $MAVEN_M2_PATH = "$env:MAVEN_USER_HOME" +} + +if (-not (Test-Path -Path $MAVEN_M2_PATH)) { + New-Item -Path $MAVEN_M2_PATH -ItemType Directory | Out-Null +} + +$MAVEN_WRAPPER_DISTS = $null +if ((Get-Item $MAVEN_M2_PATH).Target[0] -eq $null) { + $MAVEN_WRAPPER_DISTS = "$MAVEN_M2_PATH/wrapper/dists" +} else { + $MAVEN_WRAPPER_DISTS = (Get-Item $MAVEN_M2_PATH).Target[0] + "/wrapper/dists" +} + +$MAVEN_HOME_PARENT = "$MAVEN_WRAPPER_DISTS/$distributionUrlNameMain" +$MAVEN_HOME_NAME = ([System.Security.Cryptography.SHA256]::Create().ComputeHash([byte[]][char[]]$distributionUrl) | ForEach-Object {$_.ToString("x2")}) -join '' +$MAVEN_HOME = "$MAVEN_HOME_PARENT/$MAVEN_HOME_NAME" + +if (Test-Path -Path "$MAVEN_HOME" -PathType Container) { + Write-Verbose "found existing MAVEN_HOME at $MAVEN_HOME" + Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" + exit $? +} + +if (! $distributionUrlNameMain -or ($distributionUrlName -eq $distributionUrlNameMain)) { + Write-Error "distributionUrl is not valid, must end with *-bin.zip, but found $distributionUrl" +} + +# prepare tmp dir +$TMP_DOWNLOAD_DIR_HOLDER = New-TemporaryFile +$TMP_DOWNLOAD_DIR = New-Item -Itemtype Directory -Path "$TMP_DOWNLOAD_DIR_HOLDER.dir" +$TMP_DOWNLOAD_DIR_HOLDER.Delete() | Out-Null +trap { + if ($TMP_DOWNLOAD_DIR.Exists) { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } + } +} + +New-Item -Itemtype Directory -Path "$MAVEN_HOME_PARENT" -Force | Out-Null + +# Download and Install Apache Maven +Write-Verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +Write-Verbose "Downloading from: $distributionUrl" +Write-Verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" + +$webclient = New-Object System.Net.WebClient +if ($env:MVNW_USERNAME -and $env:MVNW_PASSWORD) { + $webclient.Credentials = New-Object System.Net.NetworkCredential($env:MVNW_USERNAME, $env:MVNW_PASSWORD) +} +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +$webclient.DownloadFile($distributionUrl, "$TMP_DOWNLOAD_DIR/$distributionUrlName") | Out-Null + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +$distributionSha256Sum = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionSha256Sum +if ($distributionSha256Sum) { + if ($USE_MVND) { + Write-Error "Checksum validation is not supported for maven-mvnd. `nPlease disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." + } + Import-Module $PSHOME\Modules\Microsoft.PowerShell.Utility -Function Get-FileHash + if ((Get-FileHash "$TMP_DOWNLOAD_DIR/$distributionUrlName" -Algorithm SHA256).Hash.ToLower() -ne $distributionSha256Sum) { + Write-Error "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised. If you updated your Maven version, you need to update the specified distributionSha256Sum property." + } +} + +# unzip and move +Expand-Archive "$TMP_DOWNLOAD_DIR/$distributionUrlName" -DestinationPath "$TMP_DOWNLOAD_DIR" | Out-Null + +# Find the actual extracted directory name (handles snapshots where filename != directory name) +$actualDistributionDir = "" + +# First try the expected directory name (for regular distributions) +$expectedPath = Join-Path "$TMP_DOWNLOAD_DIR" "$distributionUrlNameMain" +$expectedMvnPath = Join-Path "$expectedPath" "bin/$MVN_CMD" +if ((Test-Path -Path $expectedPath -PathType Container) -and (Test-Path -Path $expectedMvnPath -PathType Leaf)) { + $actualDistributionDir = $distributionUrlNameMain +} + +# If not found, search for any directory with the Maven executable (for snapshots) +if (!$actualDistributionDir) { + Get-ChildItem -Path "$TMP_DOWNLOAD_DIR" -Directory | ForEach-Object { + $testPath = Join-Path $_.FullName "bin/$MVN_CMD" + if (Test-Path -Path $testPath -PathType Leaf) { + $actualDistributionDir = $_.Name + } + } +} + +if (!$actualDistributionDir) { + Write-Error "Could not find Maven distribution directory in extracted archive" +} + +Write-Verbose "Found extracted Maven distribution directory: $actualDistributionDir" +Rename-Item -Path "$TMP_DOWNLOAD_DIR/$actualDistributionDir" -NewName $MAVEN_HOME_NAME | Out-Null +try { + Move-Item -Path "$TMP_DOWNLOAD_DIR/$MAVEN_HOME_NAME" -Destination $MAVEN_HOME_PARENT | Out-Null +} catch { + if (! (Test-Path -Path "$MAVEN_HOME" -PathType Container)) { + Write-Error "fail to move MAVEN_HOME" + } +} finally { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } +} + +Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" diff --git a/pom.xml b/pom.xml index 4c5e555..dcf7c75 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.springframework.boot spring-boot-starter-parent - 3.4.1 + 4.1.1 @@ -19,13 +19,16 @@ 21 + + 11.0.25 + 2026-09-21T00:00:00Z org.springframework.boot - spring-boot-starter-web + spring-boot-starter-webmvc @@ -46,7 +49,7 @@ spring-boot-starter-actuator - + org.springframework.boot spring-boot-starter-security @@ -61,15 +64,24 @@ - org.flywaydb - flyway-core + org.springframework.boot + spring-boot-starter-flyway org.flywaydb flyway-database-postgresql - + + org.apache.httpcomponents.client5 + httpclient5 + + + io.micrometer + micrometer-registry-prometheus + + + org.projectlombok lombok @@ -84,6 +96,11 @@ + + org.springframework.boot + spring-boot-starter-webmvc-test + test + com.h2database h2 @@ -115,4 +132,19 @@ + + + postgres-it + + + + org.apache.maven.plugins + maven-failsafe-plugin + integration-testverify + + + + + + diff --git a/scripts/release_manifest.py b/scripts/release_manifest.py new file mode 100644 index 0000000..79e8ea4 --- /dev/null +++ b/scripts/release_manifest.py @@ -0,0 +1,25 @@ +"""Write an artifact identity manifest; no publishing or credentials required.""" +import hashlib +import json +import pathlib +import subprocess +import sys + +artifact = pathlib.Path(sys.argv[1]) +revision = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip() +dirty = bool(subprocess.check_output(["git", "status", "--porcelain"], text=True).strip()) +with artifact.open("rb") as stream: + digest = hashlib.file_digest(stream, "sha256").hexdigest() +data = { + "schemaVersion": 1, + "gitRevision": revision, + "workingTreeDirty": dirty, + "artifact": artifact.name, + "sizeBytes": artifact.stat().st_size, + "sha256": digest, + "javaTarget": 21, + "mavenWrapper": "3.9.11", +} +output = artifact.parent / "release-manifest.json" +output.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8") +print(output) diff --git a/src/main/java/com/gateway/GatewayApplication.java b/src/main/java/com/gateway/GatewayApplication.java index a2261ae..86e7095 100644 --- a/src/main/java/com/gateway/GatewayApplication.java +++ b/src/main/java/com/gateway/GatewayApplication.java @@ -2,8 +2,11 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.context.properties.ConfigurationPropertiesScan; +import org.springframework.boot.security.autoconfigure.UserDetailsServiceAutoConfiguration; -@SpringBootApplication +@SpringBootApplication(exclude = UserDetailsServiceAutoConfiguration.class) +@ConfigurationPropertiesScan public class GatewayApplication { public static void main(String[] args) { diff --git a/src/main/java/com/gateway/api/InferenceController.java b/src/main/java/com/gateway/api/InferenceController.java index af40d0e..8c7726f 100644 --- a/src/main/java/com/gateway/api/InferenceController.java +++ b/src/main/java/com/gateway/api/InferenceController.java @@ -34,7 +34,7 @@ public InferenceController(InferenceService inferenceService) { * @param httpRequest provides the requestId set by {@link com.gateway.common.RequestIdFilter} * @return 200 with the standardized {@link InferenceResponse} */ - @PostMapping("/inference") + @PostMapping(value = "/inference", consumes = "application/json", produces = "application/json") public ResponseEntity infer( @Valid @RequestBody InferenceRequest request, HttpServletRequest httpRequest) { diff --git a/src/main/java/com/gateway/auth/ApiKeyFilter.java b/src/main/java/com/gateway/auth/ApiKeyFilter.java index ffa5837..030034b 100644 --- a/src/main/java/com/gateway/auth/ApiKeyFilter.java +++ b/src/main/java/com/gateway/auth/ApiKeyFilter.java @@ -1,94 +1,37 @@ package com.gateway.auth; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.gateway.common.RequestIdFilter; -import com.gateway.error.ApiError; +import com.gateway.error.ApiErrorWriter; import com.gateway.error.ErrorCode; import jakarta.servlet.FilterChain; import jakarta.servlet.ServletException; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.http.MediaType; import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.web.filter.OncePerRequestFilter; - import java.io.IOException; import java.util.Collections; -/** - * Servlet filter that authenticates requests using the {@code X-API-Key} header. - *

- * Registered exclusively in the Spring Security filter chain via {@link SecurityConfig} - * (not as a servlet-level filter via {@code @Component}) so that the {@code SecurityContext} - * set here is not cleared by Spring Security's own context management. - *

- * On auth failure, writes a JSON {@link ApiError} directly to the response — - * {@code GlobalExceptionHandler} doesn't cover filters (they run before DispatcherServlet). - */ +/** Registered only in the security chain; never logs supplied credentials or paths. */ public class ApiKeyFilter extends OncePerRequestFilter { - - private static final Logger log = LoggerFactory.getLogger(ApiKeyFilter.class); - private static final String API_KEY_HEADER = "X-API-Key"; - - private final ApiKeyService apiKeyService; - private final ObjectMapper objectMapper; - - public ApiKeyFilter(ApiKeyService apiKeyService, ObjectMapper objectMapper) { - this.apiKeyService = apiKeyService; - this.objectMapper = objectMapper; + private final ApiKeyService keys; + private final ApiErrorWriter errors; + public ApiKeyFilter(ApiKeyService keys, ApiErrorWriter errors) { this.keys = keys; this.errors = errors; } + @Override protected boolean shouldNotFilter(HttpServletRequest request) { + return !request.getRequestURI().startsWith("/v1/"); } - - @Override - protected void doFilterInternal(HttpServletRequest request, - HttpServletResponse response, - FilterChain filterChain) throws ServletException, IOException { - String apiKey = request.getHeader(API_KEY_HEADER); - - if (apiKey == null || apiKey.isBlank()) { - log.warn("Missing API key on {} {}", request.getMethod(), request.getRequestURI()); - writeUnauthorizedResponse(response, request, "Missing API key — provide X-API-Key header"); - return; - } - - if (!apiKeyService.validate(apiKey)) { - log.warn("Invalid API key on {} {}", request.getMethod(), request.getRequestURI()); - writeUnauthorizedResponse(response, request, "Invalid API key"); - return; + @Override protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, + FilterChain chain) throws IOException, ServletException { + String key = request.getHeader("X-API-Key"); + try { + if (key == null || key.isBlank() || key.getBytes(java.nio.charset.StandardCharsets.UTF_8).length > 72 || !keys.validate(key)) { + errors.write(request, response, ErrorCode.UNAUTHORIZED); return; + } + } catch (RuntimeException failure) { + errors.write(request, response, ErrorCode.PROVIDER_UNAVAILABLE); return; } - - // Set authentication in SecurityContext so Spring Security considers the request authenticated - UsernamePasswordAuthenticationToken authentication = - new UsernamePasswordAuthenticationToken("api-key-user", null, Collections.emptyList()); - SecurityContextHolder.getContext().setAuthentication(authentication); - - filterChain.doFilter(request, response); - } - - /** - * Only apply this filter to /v1/** paths. - * Health/actuator endpoints and other paths are excluded. - */ - @Override - protected boolean shouldNotFilter(HttpServletRequest request) { - String path = request.getRequestURI(); - return !path.startsWith("/v1/"); - } - - private void writeUnauthorizedResponse(HttpServletResponse response, - HttpServletRequest request, - String message) throws IOException { - String requestId = (String) request.getAttribute(RequestIdFilter.REQUEST_ID_ATTRIBUTE); - if (requestId == null) { - requestId = "unknown"; - } - - ApiError error = ApiError.of(ErrorCode.UNAUTHORIZED, message, request.getRequestURI(), requestId); - - response.setStatus(ErrorCode.UNAUTHORIZED.getStatusCode()); - response.setContentType(MediaType.APPLICATION_JSON_VALUE); - response.getWriter().write(objectMapper.writeValueAsString(error)); + SecurityContextHolder.getContext().setAuthentication( + new UsernamePasswordAuthenticationToken("api-key-user", null, Collections.emptyList())); + chain.doFilter(request, response); } } diff --git a/src/main/java/com/gateway/auth/SecurityConfig.java b/src/main/java/com/gateway/auth/SecurityConfig.java index 7ccfb3d..53b47b9 100644 --- a/src/main/java/com/gateway/auth/SecurityConfig.java +++ b/src/main/java/com/gateway/auth/SecurityConfig.java @@ -1,48 +1,46 @@ package com.gateway.auth; -import com.fasterxml.jackson.databind.ObjectMapper; +import com.gateway.error.ApiErrorWriter; +import com.gateway.error.ErrorCode; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.Environment; +import org.springframework.boot.web.servlet.FilterRegistrationBean; import org.springframework.security.config.annotation.web.builders.HttpSecurity; import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; import org.springframework.security.config.http.SessionCreationPolicy; -import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; import org.springframework.security.web.SecurityFilterChain; import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter; -/** - * Spring Security configuration — stateless API key authentication. - *

- * CSRF is disabled (stateless REST API, no browser sessions). - * Sessions are disabled (every request authenticates independently via X-API-Key). - * The {@link ApiKeyFilter} is the sole authentication mechanism, created as a bean - * here and inserted into the security filter chain. - */ @Configuration @EnableWebSecurity public class SecurityConfig { - @Bean - public SecurityFilterChain filterChain(HttpSecurity http, ApiKeyFilter apiKeyFilter) throws Exception { - return http - .csrf(csrf -> csrf.disable()) + public SecurityFilterChain filterChain(HttpSecurity http, ApiKeyFilter keys, ApiErrorWriter errors, Environment env) throws Exception { + return http.csrf(csrf -> csrf.disable()) .sessionManagement(sm -> sm.sessionCreationPolicy(SessionCreationPolicy.STATELESS)) .authorizeHttpRequests(auth -> auth - .requestMatchers("/actuator/**").permitAll() + .requestMatchers(request -> request.getLocalPort() == env.getProperty("local.management.port", Integer.class, + env.getProperty("management.server.port", Integer.class, 9090)) + && request.getLocalPort() != env.getProperty("local.server.port", Integer.class, + env.getProperty("server.port", Integer.class, 8080)) + && request.getRequestURI().equals("/actuator/prometheus")).permitAll() + .requestMatchers("/actuator/health", "/actuator/health/liveness", "/actuator/health/readiness").permitAll() .requestMatchers("/v1/**").authenticated() - .anyRequest().denyAll() - ) - .addFilterBefore(apiKeyFilter, UsernamePasswordAuthenticationFilter.class) - .build(); + .anyRequest().denyAll()) + .exceptionHandling(ex -> ex + .authenticationEntryPoint((req, res, ignored) -> errors.write(req, res, ErrorCode.UNAUTHORIZED)) + .accessDeniedHandler((req, res, ignored) -> errors.write(req, res, ErrorCode.FORBIDDEN))) + .addFilterBefore(keys, UsernamePasswordAuthenticationFilter.class).build(); } - - @Bean - public ApiKeyFilter apiKeyFilter(ApiKeyService apiKeyService, ObjectMapper objectMapper) { - return new ApiKeyFilter(apiKeyService, objectMapper); + @Bean public ApiKeyFilter apiKeyFilter(ApiKeyService keys, ApiErrorWriter errors) { + return new ApiKeyFilter(keys, errors); } - - @Bean - public BCryptPasswordEncoder passwordEncoder() { - return new BCryptPasswordEncoder(); + @Bean public FilterRegistrationBean disableServletRegistration(ApiKeyFilter filter) { + FilterRegistrationBean registration = new FilterRegistrationBean<>(filter); + registration.setEnabled(false); return registration; + } + @Bean public org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder passwordEncoder() { + return new org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder(); } } diff --git a/src/main/java/com/gateway/common/AdmissionFilter.java b/src/main/java/com/gateway/common/AdmissionFilter.java new file mode 100644 index 0000000..9e69313 --- /dev/null +++ b/src/main/java/com/gateway/common/AdmissionFilter.java @@ -0,0 +1,100 @@ +package com.gateway.common; + +import com.gateway.config.GatewayProperties; +import com.gateway.error.ApiErrorWriter; +import com.gateway.error.ErrorCode; +import io.micrometer.core.instrument.MeterRegistry; +import jakarta.servlet.FilterChain; +import jakarta.servlet.ReadListener; +import jakarta.servlet.ServletException; +import jakarta.servlet.ServletInputStream; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletRequestWrapper; +import jakarta.servlet.http.HttpServletResponse; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; +import org.springframework.web.filter.OncePerRequestFilter; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; + +/** Bounded admission before BCrypt/JSON work, including chunked bodies without Content-Length. */ +@Component +@Order(Ordered.HIGHEST_PRECEDENCE + 1) +public class AdmissionFilter extends OncePerRequestFilter { + private final GatewayProperties properties; + private final ApiErrorWriter errors; + private final MeterRegistry meters; + private final Semaphore permits; + public AdmissionFilter(GatewayProperties properties, ApiErrorWriter errors, MeterRegistry meters) { + this.properties = properties; this.errors = errors; this.meters = meters; + this.permits = new Semaphore(properties.getMaxConcurrentRequests()); + meters.gauge("gateway.requests.active", permits, + s -> properties.getMaxConcurrentRequests() - s.availablePermits()); + } + @Override protected boolean shouldNotFilter(HttpServletRequest request) { + return !request.getRequestURI().startsWith("/v1/"); + } + @Override protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, + FilterChain chain) throws IOException, ServletException { + long start = System.nanoTime(); + if (!permits.tryAcquire()) { + errors.write(request, response, ErrorCode.OVERLOADED); + meters.counter("gateway.admission.rejected", "reason", "concurrency").increment(); + return; + } + RequestDeadline.start(properties.getTotalTimeoutMs()); + try { + int limit = properties.getMaxRequestBytes(); + if (request.getContentLengthLong() > limit) { + errors.write(request, response, ErrorCode.PAYLOAD_TOO_LARGE); return; + } + ByteArrayOutputStream buffer = new ByteArrayOutputStream(Math.min(limit, 8192)); + byte[] chunk = new byte[8192]; + var input = request.getInputStream(); + while (buffer.size() <= limit) { + if (RequestDeadline.remainingMillis(properties.getTotalTimeoutMs()) == 0) { + errors.write(request, response, ErrorCode.PROVIDER_TIMEOUT); return; + } + int read = input.read(chunk, 0, Math.min(chunk.length, limit + 1 - buffer.size())); + if (read < 0) break; + buffer.write(chunk, 0, read); + } + byte[] body = buffer.toByteArray(); + if (body.length > limit) { + errors.write(request, response, ErrorCode.PAYLOAD_TOO_LARGE); return; + } + if (RequestDeadline.remainingMillis(properties.getTotalTimeoutMs()) == 0) { + errors.write(request, response, ErrorCode.PROVIDER_TIMEOUT); return; + } + chain.doFilter(new BufferedRequest(request, body), response); + } finally { + RequestDeadline.clear(); permits.release(); + // Only bounded categories: never URL, user key, model or exception text. + meters.timer("gateway.requests", "status", Integer.toString(response.getStatus())) + .record(System.nanoTime() - start, TimeUnit.NANOSECONDS); + } + } + private static final class BufferedRequest extends HttpServletRequestWrapper { + private final byte[] body; + BufferedRequest(HttpServletRequest request, byte[] body) { super(request); this.body = body; } + @Override public ServletInputStream getInputStream() { + ByteArrayInputStream source = new ByteArrayInputStream(body); + return new ServletInputStream() { + @Override public int read() { return source.read(); } + @Override public int read(byte[] b, int off, int len) { return source.read(b, off, len); } + @Override public boolean isFinished() { return source.available() == 0; } + @Override public boolean isReady() { return true; } + @Override public void setReadListener(ReadListener listener) { + throw new IllegalStateException("Async request-body reading is not supported by this endpoint"); + } + }; + } + @Override public BufferedReader getReader() { + return new BufferedReader(new InputStreamReader(getInputStream(), StandardCharsets.UTF_8)); + } + } +} diff --git a/src/main/java/com/gateway/common/RequestDeadline.java b/src/main/java/com/gateway/common/RequestDeadline.java new file mode 100644 index 0000000..ff38959 --- /dev/null +++ b/src/main/java/com/gateway/common/RequestDeadline.java @@ -0,0 +1,17 @@ +package com.gateway.common; + +import java.util.concurrent.TimeUnit; + +/** Monotonic deadline shared by request admission and provider dispatch; never sourced from client headers. */ +public final class RequestDeadline { + private static final ThreadLocal DEADLINE = new ThreadLocal<>(); + private RequestDeadline() {} + public static void start(int timeoutMs) { + DEADLINE.set(System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeoutMs)); + } + public static long remainingMillis(int fallbackMs) { + Long deadline = DEADLINE.get(); + return deadline == null ? fallbackMs : Math.max(0, TimeUnit.NANOSECONDS.toMillis(deadline - System.nanoTime())); + } + public static void clear() { DEADLINE.remove(); } +} diff --git a/src/main/java/com/gateway/config/DatabaseBootstrap.java b/src/main/java/com/gateway/config/DatabaseBootstrap.java new file mode 100644 index 0000000..60f5979 --- /dev/null +++ b/src/main/java/com/gateway/config/DatabaseBootstrap.java @@ -0,0 +1,66 @@ +package com.gateway.config; + +import com.gateway.auth.ApiKey; +import com.gateway.auth.ApiKeyRepository; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.core.env.Environment; +import org.springframework.core.env.Profiles; +import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; +import org.springframework.stereotype.Component; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.support.TransactionTemplate; +import java.time.LocalDateTime; +import java.util.UUID; + +/** Runs before provider registration and readiness. Local convenience requires an explicit profile and opt-in. */ +@Component +public class DatabaseBootstrap implements InitializingBean { + public static final String DEMO_KEY = "test-api-key-1"; + private static final UUID LOCAL_KEY_ID = UUID.fromString("c3d4e5f6-a7b8-9012-cdef-123456789012"); + private final ApiKeyRepository keys; + private final ProviderConfigRepository providers; + private final GatewayProperties properties; + private final Environment environment; + private final BCryptPasswordEncoder encoder; + private final TransactionTemplate transaction; + public DatabaseBootstrap(ApiKeyRepository keys, ProviderConfigRepository providers, + GatewayProperties properties, Environment environment, + BCryptPasswordEncoder encoder, PlatformTransactionManager transactionManager) { + this.keys = keys; this.providers = providers; this.properties = properties; + this.environment = environment; this.encoder = encoder; + this.transaction = new TransactionTemplate(transactionManager); + } + @Override public void afterPropertiesSet() { + boolean local = environment.acceptsProfiles(Profiles.of("local")); + boolean production = environment.acceptsProfiles(Profiles.of("production")); + var bootstrap = properties.getBootstrap(); + if (bootstrap.isLocalEnabled() && (!local || production)) { + throw new IllegalStateException("Local bootstrap requires the local profile without production"); + } + if (production) { + String password = environment.getProperty("spring.datasource.password", ""); + if (password.isBlank() || password.equals("gateway")) { + throw new IllegalStateException("Production requires an explicit non-demo database password"); + } + } + transaction.executeWithoutResult(status -> { + if (bootstrap.isLocalEnabled()) { + String raw = bootstrap.getApiKey(); + if (raw.isBlank() || raw.getBytes(java.nio.charset.StandardCharsets.UTF_8).length > 72) + throw new IllegalStateException("Local bootstrap requires a valid API key"); + keys.save(ApiKey.builder().id(LOCAL_KEY_ID).keyHash(encoder.encode(raw)) + .label("local-bootstrap").active(true).createdAt(LocalDateTime.now()).build()); + providers.findByName("ollama-local").ifPresent(provider -> { + provider.setBaseUrl(bootstrap.getOllamaBaseUrl()); providers.save(provider); + }); + } + if (!bootstrap.isLocalEnabled()) { + for (ApiKey key : keys.findByActiveTrue()) { + if (key.getId().equals(LOCAL_KEY_ID) || encoder.matches(DEMO_KEY, key.getKeyHash())) { + throw new IllegalStateException("Active demo credential detected; deactivate it before starting"); + } + } + } + }); + } +} diff --git a/src/main/java/com/gateway/config/GatewayProperties.java b/src/main/java/com/gateway/config/GatewayProperties.java new file mode 100644 index 0000000..4c2896c --- /dev/null +++ b/src/main/java/com/gateway/config/GatewayProperties.java @@ -0,0 +1,40 @@ +package com.gateway.config; + +import jakarta.validation.Valid; +import jakarta.validation.constraints.Max; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotBlank; +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.validation.annotation.Validated; + +/** Explicit resource limits. Durations use milliseconds for unambiguous deployment configuration. */ +@Getter +@Setter +@Validated +@ConfigurationProperties("gateway") +public class GatewayProperties { + @Min(1) @Max(16777216) private int maxRequestBytes = 1048576; + @Min(1) @Max(1024) private int maxConcurrentRequests = 64; + @Min(1) @Max(600000) private int totalTimeoutMs = 120000; + @Valid private Transport transport = new Transport(); + @Valid private Bootstrap bootstrap = new Bootstrap(); + + @Getter @Setter + public static class Transport { + @Min(1) @Max(60000) private int connectTimeoutMs = 2000; + @Min(1) @Max(600000) private int readTimeoutMs = 30000; + @Min(1) @Max(60000) private int poolAcquireTimeoutMs = 250; + @Min(1) @Max(512) private int maxConnections = 32; + @Min(1) @Max(512) private int maxConnectionsPerRoute = 8; + @Min(1) @Max(16777216) private int maxResponseBytes = 2097152; + } + + @Getter @Setter + public static class Bootstrap { + private boolean localEnabled; + private String apiKey = ""; + @NotBlank private String ollamaBaseUrl = "http://localhost:11434"; + } +} diff --git a/src/main/java/com/gateway/error/ApiErrorWriter.java b/src/main/java/com/gateway/error/ApiErrorWriter.java new file mode 100644 index 0000000..856fb22 --- /dev/null +++ b/src/main/java/com/gateway/error/ApiErrorWriter.java @@ -0,0 +1,21 @@ +package com.gateway.error; + +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.springframework.http.MediaType; +import org.springframework.stereotype.Component; +import tools.jackson.databind.ObjectMapper; +import java.io.IOException; + +/** Shared error serialization for failures that occur before MVC. */ +@Component +public class ApiErrorWriter { + private final ObjectMapper mapper; + public ApiErrorWriter(ObjectMapper mapper) { this.mapper = mapper; } + public void write(HttpServletRequest request, HttpServletResponse response, ErrorCode code) throws IOException { + response.setStatus(code.getStatusCode()); + response.setContentType(MediaType.APPLICATION_JSON_VALUE); + mapper.writeValue(response.getOutputStream(), ApiError.of(code, code.safeMessage(), + request.getRequestURI(), String.valueOf(request.getAttribute("requestId")))); + } +} diff --git a/src/main/java/com/gateway/error/ErrorCode.java b/src/main/java/com/gateway/error/ErrorCode.java index 7c98a57..481d347 100644 --- a/src/main/java/com/gateway/error/ErrorCode.java +++ b/src/main/java/com/gateway/error/ErrorCode.java @@ -13,12 +13,18 @@ public enum ErrorCode { INVALID_REQUEST(HttpStatus.BAD_REQUEST), PROVIDER_NOT_FOUND(HttpStatus.NOT_FOUND), - PROVIDER_UNAVAILABLE(HttpStatus.BAD_GATEWAY), + PROVIDER_UNAVAILABLE(HttpStatus.SERVICE_UNAVAILABLE), MODEL_NOT_FOUND(HttpStatus.NOT_FOUND), PROVIDER_TIMEOUT(HttpStatus.GATEWAY_TIMEOUT), INTERNAL_ERROR(HttpStatus.INTERNAL_SERVER_ERROR), BAD_CONFIGURATION(HttpStatus.INTERNAL_SERVER_ERROR), - UNAUTHORIZED(HttpStatus.UNAUTHORIZED); + UNAUTHORIZED(HttpStatus.UNAUTHORIZED), + FORBIDDEN(HttpStatus.FORBIDDEN), + PAYLOAD_TOO_LARGE(HttpStatus.PAYLOAD_TOO_LARGE), + UNSUPPORTED_MEDIA_TYPE(HttpStatus.UNSUPPORTED_MEDIA_TYPE), + METHOD_NOT_ALLOWED(HttpStatus.METHOD_NOT_ALLOWED), + RESOURCE_NOT_FOUND(HttpStatus.NOT_FOUND), + OVERLOADED(HttpStatus.SERVICE_UNAVAILABLE); private final HttpStatus httpStatus; @@ -33,4 +39,23 @@ public HttpStatus getHttpStatus() { public int getStatusCode() { return httpStatus.value(); } + + public String safeMessage() { + return switch (this) { + case INVALID_REQUEST -> "Invalid request parameters"; + case PROVIDER_NOT_FOUND -> "Provider not found or not active"; + case MODEL_NOT_FOUND -> "Model not found or not active"; + case PROVIDER_UNAVAILABLE -> "Provider is unavailable"; + case PROVIDER_TIMEOUT -> "Request deadline or provider timeout exceeded"; + case BAD_CONFIGURATION -> "Gateway configuration is invalid"; + case UNAUTHORIZED -> "Missing or invalid API key"; + case FORBIDDEN -> "Access denied"; + case PAYLOAD_TOO_LARGE -> "Request body exceeds the configured limit"; + case UNSUPPORTED_MEDIA_TYPE -> "Unsupported media type"; + case METHOD_NOT_ALLOWED -> "HTTP method is not supported"; + case RESOURCE_NOT_FOUND -> "Resource not found"; + case OVERLOADED -> "Gateway capacity is exhausted"; + case INTERNAL_ERROR -> "An unexpected error occurred"; + }; + } } diff --git a/src/main/java/com/gateway/error/GlobalExceptionHandler.java b/src/main/java/com/gateway/error/GlobalExceptionHandler.java index c14a4e6..bcbac37 100644 --- a/src/main/java/com/gateway/error/GlobalExceptionHandler.java +++ b/src/main/java/com/gateway/error/GlobalExceptionHandler.java @@ -1,111 +1,76 @@ package com.gateway.error; import jakarta.servlet.http.HttpServletRequest; +import jakarta.validation.ConstraintViolationException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; +import org.springframework.http.converter.HttpMessageNotReadableException; import org.springframework.security.access.AccessDeniedException; +import org.springframework.web.HttpMediaTypeNotSupportedException; +import org.springframework.web.HttpRequestMethodNotSupportedException; import org.springframework.web.bind.MethodArgumentNotValidException; +import org.springframework.web.bind.ServletRequestBindingException; import org.springframework.web.bind.annotation.ExceptionHandler; import org.springframework.web.bind.annotation.RestControllerAdvice; +import org.springframework.web.method.annotation.HandlerMethodValidationException; +import org.springframework.web.method.annotation.MethodArgumentTypeMismatchException; +import org.springframework.web.servlet.resource.NoResourceFoundException; -/** - * Central exception handler — converts all exceptions into standardized - * {@link ApiError} responses. No ad-hoc try/catch blocks in controllers (Rules.md §4). - */ +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.Set; + +/** Public errors and application logs never include exception messages or upstream payloads. */ @RestControllerAdvice public class GlobalExceptionHandler { - private static final Logger log = LoggerFactory.getLogger(GlobalExceptionHandler.class); - /** - * Handles all gateway business errors (routing, provider, auth, etc.). - */ @ExceptionHandler(GatewayException.class) public ResponseEntity handleGatewayException(GatewayException ex, HttpServletRequest request) { - ErrorCode code = ex.getErrorCode(); - String requestId = getRequestId(request); - - log.warn("Gateway error [{}] requestId={}: {}", code, requestId, ex.getMessage()); - - ApiError error = ApiError.of(code, ex.getMessage(), request.getRequestURI(), requestId); - return ResponseEntity.status(code.getHttpStatus()).body(error); + return error(ex.getErrorCode(), request); } - /** - * Handles Jakarta validation failures (e.g., @NotBlank on InferenceRequest fields). - */ - @ExceptionHandler(MethodArgumentNotValidException.class) - public ResponseEntity handleValidationException(MethodArgumentNotValidException ex, HttpServletRequest request) { - String requestId = getRequestId(request); + @ExceptionHandler({MethodArgumentNotValidException.class, HttpMessageNotReadableException.class, + MethodArgumentTypeMismatchException.class, ServletRequestBindingException.class, + HandlerMethodValidationException.class, ConstraintViolationException.class}) + public ResponseEntity invalidRequest(Exception ex, HttpServletRequest request) { + return error(ErrorCode.INVALID_REQUEST, request); + } - String message = ex.getBindingResult().getFieldErrors().stream() - .map(fe -> fe.getField() + ": " + fe.getDefaultMessage()) - .reduce((a, b) -> a + "; " + b) - .orElse("Validation failed"); + @ExceptionHandler(HttpMediaTypeNotSupportedException.class) + public ResponseEntity unsupportedMedia(Exception ex, HttpServletRequest request) { + return error(ErrorCode.UNSUPPORTED_MEDIA_TYPE, request); + } - log.warn("Validation error requestId={}: {}", requestId, message); + @ExceptionHandler(HttpRequestMethodNotSupportedException.class) + public ResponseEntity unsupportedMethod(Exception ex, HttpServletRequest request) { + return error(ErrorCode.METHOD_NOT_ALLOWED, request); + } - ApiError error = ApiError.of(ErrorCode.INVALID_REQUEST, message, request.getRequestURI(), requestId); - return ResponseEntity.status(ErrorCode.INVALID_REQUEST.getHttpStatus()).body(error); + @ExceptionHandler(NoResourceFoundException.class) + public ResponseEntity notFound(Exception ex, HttpServletRequest request) { + return error(ErrorCode.RESOURCE_NOT_FOUND, request); } - /** - * Handles Spring Security access denied errors (e.g., authenticated but not authorized). - * Returns 403 using the standard ApiError schema. - */ @ExceptionHandler(AccessDeniedException.class) public ResponseEntity handleAccessDenied(AccessDeniedException ex, HttpServletRequest request) { - String requestId = getRequestId(request); - - log.warn("Access denied requestId={}: {}", requestId, ex.getMessage()); - - ApiError error = ApiError.of(ErrorCode.UNAUTHORIZED, "Access denied", request.getRequestURI(), requestId); - return ResponseEntity.status(HttpStatus.FORBIDDEN).body(error); + return error(ErrorCode.FORBIDDEN, request); } - /** - * Catch-all for any unhandled exception — never leaks stack traces (Rules.md §4). - * Also unwraps wrapped exceptions to check for GatewayException causes. - */ @ExceptionHandler(Exception.class) public ResponseEntity handleGenericException(Exception ex, HttpServletRequest request) { - // Check if the root cause is a GatewayException (can happen when Spring wraps exceptions) - GatewayException gatewayException = findGatewayException(ex); - if (gatewayException != null) { - return handleGatewayException(gatewayException, request); + Set visited = Collections.newSetFromMap(new IdentityHashMap<>()); + for (Throwable cause = ex; cause != null && visited.add(cause); cause = cause.getCause()) { + if (cause instanceof GatewayException gateway) return error(gateway.getErrorCode(), request); } - - String requestId = getRequestId(request); - - log.error("Unexpected error requestId={}: {}", requestId, ex.getMessage(), ex); - - ApiError error = ApiError.of( - ErrorCode.INTERNAL_ERROR, - "An unexpected error occurred", - request.getRequestURI(), - requestId - ); - return ResponseEntity.status(ErrorCode.INTERNAL_ERROR.getHttpStatus()).body(error); + return error(ErrorCode.INTERNAL_ERROR, request); } - private String getRequestId(HttpServletRequest request) { - Object requestId = request.getAttribute("requestId"); - return requestId != null ? requestId.toString() : "unknown"; - } - - /** - * Walk the exception cause chain to find a GatewayException, if present. - */ - private GatewayException findGatewayException(Throwable ex) { - Throwable current = ex; - while (current != null) { - if (current instanceof GatewayException ge) { - return ge; - } - current = current.getCause(); - } - return null; + private ResponseEntity error(ErrorCode code, HttpServletRequest request) { + String id = String.valueOf(request.getAttribute("requestId")); + log.debug("Request rejected requestId={} code={}", id, code); + return ResponseEntity.status(code.getHttpStatus()) + .body(ApiError.of(code, code.safeMessage(), request.getRequestURI(), id)); } } diff --git a/src/main/java/com/gateway/inference/InferenceRequest.java b/src/main/java/com/gateway/inference/InferenceRequest.java index d010982..b3a7c0e 100644 --- a/src/main/java/com/gateway/inference/InferenceRequest.java +++ b/src/main/java/com/gateway/inference/InferenceRequest.java @@ -1,6 +1,7 @@ package com.gateway.inference; import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.Size; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Getter; @@ -23,17 +24,20 @@ public class InferenceRequest { * Target provider name (e.g., "ollama-local"). Optional — if omitted, * the routing engine uses the active default provider. */ + @Size(max = 200) private String provider; /** * Model name to use for inference (e.g., "qwen3"). */ @NotBlank(message = "model is required") + @Size(max = 200) private String model; /** * The prompt to send to the AI model. */ @NotBlank(message = "prompt is required") + @Size(max = 262144) private String prompt; } diff --git a/src/main/java/com/gateway/inference/InferenceService.java b/src/main/java/com/gateway/inference/InferenceService.java index c6790e9..3222ece 100644 --- a/src/main/java/com/gateway/inference/InferenceService.java +++ b/src/main/java/com/gateway/inference/InferenceService.java @@ -1,84 +1,60 @@ package com.gateway.inference; +import com.gateway.error.ErrorCode; import com.gateway.error.GatewayException; import com.gateway.logging.RequestLogService; import com.gateway.routing.RoutingEngine; import com.gateway.routing.RoutingEngine.RoutingResult; +import io.micrometer.core.instrument.MeterRegistry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; +import java.util.concurrent.TimeUnit; -/** - * Core orchestration service implementing the full inference flow: - *

    - *
  1. Resolve provider via {@link RoutingEngine}
    2. - *
  2. Call provider adapter ({@link com.gateway.provider.AIProvider#infer})
    3. - *
  3. Stamp {@code requestId} and end-to-end latency onto the response
    4. - *
  4. Log the outcome (success or failure) to {@code request_logs}
    5. - *
- * This is the core value delivery of the MVP — wiring everything together. - */ +/** Owns one terminal diagnostic outcome for every request admitted to inference, including routing errors. */ @Service public class InferenceService { - private static final Logger log = LoggerFactory.getLogger(InferenceService.class); - private final RoutingEngine routingEngine; private final RequestLogService requestLogService; + private final MeterRegistry meters; - public InferenceService(RoutingEngine routingEngine, RequestLogService requestLogService) { - this.routingEngine = routingEngine; - this.requestLogService = requestLogService; + public InferenceService(RoutingEngine routingEngine, RequestLogService requestLogService, MeterRegistry meters) { + this.routingEngine = routingEngine; this.requestLogService = requestLogService; this.meters = meters; } - /** - * Execute the full inference pipeline. - * - * @param request the validated inference request - * @param requestId the unique request tracking ID from {@link com.gateway.common.RequestIdFilter} - * @return the enriched response with requestId and end-to-end latency - * @throws GatewayException re-thrown after logging so GlobalExceptionHandler produces ApiError - */ public InferenceResponse infer(InferenceRequest request, String requestId) { - long startTime = System.currentTimeMillis(); - - // 1. Resolve provider via routing engine - RoutingResult routing = routingEngine.resolve(request); - - log.info("Inference request {} → provider={}, model={}", - requestId, routing.providerName(), request.getModel()); - + long start = System.nanoTime(); + String provider = null; + ErrorCode failure = null; try { - // 2. Call provider adapter - InferenceResponse providerResponse = routing.provider().infer(request); - long latencyMs = System.currentTimeMillis() - startTime; - - // 3. Stamp requestId and end-to-end latency - InferenceResponse enriched = InferenceResponse.builder() - .requestId(requestId) - .text(providerResponse.getText()) - .model(providerResponse.getModel()) - .provider(providerResponse.getProvider()) - .latencyMs(latencyMs) - .build(); - - // 4. Log success - requestLogService.log(requestId, routing.providerName(), request.getModel(), - "SUCCESS", null, (int) latencyMs); - - log.info("Inference request {} completed in {}ms", requestId, latencyMs); - return enriched; - + RoutingResult routing = routingEngine.resolve(request); + provider = routing.providerName(); + InferenceResponse result = routing.provider().infer(request); + return InferenceResponse.builder().requestId(requestId).text(result.getText()) + .model(result.getModel()).provider(result.getProvider()).latencyMs(elapsed(start)).build(); } catch (GatewayException ex) { - long latencyMs = System.currentTimeMillis() - startTime; - - // 4. Log failure - requestLogService.log(requestId, routing.providerName(), request.getModel(), - "FAILURE", ex.getErrorCode().name(), (int) latencyMs); - - log.warn("Inference request {} failed in {}ms: {} — {}", - requestId, latencyMs, ex.getErrorCode(), ex.getMessage()); - throw ex; // re-throw for GlobalExceptionHandler + failure = ex.getErrorCode(); + throw ex; + } catch (RuntimeException ex) { + failure = ErrorCode.INTERNAL_ERROR; + // Do not pass unknown exception messages into normal response or diagnostic logs. + throw new GatewayException(failure, failure.safeMessage()); + } finally { + String outcome = failure == null ? "SUCCESS" : "FAILURE"; + meters.counter("gateway.inference.outcomes", "outcome", outcome, + "error", failure == null ? "none" : failure.name()).increment(); + meters.timer("gateway.inference.duration", "outcome", outcome) + .record(System.nanoTime() - start, TimeUnit.NANOSECONDS); + try { + requestLogService.log(requestId, provider, request.getModel(), outcome, + failure == null ? null : failure.name(), (int) Math.min(Integer.MAX_VALUE, elapsed(start))); + } catch (RuntimeException loggingFailure) { + meters.counter("gateway.diagnostics.dropped").increment(); + log.warn("Diagnostic persistence failed requestId={}", requestId); + } } } + + private long elapsed(long start) { return TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start); } } diff --git a/src/main/java/com/gateway/logging/RequestLogService.java b/src/main/java/com/gateway/logging/RequestLogService.java index 87efa3f..36ba4e2 100644 --- a/src/main/java/com/gateway/logging/RequestLogService.java +++ b/src/main/java/com/gateway/logging/RequestLogService.java @@ -7,6 +7,8 @@ import org.springframework.data.domain.Sort; import org.springframework.data.jpa.domain.Specification; import org.springframework.stereotype.Service; +import com.gateway.error.ErrorCode; +import com.gateway.error.GatewayException; import java.time.LocalDateTime; import java.util.UUID; @@ -69,10 +71,15 @@ public void log(String requestId, String provider, String model, */ public Page getLogs(int page, int size, String provider, String status, LocalDateTime from, LocalDateTime to) { + if (page < 0 || size < 1 || (provider != null && provider.length() > 200) + || (status != null && !status.equals("SUCCESS") && !status.equals("FAILURE")) + || (from != null && to != null && from.isAfter(to))) { + throw new GatewayException(ErrorCode.INVALID_REQUEST, ErrorCode.INVALID_REQUEST.safeMessage()); + } int cappedSize = Math.min(size, 100); PageRequest pageRequest = PageRequest.of(page, cappedSize, Sort.by(Sort.Direction.DESC, "timestamp")); - Specification spec = Specification.where(null); + Specification spec = Specification.unrestricted(); if (provider != null && !provider.isBlank()) { spec = spec.and((root, query, cb) -> cb.equal(root.get("provider"), provider)); diff --git a/src/main/java/com/gateway/provider/ProviderRegistry.java b/src/main/java/com/gateway/provider/ProviderRegistry.java index bcac5a8..88c24da 100644 --- a/src/main/java/com/gateway/provider/ProviderRegistry.java +++ b/src/main/java/com/gateway/provider/ProviderRegistry.java @@ -8,7 +8,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; -import org.springframework.web.client.RestClient; +import tools.jackson.databind.ObjectMapper; +import org.springframework.context.annotation.DependsOn; import java.util.Collections; import java.util.LinkedHashMap; @@ -22,22 +23,25 @@ * appropriate adapter for each based on the provider type. */ @Service +@DependsOn("databaseBootstrap") public class ProviderRegistry { private static final Logger log = LoggerFactory.getLogger(ProviderRegistry.class); private final ProviderConfigRepository providerConfigRepository; private final ModelConfigRepository modelConfigRepository; - private final RestClient.Builder restClientBuilder; + private final ProviderTransport transport; + private final ObjectMapper mapper; private Map providers = Collections.emptyMap(); public ProviderRegistry(ProviderConfigRepository providerConfigRepository, ModelConfigRepository modelConfigRepository, - RestClient.Builder restClientBuilder) { + ProviderTransport transport, ObjectMapper mapper) { this.providerConfigRepository = providerConfigRepository; this.modelConfigRepository = modelConfigRepository; - this.restClientBuilder = restClientBuilder; + this.transport = transport; + this.mapper = mapper; } @PostConstruct @@ -80,7 +84,7 @@ public Map getAllProviders() { private AIProvider createAdapter(ProviderConfig config) { return switch (config.getType().toLowerCase()) { - case "ollama" -> new OllamaProvider(config, modelConfigRepository, restClientBuilder); + case "ollama" -> new OllamaProvider(config, modelConfigRepository, transport, mapper); default -> { log.warn("Unknown provider type '{}' for provider '{}' — skipping", config.getType(), config.getName()); diff --git a/src/main/java/com/gateway/provider/ProviderTransport.java b/src/main/java/com/gateway/provider/ProviderTransport.java new file mode 100644 index 0000000..8c3cf62 --- /dev/null +++ b/src/main/java/com/gateway/provider/ProviderTransport.java @@ -0,0 +1,103 @@ +package com.gateway.provider; + +import com.gateway.common.RequestDeadline; +import com.gateway.config.GatewayProperties; +import com.gateway.error.ErrorCode; +import com.gateway.error.GatewayException; +import io.micrometer.core.instrument.MeterRegistry; +import jakarta.annotation.PreDestroy; +import org.apache.hc.client5.http.classic.methods.HttpUriRequestBase; +import org.apache.hc.client5.http.config.ConnectionConfig; +import org.apache.hc.client5.http.config.RequestConfig; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.HttpClients; +import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder; +import org.apache.hc.core5.http.ContentType; +import org.apache.hc.core5.http.io.entity.ByteArrayEntity; +import org.apache.hc.core5.pool.PoolConcurrencyPolicy; +import org.apache.hc.core5.util.Timeout; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.net.SocketTimeoutException; +import java.net.URI; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; + +/** Shared bounded HTTP pool. No automatic retry, redirect, cookies or raw response error logging. */ +@Component +public class ProviderTransport implements AutoCloseable { + private final GatewayProperties properties; + private final MeterRegistry meters; + private final CloseableHttpClient client; + private final ScheduledThreadPoolExecutor deadlines; + public ProviderTransport(GatewayProperties properties, MeterRegistry meters) { + this.properties = properties; this.meters = meters; + var config = properties.getTransport(); + var pool = PoolingHttpClientConnectionManagerBuilder.create() + .setPoolConcurrencyPolicy(PoolConcurrencyPolicy.STRICT) + .setMaxConnTotal(config.getMaxConnections()) + .setMaxConnPerRoute(config.getMaxConnectionsPerRoute()) + .setDefaultConnectionConfig(ConnectionConfig.custom() + .setConnectTimeout(Timeout.ofMilliseconds(config.getConnectTimeoutMs())) + .setSocketTimeout(Timeout.ofMilliseconds(config.getReadTimeoutMs())).build()).build(); + client = HttpClients.custom().setConnectionManager(pool) + .setDefaultRequestConfig(RequestConfig.custom() + .setConnectionRequestTimeout(Timeout.ofMilliseconds(config.getPoolAcquireTimeoutMs())) + .setResponseTimeout(Timeout.ofMilliseconds(config.getReadTimeoutMs())).build()) + .disableAutomaticRetries().disableRedirectHandling().disableCookieManagement().build(); + deadlines = new ScheduledThreadPoolExecutor(1, Thread.ofPlatform().daemon().name("provider-deadlines").factory()); + deadlines.setRemoveOnCancelPolicy(true); + } + + public byte[] exchange(String method, URI uri, byte[] body) { + long remaining = RequestDeadline.remainingMillis(properties.getTotalTimeoutMs()); + if (remaining <= 0) throw failure(ErrorCode.PROVIDER_TIMEOUT); + HttpUriRequestBase request = new HttpUriRequestBase(method, uri); + if (body != null) request.setEntity(new ByteArrayEntity(body, ContentType.APPLICATION_JSON)); + AtomicBoolean expired = new AtomicBoolean(); + ScheduledFuture cancellation = deadlines.schedule(() -> { + expired.set(true); request.cancel(); + }, remaining, TimeUnit.MILLISECONDS); + long start = System.nanoTime(); + String outcome = "success"; + try { + byte[] result = client.execute(request, response -> { + if (response.getCode() < 200 || response.getCode() >= 300) { + // Abort instead of consuming an arbitrary upstream error body. + request.cancel(); + throw failure(ErrorCode.PROVIDER_UNAVAILABLE); + } + if (response.getEntity() == null) throw failure(ErrorCode.PROVIDER_UNAVAILABLE); + int max = properties.getTransport().getMaxResponseBytes(); + try (var stream = response.getEntity().getContent()) { + byte[] bytes = stream.readNBytes(max + 1); + if (bytes.length > max) { + request.cancel(); throw failure(ErrorCode.PROVIDER_UNAVAILABLE); + } + return bytes; + } + }); + if (expired.get()) throw failure(ErrorCode.PROVIDER_TIMEOUT); + return result; + } catch (IOException ex) { + outcome = expired.get() || hasTimeout(ex) ? "timeout" : "unavailable"; + throw failure(outcome.equals("timeout") ? ErrorCode.PROVIDER_TIMEOUT : ErrorCode.PROVIDER_UNAVAILABLE); + } catch (GatewayException ex) { + outcome = expired.get() || ex.getErrorCode() == ErrorCode.PROVIDER_TIMEOUT ? "timeout" : "unavailable"; + throw failure(outcome.equals("timeout") ? ErrorCode.PROVIDER_TIMEOUT : ex.getErrorCode()); + } finally { + cancellation.cancel(false); + meters.timer("gateway.provider.duration", "outcome", outcome) + .record(System.nanoTime() - start, TimeUnit.NANOSECONDS); + } + } + private boolean hasTimeout(Throwable error) { + for (Throwable cause = error; cause != null; cause = cause.getCause()) { + if (cause instanceof SocketTimeoutException || cause instanceof TimeoutException) return true; + } + return false; + } + private GatewayException failure(ErrorCode code) { return new GatewayException(code, code.safeMessage()); } + @Override @PreDestroy public void close() throws IOException { deadlines.shutdownNow(); client.close(); } +} diff --git a/src/main/java/com/gateway/provider/ollama/OllamaProvider.java b/src/main/java/com/gateway/provider/ollama/OllamaProvider.java index 75dc9ac..3e29827 100644 --- a/src/main/java/com/gateway/provider/ollama/OllamaProvider.java +++ b/src/main/java/com/gateway/provider/ollama/OllamaProvider.java @@ -7,113 +7,57 @@ import com.gateway.error.GatewayException; import com.gateway.inference.InferenceRequest; import com.gateway.inference.InferenceResponse; -import com.gateway.provider.AIProvider; -import com.gateway.provider.HealthStatus; -import com.gateway.provider.ProviderInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.web.client.ResourceAccessException; -import org.springframework.web.client.RestClient; - -import java.util.List; +import com.gateway.provider.*; +import tools.jackson.databind.ObjectMapper; +import java.net.URI; import java.util.Map; +import java.util.concurrent.TimeUnit; -/** - * {@link AIProvider} implementation for Ollama. - * Calls the local Ollama HTTP API via Spring {@link RestClient}. - */ +/** Typed Ollama boundary using bounded transport; never retains raw provider errors. */ public class OllamaProvider implements AIProvider { - - private static final Logger log = LoggerFactory.getLogger(OllamaProvider.class); - private final ProviderConfig config; - private final ModelConfigRepository modelConfigRepository; - private final RestClient restClient; - - public OllamaProvider(ProviderConfig config, - ModelConfigRepository modelConfigRepository, - RestClient.Builder restClientBuilder) { - this.config = config; - this.modelConfigRepository = modelConfigRepository; - this.restClient = restClientBuilder - .baseUrl(config.getBaseUrl()) - .build(); + private final ModelConfigRepository models; + private final ProviderTransport transport; + private final ObjectMapper mapper; + public OllamaProvider(ProviderConfig config, ModelConfigRepository models, + ProviderTransport transport, ObjectMapper mapper) { + this.config = config; this.models = models; this.transport = transport; this.mapper = mapper; + URI endpoint; + try { + endpoint = URI.create(config.getBaseUrl()); + } catch (IllegalArgumentException | NullPointerException invalidEndpoint) { + throw new GatewayException(ErrorCode.BAD_CONFIGURATION, ErrorCode.BAD_CONFIGURATION.safeMessage()); + } + if (!("http".equals(endpoint.getScheme()) || "https".equals(endpoint.getScheme())) + || endpoint.getHost() == null || endpoint.getUserInfo() != null + || endpoint.getQuery() != null || endpoint.getFragment() != null) { + throw new GatewayException(ErrorCode.BAD_CONFIGURATION, ErrorCode.BAD_CONFIGURATION.safeMessage()); + } } - - @Override - public InferenceResponse infer(InferenceRequest request) { - long startTime = System.currentTimeMillis(); - + @Override public InferenceResponse infer(InferenceRequest request) { + long start = System.nanoTime(); + byte[] body = mapper.writeValueAsBytes(Map.of("model", request.getModel(), "prompt", request.getPrompt(), "stream", false)); + byte[] bytes = transport.exchange("POST", endpoint("/api/generate"), body); try { - Map ollamaRequest = Map.of( - "model", request.getModel(), - "prompt", request.getPrompt(), - "stream", false - ); - - @SuppressWarnings("unchecked") - Map ollamaResponse = restClient.post() - .uri("/api/generate") - .body(ollamaRequest) - .retrieve() - .body(Map.class); - - long latencyMs = System.currentTimeMillis() - startTime; - - if (ollamaResponse == null) { - throw new GatewayException(ErrorCode.PROVIDER_UNAVAILABLE, - "Ollama returned an empty response"); - } - - String text = (String) ollamaResponse.get("response"); - - return InferenceResponse.builder() - .text(text) - .model(request.getModel()) - .provider(config.getName()) - .latencyMs(latencyMs) - .build(); - - } catch (ResourceAccessException ex) { - long latencyMs = System.currentTimeMillis() - startTime; - log.error("Ollama connection error after {}ms: {}", latencyMs, ex.getMessage()); - - if (ex.getMessage() != null && ex.getMessage().contains("timed out")) { - throw new GatewayException(ErrorCode.PROVIDER_TIMEOUT, - "Ollama request timed out", ex); + var json = mapper.readTree(bytes); + if (json == null || !json.path("response").isString() || !json.path("done").asBoolean(false)) { + throw new GatewayException(ErrorCode.PROVIDER_UNAVAILABLE, ErrorCode.PROVIDER_UNAVAILABLE.safeMessage()); } - throw new GatewayException(ErrorCode.PROVIDER_UNAVAILABLE, - "Ollama is not reachable: " + ex.getMessage(), ex); + return InferenceResponse.builder().text(json.get("response").asString()).model(request.getModel()) + .provider(config.getName()).latencyMs(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)).build(); + } catch (GatewayException ex) { + throw ex; + } catch (RuntimeException ex) { + throw new GatewayException(ErrorCode.PROVIDER_UNAVAILABLE, ErrorCode.PROVIDER_UNAVAILABLE.safeMessage()); } } - - @Override - public ProviderInfo info() { - List modelNames = modelConfigRepository - .findByProviderIdAndActiveTrue(config.getId()) - .stream() - .map(ModelConfig::getName) - .toList(); - - return new ProviderInfo( - config.getName(), - config.getType(), - config.getBaseUrl(), - modelNames - ); + @Override public ProviderInfo info() { + return new ProviderInfo(config.getName(), config.getType(), config.getBaseUrl(), + models.findByProviderIdAndActiveTrue(config.getId()).stream().map(ModelConfig::getName).toList()); } - - @Override - public HealthStatus health() { - try { - restClient.get() - .uri("/") - .retrieve() - .toBodilessEntity(); - return HealthStatus.up(); - } catch (Exception ex) { - log.warn("Ollama health check failed: {}", ex.getMessage()); - return HealthStatus.down(ex.getMessage()); - } + @Override public HealthStatus health() { + try { transport.exchange("GET", endpoint("/"), null); return HealthStatus.up(); } + catch (RuntimeException ex) { return HealthStatus.down("Provider probe failed"); } } + private URI endpoint(String path) { return URI.create(config.getBaseUrl().replaceAll("/+$", "") + path); } } diff --git a/src/main/resources/application-local.yml b/src/main/resources/application-local.yml new file mode 100644 index 0000000..77c1c61 --- /dev/null +++ b/src/main/resources/application-local.yml @@ -0,0 +1,9 @@ +# Explicitly opted-in demo settings. Never combine this profile with production. +spring: + datasource: + password: ${DB_PASSWORD:gateway} +gateway: + bootstrap: + local-enabled: true + api-key: ${LOCAL_API_KEY:test-api-key-1} + ollama-base-url: ${OLLAMA_BASE_URL:http://localhost:11434} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 64279a4..223da4f 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -1,34 +1,82 @@ spring: application: name: ai-inference-gateway - + profiles: + default: production datasource: - url: jdbc:postgresql://${DB_HOST:localhost}:${DB_PORT:5432}/${DB_NAME:gateway_db} + url: jdbc:postgresql://${DB_HOST:localhost}:${DB_PORT:5432}/${DB_NAME:gateway_db}?connectTimeout=3&socketTimeout=5 username: ${DB_USERNAME:gateway} - password: ${DB_PASSWORD:gateway} - + password: ${DB_PASSWORD:} + hikari: + maximum-pool-size: 10 + connection-timeout: 2000 + validation-timeout: 1000 + connection-init-sql: SET statement_timeout = 2000 jpa: hibernate: ddl-auto: validate open-in-view: false - + properties: + jakarta.persistence.query.timeout: 2000 + transaction: + default-timeout: 5s flyway: enabled: true locations: classpath:db/migration server: - port: 8080 + port: ${SERVER_PORT:8080} + shutdown: graceful + max-http-request-header-size: 16KB + tomcat: + threads: + max: 100 + min-spare: 10 + max-connections: 256 + accept-count: 32 + connection-timeout: 5s + error: + include-message: never + include-stacktrace: never + include-binding-errors: never -# ── Custom Properties ──────────────────────────────── -ollama: - base-url: ${OLLAMA_BASE_URL:http://localhost:11434} +gateway: + max-request-bytes: 1048576 + max-concurrent-requests: 64 + total-timeout-ms: 120000 + transport: + connect-timeout-ms: 2000 + read-timeout-ms: 30000 + pool-acquire-timeout-ms: 250 + max-connections: 32 + max-connections-per-route: 8 + max-response-bytes: 2097152 + bootstrap: + local-enabled: false -# ── Actuator ───────────────────────────────────────── management: + server: + port: ${MANAGEMENT_PORT:9090} + address: ${MANAGEMENT_ADDRESS:127.0.0.1} endpoints: web: exposure: - include: health + include: health,prometheus endpoint: health: - show-details: always + show-details: never + probes: + enabled: true + group: + readiness: + include: readinessState,db + metrics: + distribution: + percentiles-histogram: + gateway.requests: true + gateway.provider.duration: true + +logging: + level: + org.apache.hc.client5.http: WARN + org.apache.hc.core5.http: WARN diff --git a/src/main/resources/db/migration/V6__disable_demo_credentials.sql b/src/main/resources/db/migration/V6__disable_demo_credentials.sql new file mode 100644 index 0000000..9a7ba1d --- /dev/null +++ b/src/main/resources/db/migration/V6__disable_demo_credentials.sql @@ -0,0 +1,5 @@ +-- Preserve V1-V5 checksums. Disable the public MVP credential and its known hash. +-- Startup additionally detects rehashed copies by verifying active hashes. +UPDATE api_keys SET active = FALSE +WHERE id = 'c3d4e5f6-a7b8-9012-cdef-123456789012' + OR key_hash = '$2b$10$iemW2pbe8LCoClCT4DdP2ulCHQey3p3JfzX3bM0uFvXWLzU43HyuK'; diff --git a/src/test/java/com/gateway/api/InferenceControllerIntegrationTest.java b/src/test/java/com/gateway/api/InferenceControllerIntegrationTest.java index 881c132..0f12e4e 100644 --- a/src/test/java/com/gateway/api/InferenceControllerIntegrationTest.java +++ b/src/test/java/com/gateway/api/InferenceControllerIntegrationTest.java @@ -13,7 +13,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.webmvc.test.autoconfigure.AutoConfigureMockMvc; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.http.MediaType; import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; @@ -134,6 +134,43 @@ void inference_happyPath_returns200WithResponse() throws Exception { .andExpect(jsonPath("$.latencyMs").isNumber()); } + @Test void malformedJsonReturnsSafe400() throws Exception { + mockMvc.perform(post(INFERENCE_URL).header("X-API-Key", VALID_RAW_KEY) + .contentType(MediaType.APPLICATION_JSON).content("{SECRET_BODY")) + .andExpect(status().isBadRequest()) + .andExpect(jsonPath("$.message").value("Invalid request parameters")); + } + + @Test void unsupportedMediaReturns415() throws Exception { + mockMvc.perform(post(INFERENCE_URL).header("X-API-Key", VALID_RAW_KEY) + .contentType(MediaType.TEXT_PLAIN).content("SECRET_BODY")) + .andExpect(status().isUnsupportedMediaType()).andExpect(jsonPath("$.status").value(415)); + } + + @Test void oversizedBodyIsRejectedBeforeProvider() throws Exception { + mockMvc.perform(post(INFERENCE_URL).header("X-API-Key", VALID_RAW_KEY) + .contentType(MediaType.APPLICATION_JSON).content("x".repeat(1048577))) + .andExpect(status().isPayloadTooLarge()).andExpect(header().exists("X-Request-Id")); + org.mockito.Mockito.verifyNoInteractions(mockProvider); + assertThat(requestLogRepository.count()).isZero(); + } + + @Test void upstreamFailureDoesNotLeakToResponseLogsOrDatabase() throws Exception { + var appender = new ch.qos.logback.core.read.ListAppender(); + var root = (ch.qos.logback.classic.Logger) org.slf4j.LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME); + appender.start(); root.addAppender(appender); + when(mockProvider.infer(any())).thenThrow(new com.gateway.error.GatewayException( + com.gateway.error.ErrorCode.PROVIDER_UNAVAILABLE, "SECRET_PROVIDER_RESPONSE")); + try { + mockMvc.perform(post(INFERENCE_URL).header("X-API-Key", VALID_RAW_KEY).contentType(MediaType.APPLICATION_JSON) + .content("{\"model\":\"qwen3\",\"prompt\":\"SECRET_PROMPT\"}")) + .andExpect(status().isServiceUnavailable()).andExpect(jsonPath("$.message").value("Provider is unavailable")); + assertThat(appender.list).noneMatch(event -> event.getFormattedMessage().contains("SECRET_")); + assertThat(requestLogRepository.findAll()).singleElement() + .satisfies(row -> assertThat(row.getErrorCode()).isEqualTo("PROVIDER_UNAVAILABLE")); + } finally { root.detachAppender(appender); appender.stop(); } + } + @Test void inference_withExplicitProvider_routesToThatProvider() throws Exception { InferenceResponse providerResponse = InferenceResponse.builder() diff --git a/src/test/java/com/gateway/api/LogControllerIntegrationTest.java b/src/test/java/com/gateway/api/LogControllerIntegrationTest.java index 24bc07b..cb3af9e 100644 --- a/src/test/java/com/gateway/api/LogControllerIntegrationTest.java +++ b/src/test/java/com/gateway/api/LogControllerIntegrationTest.java @@ -9,7 +9,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.webmvc.test.autoconfigure.AutoConfigureMockMvc; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; import org.springframework.test.context.bean.override.mockito.MockitoBean; @@ -97,6 +97,17 @@ private void saveLog(String requestId, LocalDateTime timestamp, String provider, // --- Pagination tests --- + @org.junit.jupiter.params.ParameterizedTest + @org.junit.jupiter.params.provider.ValueSource(strings = { + "page=-1", "page=invalid", "size=0", "size=-1", "status=INVALID", + "from=not-a-date", "from=2026-09-02T00:00:00&to=2026-09-01T00:00:00"}) + void rejectsInvalidFiltersWithSafe400(String query) throws Exception { + mockMvc.perform(get(LOGS_URL + "?" + query).header("X-API-Key", VALID_RAW_KEY)) + .andExpect(status().isBadRequest()).andExpect(jsonPath("$.status").value(400)) + .andExpect(jsonPath("$.error").value("INVALID_REQUEST")) + .andExpect(header().exists("X-Request-Id")); + } + @Test void getLogs_defaultPagination_returns200WithAllEntries() throws Exception { mockMvc.perform(get(LOGS_URL) diff --git a/src/test/java/com/gateway/api/ProviderControllerIntegrationTest.java b/src/test/java/com/gateway/api/ProviderControllerIntegrationTest.java index 82c3fc5..b410e31 100644 --- a/src/test/java/com/gateway/api/ProviderControllerIntegrationTest.java +++ b/src/test/java/com/gateway/api/ProviderControllerIntegrationTest.java @@ -8,7 +8,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.webmvc.test.autoconfigure.AutoConfigureMockMvc; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; import org.springframework.test.context.bean.override.mockito.MockitoBean; diff --git a/src/test/java/com/gateway/auth/ApiKeyFilterIntegrationTest.java b/src/test/java/com/gateway/auth/ApiKeyFilterIntegrationTest.java index 8063c73..1e54e0d 100644 --- a/src/test/java/com/gateway/auth/ApiKeyFilterIntegrationTest.java +++ b/src/test/java/com/gateway/auth/ApiKeyFilterIntegrationTest.java @@ -3,7 +3,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.webmvc.test.autoconfigure.AutoConfigureMockMvc; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.context.TestConfiguration; import org.springframework.http.MediaType; @@ -78,7 +78,7 @@ void request_withoutApiKey_returns401() throws Exception { .andExpect(status().isUnauthorized()) .andExpect(jsonPath("$.status").value(401)) .andExpect(jsonPath("$.error").value("UNAUTHORIZED")) - .andExpect(jsonPath("$.message").value("Missing API key — provide X-API-Key header")) + .andExpect(jsonPath("$.message").value("Missing or invalid API key")) .andExpect(jsonPath("$.path").value("/v1/test-auth")) .andExpect(jsonPath("$.requestId").value(startsWith("req_"))) .andExpect(jsonPath("$.timestamp").exists()); @@ -92,7 +92,7 @@ void request_withInvalidApiKey_returns401() throws Exception { .andExpect(status().isUnauthorized()) .andExpect(jsonPath("$.status").value(401)) .andExpect(jsonPath("$.error").value("UNAUTHORIZED")) - .andExpect(jsonPath("$.message").value("Invalid API key")) + .andExpect(jsonPath("$.message").value("Missing or invalid API key")) .andExpect(jsonPath("$.requestId").value(startsWith("req_"))); } @@ -128,7 +128,7 @@ void request_withEmptyApiKey_returns401() throws Exception { .accept(MediaType.APPLICATION_JSON)) .andExpect(status().isUnauthorized()) .andExpect(jsonPath("$.error").value("UNAUTHORIZED")) - .andExpect(jsonPath("$.message").value("Missing API key — provide X-API-Key header")); + .andExpect(jsonPath("$.message").value("Missing or invalid API key")); } @Test diff --git a/src/test/java/com/gateway/common/AdmissionFilterTest.java b/src/test/java/com/gateway/common/AdmissionFilterTest.java new file mode 100644 index 0000000..7f77767 --- /dev/null +++ b/src/test/java/com/gateway/common/AdmissionFilterTest.java @@ -0,0 +1,52 @@ +package com.gateway.common; + +import com.gateway.config.GatewayProperties; +import com.gateway.error.ApiErrorWriter; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import org.junit.jupiter.api.Test; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; +import tools.jackson.databind.json.JsonMapper; +import java.util.concurrent.*; +import static org.assertj.core.api.Assertions.*; + +class AdmissionFilterTest { + private AdmissionFilter filter(GatewayProperties properties) { + return new AdmissionFilter(properties, new ApiErrorWriter(JsonMapper.builder().build()), new SimpleMeterRegistry()); + } + @Test void unknownContentLengthCannotBypassBodyBound() throws Exception { + var props = new GatewayProperties(); props.setMaxRequestBytes(8); + var request = new MockHttpServletRequest("POST", "/v1/inference") { + @Override public long getContentLengthLong() { return -1; } + }; + request.setContent("secret-too-long".getBytes()); request.setAttribute("requestId", "req_bounded"); + var response = new MockHttpServletResponse(); + filter(props).doFilter(request, response, (req, res) -> { throw new AssertionError("Must not reach authentication"); }); + assertThat(response.getStatus()).isEqualTo(413); + assertThat(response.getContentAsString()).doesNotContain("secret-too-long").contains("req_bounded"); + } + @Test void saturationRejectsPromptlyAndPermitIsReleased() throws Exception { + var props = new GatewayProperties(); props.setMaxConcurrentRequests(1); var filter = filter(props); + CountDownLatch entered = new CountDownLatch(1), release = new CountDownLatch(1); + try (var worker = Executors.newSingleThreadExecutor()) { + var first = worker.submit(() -> { + try { + filter.doFilter(new MockHttpServletRequest("POST", "/v1/inference"), new MockHttpServletResponse(), (req, res) -> { + entered.countDown(); + try { release.await(3, TimeUnit.SECONDS); } catch (InterruptedException ex) { Thread.currentThread().interrupt(); } + }); + } catch (Exception ex) { throw new RuntimeException(ex); } + }); + assertThat(entered.await(2, TimeUnit.SECONDS)).isTrue(); + try { + var rejected = new MockHttpServletResponse(); + filter.doFilter(new MockHttpServletRequest("POST", "/v1/inference"), rejected, (req, res) -> { throw new AssertionError(); }); + assertThat(rejected.getStatus()).isEqualTo(503); + } finally { release.countDown(); } + first.get(2, TimeUnit.SECONDS); + var admitted = new MockHttpServletResponse(); + filter.doFilter(new MockHttpServletRequest("GET", "/v1/providers"), admitted, (req, res) -> {}); + assertThat(admitted.getStatus()).isEqualTo(200); + } + } +} diff --git a/src/test/java/com/gateway/config/DatabaseBootstrapTest.java b/src/test/java/com/gateway/config/DatabaseBootstrapTest.java new file mode 100644 index 0000000..b71b956 --- /dev/null +++ b/src/test/java/com/gateway/config/DatabaseBootstrapTest.java @@ -0,0 +1,51 @@ +package com.gateway.config; + +import com.gateway.auth.*; +import org.junit.jupiter.api.Test; +import org.springframework.mock.env.MockEnvironment; +import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; +import org.springframework.transaction.PlatformTransactionManager; +import java.util.List; +import java.util.UUID; +import static org.assertj.core.api.Assertions.*; +import static org.mockito.Mockito.*; + +class DatabaseBootstrapTest { + private final ApiKeyRepository keys = mock(ApiKeyRepository.class); + private final ProviderConfigRepository providers = mock(ProviderConfigRepository.class); + private final BCryptPasswordEncoder encoder = new BCryptPasswordEncoder(); + private DatabaseBootstrap bootstrap(GatewayProperties props, MockEnvironment env) { + return new DatabaseBootstrap(keys, providers, props, env, encoder, mock(PlatformTransactionManager.class)); + } + @Test void detectsRehashedCopyOfPublicKeyBeforeStartup() { + when(keys.findByActiveTrue()).thenReturn(List.of(ApiKey.builder().id(UUID.randomUUID()) + .keyHash(encoder.encode(DatabaseBootstrap.DEMO_KEY)).active(true).build())); + assertThatThrownBy(() -> bootstrap(new GatewayProperties(), new MockEnvironment()).afterPropertiesSet()) + .isInstanceOf(IllegalStateException.class).hasMessageContaining("Active demo credential"); + } + @Test void refusesLocalBootstrapWithoutLocalProfile() { + var props = new GatewayProperties(); props.getBootstrap().setLocalEnabled(true); + assertThatThrownBy(() -> bootstrap(props, new MockEnvironment()).afterPropertiesSet()).isInstanceOf(IllegalStateException.class); + verifyNoInteractions(keys); + } + @Test void refusesCombinedLocalProductionProfiles() { + var props = new GatewayProperties(); props.getBootstrap().setLocalEnabled(true); + var env = new MockEnvironment(); env.setActiveProfiles("local", "production"); + assertThatThrownBy(() -> bootstrap(props, env).afterPropertiesSet()).isInstanceOf(IllegalStateException.class); + } + @Test void productionRejectsDemoDatabasePassword() { + var env = new MockEnvironment().withProperty("spring.datasource.password", "gateway"); env.setActiveProfiles("production"); + assertThatThrownBy(() -> bootstrap(new GatewayProperties(), env).afterPropertiesSet()).hasMessageContaining("database password"); + } + @Test void explicitLocalBootstrapHashesKeyAndUpdatesProviderEndpoint() { + var props = new GatewayProperties(); props.getBootstrap().setLocalEnabled(true); + props.getBootstrap().setApiKey("local-custom-key"); props.getBootstrap().setOllamaBaseUrl("http://localhost:1234"); + var env = new MockEnvironment(); env.setActiveProfiles("local"); + var provider = ProviderConfig.builder().name("ollama-local").build(); + when(providers.findByName("ollama-local")).thenReturn(java.util.Optional.of(provider)); + bootstrap(props, env).afterPropertiesSet(); + var saved = org.mockito.ArgumentCaptor.forClass(ApiKey.class); verify(keys).save(saved.capture()); + assertThat(encoder.matches("local-custom-key", saved.getValue().getKeyHash())).isTrue(); + assertThat(provider.getBaseUrl()).isEqualTo("http://localhost:1234"); + } +} diff --git a/src/test/java/com/gateway/config/PostgresMigrationIT.java b/src/test/java/com/gateway/config/PostgresMigrationIT.java new file mode 100644 index 0000000..99dd9d7 --- /dev/null +++ b/src/test/java/com/gateway/config/PostgresMigrationIT.java @@ -0,0 +1,109 @@ +package com.gateway.config; + +import org.flywaydb.core.Flyway; +import org.flywaydb.core.api.MigrationVersion; +import org.junit.jupiter.api.*; +import java.sql.*; +import java.util.UUID; +import static org.assertj.core.api.Assertions.*; + +/** Opt-in via -Ppostgres-it; missing PostgreSQL fails rather than silently skipping the release gate. */ +class PostgresMigrationIT { + private String url, user, password, schema; + @BeforeEach void prepare() throws Exception { + url = System.getenv("TEST_DATABASE_URL"); user = System.getenv("TEST_DATABASE_USER"); + password = System.getenv("TEST_DATABASE_PASSWORD"); + assertThat(url).as("TEST_DATABASE_URL is required for -Ppostgres-it").isNotBlank(); + schema = "p6_" + UUID.randomUUID().toString().replace("-", ""); + try (Connection connection = connect()) { connection.createStatement().execute("CREATE SCHEMA " + schema); } + } + private Connection connect() throws SQLException { return DriverManager.getConnection(url, user, password); } + private Flyway migration(String version) { + var config = Flyway.configure().dataSource(url, user, password).schemas(schema).defaultSchema(schema) + .locations("classpath:db/migration"); + if (version != null) config.target(MigrationVersion.fromVersion(version)); + return config.load(); + } + @AfterEach void cleanup() throws Exception { + if (schema != null && schema.matches("p6_[a-f0-9]{32}")) { + try (Connection connection = connect()) { connection.createStatement().execute("DROP SCHEMA " + schema + " CASCADE"); } + } + } + private int scalar(String sql) throws Exception { + try (Connection connection = connect(); Statement statement = connection.createStatement()) { + statement.execute("SET search_path TO " + schema); + try (ResultSet result = statement.executeQuery(sql)) { result.next(); return result.getInt(1); } + } + } + @Test void freshInstallRunsEveryMigrationAndDisablesPublicKey() throws Exception { + assertThat(migration(null).migrate().migrationsExecuted).isEqualTo(6); + migration(null).validate(); + assertThat(scalar("SELECT count(*) FROM api_keys WHERE active")).isZero(); + assertThat(scalar("SELECT count(*) FROM providers")).isEqualTo(1); + assertThat(scalar("SELECT count(*) FROM models")).isEqualTo(1); + assertThat(migration(null).migrate().migrationsExecuted).isZero(); + } + @Test void populatedMvpUpgradesWithoutLosingExistingData() throws Exception { + migration("5").migrate(); + assertThat(scalar("SELECT count(*) FROM api_keys WHERE active")).isEqualTo(1); + try (Connection connection = connect(); Statement statement = connection.createStatement()) { + statement.execute("SET search_path TO " + schema); + statement.execute("INSERT INTO request_logs(id,request_id,status,latency_ms) VALUES ('" + UUID.randomUUID() + "','mvp-request','SUCCESS',17)"); + statement.execute("INSERT INTO api_keys(id,key_hash,label,active) VALUES ('" + UUID.randomUUID() + "','existing-customer-hash','existing',true)"); + } + assertThat(migration(null).migrate().migrationsExecuted).isEqualTo(1); + migration(null).validate(); + assertThat(scalar("SELECT count(*) FROM request_logs WHERE request_id='mvp-request'")).isEqualTo(1); + assertThat(scalar("SELECT count(*) FROM api_keys WHERE active AND label='existing'")).isEqualTo(1); + assertThat(scalar("SELECT count(*) FROM api_keys WHERE active AND label='local-test-key'")).isZero(); + } + + private org.springframework.context.ConfigurableApplicationContext startApplication() { + return new org.springframework.boot.builder.SpringApplicationBuilder(com.gateway.GatewayApplication.class) + .run("--spring.profiles.active=production", + "--spring.datasource.url=" + url + (url.contains("?") ? "&" : "?") + "currentSchema=" + schema, + "--spring.datasource.username=" + user, "--spring.datasource.password=" + password, + "--spring.datasource.driver-class-name=org.postgresql.Driver", + "--spring.jpa.hibernate.ddl-auto=validate", "--spring.flyway.enabled=true", + "--spring.flyway.schemas=" + schema, "--spring.flyway.default-schema=" + schema, + "--server.port=0", "--management.server.port=0", + "--management.endpoints.web.exposure.include=health,prometheus", + "--management.endpoint.health.show-details=never", "--spring.main.banner-mode=off"); + } + + @Test void productionStartsWithRealMigrationsAndMinimalHealth() throws Exception { + try (var context = startApplication(); var http = java.net.http.HttpClient.newHttpClient()) { + var environment = context.getEnvironment(); + int port = environment.getProperty("local.server.port", Integer.class); + int management = environment.getProperty("local.management.port", Integer.class); + var api = http.send(java.net.http.HttpRequest.newBuilder(java.net.URI.create("http://localhost:" + port + "/v1/providers")).build(), + java.net.http.HttpResponse.BodyHandlers.ofString()); + assertThat(api.statusCode()).isEqualTo(401); + assertThat(api.headers().firstValue("X-Request-Id")).isPresent(); + var health = http.send(java.net.http.HttpRequest.newBuilder(java.net.URI.create("http://localhost:" + management + "/actuator/health")).build(), + java.net.http.HttpResponse.BodyHandlers.ofString()); + assertThat(health.statusCode()).isEqualTo(200); + assertThat(health.body()).doesNotContain("components", "jdbc", "password"); + var metrics = http.send(java.net.http.HttpRequest.newBuilder(java.net.URI.create("http://localhost:" + management + "/actuator/prometheus")).build(), + java.net.http.HttpResponse.BodyHandlers.ofString()); + assertThat(metrics.statusCode()).isEqualTo(200); + assertThat(metrics.body()).contains("gateway_requests"); + var publicMetrics = http.send(java.net.http.HttpRequest.newBuilder(java.net.URI.create("http://localhost:" + port + "/actuator/prometheus")).build(), + java.net.http.HttpResponse.BodyHandlers.ofString()); + assertThat(publicMetrics.statusCode()).isEqualTo(401); + assertThat(scalar("SELECT count(*) FROM api_keys WHERE active")).isZero(); + } + } + + @Test void productionRefusesRehashedDemoCredentialAfterMigration() throws Exception { + migration(null).migrate(); + try (Connection connection = connect(); var statement = connection.prepareStatement( + "INSERT INTO " + schema + ".api_keys(id,key_hash,label,active) VALUES (?,?,?,true)")) { + statement.setObject(1, UUID.randomUUID()); + statement.setString(2, new org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder().encode(DatabaseBootstrap.DEMO_KEY)); + statement.setString(3, "accidentally-reintroduced-demo"); statement.executeUpdate(); + } + assertThatThrownBy(() -> { try (var ignored = startApplication()) { } }) + .hasRootCauseInstanceOf(IllegalStateException.class).hasStackTraceContaining("Active demo credential"); + } +} diff --git a/src/test/java/com/gateway/error/GlobalExceptionHandlerTest.java b/src/test/java/com/gateway/error/GlobalExceptionHandlerTest.java index 09a1b49..6582b22 100644 --- a/src/test/java/com/gateway/error/GlobalExceptionHandlerTest.java +++ b/src/test/java/com/gateway/error/GlobalExceptionHandlerTest.java @@ -38,7 +38,7 @@ void gatewayException_providerNotFound_returns404() throws Exception { .andExpect(status().isNotFound()) .andExpect(jsonPath("$.status").value(404)) .andExpect(jsonPath("$.error").value("PROVIDER_NOT_FOUND")) - .andExpect(jsonPath("$.message").value("Test provider not found")) + .andExpect(jsonPath("$.message").value("Provider not found or not active")) .andExpect(jsonPath("$.path").value("/test/provider-not-found")) .andExpect(jsonPath("$.requestId").value(startsWith("req_"))) .andExpect(jsonPath("$.timestamp").exists()); @@ -51,7 +51,7 @@ void gatewayException_badConfiguration_returns500() throws Exception { .andExpect(status().isInternalServerError()) .andExpect(jsonPath("$.status").value(500)) .andExpect(jsonPath("$.error").value("BAD_CONFIGURATION")) - .andExpect(jsonPath("$.message").value("No default provider")) + .andExpect(jsonPath("$.message").value("Gateway configuration is invalid")) .andExpect(jsonPath("$.requestId").value(startsWith("req_"))); } @@ -76,12 +76,25 @@ void responseContains_xRequestIdHeader() throws Exception { .andExpect(header().string("X-Request-Id", startsWith("req_"))); } + @Test void forbiddenIsSafeAndCorrelated() throws Exception { + mockMvc.perform(get("/test/forbidden")) + .andExpect(status().isForbidden()) + .andExpect(jsonPath("$.error").value("FORBIDDEN")) + .andExpect(jsonPath("$.message").value(not(containsString("SECRET")))) + .andExpect(header().string("X-Request-Id", startsWith("req_"))); + } + /** * Fake controller used only by these tests to trigger specific exceptions. */ @RestController static class TestController { + @GetMapping("/test/forbidden") + public String forbidden() { + throw new org.springframework.security.access.AccessDeniedException("SECRET_RESOURCE"); + } + @GetMapping("/test/provider-not-found") public String providerNotFound() { throw new GatewayException(ErrorCode.PROVIDER_NOT_FOUND, "Test provider not found"); diff --git a/src/test/java/com/gateway/inference/InferenceServiceTest.java b/src/test/java/com/gateway/inference/InferenceServiceTest.java index 58326e0..665c76b 100644 --- a/src/test/java/com/gateway/inference/InferenceServiceTest.java +++ b/src/test/java/com/gateway/inference/InferenceServiceTest.java @@ -10,6 +10,9 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.Spy; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; import org.mockito.junit.jupiter.MockitoExtension; import static org.assertj.core.api.Assertions.assertThat; @@ -22,6 +25,7 @@ */ @ExtendWith(MockitoExtension.class) class InferenceServiceTest { + @Spy private MeterRegistry meters = new SimpleMeterRegistry(); @Mock private RoutingEngine routingEngine; @@ -43,6 +47,34 @@ private InferenceRequest createRequest(String provider, String model, String pro .build(); } + @Test void unexpectedFailureIsRecordedAndSanitized() { + var request = createRequest(null, "qwen3", "secret prompt"); + when(routingEngine.resolve(request)).thenThrow(new RuntimeException("SECRET_INTERNAL")); + assertThatThrownBy(() -> inferenceService.infer(request, "req_unexpected")) + .hasMessage("An unexpected error occurred"); + verify(requestLogService).log(eq("req_unexpected"), isNull(), eq("qwen3"), eq("FAILURE"), eq("INTERNAL_ERROR"), anyInt()); + } + + @Test void diagnosticFailurePreservesProviderOutcome() { + var request = createRequest(null, "qwen3", "hello"); + when(routingEngine.resolve(request)).thenReturn(new RoutingResult(mockProvider, "ollama-local")); + when(mockProvider.infer(request)).thenReturn(InferenceResponse.builder().text("ok").build()); + doThrow(new RuntimeException("SECRET_DB_ERROR")).when(requestLogService).log(any(), any(), any(), any(), any(), anyInt()); + assertThat(inferenceService.infer(request, "req_logfail").getText()).isEqualTo("ok"); + assertThat(meters.get("gateway.diagnostics.dropped").counter().count()).isEqualTo(1); + verify(requestLogService, times(1)).log(any(), any(), any(), any(), any(), anyInt()); + } + + @Test void diagnosticFailurePreservesOriginalProviderFailure() { + var request = createRequest(null, "qwen3", "hello"); + var failure = new GatewayException(ErrorCode.PROVIDER_TIMEOUT, "Provider request timed out"); + when(routingEngine.resolve(request)).thenReturn(new RoutingResult(mockProvider, "ollama-local")); + when(mockProvider.infer(request)).thenThrow(failure); + doThrow(new RuntimeException("SECRET_DB_ERROR")).when(requestLogService).log(any(), any(), any(), any(), any(), anyInt()); + assertThatThrownBy(() -> inferenceService.infer(request, "req_bothfail")).isSameAs(failure); + verify(requestLogService, times(1)).log(eq("req_bothfail"), any(), any(), eq("FAILURE"), eq("PROVIDER_TIMEOUT"), anyInt()); + } + @Test void infer_happyPath_returnsEnrichedResponseAndLogsSuccess() { InferenceRequest request = createRequest(null, "qwen3", "Hello"); @@ -92,7 +124,7 @@ void infer_providerFailure_logsFailureAndRethrows() { } @Test - void infer_routingFailure_throwsWithoutLogging() { + void infer_routingFailure_recordsTerminalFailure() { InferenceRequest request = createRequest("nonexistent", "qwen3", "Hello"); String requestId = "req_route789"; @@ -103,8 +135,8 @@ void infer_routingFailure_throwsWithoutLogging() { .isInstanceOf(GatewayException.class) .hasFieldOrPropertyWithValue("errorCode", ErrorCode.PROVIDER_NOT_FOUND); - // No log written because provider was never resolved - verifyNoInteractions(requestLogService); + verify(requestLogService).log(eq(requestId), isNull(), eq("qwen3"), eq("FAILURE"), + eq("PROVIDER_NOT_FOUND"), anyInt()); } @Test diff --git a/src/test/java/com/gateway/performance/BaselineBenchmarkTest.java b/src/test/java/com/gateway/performance/BaselineBenchmarkTest.java new file mode 100644 index 0000000..8e56019 --- /dev/null +++ b/src/test/java/com/gateway/performance/BaselineBenchmarkTest.java @@ -0,0 +1,82 @@ +package com.gateway.performance; + +import com.gateway.auth.*; +import com.gateway.config.*; +import com.gateway.provider.ProviderRegistry; +import com.gateway.support.MockUpstream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.core.env.Environment; +import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; +import tools.jackson.databind.json.JsonMapper; +import java.net.URI; +import java.net.http.*; +import java.nio.file.*; +import java.time.LocalDateTime; +import java.util.*; +import java.util.concurrent.*; +import static org.assertj.core.api.Assertions.*; + +/** Reproducible small non-streaming baseline, deliberately separate from the production load gate. */ +@EnabledIfSystemProperty(named = "gateway.benchmark", matches = "true") +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + properties = {"spring.datasource.url=jdbc:h2:mem:benchmark;DB_CLOSE_DELAY=-1", "management.server.port=0"}) +class BaselineBenchmarkTest { + @Autowired Environment env; + @Autowired ApiKeyRepository keys; + @Autowired ProviderConfigRepository providers; + @Autowired ModelConfigRepository models; + @Autowired ProviderRegistry registry; + @Autowired BCryptPasswordEncoder encoder; + + @Test void measureLoopbackGatewayWithAuthenticationAndDiagnosticWrites() throws Exception { + try (var upstream = new MockUpstream(); var http = HttpClient.newHttpClient()) { + var provider = providers.save(ProviderConfig.builder().id(UUID.randomUUID()).name("benchmark") + .type("ollama").baseUrl(upstream.url()).active(true).defaultProvider(true).build()); + models.save(ModelConfig.builder().id(UUID.randomUUID()).provider(provider).name("fixture").active(true).build()); + keys.save(ApiKey.builder().id(UUID.randomUUID()).keyHash(encoder.encode("benchmark-local-key")) + .label("benchmark").active(true).createdAt(LocalDateTime.now()).build()); + registry.init(); + String prompt = "a".repeat(1024); + byte[] body = JsonMapper.builder().build().writeValueAsBytes(Map.of("model", "fixture", "prompt", prompt)); + var request = HttpRequest.newBuilder(URI.create("http://localhost:" + env.getProperty("local.server.port") + "/v1/inference")) + .header("Content-Type", "application/json").header("X-API-Key", "benchmark-local-key") + .POST(HttpRequest.BodyPublishers.ofByteArray(body)).build(); + var direct = HttpRequest.newBuilder(URI.create(upstream.url() + "/api/generate")) + .POST(HttpRequest.BodyPublishers.ofByteArray(body)).build(); + for (int i = 0; i < 20; i++) execute(http, request); + var output = new LinkedHashMap(); + output.put("scope", "loopback HTTP, real BCrypt, H2 diagnostic writes, synthetic Ollama; not production capacity"); + output.put("java", System.getProperty("java.version")); output.put("os", System.getProperty("os.name")); + output.put("cpus", Runtime.getRuntime().availableProcessors()); output.put("inputCharacters", 1024); + output.put("warmupRequests", 20); output.put("directSequential", measure(http, direct, 100, 1)); + output.put("gatewaySequential", measure(http, request, 100, 1)); + output.put("gatewayConcurrent", measure(http, request, 100, 10)); + output.put("heapUsedBytesAtEnd", Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()); + output.put("maxHeapBytes", Runtime.getRuntime().maxMemory()); + Files.createDirectories(Path.of("target")); + Files.writeString(Path.of("target/phase6-baseline.json"), JsonMapper.builder().build() + .writerWithDefaultPrettyPrinter().writeValueAsString(output)); + } + } + private Map measure(HttpClient http, HttpRequest request, int count, int concurrency) throws Exception { + long start = System.nanoTime(); List timings = new ArrayList<>(); + try (var pool = Executors.newFixedThreadPool(concurrency)) { + List> futures = new ArrayList<>(); + for (int i = 0; i < count; i++) futures.add(pool.submit(() -> execute(http, request))); + for (var future : futures) timings.add(future.get(30, TimeUnit.SECONDS)); + } + Collections.sort(timings); + return Map.of("requests", count, "concurrency", concurrency, "p50Ms", timings.get(49), + "p95Ms", timings.get(94), "p99Ms", timings.get(98), + "requestsPerSecond", count * 1e9 / (System.nanoTime() - start)); + } + private double execute(HttpClient http, HttpRequest request) throws Exception { + long start = System.nanoTime(); + var result = http.send(request, HttpResponse.BodyHandlers.discarding()); + assertThat(result.statusCode()).isEqualTo(200); + return (System.nanoTime() - start) / 1e6; + } +} diff --git a/src/test/java/com/gateway/provider/ProviderTransportTest.java b/src/test/java/com/gateway/provider/ProviderTransportTest.java new file mode 100644 index 0000000..49fb240 --- /dev/null +++ b/src/test/java/com/gateway/provider/ProviderTransportTest.java @@ -0,0 +1,81 @@ +package com.gateway.provider; + +import com.gateway.common.RequestDeadline; +import com.gateway.config.GatewayProperties; +import com.gateway.error.ErrorCode; +import com.gateway.error.GatewayException; +import com.gateway.support.MockUpstream; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import org.junit.jupiter.api.Test; +import java.net.URI; +import java.util.concurrent.*; +import static org.assertj.core.api.Assertions.*; + +class ProviderTransportTest { + @Test void totalDeadlineAbortsSlowBodyAndReleasesPool() throws Exception { + var properties = new GatewayProperties(); + properties.setTotalTimeoutMs(150); + properties.getTransport().setReadTimeoutMs(5000); + properties.getTransport().setMaxConnections(1); + properties.getTransport().setMaxConnectionsPerRoute(1); + try (var upstream = new MockUpstream(); var transport = new ProviderTransport(properties, new SimpleMeterRegistry())) { + upstream.respondsWith(exchange -> { + exchange.sendResponseHeaders(200, 0); + try { + for (int i = 0; i < 100; i++) { + exchange.getResponseBody().write('x'); exchange.getResponseBody().flush(); Thread.sleep(20); + } + } catch (InterruptedException e) { Thread.currentThread().interrupt(); } + }); + long start = System.nanoTime(); + assertThatThrownBy(() -> transport.exchange("GET", URI.create(upstream.url()), null)) + .isInstanceOf(GatewayException.class).hasFieldOrPropertyWithValue("errorCode", ErrorCode.PROVIDER_TIMEOUT); + assertThat(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)).isLessThan(1500); + assertThat(upstream.calls.get()).isEqualTo(1); + upstream.respondsWith(exchange -> MockUpstream.respond(exchange, 200, "ok")); + assertThat(new String(transport.exchange("GET", URI.create(upstream.url()), null))).isEqualTo("ok"); + } + } + @Test void responseBoundAndStatusErrorsDoNotLeakOrRetry() throws Exception { + var properties = new GatewayProperties(); properties.getTransport().setMaxResponseBytes(32); + try (var upstream = new MockUpstream(); var transport = new ProviderTransport(properties, new SimpleMeterRegistry())) { + for (int status : new int[]{429, 500, 302, 200}) { + upstream.respondsWith(exchange -> MockUpstream.respond(exchange, status, "SENSITIVE_PROVIDER_BODY".repeat(100))); + assertThatThrownBy(() -> transport.exchange("GET", URI.create(upstream.url()), null)) + .isInstanceOf(GatewayException.class).hasMessage("Provider is unavailable") + .hasFieldOrPropertyWithValue("errorCode", ErrorCode.PROVIDER_UNAVAILABLE); + } + assertThat(upstream.calls.get()).isEqualTo(4); + } + } + @Test void poolWaitIsBoundedAndDoesNotOpenAnotherConnection() throws Exception { + var properties = new GatewayProperties(); properties.getTransport().setMaxConnections(1); + properties.getTransport().setMaxConnectionsPerRoute(1); properties.getTransport().setPoolAcquireTimeoutMs(80); + CountDownLatch entered = new CountDownLatch(1); CountDownLatch release = new CountDownLatch(1); + try (var upstream = new MockUpstream(); var transport = new ProviderTransport(properties, new SimpleMeterRegistry()); + var workers = Executors.newSingleThreadExecutor()) { + upstream.respondsWith(exchange -> { + entered.countDown(); + try { release.await(3, TimeUnit.SECONDS); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } + MockUpstream.respond(exchange, 200, "ok"); + }); + Future first = workers.submit(() -> transport.exchange("GET", URI.create(upstream.url()), null)); + assertThat(entered.await(2, TimeUnit.SECONDS)).isTrue(); + try { + assertThatThrownBy(() -> transport.exchange("GET", URI.create(upstream.url()), null)).isInstanceOf(GatewayException.class); + assertThat(upstream.calls.get()).isEqualTo(1); + } finally { release.countDown(); } + assertThat(first.get(2, TimeUnit.SECONDS)).isEqualTo("ok".getBytes()); + } + } + @Test void expiredAdmissionDeadlinePreventsDispatch() throws Exception { + try (var upstream = new MockUpstream(); var transport = new ProviderTransport(new GatewayProperties(), new SimpleMeterRegistry())) { + RequestDeadline.start(1); Thread.sleep(10); + try { + assertThatThrownBy(() -> transport.exchange("GET", URI.create(upstream.url()), null)) + .hasFieldOrPropertyWithValue("errorCode", ErrorCode.PROVIDER_TIMEOUT); + assertThat(upstream.calls.get()).isZero(); + } finally { RequestDeadline.clear(); } + } + } +} diff --git a/src/test/java/com/gateway/provider/ollama/OllamaProviderTest.java b/src/test/java/com/gateway/provider/ollama/OllamaProviderTest.java index 68e3400..9d9d066 100644 --- a/src/test/java/com/gateway/provider/ollama/OllamaProviderTest.java +++ b/src/test/java/com/gateway/provider/ollama/OllamaProviderTest.java @@ -1,153 +1,56 @@ package com.gateway.provider.ollama; -import com.gateway.config.ModelConfig; -import com.gateway.config.ModelConfigRepository; -import com.gateway.config.ProviderConfig; -import com.gateway.error.ErrorCode; -import com.gateway.error.GatewayException; +import com.gateway.config.*; +import com.gateway.error.*; import com.gateway.inference.InferenceRequest; -import com.gateway.inference.InferenceResponse; +import com.gateway.provider.ProviderTransport; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.springframework.http.HttpMethod; -import org.springframework.http.MediaType; -import org.springframework.test.web.client.MockRestServiceServer; -import org.springframework.web.client.RestClient; - +import tools.jackson.databind.json.JsonMapper; import java.util.List; import java.util.UUID; +import static org.assertj.core.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.Mockito.when; -import static org.springframework.test.web.client.match.MockRestRequestMatchers.*; -import static org.springframework.test.web.client.response.MockRestResponseCreators.*; - -@ExtendWith(MockitoExtension.class) class OllamaProviderTest { - - @Mock - private ModelConfigRepository modelConfigRepository; - - private MockRestServiceServer mockServer; - private OllamaProvider ollamaProvider; - - private static final UUID PROVIDER_ID = UUID.fromString("a1b2c3d4-e5f6-7890-abcd-ef1234567890"); - private static final String PROVIDER_NAME = "ollama-local"; - - @BeforeEach - void setUp() { - ProviderConfig config = ProviderConfig.builder() - .id(PROVIDER_ID) - .name(PROVIDER_NAME) - .type("ollama") - .baseUrl("http://localhost:11434") - .active(true) - .defaultProvider(true) - .build(); - - RestClient.Builder builder = RestClient.builder(); - mockServer = MockRestServiceServer.bindTo(builder).build(); - ollamaProvider = new OllamaProvider(config, modelConfigRepository, builder); + private final ProviderTransport transport = mock(ProviderTransport.class); + private final ModelConfigRepository models = mock(ModelConfigRepository.class); + private final UUID id = UUID.randomUUID(); + private OllamaProvider provider; + @BeforeEach void prepare() { + provider = new OllamaProvider(ProviderConfig.builder().id(id).name("ollama-local").type("ollama") + .baseUrl("http://localhost:11434").build(), models, transport, JsonMapper.builder().build()); } - - @Test - void infer_validResponse_mapsCorrectly() { - String ollamaJson = """ - { - "model": "qwen3", - "response": "Hello! How can I help you today?", - "done": true - } - """; - - mockServer.expect(requestTo("http://localhost:11434/api/generate")) - .andExpect(method(HttpMethod.POST)) - .andRespond(withSuccess(ollamaJson, MediaType.APPLICATION_JSON)); - - InferenceRequest request = InferenceRequest.builder() - .model("qwen3") - .prompt("Hi") - .build(); - - InferenceResponse response = ollamaProvider.infer(request); - - assertNotNull(response); - assertEquals("Hello! How can I help you today?", response.getText()); - assertEquals("qwen3", response.getModel()); - assertEquals(PROVIDER_NAME, response.getProvider()); - assertTrue(response.getLatencyMs() >= 0); - - mockServer.verify(); + private InferenceRequest request() { return InferenceRequest.builder().model("qwen3").prompt("hello").build(); } + @Test void mapsValidResponse() { + when(transport.exchange(eq("POST"), any(), any())).thenReturn("{\"response\":\"hello\",\"done\":true}".getBytes()); + var response = provider.infer(request()); + assertThat(response.getText()).isEqualTo("hello"); + assertThat(response.getProvider()).isEqualTo("ollama-local"); } - - @Test - void infer_connectionFailure_throwsProviderUnavailable() { - mockServer.expect(requestTo("http://localhost:11434/api/generate")) - .andExpect(method(HttpMethod.POST)) - .andRespond(withServerError()); - - InferenceRequest request = InferenceRequest.builder() - .model("qwen3") - .prompt("Hi") - .build(); - - // Server error (5xx from Ollama) will result in an exception - // which gets mapped to PROVIDER_UNAVAILABLE or INTERNAL_ERROR - // depending on the actual exception type. For a 500, RestClient - // throws HttpServerErrorException which is not ResourceAccessException, - // so it won't be caught by our specific handler — it'll bubble up. - // This tests the connection path. - assertThrows(Exception.class, () -> ollamaProvider.infer(request)); - - mockServer.verify(); + @Test void rejectsMalformedOrIncompleteResponseWithoutLeakingBody() { + for (String body : List.of("SECRET_RAW_PAYLOAD", "{}", "{\"response\":17,\"done\":true}", "{\"response\":\"secret\",\"done\":false}")) { + when(transport.exchange(eq("POST"), any(), any())).thenReturn(body.getBytes()); + assertThatThrownBy(() -> provider.infer(request())).isInstanceOf(GatewayException.class) + .hasMessage("Provider is unavailable"); + } } - - @Test - void info_returnsProviderMetadata() { - ModelConfig model = ModelConfig.builder() - .name("qwen3") - .active(true) - .build(); - - when(modelConfigRepository.findByProviderIdAndActiveTrue(PROVIDER_ID)) - .thenReturn(List.of(model)); - - var info = ollamaProvider.info(); - - assertEquals(PROVIDER_NAME, info.name()); - assertEquals("ollama", info.type()); - assertEquals("http://localhost:11434", info.baseUrl()); - assertEquals(List.of("qwen3"), info.models()); + @Test void infoReadsConfiguredModels() { + when(models.findByProviderIdAndActiveTrue(id)).thenReturn(List.of(ModelConfig.builder().name("qwen3").build())); + assertThat(provider.info().models()).containsExactly("qwen3"); } - - @Test - void health_serverUp_returnsHealthy() { - mockServer.expect(requestTo("http://localhost:11434/")) - .andExpect(method(HttpMethod.GET)) - .andRespond(withSuccess("Ollama is running", MediaType.TEXT_PLAIN)); - - var health = ollamaProvider.health(); - - assertTrue(health.healthy()); - assertNull(health.details()); - - mockServer.verify(); + @Test void healthDoesNotExposeExceptionText() { + when(transport.exchange(eq("GET"), any(), isNull())).thenThrow(new RuntimeException("SECRET_ENDPOINT")); + assertThat(provider.health().healthy()).isFalse(); + assertThat(provider.health().details()).isEqualTo("Provider probe failed"); } - - @Test - void health_serverDown_returnsUnhealthy() { - mockServer.expect(requestTo("http://localhost:11434/")) - .andExpect(method(HttpMethod.GET)) - .andRespond(withServerError()); - - var health = ollamaProvider.health(); - - assertFalse(health.healthy()); - assertNotNull(health.details()); - - mockServer.verify(); + @Test void invalidEndpointsAreRejectedWithoutLeakingTheirValues() { + for (String endpoint : List.of("http://user:SECRET@host", "http://SECRET/%broken", "file:///SECRET")) { + assertThatThrownBy(() -> new OllamaProvider(ProviderConfig.builder().baseUrl(endpoint).build(), + models, transport, JsonMapper.builder().build())) + .hasFieldOrPropertyWithValue("errorCode", ErrorCode.BAD_CONFIGURATION) + .hasMessage("Gateway configuration is invalid").hasNoCause(); + } } } diff --git a/src/test/java/com/gateway/support/MockUpstream.java b/src/test/java/com/gateway/support/MockUpstream.java new file mode 100644 index 0000000..19a616a --- /dev/null +++ b/src/test/java/com/gateway/support/MockUpstream.java @@ -0,0 +1,39 @@ +package com.gateway.support; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; + +/** Real loopback HTTP fixture; supports delays, status failures and malformed/oversized bodies. */ +public final class MockUpstream implements AutoCloseable { + private final HttpServer server; + private final ExecutorService executor = Executors.newFixedThreadPool(8); + private volatile HttpHandler responder = exchange -> respond(exchange, 200, "{\"response\":\"ok\",\"done\":true}"); + public final AtomicInteger calls = new AtomicInteger(); + public MockUpstream() throws IOException { + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 16); + server.createContext("/", exchange -> { + calls.incrementAndGet(); + try { responder.handle(exchange); } catch (IOException ignored) { + // Client cancellation is expected in deadline tests. + } finally { exchange.close(); } + }); + server.setExecutor(executor); server.start(); + } + public String url() { return "http://127.0.0.1:" + server.getAddress().getPort(); } + public void respondsWith(HttpHandler responder) { this.responder = responder; } + public static void respond(HttpExchange exchange, int status, String body) throws IOException { + byte[] bytes = body.getBytes(StandardCharsets.UTF_8); + exchange.getRequestBody().readAllBytes(); + exchange.getResponseHeaders().set("Content-Type", "application/json"); + exchange.sendResponseHeaders(status, bytes.length); + exchange.getResponseBody().write(bytes); + } + @Override public void close() { server.stop(0); executor.shutdownNow(); } +} diff --git a/src/test/resources/application.yml b/src/test/resources/application.yml index 04f90ed..89b82be 100644 --- a/src/test/resources/application.yml +++ b/src/test/resources/application.yml @@ -3,6 +3,8 @@ # Hibernate auto-DDL creates the schema from JPA entities instead. spring: + profiles: + active: test datasource: url: jdbc:h2:mem:testdb;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=false driver-class-name: org.h2.Driver