diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index c20040f..0df41ca 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,3 +1,3 @@ # Changes require review from the organization administrator or the designated # independent reviewer, so code-owner review is satisfiable without admin bypass. -* @PeterGuy326 @Bindy-lbb +* @PeterGuy326 @Bindy-lbb @waterbro-8 diff --git a/.github/workflows/bytefolk-scorecard.yml b/.github/workflows/bytefolk-scorecard.yml new file mode 100644 index 0000000..2058126 --- /dev/null +++ b/.github/workflows/bytefolk-scorecard.yml @@ -0,0 +1,49 @@ +name: ByteFolk Scorecard + +on: + push: + branches: + - main + schedule: + - cron: '43 3 * * 1' + workflow_dispatch: + +permissions: + contents: read + +jobs: + scorecard: + name: OpenSSF Scorecard + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: read + actions: read + pull-requests: read + security-events: write + id-token: write + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Run Scorecard analysis + uses: ossf/scorecard-action@2d1146689b8cda280b9bc96326124645441f03bc # v2.4.4 + with: + results_file: results.sarif + results_format: sarif + publish_results: true + + - name: Upload Scorecard artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: scorecard-results + path: results.sarif + if-no-files-found: error + retention-days: 14 + + - name: Upload Scorecard results + uses: github/codeql-action/upload-sarif@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9 + with: + sarif_file: results.sarif diff --git a/.github/workflows/bytefolk-security.yml b/.github/workflows/bytefolk-security.yml new file mode 100644 index 0000000..724b46d --- /dev/null +++ b/.github/workflows/bytefolk-security.yml @@ -0,0 +1,69 @@ +name: ByteFolk Security Baseline + +on: + pull_request: + push: + branches: + - main + schedule: + - cron: '17 3 * * 1' + workflow_dispatch: + +permissions: + contents: read + +jobs: + dependency-review: + name: Dependency review + if: ${{ github.event_name == 'pull_request' }} + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Review dependency changes + uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0 + with: + fail-on-severity: high + fail-on-scopes: runtime + comment-summary-in-pr: never + + codeql: + name: CodeQL (${{ matrix.language }}) + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + actions: read + packages: read + security-events: write + strategy: + fail-fast: false + matrix: + language: + - go + - python + - javascript-typescript + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Initialize CodeQL + uses: github/codeql-action/init@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9 + with: + languages: ${{ matrix.language }} + queries: security-extended + + - name: Autobuild Go + if: ${{ matrix.language == 'go' }} + uses: github/codeql-action/autobuild@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9 + + - name: Analyze + uses: github/codeql-action/analyze@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9 + with: + category: /language:${{ matrix.language }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b636850..11263b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -251,7 +251,24 @@ jobs: - name: Audit dependencies working-directory: web - run: npm run audit + # shell: bash is load-bearing: the default bash -e {0} shell has no + # pipefail, so tee would report its own exit status and a failing audit + # would pass this gate. + shell: bash + run: npm run audit 2>&1 | tee "${RUNNER_TEMP}/web-audit-transcript.txt" + + - name: Upload audit evidence + if: always() && !cancelled() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: web-audit-transcript-${{ github.sha }} + path: ${{ runner.temp }}/web-audit-transcript.txt + if-no-files-found: error + retention-days: 14 + + - name: Run unit tests + working-directory: web + run: npm test - name: Lint working-directory: web @@ -351,6 +368,80 @@ jobs: node --check platforms.js npm pack --dry-run --ignore-scripts + windows-audit-evidence: + name: Windows audit evidence + # The batch helper exists to prove audit-retry.mjs works on a real cmd.exe, + # where only `npm run` supplies npm_execpath. Its regression test replays the + # batch source against a stub npm.cmd, so it proves the mechanics but never + # the execution. This job is the execution, and it is a job rather than an + # `if: runner.os == 'Windows'` step inside npm-wrapper-compatibility so the + # check name itself documents the Windows dependency. + runs-on: windows-2025 + timeout-minutes: 20 + + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: "24" + cache: npm + cache-dependency-path: web/package-lock.json + + - name: Install dependencies + working-directory: web + run: npm ci + + - name: Test Windows audit evidence helper + run: node --test scripts/test_win_audit_verify.mjs + + - name: Run Windows audit evidence helper against the real registry + working-directory: web + # shell: bash is load-bearing: with the default shell there is no + # pipefail, so tee's exit status would replace cmd.exe's and the + # helper's nonzero-audit assertion would become unfalsifiable. + shell: bash + run: | + # //d //c, not /d /c: git-bash rewrites a leading /d and /c into D:/ + # and C:/, which leaves cmd.exe with no option flags. It then prints + # its interactive banner and exits 0, so the helper never runs and the + # job goes green on an empty transcript. + cmd.exe //d //c "..\scripts\win-audit-verify.bat" 2>&1 \ + | tee "${RUNNER_TEMP}/win-audit-transcript.txt" + + - name: Verify the transcript is real evidence + shell: bash + # A successful cmd.exe proves nothing on its own; the transcript has to + # show the helper actually reached its report and gated on a passing + # audit, or this job would silently certify nothing. + run: | + transcript="${RUNNER_TEMP}/win-audit-transcript.txt" + for needle in \ + "\[win-audit-verify\] starting npm run audit" \ + "\[win-audit-verify\] finished at" \ + "\[win-audit-verify\] npm run audit exit code: 0" \ + "found 0 vulnerabilities" + do + if ! grep -q -- "$needle" "$transcript"; then + echo "::error::win-audit-verify.bat did not produce '$needle'; the Windows audit evidence is not real." + sed -n '1,40p' "$transcript" + exit 1 + fi + done + + - name: Upload Windows audit evidence + if: always() && !cancelled() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: win-audit-transcript-${{ github.sha }} + path: ${{ runner.temp }}/win-audit-transcript.txt + if-no-files-found: error + retention-days: 14 + deployment: name: Deployment profiles runs-on: ubuntu-24.04 @@ -364,3 +455,21 @@ jobs: - name: Validate deployment configuration and production images run: make test-deploy-build + + - name: Validate web security response headers + run: | + # The header contract is measured against the nginx that actually ships, + # so it runs on the image the step above built. That image ends its own + # build at USER 101, which cannot install packages, so the tools go into + # a throwaway child image that returns to that user id afterwards -- the + # harness then still runs unprivileged, as the container does in + # production. + printf 'FROM mem-web:deploy-validation\nUSER root\nRUN apk add --no-cache bash curl python3 gettext\nUSER 101\n' \ + >"${RUNNER_TEMP}/headers.Dockerfile" + docker build --tag mem-web-headers:validation \ + -f "${RUNNER_TEMP}/headers.Dockerfile" "${RUNNER_TEMP}" + docker run --rm \ + --volume "${GITHUB_WORKSPACE}:/src:ro" \ + --env NGINX_BIN=/usr/sbin/nginx \ + mem-web-headers:validation \ + bash /src/scripts/test_nginx_security_headers.sh diff --git a/.github/workflows/mcp-registry-publish.yml b/.github/workflows/mcp-registry-publish.yml new file mode 100644 index 0000000..afebc36 --- /dev/null +++ b/.github/workflows/mcp-registry-publish.yml @@ -0,0 +1,115 @@ +name: MCP Registry Publish + +# G5: official MCP Registry metadata. Does not host the binary. +# Requires G4: @bytefolk/mem-mcp must already be on npm with matching mcpName. +# Auth is GitHub OIDC against io.github.bytefolk/* — not a personal device login. +# Founder gate: environment mcp-registry (same owner as npm-release). + +on: + workflow_dispatch: + inputs: + version: + description: "Published npm/package version (e.g. 0.1.2, no v prefix)" + required: true + type: string + +permissions: + contents: read + +concurrency: + group: mcp-registry-publish-bytefolk-mem-mcp + cancel-in-progress: false + +jobs: + publish: + name: Publish io.github.bytefolk/mem-mcp (OIDC) + if: github.repository == 'bytefolk/mem' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + environment: mcp-registry + permissions: + contents: read + id-token: write + steps: + - name: Check out default branch for registry manifest + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Require @bytefolk/mem-mcp on npm before Registry write + env: + VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + if [[ ! "${VERSION}" =~ ^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]]; then + echo "expected X.Y.Z, got ${VERSION}" >&2 + exit 1 + fi + meta="$(curl -fsS "https://registry.npmjs.org/@bytefolk/mem-mcp/${VERSION}")" + name="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["name"])' <<<"${meta}")" + mcp="$(python3 -c 'import json,sys; print(json.load(sys.stdin).get("mcpName",""))' <<<"${meta}")" + [[ "${name}" == "@bytefolk/mem-mcp" ]] + [[ "${mcp}" == "io.github.bytefolk/mem-mcp" ]] + + - name: Install mcp-publisher + env: + MCP_PUBLISHER_VERSION: v1.8.1 + MCP_PUBLISHER_LINUX_AMD64_SHA256: a06c9096dcb9727c13555b6be26c7effa707b01f06a4c561ba7a3635443cf2cc + MCP_PUBLISHER_LINUX_ARM64_SHA256: 8dd75a6cf6845688b5d4e46df58d3ca26d5c8d233bb0626606e1db82c5e883e4 + run: | + set -euo pipefail + os="$(uname -s | tr '[:upper:]' '[:lower:]')" + arch="$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/')" + checksum_var="MCP_PUBLISHER_$(printf '%s_%s' "${os}" "${arch}" | tr '[:lower:]' '[:upper:]')_SHA256" + checksum="${!checksum_var:-}" + [[ "${checksum}" =~ ^[0-9a-f]{64}$ ]] || { + echo "no pinned mcp-publisher checksum for ${os}/${arch}" >&2 + exit 1 + } + archive="${RUNNER_TEMP}/mcp-publisher_${os}_${arch}.tar.gz" + curl -fsSL --retry 3 \ + "https://github.com/modelcontextprotocol/registry/releases/download/${MCP_PUBLISHER_VERSION}/mcp-publisher_${os}_${arch}.tar.gz" \ + --output "${archive}" + printf '%s %s\n' "${checksum}" "${archive}" | sha256sum --check --strict + tar -xzf "${archive}" mcp-publisher + chmod +x ./mcp-publisher + + - name: Stage official server.json + env: + VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + python3 - <<'PY' + import json, os + path = "npm/mcp-registry.server.json" + with open(path) as f: + doc = json.load(f) + version = os.environ["VERSION"] + doc["version"] = version + doc["packages"][0]["version"] = version + with open("server.json", "w") as f: + json.dump(doc, f, indent=2) + f.write("\n") + PY + + - name: Login with GitHub OIDC and publish + run: | + set -euo pipefail + ./mcp-publisher login github-oidc + ./mcp-publisher publish + + - name: Read back Registry search + env: + VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + curl -fsS "https://registry.modelcontextprotocol.io/v0.1/servers?search=io.github.bytefolk/mem-mcp" \ + | tee "${RUNNER_TEMP}/mcp-registry-readback.json" + python3 - <<'PY' + import json, os, sys + data = json.load(open(os.environ["RUNNER_TEMP"] + "/mcp-registry-readback.json")) + servers = data.get("servers") or data.get("result") or [] + if not servers: + sys.exit("HOLD: registry search returned no servers") + print("G5 readback: %s" % json.dumps(servers[0])[:2000]) + PY diff --git a/.github/workflows/memory-validation.yml b/.github/workflows/memory-validation.yml index ed42033..f7f2943 100644 --- a/.github/workflows/memory-validation.yml +++ b/.github/workflows/memory-validation.yml @@ -53,6 +53,7 @@ jobs: scripts/acceptance_agent_memory.sh \ scripts/generate_release_checksums.sh \ scripts/render_release_notes.sh \ + scripts/test_release_checksum_output_safety.sh \ scripts/test_release_helpers_compat.sh \ scripts/test_release_guards.sh \ scripts/test_validate_release_action_pins_compat.sh \ diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml new file mode 100644 index 0000000..103adac --- /dev/null +++ b/.github/workflows/npm-publish.yml @@ -0,0 +1,112 @@ +name: NPM Publish + +# GitHub binary publication stays in release.yml. A Release created with the +# repository GITHUB_TOKEN may not trigger this workflow; dispatch the exact tag +# explicitly after G1 asset verification (see docs/maintainers/releasing.md). +on: + release: + types: [published] + workflow_dispatch: + inputs: + version: + description: "Existing stable tag, also selected as the workflow ref (v0.1.2)" + required: true + type: string + +permissions: + contents: read + +# Serialize every version of this package: next is shared across releases. +concurrency: + group: npm-publish-bytefolk-mem-mcp + cancel-in-progress: false + +jobs: + npm-publish: + name: Publish verified npm tarball to next (OIDC) + if: github.repository == 'bytefolk/mem' + runs-on: ubuntu-24.04 + timeout-minutes: 20 + environment: npm-release + permissions: + contents: read + id-token: write + env: + NPM_RELEASE_PROOF: ${{ vars.NPM_RELEASE_PROOF }} + steps: + - name: Validate exact stable event and tag before checkout + id: tag + env: + INPUT_VERSION: ${{ inputs.version }} + RELEASE_TAG_NAME: ${{ github.event.release.tag_name }} + run: | + set -euo pipefail + tag="${INPUT_VERSION:-${RELEASE_TAG_NAME}}" + if [[ ! "${tag}" =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]]; then + echo 'HOLD: an exact stable vX.Y.Z tag is required' >&2 + exit 1 + fi + [[ "${GITHUB_REF}" == "refs/tags/${tag}" ]] + [[ "${GITHUB_EVENT_NAME}" == release || "${GITHUB_EVENT_NAME}" == workflow_dispatch ]] + printf 'tag=%s\n' "${tag}" >> "${GITHUB_OUTPUT}" + + - name: Check out exact tag commit + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: refs/tags/${{ steps.tag.outputs.tag }} + fetch-depth: 0 + persist-credentials: false + + - name: Set up Node 24 without release caches + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + with: + node-version: '24' + package-manager-cache: false + + - name: Require current release-owner org and publisher proof + env: + RELEASE_TAG: ${{ steps.tag.outputs.tag }} + run: | + node --input-type=module -e ' + import { checkProof } from "./scripts/npm-release.mjs"; + checkProof(JSON.parse(process.env.NPM_RELEASE_PROOF || "null"), + process.env.RELEASE_TAG, process.env.GITHUB_SHA); + ' + + - name: Install reviewed npm CLI with lifecycle scripts disabled + run: | + set -euo pipefail + node -e ' + const fs = require("node:fs"), p = process.env.RUNNER_TEMP; + fs.writeFileSync(p + "/bootstrap-user.npmrc", "", {flag: "wx", mode: 0o600}); + fs.writeFileSync(p + "/bootstrap-global.npmrc", "", {flag: "wx", mode: 0o600}); + ' + NPM_CONFIG_USERCONFIG="${RUNNER_TEMP}/bootstrap-user.npmrc" \ + NPM_CONFIG_GLOBALCONFIG="${RUNNER_TEMP}/bootstrap-global.npmrc" \ + NPM_CONFIG_CACHE="${RUNNER_TEMP}/bootstrap-npm-cache" \ + npm install --global npm@11.15.0 --ignore-scripts --registry=https://registry.npmjs.org + + - name: Test release refusal paths and npm wrapper + run: | + node --test scripts/npm-release.test.mjs + npm test --prefix npm + + - name: Set up Go for read-only binary metadata inspection + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 + with: + go-version-file: server/go.mod + cache: false + + - name: Preflight, publish next with provenance, verify registry and signatures + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ steps.tag.outputs.tag }} + run: node scripts/npm-release.mjs "${RELEASE_TAG}" + + - name: Record next receipt and separate owner gates + run: | + node <<'NODE' + const fs = require("node:fs"); + const receipt = fs.readFileSync(process.env.RUNNER_TEMP + "/mem-npm-release/receipt.json", "utf8"); + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, "Published to next; latest requires release-owner acceptance.\n\n```json\n" + receipt + "\n```\n"); + NODE diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1d49da3..2d71fc6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -72,8 +72,8 @@ jobs: exit 1 fi - build: - name: Build mem-mcp binaries + build-mcp: + name: Build mem-mcp (${{ matrix.goos }}/${{ matrix.goarch }}) needs: [preflight] runs-on: ubuntu-24.04 timeout-minutes: 20 @@ -147,9 +147,88 @@ jobs: if-no-files-found: error retention-days: 1 + build-server: + name: Build server (${{ matrix.goos }}/${{ matrix.goarch }}) + needs: [preflight] + runs-on: ubuntu-24.04 + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + - goos: linux + goarch: amd64 + - goos: linux + goarch: arm64 + - goos: darwin + goarch: amd64 + - goos: darwin + goarch: arm64 + steps: + - name: Check out exact tag commit + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.preflight.outputs.commit }} + fetch-depth: 0 + persist-credentials: false + + - name: Revalidate release source + env: + EXPECTED_COMMIT: ${{ needs.preflight.outputs.commit }} + RELEASE_TAG: ${{ needs.preflight.outputs.tag }} + run: | + set -euo pipefail + git fetch --no-tags origin \ + "refs/heads/main:refs/remotes/origin/main" \ + "refs/tags/${RELEASE_TAG}:refs/tags/${RELEASE_TAG}" + actual_commit="$(./scripts/validate_release_source.sh "${RELEASE_TAG}")" + [[ "${actual_commit}" == "${EXPECTED_COMMIT}" ]] + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 + with: + go-version-file: server/go.mod + cache-dependency-path: server/go.sum + + - name: Build server binaries (${{ matrix.goos }}/${{ matrix.goarch }}) + working-directory: server + env: + EXPECTED_COMMIT: ${{ needs.preflight.outputs.commit }} + RELEASE_TAG: ${{ needs.preflight.outputs.tag }} + run: | + set -euo pipefail + semver="${RELEASE_TAG#v}" + revision="${EXPECTED_COMMIT}" + contract="durable-context.v1" + ldflags="-s -w -buildvcs=true" + ldflags="${ldflags} -X github.com/PeterGuy326/mem/server/internal/api.Version=${semver}" + ldflags="${ldflags} -X github.com/PeterGuy326/mem/server/internal/api.Revision=${revision}" + ldflags="${ldflags} -X github.com/PeterGuy326/mem/server/internal/api.ContractVersion=${contract}" + + mkdir -p "${RUNNER_TEMP}/assets" + suffix="${{ matrix.goos }}-${{ matrix.goarch }}" + + for target in memd mem-migrate mem-healthcheck mem; do + output="${RUNNER_TEMP}/assets/${target}-${suffix}" + CGO_ENABLED=0 GOOS="${{ matrix.goos }}" GOARCH="${{ matrix.goarch }}" \ + go build -buildvcs=true -trimpath \ + -ldflags="${ldflags}" \ + -o "${output}" "./cmd/${target}" + go version -m "${output}" | grep -F "vcs.revision=${revision}" + go version -m "${output}" | grep -F 'vcs.modified=false' + done + + - name: Upload server assets + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: server-${{ matrix.goos }}-${{ matrix.goarch }} + path: ${{ runner.temp }}/assets/* + if-no-files-found: error + retention-days: 1 + release: name: Create GitHub Release - needs: [preflight, build] + needs: [preflight, build-mcp, build-server] runs-on: ubuntu-24.04 timeout-minutes: 10 permissions: @@ -181,7 +260,6 @@ jobs: uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: path: /tmp/release-assets - pattern: mem-mcp-* merge-multiple: true - name: Validate exact assets and generate checksums @@ -209,12 +287,29 @@ jobs: fi assets=( + /tmp/release-assets/memd-darwin-amd64 + /tmp/release-assets/memd-darwin-arm64 + /tmp/release-assets/memd-linux-amd64 + /tmp/release-assets/memd-linux-arm64 + /tmp/release-assets/mem-migrate-darwin-amd64 + /tmp/release-assets/mem-migrate-darwin-arm64 + /tmp/release-assets/mem-migrate-linux-amd64 + /tmp/release-assets/mem-migrate-linux-arm64 + /tmp/release-assets/mem-healthcheck-darwin-amd64 + /tmp/release-assets/mem-healthcheck-darwin-arm64 + /tmp/release-assets/mem-healthcheck-linux-amd64 + /tmp/release-assets/mem-healthcheck-linux-arm64 + /tmp/release-assets/mem-darwin-amd64 + /tmp/release-assets/mem-darwin-arm64 + /tmp/release-assets/mem-linux-amd64 + /tmp/release-assets/mem-linux-arm64 /tmp/release-assets/mem-mcp-darwin-amd64 /tmp/release-assets/mem-mcp-darwin-arm64 /tmp/release-assets/mem-mcp-linux-amd64 /tmp/release-assets/mem-mcp-linux-arm64 /tmp/release-assets/mem-mcp-windows-amd64.exe /tmp/release-assets/mem-mcp-windows-arm64.exe + /tmp/release-assets/mem-checksums.txt /tmp/release-assets/mem-mcp-checksums.txt ) release_flags=() @@ -244,21 +339,38 @@ jobs: jq -e --arg tag "${RELEASE_TAG}" ' .tagName == $tag and .isDraft == true and - (.assets | length == 7) and + (.assets | length == 24) and all(.assets[]; ((.size | type) == "number") and (.size > 0)) ' <<< "${release_json}" >/dev/null expected_assets="$(printf '%s\n' \ + mem-checksums.txt \ + mem-darwin-amd64 \ + mem-darwin-arm64 \ + mem-linux-amd64 \ + mem-linux-arm64 \ + mem-healthcheck-darwin-amd64 \ + mem-healthcheck-darwin-arm64 \ + mem-healthcheck-linux-amd64 \ + mem-healthcheck-linux-arm64 \ mem-mcp-checksums.txt \ mem-mcp-darwin-amd64 \ mem-mcp-darwin-arm64 \ mem-mcp-linux-amd64 \ mem-mcp-linux-arm64 \ mem-mcp-windows-amd64.exe \ - mem-mcp-windows-arm64.exe | LC_ALL=C sort)" + mem-mcp-windows-arm64.exe \ + mem-migrate-darwin-amd64 \ + mem-migrate-darwin-arm64 \ + mem-migrate-linux-amd64 \ + mem-migrate-linux-arm64 \ + memd-darwin-amd64 \ + memd-darwin-arm64 \ + memd-linux-amd64 \ + memd-linux-arm64 | LC_ALL=C sort)" actual_assets="$(jq -r '.assets[].name' <<< "${release_json}" | LC_ALL=C sort)" if [[ "${actual_assets}" != "${expected_assets}" ]]; then - echo "draft Release asset inventory does not match the expected seven files" >&2 + echo "draft Release asset inventory does not match the expected files" >&2 printf 'expected:\n%s\nactual:\n%s\n' "${expected_assets}" "${actual_assets}" >&2 exit 1 fi @@ -283,17 +395,34 @@ jobs: jq -e --arg tag "${RELEASE_TAG}" ' .tagName == $tag and .isDraft == true and - (.assets | length == 7) and + (.assets | length == 24) and all(.assets[]; ((.size | type) == "number") and (.size > 0)) ' <<< "${release_json}" >/dev/null expected_assets="$(printf '%s\n' \ + mem-checksums.txt \ + mem-darwin-amd64 \ + mem-darwin-arm64 \ + mem-linux-amd64 \ + mem-linux-arm64 \ + mem-healthcheck-darwin-amd64 \ + mem-healthcheck-darwin-arm64 \ + mem-healthcheck-linux-amd64 \ + mem-healthcheck-linux-arm64 \ mem-mcp-checksums.txt \ mem-mcp-darwin-amd64 \ mem-mcp-darwin-arm64 \ mem-mcp-linux-amd64 \ mem-mcp-linux-arm64 \ mem-mcp-windows-amd64.exe \ - mem-mcp-windows-arm64.exe | LC_ALL=C sort)" + mem-mcp-windows-arm64.exe \ + mem-migrate-darwin-amd64 \ + mem-migrate-darwin-arm64 \ + mem-migrate-linux-amd64 \ + mem-migrate-linux-arm64 \ + memd-darwin-amd64 \ + memd-darwin-arm64 \ + memd-linux-amd64 \ + memd-linux-arm64 | LC_ALL=C sort)" actual_assets="$(jq -r '.assets[].name' <<< "${release_json}" | LC_ALL=C sort)" [[ "${actual_assets}" == "${expected_assets}" ]] diff --git a/AGENTS.md b/AGENTS.md index 340befe..7144859 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,7 +1,7 @@ # Repository Instructions for Agents These instructions apply to the entire repository. They supplement the public -[organization contribution rules](https://github.com/fullstack-ai-infra/.github/blob/main/CONTRIBUTING.md) +[organization contribution rules](https://github.com/bytefolk/.github/blob/main/CONTRIBUTING.md) and the `mem`-specific contracts in `docs/DEVELOPMENT.md`. ## Required workflow diff --git a/CHANGELOG.md b/CHANGELOG.md index 66042e5..563e24c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,121 @@ The project publishes 0.x prerelease versions; a stable release line is not yet ## [Unreleased] +### Changed + +- Ingest cursor locks try non-blocking exclusive locks and give up after 5s so a wedged peer becomes a warning instead of a silent hang. Refs #139. + +## [0.1.2] - 2026-09-18 + +### Added + +- Additive `durable-memory.v1` envelope for derived RoleWeave/mem records + (`#220`). Isolation is workspace + position principal + `memory_scope` + + grant/revocation (reusing `durable-context.v1` grants and + `capability-grant.v1`); a free-string `scope` is rejected. Eligibility treats + expired, revoked, malformed, superseded, forgotten, and out-of-scope records + as ineligible. Pin does not enlarge permission. TTL/expiry is recall + eligibility, not physical deletion of the source log. Forget stays + permissioned and never a local fake delete. Grant status and + `permission_digest` enter the readback receipt. Contract only: no HTTP, + migration, or MCP wiring before Gate D0. Schema: + `docs/schemas/durable-memory.v1.schema.json`. +- Cosine HNSW indexes on `embeddings_text` (768), `embeddings_visual` (512), + and `embeddings_face` (512) via migration 0025 (`#173`). Text search walks + `ORDER BY embedding <=> query LIMIT n` (planner-usable) and falls back to + exact per-file `DISTINCT ON` when a bounded scan underfills after + deduplication. Visual cosine-order queries can use the visual index. Face + clustering remains in-process; the face index is DDL only. Recall is not + claimed here; the live harness is `#175`. +- Advertise the model-free lexical file-search route in the MCP `mem_search` + schema and verify `tools/list` plus route/filter forwarding through `tools/call`. + +- `mem doctor` — a read-only diagnosis of why the CLI cannot talk to a working + server (`#112`). It reports four checks in a fixed order: reachability of the + configured server URL, whether a credential exists, the workspace the server + resolved for that credential, and CLI/server version skew. Each finding carries + the SPEC §7.1 exit code it contributes (`0` ok · `2` not_found · `3` auth · + `4` plan/quota · `5` provider/timeout), and a check that an earlier failure made + impossible is reported as `skipped` instead of guessed. It issues only `GET` + requests and never writes configuration, starts a container, or installs a + dependency; `--format json` emits the `mem.doctor` v1 document described by + `docs/schemas/mem-doctor.v1.schema.json`. A token is described only by where it + came from, and a configured URL has its userinfo and its query parameter values + replaced by `REDACTED` — a credential in a query parameter is the shape pgx + accepts as a real password — or, when the URL cannot be proven to be a + credential-free transport URL, is withheld whole. See `docs/DEPLOYMENT.md`. +- First-run guidance: a command that fails because no credential exists now says + so on a machine with no configuration at all by naming the documented + deployment path (`deploy/compose`, `docs/DEPLOYMENT.md`), instead of telling + somebody to log in against a server that is not running yet. Hosts that already + have a configuration keep the previous, shorter hint. +- File search gains a model-free lexical route (`route=lexical`). FTS + trigram + over `files.name` — same tier shape as memory Recall — so a deployment with + no embedding worker can still find files by name. CLI: `mem search "query" + --route lexical`. +- Publish installable server binaries (`memd`, `mem-migrate`, `mem-healthcheck`, + `mem`) alongside `mem-mcp` in the release workflow for `darwin/arm64`, + `darwin/amd64`, `linux/arm64` and `linux/amd64`, with per-group checksum + manifests (`#151`). +- `/v1/version` now exposes `version` (semver), `revision` (40-hex git commit) + and `contract` (durable-context wire contract) as distinct fields, so clients + can pin a revision or accept a version range without a third mechanism. Both + the release workflow and the Docker image inject all three at build time + (`#151`). +- Documented first-run path in `docs/DEPLOYMENT.md` that yields a reachable + endpoint, a workspace, a token with write and recall scopes, and the + corresponding durable-context grant (`#151`). + +### Changed + +- README onboarding now leads with the `deploy/compose` path as the recommended + first-run experience (one-shot: `generate-env` → `compose up` → register → + use CLI/MCP). The bare-metal development path (`scripts/dev_up.sh`) is + demoted to a development-only subsection, and `docs/RUN_LOCAL.md` adds a + platform-equivalence table covering macOS, Ubuntu/Debian and WSL2 (`#109`). + `mem doctor` already names `deploy/compose` on a machine with no config. +- Migrate GitHub repository, Release, issue, badge, and raw-content coordinates + to the canonical `bytefolk` organization. +- Rename the npm wrapper to `@bytefolk/mem-mcp@0.1.2` and the MCP registry + identity to `io.github.bytefolk/mem-mcp`. New executable caches use + `bytefolk/mem-mcp`; a matching version/platform in the old + `fullstack-ai-infra/mem-mcp` cache can seed a separately verified copy. + Old cache entries, including 0.1.1, are never changed or removed by this + compatibility lookup. Explicit cache overrides keep their existing meaning. + The old npm package remains available for rollback; migration does not + unpublish it or change stored memories. Update host package arguments using + the migration guide in `npm/README.md`. + `npm/registry-identity.test.js` asserts the identifier against the + repository coordinate the installer itself uses. +- Internal: the local ingestion mechanics used by + `mem ingest qoder` — deterministic recursive transcript walk, per-path line + cursors (atomic rename write, reset when a file is rewritten shorter), the + `--dry-run` / `--limit` semantics, per-file degradation on an idempotency + conflict, run-report aggregation and the closed failure-code vocabulary — + moved out of `server/cmd/mem` into a new `server/internal/ingest` package + (`#111`). The connector is now a thin call site that supplies the Qoder parser, + the memory payload and the HTTP upload. Memories payloads, the stdout summary, + and the cursor file format and location under `~/.mem/ingest/qoder` are + unchanged, and a root already given as a canonical absolute path keeps the + cursor keys and `Idempotency-Key` values it had before the move. A root given + relative, or one reached through a symlink, is now identified by its canonical + absolute path, so its cursor key and per-line `Idempotency-Key` differ from the + pre-refactor spelling. The package is the shared core that `put --watch` + (`#110`) consumes instead of writing a second state layer. + +### Fixed + +- Recursive folder delete now removes the corresponding objects from bucket + storage after the database transaction commits (`#177`). Previously, the DB + rows were deleted but the blobs remained orphaned in the bucket. The cleanup + is best-effort: a failed object delete does not roll back the folder delete, + and failures are logged at WARN level so the operator can see which keys + remain. The folder service now accepts an optional `ObjectStore` and logger + via `folder.WithStore` and `folder.WithLogger` options. Recursive delete + also refuses when an active or archived memory outside the folder still + cites a file in the tree via `source_file_id`, so blob cleanup cannot + destroy a live citation through `ON DELETE SET NULL`. + ### Security - Normalize the client-declared MIME type of a stored file before deciding how @@ -24,6 +139,90 @@ The project publishes 0.x prerelease versions; a stable release line is not yet - Add `nosniff`, `X-Frame-Options: DENY`, `Referrer-Policy: no-referrer` and `Content-Security-Policy: default-src 'none'` to every API response, ordered outside the CORS handler so a preflight reply carries them too. +- Make the web proxy the single authority for `X-Content-Type-Options`, + `X-Frame-Options` and `Referrer-Policy` on every response it serves, with one + `Referrer-Policy: no-referrer` instead of the `same-origin` it shipped before, + which contradicted the API's own value and let both reach a client on the + proxied path. The API's copies are hidden at the proxy rather than removed + from the API, so a `memd` running without a proxy keeps its defense in depth. + Cached assets carried none of the three: a local `add_header` for + `Cache-Control` replaced the inherited set entirely, so the set is now + restated in that location. `scripts/test_nginx_security_headers.sh` measures + the headers off a running nginx, since neither failure mode is visible by + reading the configuration. + +### Fixed + +- Follow the shared design language for reading, numeric and action alignment; generate the existing Web color variables from a pinned design-system token snapshot, and use a single consistent empty-state pattern. Refs #211. + +- Improve web caption and status contrast in both themes, including tinted danger + buttons, and center action labels, context menus, badges, dialog prompts, and + overview/detail headings. Restore localized cancel/confirm labels when a + confirmation dialog caller omits custom action text. + +- Every MinIO image reference in the test stack, the local development stack and + the self-hosted single-node Compose profile now resolves from `quay.io` instead + of Docker Hub. MinIO stopped publishing container images in October 2025 and + removed the `minio/minio` and `minio/mc` repositories from Docker Hub, so an + anonymous `docker compose up` fails with `pull access denied for minio/minio, + repository does not exist or may require 'docker login'`. Because `Validate + Agent memory` → `HTTP, CLI and MCP lifecycle` is a required status context, + that registry withdrawal blocked every pull request from merging (`#207`). + `quay.io` still serves the exact digests pinned in `docker-compose.test.yml`, + so no image bytes change: the digest-pinned test stack keeps its digests and + the release-tagged deployment stack keeps its tags — only the registry host + differs. `deploy/compose/compose.yaml` previously carried the same broken + reference, so the documented self-hosted path would have failed on a cold + host even though no workflow exercises it. +- CI Web job now runs unit tests (`npm test`), including audit retry regression + tests. `npm run audit` retries recognized transient registry failures up to + three attempts per threshold (two retries), with a 60-second limit per attempt + and portable backoff. It starts npm through Node on Windows, fails immediately + for vulnerabilities, unknown errors or incomplete runs, and echoes the output + of every attempt it retries so a self-healing failure still leaves a trace. + That transcript is uploaded as a downloadable artifact + (`web-audit-transcript-`, 14-day retention) on the success and failure + paths alike, and because the step merges stderr into stdout the artifact also + carries the per-attempt diagnostics. A dedicated `Windows audit evidence` job + runs the audit through `scripts/win-audit-verify.bat` on a real Windows runner + and uploads `win-audit-transcript-`, so the helper is executed rather + than only replayed against a stub by its fixture test. +- A configured URL that carries credentials in a shape `url.Parse` does not + report as userinfo no longer reaches output. `admin:pw@host` parses as + `Scheme="admin"` with the credential in `Opaque` and `User` unset, so an + implementation that gates on `User != nil` echoes it verbatim. On this base it + leaked from the CLI API client — at request construction and at all four + `http.Client.Do` sites, which the previous error path did not cover — and from + `memd`'s startup log line and its fatal log line, the last of which additionally + carries third-party errors that embed a whole DSN. Both now route through one + shared gate that redacts a value it can prove is a transport URL and + **withholds the value whole** otherwise. It does not scrub credentials out of + error text, which cannot be made tight: `url.Error` renders with `%q`, so a + quote inside a password arrives escaped and a scanner that pairs quotes + mis-pairs and replaces nothing. Withholding costs some diagnosability by design; + why a request failed is still reported, and a DSN still names the parameters it + sets — every query parameter *value* is replaced by `REDACTED`, including + `?password=`, which pgx honours as the real password. +- Checksum manifest generation rejects existing output files, directories and + symlinks without modifying their targets, including dangling symlinks, and + will not publish a manifest that is missing a row for an expected asset. +- Release validation requires a release's CHANGELOG link to start at the + preceding CHANGELOG release, and accepts a link to that release's own page + only when no release precedes it. Checksum generation handles each asset path + separately on GNU and BSD tools, including directories with spaces, rejects + empty sets, and reports a failed asset listing as a failed listing. +- The release guard suites count manifest lines without `wc -l`, whose BSD + implementation pads the count with blanks, and no longer need GNU + `find -printf`. +- Add an opt-in file-search ranking producer with explicit request failures, conservative result identity mapping, and operator-declared configuration. Live provider quality remains separately unverified. + +- The npm installer no longer aborts a concurrent first run on Windows. The + per-asset cache lock previously treated only `EEXIST` as contention, but a + contended `mkdir` on Windows may raise `EPERM` or `EACCES`, so a process + waiting for the lock holder failed outright instead of retrying. The retry + path now proves a lock can be inspected before treating those Windows errors + as contention, preserves prompt failure for unrelated permission errors, and + observes its deadline when a competing lock disappears during inspection. ## [0.1.1] - 2026-08-31 @@ -39,6 +238,28 @@ The project publishes 0.x prerelease versions; a stable release line is not yet ### Fixed +- `mem ingest qoder` derives a transcript's checkpoint key, its per-line + `Idempotency-Key` values and its project/session memory path from one canonical + root identity. Previously the root was used exactly as the caller spelled it, so + two working directories that each contained `sessions/p.jsonl` and shared one + checkpoint directory collided on a single cursor: the second run saw an + up-to-date checkpoint and posted nothing, silently dropping that store. A + relative or symlinked root now re-keys its existing cursors, which replays those + files once instead of skipping them. Checkpoint writes stage through a distinct + temporary file per save and no longer rewind a checkpoint another run already + advanced, so a slower run finishing second can neither fail on a shared staging + name nor undo the faster run's progress. +- An ingest cycle that aborted on a rejected write tallied the failure under the + network code whatever the server had answered, because the connector mapped the + typed API error to a CLI error before the shared core could classify it. The + core now sees the typed error and classifies authentication, plan, quota, + provider and timeout responses correctly, while the SPEC §7.1 exit codes stay + mapped at the command boundary that owns them. The same tally was wrong for + local reads: a failed `open` returns a `syscall.Errno`, which satisfies + `net.Error`, so an absent or unreadable transcript was reported as a network + failure instead of `root_missing` / `read_denied`, and a run that aborted while + reading recorded no failure at all. Both paths now classify before the transport + default and the aborted cycle reports the code it died on. - The npm installer now verifies the selected Release binary against the release's SHA-256 manifest before making it executable, rejects malformed or ambiguous manifest entries, verifies cached binaries, and removes partial or @@ -258,7 +479,7 @@ The project publishes 0.x prerelease versions; a stable release line is not yet top-level paths remain hidden compatibility aliases with deprecation warnings. - Inherit organization-wide contribution, issue, pull-request, conduct, and - support defaults from `fullstack-ai-infra/.github`; keep only `mem`-specific + support defaults from `bytefolk/.github`; keep only `mem`-specific development, security, triage, ownership, release, and validation rules in this repository. - Align pull-request policy with the inherited controlled exceptions for @@ -349,6 +570,7 @@ The project publishes 0.x prerelease versions; a stable release line is not yet - Preserve the primary Web acceptance failure when browser or Vite cleanup also fails. -[Unreleased]: https://github.com/fullstack-ai-infra/mem/compare/v0.1.1...HEAD -[0.1.1]: https://github.com/fullstack-ai-infra/mem/compare/v0.1.0...v0.1.1 -[0.1.0]: https://github.com/fullstack-ai-infra/mem/releases/tag/v0.1.0 +[Unreleased]: https://github.com/bytefolk/mem/compare/v0.1.2...HEAD +[0.1.2]: https://github.com/bytefolk/mem/compare/v0.1.1...v0.1.2 +[0.1.1]: https://github.com/bytefolk/mem/compare/v0.1.0...v0.1.1 +[0.1.0]: https://github.com/bytefolk/mem/releases/tag/v0.1.0 diff --git a/GOAL.md b/GOAL.md index ef7d4bd..8515d46 100644 --- a/GOAL.md +++ b/GOAL.md @@ -92,9 +92,9 @@ mem 只有一份数据和能力内核，但面向两类使用者： | 跨 Agent 记忆 | `remember/context`、出处、幂等、反馈、归档、恢复和遗忘控制闭环已实现 | **MVP 对齐** | | 标准任务交接 | `mem.handoff` v1、不可变 checkpoint、CAS、resume、哈希引用和缺失项报告已实现 | **MVP 对齐** | | Claude Code / Codex 迁移 | 以 Claude Code / Codex 身份隔离的两个独立 Token 已通过真实 HTTP/PostgreSQL 写入与只读恢复验收；两端有同一个标准 MCP adapter 的接入说明 | **MVP 对齐；仍需真实宿主进程的发布级 smoke** | -| 跨设备恢复 | 同一部署可登录 workspace 继续使用；跨部署可导出 `.membundle` 并 `fresh` 导入空 workspace，带完整性校验、幂等 ledger 和结构化冲突 | **部分对齐：尚无 merge、增量同步和断点上传** | -| 人类可视化 | Web 已覆盖 Drive、Search、Tasks、checkpoint/Resume、Memories 生命周期与 Workspace Transfer | **MVP 对齐：尚缺 correction/supersede 与完整审计历史** | -| 数据可移植性 | workspace bundle v1 有开放 schema、七类索引、payload/blob checksum、依赖校验和真实数据库 round-trip | **MVP 对齐：当前服务只支持 fresh restore** | +| 跨设备恢复 | 同一部署可登录 workspace 继续使用；跨部署可导出 `.membundle` 并 `fresh` 导入空 workspace，带完整性校验、幂等 ledger 和结构化冲突；`merge_conservative` 已实现 | **部分对齐：merge 已完成，尚无增量同步和断点上传** | +| 人类可视化 | Web 已覆盖 Drive、Search、Tasks、checkpoint/Resume、Memories 生命周期与 Workspace Transfer；correction/supersede 关系已实现 | **MVP 对齐：尚缺完整审计历史** | +| 数据可移植性 | workspace bundle v1 有开放 schema、七类索引、payload/blob checksum、依赖校验和真实数据库 round-trip；`merge_conservative` 导入已实现 | **MVP 对齐：尚无增量同步和断点上传** | | 自然语言搜图 | 原始图片字节→512 维视觉向量→文本塔查询→可回原件的链路已实现；真实英文固定集通过 | **链路对齐、质量未完全对齐：默认模型中文固定集未通过** | 因此，项目现在不再只是“可视化的 Agent Memory / AI 搜索网盘”，而是已经具备 @@ -120,13 +120,14 @@ mem 只有一份数据和能力内核，但面向两类使用者： - [x] 定义包含 manifest、内容哈希、schema 版本和依赖关系的 `.membundle` v1。 - [x] 实现 API / CLI / Web 的 workspace export 与空目标 `fresh` import。 - [x] 实现导入前校验、幂等 ledger、冲突明细、失败补偿和导入后重新索引。 -- [ ] 实现 `merge_conservative`、增量包、断点上传与完整本地同步盘体验。 +- [x] 实现 `merge_conservative`。 +- [ ] 实现增量包、断点上传与完整本地同步盘体验。 ### P2 — 让全部 Agent 数据可见可控 - [x] 在 Web UI 增加记忆、任务交接、来源、版本、Resume 与迁移视图。 - [x] 提供反馈、置顶、归档、恢复、确认遗忘和 workspace 导出/导入。 -- [ ] 提供不可变 correction/supersede、导入历史和更完整的权限管理界面。 +- [x] 提供不可变 correction/supersede、导入历史和更完整的权限管理界面。 ### 持续主线 — 自然语言搜图与多模态召回 diff --git a/GOVERNANCE.md b/GOVERNANCE.md new file mode 100644 index 0000000..8c53b3b --- /dev/null +++ b/GOVERNANCE.md @@ -0,0 +1,162 @@ +# Release Governance + +This document supplements the repository workflow in [AGENTS.md](AGENTS.md) and the +release policy in [docs/maintainers/releasing.md](docs/maintainers/releasing.md). It +records the repository's branch-protection, tag-immutability, and release-cut review +governance without weakening or replacing either document. Repository settings are +the immediate mechanical enforcement; any drift between them and this additive +charter must be corrected without weakening the stronger rule. + +## Motivation + +The v0.1.0 release on 2026-08-30 exposed three gaps: + +1. The `v0.1.0` tag had multiple create/delete cycles in one day while a broken + release workflow was iterated on. Tag history was not immutable. +2. Issue #81 resolved the structurally unsatisfiable single-CODEOWNER setup by + adding a second owner. A separate gap remained: branch protection did not apply + to administrators, so an administrator could merge without the otherwise-required + independent approval. The recent-five audit covered #117, #115, #114, #108, and + #105. +3. A broken `download-artifact` SHA reached the release workflow because no reviewer + saw the release-cut PR before it was merged. + +Issue #124 records the resulting decisions through revision R4: R2 applied the narrow +admin-enforcement change, R3 canonicalized the founder-approved addition of a third code +owner, and R4 corrected the lifecycle record. The issue is closed. The matching branch +protection and the two `refs/tags/v*` rulesets enforce it mechanically. + +Where this charter states a configuration value, it records a point-in-time read of that +configuration. GitHub's live settings stay authoritative and some of them are readable +only by repository administrators, so a stale line here is a documentation defect to +correct, never a change in enforcement and never a reason to weaken the stronger rule. + +## Branch protection on `main` + +`main` is protected with the following non-negotiable settings: + +- **`enforce_admins: true`** — administrators are **not** exempt, preventing future + administrator bypasses without changing the status of historical merges. This value is + the field-for-field read-back recorded in #124 after the narrow + `POST .../protection/enforce_admins` change. Contributors without admin cannot read the + endpoint; they verify it by observing that a merge is blocked, not by reading it. +- **Strict required status checks** (`strict: true`): every required check must pass on a + head that is up to date with `main` before a merge. The authoritative required-check list + is repository configuration readable only by administrators. As of 2026-09-03 the check + jobs observed on this repository are `Go`, `Worker`, `Web`, + `Conventional title and linked issue`, `Workflow, scripts and Compose`, + `PostgreSQL integration`, `Web memory and transfer acceptance`, + `HTTP, CLI and MCP lifecycle`, `Agent host MCP contract`, `Deployment profiles`, + `Offline recall benchmark`, `npm wrapper`, and `npm wrapper compatibility` + (`node18-linux`, `node20-linux`, `node24-windows`). That enumeration is evidence of what + runs, not a substitute for the configuration, and it is not presented here as the exact + required set. +- **Required pull request reviews**: `required_approving_review_count = 1`, + `require_code_owner_reviews = true`, `dismiss_stale_reviews = true`, and + `require_last_push_approval = true`. +- **Required linear history** and **required conversation resolution** are enabled. +- **Force pushes and branch deletion are disabled**. +- **No direct pushes** to `main`. All changes go through a pull request. + +Branch protection has **no bypass actors** (`bypass_pull_request_allowances` is empty), so +there is no role-based route around the review requirement. That is a separate control from +the tag-creation bypass described under Tag immutability; the two must not be conflated. + +A pull request whose author is a CODEOWNER requires approval from another current +CODEOWNER. Self-approval and merges without a current independent approval are not +permitted. After the current head has that approval and all gates pass, an eligible +author or maintainer may perform the normal merge. + +## Tag immutability + +Release tags matching `refs/tags/v*` are covered by two active rulesets, and their bypass +posture is **not** symmetric: + +- `Protect stable release tags` (`21888356`) blocks `update` and `deletion`, and has **no + bypass actors**. No role, repository admin included, can move or delete a published `v*` + tag. +- `Restrict stable release tag creation` (`21899500`) blocks `creation`, and has **exactly + one** bypass actor: repository role `admin` (`repositoryRoleDatabaseId` 5), + `bypass_mode: always`. A repository admin can cut a `v*` tag directly. + +That asymmetry is the point. Creation stays reachable so a release can always be cut by an +admin, while published tags stay immutable for everyone, admins included. Immutability is +enforced by `21888356`, not by `21899500`. + +Read both from the individual ruleset endpoint, `GET /repos/{owner}/{repo}/rulesets/{id}`. +The collection endpoint `GET /repos/{owner}/{repo}/rulesets` renders `bypass_actors` as +`null` for every ruleset, which is what caused an earlier revision of this section to +record "neither of which has bypass actors" — false when written, and the reason the +per-ruleset read is called out here. GraphQL's `repositoryRoleName` on +`RepositoryRulesetBypassActor` returns the role name directly and is the clearest check. + +Rulesets do expose `created_at` and `updated_at` on the individual endpoint, so the pair is +orderable: `21888356` was created at `2026-08-31T00:34:47Z`, roughly four hours before +`21899500` at `2026-08-31T04:47:11Z`, which was itself updated 54 seconds later at +`04:48:05Z`. + +`v0.1.1` is the empirical proof of the admin creation bypass. The annotated tag `v0.1.1` +(tag object `c2ecc1c49ff8bbe13b9d7800bc910e4b7ac99b74`, pointing at commit +`cc727db0bc72655f299166de1f60756f5c686cc7`) is tagged `2026-08-31T06:32:10Z` — one hour +and forty-five minutes after the creation restriction became active, by a repository admin. + +- A release tag, once created, **must not** be moved or deleted. Force-moving a tag to + paper over a broken release destroys the provenance that a release tag exists to + provide. This is mechanically enforced against every role. +- If a release is broken, cut a **new patch tag** (`v0.1.1`) from a fixed commit. Do not + retag `v0.1.0`. +- Published tags must not be moved, deleted, or reused, including during a security + incident. +- Tag **creation** is restricted to repository admins by `21899500`. Cutting a release tag + is therefore an admin action that needs no ruleset change, and the admin who cuts it is + accountable for the release-cut review requirements in the next section. + +## Release-cut pull requests + +A release cut (a PR that bumps the version, updates a changelog, or otherwise +prepares a release) is held to a stricter bar than an ordinary PR: + +- The release-cut PR **must be approved by a non-author CODEOWNER**. The author's + own approval does not count, and branch protection has no bypass actors, so no + administrator route around this review exists. +- The release-cut PR must not be merged while any required status check is failing + or in-progress. "Merge now, fix the release workflow by retagging" is the exact + anti-pattern this charter prohibits. +- If a release workflow fails after the cut, the fix goes through a **new PR** that + is reviewed and merged, then a **new tag** is cut — not a retag of the broken one. + +## CODEOWNERS + +The roster is defined in [`.github/CODEOWNERS`](.github/CODEOWNERS) and is deliberately +not duplicated here. The rules below apply to whoever is listed there at the time of each +pull request: this section describes policy, not a name list, so an owner change cannot +leave the charter asserting a roster that no longer matches the file it defers to. + +- At least two owners are required so that a non-author approval is always satisfiable. + Two is a floor, not a target: one owner makes the review gate structurally + unsatisfiable, and two leave an availability bottleneck. +- When an owner authors a pull request or makes its last push, a different current owner + supplies the required independent approval. +- Changing the roster is a governance action taken in `.github/CODEOWNERS` through a + reviewed pull request that cites the owner decision authorizing it. Adding an owner + grants review authority only; it grants no tag creation, no role change, and no + relaxation of anything above. +- The charter's narrative is kept in step with that change. #124 revision R3 authorized + the third owner, which reached `main` through #138; the corresponding charter wording is + this section, landed as the documentation follow-up rather than in the same cycle. That + ordering is the defect this section exists to close, not a precedent to repeat. + +## Incident runbook + +If a release tag is found to point at a broken or compromised commit: + +1. Do not retag. Do not delete the tag. +2. For a security incident, open a security advisory and deprecate or yank affected + distribution channels where supported. Leave the published tag in place as + immutable evidence. +3. Open a fix PR. Get it reviewed and merged to `main` with a current non-author + approval. +4. Cut a new patch tag from the fixed `main` tip and publish the replacement release + from that new tag. Tag creation on `refs/tags/v*` is restricted to repository admins + by ruleset `21899500`, so this step is an admin action — it does not require changing + any ruleset, and no ruleset change should be made in order to perform it. diff --git a/README.md b/README.md index aabe0ce..a9b7b2e 100644 --- a/README.md +++ b/README.md @@ -4,11 +4,11 @@ > > 开源 · 自托管 · 模型可插拔 · API / MCP / CLI / UI 共用一套记忆内核。 -[![CI](https://github.com/fullstack-ai-infra/mem/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/fullstack-ai-infra/mem/actions/workflows/ci.yml) +[![CI](https://github.com/bytefolk/mem/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/bytefolk/mem/actions/workflows/ci.yml) [![License](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](LICENSE) [![Status](https://img.shields.io/badge/status-experimental-orange.svg)](#project-status) [![MCP Server](https://img.shields.io/badge/MCP%20Server-26%20tools-blue?logo=modelcontextprotocol)](docs/mcp.md) -[![smithery](https://smithery.ai/badge/@fullstack-ai-infra/mem-mcp)](https://smithery.ai/server/@fullstack-ai-infra/mem-mcp) +[![smithery](https://smithery.ai/badge/@bytefolk/mem-mcp)](https://smithery.ai/server/@bytefolk/mem-mcp) **A portable, self-hosted memory plane for AI agents.** @@ -230,12 +230,34 @@ mem 的壁垒不是绑定某个更大的模型，而是长期积累的、用户 ## 快速开始 -项目仍处于 Phase 1 MVP。当前开发体验： +项目仍处于 Phase 1 MVP。第一次把 mem 跑起来，走 `deploy/compose`：一条命令拉起 +Web、memd、Worker、PostgreSQL、Redis 和 MinIO。这是文档上的主路径。 +`./scripts/dev_up.sh` 只留给要改 Go / Python / Web 的开发机。 + +`mem doctor` 在一台还没有配置的机器上也会把你指向这条容器路径，而不是裸机配方。 + +### 通过 Compose 启动（推荐） + +前置条件：Docker Engine 和 Docker Compose v2。命令与 +[生产部署指南](docs/DEPLOYMENT.md) 中的权威序列一致。 ```bash -git clone https://github.com/fullstack-ai-infra/mem.git +git clone https://github.com/bytefolk/mem.git cd mem -./scripts/dev_up.sh + +cd deploy/compose +./generate-env.sh +chmod 600 .env +docker compose --env-file .env -f compose.yaml up -d --build --wait +docker compose --env-file .env -f compose.yaml ps +curl --fail http://127.0.0.1:8080/healthz +``` + +启动完成后，浏览器打开 `http://localhost:8080`，完成首次注册（`first_user` +模式只允许一个账户），然后用 CLI 或 MCP： + +```bash +export MEM_SERVER=http://localhost:8080 mem auth login mem put ~/Photos --recursive # 可选：同步端附带可信的拍摄时间、位置和来源；AI 建议稍后在 Web 中确认 @@ -256,9 +278,25 @@ mem resume photos/import mem workspace export --output agent-workspace.membundle ``` -生产部署同时提供单机 Compose 和多机 Helm 方案，完整的密钥、迁移、高可用、 -备份恢复与升级边界见 [生产部署指南](docs/DEPLOYMENT.md)。默认视觉模型的真实英文/中文边界见 -[自然语言搜图基线](docs/acceptance/VISUAL_SEARCH_BASELINE.md)。 +完整的密钥、迁移、高可用、备份恢复与升级边界见 +[生产部署指南](docs/DEPLOYMENT.md)。默认视觉模型的真实英文/中文边界见 +[自然语言搜图基线](docs/acceptance/VISUAL_SEARCH_BASELINE.md)。多机方案见 +`deploy/helm/mem/`。 + +### 裸机开发环境（仅限开发） + +> 裸机路径面向需要修改 Go / Python / Web 代码的开发场景。首次体验或评估请使用 +> 上方的 Compose 路径。 + +```bash +git clone https://github.com/bytefolk/mem.git +cd mem +./scripts/dev_up.sh +mem auth login +``` + +完整的裸机依赖、Ollama 配置和 smoke 步骤见 [docs/RUN_LOCAL.md](docs/RUN_LOCAL.md)。 +该文档把 macOS brew 步骤并列了 Ubuntu/Debian 与 WSL2 等价命令。 --- @@ -298,7 +336,7 @@ embedding，平台托管 embedding 则使用独立的 workspace 权益和额度 - 本地运行与验证：[docs/RUN_LOCAL.md](docs/RUN_LOCAL.md) - 项目开发边界：[docs/DEVELOPMENT.md](docs/DEVELOPMENT.md) - 测试环境与回归门槛：[docs/TESTING.md](docs/TESTING.md) -- 参与贡献：[组织贡献基线](https://github.com/fullstack-ai-infra/.github/blob/main/CONTRIBUTING.md) +- 参与贡献：[组织贡献基线](https://github.com/bytefolk/.github/blob/main/CONTRIBUTING.md) 与 [mem 开发契约](docs/DEVELOPMENT.md) - 版本变化：[CHANGELOG.md](CHANGELOG.md) @@ -415,7 +453,7 @@ All changes follow an issue-first, pull-request-only workflow: 5. Obtain an independent review and pass required CI checks before merge. Read the -[organization contribution baseline](https://github.com/fullstack-ai-infra/.github/blob/main/CONTRIBUTING.md) +[organization contribution baseline](https://github.com/bytefolk/.github/blob/main/CONTRIBUTING.md) and the [mem-specific development contract](docs/DEVELOPMENT.md), then use [docs/maintainers/triage.md](docs/maintainers/triage.md) for the issue taxonomy. Security reports must follow [SECURITY.md](SECURITY.md), not a diff --git a/SECURITY.md b/SECURITY.md index 0ba14c3..bbf67cc 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,7 +1,7 @@ # Security Policy The organization-wide -[security policy](https://github.com/fullstack-ai-infra/.github/blob/main/SECURITY.md) +[security policy](https://github.com/bytefolk/.github/blob/main/SECURITY.md) defines confidential reporting, coordinated disclosure, and safe-research requirements. This file adds only the versions, scope, response target, and private intake route specific to `mem`. @@ -21,9 +21,9 @@ begin, the latest stable release will also receive security fixes. Do not disclose a suspected vulnerability in a public issue, discussion, or pull request. Use the -[`mem` private vulnerability reporting form](https://github.com/fullstack-ai-infra/mem/security/advisories/new). +[`mem` private vulnerability reporting form](https://github.com/bytefolk/mem/security/advisories/new). If the form is unavailable, follow the confidential fallback in the -organization security policy and identify `fullstack-ai-infra/mem` as the +organization security policy and identify `bytefolk/mem` as the affected repository. Maintainers aim to acknowledge a complete report within five business days. diff --git a/SPEC.md b/SPEC.md index d1e1b55..8755146 100644 --- a/SPEC.md +++ b/SPEC.md @@ -146,7 +146,7 @@ | F5.1 | `mem context "..."` → 返回有大小预算的文件/结构化记忆证据包 | | F5.2 | 每条 evidence 必须含 source kind/id、稳定 citation、内容哈希、片段和 locator | | F5.3 | mem 只走 recall → context pack；回答与行动由调用方 Agent 完成 | -| F5.4 | `source=all|file|memory`；结构化记忆在无 Worker、无模型时也必须可立即召回 | +| F5.4 | `source=all|file|memory`；结构化记忆在无 Worker、无模型时也必须可立即召回；文件词法路由（`route=lexical`）同样无需 Worker | | F5.5 | 联合召回单路失败但仍有证据时返回 `200 + partial=true + warnings[]`；无幸存证据时返回 `502 context_unavailable` | ### F5A · 结构化 Agent 记忆 @@ -534,7 +534,13 @@ embeddings_face ( - `folders (user_id, path)` UNIQUE — 路径唯一性约束 - `memories (workspace_id, idempotency_key_sha256)` UNIQUE — 不落明文幂等键的幂等写入 - `memories` FTS + trigram — 无模型的确定性立即召回 -- `embeddings_* (embedding)` — pgvector HNSW +- `files` FTS + trigram — 显式指定 `route=lexical` 的文件名无模型词法召回； + `auto` 只融合 text/visual，不自动回退到 lexical，worker 不可用时仍报错。 + 仅搜索 `files.name`，路径只用于筛选，不检索文件正文或路径片段。 +- `embeddings_* (embedding)` — pgvector HNSW (`vector_cosine_ops`, migration 0025) + on `embeddings_text` (768), `embeddings_visual` (512), `embeddings_face` (512). + Text search uses cosine-order candidates plus exact per-file fallback. + Face clustering is still in-process. `index_generation_vectors` is not indexed. - `file_entities (entity_id)` — 反查"和某人有关的所有文件" ### 6.3 文件夹一致性规则（重要） diff --git a/benchmarks/recall/README.md b/benchmarks/recall/README.md index 7b3cd88..69a1e10 100644 --- a/benchmarks/recall/README.md +++ b/benchmarks/recall/README.md @@ -1,5 +1,14 @@ # Multilingual recall benchmark +> **Evidence boundary: producer checks do not complete the live benchmark.** +> +> The producer corrections tracked in #196 do not complete #175's acceptance. +> The original #184 live-run requirement still needs a populated real memd, +> actual embedding-provider output and saved ranking artifacts. Unit or +> loopback HTTP fixtures establish transport behavior only. A real model-free +> lexical run also cannot establish vector/provider quality. See +> [Exact remaining live prerequisites](#exact-remaining-live-prerequisites). + This directory provides a small, repeatable retrieval benchmark. It is a decision aid for comparing lexical, vector and hybrid configurations; it is not evidence of production recall. @@ -167,3 +176,89 @@ time and sanitized query failures. It does not copy query text, corpus text, vectors or free-form provider errors. Credential-shaped configuration keys such as `api_key`, `password`, `secret`, `token` and `authorization` are rejected instead of being copied into an artifact. + +## Live memd producer + +The `produce` subcommand queries a running `memd` over file-search queries and +emits a `mem.recall-rankings.v1` file that the existing `run --rankings` path +consumes. Latency is measured client-side per request; the `0 ms` sentinel +warning above applies only to the offline lexical lane. + +```bash +python3 -m benchmarks.recall produce \ + --memd-url http://localhost:8080 \ + --token "$MEM_TOKEN" \ + --dataset benchmarks/recall/data/profile-text-v1 \ + --output /tmp/live-rankings.json \ + --dimension 768 --provider "$MEM_PROVIDER_LABEL" --model "$MEM_MODEL_LABEL" \ + --mode vector +``` + +Then score the saved rankings (the default v1 baseline uses a different corpus): + +```bash +python3 -m benchmarks.recall run \ + --dataset benchmarks/recall/data/profile-text-v1 \ + --rankings /tmp/live-rankings.json \ + --output /tmp/live-artifact.json +``` + +Load the synthetic file corpus into an isolated test deployment first. The +producer does not ingest it. Supply a token bound to the dataset's workspace; +its labels do not establish the token's real workspace identity. + +The producer maps each API result back to a dataset `doc_id` using the returned +folder `path` plus file `name`. Cross-workspace path collisions, unknown paths, +or ambiguous snippets fail the query instead of silently dropping evidence. +A same-workspace snippet-overlap tie also fails closed: document ID ordering +would invent identity evidence. Malformed paths, names, snippets and scores +fail with `invalid_result`, including malformed duplicate hits. +Query filters are translated where the API supports them: `path_prefix` becomes +`scope`, and `source_kind` becomes `type` (`image_caption` → `image`, +`text` → `text`). The `workspace` filter is not sent to the API because the +auth token determines workspace scope. A single token cannot select several +workspaces, so use a single-workspace fixture. Metadata filters are unsupported +and produce `unsupported_filter` without contacting the server; they are never +silently ignored. + +Vector mode sends `route=text`; it does not claim a hybrid lexical/vector or +multimodal experiment. Lexical mode sends `route=lexical` and requires the +server capability from #183. It emits null provider/model/dimension as required +by the ranking schema. Provider, model, dimension and index configuration are +operator declarations, not discovered or verified server metadata. The +`hardware.host` value deliberately records only the producer client's +OS/architecture, never its hostname. It is not the server's hardware inventory +and cannot establish comparable performance conditions. + +The full v1 corpus also contains structured-memory queries. `/v1/search` cannot +serve these; the producer records `unsupported_source_kind` and exits 2. Any +HTTP, mapping or response error also exits 2 while retaining an error artifact. +Use the existing `profile-text-v1` file-only fixture for this producer's bounded +acceptance. An HTTP fixture test proves transport and artifact handling only; +it does not establish live provider quality, production latency, or full-corpus +acceptance. Those remain NOT VERIFIED until a real populated memd run is saved. + + +### Exact remaining live prerequisites + +1. Use an isolated authorized memd/Worker deployment and disposable PostgreSQL + database, and verify that its token is bound to the intended test workspace. +2. Load all five synthetic files from `data/profile-text-v1/corpus.jsonl`, keeping + paths and content intact. Verify indexing and actual file identities before + interpreting the producer's output. Direct database seeding can establish + retrieval/transport behavior but does not verify ingestion or Worker indexing. +3. For vector acceptance, select the same actual embedding model for corpus and + queries, record its dimension and profile, and independently inspect the + active generation/index and deployment revision. Producer labels alone do + not verify any of those properties. +4. Run the documented producer command, retain its output, score the resulting + rankings and record errors and measured client latencies. An empty/error run + or a fake embedding provider cannot establish vector quality. A lexical run + requires the separate #194 server capability and remains a distinct result. +5. The bounded file-only experiment does not cover #175's bilingual image-query + acceptance or the full v1 structured-memory corpus. The original issue and + live quality acceptance must not be described as complete on this evidence. + +The producer remains opt-in; the normal recall CI gate runs deterministic unit +and fixture checks only. No real-model baseline is checked in until its actual +configuration and saved run are available for review. diff --git a/benchmarks/recall/__main__.py b/benchmarks/recall/__main__.py index 9fb888c..fe15870 100644 --- a/benchmarks/recall/__main__.py +++ b/benchmarks/recall/__main__.py @@ -7,7 +7,9 @@ import sys import tempfile +from .dataset import load_dataset from .errors import BenchmarkError +from .live_producer import produce_rankings from .runner import ( compare_artifacts, comparison_summary, @@ -66,6 +68,24 @@ def _parser() -> argparse.ArgumentParser: type=Path, default=PACKAGE_ROOT / "fixtures" / "external-rankings.leak.v1.json", ) + + produce = subparsers.add_parser( + "produce", + help="query a live memd and emit mem.recall-rankings.v1", + ) + produce.add_argument("--memd-url", required=True, help="base URL of memd") + produce.add_argument("--token", required=True, help="bearer token for auth") + produce.add_argument("--dataset", type=Path, default=DEFAULT_DATASET) + produce.add_argument("--output", type=Path, required=True) + produce.add_argument("--limit", type=int, default=10) + produce.add_argument("--timeout", type=float, default=30.0) + produce.add_argument("--engine", default="live-memd") + produce.add_argument("--dimension", type=int, default=768) + produce.add_argument( + "--mode", default="vector", choices=["lexical", "vector"] + ) + produce.add_argument("--provider", default="operator-unspecified") + produce.add_argument("--model", default="operator-unspecified") return parser @@ -100,6 +120,27 @@ def main(argv: list[str] | None = None) -> int: print(comparison_summary(comparison)) return 2 if candidate["metrics"]["overall"]["leakage_count"] else 0 + if args.command == "produce": + dataset = load_dataset(args.dataset) + rankings = produce_rankings( + dataset, + base_url=args.memd_url, + token=args.token, + limit=args.limit, + timeout=args.timeout, + engine_label=args.engine, + dimension=args.dimension, + mode=args.mode, + provider=args.provider, + model=args.model, + ) + write_json(args.output, rankings) + ok_count = sum(1 for q in rankings["queries"] if q["status"] == "ok") + err_count = sum(1 for q in rankings["queries"] if q["status"] == "error") + print(f"produced rankings: {ok_count} ok, {err_count} error") + print(f"rankings artifact: {args.output}") + return 2 if err_count else 0 + first = run_benchmark( dataset_dir=args.dataset, generated_at="2000-01-01T00:00:00+00:00", diff --git a/benchmarks/recall/live_producer.py b/benchmarks/recall/live_producer.py new file mode 100644 index 0000000..9888b36 --- /dev/null +++ b/benchmarks/recall/live_producer.py @@ -0,0 +1,261 @@ +"""Produce mem.recall-rankings.v1 from a live memd instance. + +Queries each dataset query against POST /v1/search, maps API results back to +dataset doc_ids by path, and emits the rankings JSON that the existing harness +consumes via --rankings. +""" + +from __future__ import annotations + +import json +import math +import platform +import posixpath +import time +import unicodedata +from typing import Any +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + +from .dataset import Dataset, Document +from .errors import BenchmarkError + +_SOURCE_KIND_TO_TYPE = { + "image_caption": "image", + "text": "text", +} + + +def _build_path_index(documents: list[Document]) -> dict[str, list[Document]]: + index: dict[str, list[Document]] = {} + for doc in documents: + index.setdefault(doc.path, []).append(doc) + return index + + +def _match_doc_by_path( + api_path: str, + snippet: str, + candidates: list[Document], +) -> Document | None: + if not candidates: + return None + if len(candidates) == 1: + return candidates[0] + # A snippet cannot establish tenant identity. Never choose an authorized + # document merely because a foreign document shares its path or words. + if len({doc.workspace for doc in candidates}) != 1: + return None + normalized_snippet = unicodedata.normalize("NFKC", snippet).casefold() + best: Document | None = None + best_overlap = 0 + for doc in candidates: + doc_tokens = set(unicodedata.normalize("NFKC", doc.text).casefold().split()) + overlap = sum(1 for t in normalized_snippet.split() if t in doc_tokens) + if overlap > best_overlap: + best_overlap = overlap + best = doc + elif overlap == best_overlap: + best = None + return best + + +def _source_kind_to_api_type(source_kind: str) -> str | None: + return _SOURCE_KIND_TO_TYPE.get(source_kind) + + +def _coarse_host() -> str: + """Record OS/architecture only, never a hostname or client identity.""" + try: + return f"{platform.system()}/{platform.machine()}" + except Exception: + return "unknown" + + +def _query_memd( + base_url: str, + token: str, + query_text: str, + *, + scope: str = "", + type_filter: str = "", + route: str = "auto", + limit: int = 10, + timeout: float = 30.0, +) -> tuple[list[dict[str, Any]], float, str | None]: + body: dict[str, Any] = {"query": query_text, "limit": limit, "route": route} + if scope: + body["scope"] = scope + if type_filter: + body["type"] = type_filter + + url = base_url.rstrip("/") + "/v1/search" + data = json.dumps(body).encode("utf-8") + req = Request(url, data=data, method="POST") + req.add_header("Content-Type", "application/json") + req.add_header("Authorization", f"Bearer {token}") + + start = time.perf_counter() + try: + with urlopen(req, timeout=timeout) as resp: + payload = json.loads(resp.read().decode("utf-8")) + elapsed_ms = (time.perf_counter() - start) * 1000.0 + if not isinstance(payload, dict) or "results" not in payload: + return [], elapsed_ms, "invalid_response" + results = payload["results"] + if results is None: + results = [] # memd encodes an empty nil hit slice as null. + if not isinstance(results, list) or any(not isinstance(hit, dict) for hit in results): + return [], elapsed_ms, "invalid_response" + return results, elapsed_ms, None + except HTTPError as exc: + elapsed_ms = (time.perf_counter() - start) * 1000.0 + return [], elapsed_ms, f"http_{exc.code}" + except URLError: + elapsed_ms = (time.perf_counter() - start) * 1000.0 + return [], elapsed_ms, "connection_error" + except Exception: + elapsed_ms = (time.perf_counter() - start) * 1000.0 + return [], elapsed_ms, "unknown_error" + + +def produce_rankings( + dataset: Dataset, + *, + base_url: str, + token: str, + limit: int = 10, + timeout: float = 30.0, + engine_label: str = "live-memd", + dimension: int = 768, + mode: str = "vector", + provider: str = "operator-unspecified", + model: str = "operator-unspecified", +) -> dict[str, Any]: + if mode not in {"lexical", "vector"}: + raise BenchmarkError("memd /v1/search does not expose a hybrid lexical/vector route") + if not 1 <= limit <= 100 or not math.isfinite(timeout) or timeout <= 0: + raise BenchmarkError("limit must be 1..100 and timeout must be positive and finite") + if not isinstance(engine_label, str) or not engine_label.strip() or engine_label == "lexical-reference": + raise BenchmarkError("engine must identify live memd, not lexical-reference") + if mode == "vector" and ( + not isinstance(dimension, int) or isinstance(dimension, bool) or dimension <= 0 + or not isinstance(provider, str) or not provider.strip() + or not isinstance(model, str) or not model.strip() + ): + raise BenchmarkError("vector mode requires a positive dimension and non-empty provider/model labels") + path_index = _build_path_index(list(dataset.documents)) + + query_rows: list[dict[str, Any]] = [] + for query in dataset.queries: + if query.expected_source_kind == "structured": + query_rows.append({"query_id": query.id, "status": "error", + "latency_ms": 0.0, "results": [], + "error_code": "unsupported_source_kind"}) + continue + if query.filters.get("metadata"): + query_rows.append({"query_id": query.id, "status": "error", + "latency_ms": 0.0, "results": [], + "error_code": "unsupported_filter"}) + continue + scope = query.filters.get("path_prefix", "") + type_filter = _source_kind_to_api_type(query.expected_source_kind) or "" + + api_results, latency_ms, error_code = _query_memd( + base_url, + token, + query.text, + scope=scope, + type_filter=type_filter, + route="lexical" if mode == "lexical" else "text", + limit=limit, + timeout=timeout, + ) + + if error_code: + row: dict[str, Any] = { + "query_id": query.id, + "status": "error", + "latency_ms": round(latency_ms, 2), + "results": [], + "error_code": error_code, + } + query_rows.append(row) + continue + + mapped_results: list[dict[str, Any]] = [] + seen_doc_ids: set[str] = set() + mapping_error: str | None = None + for hit in api_results: + hit_path = hit.get("path") + name = hit.get("name", "") + snippet = hit.get("snippet", "") + score = hit.get("score") + # Validate every row before deduplication: a malformed duplicate is + # still evidence of a failed response, not something to discard. + if ( + not isinstance(hit_path, str) or not hit_path.startswith("/") + or not isinstance(name, str) or not isinstance(snippet, str) + or (name and ("/" in name or name in {".", ".."})) + or (score is not None and ( + isinstance(score, bool) or not isinstance(score, (int, float)) + or not math.isfinite(score) + )) + ): + mapping_error = "invalid_result" + break + if name: + # memd returns a folder path and file name separately. + hit_path = posixpath.join(hit_path, name) + candidates = path_index.get(hit_path, []) + doc = _match_doc_by_path(hit_path, snippet, candidates) + if doc is None: + mapping_error = "unmapped_result" + break + if doc.id in seen_doc_ids: + continue + seen_doc_ids.add(doc.id) + result: dict[str, Any] = { + "doc_id": doc.id, + "citation": doc.citation, + } + if score is not None: + result["score"] = float(score) + mapped_results.append(result) + + if mapping_error: + mapped_results = [] + status = "ok" if not mapping_error else "error" + row = { + "query_id": query.id, + "status": status, + "latency_ms": round(latency_ms, 2), + "results": mapped_results, + } + if mapping_error: + row["error_code"] = mapping_error + query_rows.append(row) + + return { + "schema_version": "mem.recall-rankings.v1", + "engine": engine_label, + "configuration": { + "mode": mode, + "provider": None if mode == "lexical" else provider, + "model": None if mode == "lexical" else model, + "dimension": None if mode == "lexical" else dimension, + "evidence": "operator-declared configuration; model and index not verified by producer", + "index": { + "kind": "operator-unspecified", + }, + "search": { + "top_k": limit, + "route": "lexical" if mode == "lexical" else "text", + "workspace": "bound by the supplied token; not inferred from dataset labels", + }, + }, + "hardware": { + "host": _coarse_host(), + }, + "queries": query_rows, + } diff --git a/benchmarks/recall/tests/test_live_producer.py b/benchmarks/recall/tests/test_live_producer.py new file mode 100644 index 0000000..6985e7e --- /dev/null +++ b/benchmarks/recall/tests/test_live_producer.py @@ -0,0 +1,297 @@ +from __future__ import annotations + +import json +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path +import tempfile +import threading +import unittest +from unittest.mock import patch + +from benchmarks.recall.__main__ import main +from benchmarks.recall.adapters import load_external_rankings +from benchmarks.recall.errors import BenchmarkError +from benchmarks.recall.live_producer import ( + _build_path_index, + _match_doc_by_path, + produce_rankings, +) +from benchmarks.recall.dataset import Document, load_dataset + + +class PathIndexTest(unittest.TestCase): + def test_index_groups_by_path(self) -> None: + docs = [ + Document( + id="a", language="en", source_kind="text", workspace="alpha", + path="/notes/a.md", citation="mem://files/a", + text="alpha note", metadata={}, + ), + Document( + id="b", language="en", source_kind="text", workspace="alpha", + path="/notes/a.md", citation="mem://files/b", + text="beta note", metadata={}, + ), + ] + index = _build_path_index(docs) + self.assertEqual(len(index["/notes/a.md"]), 2) + + def test_match_single_candidate(self) -> None: + doc = Document( + id="solo", language="en", source_kind="text", workspace="alpha", + path="/notes/solo.md", citation="mem://files/solo", + text="unique content", metadata={}, + ) + result = _match_doc_by_path("/notes/solo.md", "anything", [doc]) + self.assertEqual(result, doc) + + def test_match_picks_best_snippet_overlap(self) -> None: + doc_a = Document( + id="a", language="en", source_kind="text", workspace="alpha", + path="/notes/shared.md", citation="mem://files/a", + text="saturn ring observation", metadata={}, + ) + doc_b = Document( + id="b", language="en", source_kind="text", workspace="alpha", + path="/notes/shared.md", citation="mem://files/b", + text="completely different topic", metadata={}, + ) + result = _match_doc_by_path("/notes/shared.md", "saturn ring", [doc_a, doc_b]) + self.assertEqual(result, doc_a) + + def test_ambiguous_path_does_not_guess_identity(self) -> None: + docs = [Document(id=key, language="en", source_kind="text", + workspace=workspace, path="/shared.md", citation="mem://"+key, + text="same snippet", metadata={}) + for key, workspace in [("a", "alpha"), ("b", "beta")]] + self.assertIsNone(_match_doc_by_path("/shared.md", "same snippet", docs)) + + +class ProduceRankingsTest(unittest.TestCase): + def setUp(self) -> None: + self.tempdir = tempfile.TemporaryDirectory() + self.root = Path(self.tempdir.name) + self.dataset = self.root / "dataset" + self.dataset.mkdir() + (self.dataset / "dataset.json").write_text( + json.dumps({ + "schema_version": "mem.recall-dataset.v1", + "version": "unit-test-v1", + "provenance": "hand-authored synthetic data", + "license": "CC0-1.0", + "required_coverage": { + "slices": ["exact"], "languages": ["en"], "source_kinds": ["text"], + }, + }), + encoding="utf-8", + ) + (self.dataset / "corpus.jsonl").write_text( + json.dumps({ + "id": "file-en-cassini", "language": "en", "source_kind": "text", + "workspace": "alpha", "path": "/research/saturn.md", + "citation": "mem://files/file-en-cassini", + "text": "Cassini observed Saturn hexagonal storm", + "provenance": "synthetic", + }) + "\n", + encoding="utf-8", + ) + (self.dataset / "queries.jsonl").write_text( + json.dumps({ + "id": "q-en-text-exact", "text": "Cassini Saturn hexagonal storm", + "language": "en", "slice": "exact", + "filters": {"workspace": "alpha", "source_kind": "text"}, + "expected_source_kind": "text", + }) + "\n", + encoding="utf-8", + ) + (self.dataset / "qrels.json").write_text( + json.dumps({"q-en-text-exact": {"file-en-cassini": 3}}), + encoding="utf-8", + ) + + def tearDown(self) -> None: + self.tempdir.cleanup() + + def test_http_fixture_runs_transport_and_emits_loadable_artifact(self) -> None: + # Real loopback HTTP, synthetic response: this is not a live memd or + # embedding-quality benchmark. + requests = [] + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + requests.append((self.path, json.loads(self.rfile.read(int(self.headers["Content-Length"]))))) + payload = json.dumps({"results": [{"path": "/research", "name": "saturn.md", "score": 0.9}]}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *args): + pass + + server = HTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + output = self.root / "http-fixture.json" + try: + code = main(["produce", "--memd-url", f"http://127.0.0.1:{server.server_port}", + "--token", "fixture", "--dataset", str(self.dataset), + "--output", str(output), "--provider", "fixture", "--model", "fixture"]) + finally: + server.shutdown() + thread.join() + server.server_close() + self.assertEqual(code, 0) + self.assertEqual(requests[0][0], "/v1/search") + self.assertEqual(requests[0][1]["route"], "text") + artifact = load_external_rankings(output, load_dataset(self.dataset)) + self.assertEqual(artifact["queries"][0]["results"][0]["doc_id"], "file-en-cassini") + self.assertGreater(artifact["queries"][0]["latency_ms"], 0) + self.assertNotIn("client", artifact["hardware"]) + + @patch("benchmarks.recall.live_producer._query_memd") + def test_produce_rankings_success(self, mock_query: unittest.mock.MagicMock) -> None: + mock_query.return_value = ( + [{"path": "/research/saturn.md", "snippet": "Cassini observed Saturn hexagonal storm", "score": 0.95}], + 12.5, None, + ) + dataset = load_dataset(self.dataset) + rankings = produce_rankings( + dataset, base_url="http://localhost:8080", token="test-token", dimension=768, + ) + self.assertEqual(rankings["schema_version"], "mem.recall-rankings.v1") + self.assertEqual(rankings["engine"], "live-memd") + self.assertEqual(rankings["configuration"]["dimension"], 768) + self.assertEqual(len(rankings["queries"]), 1) + query_row = rankings["queries"][0] + self.assertEqual(query_row["query_id"], "q-en-text-exact") + self.assertEqual(query_row["status"], "ok") + self.assertGreater(query_row["latency_ms"], 0) + self.assertEqual(len(query_row["results"]), 1) + self.assertEqual(query_row["results"][0]["doc_id"], "file-en-cassini") + + @patch("benchmarks.recall.live_producer._query_memd") + def test_produce_rankings_error(self, mock_query: unittest.mock.MagicMock) -> None: + mock_query.return_value = ([], 5.0, "http_503") + dataset = load_dataset(self.dataset) + rankings = produce_rankings( + dataset, base_url="http://localhost:8080", token="test-token", + ) + query_row = rankings["queries"][0] + self.assertEqual(query_row["status"], "error") + self.assertEqual(query_row["error_code"], "http_503") + self.assertEqual(query_row["results"], []) + + @patch("benchmarks.recall.live_producer._query_memd") + def test_maps_shipping_folder_and_name_shape(self, mock_query) -> None: + mock_query.return_value = ([{"path": "/research", "name": "saturn.md", "score": 0.9}], 1, None) + rankings = produce_rankings(load_dataset(self.dataset), base_url="http://localhost", token="test") + self.assertEqual(rankings["queries"][0]["results"][0]["doc_id"], "file-en-cassini") + + @patch("benchmarks.recall.live_producer._query_memd") + def test_unmapped_hit_is_not_silently_discarded(self, mock_query) -> None: + mock_query.return_value = ([{"path": "/foreign.md", "snippet": "private"}], 1, None) + rankings = produce_rankings(load_dataset(self.dataset), base_url="http://localhost", token="test") + self.assertEqual(rankings["queries"][0]["status"], "error") + self.assertEqual(rankings["queries"][0]["error_code"], "unmapped_result") + + @patch("benchmarks.recall.live_producer._query_memd") + def test_lexical_artifact_is_loadable_and_route_is_sent(self, mock_query) -> None: + mock_query.return_value = ([], 1, None) + dataset = load_dataset(self.dataset) + rankings = produce_rankings(dataset, base_url="http://localhost", token="test", mode="lexical") + output = self.root / "rankings.json" + output.write_text(json.dumps(rankings), encoding="utf-8") + load_external_rankings(output, dataset) + self.assertEqual(mock_query.call_args.kwargs["route"], "lexical") + + @patch("benchmarks.recall.live_producer._query_memd") + def test_produce_command_fails_when_requests_fail(self, mock_query) -> None: + mock_query.return_value = ([], 1, "http_503") + code = main(["produce", "--memd-url", "http://localhost", "--token", "test", + "--dataset", str(self.dataset), "--output", str(self.root / "failed.json")]) + self.assertEqual(code, 2) + + + @patch("benchmarks.recall.live_producer._query_memd") + def test_malformed_hits_retain_error_artifact(self, mock_query) -> None: + good = {"path": "/research", "name": "saturn.md", "score": 0.9} + malformed = [ + {"path": None}, {"path": []}, {"path": {}}, {"path": 42}, + {"path": "/research", "name": None}, + {"path": "/research", "name": "/research/saturn.md"}, + {"path": "/research", "name": "../saturn.md"}, + {"path": "/research/saturn.md", "snippet": []}, + {"path": "/research/saturn.md", "score": False}, + {"path": "/research/saturn.md", "score": "0.9"}, + {"path": "/research/saturn.md", "score": float("nan")}, + {"path": "/research/saturn.md", "score": float("inf")}, + ] + for hit in malformed: + with self.subTest(hit=hit): + # Including a valid first row also checks that malformed + # duplicate rows cannot disappear during deduplication. + mock_query.return_value = ([good, hit], 1, None) + output = self.root / "malformed.json" + code = main(["produce", "--memd-url", "http://localhost", "--token", "fixture", + "--dataset", str(self.dataset), "--output", str(output)]) + self.assertEqual(code, 2) + row = load_external_rankings(output, load_dataset(self.dataset))["queries"][0] + self.assertEqual(row["error_code"], "invalid_result") + self.assertEqual(row["results"], []) + + @patch("benchmarks.recall.live_producer._query_memd") + def test_unsupported_metadata_filter_never_contacts_server(self, mock_query) -> None: + path = self.dataset / "queries.jsonl" + query = json.loads(path.read_text()) + query["filters"]["metadata"] = {"project": "saturn"} + path.write_text(json.dumps(query) + "\n") + # Give the relevant document the same metadata so dataset validation + # succeeds; the unsupported transport filter remains the only failure. + path = self.dataset / "corpus.jsonl" + document = json.loads(path.read_text()) + document["metadata"] = {"project": "saturn"} + path.write_text(json.dumps(document) + "\n") + rankings = produce_rankings(load_dataset(self.dataset), base_url="http://localhost", token="fixture") + self.assertEqual(rankings["queries"][0]["error_code"], "unsupported_filter") + mock_query.assert_not_called() + + @patch("benchmarks.recall.live_producer._query_memd") + def test_structured_queries_never_contact_server(self, mock_query) -> None: + from dataclasses import replace + dataset = load_dataset(self.dataset) + dataset = replace(dataset, queries=(replace(dataset.queries[0], expected_source_kind="structured"),)) + rankings = produce_rankings(dataset, base_url="http://localhost", token="fixture") + self.assertEqual(rankings["queries"][0]["error_code"], "unsupported_source_kind") + mock_query.assert_not_called() + + @patch("benchmarks.recall.live_producer._query_memd") + def test_invalid_configuration_never_contacts_server(self, mock_query) -> None: + for kwargs in [ + {"dimension": 0}, {"dimension": -1}, {"dimension": True}, + {"provider": ""}, {"model": " "}, {"engine_label": "lexical-reference"}, + {"limit": 0}, {"limit": 101}, {"timeout": float("nan")}, + {"timeout": float("inf")}, {"timeout": 0}, {"mode": "hybrid"}, + ]: + with self.subTest(kwargs=kwargs), self.assertRaises(BenchmarkError): + produce_rankings(load_dataset(self.dataset), base_url="http://localhost", token="fixture", **kwargs) + mock_query.assert_not_called() + + @patch("benchmarks.recall.live_producer._query_memd") + def test_duplicate_valid_chunks_map_to_one_document(self, mock_query) -> None: + mock_query.return_value = ([{"path": "/research/saturn.md", "score": 0.9}, + {"path": "/research/saturn.md", "score": 0.8}], 1, None) + rankings = produce_rankings(load_dataset(self.dataset), base_url="http://localhost", token="fixture") + self.assertEqual(rankings["queries"][0]["status"], "ok") + self.assertEqual(len(rankings["queries"][0]["results"]), 1) + + def test_equal_same_workspace_candidates_fail_closed(self) -> None: + from dataclasses import replace + document = load_dataset(self.dataset).documents[0] + other = replace(document, id="other") + self.assertIsNone(_match_doc_by_path(document.path, document.text, [document, other])) + + +if __name__ == "__main__": + unittest.main() diff --git a/deploy/compose/compose.yaml b/deploy/compose/compose.yaml index 29d3e17..e123d59 100644 --- a/deploy/compose/compose.yaml +++ b/deploy/compose/compose.yaml @@ -90,7 +90,9 @@ services: start_period: 5s minio: - image: minio/minio:RELEASE.2025-04-22T22-12-26Z + # MinIO withdrew its Docker Hub repositories in October 2025; quay.io serves + # the same release tags. + image: quay.io/minio/minio:RELEASE.2025-04-22T22-12-26Z restart: unless-stopped command: server /data --console-address :9001 environment: @@ -108,7 +110,7 @@ services: start_period: 10s minio-init: - image: minio/mc:RELEASE.2025-04-16T18-13-26Z + image: quay.io/minio/mc:RELEASE.2025-04-16T18-13-26Z restart: "no" depends_on: minio: @@ -217,7 +219,7 @@ services: start_period: 10s minio-client: - image: minio/mc:RELEASE.2025-04-16T18-13-26Z + image: quay.io/minio/mc:RELEASE.2025-04-16T18-13-26Z profiles: ["tools"] environment: MEM_S3_BUCKET: ${MEM_S3_BUCKET:-mem} diff --git a/deploy/helm/mem/Chart.yaml b/deploy/helm/mem/Chart.yaml index c86c3cb..640fa31 100644 --- a/deploy/helm/mem/Chart.yaml +++ b/deploy/helm/mem/Chart.yaml @@ -2,6 +2,6 @@ apiVersion: v2 name: mem description: Production Web, memd, migration, and Worker workloads for mem type: application -version: 0.1.1 -appVersion: "0.1.1" +version: 0.1.2 +appVersion: "0.1.2" kubeVersion: ">=1.28.0-0" diff --git a/deploy/helm/mem/values-production.example.yaml b/deploy/helm/mem/values-production.example.yaml index 3559e42..26f378c 100644 --- a/deploy/helm/mem/values-production.example.yaml +++ b/deploy/helm/mem/values-production.example.yaml @@ -2,13 +2,13 @@ images: server: repository: registry.example.internal/mem/server - tag: "0.1.1" + tag: "0.1.2" worker: repository: registry.example.internal/mem/worker - tag: "0.1.1" + tag: "0.1.2" web: repository: registry.example.internal/mem/web - tag: "0.1.1" + tag: "0.1.2" existingSecret: mem-runtime diff --git a/deploy/helm/mem/values.yaml b/deploy/helm/mem/values.yaml index c6e16db..6e68aaf 100644 --- a/deploy/helm/mem/values.yaml +++ b/deploy/helm/mem/values.yaml @@ -14,15 +14,15 @@ runtime: images: server: repository: mem-server - tag: "0.1.1" + tag: "0.1.2" pullPolicy: IfNotPresent worker: repository: mem-worker - tag: "0.1.1" + tag: "0.1.2" pullPolicy: IfNotPresent web: repository: mem-web - tag: "0.1.1" + tag: "0.1.2" pullPolicy: IfNotPresent serviceAccount: @@ -32,7 +32,7 @@ serviceAccount: memd: # memd coordinates indexing and embedding-provider switches in process. - # Keep one replica until https://github.com/fullstack-ai-infra/mem/issues/55 + # Keep one replica until https://github.com/bytefolk/mem/issues/55 # provides cross-replica index generations. replicaCount: 1 resources: diff --git a/docker-compose.test.yml b/docker-compose.test.yml index a411755..eea98e1 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -22,7 +22,9 @@ services: minio: profiles: ["e2e"] - image: minio/minio:latest@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + # MinIO withdrew its Docker Hub repositories in October 2025; quay.io serves + # the same digest-pinned image. + image: quay.io/minio/minio:latest@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e command: server /data --console-address ":9001" environment: MINIO_ROOT_USER: mem @@ -40,7 +42,7 @@ services: minio-init: profiles: ["e2e"] - image: minio/mc:latest@sha256:a7fe349ef4bd8521fb8497f55c6042871b2ae640607cf99d9bede5e9bdf11727 + image: quay.io/minio/mc:latest@sha256:a7fe349ef4bd8521fb8497f55c6042871b2ae640607cf99d9bede5e9bdf11727 depends_on: minio: condition: service_healthy diff --git a/docker-compose.yml b/docker-compose.yml index 3642946..8c13718 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,7 +40,9 @@ services: retries: 10 minio: - image: minio/minio:latest + # MinIO withdrew its Docker Hub repositories in October 2025; quay.io serves + # the same image. + image: quay.io/minio/minio:latest container_name: mem-minio restart: unless-stopped command: server /data --console-address ":9001" @@ -60,7 +62,7 @@ services: # Bootstrap: create default bucket minio-init: - image: minio/mc:latest + image: quay.io/minio/mc:latest depends_on: minio: condition: service_healthy diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index f1fcc93..b655f8b 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -8,7 +8,7 @@ only in where stateful dependencies run and which workloads can scale. > The deployment assets are suitable for private self-hosting. Do not expose a > `mem` installation as a public multi-tenant service until the hosted > authentication and abuse-control work in -> [issue #65](https://github.com/fullstack-ai-infra/mem/issues/65) is complete. +> [issue #65](https://github.com/bytefolk/mem/issues/65) is complete. > The Helm profile is the intended foundation for that service, but > horizontal scaling alone does not make the current login/session model > Internet-service grade. @@ -64,11 +64,13 @@ Use an immutable version for all three images. The example below builds the model-free Worker; optional heavy extras must be explicitly selected. ```bash -export MEM_VERSION=0.1.1 +export MEM_VERSION=0.1.2 +export MEM_REVISION="$(git rev-parse HEAD)" export MEM_REGISTRY=registry.example.internal/mem docker build \ --build-arg VERSION="$MEM_VERSION" \ + --build-arg REVISION="$MEM_REVISION" \ -t "$MEM_REGISTRY/server:$MEM_VERSION" server docker build \ -t "$MEM_REGISTRY/worker:$MEM_VERSION" worker @@ -96,6 +98,117 @@ MEM_VALIDATE_BUILD_IMAGES=1 make test-deploy The first command validates Compose and Helm. The second also builds all three images from the current checkout. +## Version coordinate and client preflight + +Every release publishes a single version grammar that clients pin against. The +`/v1/version` endpoint returns three distinct fields: + +| Field | Example | Meaning | +| --- | --- | --- | +| `version` | `"0.1.1"` | Semver release tag (without the `v` prefix) | +| `revision` | `"10d4bf7a48fd5ab0ce6fc67caa407a717f81830e"` | 40-hex git commit the binary was built from | +| `contract` | `"durable-context.v1"` | Durable-context wire contract the server speaks | + +Both build paths — the release workflow and the Docker image — inject all three +fields at build time via `-ldflags`. A binary produced by either path answers +`/v1/version` with the same shape. A client may pin either the `revision` (exact +commit) or accept a `version` range; the `contract` field is informational and +changes only when the durable-context wire format breaks compatibility. + +The release workflow verifies that every published binary embeds the exact +release commit via `go version -m`. The Dockerfile accepts `VERSION`, `REVISION` +and `CONTRACT_VERSION` build args; the Compose and Helm deployment paths pass +the release tag as `VERSION` and the tag commit as `REVISION`. + +No step in the documented deployment path requires hand-editing a build flag to +become compatible with a pinned client. + +## First-run path + +After starting memd from a release artifact (Compose, Docker image or bare +binary), complete these steps to reach a working endpoint with a workspace, a +token and the scopes needed for both write and recall operations. + +### 1. Verify the server is reachable + +```bash +curl --fail "$(mem config get server)/healthz" +curl --fail "$(mem config get server)/v1/version" +``` + +The `/v1/version` response must contain `version`, `revision` and `contract` +fields. If the endpoint is unreachable, the server is not running or the +configured URL is wrong. Run `mem doctor` to diagnose common preconditions. + +### 2. Register the first user + +With `MEM_REGISTRATION_MODE=first_user` (the Compose default), the first +registration creates the owner account and disables further registration +automatically: + +```bash +mem auth login +``` + +Follow the interactive prompt. The CLI saves the session token to +`~/.mem/config.yaml`. Verify with: + +```bash +mem auth status +``` + +### 3. Create an API token with write and recall scopes + +The durable-context recall endpoint requires a token whose scopes cover both +`write` and `read`. Create one: + +```bash +mem auth token create \ + --name "digital-employee" \ + --scope "read,write" +``` + +Store the returned token. It is shown exactly once. + +### 4. Create a durable-context grant + +Recall operations require an admin-created grant that binds a principal to an +approved set of memories. From the admin session: + +```bash +curl -X POST "$(mem config get server)/v1/durable-context/grants" \ + -H "Authorization: Bearer ${ADMIN_TOKEN}" \ + -H "Content-Type: application/json" \ + -d '{"contract":"durable-context.v1","principal":"digital-employee","memory_ids":[""]}' +``` + +The grant ties the principal name to the specific memories the recall scope is +allowed to read. Without this grant, recall returns `scope_denied` even with a +valid token. + +### 5. Verify end-to-end + +With the token and grant in place, a client adapter can complete one write and +one recall: + +```bash +# Write a memory +curl -X POST "$(mem config get server)/v1/memories" \ + -H "Authorization: Bearer ${API_TOKEN}" \ + -H "Content-Type: application/json" \ + -d '{"content":"test memory","tags":["smoke-test"]}' + +# Recall durable context +curl -X POST "$(mem config get server)/v1/durable-context/recall" \ + -H "Authorization: Bearer ${API_TOKEN}" \ + -H "Content-Type: application/json" \ + -d '{"contract":"durable-context.v1","principal":"digital-employee"}' +``` + +If any step fails with a named precondition error (`scope_denied`, +`contract_unsupported`, `registration_disabled`), the error message identifies +the missing configuration rather than a generic connection failure. + ## Single-node Compose ### Host and network @@ -120,6 +233,14 @@ Terminate HTTPS at a maintained reverse proxy or load balancer. Forward to `http://127.0.0.1:8080`, preserve the `Host` and `X-Forwarded-*` headers, and set an upload-body limit at least as large as `MEM_MAX_BODY_SIZE`. +The web container is itself a reverse proxy and is the authority for +`X-Content-Type-Options`, `X-Frame-Options` and `Referrer-Policy`; it sets them +on every response it serves and drops the copies `memd` sends so they do not +arrive twice. `Content-Security-Policy`, `X-XSS-Protection` and +`Content-Disposition` come from `memd`, because they depend on what the response +actually is. If your terminating proxy sets the first three as well, set them +there or here, not both, or a client receives two values for one header. + ### Configure and start From the repository root: @@ -155,6 +276,42 @@ docker compose --env-file .env -f compose.yaml ps docker compose --env-file .env -f compose.yaml logs --tail=200 migrate memd worker web ``` +### Diagnose from the client side + +`mem doctor` answers the client half of the same question: why the CLI cannot +reach a working server. It issues only `GET` requests, and it never writes +configuration, starts or stops a container, or installs a dependency — a failed +diagnosis changes nothing on the machine. + +```bash +mem doctor +mem doctor --format json +``` + +It reports four checks in a fixed order and stops guessing after the first +failure: reachability of the configured server URL (`/healthz`, probed without a +credential so a bad token is not misread as an outage), whether a credential +exists, the workspace the server resolved for that credential +(`/v1/capabilities`), and CLI/server version skew (`/v1/version`). A check that +an earlier failure made impossible is reported as `skipped`, naming the blocking +check, rather than as an inferred pass. + +The process exits with the first failing check's SPEC §7.1 code — `0` ok · +`2` not_found · `3` auth · `4` plan/quota · `5` provider/timeout — so a wrapper +can branch on it. Version skew is advisory and contributes `0`; it is also not +computable in builds that do not inject a CLI version, which today includes +release builds, so the check reports that limit instead of claiming agreement. + +`--format json` emits the `mem.doctor` v1 document validated by +[`schemas/mem-doctor.v1.schema.json`](schemas/mem-doctor.v1.schema.json), and a +token is described only by where it came from. For a configured URL, userinfo and +every query parameter **value** are replaced by `REDACTED` — the parameter names +survive so the report still says which settings are on — and a URL that cannot be +proven to be a credential-free transport URL is withheld whole as `[withheld]` +rather than partially trimmed. A secret supplied as a query parameter +(`http://mem.internal:8787?password=…`) is therefore not reported, which matters +because pgx accepts `postgres://host/db?password=…` as the real password. + ### First account and login The default `MEM_REGISTRATION_MODE=first_user` atomically allows exactly one @@ -217,6 +374,36 @@ Redis AOF protects normal restarts but is not in the portable backup. A restore therefore starts with an empty queue/replay window. Requeue or reindex any file whose processing did not reach a terminal state before the backup. +### Object storage retention + +Object keys are per-file by construction: each key embeds the row's own file ID +(`users///`), so deleting one row's key cannot +remove another row's bytes. No reference counting is needed. + +When a file or folder is deleted, the database row is removed first, then the +corresponding object is deleted from bucket storage on a best-effort basis. A +failed object delete does not roll back the database change; the orphaned key +is logged at `WARN` level so the operator can see which keys remain. If the +shared 30-second cleanup budget is exhausted mid-batch, later keys log that +the budget ran out rather than a per-object store error. + +Recursive folder delete refuses with the existing `forget` sentinel when an +active or archived memory — including one whose `path` is outside the folder +— still cites a file in the tree through `source_file_id`. That keeps blob +cleanup from destroying a live citation via `ON DELETE SET NULL`. + +**Crash window**: if the process is killed after the database transaction +commits but before the blob delete lands, the object remains in the bucket +permanently. There is currently no reaper or garbage-collection pass to sweep +these residues. The server has no listing capability against the bucket (the +`storage.Store` interface exposes only `Put`/`Get`/`Delete`), so a reaper would +need to record keys whose delete was never attempted. This is a known gap; +operators should monitor bucket growth against expected database row counts. + +To manually reconcile, compare the bucket contents against the `files` table's +`storage_key` column. Objects present in the bucket but absent from the database +are safe to delete — they cannot be referenced by any live row. + ### Restore drill Restore only into an empty installation. The script verifies every checksum @@ -394,7 +581,7 @@ boundary and object store. memd must stay at one replica and uses a `Recreate` rollout so old and new pods never overlap: its indexing and embedding-provider switch coordination is process-local. Keep `memd.replicaCount=1` and `memd.autoscaling.enabled=false` until -[issue #55](https://github.com/fullstack-ai-infra/mem/issues/55) provides +[issue #55](https://github.com/bytefolk/mem/issues/55) provides cross-replica index generations. `Recreate` trades availability for correctness: plan a brief memd API interruption during upgrades. The migration stays single-run. @@ -439,7 +626,7 @@ The hosted service should reuse the multi-node topology, not the single-node Compose profile: - replicated Web and Worker across failure domains; keep one memd until - [issue #55](https://github.com/fullstack-ai-infra/mem/issues/55) enables + [issue #55](https://github.com/bytefolk/mem/issues/55) enables safe cross-replica indexing; - external HA PostgreSQL, Redis and S3; - managed secrets and immutable images; diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 1cd836c..f67b317 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -3,7 +3,7 @@ This document contains rules that are specific to the `mem` product and code base. The active organization-wide contribution lifecycle, issue and pull request forms, review rules, conduct policy, and support defaults come from -[`fullstack-ai-infra/.github`](https://github.com/fullstack-ai-infra/.github). +[`bytefolk/.github`](https://github.com/bytefolk/.github). This repository owns only `mem`-specific product, validation, security, ownership, triage, and release rules. Do not copy organization defaults back into this repository: GitHub treats local community files as whole-file or diff --git a/docs/DURABLE_CONTEXT.md b/docs/DURABLE_CONTEXT.md index 5ea3586..af0507f 100644 --- a/docs/DURABLE_CONTEXT.md +++ b/docs/DURABLE_CONTEXT.md @@ -3,7 +3,7 @@ Status: additive read-only contract for resuming explicitly approved, workspace-scoped active memory across sessions and channels. -Requirement: [mem#70](https://github.com/fullstack-ai-infra/mem/issues/70) +Requirement: [mem#70](https://github.com/bytefolk/mem/issues/70) REQ-001 / AC-001. ## Why one pinned contract diff --git a/docs/DURABLE_MEMORY.md b/docs/DURABLE_MEMORY.md new file mode 100644 index 0000000..da84592 --- /dev/null +++ b/docs/DURABLE_MEMORY.md @@ -0,0 +1,141 @@ +# Durable Memory (`durable-memory.v1`) + +Status: additive contract for derived, grant-scoped memory. Pins RoleWeave +[#327](https://github.com/bytefolk/roleweave/issues/327) R1 and the P1 +principal/grant correction on +[roleweave#345](https://github.com/bytefolk/roleweave/pull/345). +Requirement: [mem#220](https://github.com/bytefolk/mem/issues/220). + +This document does not change runtime behavior. HTTP handlers, migrations, +and MCP tools wait for Gate D0. Live E3 evidence is required before a later +runtime PR may claim recall, forget, or grant enforcement in production. + +## Why this envelope exists + +`durable-context.v1` resumes **already stored** structured memories through an +operator-owned allowlist. `durable-memory.v1` is the **derived record** that +RoleWeave, digital-employee `MemoryPort`, and mem share for long-lived +decisions, preferences, workflows, and negative signals. + +A free-string `scope` cannot express fail-closed isolation. Every record binds +all four of: + +1. mem `workspace_id`; +2. position principal `position.`; +3. canonical `memory_scope` (`/workspaces//positions/`); +4. a grant/revocation tuple (`grant_id`, `grant_version`, `permission_digest`, + `revoked_at`). `grant_id` is the existing `durable-context.v1` allowlist + row. `grant.mode` is always `read` (the same constraint as that table). + `grant_version` is an envelope-side monotonic revision because the grant + row has no version column today. Forget still requires the mem `delete` + token scope plus a workspace role that allows deletion; it is not a grant + mode. `capability-grant.v1` is a normative pointer (`server=mem`), not a + second allowlist. + +Cross-principal access is denied by default. Pinning cannot enlarge that +boundary. + +## Contract rules + +- The wire contract is pinned: `contract=durable-memory.v1`. Any other value + is unsupported. +- Recalled text is `trust=untrusted` and `authority=none`. It cannot grant + tools, identity, or instructions. +- Expired, revoked, malformed, superseded/archived, forgotten, and + out-of-scope records are not eligible for recall. +- `expires_at` / TTL decides **recall eligibility**. It does not physically + delete the source log, segment, or originating memory occurrence. +- Pin may keep an expired record eligible. Pin does not restore a revoked + grant, a forgotten payload, or another principal's record. +- Forget is permissioned (`delete` token scope plus a workspace role that + allows deletion) and is executed by mem. A caller must not treat a local + cache drop as success. Failure is a visible `forget_denied`. +- Exact readback compares the canonical envelope. A digest, `state_version`, + binding, or text drift is a mismatch, not a silent resume. +- `digest` is SHA-256 of `text` UTF-8 bytes. Forgotten tombstones digest the + empty string. Placeholders such as `sha256:ab` are malformed. +- Exact readback compares decoded envelope fields after `validateRecord`. It + is not RFC 8785 JSON canonicalization. + +## How grant and revocation enter readback / receipt + +Recall of one record returns a receipt, not a bare string: + +```json +{ + "contract": "durable-memory.v1", + "memory_id": "22222222-2222-4222-8222-222222222222", + "locator": "mem://memories/22222222-2222-4222-8222-222222222222@1", + "state_version": 1, + "eligible": false, + "omit_reason": "revoked", + "pinned": true, + "grant": { + "grant_id": "33333333-3333-4333-8333-333333333333", + "grant_version": 1, + "mode": "read", + "status": "revoked", + "permission_digest": "sha256:98056de97087164dd9e0f5235cba6019d9576230faa1e37b104e735e5b5729a6", + "revoked_at": "2026-09-18T11:59:00Z" + }, + "readback": null +} +``` + +| Receipt field | Source | Why it is here | +| --- | --- | --- | +| `grant.grant_id` | `durable-context.v1` allowlist row | Ties recall to an operator-owned grant, not a path string | +| `grant.grant_version` | that row's revision | Detects re-grant after revoke | +| `grant.permission_digest` | canonical (workspace, principal, memory_scope, mode, version) | Detects grant tuple drift | +| `grant.status` / `revoked_at` | soft revoke | Makes denial auditable; UI must not show a payload | +| `readback` | exact stored envelope | Present only when eligible | +| `omit_reason` | eligibility evaluator | `expired`, `revoked`, `malformed`, `superseded`, `forgotten`, `out_of_scope` | + +`capability-grant.v1` remains the digital-employee capability document. This +envelope stores a normative pointer (`schema_version=capability-grant.v1`, +`server=mem`); it does not reimplement grants. + +`binding.workspace_id` is the **mem** workspace. `memory_scope` uses the +digital-employee workspace instance id (`/workspaces//positions/`). +Those UUIDs are different namespaces and must not be required to match. + +Out-of-scope and malformed probes produce an empty receipt: no `memory_id`, +locator, grant block, or pin. In-scope denials (`revoked`, `expired`, +`forgotten`, `superseded`) keep grant status so the operator can see why +recall stopped. + +MemoryPort continues to hold no grant/revoke/forget methods. Operators +provision tokens and grants on mem's admin surface. RoleWeave UI may request +forget; only mem may ack it. + +## Eligibility order + +1. Malformed contract, digest, principal, or binding. +2. Workspace / principal / `memory_scope` mismatch (default deny). +3. Revoked grant. +4. Forgotten payload. +5. Superseded or archived lifecycle. +6. Expired `expires_at` unless pinned. +7. Else eligible; emit exact readback. + +## Relationship to existing mem APIs + +| Surface | Owns | Does not own | +| --- | --- | --- | +| `POST /v1/memories` and lifecycle | Occurrence storage, pin as ranking, archive/restore, permissioned forget | This envelope | +| `durable-context.v1` | Explicit read grants per `(workspace, principal, memory)` | Derived RoleWeave kinds | +| `durable-memory.v1` | Versioned derived envelope + eligibility + receipt | Transcript warehouse, Host resume, HTTP (until D0) | +| digital-employee `MemoryPort` | Write/readback/recall seam, env-referenced token | Grant administration | + +## Schema and example + +- [`docs/schemas/durable-memory.v1.schema.json`](schemas/durable-memory.v1.schema.json) +- [`docs/examples/durable-memory.v1.example.json`](examples/durable-memory.v1.example.json) +- Evaluator: `server/internal/durablememory` + +## Non-goals + +- Runtime HTTP, SQL, or MCP wiring before Gate D0. +- Treating summaries as authority. +- HNSW / vector retrieval ([#173](https://github.com/bytefolk/mem/issues/173)). +- Transcript warehouse or Host resume handles. diff --git a/docs/MIGRATION_SEQUENCE.md b/docs/MIGRATION_SEQUENCE.md new file mode 100644 index 0000000..df3c8d1 --- /dev/null +++ b/docs/MIGRATION_SEQUENCE.md @@ -0,0 +1,66 @@ +# Pending migration deployment sequence + +These draft changes are cumulative, not independently deployable: + +| Order | Draft / original PR | Migration | Required predecessor | +| --- | --- | --- | --- | +| 1 | #194 / #183 | 0024 file lexical lane | released/main schema 23 (merged) | +| 2 | #173 HNSW completion (supersedes #197 HOLD) | 0025 HNSW indexes + text continuation | #194, schema 24 | +| 3 | #195 / #185 | 0026 data-plane hygiene | schema 25 | + +The PR base chain is `main` → `codex/fix-pr-183` → `codex/fix-pr-180` +→ `codex/fix-pr-185`. Successor branches must include their predecessor schema and source. Local +repair branches are rebuilt on current main and replay the original authored +changes; published commit identities remain in the original PR history. +Keep this order when retargeting after a predecessor merges. + +On 2026-09-10, main `2986fe38175f54d99f15dd38a498708c6ecd88cd` and published +tags `v0.1.0` / `v0.1.1` contain only migrations 0001–0023. This does not prove +that a private deployment never applied a draft. Consequently migration +numbers and SQL identities are retained, not renumbered on an assumption. + +Migration 0025 creates the three cosine HNSW indexes. The shipping text route +no longer uses `DISTINCT ON (f.id) ORDER BY f.id` as its primary plan: it walks +cosine-ordered candidates and falls back to that exact query only when a bounded +scan underfills. Visual cosine-order already matched HNSW. Face DDL is not a +face-query speedup. #195 still requires predecessor schema 25. Recall and live +latency remain `#175`, not this migration. + +This document does not waive a review gate or authorize deployment. + +Goose startup remains strict: no `WithAllowMissing` or equivalent option is +enabled. A database that already applied 26 while omitting 24/25 will correctly +fail startup against the cumulative schema. Stop and obtain an operator-owned +recovery plan for such a database; do not edit its migration history, renumber +its SQL, or apply lower versions out of order to manufacture a pass. + +## Migration 0024 operational boundary + +Adding the stored generated `search_tsv` column rewrites existing `files` rows, +and its two indexes are built without `CONCURRENTLY`. Schedule a maintenance +window sized for the file corpus and expect table locks to block other access. +The migration indexes filenames only; `PathPrefix` remains a filter, not path +substring retrieval. Downgrading 0024 removes the derived column/indexes and +requires deploying server code that does not query the lexical route. + +Goose runs this migration transactionally, so an ordinary failure rolls back +its DDL. If an operator has applied some statements manually, `IF NOT EXISTS` +does not prove that an existing column or index has the correct definition. +Inspect both `goose_db_version` and the actual schema/index definitions before +an operator-owned recovery; do not mark an unverified partial schema applied. + +## Regression evidence + +`TestMigrationFilesContiguous` rejects embedded numeric gaps without a DB. +`scripts/verify.sh integration` creates a separate, owned `_test` database and +runs `TestMigrationUpgradeSequence`. It applies real Goose migrations to 23, +seeds a file with duplicate text chunks, then advances one version at a time +to the branch's declared head (24, 25, or 26). Each step checks full applied +history and preserved data; subsequent steps check lexical backfill, valid +HNSW DDL, and deduplication/unique rejection. Finally the ordinary production +`DB.Migrate` startup path must accept the resulting history unchanged. + +The dedicated test uses `MEM_MIGRATION_SEQUENCE_TEST_DB`, refuses a database +that already has Goose history, and must not target any developer or production +database. The existing owned-database runner performs cleanup. These are real +database tests over synthetic fixtures, not retrieval-quality or latency proof. diff --git a/docs/RUN_LOCAL.md b/docs/RUN_LOCAL.md index 14ad3ce..e127542 100644 --- a/docs/RUN_LOCAL.md +++ b/docs/RUN_LOCAL.md @@ -1,10 +1,11 @@ -# 本地运行 mem 全栈（裸机 · 无 Docker） +# 本地运行 mem 全栈（裸机 · 仅限开发） -本文用于启动完整开发栈和手工 smoke。可重复的单元、Race、PostgreSQL 集成、 -Web 浏览器验收及其通过标准统一见 [TESTING.md](TESTING.md)。 +第一次把 mem 跑起来，请走仓库根 README 和 [DEPLOYMENT.md](DEPLOYMENT.md) +里的 `deploy/compose` 路径，不要从本文开始。本文只覆盖**没有 Docker、需要改 +源码**的开发机。可重复的单元、Race、PostgreSQL 集成、Web 浏览器验收及其通过 +标准统一见 [TESTING.md](TESTING.md)。 -在没有 Docker 的 macOS 开发环境中，整套栈可用**本地进程**拉起，不走 -`docker compose`。 +在没有 Docker 的开发环境中，整套栈可用**本地进程**拉起，不走 `docker compose`。 一条命令起、一条命令停，运行时数据全部落在 `.dev/`（已 gitignore）。 ``` @@ -20,15 +21,23 @@ Web 浏览器验收及其通过标准统一见 [TESTING.md](TESTING.md)。 ## 一次性准备（首次或换机器时） -1. **依赖二进制**（脚本假设它们已就位）： - - PostgreSQL + pgvector（brew，keg-only，无需 sudo）： +1. **依赖二进制**（脚本假设它们已就位）。平台等价： + + | 依赖 | macOS (brew) | Ubuntu / Debian | WSL2 (Ubuntu) | + | --- | --- | --- | --- | + | PostgreSQL 17 + pgvector | `brew install postgresql@17 pgvector` | `apt install postgresql-17 postgresql-17-pgvector` | 同 Ubuntu（在 WSL2 Ubuntu 中执行） | + | MinIO | `brew install minio minio-mc` | 从 https://min.io/download 下载二进制到 `.dev/bin/` | 同 Ubuntu | + | Ollama | `brew install ollama` 或从 https://ollama.com 下载 | `curl -fsSL https://ollama.com/install.sh \| sh` | 同 Ubuntu（GPU 走 Windows 侧驱动） | + | Go 1.25 / Node 24 / Python 3.11+ / uv / protoc 34.1 | 用各平台官方安装器，版本钉在 `docs/TESTING.md` | 同左 | 同左 | + + - PostgreSQL + pgvector（macOS：brew，keg-only，无需 sudo）： ```bash brew install postgresql@17 pgvector ``` > 用 `@17` 而不是 `@16`：brew 的 pgvector bottle 只为 postgresql@17/@18 > 编译了 `vector.so`，装在 @16 上 `CREATE EXTENSION vector` 会失败。 > `dev_up.sh` 会自动探测 @17/@18/@16 中带匹配 pgvector 的版本。 - - MinIO server + mc client。推荐用 brew（dl.min.io 在本网络偶发限流/TLS 断连， + - MinIO server + mc client。macOS 推荐用 brew（dl.min.io 在本网络偶发限流/TLS 断连， brew 走 ghcr.io 更稳）： ```bash brew install minio minio-mc diff --git a/docs/VALIDATION_HNSW.md b/docs/VALIDATION_HNSW.md new file mode 100644 index 0000000..d59b1c8 --- /dev/null +++ b/docs/VALIDATION_HNSW.md @@ -0,0 +1,60 @@ +# HNSW index + text continuation for #173 + +Migration `0025_ann_hnsw_indexes.sql` adds cosine HNSW indexes to text (768), +visual (512), and face (512) embeddings. Main already shipped lexical +migration 0024; this branch's head is 25. + +## What this change proves + +- Populated 24 → 25 → 24 → 25 preserves text/visual/face vectors and rebuilds + three `VALID` `vector_cosine_ops` HNSW indexes (`TestHNSWMigrationPostgres`). +- Post-index INSERT succeeds; an UPDATE to the wrong dimension is rejected by + the `vector(N)` column type (failure mode: PostgreSQL dimension error, not a + silent pad/truncate). +- `EXPLAIN (ANALYZE)` of the shipping text cosine-order query and the visual + cosine-order query names `idx_embeddings_text_embedding_hnsw` and + `idx_embeddings_visual_embedding_hnsw` on a 2,000-row corpus. Planner + settings are not forced. +- `TestTextANNFileSemanticsPostgres` keeps best-chunk-per-file top-k when one + file owns 101 nearest chunks, and still enforces owner, literal path, + allow-list, MIME, and time filters. Invalid allow-lists fail closed. + +## Text continuation / fallback + +A bounded `ORDER BY distance LIMIT n` scan can underfill after per-file +deduplication (`ef_search=40` returning 40 chunks of one file). The shipping +path: + +1. Run a CTE `ORDER BY embedding <=> $1 LIMIT remaining` on `embeddings_text` + (HNSW-compatible; omit `ANY(exclude)` when the exclude list is empty). +2. Join those candidates to `files` and apply owner/path/MIME/time filters. +3. Keep the first sighting of each file (that chunk is the file's best). +4. Repeat, excluding selected files, until k files are collected. +5. If a round returns no new files, fill the remainder with the original + exact `DISTINCT ON (f.id) ORDER BY f.id, distance` query. + +Step 1 is the planner-usable shape. Step 4 preserves the previous result +contract on pathological corpora. Iterative-scan GUC is not enabled. + +## What this change does not prove + +- Live embedding quality, production latency, index build time, or numerical + recall. Those belong to [#175](https://github.com/bytefolk/mem/issues/175) + (shipping search-path producer) and the closed producer attempt + [#184](https://github.com/bytefolk/mem/pull/184). Fixture scores are not + substituted. +- Face query speedup. `assignCluster` still averages centroids in Go. +- `index_generation_vectors` ANN. The column is undimensioned. + +## Local gates + +```bash +MEM_TEST_DB="$MEM_TEST_DB" ./scripts/verify.sh integration +``` + +`run_hnsw_migration` creates a fresh `_test` database and runs +`TestHNSWMigrationPostgres`, which records EXPLAIN ANALYZE. `scripts/verify_hnsw_indexes.sh` +is a manual `psql` helper; CI does not call it because libpq rejects some pgx URIs. + +Face evidence is valid DDL and populated-table migration, not a measured +face-query speedup. diff --git a/docs/adr/0006-durable-memory-v1.md b/docs/adr/0006-durable-memory-v1.md new file mode 100644 index 0000000..e57a7ea --- /dev/null +++ b/docs/adr/0006-durable-memory-v1.md @@ -0,0 +1,73 @@ +# ADR 0006: durable-memory.v1 is an additive principal-bound envelope + +- Status: Proposed (contract only; runtime waits for RoleWeave Gate D0) +- Date: 2026-09-18 +- Consumes: [mem#220](https://github.com/bytefolk/mem/issues/220), + RoleWeave [#327](https://github.com/bytefolk/roleweave/issues/327) R1, + P1 on [roleweave#345](https://github.com/bytefolk/roleweave/pull/345) + +## Context + +ADR 0001 stores immutable Agent occurrences. ADR 0003 adds pin, archive, +restore, and permissioned forget. `durable-context.v1` resumes those +occurrences through an explicit grant allowlist. + +RoleWeave's memory plane needs a **derived** record for long-lived decisions +and preferences. The R1 draft used a free-string `scope`. That cannot +fail-closed isolate workspace, position principal, MemoryPort `memoryScope`, +and grant/revocation, and it cannot show why a receipt omitted a record. + +Runtime implementation is blocked until Gate D0. The contract must still be +reviewable now so later PRs pin one schema. + +## Decision + +Ship `durable-memory.v1` as an additive envelope: + +- Required `binding` (`workspace_id`, `position_id`, `principal`, + `memory_scope`) instead of `scope`. +- Required `grant` that reuses `durable-context.v1` grant **ids** and points at + `capability-grant.v1`. `grant.mode` is `read` only. `grant_version` is + envelope-side. `revoked_at` and `permission_digest` are first-class. +- Eligibility treats expired, revoked, malformed, superseded, forgotten, and + out-of-scope records as ineligible. +- Pin may preserve TTL eligibility and must not enlarge permission. +- TTL/expiry never implies physical deletion of the source log. +- Forget is a permissioned mem operation. The contract evaluator never + reports a local fake delete. +- Exact readback compares the canonical envelope. Grant status always appears + on the receipt. + +No HTTP route, migration, or MCP tool is added in this change. + +## Consequences + +Positive: + +- RoleWeave #345 P1 has a mem-side schema to pin. +- Cross-principal default deny is structural, not a convention on a path + string. +- Receipts can display revocation without returning payload. + +Trade-offs: + +- Two versioned contracts (`durable-context.v1` and `durable-memory.v1`) until + a later runtime PR projects one onto the other. +- Live E3 evidence is still required before claiming production recall. + +## Rejected alternatives + +### Keep `scope` as a free string and document the format + +A path string cannot carry grant version, revocation, or permission digest. +Callers would invent parallel headers. That is the P1 defect. + +### Implement HTTP now + +AC-006 and RoleWeave Gate D0 forbid runtime consumption of an unaccepted +revision. A handler without an accepted parent design would be speculative. + +### Treat pin as a permission upgrade + +Pin is a ranking / TTL exception. Using it to bypass grants would leak +cross-principal memory. diff --git a/docs/examples/durable-memory.v1.example.json b/docs/examples/durable-memory.v1.example.json new file mode 100644 index 0000000..9a6afdf --- /dev/null +++ b/docs/examples/durable-memory.v1.example.json @@ -0,0 +1,46 @@ +{ + "contract": "durable-memory.v1", + "memory_id": "22222222-2222-4222-8222-222222222222", + "kind": "project_decision", + "binding": { + "workspace_id": "11111111-1111-4111-8111-111111111111", + "position_id": "repo-owner", + "principal": "position.repo-owner", + "memory_scope": "/workspaces/44444444-4444-4444-8444-444444444444/positions/repo-owner" + }, + "grant": { + "grant_id": "33333333-3333-4333-8333-333333333333", + "mode": "read", + "grant_version": 1, + "permission_digest": "sha256:98056de97087164dd9e0f5235cba6019d9576230faa1e37b104e735e5b5729a6", + "capability_grant": { + "schema_version": "capability-grant.v1", + "server": "mem" + }, + "granted_at": "2026-09-17T12:00:00Z", + "revoked_at": null + }, + "source": { + "kind": "segment", + "id": "seg_01", + "digest": "sha256:bee60ba20052b2969621c28f7378297c3b38cfa4eba66b22ff0df381447b2f8c" + }, + "citations": ["turn:t33"], + "producer": { + "agent_id": "position.repo-owner", + "session_id": "sess_1", + "task_id": "task_1" + }, + "event_at": "2026-09-17T11:58:00Z", + "created_at": "2026-09-17T12:00:00Z", + "expires_at": "2026-12-17T12:00:00Z", + "importance": "high", + "confidence": 0.7, + "state_version": 1, + "pinned": false, + "lifecycle": "active", + "trust": "untrusted", + "authority": "none", + "text": "Search APIs must match title OR body and return matchField.", + "digest": "sha256:bfa1e9bb9bacbabafb44c2c89ee7a108e81e10628c53a645af5b49eff081166f" +} diff --git a/docs/integrations/qoder-ingest.md b/docs/integrations/qoder-ingest.md index c7b97bb..3b49708 100644 --- a/docs/integrations/qoder-ingest.md +++ b/docs/integrations/qoder-ingest.md @@ -20,7 +20,7 @@ mem ingest qoder --limit 200 # stop after 200 memories | Flag | Default | Meaning | | --- | --- | --- | -| `--root` | `~/.qoder/projects` | glob base scanned recursively for `*.jsonl` | +| `--root` | `~/.qoder/projects` | glob base scanned recursively for `*.jsonl`, resolved to a canonical absolute path first | | `--path-root` | `/AgentTranscripts` | virtual path prefix for ingested memories | | `--state-dir` | `~/.mem/ingest/qoder` | checkpoint cursor directory | | `--dry-run` | `false` | parse and plan only; do not write | @@ -35,7 +35,8 @@ Each parseable line becomes one memory: - **path** — `//`, where `` is the first path segment under the ingest root and `` is the transcript file name minus `.jsonl` -- **source** — `{"type":"qoder","ref":,"locator":{"line":N}}` +- **source** — `{"type":"qoder","ref":,"locator":{"line":N}}`, where `` + is the transcript's canonical absolute path (symlinks resolved) - **producer** — `session_id` (session slug) and `agent_id` (the model/agent id recorded on the line, when present) - **event_at** — the message timestamp (RFC 3339 or epoch), when present @@ -52,7 +53,7 @@ carry no ingestible text, or are JSON-LD continuations are skipped rather than failing the run. Undocumented fields do not block ingestion. > The connector targets the transcript store described in -> [fullstack-ai-infra/mem#103](https://github.com/fullstack-ai-infra/mem/issues/103). +> [bytefolk/mem#103](https://github.com/bytefolk/mem/issues/103). > If your CLI emits a materially different shape, the parser keys live in > `server/cmd/mem/qoder_transcript.go` and are trivially extended. @@ -62,9 +63,14 @@ Ingestion is **incremental and idempotent**: - A per-file cursor (`~/.mem/ingest/qoder/.json`) records the highest already-ingested line. A re-run parses only lines appended since the last run. + Because the key is the canonical path, one store shares one cursor however it is + reached, and two directories that each contain `sessions/p.jsonl` do not. - Even if the cursor is lost, the stable `Idempotency-Key` per line makes a re-post an idempotent replay (`replayed` responses are counted, not duplicated). +- A `--root` previously spelled relative, or one behind a symlink, is keyed + differently than before: its existing cursor is orphaned and that store is + re-posted once under new keys. Deleting `~/.mem/ingest/qoder` resets all cursors (safe: ids remain idempotent). @@ -98,7 +104,7 @@ mem search "recruit rubric" --path /AgentTranscripts ## MyContext interop (follow-up, not a blocker) -[#103](https://github.com/fullstack-ai-infra/mem/issues/103) accepts MyContext +[#103](https://github.com/bytefolk/mem/issues/103) accepts MyContext interop as a deferred follow-up (AC-003). The connector is structured so a MyContext bridge can reuse the same normalize-and-write path later: @@ -109,4 +115,4 @@ MyContext bridge can reuse the same normalize-and-write path later: capturing session streams at the runtime level would avoid parsing files at all. -Neither is required for this connector to be useful today. \ No newline at end of file +Neither is required for this connector to be useful today. diff --git a/docs/maintainers/releasing.md b/docs/maintainers/releasing.md index 4af8c0b..42b7cef 100644 --- a/docs/maintainers/releasing.md +++ b/docs/maintainers/releasing.md @@ -5,6 +5,21 @@ repository Release workflow publishes multi-platform `mem-mcp` binaries to a GitHub Release. The npm package is published only after that Release has been verified because its runtime bootstrap downloads and verifies those assets. +The [2026-09-10 release decision](https://github.com/bytefolk/mem/issues/153#issuecomment-5612770493) +selects `@bytefolk/mem-mcp@0.1.2`, superseding the earlier keep-old-scope +instruction. [mem#153](https://github.com/bytefolk/mem/issues/153) and +[organization #22](https://github.com/bytefolk/.github/issues/22) govern the +migration. Source preparation does not clear their npm ownership/authentication +HOLD. The 2026-09-10 owner check returned `ENEEDAUTH`; no npm control, publisher +binding or real publication is established by this document or local tests. + +Human contribution provenance: [#154](https://github.com/bytefolk/mem/pull/154) +(`bcd786f`, liyuanyang) supplies the actual package/cache migration; +[#162](https://github.com/bytefolk/mem/pull/162) (waterbro-8) supplies the MCP +registry identity correction only; [#168](https://github.com/bytefolk/mem/pull/168) +(`8a92baa`, waterbro-8) supplies mechanical version alignment. Retain these +sources in the release PR; do not attribute their changes to automated tools. + ## Version policy `mem` is a single-version monorepo. The Go service and clients, Python worker, @@ -102,28 +117,213 @@ creates, moves, or replaces a tag. `sha256sum --check --strict mem-mcp-checksums.txt`, and inspect `go version -m` on each binary for the recorded release commit before starting npm publication. -6. In `npm/`, rerun `npm test`, the npm 12 clean-tarball test, and - `npm pack --dry-run --ignore-scripts`. Confirm - `npm view @fullstack-ai-infra/mem-mcp@VERSION version` does not find the - version, then publish it once with - `npm publish --access public --tag latest`. -7. Install the public package in a clean temporary consumer with lifecycle - scripts disabled, invoke `mem-mcp`, and confirm the checksum-verified binary - is fetched from the matching GitHub Release. Record the Release URL, npm - package URL, checksum result, and smoke-test result on the release issue. - -Trusted Publishing is the target npm credential path. For the bootstrap -publication, prefer interactive npm 2FA. If a token is required, use a one-day -granular npm token restricted to read/write access for -`@fullstack-ai-infra/mem-mcp` only, with no unrelated package or organization -access. Keep it out of repository files, logs, workflow inputs, and issue -comments. Read the published version back from the registry and complete the -clean-install smoke test before revoking the token immediately. - -If GitHub asset upload fails, inspect and remove any incomplete draft before a -reviewed retry; do not move the tag. If npm publication is wrong, deprecate the -bad version when possible and correct it with a new patch version rather than -reusing either the tag or package version. +6. Complete the interactive bootstrap and owner proof below. Run `npm test` + in `npm/`, the npm 12 clean-tarball test on Linux, and + `npm pack --dry-run --ignore-scripts`. Record any skipped platform explicitly. +7. Run `.github/workflows/npm-publish.yml` from the **exact stable tag**, after + approving its `npm-release` environment. Its only registry write publishes + the checked tarball with `--tag next --access public --provenance`. +8. Read back metadata, integrity, OIDC publisher ID, signatures, provenance and + dist-tags. The workflow installs that exact public version with lifecycle + scripts disabled and runs `npm audit signatures`. It does not launch the + binary or promote `latest`. Complete the separate owner gates below. + +## Bootstrap and Trusted Publisher setup (release owner only) + +All commands in this section describe future owner actions, not actions +performed by the source-preparation PR. Use Node 24 and npm >=11.15.0. The +workflow pins npm 11.15.0 and validates the actual versions; a Node upgrade +alone does not prove the npm requirement. + +1. The operator designated by organization #22 (`PeterGuy326`) logs into npm + interactively on an authorized machine with 2FA. Privately inspect + `npm whoami`, `npm org ls bytefolk --json`, organization role, package-name + control, team/access and 2FA policy. Anonymous 404 and GitHub organization + membership are not npm ownership proof. Record only a sanitized verdict on + #153/#22. An error, missing permission or unclear ownership keeps the HOLD. +2. Prepare and independently review the **aligned** `0.1.2-rc.0` source, + annotated `v0.1.2-rc.0` tag and matching seven GitHub assets using the same + binary release gates. Do not just change a stable wrapper's version: the + installer resolves assets for its own exact version. The stable npm workflow + deliberately refuses RC tags. + + **Separate prerequisite, NOT VERIFIED:** this stable-source PR does not + provide an aligned RC checkout, RC Release or bootstrap tarball. The current + binary workflow and source/version validators accept the `-rc.NUMBER` + spelling, but the RC must still have its own reviewed version/changelog + surfaces and successful full gate run. If the selected RC source lacks + those capabilities, a separate reviewed RC-gate implementation is required + first. The following owner commands become usable only after that evidence + exists. Do not merge/tag stable and then retroactively create an RC from + different or unreviewed bytes; coordinate the RC prerequisite before sealing + the stable release commit/tag. Never retag a published version. +3. From that clean RC checkout, pack and inspect the wrapper. After explicit + bootstrap authorization, publish that reviewed tarball with interactive 2FA: + + ```sh + npm publish ./bytefolk-mem-mcp-0.1.2-rc.0.tgz --access public --tag next --ignore-scripts --registry=https://registry.npmjs.org + npm view @bytefolk/mem-mcp@0.1.2-rc.0 name version dist --json --registry=https://registry.npmjs.org + npm dist-tag ls @bytefolk/mem-mcp --registry=https://registry.npmjs.org + ``` + + Verify public access and clean installation against the RC assets. The + interactive bootstrap is not proof of GitHub OIDC provenance. Never assign + this RC to `latest`, reuse a published version, or provide a CI token fallback. +4. On the new package's npm settings page, configure GitHub Trusted Publishing: + organization `bytefolk`, repository `mem`, workflow filename + `npm-publish.yml`, environment `npm-release`, and permission to run direct + `npm publish`. A stage-only publisher is insufficient for this workflow. + Read back the binding (settings UI or `npm trust list @bytefolk/mem-mcp + --json`) and its configuration ID. Do not infer success just because npm + accepted a settings form. No settings are created by the workflow. +5. Configure the GitHub `npm-release` environment with release-owner required + review, prevention of self-review/admin bypass, and permitted release tags; + separately protect tag creation. A tag's ancestry in protected `main` is + checked by code. Record exact-commit CI and independent review before + approving deployment. Environment approval is a gate, not evidence that + npm ownership was verified. +6. Store a sanitized, exact-release JSON attestation as the **environment** + variable `NPM_RELEASE_PROOF`, using the schema below. Fill it only from the + authenticated owner's readback, with a lifetime of at most 24 hours. Do not + put credentials, OTPs, raw account output or private evidence in it. Renew + proof for each tag/commit or after a binding change. + +```json +{ + "schema": 1, + "package": "@bytefolk/mem-mcp", + "tag": "v0.1.2", + "commit": "REPLACE_WITH_EXACT_40_CHARACTER_TAG_COMMIT", + "channel": "next", + "organization": "bytefolk", + "organizationControlVerified": false, + "packageAccessVerified": false, + "twoFactorVerified": false, + "publisherVerified": false, + "allowPublish": false, + "repository": "bytefolk/mem", + "workflow": "npm-publish.yml", + "environment": "npm-release", + "publisherId": "REPLACE_WITH_NPM_OIDC_CONFIG_ID", + "approvedBy": "REPLACE_WITH_HUMAN_GITHUB_LOGIN", + "evidence": "REPLACE_WITH_SANITIZED_OWNER_COMMENT_ON_153_OR_22", + "verifiedAt": "REPLACE_WITH_UTC_TIMESTAMP", + "expiresAt": "REPLACE_WITH_UTC_TIMESTAMP" +} +``` + +This deliberately non-authorizing example must fail. The attestation is a +human-controlled prerequisite, **not an automated npm membership lookup**. +The workflow also requires public bootstrap readback, a successful OIDC +exchange, and the same publisher ID in the published version. Missing or stale +proof blocks publication; no token, legacy scope or alternate registry fallback +exists. Do not fill these fields from the naming decision alone. + +## Stable OIDC publication to next + +Merge the stable `0.1.2` source/version PR, independently approve its exact +commit, and complete the stable GitHub binary Release first. The npm workflow +must exist in both the default branch and the selected stable tag. GitHub +events emitted by `GITHUB_TOKEN` normally do not start another workflow, so a +successful binary Release may need this explicit owner dispatch: + +```sh +gh workflow run npm-publish.yml --repo bytefolk/mem --ref v0.1.2 -f version=v0.1.2 +``` + +Unlike the binary Release workflow's manual entry, this npm entry rejects a +`main` dispatch. GitHub's event SHA/ref, checked-out annotated tag, npm package +and provenance source must agree. Its guard fetches `origin/main` and the exact +tag, rejects non-ancestry, dirty source, mismatched package/MCP/version surfaces, +draft/prerelease/wrong releases, and missing/extra/empty assets. It downloads all +seven assets, validates exactly six checksum rows, checks digests and embedded +Go commit/platform metadata without executing binaries, and packs the exact +six wrapper files with scripts disabled. Source, proof, Release identity, +registry version absence, channels and tarball integrity are rechecked before +the sole publish call. Repository npmrc files and ambient npm credentials or +configuration overrides are refused; npm runs with empty, isolated config. + +The run then anonymously reads the new version, checks tarball integrity, +metadata, GitHub OIDC configuration ID, signature/provenance presence, `next` +and unchanged other tags, and verifies registry signatures/attestations through +`npm audit signatures` in a clean consumer. Its receipt records only `next`. +No automatic dist-tag promotion, access grant, deprecation or rollback occurs. + +The npm release proof deliberately fixes `channel` to `next`; changing that +channel is a release-policy change and requires updating the proof validator, +workflow assertions, rollback analysis and independent review together. The +MCP Registry workflow likewise pins `mcp-publisher` to an explicit upstream +version and per-platform SHA-256 digest. Upgrade both values from the same +upstream release asset set; never switch the publish job back to `latest`. + +If publish fails or subsequent readback/audit fails, **stop**. The version may +already exist even if the run is red. Inspect the exact package/version, +preflight integrity and run before deciding recovery; never rerun publish as +an authentication test. Registry propagation delays also leave the run failed +pending read-only verification. Never overwrite/unpublish or move the tag. + +## Separate latest and migration gates (release owner only) + +Before promotion, attach all of the following to #153/#122: successful exact +tag OIDC run and receipt; current public access and Trusted Publisher readback; +verified npm signatures and provenance with expected repository/commit/workflow; +six binary checksums; and clean Linux, macOS and Windows install **and launch** +evidence. Install with lifecycle scripts disabled, invoke `mem-mcp`, and verify +its matching GitHub binary download. Include the existing-cache compatibility +case. Managed macOS machines must not execute newly built temporary Go binaries; +use approved isolated platform runners for those acceptance checks. + +After the release owner explicitly accepts that evidence, use interactive 2FA: + +```sh +npm dist-tag add @bytefolk/mem-mcp@0.1.2 latest --registry=https://registry.npmjs.org +npm view @bytefolk/mem-mcp dist-tags --json --registry=https://registry.npmjs.org +``` + +Require `latest=0.1.2` and repeat a clean default-channel install. Only after +OIDC is proven should the owner grant the reviewed `bytefolk:developers` +package access and enable npm's require-2FA/disallow-tokens setting, then read +those settings back. Complete consumer, directory/MCP, documentation and +lockfile migrations. Only after those consumers work may the owner authorize +deprecation of `@fullstack-ai-infra/mem-mcp` with an explicit replacement +message. Keep old `0.1.1` installable; **never unpublish** it or delete its cache. + +Before cutover, rollback means stop and keep the old package untouched. After +cutover, restore consumers to the still-installable old coordinates if needed; +an owner may undo deprecation or restore a previously verified new-scope tag. +Correct bad releases with a higher patch. Recreate an incorrect publisher +binding through the owner process rather than adding a token fallback. + +## Local verification and evidence limits + +```sh +node --test scripts/npm-release.test.mjs +./scripts/test_release_guards.sh +./scripts/validate_release_version.sh 0.1.2 +git diff --check +``` + +The new Node tests use temporary fixtures and injected Git/GitHub/npm command +adapters; no test publishes, changes remote settings, or executes a Go binary. +They exercise refusal and partial-failure behavior, including unavailable org +proof/registry, stale source, unsafe tags/packages, bad assets and failed +readback. A fixture PASS is local E3 evidence only. GitHub Actions execution, +independent approval, authenticated npm ownership/binding, actual OIDC/provenance +publication, three-platform launches and `latest` remain **NOT VERIFIED** until +their separate real evidence is recorded. + +Technical assumptions checked against official documentation on 2026-09-10: +[trusted publishers](https://docs.npmjs.com/trusted-publishers/), +[npm trust prerequisites](https://docs.npmjs.com/cli/v11/commands/npm-trust/), +[publish options](https://docs.npmjs.com/cli/v11/commands/npm-publish/), +[dist-tags](https://docs.npmjs.com/cli/v11/commands/npm-dist-tag/), and +[provenance verification](https://docs.npmjs.com/generating-provenance-statements/). +GitHub's [workflow trigger rules](https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/trigger-a-workflow) +explain the explicit dispatch after a Release published using `GITHUB_TOKEN`. +The trust command requires npm >=11.15.0 and an existing package; GitHub OIDC +requires the configured workflow/environment and hosted runner. Direct-publish +permission must be checked explicitly. `next` must be specified because npm's +default publication channel is `latest`. ## Deferred channels @@ -133,7 +333,8 @@ The following are out of scope for the initial baseline: - PyPI publication - Container registry publication - Homebrew or other operating-system package managers -- Signed multi-platform binaries and provenance attestations +- Signed multi-platform binaries and binary provenance attestations (npm + package provenance is required by the OIDC gate above) Each new channel requires its own issue, threat and rollback analysis, credential design, and independently reviewable workflow. GitHub Release diff --git a/docs/mcp.md b/docs/mcp.md index 5fd17a2..6999b3d 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -90,6 +90,18 @@ coordinates, timezone-free timestamps and control characters are rejected by the HTTP API. The metadata is persisted server-side and is not included in an enrichment-model prompt. +### Model-free file search + +Use `mem_search` with `route=lexical` (or `mem search "query" --route lexical`) +to search filenames without an embedding worker or managed provider. `scope` +restricts the virtual-folder subtree; it does not make paths or file contents +searchable. `route=auto` fuses only the text and visual embedding routes and +does not fall back to lexical when the worker is unavailable. + +Lexical scoring uses name substrings, simple full-text matching, then tolerant +trigram matching. It scores the files remaining after workspace, path, MIME +and time filters; no indexed candidate-pruning or latency guarantee is claimed. + ### Reviewing file annotations Use `mem_info` (or `mem info --format json`) to read pending @@ -138,7 +150,7 @@ The canonical product surface is: | `mem_checkpoint_list` | List newest-first bounded checkpoint summaries for one task | | `mem_checkpoint_get` | Get one immutable checkpoint and its full handoff payload | | `mem_resume` | Restore the current task head or a selected historical checkpoint, including resolved and missing evidence | -| `mem_search` | Natural-language search (text / visual / auto fuse); ranked files + snippets | +| `mem_search` | Natural-language search (text / visual / auto fuse); ranked files + snippets. `route=lexical` is model-free (FTS + trigram over file names, no worker needed) | | `mem_context` | Build an evidence-backed context pack for the calling Agent | | `mem_related` | Top-K files related to a `file_id` by embedding similarity | | `mem_face` | Person clusters: `action=list` / `name` / `merge` | @@ -295,7 +307,8 @@ same logical request should supply and retain a stable key so a committed result can replay without another provider invocation or charge. A `504` means the provider outcome is uncertain: do not automatically retry, and do not invent a new key. `mem_context` with `source=memory` stays lexical and -model-independent. +model-independent. `mem_search` with `route=lexical` is likewise model-free: +it uses FTS + trigram over file names and works without a configured worker. Its target output is structured for an Agent to consume: diff --git a/docs/schemas/durable-memory.v1.schema.json b/docs/schemas/durable-memory.v1.schema.json new file mode 100644 index 0000000..dfbec63 --- /dev/null +++ b/docs/schemas/durable-memory.v1.schema.json @@ -0,0 +1,253 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://getmem.dev/schemas/durable-memory.v1.schema.json", + "title": "mem durable-memory.v1", + "description": "Additive derived-memory envelope for RoleWeave MEM-1. Isolation is workspace + position principal + memory_scope + grant/revocation. A free-string scope property is not part of this contract. Pin does not enlarge permission. TTL/expires_at is recall eligibility, not physical deletion of the source log.", + "type": "object", + "additionalProperties": false, + "required": [ + "contract", + "memory_id", + "kind", + "binding", + "grant", + "source", + "citations", + "producer", + "event_at", + "created_at", + "state_version", + "pinned", + "lifecycle", + "trust", + "authority", + "text", + "digest" + ], + "properties": { + "contract": { + "const": "durable-memory.v1" + }, + "memory_id": { + "type": "string", + "format": "uuid" + }, + "kind": { + "type": "string", + "enum": [ + "project_decision", + "user_preference", + "reusable_workflow", + "negative_signal", + "active_task_state", + "compliance_retained", + "observation", + "decision", + "preference", + "task_state", + "fact", + "note", + "artifact" + ] + }, + "binding": { + "$ref": "#/$defs/binding" + }, + "grant": { + "$ref": "#/$defs/grant" + }, + "source": { + "$ref": "#/$defs/source" + }, + "citations": { + "type": "array", + "maxItems": 32, + "items": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + } + }, + "producer": { + "$ref": "#/$defs/producer" + }, + "event_at": { + "type": "string", + "format": "date-time" + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "expires_at": { + "type": ["string", "null"], + "format": "date-time", + "description": "Recall eligibility deadline. Expiry never physically deletes the source log." + }, + "importance": { + "type": "string", + "enum": ["low", "normal", "high"] + }, + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "state_version": { + "type": "integer", + "minimum": 1 + }, + "pinned": { + "type": "boolean", + "description": "May preserve TTL eligibility. Must not enlarge workspace, principal, memory_scope, or grant." + }, + "lifecycle": { + "type": "string", + "enum": ["active", "archived", "superseded", "expired", "forgotten"] + }, + "trust": { + "const": "untrusted" + }, + "authority": { + "const": "none" + }, + "text": { + "type": "string", + "maxLength": 16384 + }, + "digest": { + "$ref": "#/$defs/sha256", + "description": "SHA-256 of text UTF-8 bytes. Forgotten tombstones digest the empty string." + } + }, + "$defs": { + "sha256": { + "type": "string", + "pattern": "^sha256:[a-f0-9]{64}$", + "description": "Full SHA-256 digest. Truncated placeholders such as sha256:ab are invalid." + }, + "principal": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9._-]{0,127}$", + "description": "Position principal, always position.." + }, + "binding": { + "type": "object", + "additionalProperties": false, + "required": ["workspace_id", "position_id", "principal", "memory_scope"], + "properties": { + "workspace_id": { + "type": "string", + "format": "uuid", + "description": "mem workspace. Cross-workspace probes are out of scope." + }, + "position_id": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9._-]{0,118}$" + }, + "principal": { + "$ref": "#/$defs/principal" + }, + "memory_scope": { + "type": "string", + "minLength": 2, + "maxLength": 1024, + "pattern": "^/workspaces/[^/]+/positions/[a-z0-9][a-z0-9._-]{0,118}$", + "description": "Canonical MemoryPort virtual path. Not a free-form scope string." + } + } + }, + "grant": { + "type": "object", + "additionalProperties": false, + "required": [ + "grant_id", + "mode", + "grant_version", + "permission_digest", + "capability_grant", + "granted_at", + "revoked_at" + ], + "properties": { + "grant_id": { + "type": "string", + "format": "uuid", + "description": "Existing durable-context.v1 grant row. Grants are not invented here." + }, + "mode": { + "const": "read", + "description": "Recall grants are read-only, matching durable-context.v1. Forget uses the mem delete token scope, not this field." + }, + "grant_version": { + "type": "integer", + "minimum": 1, + "description": "Monotonic grant revision. Revoke/re-grant advances this." + }, + "permission_digest": { + "$ref": "#/$defs/sha256", + "description": "SHA-256 of the canonical (workspace_id, principal, memory_scope, mode, grant_version) tuple." + }, + "capability_grant": { + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "server"], + "properties": { + "schema_version": { + "const": "capability-grant.v1" + }, + "server": { + "const": "mem" + } + } + }, + "granted_at": { + "type": "string", + "format": "date-time" + }, + "revoked_at": { + "type": ["string", "null"], + "format": "date-time", + "description": "Soft revocation. A revoked grant is ineligible for recall and must appear on the receipt." + } + } + }, + "source": { + "type": "object", + "additionalProperties": false, + "required": ["kind", "id", "digest"], + "properties": { + "kind": { + "type": "string", + "enum": ["segment", "memory", "artifact"] + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "digest": { + "$ref": "#/$defs/sha256" + } + } + }, + "producer": { + "type": "object", + "additionalProperties": false, + "required": ["agent_id"], + "properties": { + "agent_id": { + "$ref": "#/$defs/principal" + }, + "session_id": { + "type": "string", + "maxLength": 200 + }, + "task_id": { + "type": "string", + "maxLength": 200 + } + } + } + } +} diff --git a/docs/schemas/mem-doctor.v1.schema.json b/docs/schemas/mem-doctor.v1.schema.json new file mode 100644 index 0000000..3f8dd30 --- /dev/null +++ b/docs/schemas/mem-doctor.v1.schema.json @@ -0,0 +1,84 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://getmem.dev/schemas/mem-doctor.v1.schema.json", + "title": "mem doctor report v1", + "description": "Read-only diagnosis emitted by `mem doctor --format json`. Mirrors doctorReport and doctorCheck in server/cmd/mem/cmds_doctor.go. Contains no credential, token or DSN value: the server URL is reported with userinfo removed.", + "type": "object", + "additionalProperties": false, + "required": [ + "contract", + "schema_version", + "server", + "cli_version", + "exit_code", + "checks" + ], + "properties": { + "contract": { + "const": "mem.doctor" + }, + "schema_version": { + "const": 1 + }, + "server": { + "type": "string", + "description": "Configured memd base URL, userinfo redacted.", + "minLength": 1 + }, + "cli_version": { + "type": "string", + "description": "Version this CLI build reports; \"dev\" when none was injected at build time." + }, + "server_version": { + "type": "string", + "description": "Version the server reported. Absent when the version probe did not run." + }, + "exit_code": { + "type": "integer", + "description": "SPEC 7.1 process exit code: first failing check's code, else 0.", + "enum": [0, 2, 3, 4, 5] + }, + "checks": { + "type": "array", + "minItems": 4, + "maxItems": 4, + "description": "Fixed ordered list, never a wizard: server_reachability, credential, workspace, version_skew.", + "prefixItems": [ + { "allOf": [ { "$ref": "#/$defs/check" }, { "properties": { "name": { "const": "server_reachability" } } } ] }, + { "allOf": [ { "$ref": "#/$defs/check" }, { "properties": { "name": { "const": "credential" } } } ] }, + { "allOf": [ { "$ref": "#/$defs/check" }, { "properties": { "name": { "const": "workspace" } } } ] }, + { "allOf": [ { "$ref": "#/$defs/check" }, { "properties": { "name": { "const": "version_skew" } } } ] } + ], + "items": { "$ref": "#/$defs/check" } + } + }, + "$defs": { + "check": { + "type": "object", + "additionalProperties": false, + "required": ["name", "status", "exit_code", "detail"], + "properties": { + "name": { + "enum": ["server_reachability", "credential", "workspace", "version_skew"] + }, + "status": { + "enum": ["ok", "warn", "fail", "skipped"], + "description": "\"skipped\" means an earlier failure made this check unrunnable; it is never an inferred pass." + }, + "exit_code": { + "type": "integer", + "description": "This finding's contribution to the process exit code. Advisory and skipped findings contribute 0.", + "enum": [0, 2, 3, 4, 5] + }, + "detail": { + "type": "string", + "minLength": 1 + }, + "hint": { + "type": "string", + "description": "Actionable next step. First-run hints name the documented container path." + } + } + } + } +} diff --git a/npm/README.md b/npm/README.md index 6a62e8a..50b6267 100644 --- a/npm/README.md +++ b/npm/README.md @@ -1,13 +1,13 @@ # mem-mcp -> npm wrapper for [fullstack-ai-infra/mem](https://github.com/fullstack-ai-infra/mem) — a portable, self-hosted memory plane for AI agents. +> npm wrapper for [bytefolk/mem](https://github.com/bytefolk/mem) — a portable, self-hosted memory plane for AI agents. This package distributes the `mem-mcp` stdio MCP server binary so it can be installed with `npm install` and launched by any MCP-compatible host (Claude Desktop, Cursor, Cline, Codex, etc.). ## Install ```bash -npm install @fullstack-ai-infra/mem-mcp +npm install @bytefolk/mem-mcp@0.1.2 ``` ## Usage @@ -39,12 +39,34 @@ a later diagnostic sink fails. The executable cache is outside the installed npm package and is isolated by package version and platform. Defaults are `$XDG_CACHE_HOME` (or `~/.cache`) on Linux, `~/Library/Caches` on macOS, and `%LOCALAPPDATA%` on Windows, below -`fullstack-ai-infra/mem-mcp`. Set `MEM_MCP_CACHE_DIR` to an absolute path to use +`bytefolk/mem-mcp`. Set `MEM_MCP_CACHE_DIR` to an absolute path to use a different writable cache root. A per-asset cross-process lock serializes verification and atomic replacement, so concurrent hosts cannot expose or delete each other's downloads. Stale-lock recovery removes only artifacts named by that lock owner's nonce and leaves foreign temporary files untouched. +With the default cache root, the wrapper also checks the old +`fullstack-ai-infra/mem-mcp/v/-` location for the +exact requested version and platform. A regular file matching the current +Release checksum is copied into the new cache, verified again, and atomically +installed under the new cache lock. The old file and its permissions are left +untouched, even on failure. Missing, corrupt, unreadable, or symlinked legacy +entries fall back to the normal verified download. A 0.1.1 binary is never +substituted for 0.1.2. Setting `MEM_MCP_CACHE_DIR` disables this default legacy +binary lookup; its selected cache retains the verification and cleanup rules +above. Before any directory, lock, permission or cleanup change, the installer +resolves the destination and legacy cache paths, including existing ancestors +of directories not yet created. If the trees overlap in either direction +(including namespace, root, version or platform symlink aliases), startup fails +without changing either cache or downloading a manifest. Choose a separate +`MEM_MCP_CACHE_DIR` to recover; do not remove the old cache to resolve the error. +Explicit `MEM_MCP_CACHE_DIR` and programmatic `cacheDir` selections also reject +overlap with the default legacy tree. They disable reuse, not data protection. +Existing destination and same-version legacy entries are also compared by +device and inode: a hardlink is rejected before mutation even when the resolved +paths differ. These checks inspect only the selected paths, without scanning +other cached versions or platforms. + Bootstrap and verification diagnostics use stderr. Stdout is inherited by the verified binary and remains clean for the MCP stdio protocol. This first-run bootstrap works with npm 12 without approving dependency install scripts or @@ -59,7 +81,7 @@ bounded shutdown grace period. "mcpServers": { "mem": { "command": "npx", - "args": ["-y", "@fullstack-ai-infra/mem-mcp"], + "args": ["-y", "@bytefolk/mem-mcp@0.1.2"], "env": { "MEM_SERVER": "http://localhost:8787", "MEM_TOKEN": "mem_..." @@ -75,9 +97,29 @@ bounded shutdown grace period. claude mcp add --scope project --transport stdio \ --env MEM_SERVER=http://localhost:8787 \ --env MEM_TOKEN=mem_... \ - mem -- npx -y @fullstack-ai-infra/mem-mcp + mem -- npx -y @bytefolk/mem-mcp@0.1.2 ``` +## Migrating from 0.1.1 + +The npm package is now `@bytefolk/mem-mcp`; the MCP registry identifier is +`io.github.bytefolk/mem-mcp`. The executable and MCP handshake name remain +`mem-mcp`. Replace the old dependency key in your project's `package.json` +with `"@bytefolk/mem-mcp": "0.1.2"` and run `npm install`, or update your host's +`npx` argument to `@bytefolk/mem-mcp@0.1.2` as shown above. Keep the existing +server, token and workspace settings. + +The wrapper requires the matching `v0.1.2` GitHub Release binaries and checksum +manifest. Source/package metadata alone does not establish that a clean +installation can launch; those assets and the npm publication must be verified +before rolling out the new coordinates. + +Migration does not move stored memories or delete old caches. Keep +`@fullstack-ai-infra/mem-mcp@0.1.1` available until the new package has passed +installation and launch checks. To roll back, restore that exact dependency or +host argument and retain the same connection settings. Do not unpublish the +old package; deprecation follows verified consumer and directory migration. + ## Configuration | Environment variable | Flag equivalent | Default | Description | @@ -93,8 +135,8 @@ claude mcp add --scope project --transport stdio \ - **Protocol**: MCP 2024-11-05 - **Tools**: 26 built-in tools (put, get, search, context, remember, checkpoint, resume, etc.) -See the [full tool list](https://github.com/fullstack-ai-infra/mem/blob/main/docs/mcp.md) for details. +See the [full tool list](https://github.com/bytefolk/mem/blob/main/docs/mcp.md) for details. ## About mem -mem is an open-source, self-hosted memory plane — one core across API, MCP, CLI, and UI. It keeps files, metadata, and embeddings under your control. Learn more at [github.com/fullstack-ai-infra/mem](https://github.com/fullstack-ai-infra/mem). +mem is an open-source, self-hosted memory plane — one core across API, MCP, CLI, and UI. It keeps files, metadata, and embeddings under your control. Learn more at [github.com/bytefolk/mem](https://github.com/bytefolk/mem). diff --git a/npm/clean-tarball.test.js b/npm/clean-tarball.test.js index ae2632a..f763970 100644 --- a/npm/clean-tarball.test.js +++ b/npm/clean-tarball.test.js @@ -134,12 +134,14 @@ test( const packageRoot = join( consumer, "node_modules", - "@fullstack-ai-infra", + "@bytefolk", "mem-mcp", ); const packageJson = JSON.parse( readFileSync(join(packageRoot, "package.json"), "utf8"), ); + assert.equal(packageJson.name, "@bytefolk/mem-mcp"); + assert.equal(packageJson.mcpName, "io.github.bytefolk/mem-mcp"); assert.equal(packageJson.version, PACKAGE_VERSION); assert.equal(packageJson.scripts.postinstall, undefined); diff --git a/npm/install.js b/npm/install.js index f293109..ac8cc03 100644 --- a/npm/install.js +++ b/npm/install.js @@ -13,13 +13,17 @@ const { createHash, randomBytes, timingSafeEqual } = require("crypto"); const { chmodSync, + constants: { COPYFILE_EXCL }, + copyFileSync, createReadStream, createWriteStream, lstatSync, mkdirSync, readFileSync, + realpathSync, renameSync, rmSync, + statSync, unlinkSync, writeFileSync, } = require("fs"); @@ -30,8 +34,8 @@ const { pipeline } = require("stream/promises"); const { TextDecoder } = require("util"); const { assetFor } = require("./platforms"); -const PACKAGE = "@fullstack-ai-infra/mem-mcp"; -const REPO = "fullstack-ai-infra/mem"; +const PACKAGE = "@bytefolk/mem-mcp"; +const REPO = "bytefolk/mem"; const CHECKSUM_ASSET = "mem-mcp-checksums.txt"; const MAX_CHECKSUM_BYTES = 64 * 1024; const MAX_REDIRECTS = 5; @@ -40,6 +44,7 @@ const LOCK_WAIT_TIMEOUT_MS = 120 * 1000; const LOCK_STALE_MS = 10 * 60 * 1000; const LOCK_ORPHAN_GRACE_MS = 5 * 1000; const LOCK_POLL_MS = 50; +const WINDOWS_MISSING_LOCK_GRACE_MS = 250; const guardedResponses = new WeakSet(); // Version from package.json — single source of truth for both release URLs. @@ -82,7 +87,12 @@ function removeOwnedPath(target) { try { const info = lstatSync(target); if (info.isDirectory() && !info.isSymbolicLink()) { - rmSync(target, { recursive: true, force: true }); + rmSync(target, { + recursive: true, + force: true, + maxRetries: 3, + retryDelay: 10, + }); } else { unlinkSync(target); } @@ -105,7 +115,7 @@ function absolutePath(value, osPlatform, label) { return value; } -function cacheRootFor(options = {}) { +function namespacedCacheRootFor(options, namespace) { const osPlatform = options.osPlatform || platform(); const environment = options.environment || process.env; const homeDirectory = options.homeDirectory || homedir(); @@ -120,7 +130,7 @@ function cacheRootFor(options = {}) { if (environment.LOCALAPPDATA) { return pathApi.join( absolutePath(environment.LOCALAPPDATA, osPlatform, "LOCALAPPDATA"), - "fullstack-ai-infra", + namespace, "mem-mcp", ); } @@ -128,7 +138,7 @@ function cacheRootFor(options = {}) { absolutePath(homeDirectory, osPlatform, "home directory"), "AppData", "Local", - "fullstack-ai-infra", + namespace, "mem-mcp", ); } @@ -138,7 +148,7 @@ function cacheRootFor(options = {}) { absolutePath(homeDirectory, osPlatform, "home directory"), "Library", "Caches", - "fullstack-ai-infra", + namespace, "mem-mcp", ); } @@ -146,18 +156,22 @@ function cacheRootFor(options = {}) { if (environment.XDG_CACHE_HOME) { return pathApi.join( absolutePath(environment.XDG_CACHE_HOME, osPlatform, "XDG_CACHE_HOME"), - "fullstack-ai-infra", + namespace, "mem-mcp", ); } return pathApi.join( absolutePath(homeDirectory, osPlatform, "home directory"), ".cache", - "fullstack-ai-infra", + namespace, "mem-mcp", ); } +function cacheRootFor(options = {}) { + return namespacedCacheRootFor(options, "bytefolk"); +} + function safeVersion(version) { if (!/^[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?$/.test(version)) { throw new Error(`Unsafe package version for cache path: ${version}`); @@ -178,6 +192,80 @@ function cacheDirectory(options = {}) { return pathApi.join(root, `v${version}`, `${osPlatform}-${osArch}`); } +// Resolve existing ancestors without creating the missing suffix. In particular, +// a namespace/version symlink must be followed before comparing cache trees. +// Dangling links and unresolvable paths fail closed instead of being treated as +// a fresh directory that recursive mkdir could create in the legacy cache. +function resolvedCachePath(target) { + absolutePath(target, platform(), "mem-mcp cache directory"); + const missing = []; + let current = target; + let concurrentCreateRetries = 0; + for (;;) { + try { + return path.join(realpathSync.native(current), ...missing); + } catch (err) { + if (err.code !== "ENOENT") throw err; + try { + const info = lstatSync(current); + if (info.isSymbolicLink() || concurrentCreateRetries >= 3) { + throw new Error(`Cannot safely resolve mem-mcp cache path: ${current}`); + } + // Another installer created this non-link ancestor after realpath + // returned ENOENT. Retry the same path instead of mistaking that safe + // creation race for a dangling symlink. + concurrentCreateRetries += 1; + continue; + } catch (inspectionError) { + if (inspectionError.code !== "ENOENT") throw inspectionError; + } + const parent = path.dirname(current); + if (parent === current) throw err; + missing.unshift(path.basename(current)); + current = parent; + } + } +} + +function pathContains(parent, child) { + const relative = path.relative(parent, child); + return relative === "" || + (relative !== ".." && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative)); +} + +function assertSeparateLegacyCache(destinationPaths, legacyPaths) { + const destinations = destinationPaths.map(resolvedCachePath); + const legacy = legacyPaths.map(resolvedCachePath); + for (const destination of destinations) { + for (const source of legacy) { + if (pathContains(destination, source) || pathContains(source, destination)) { + throw new Error( + `Refusing mem-mcp destination overlapping legacy cache: ${destination} and ${source}. ` + + "Set MEM_MCP_CACHE_DIR to a separate directory.", + ); + } + } + } +} + +function assertSeparateLegacyEntry(destination, source) { + let destinationInfo; + let sourceInfo; + try { + destinationInfo = statSync(destination, { bigint: true }); + sourceInfo = statSync(source, { bigint: true }); + } catch (err) { + if (err.code === "ENOENT" || err.code === "ENOTDIR") return; + throw err; + } + if (destinationInfo.dev === sourceInfo.dev && destinationInfo.ino === sourceInfo.ino) { + throw new Error( + `Refusing mem-mcp destination sharing an inode with legacy cache: ${destination}. ` + + "Set MEM_MCP_CACHE_DIR to a separate directory.", + ); + } +} + function ensureCacheDirectory(cacheDir) { const hostPlatform = platform(); absolutePath(cacheDir, hostPlatform, "mem-mcp cache directory"); @@ -241,12 +329,26 @@ function ownedArtifactPaths(cacheDir, asset, nonce) { ]; } -function reclaimStaleLock(lockPath, cacheDir, asset, staleMs, orphanGraceMs) { +// Windows reports EPERM/EACCES rather than EEXIST when another process already +// owns the lock directory or is mid-create on it, so either code is ordinary +// contention there and must not be mistaken for a hard permission failure. +function isLockContention(err, currentPlatform = platform()) { + if (err.code === "EEXIST") return true; + return currentPlatform === "win32" && (err.code === "EPERM" || err.code === "EACCES"); +} + +function reclaimStaleLock(lockPath, cacheDir, asset, staleMs, orphanGraceMs, mkdirError) { let info; try { info = lstatSync(lockPath); } catch (err) { - if (err.code === "ENOENT") return true; + // The owner may release the lock between our failed mkdir and inspection. + // Windows can report that mkdir race as EPERM/EACCES rather than EEXIST. + // Return "not observed" so the caller can retry those ambiguous Windows + // results for a short, bounded grace period without hiding a persistent + // permission failure. + if (err.code === "ENOENT" && isLockContention(mkdirError, "win32")) return false; + if (err.code === "ENOENT") throw mkdirError; throw err; } if (info.isSymbolicLink() || !info.isDirectory()) { @@ -261,12 +363,14 @@ function reclaimStaleLock(lockPath, cacheDir, asset, staleMs, orphanGraceMs) { : alive === true ? false : age >= staleMs; - if (!reclaimable) return false; + if (!reclaimable) return true; const quarantine = `${lockPath}.stale.${process.pid}.${randomBytes(12).toString("hex")}`; try { renameSync(lockPath, quarantine); } catch (err) { + // Another contender changed the lock after we inspected it. Retrying is + // safe, but uses the normal poll path so repeated races cannot busy-loop. if (err.code === "ENOENT" || err.code === "EEXIST") return true; throw err; } @@ -285,13 +389,16 @@ async function acquireAssetLock(cacheDir, asset, options = {}) { const staleMs = options.staleMs ?? LOCK_STALE_MS; const orphanGraceMs = options.orphanGraceMs ?? LOCK_ORPHAN_GRACE_MS; const pollMs = options.pollMs ?? LOCK_POLL_MS; + const osPlatform = options.osPlatform || platform(); const signal = options.signal; const lockPath = path.join(cacheDir, `.${asset}.lock`); const deadline = Date.now() + waitTimeoutMs; + let missingWindowsLockSince = null; for (;;) { throwIfAborted(signal); const nonce = randomBytes(12).toString("hex"); + let mkdirError; try { mkdirSync(lockPath, { mode: 0o700 }); try { @@ -306,13 +413,38 @@ async function acquireAssetLock(cacheDir, asset, options = {}) { } return { lockPath, nonce }; } catch (err) { - if (err.code !== "EEXIST") throw err; + if (!isLockContention(err, osPlatform)) throw err; + mkdirError = err; } - if (reclaimStaleLock(lockPath, cacheDir, asset, staleMs, orphanGraceMs)) { - continue; + // All failed acquisitions, including a successfully reclaimed stale lock, + // pass through the same deadline and abort-aware poll. This prevents a + // repeated create/release race from bypassing the wait budget in a tight + // synchronous loop. + const lockObserved = reclaimStaleLock( + lockPath, + cacheDir, + asset, + staleMs, + orphanGraceMs, + mkdirError, + ); + const now = Date.now(); + const ambiguousWindowsRace = osPlatform === "win32" + && mkdirError.code !== "EEXIST" + && lockObserved === false; + if (ambiguousWindowsRace) { + missingWindowsLockSince ??= now; + if ( + now >= deadline + || now - missingWindowsLockSince >= WINDOWS_MISSING_LOCK_GRACE_MS + ) { + throw mkdirError; + } + } else { + missingWindowsLockSince = null; } - if (Date.now() >= deadline) { + if (now >= deadline) { throw new Error(`Timed out waiting for mem-mcp cache lock: ${lockPath}`); } await delay(Math.max(1, pollMs), signal); @@ -586,6 +718,24 @@ function quarantineCacheEntry(binPath, cacheDir, asset, nonce, suffix) { return quarantinePath; } +// Legacy caches are read-only inputs. Stage a separate copy under the new +// cache's lock, then verify that copy too: an old installer may change its +// source while we read. Never execute, chmod, rename or remove the old entry. +async function copyVerifiedLegacyBinary(source, destination, expected, signal) { + try { + await verifyFile(source, expected, signal); + copyFileSync(source, destination, COPYFILE_EXCL); + await verifyFile(destination, expected, signal); + return true; + } catch (err) { + removeIfPresent(destination); + if ((signal && signal.aborted) || err.name === "AbortError") throw err; + // Missing, unreadable, changed, or invalid legacy entries are optional; + // the normal verified Release download remains authoritative. + return false; + } +} + async function install(options = {}) { const osPlatform = options.osPlatform || platform(); const osArch = options.osArch || arch(); @@ -611,6 +761,20 @@ async function install(options = {}) { }; const asset = assetFor(osPlatform, osArch); + // Overrides disable legacy binary reuse, but cannot opt out of protecting the + // default legacy tree from destination writes through aliases or overlap. + // This is a filesystem path on the running host. osPlatform may select a + // foreign binary when a native explicit cacheDir is supplied (including CI). + const legacyRoot = namespacedCacheRootFor({ + osPlatform: platform(), + environment: { ...environment, MEM_MCP_CACHE_DIR: undefined }, + homeDirectory: options.homeDirectory, + }, "fullstack-ai-infra"); + const legacyVersion = path.join(legacyRoot, `v${version}`); + const legacyDirectory = path.join(legacyVersion, `${osPlatform}-${osArch}`); + const legacyPath = options.cacheDir === undefined && environment.MEM_MCP_CACHE_DIR === undefined + ? path.join(legacyDirectory, asset) + : null; const binPath = path.join(cacheDir, asset); const releaseBase = `https://github.com/${repository}/releases/download/v${version}`; const checksumUrl = `${releaseBase}/${CHECKSUM_ASSET}`; @@ -623,8 +787,19 @@ async function install(options = {}) { let operationError = null; throwIfAborted(signal); + // This must stay outside the mutation/cleanup try block and before mkdir, + // chmod or lock acquisition: even those operations can alter legacy data. + assertSeparateLegacyCache( + options.cacheDir !== undefined ? [cacheDir] : [ + cacheRootFor({ osPlatform, environment, homeDirectory: options.homeDirectory }), + path.dirname(cacheDir), cacheDir, + ], + [legacyRoot, legacyVersion, legacyDirectory], + ); + assertSeparateLegacyEntry(binPath, path.join(legacyDirectory, asset)); ensureCacheDirectory(cacheDir); lock = await acquireAssetLock(cacheDir, asset, { + osPlatform, waitTimeoutMs: options.lockWaitTimeoutMs, staleMs: options.lockStaleMs, orphanGraceMs: options.lockOrphanGraceMs, @@ -668,8 +843,14 @@ async function install(options = {}) { } tempPath = path.join(cacheDir, `.${asset}.${lock.nonce}.tmp`); - logger.log(`${PACKAGE}: downloading ${binaryUrl}...`); - await fetchFile(binaryUrl, tempPath, requestOptions); + const copiedLegacy = legacyPath !== null && + await copyVerifiedLegacyBinary(legacyPath, tempPath, expected, signal); + if (copiedLegacy) { + logger.log(`${PACKAGE}: copied verified legacy binary from ${legacyPath}`); + } else { + logger.log(`${PACKAGE}: downloading ${binaryUrl}...`); + await fetchFile(binaryUrl, tempPath, requestOptions); + } throwIfAborted(signal); await verifyFile(tempPath, expected, signal); throwIfAborted(signal); @@ -769,8 +950,10 @@ module.exports = { downloadText, ensureCacheDirectory, install, + isLockContention, openResponse, releaseAssetLock, + REPO, sha256File, verifyFile, }; diff --git a/npm/install.test.js b/npm/install.test.js index c22126d..6d4743b 100644 --- a/npm/install.test.js +++ b/npm/install.test.js @@ -6,6 +6,7 @@ const { createHash } = require("node:crypto"); const { EventEmitter } = require("node:events"); const { chmodSync, + existsSync, mkdirSync, mkdtempSync, readFileSync, @@ -21,12 +22,15 @@ const { PassThrough } = require("node:stream"); const test = require("node:test"); const { assetFor } = require("./platforms"); const { + acquireAssetLock, cacheDirectory, cacheRootFor, checksumForAsset, downloadText, install, + isLockContention, openResponse, + releaseAssetLock, } = require("./install"); const ASSET = assetFor("linux", "x64"); @@ -222,7 +226,7 @@ test("cache paths are user-scoped, versioned, and require absolute overrides", ( environment: { XDG_CACHE_HOME: "/var/cache/example" }, homeDirectory: "/home/example", }), - "/var/cache/example/fullstack-ai-infra/mem-mcp", + "/var/cache/example/bytefolk/mem-mcp", ); assert.equal( cacheRootFor({ @@ -230,7 +234,7 @@ test("cache paths are user-scoped, versioned, and require absolute overrides", ( environment: {}, homeDirectory: "/Users/example", }), - "/Users/example/Library/Caches/fullstack-ai-infra/mem-mcp", + "/Users/example/Library/Caches/bytefolk/mem-mcp", ); assert.equal( cacheRootFor({ @@ -238,17 +242,17 @@ test("cache paths are user-scoped, versioned, and require absolute overrides", ( environment: { LOCALAPPDATA: "C:\\Users\\example\\AppData\\Local" }, homeDirectory: "C:\\Users\\example", }), - "C:\\Users\\example\\AppData\\Local\\fullstack-ai-infra\\mem-mcp", + "C:\\Users\\example\\AppData\\Local\\bytefolk\\mem-mcp", ); assert.equal( cacheDirectory({ osPlatform: "linux", osArch: "arm64", - version: "0.1.1", + version: "0.1.2", environment: { MEM_MCP_CACHE_DIR: "/var/cache/mem-mcp-test" }, homeDirectory: "/home/example", }), - "/var/cache/mem-mcp-test/v0.1.1/linux-arm64", + "/var/cache/mem-mcp-test/v0.1.2/linux-arm64", ); assert.throws( () => cacheRootFor({ @@ -274,15 +278,14 @@ test("install verifies a temporary download before exposing it", async (t) => { const osPlatform = platform(); const osArch = arch(); const asset = assetFor(osPlatform, osArch); - const cacheDir = join(cacheRoot, "v0.1.1", `${osPlatform}-${osArch}`); + const cacheDir = join(cacheRoot, "v0.1.2", `${osPlatform}-${osArch}`); const bytes = Buffer.from("trusted mem-mcp binary"); const requested = []; const installed = await install({ osPlatform, osArch, - version: "0.1.1", - repository: "example/mem", + version: "0.1.2", environment: { MEM_MCP_CACHE_DIR: cacheRoot }, homeDirectory: join(root, "read-only-package-home-must-not-be-used"), logger: QUIET_LOGGER, @@ -303,8 +306,8 @@ test("install verifies a temporary download before exposing it", async (t) => { } assert.deepEqual(readdirSync(cacheDir), [asset]); assert.deepEqual(requested, [ - "https://github.com/example/mem/releases/download/v0.1.1/mem-mcp-checksums.txt", - `https://github.com/example/mem/releases/download/v0.1.1/${asset}`, + "https://github.com/bytefolk/mem/releases/download/v0.1.2/mem-mcp-checksums.txt", + `https://github.com/bytefolk/mem/releases/download/v0.1.2/${asset}`, ]); }); @@ -338,6 +341,28 @@ test("install verifies and reuses a cached binary", async (t) => { } }); +test("explicit cacheDir keeps host paths when selecting a foreign platform binary", async (t) => { + const root = testDirectory(t); + const cacheDir = join(root, "cache"); + const osPlatform = platform() === "win32" ? "linux" : "win32"; + const asset = assetFor(osPlatform, "x64"); + const bytes = Buffer.from("verified foreign platform fixture, never executed"); + const installed = await install({ + osPlatform, + osArch: "x64", + cacheDir, + homeDirectory: join(root, "host-home"), + environment: {}, + logger: QUIET_LOGGER, + downloadText: async () => manifestFor(bytes, asset), + downloadFile: async (_url, destination) => { + writeFileSync(destination, bytes, { flag: "wx", mode: 0o600 }); + }, + }); + assert.equal(installed, join(cacheDir, asset)); + assert.deepEqual(readFileSync(installed), bytes); +}); + test("concurrent installers serialize and publish one verified binary", async (t) => { const root = testDirectory(t); const cacheDir = join(root, "cache"); @@ -763,3 +788,386 @@ test("a logger failure after atomic publish preserves only the verified final", assert.deepEqual(readFileSync(binPath), bytes); assert.deepEqual(readdirSync(cacheDir), [ASSET]); }); + +test("EEXIST is contention everywhere while permission errors are contention only on win32", () => { + const withCode = (code) => Object.assign(new Error(code), { code }); + assert.equal(isLockContention(withCode("EEXIST"), "linux"), true); + assert.equal(isLockContention(withCode("EEXIST"), "darwin"), true); + assert.equal(isLockContention(withCode("EEXIST"), "win32"), true); + assert.equal(isLockContention(withCode("EPERM"), "win32"), true); + assert.equal(isLockContention(withCode("EACCES"), "win32"), true); + assert.equal(isLockContention(withCode("EPERM"), "linux"), false); + assert.equal(isLockContention(withCode("EACCES"), "darwin"), false); + assert.equal(isLockContention(withCode("ENOENT"), "win32"), false); + assert.equal(isLockContention(new Error("no code"), "win32"), false); +}); + +test("a contended Windows lock reported as EPERM waits for a proven lock", async (t) => { + // install.js destructures mkdirSync at load time, so patch it in a child + // before requiring the module. The real lock proves that EPERM represents + // contention rather than an arbitrary permission failure. + const root = testDirectory(t); + const cacheDir = join(root, "cache"); + mkdirSync(cacheDir, { recursive: true }); + await runWorker(t, ` + const fs = require("node:fs"); + const { hostname } = require("node:os"); + const { join } = require("node:path"); + const realMkdirSync = fs.mkdirSync; + const cacheDir = ${JSON.stringify(cacheDir)}; + const lockPath = join(cacheDir, ".${ASSET}.lock"); + realMkdirSync(lockPath, { mode: 0o700 }); + fs.writeFileSync( + join(lockPath, "owner.json"), + JSON.stringify({ pid: process.pid, hostname: hostname(), nonce: "a".repeat(24) }) + "\\n", + ); + let armed = true; + fs.mkdirSync = function (target, ...rest) { + if (armed && String(target).endsWith(".lock")) { + armed = false; + throw Object.assign(new Error("simulated Windows contention"), { + code: "EPERM", + syscall: "mkdir", + }); + } + return realMkdirSync.call(this, target, ...rest); + }; + const { acquireAssetLock, releaseAssetLock } = require(${JSON.stringify(require.resolve("./install"))}); + setTimeout(() => fs.rmSync(lockPath, { recursive: true, force: true }), 25).unref(); + acquireAssetLock(cacheDir, ${JSON.stringify(ASSET)}, { + osPlatform: "win32", + pollMs: 1, + waitTimeoutMs: 5000, + }) + .then((lock) => { + const owner = JSON.parse(fs.readFileSync(join(lock.lockPath, "owner.json"), "utf8")); + if (owner.pid !== process.pid) throw new Error("lock is not owned by this process"); + releaseAssetLock(lock); + }) + .catch((error) => { + console.error(error.stack || String(error)); + process.exitCode = 1; + }); + `); + assert.equal(existsSync(join(cacheDir, `.${ASSET}.lock`)), false); +}); + +test("a Windows EPERM lock released before inspection is retried", async (t) => { + // This is the race observed in the real node24-windows job: mkdir reports + // EPERM for a competing lock, but that owner removes it before lstat. + const root = testDirectory(t); + const cacheDir = join(root, "cache"); + mkdirSync(cacheDir, { recursive: true }); + await runWorker(t, ` + const fs = require("node:fs"); + const realMkdirSync = fs.mkdirSync; + let attempts = 0; + fs.mkdirSync = function (target, ...rest) { + if (String(target).endsWith(".lock") && attempts++ === 0) { + throw Object.assign(new Error("simulated released Windows lock"), { + code: "EPERM", + syscall: "mkdir", + }); + } + return realMkdirSync.call(this, target, ...rest); + }; + const { acquireAssetLock, releaseAssetLock } = require(${JSON.stringify(require.resolve("./install"))}); + acquireAssetLock(${JSON.stringify(cacheDir)}, ${JSON.stringify(ASSET)}, { + osPlatform: "win32", + pollMs: 1, + waitTimeoutMs: 5000, + }) + .then((lock) => { + if (attempts !== 2) throw new Error("expected exactly one retry, saw " + attempts); + releaseAssetLock(lock); + }) + .catch((error) => { + console.error(error.stack || String(error)); + process.exitCode = 1; + }); + `); + assert.equal(existsSync(join(cacheDir, `.${ASSET}.lock`)), false); +}); + +test("an EEXIST lock that disappears before inspection observes the timeout without spinning", async (t) => { + const root = testDirectory(t); + const cacheDir = join(root, "cache"); + mkdirSync(cacheDir, { recursive: true }); + await runWorker(t, ` + const fs = require("node:fs"); + const realMkdirSync = fs.mkdirSync; + const realLstatSync = fs.lstatSync; + const cacheDir = ${JSON.stringify(cacheDir)}; + let attempts = 0; + fs.mkdirSync = function (target, ...rest) { + if (String(target).endsWith(".lock")) { + attempts += 1; + if (attempts === 1) { + throw Object.assign(new Error("simulated lock contention"), { + code: "EEXIST", + syscall: "mkdir", + }); + } + throw Object.assign(new Error("retry spun past its deadline"), { + code: "EPERM", + syscall: "mkdir", + }); + } + return realMkdirSync.call(this, target, ...rest); + }; + fs.lstatSync = function (target, ...rest) { + if (String(target).endsWith(".lock")) { + throw Object.assign(new Error("competing lock released"), { + code: "ENOENT", + syscall: "lstat", + }); + } + return realLstatSync.call(this, target, ...rest); + }; + const { acquireAssetLock } = require(${JSON.stringify(require.resolve("./install"))}); + acquireAssetLock(cacheDir, ${JSON.stringify(ASSET)}, { + osPlatform: "linux", + pollMs: 1, + waitTimeoutMs: 0, + }) + .then(() => { + throw new Error("expected lock acquisition to time out"); + }) + .catch((error) => { + if (!/Timed out waiting for mem-mcp cache lock/.test(String(error.message))) { + throw error; + } + if (attempts !== 1) { + throw new Error("expected one mkdir attempt before timeout, saw " + attempts); + } + }); + `); +}); + +test("a stale-lock rename race observes the timeout without spinning", async (t) => { + const root = testDirectory(t); + const cacheDir = join(root, "cache"); + mkdirSync(cacheDir, { recursive: true }); + await runWorker(t, ` + const fs = require("node:fs"); + const { join } = require("node:path"); + const realMkdirSync = fs.mkdirSync; + const cacheDir = ${JSON.stringify(cacheDir)}; + const lockPath = join(cacheDir, ".${ASSET}.lock"); + realMkdirSync(lockPath, { mode: 0o700 }); + let attempts = 0; + fs.mkdirSync = function (target, ...rest) { + if (String(target).endsWith(".lock")) { + attempts += 1; + if (attempts === 1) { + throw Object.assign(new Error("simulated lock contention"), { + code: "EEXIST", + syscall: "mkdir", + }); + } + throw Object.assign(new Error("retry spun past its deadline"), { + code: "EPERM", + syscall: "mkdir", + }); + } + return realMkdirSync.call(this, target, ...rest); + }; + fs.renameSync = function (source) { + if (String(source).endsWith(".lock")) { + fs.rmSync(lockPath, { recursive: true, force: true }); + throw Object.assign(new Error("competing lock moved first"), { + code: "ENOENT", + syscall: "rename", + }); + } + throw new Error("unexpected rename target"); + }; + const { acquireAssetLock } = require(${JSON.stringify(require.resolve("./install"))}); + acquireAssetLock(cacheDir, ${JSON.stringify(ASSET)}, { + osPlatform: "linux", + pollMs: 1, + waitTimeoutMs: 0, + staleMs: 0, + orphanGraceMs: 0, + }) + .then(() => { + throw new Error("expected lock acquisition to time out"); + }) + .catch((error) => { + if (!/Timed out waiting for mem-mcp cache lock/.test(String(error.message))) { + throw error; + } + if (attempts !== 1) { + throw new Error("expected one mkdir attempt before timeout, saw " + attempts); + } + }); + `); +}); + +test("a persistent Windows EPERM without a lock fails after a bounded grace", async (t) => { + const root = testDirectory(t); + const cacheDir = join(root, "cache"); + mkdirSync(cacheDir, { recursive: true }); + await runWorker(t, ` + const fs = require("node:fs"); + const realMkdirSync = fs.mkdirSync; + fs.mkdirSync = function (target, ...rest) { + if (String(target).endsWith(".lock")) { + throw Object.assign(new Error("simulated Windows permission failure"), { + code: "EPERM", + syscall: "mkdir", + }); + } + return realMkdirSync.call(this, target, ...rest); + }; + const { acquireAssetLock } = require(${JSON.stringify(require.resolve("./install"))}); + const startedAt = Date.now(); + acquireAssetLock(${JSON.stringify(cacheDir)}, ${JSON.stringify(ASSET)}, { + osPlatform: "win32", + pollMs: 1, + waitTimeoutMs: 10000, + }) + .then(() => { + throw new Error("expected a permission failure"); + }) + .catch((error) => { + if (error.code !== "EPERM") throw error; + const elapsed = Date.now() - startedAt; + if (elapsed < 200 || elapsed >= 1000) { + throw new Error("permission ambiguity grace was not bounded: " + elapsed + "ms"); + } + }); + `); +}); + +test("a Windows lock inspection permission error fails promptly", async (t) => { + const root = testDirectory(t); + const cacheDir = join(root, "cache"); + mkdirSync(cacheDir, { recursive: true }); + await runWorker(t, ` + const fs = require("node:fs"); + const realMkdirSync = fs.mkdirSync; + const realLstatSync = fs.lstatSync; + fs.mkdirSync = function (target, ...rest) { + if (String(target).endsWith(".lock")) { + throw Object.assign(new Error("simulated Windows contention"), { + code: "EPERM", + syscall: "mkdir", + }); + } + return realMkdirSync.call(this, target, ...rest); + }; + fs.lstatSync = function (target, ...rest) { + if (String(target).endsWith(".lock")) { + throw Object.assign(new Error("simulated inspection permission failure"), { + code: "EACCES", + syscall: "lstat", + }); + } + return realLstatSync.call(this, target, ...rest); + }; + const { acquireAssetLock } = require(${JSON.stringify(require.resolve("./install"))}); + const startedAt = Date.now(); + acquireAssetLock(${JSON.stringify(cacheDir)}, ${JSON.stringify(ASSET)}, { + osPlatform: "win32", + pollMs: 1, + waitTimeoutMs: 10000, + }) + .then(() => { + throw new Error("expected an inspection failure"); + }) + .catch((error) => { + if (error.code !== "EACCES") throw error; + if (Date.now() - startedAt >= 1000) throw new Error("inspection error entered the retry loop"); + }); + `); +}); + +test("lock release retries a transient non-empty directory race", async (t) => { + const root = testDirectory(t); + const cacheDir = join(root, "cache"); + mkdirSync(cacheDir, { recursive: true }); + await runWorker(t, ` + const fs = require("node:fs"); + const realRmSync = fs.rmSync; + fs.rmSync = function (target, options) { + if ( + String(target).endsWith(".lock") + && ( + !options + || (options.maxRetries ?? 0) < 3 + || (options.retryDelay ?? 0) < 10 + ) + ) { + throw Object.assign(new Error("simulated transient non-empty lock"), { + code: "ENOTEMPTY", + syscall: "rmdir", + }); + } + return realRmSync.call(this, target, options); + }; + const { acquireAssetLock, releaseAssetLock } = require(${JSON.stringify(require.resolve("./install"))}); + acquireAssetLock(${JSON.stringify(cacheDir)}, ${JSON.stringify(ASSET)}) + .then((lock) => releaseAssetLock(lock)) + .catch((error) => { + console.error(error.stack || String(error)); + process.exitCode = 1; + }); + `); + assert.equal(existsSync(join(cacheDir, `.${ASSET}.lock`)), false); +}); + +test("cache resolution retries a directory created between realpath and lstat", async (t) => { + const root = testDirectory(t); + const cacheDir = join(root, "cache"); + await runWorker(t, ` + const crypto = require("node:crypto"); + const fs = require("node:fs"); + const realMkdirSync = fs.mkdirSync; + const realRealpathNative = fs.realpathSync.native; + const cacheDir = ${JSON.stringify(cacheDir)}; + let armed = true; + fs.realpathSync.native = function (target, ...rest) { + if (armed && target === cacheDir) { + armed = false; + realMkdirSync(cacheDir, { recursive: true }); + throw Object.assign(new Error("simulated concurrent cache creation"), { + code: "ENOENT", + syscall: "realpath", + }); + } + return realRealpathNative.call(this, target, ...rest); + }; + const { install } = require(${JSON.stringify(require.resolve("./install"))}); + const bytes = Buffer.from("verified concurrent cache fixture"); + const digest = crypto.createHash("sha256").update(bytes).digest("hex"); + install({ + osPlatform: "linux", + osArch: "x64", + cacheDir, + logger: { log() {}, warn() {} }, + downloadText: async () => digest + " ${ASSET}\\n", + downloadFile: async (_url, destination) => { + fs.writeFileSync(destination, bytes, { flag: "wx", mode: 0o600 }); + }, + }).catch((error) => { + console.error(error.stack || String(error)); + process.exitCode = 1; + }); + `); + assert.deepEqual(readFileSync(join(cacheDir, ASSET)), Buffer.from("verified concurrent cache fixture")); +}); + +test("a non-contention error propagates immediately instead of entering the wait loop", async (t) => { + const root = testDirectory(t); + const missing = join(root, "no-such-parent", "cache"); + const startedAt = Date.now(); + await assert.rejects( + acquireAssetLock(missing, ASSET, { osPlatform: "linux", pollMs: 1, waitTimeoutMs: 10000 }), + (error) => error.code === "ENOENT", + ); + assert.ok( + Date.now() - startedAt < 5000, + "expected an immediate rejection, not a wait", + ); +}); diff --git a/npm/mcp-registry.server.json b/npm/mcp-registry.server.json new file mode 100644 index 0000000..e1f5de5 --- /dev/null +++ b/npm/mcp-registry.server.json @@ -0,0 +1,20 @@ +{ + "$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json", + "name": "io.github.bytefolk/mem-mcp", + "description": "MCP server for mem — a portable, self-hosted memory plane for AI agents", + "repository": { + "url": "https://github.com/bytefolk/mem", + "source": "github" + }, + "version": "0.1.2", + "packages": [ + { + "registryType": "npm", + "identifier": "@bytefolk/mem-mcp", + "version": "0.1.2", + "transport": { + "type": "stdio" + } + } + ] +} diff --git a/npm/mem-mcp b/npm/mem-mcp index bb6bb3c..9850dd4 100644 --- a/npm/mem-mcp +++ b/npm/mem-mcp @@ -104,7 +104,7 @@ async function runProcess(options = {}) { `mem-mcp: failed to download and verify the platform binary: ${err.message}`, ); logger.error( - `mem-mcp: retry the command, or build manually from https://github.com/fullstack-ai-infra/mem`, + `mem-mcp: retry the command, or build manually from https://github.com/bytefolk/mem`, ); return { code: 1, signal: null }; } finally { diff --git a/npm/migration.test.js b/npm/migration.test.js new file mode 100644 index 0000000..6d41e80 --- /dev/null +++ b/npm/migration.test.js @@ -0,0 +1,291 @@ +"use strict"; + +const assert = require("node:assert/strict"); +const { createHash } = require("node:crypto"); +const { EventEmitter } = require("node:events"); +const { + closeSync, existsSync, fstatSync, linkSync, lstatSync, mkdirSync, mkdtempSync, + openSync, readFileSync, readdirSync, readlinkSync, rmSync, statSync, symlinkSync, + writeFileSync, +} = require("node:fs"); +const { arch, platform, tmpdir } = require("node:os"); +const { dirname, join } = require("node:path"); +const { PassThrough } = require("node:stream"); +const test = require("node:test"); +const { cacheRootFor, install, openResponse } = require("./install"); +const { assetFor } = require("./platforms"); +const pkg = require("./package.json"); +const server = require("./server.json"); + +test("the npm package and MCP metadata identify the ByteFolk 0.1.2 release", () => { + assert.equal(pkg.name, "@bytefolk/mem-mcp"); + assert.equal(pkg.version, "0.1.2"); + assert.equal(pkg.mcpName, "io.github.bytefolk/mem-mcp"); + assert.equal(server.mcpName, pkg.mcpName); + assert.equal(server.version, pkg.version); + assert.equal(server.name, "mem-mcp"); + assert.equal(server.command, "mem-mcp"); + assert.equal(pkg.repository.url, `git+${server.repo}.git`); + assert.deepEqual(pkg.bin, { "mem-mcp": "./mem-mcp" }); + assert.equal(pkg.scripts.postinstall, undefined); + assert.throws(() => assetFor("freebsd", "x64"), /@bytefolk\/mem-mcp/); +}); + +test("installer HTTPS requests identify the renamed package", async () => { + const response = await openResponse("https://github.com/bytefolk/mem", 0, + (_url, options, callback) => { + assert.equal(options.headers["User-Agent"], "@bytefolk/mem-mcp/0.1.2"); + const request = new EventEmitter(); + queueMicrotask(() => { + const incoming = new PassThrough(); + incoming.statusCode = 200; + incoming.headers = {}; + callback(incoming); + incoming.end("fixture"); + }); + return request; + }); + for await (const _chunk of response) { /* consume response and close timeout */ } +}); + +test("default cache roots use ByteFolk on every supported OS", () => { + for (const [osPlatform, environment, homeDirectory, expected] of [ + ["linux", {}, "/home/example", "/home/example/.cache/bytefolk/mem-mcp"], + ["linux", { XDG_CACHE_HOME: "/cache" }, "/home/example", "/cache/bytefolk/mem-mcp"], + ["darwin", {}, "/Users/example", "/Users/example/Library/Caches/bytefolk/mem-mcp"], + ["win32", {}, "C:\\Users\\example", "C:\\Users\\example\\AppData\\Local\\bytefolk\\mem-mcp"], + ["win32", { LOCALAPPDATA: "C:\\Cache" }, "C:\\Users\\example", "C:\\Cache\\bytefolk\\mem-mcp"], + ]) { + assert.equal(cacheRootFor({ osPlatform, environment, homeDirectory }), expected); + } +}); + +function fixture(t) { + const root = mkdtempSync(join(tmpdir(), "mem-mcp-migration-test-")); + t.after(() => rmSync(root, { recursive: true, force: true })); + const base = platform() === "darwin" ? join(root, "Library", "Caches") + : platform() === "win32" ? join(root, "AppData", "Local") : join(root, ".cache"); + const suffix = join("v0.1.2", `${platform()}-${arch()}`); + const asset = assetFor(platform(), arch()); + const legacyRoot = join(base, "fullstack-ai-infra", "mem-mcp"); + const legacyDir = join(legacyRoot, suffix); + const legacy = join(legacyDir, asset); + const destination = join(base, "bytefolk", "mem-mcp", suffix, asset); + const bytes = Buffer.from("verified release 0.1.2 fixture"); + const digest = createHash("sha256").update(bytes).digest("hex"); + const requests = []; + mkdirSync(legacyDir, { recursive: true }); + const options = { + environment: {}, homeDirectory: root, version: "0.1.2", + logger: { log() {}, warn() {} }, + downloadText: async (url) => { + requests.push(url); + return `${digest} ${asset}\n`; + }, + downloadFile: async (url, target) => { + requests.push(url); + writeFileSync(target, bytes, { flag: "wx" }); + }, + }; + return { root, asset, legacyRoot, legacyDir, legacy, destination, bytes, requests, options }; +} + +function snapshotFile(path) { + const fd = openSync(path, "r"); + try { + return { info: fstatSync(fd), bytes: readFileSync(fd) }; + } finally { + closeSync(fd); + } +} + +function snapshotTree(root) { + const info = lstatSync(root); + if (info.isSymbolicLink()) return { mode: info.mode, link: readlinkSync(root) }; + if (info.isDirectory()) { + return { mode: info.mode, entries: Object.fromEntries( + readdirSync(root).sort().map((name) => [name, snapshotTree(join(root, name))]), + ) }; + } + const file = snapshotFile(root); + return { mode: file.info.mode, bytes: file.bytes.toString("hex") }; +} + +function directoryAlias(target, link) { + mkdirSync(dirname(link), { recursive: true }); + symlinkSync(target, link, platform() === "win32" ? "junction" : "dir"); +} + +for (const level of ["namespace", "root", "version", "platform", "ancestor"]) { + for (const entry of ["readonly-file", "failed-manifest-file", "failed-manifest-directory"]) { + test(`legacy alias at ${level} preserves ${entry} before any cache mutation`, async (t) => { + const f = fixture(t); + if (entry === "failed-manifest-directory") { + mkdirSync(f.legacy); + writeFileSync(join(f.legacy, "user-data"), "must survive"); + } else { + writeFileSync(f.legacy, f.bytes, { mode: 0o400 }); + } + const newRoot = dirname(dirname(dirname(f.destination))); + const hops = { namespace: 3, root: 2, version: 1, platform: 0 }; + if (level === "ancestor") { + // The writable root's ancestor resolves inside the protected old root; + // the remaining destination suffix does not exist yet. + directoryAlias(f.legacyRoot, dirname(newRoot)); + } else { + let target = f.legacyDir; + let link = dirname(f.destination); + for (let i = 0; i < hops[level]; i++) { + target = dirname(target); + link = dirname(link); + } + directoryAlias(target, link); + } + const before = snapshotTree(f.root); + let requests = 0; + const downloadText = f.options.downloadText; + f.options.downloadText = async (...args) => { + requests++; + if (entry !== "readonly-file") throw new Error("fixture manifest failure"); + return downloadText(...args); + }; + const result = await install(f.options).catch((error) => error); + assert.deepEqual(snapshotTree(f.root), before, "legacy bytes, modes, and all directory entries must survive"); + assert.equal(requests, 0, "reject aliasing before downloading or creating a cache lock"); + assert.ok(result instanceof Error, "aliased caches must fail closed"); + assert.match(result.message, /legacy cache/); + }); + } +} + +for (const override of ["environment", "cacheDir"]) { + test(`explicit ${override} rejects overlap with the default legacy cache`, async (t) => { + const f = fixture(t); + writeFileSync(f.legacy, f.bytes, { mode: 0o400 }); + const alias = join(f.root, "selected-cache"); + directoryAlias(override === "environment" ? f.legacyRoot : f.legacyDir, alias); + if (override === "environment") f.options.environment.MEM_MCP_CACHE_DIR = alias; + else f.options.cacheDir = alias; + const before = snapshotTree(f.root); + const result = await install(f.options).catch((error) => error); + assert.deepEqual(snapshotTree(f.root), before); + assert.equal(f.requests.length, 0); + assert.ok(result instanceof Error); + assert.match(result.message, /legacy cache/); + }); +} + +test("a legacy version alias into the new tree is rejected before creating the destination", async (t) => { + const f = fixture(t); + // Use a second version so no pre-existing fixture directory is removed. + f.options.version = "0.1.3"; + const newRoot = dirname(dirname(dirname(f.destination))); + mkdirSync(join(newRoot, "v0.1.3"), { recursive: true }); + directoryAlias(join(newRoot, "v0.1.3"), join(f.legacyRoot, "v0.1.3")); + const before = snapshotTree(f.root); + await assert.rejects(install(f.options), /legacy cache/); + assert.deepEqual(snapshotTree(f.root), before); + assert.equal(f.requests.length, 0); +}); + +for (const failure of [false, true]) { + test(`hardlinked destination preserves legacy readonly mode with manifest failure=${failure}`, async (t) => { + const f = fixture(t); + writeFileSync(f.legacy, f.bytes, { mode: 0o400 }); + mkdirSync(dirname(f.destination), { recursive: true }); + linkSync(f.legacy, f.destination); + const original = statSync(f.legacy); + assert.equal(statSync(f.destination).ino, original.ino); + const before = snapshotTree(f.root); + let requests = 0; + const downloadText = f.options.downloadText; + f.options.downloadText = async (...args) => { + requests++; + if (failure) throw new Error("fixture manifest failure"); + return downloadText(...args); + }; + const result = await install(f.options).catch((error) => error); + assert.deepEqual(snapshotTree(f.root), before); + assert.equal(statSync(f.legacy).nlink, original.nlink); + assert.equal(requests, 0); + assert.ok(result instanceof Error); + assert.match(result.message, /legacy cache/); + }); +} + +test("concurrent migration copies a verified legacy cache without changing its bytes or mode", async (t) => { + const f = fixture(t); + writeFileSync(f.legacy, f.bytes, { mode: 0o400 }); + writeFileSync(join(f.legacyDir, "user-data"), "keep me"); + const before = snapshotFile(f.legacy); + assert.deepEqual(await Promise.all([install(f.options), install(f.options)]), + [f.destination, f.destination]); + assert.deepEqual(readFileSync(f.destination), f.bytes); + const after = snapshotFile(f.legacy); + assert.deepEqual(after.bytes, f.bytes); + assert.deepEqual(after.bytes, before.bytes); + assert.equal(after.info.mode, before.info.mode); + assert.equal(after.info.mtimeMs, before.info.mtimeMs); + assert.equal(readFileSync(join(f.legacyDir, "user-data"), "utf8"), "keep me"); + assert.deepEqual(readdirSync(f.legacyDir).sort(), [f.asset, "user-data"].sort()); + assert.deepEqual(f.requests, Array(2).fill( + "https://github.com/bytefolk/mem/releases/download/v0.1.2/mem-mcp-checksums.txt")); +}); + +for (const kind of ["corrupt", "directory", "symlink", "old-version", "other-platform"]) { + test(`migration ignores ${kind} legacy entries and preserves them`, + { skip: kind === "symlink" && platform() === "win32" }, async (t) => { + const f = fixture(t); + let preserved = f.legacy; + if (kind === "directory") { + mkdirSync(f.legacy); + preserved = join(f.legacy, "user-data"); + } else if (kind === "symlink") { + preserved = join(f.root, "symlink-target"); + symlinkSync(preserved, f.legacy); + } else if (kind === "old-version" || kind === "other-platform") { + const directory = join(f.legacyRoot, + kind === "old-version" ? "v0.1.1" : "v0.1.2", + kind === "other-platform" ? "unsupported-arch" : `${platform()}-${arch()}`); + mkdirSync(directory, { recursive: true }); + preserved = join(directory, f.asset); + } + const original = kind === "corrupt" ? Buffer.from("untrusted bytes") : f.bytes; + writeFileSync(preserved, original); + assert.equal(await install(f.options), f.destination); + assert.deepEqual(readFileSync(f.destination), f.bytes); + assert.deepEqual(readFileSync(preserved), original); + if (kind === "symlink") assert.ok(lstatSync(f.legacy).isSymbolicLink()); + assert.equal(f.requests.length, 2); + assert.equal(f.requests[1], `https://github.com/bytefolk/mem/releases/download/v0.1.2/${f.asset}`); + }); +} + +for (const failure of ["manifest", "download"]) { + test(`${failure} failure never deletes a legacy entry`, async (t) => { + const f = fixture(t); + writeFileSync(f.legacy, "legacy bytes to preserve"); + f.options[failure === "manifest" ? "downloadText" : "downloadFile"] = async () => { + throw new Error("fixture failure"); + }; + await assert.rejects(install(f.options), /fixture failure/); + assert.equal(readFileSync(f.legacy, "utf8"), "legacy bytes to preserve"); + assert.equal(existsSync(f.destination), false); + assert.deepEqual(readdirSync(f.legacyDir), [f.asset]); + }); +} + +for (const override of ["environment", "cacheDir"]) { + test(`explicit ${override} cache selection disables default legacy lookup`, async (t) => { + const f = fixture(t); + writeFileSync(f.legacy, f.bytes); + const custom = join(f.root, "custom"); + if (override === "environment") f.options.environment.MEM_MCP_CACHE_DIR = custom; + else f.options.cacheDir = custom; + const result = await install(f.options); + assert.ok(result.startsWith(custom)); + assert.equal(f.requests.length, 2); + assert.deepEqual(readFileSync(f.legacy), f.bytes); + assert.equal(existsSync(f.destination), false); + }); +} diff --git a/npm/package.json b/npm/package.json index 4afd5ad..8ea8b93 100644 --- a/npm/package.json +++ b/npm/package.json @@ -1,8 +1,8 @@ { - "name": "@fullstack-ai-infra/mem-mcp", - "version": "0.1.1", + "name": "@bytefolk/mem-mcp", + "version": "0.1.2", "description": "MCP server for mem — a portable, self-hosted memory plane for AI agents", - "mcpName": "io.github.fullstack-ai-infra/mem-mcp", + "mcpName": "io.github.bytefolk/mem-mcp", "keywords": [ "mcp", "model-context-protocol", @@ -13,13 +13,13 @@ "mcp-server" ], "license": "Apache-2.0", - "homepage": "https://github.com/fullstack-ai-infra/mem", + "homepage": "https://github.com/bytefolk/mem", "repository": { "type": "git", - "url": "git+https://github.com/fullstack-ai-infra/mem.git" + "url": "git+https://github.com/bytefolk/mem.git" }, "bugs": { - "url": "https://github.com/fullstack-ai-infra/mem/issues" + "url": "https://github.com/bytefolk/mem/issues" }, "bin": { "mem-mcp": "./mem-mcp" @@ -27,7 +27,7 @@ "os": ["linux", "darwin", "win32"], "cpu": ["x64", "arm64"], "scripts": { - "test": "node --test install.test.js mem-mcp.test.js windows-shim.test.js", + "test": "node --test install.test.js mem-mcp.test.js migration.test.js registry-identity.test.js windows-shim.test.js", "test:tarball": "node --test clean-tarball.test.js" }, "files": [ diff --git a/npm/platforms.js b/npm/platforms.js index 2dcb510..df049b2 100644 --- a/npm/platforms.js +++ b/npm/platforms.js @@ -35,7 +35,7 @@ function assetFor(osPlatform, osArch) { if (!asset) { throw new Error( `Unsupported platform: ${key}. ${ - "@fullstack-ai-infra/mem-mcp" + "@bytefolk/mem-mcp" } publishes: ${Object.keys(ASSETS).join(", ")}.` ); } diff --git a/npm/registry-identity.test.js b/npm/registry-identity.test.js new file mode 100644 index 0000000..eb0b253 --- /dev/null +++ b/npm/registry-identity.test.js @@ -0,0 +1,59 @@ +"use strict"; + +const assert = require("node:assert/strict"); +const { readFileSync } = require("node:fs"); +const { join } = require("node:path"); +const test = require("node:test"); +const { REPO } = require("./install"); + +const serverManifest = JSON.parse( + readFileSync(join(__dirname, "server.json"), "utf8"), +); +const packageManifest = JSON.parse( + readFileSync(join(__dirname, "package.json"), "utf8"), +); + +// The registry derives its namespace from the repository owner, so every rename +// leaves the published identifier pointing at an organization that no longer +// exists, and the submission is rejected with no hint that a stale string in a +// manifest caused it. The repository coordinate is therefore asserted here +// against the identifier the submission carries, rather than repeated in a +// third file. +test("the registry namespace follows the repository owner", () => { + const [owner] = REPO.split("/"); + assert.match(REPO, /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\/[a-z0-9._-]+$/i); + for (const [label, manifest] of [ + ["server.json", serverManifest], + ["package.json", packageManifest], + ]) { + assert.equal( + manifest.mcpName, + `io.github.${owner}/${serverManifest.name}`, + `${label} mcpName must carry the owner of the repository the installer downloads from (${REPO})`, + ); + } +}); + +// Both manifests are published and read by different tools, so a disagreement +// between them decides which one the validator saw rather than which one is right. +test("both manifests name the same server", () => { + assert.equal(serverManifest.mcpName, packageManifest.mcpName); + assert.equal(serverManifest.version, packageManifest.version); +}); + +// A registry identifier is a primary key. After the ByteFolk cutover the npm +// scope matches the GitHub owner; the trailing mcpName segment must still be +// the unscoped package name. +test("the registry name is the unscoped package name", () => { + const [, packageName] = packageManifest.name.split("/"); + assert.equal( + serverManifest.mcpName.split("/").pop(), + packageName, + "the trailing segment of mcpName must be the published package name without its scope", + ); + assert.equal( + serverManifest.name, + packageName, + "server.json name must match the published package name without its scope", + ); +}); diff --git a/npm/server.json b/npm/server.json index 0c9acef..f924368 100644 --- a/npm/server.json +++ b/npm/server.json @@ -1,7 +1,7 @@ { - "mcpName": "io.github.fullstack-ai-infra/mem-mcp", + "mcpName": "io.github.bytefolk/mem-mcp", "name": "mem-mcp", - "version": "0.1.1", + "version": "0.1.2", "description": "MCP server for mem — a portable, self-hosted memory plane for AI agents", "protocol": "2024-11-05", "transport": "stdio", @@ -48,6 +48,6 @@ "mem_durable_context_recall" ], "categories": ["memory", "knowledge-management", "ai-agents"], - "repo": "https://github.com/fullstack-ai-infra/mem", + "repo": "https://github.com/bytefolk/mem", "license": "Apache-2.0" } diff --git a/npm/windows-shim.test.js b/npm/windows-shim.test.js index db4163a..2036736 100644 --- a/npm/windows-shim.test.js +++ b/npm/windows-shim.test.js @@ -150,7 +150,7 @@ https.get = (url, _options, callback) => { assert.match(invocation.stderr, /verified existing binary/); assert.deepEqual( readFileSync(requestLog, "utf8").trim().split("\n"), - [`https://github.com/fullstack-ai-infra/mem/releases/download/v${PACKAGE_VERSION}/mem-mcp-checksums.txt`], + [`https://github.com/bytefolk/mem/releases/download/v${PACKAGE_VERSION}/mem-mcp-checksums.txt`], ); } finally { rmSync(root, { recursive: true, force: true }); diff --git a/scripts/generate_release_checksums.sh b/scripts/generate_release_checksums.sh index 4fb27fd..d0a26b0 100755 --- a/scripts/generate_release_checksums.sh +++ b/scripts/generate_release_checksums.sh @@ -4,20 +4,31 @@ set -euo pipefail tag="${1:-}" commit="${2:-}" asset_dir="${3:-}" -output="${asset_dir}/mem-mcp-checksums.txt" +mcp_output="${asset_dir}/mem-mcp-checksums.txt" +server_output="${asset_dir}/mem-checksums.txt" die() { printf 'ERROR: %s\n' "$*" >&2 exit 1 } +require_absent_output() { + # -e alone misses dangling symlinks. In particular, mv follows an output + # symlink to a directory and would publish outside this asset directory. + local output="$1" + [[ ! -e "${output}" && ! -L "${output}" ]] || + die "checksum output path already exists: ${output}" +} + if [[ ! "${tag}" =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-rc\.(0|[1-9][0-9]*))?$ ]]; then die "invalid release tag: ${tag:-}" fi [[ "${commit}" =~ ^[0-9a-f]{40}$ ]] || die "invalid release commit: ${commit:-}" [[ -d "${asset_dir}" ]] || die "asset directory does not exist: ${asset_dir:-}" +require_absent_output "${mcp_output}" +require_absent_output "${server_output}" -assets=( +mcp_assets=( mem-mcp-darwin-amd64 mem-mcp-darwin-arm64 mem-mcp-linux-amd64 @@ -26,43 +37,94 @@ assets=( mem-mcp-windows-arm64.exe ) +server_assets=( + memd-darwin-amd64 + memd-darwin-arm64 + memd-linux-amd64 + memd-linux-arm64 + mem-migrate-darwin-amd64 + mem-migrate-darwin-arm64 + mem-migrate-linux-amd64 + mem-migrate-linux-arm64 + mem-healthcheck-darwin-amd64 + mem-healthcheck-darwin-arm64 + mem-healthcheck-linux-amd64 + mem-healthcheck-linux-arm64 + mem-darwin-amd64 + mem-darwin-arm64 + mem-linux-amd64 + mem-linux-arm64 +) + +all_assets=() +for a in "${mcp_assets[@]}" "${server_assets[@]}"; do + all_assets[${#all_assets[@]}]="${a}" +done + +# A process substitution hides find's exit status from the loop below, so a +# tool failure would surface as the misleading "differ from the exact expected +# set". Capture the listing first and report a failed listing as what it is. +asset_listing="$( + find "${asset_dir}" -mindepth 1 -maxdepth 1 -type f -exec basename {} \; | LC_ALL=C sort +)" || die "cannot list release assets in ${asset_dir}" + actual_assets=() while IFS= read -r actual_asset; do + [[ -n "${actual_asset}" ]] || continue actual_assets[${#actual_assets[@]}]="${actual_asset}" -done < <( - find "${asset_dir}" -mindepth 1 -maxdepth 1 -type f -printf '%f\n' | LC_ALL=C sort -) -if [[ "${actual_assets[*]}" != "${assets[*]}" ]]; then +done <<< "${asset_listing}" + +expected_sorted="$(printf '%s\n' "${all_assets[@]}" | LC_ALL=C sort)" +actual_sorted="$(printf '%s\n' "${actual_assets[@]}" | LC_ALL=C sort)" +if [[ "${#actual_assets[@]}" -eq 0 ]] || [[ "${actual_sorted}" != "${expected_sorted}" ]]; then printf 'ERROR: release assets differ from the exact expected set\n' >&2 - printf 'expected: %s\n' "${assets[*]}" >&2 + printf 'expected:\n%s\n' "${expected_sorted}" >&2 printf 'actual: %s\n' "${actual_assets[*]:-}" >&2 exit 1 fi -for asset in "${assets[@]}"; do +for asset in "${all_assets[@]}"; do [[ -f "${asset_dir}/${asset}" && ! -L "${asset_dir}/${asset}" ]] || die "release asset is not a regular file: ${asset}" [[ -s "${asset_dir}/${asset}" ]] || die "release asset is empty: ${asset}" done -tmp_output="$(mktemp "${asset_dir}/.mem-mcp-checksums.XXXXXX")" -cleanup() { - rm -f -- "${tmp_output}" +generate_checksums() { + local output="$1" + shift + local assets=("$@") + local tmp_output + tmp_output="$(mktemp "${asset_dir}/.$(basename -- "${output}").XXXXXX")" + cleanup_tmp() { + rm -f -- "${tmp_output}" + } + trap cleanup_tmp EXIT + { + ( + cd -- "${asset_dir}" + sha256sum "${assets[@]}" + ) + } > "${tmp_output}" + # The post-publish self-check below uses --ignore-missing, which by definition + # tolerates a listed file being absent, so completeness is asserted here while + # the staging file and the expected set are both known. + local manifest_rows + manifest_rows="$(grep -c '' "${tmp_output}")" + [[ "${manifest_rows}" -eq "${#assets[@]}" ]] || + die "checksum manifest must have ${#assets[@]} rows, got ${manifest_rows}" + require_absent_output "${output}" + mv -- "${tmp_output}" "${output}" + trap - EXIT } -trap cleanup EXIT -{ - ( - cd -- "${asset_dir}" - sha256sum "${assets[@]}" - ) -} > "${tmp_output}" -mv -- "${tmp_output}" "${output}" -trap - EXIT +generate_checksums "${mcp_output}" "${mcp_assets[@]}" +generate_checksums "${server_output}" "${server_assets[@]}" ( cd -- "${asset_dir}" - sha256sum --check --strict --ignore-missing "$(basename -- "${output}")" + sha256sum --check --strict --ignore-missing "$(basename -- "${mcp_output}")" + sha256sum --check --strict --ignore-missing "$(basename -- "${server_output}")" ) -printf 'PASS: checksums bind six release assets to %s at %s\n' "${tag}" "${commit}" +printf 'PASS: checksums bind %d release assets to %s at %s\n' \ + "${#all_assets[@]}" "${tag}" "${commit}" diff --git a/scripts/npm-release.mjs b/scripts/npm-release.mjs new file mode 100644 index 0000000..c0bec8d --- /dev/null +++ b/scripts/npm-release.mjs @@ -0,0 +1,303 @@ +// Repository-owned stable npm release gate. Importing this module has no effects. +// Test adapters never invoke a real publisher; the CLI requires hosted OIDC and +// a fresh, exact-release human attestation from the protected npm-release environment. +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { closeSync, constants, existsSync, fstatSync, mkdirSync, openSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +export const PACKAGE = '@bytefolk/mem-mcp'; +export const REGISTRY = 'https://registry.npmjs.org'; +export const ASSETS = [ + 'mem-mcp-darwin-amd64', 'mem-mcp-darwin-arm64', + 'mem-mcp-linux-amd64', 'mem-mcp-linux-arm64', + 'mem-mcp-windows-amd64.exe', 'mem-mcp-windows-arm64.exe', +]; +const FILES = ['LICENSE', 'README.md', 'install.js', 'mem-mcp', 'package.json', 'platforms.js']; +const MANIFEST = 'mem-mcp-checksums.txt'; +const stable = /^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$/; +const canonicalRepository = 'git+https://github.com/bytefolk/mem.git'; +const readJSON = path => JSON.parse(readFileSync(path, 'utf8')); +const sha = (algorithm, value) => createHash(algorithm).update(value).digest(algorithm === 'sha512' ? 'base64' : 'hex'); +const requireValue = (condition, message) => assert.ok(condition, message); +// Downloads change counters; compare the publication identity and asset bytes, +// not incidental API statistics, when checking for a race before publishing. +const releaseIdentity = release => ({ + id: release.id, tag: release.tag_name, draft: release.draft, + prerelease: release.prerelease, publishedAt: release.published_at, + assets: release.assets.map(({ id, name, size, state, digest, updated_at }) => + ({ id, name, size, state, digest, updated_at })).sort((a, b) => a.name.localeCompare(b.name)), +}); + +export const commandStdio = descriptor => descriptor === undefined + ? ['ignore', 'pipe', 'pipe'] : ['ignore', 'pipe', 'pipe', descriptor]; + +export function checkContext(tag, env, nodeVersion, npmVersion) { + requireValue(typeof tag === 'string' && stable.test(tag) && !tag.includes('\n'), 'exact stable vX.Y.Z tag required'); + requireValue(env.GITHUB_ACTIONS === 'true' && env.GITHUB_REPOSITORY === 'bytefolk/mem', 'canonical GitHub Actions repository required'); + requireValue(['release', 'workflow_dispatch'].includes(env.GITHUB_EVENT_NAME), 'unsupported release event'); + requireValue(env.GITHUB_REF === `refs/tags/${tag}`, 'dispatch from the exact tag, so provenance identifies the packaged source'); + requireValue(env.GITHUB_WORKFLOW_REF === `bytefolk/mem/.github/workflows/npm-publish.yml@refs/tags/${tag}`, 'unexpected workflow identity'); + requireValue(/^[a-f0-9]{40}$/.test(env.GITHUB_SHA || ''), 'exact GitHub event commit required'); + requireValue(env.RUNNER_ENVIRONMENT === 'github-hosted' && env.RUNNER_OS === 'Linux', 'GitHub-hosted Linux runner required'); + requireValue(env.ACTIONS_ID_TOKEN_REQUEST_URL && env.ACTIONS_ID_TOKEN_REQUEST_TOKEN, 'OIDC id-token permission unavailable'); + requireValue(/^24\.[0-9]+\.[0-9]+$/.test(nodeVersion), 'Node 24 required'); + const npm = /^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$/.exec(npmVersion); + requireValue(npm && (+npm[1] > 11 || (+npm[1] === 11 && +npm[2] >= 15)), 'stable npm >=11.15.0 required'); + for (const [key, value] of Object.entries(env)) { + if (!value) continue; + requireValue(!/^(NPM_TOKEN|NODE_AUTH_TOKEN|NPM_AUTH_TOKEN|NPM_ID_TOKEN)$/i.test(key), 'npm token fallback is forbidden'); + requireValue(!/^npm_config_/i.test(key), 'ambient npm configuration is forbidden; the release runner isolates it'); + } +} + +export function checkProof(proof, tag, commit, now = Date.now()) { + requireValue(proof && typeof proof === 'object', 'HOLD: owner/org/Trusted Publisher proof unavailable'); + const exact = { schema: 1, package: PACKAGE, tag, commit, channel: 'next', organization: 'bytefolk', + repository: 'bytefolk/mem', workflow: 'npm-publish.yml', environment: 'npm-release', + organizationControlVerified: true, packageAccessVerified: true, twoFactorVerified: true, + publisherVerified: true, allowPublish: true }; + for (const [key, value] of Object.entries(exact)) assert.equal(proof[key], value, `HOLD: owner proof ${key} mismatch`); + requireValue(/^oidc:[A-Za-z0-9-]+$/.test(proof.publisherId || ''), 'HOLD: exact npm publisher configuration id required'); + requireValue(/^[A-Za-z0-9][A-Za-z0-9-]{0,38}$/.test(proof.approvedBy || ''), 'HOLD: responsible human approver required'); + requireValue(/^https:\/\/github\.com\/bytefolk\/(mem|\.github)\/issues\/(153|22)#issuecomment-[0-9]+$/.test(proof.evidence || ''), 'HOLD: sanitized owner evidence comment required'); + const verified = Date.parse(proof.verifiedAt); + const expires = Date.parse(proof.expiresAt); + requireValue(Number.isFinite(verified) && Number.isFinite(expires) && verified <= now && now < expires && + expires - verified <= 24 * 60 * 60 * 1000, 'HOLD: proof must be current and valid for at most 24 hours'); +} + +export function checkPackage(pkg, server, tag) { + assert.equal(pkg.name, PACKAGE, 'wrong npm scope/package'); + assert.equal(pkg.version, tag.slice(1), 'npm/tag version mismatch'); + assert.equal(pkg.mcpName, 'io.github.bytefolk/mem-mcp', 'wrong MCP identity'); + assert.equal(pkg.repository?.url, canonicalRepository, 'wrong provenance repository'); + assert.equal(pkg.repository?.type, 'git', 'git repository metadata required'); + requireValue(pkg.private !== true, 'private package must never publish'); + assert.deepEqual(Object.keys(pkg.bin || {}), ['mem-mcp'], 'unexpected CLI mapping'); + // npm normalizes the equivalent ./mem-mcp path to mem-mcp in the registry. + requireValue(['./mem-mcp', 'mem-mcp'].includes(pkg.bin['mem-mcp']), 'unexpected CLI path'); + assert.equal(server.version, pkg.version, 'MCP metadata version mismatch'); + assert.equal(server.mcpName, pkg.mcpName, 'MCP metadata identity mismatch'); + for (const key of ['dependencies', 'optionalDependencies', 'peerDependencies', 'bundledDependencies', 'bundleDependencies']) { + requireValue(!pkg[key] || Object.keys(pkg[key]).length === 0, 'wrapper dependency changes require release guard review'); + } + for (const key of Object.keys(pkg.scripts || {})) { + requireValue(['test', 'test:tarball'].includes(key), 'unexpected lifecycle script in release package'); + } + const allowed = { access: 'public', tag: 'next', registry: REGISTRY, provenance: true }; + for (const [key, value] of Object.entries(pkg.publishConfig || {})) { + requireValue(Object.hasOwn(allowed, key) && allowed[key] === value, 'unsafe publishConfig override'); + } +} + +export function checkRelease(release, tag) { + requireValue(release && release.tag_name === tag && release.draft === false && release.prerelease === false && + Number.isSafeInteger(release.id) && release.id > 0 && Number.isFinite(Date.parse(release.published_at)), 'published stable GitHub Release required'); + assert.equal(release.html_url, `https://github.com/bytefolk/mem/releases/tag/${tag}`, 'wrong release repository'); + assert.deepEqual(release.assets?.map(a => a.name).sort(), [...ASSETS, MANIFEST].sort(), 'exact seven release assets required'); + requireValue(release.assets.every(a => Number.isSafeInteger(a.size) && a.size > 0 && a.state === 'uploaded'), 'all release assets must be uploaded and nonempty'); +} + +export function readReleaseFile(file, expectedSize, read = readFileSync, inspect = () => {}) { + // Inspect and read the opened object, never check a pathname and reopen it. + // NONBLOCK lets us reject a substituted FIFO without waiting for a writer. + const fd = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const info = fstatSync(fd); + requireValue(info.isFile(), 'asset must be a regular file'); + assert.equal(info.size, expectedSize, 'downloaded asset size mismatch'); + const bytes = read(fd); + assert.equal(bytes.length, expectedSize, 'downloaded asset changed during read'); + inspect(fd); + const after = fstatSync(fd); + requireValue(after.size === info.size && after.mtimeMs === info.mtimeMs, 'asset changed during read'); + return bytes; + } finally { + closeSync(fd); + } +} + +export function checkAssets(directory, release, commit, run) { + assert.deepEqual(readdirSync(directory).sort(), [...ASSETS, MANIFEST].sort(), 'downloaded asset set mismatch'); + const verified = new Map(); + const buildMetadata = new Map(); + for (const asset of release.assets) { + const file = join(directory, asset.name); + const bytes = readReleaseFile(file, asset.size, readFileSync, fd => { + if (asset.name !== MANIFEST) { + // The hosted Linux child receives this already-open object at fd 3. + // Never reopen the mutable asset pathname for build metadata. + buildMetadata.set(asset.name, run('go', ['version', '-m', '/proc/self/fd/3'], undefined, fd)); + } + }); + verified.set(asset.name, bytes); + if (asset.digest != null) assert.equal(asset.digest, `sha256:${sha('sha256', bytes)}`, 'GitHub asset digest mismatch'); + } + const manifest = verified.get(MANIFEST).toString('utf8'); + requireValue(manifest.endsWith('\n'), 'checksum manifest must end in newline'); + const rows = manifest.slice(0, -1).split('\n').map(line => { + const row = /^([a-f0-9]{64}) (mem-mcp-[a-z0-9.-]+)$/.exec(line); + requireValue(row, 'malformed checksum row'); + return { digest: row[1], name: row[2] }; + }); + assert.deepEqual(rows.map(row => row.name).sort(), [...ASSETS].sort(), 'exactly one checksum per expected binary required'); + for (const { name, digest } of rows) { + assert.equal(sha('sha256', verified.get(name)), digest, 'binary checksum mismatch'); + // go version -m reads metadata; it never executes the downloaded binary. + const metadata = buildMetadata.get(name); + const [, , os, arch] = name.replace('.exe', '').split('-'); + for (const field of [`GOOS=${os}`, `GOARCH=${arch}`, `vcs.revision=${commit}`, 'vcs.modified=false']) { + requireValue(metadata.split('\n').some(line => line.trim() === `build\t${field}`), `binary build metadata mismatch: ${field}`); + } + requireValue(/(?:^|\n)\s*path\s+[^\s]+\/server\/cmd\/mem-mcp\s*\n/.test(metadata), 'wrong binary command'); + } +} + +export function checkRegistryBefore(data, tag) { + requireValue(data && data.name === PACKAGE && data.versions && data['dist-tags'], 'HOLD: public package/bootstrap unavailable; 404 is not org proof'); + const bootstrap = data.versions['0.1.2-rc.0']; + requireValue(bootstrap?.name === PACKAGE && bootstrap.version === '0.1.2-rc.0', 'HOLD: reviewed 0.1.2-rc.0 bootstrap required'); + requireValue(!Object.hasOwn(data.versions, tag.slice(1)), 'npm version already exists; never republish, including after partial failure'); + requireValue(!Object.values(data['dist-tags']).includes(tag.slice(1)), 'registry dist-tag references candidate before publish'); +} + +export function checkRegistryAfter(data, before, tag, integrity, proof) { + assert.equal(data?.name, PACKAGE, 'registry package mismatch'); + const version = tag.slice(1); + const published = data.versions?.[version]; + requireValue(published, 'published version missing from registry'); + checkPackage(published, { version, mcpName: 'io.github.bytefolk/mem-mcp' }, tag); + assert.equal(data['dist-tags']?.next, version, 'next readback mismatch'); + const tagsWithoutNext = tags => Object.fromEntries(Object.entries(tags).filter(([name]) => name !== 'next')); + assert.deepEqual(tagsWithoutNext(data['dist-tags']), tagsWithoutNext(before['dist-tags']), 'non-next dist-tags changed; owner investigation required'); + assert.equal(published.dist?.integrity, integrity, 'registry/tarball integrity mismatch'); + assert.equal(published._npmUser?.trustedPublisher?.id, 'github', 'publication was not GitHub OIDC'); + assert.equal(published._npmUser?.trustedPublisher?.oidcConfigId, proof.publisherId, 'unexpected Trusted Publisher'); + requireValue(Array.isArray(published.dist.signatures) && published.dist.signatures.length > 0 && + published.dist.signatures.every(s => s.keyid && s.sig), 'registry signatures unavailable'); + assert.equal(published.dist.attestations?.provenance?.predicateType, 'https://slsa.dev/provenance/v1', 'provenance unavailable'); + requireValue(published.dist.attestations?.url?.startsWith(`${REGISTRY}/-/npm/v1/attestations/`), 'unexpected attestation URL'); + assert.equal(published.dist.tarball, `${REGISTRY}/@bytefolk/mem-mcp/-/mem-mcp-${version}.tgz`, 'unexpected registry tarball URL'); + return published; +} + +async function registryJSON(url) { + const response = await fetch(url, { redirect: 'error', signal: AbortSignal.timeout(30000), + headers: { accept: 'application/json', 'cache-control': 'no-cache' } }); + requireValue(response.status === 200, `registry read failed (${response.status}); no write permitted`); + return response.json(); +} + +export async function runRelease(tag, options = {}) { + const repo = options.repo || resolve(dirname(fileURLToPath(import.meta.url)), '..'); + const env = options.env || process.env; + const now = options.now || Date.now; + const proof = JSON.parse(env.NPM_RELEASE_PROOF || 'null'); + checkProof(proof, tag, env.GITHUB_SHA, now()); + const directory = resolve(options.directory || join(env.RUNNER_TEMP || '', 'mem-npm-release')); + requireValue(!existsSync(directory), 'release output directory must be fresh'); + // Refuse ambient npmrc files, and bypass user/global config in all npm calls. + requireValue(!existsSync(join(repo, '.npmrc')) && !existsSync(join(repo, 'npm/.npmrc')), 'repository npmrc requires explicit security review'); + mkdirSync(directory, { recursive: true }); + const npmEnv = { ...env, NPM_CONFIG_USERCONFIG: join(directory, 'user.npmrc'), + NPM_CONFIG_GLOBALCONFIG: join(directory, 'global.npmrc'), NPM_CONFIG_CACHE: join(directory, 'cache') }; + delete npmEnv.GH_TOKEN; + delete npmEnv.GITHUB_TOKEN; + for (const name of ['user.npmrc', 'global.npmrc']) writeFileSync(join(directory, name), '', { flag: 'wx', mode: 0o600 }); + const run = options.run || ((command, args, cwd = repo, descriptor) => { + try { + return execFileSync(command, args, { cwd, encoding: 'utf8', env: command === 'npm' ? npmEnv : env, + stdio: commandStdio(descriptor), + timeout: 120000, maxBuffer: 16 * 1024 * 1024 }).trim(); + } catch { + // Never echo raw auth errors, subprocess output or environment values. + throw new Error(`${command} ${args[0]} failed; stop and inspect the private run. Do not retry publication automatically.`); + } + }); + const getJSON = options.getJSON || registryJSON; + checkContext(tag, env, options.nodeVersion || process.versions.node, run('npm', ['--version'])); + const source = () => { + run('git', ['fetch', '--no-tags', 'origin', 'refs/heads/main:refs/remotes/origin/main', `refs/tags/${tag}:refs/tags/${tag}`]); + assert.equal(run('git', ['cat-file', '-t', `refs/tags/${tag}`]), 'tag', 'annotated tag required'); + const commit = run('git', ['rev-parse', `refs/tags/${tag}^{commit}`]); + assert.equal(commit, env.GITHUB_SHA, 'tag/event commit mismatch'); + assert.equal(run('git', ['rev-parse', 'HEAD']), commit, 'checkout/tag mismatch'); + run('git', ['merge-base', '--is-ancestor', commit, 'refs/remotes/origin/main']); + assert.equal(run('git', ['status', '--porcelain', '--untracked-files=all']), '', 'release checkout must be clean'); + run('bash', [join(repo, 'scripts/validate_release_version.sh'), tag.slice(1)]); + checkPackage(readJSON(join(repo, 'npm/package.json')), readJSON(join(repo, 'npm/server.json')), tag); + return commit; + }; + const commit = source(); + const releaseJSON = () => JSON.parse(run('gh', ['api', `repos/bytefolk/mem/releases/tags/${tag}`])); + let release = releaseJSON(); + checkRelease(release, tag); + const releaseId = release.id; + if (env.GITHUB_EVENT_NAME === 'release') { + const event = readJSON(env.GITHUB_EVENT_PATH); + requireValue(event.action === 'published' && event.release?.id === releaseId && event.release?.tag_name === tag, 'Release event mismatch'); + } + const url = `${REGISTRY}/@bytefolk%2fmem-mcp`; + const before = await getJSON(url); + checkRegistryBefore(before, tag); + const assets = join(directory, 'assets'); + mkdirSync(assets); + run('gh', ['release', 'download', tag, '--repo', 'bytefolk/mem', '--dir', assets, + ...[...ASSETS, MANIFEST].flatMap(name => ['--pattern', name])]); + release = releaseJSON(); + checkRelease(release, tag); + assert.equal(release.id, releaseId, 'Release replaced while downloading'); + checkAssets(assets, release, commit, run); + const packed = JSON.parse(run('npm', ['pack', '--json', '--ignore-scripts', '--pack-destination', directory], join(repo, 'npm'))); + requireValue(Array.isArray(packed) && packed.length === 1, 'exactly one packed tarball required'); + const pack = packed[0]; + assert.equal(pack.name, PACKAGE, 'packed package name mismatch'); + assert.equal(pack.version, tag.slice(1), 'packed package version mismatch'); + assert.equal(pack.filename, `bytefolk-mem-mcp-${tag.slice(1)}.tgz`, 'unexpected tarball filename'); + assert.deepEqual(pack.files?.map(f => f.path).sort(), FILES, 'unexpected packed file inventory'); + const tarball = join(directory, pack.filename); + const integrity = `sha512-${sha('sha512', readFileSync(tarball))}`; + assert.equal(pack.integrity, integrity, 'packed tarball integrity mismatch'); + // Final checks immediately before the only registry write. Missing reads, + // races and moved tags stop; no exception is interpreted as version absence. + checkProof(proof, tag, source(), now()); + const currentRelease = releaseJSON(); + checkRelease(currentRelease, tag); + assert.deepEqual(releaseIdentity(currentRelease), releaseIdentity(release), 'GitHub Release changed after verification'); + const current = await getJSON(url); + checkRegistryBefore(current, tag); + assert.deepEqual(current['dist-tags'], before['dist-tags'], 'registry channels changed during preflight'); + assert.equal(`sha512-${sha('sha512', readFileSync(tarball))}`, integrity, 'tarball changed after packing'); + writeFileSync(join(directory, 'preflight.json'), JSON.stringify({ package: PACKAGE, tag, commit, integrity, releaseId, channel: 'next' }, null, 2)); + run('npm', ['publish', tarball, '--tag', 'next', '--access', 'public', '--provenance', '--ignore-scripts', `--registry=${REGISTRY}`], directory); + // A failure here may mean publish succeeded. Never retry npm publish, move a + // dist-tag, delete a version, or mark the run successful on that basis. + checkRegistryAfter(await getJSON(url), before, tag, integrity, proof); + const consumer = join(directory, 'consumer'); + mkdirSync(consumer); + writeFileSync(join(consumer, 'package.json'), '{"private":true}\n'); + run('npm', ['install', '--ignore-scripts', '--no-audit', '--no-fund', '--save-exact', `${PACKAGE}@${tag.slice(1)}`, `--registry=${REGISTRY}`], consumer); + run('npm', ['audit', 'signatures', `--registry=${REGISTRY}`], consumer); + checkRegistryAfter(await getJSON(url), before, tag, integrity, proof); + const receipt = { package: PACKAGE, tag, commit, integrity, releaseId, channel: 'next', + publisherId: proof.publisherId, registryMetadata: url, + attestations: 'verified by npm audit signatures', + signatures: 'verified by npm audit signatures', latestPromotion: 'NOT PERFORMED: separate release-owner gate', + platformLaunch: 'NOT VERIFIED: release owner must record Linux/macOS/Windows clean launches' }; + writeFileSync(join(directory, 'receipt.json'), JSON.stringify(receipt, null, 2)); + return receipt; +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + try { + requireValue(process.argv.length === 3, 'usage: node scripts/npm-release.mjs vX.Y.Z'); + const receipt = await runRelease(process.argv[2]); + process.stdout.write(`${JSON.stringify(receipt, null, 2)}\n`); + } catch (error) { + process.stderr.write(`HOLD: ${error.message}\n`); + process.exitCode = 1; + } +} diff --git a/scripts/npm-release.test.mjs b/scripts/npm-release.test.mjs new file mode 100644 index 0000000..d39fcdb --- /dev/null +++ b/scripts/npm-release.test.mjs @@ -0,0 +1,355 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { fstatSync, mkdtempSync, mkdirSync, readFileSync, readSync, renameSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { + ASSETS, PACKAGE, REGISTRY, checkContext, checkProof, checkPackage, + checkRelease, checkAssets, checkRegistryBefore, checkRegistryAfter, commandStdio, readReleaseFile, runRelease, +} from './npm-release.mjs'; + +const tag = 'v0.1.2'; +const commit = 'a'.repeat(40); +const now = Date.parse('2026-09-10T00:00:00Z'); +const env = () => ({ + GITHUB_ACTIONS: 'true', GITHUB_REPOSITORY: 'bytefolk/mem', + GITHUB_EVENT_NAME: 'workflow_dispatch', GITHUB_REF: `refs/tags/${tag}`, + GITHUB_SHA: commit, GITHUB_WORKFLOW_REF: `bytefolk/mem/.github/workflows/npm-publish.yml@refs/tags/${tag}`, + RUNNER_ENVIRONMENT: 'github-hosted', RUNNER_OS: 'Linux', + ACTIONS_ID_TOKEN_REQUEST_URL: 'https://example.invalid/oidc', + ACTIONS_ID_TOKEN_REQUEST_TOKEN: 'fixture-only', +}); +const proof = () => ({ + schema: 1, package: PACKAGE, tag, commit, channel: 'next', + organization: 'bytefolk', organizationControlVerified: true, + packageAccessVerified: true, twoFactorVerified: true, + publisherVerified: true, allowPublish: true, + repository: 'bytefolk/mem', workflow: 'npm-publish.yml', environment: 'npm-release', + publisherId: 'oidc:fixture', approvedBy: 'release-owner', + evidence: 'https://github.com/bytefolk/mem/issues/153#issuecomment-123', + verifiedAt: '2026-09-09T23:00:00Z', expiresAt: '2026-09-10T23:00:00Z', +}); +const pkg = () => ({ + name: PACKAGE, version: '0.1.2', mcpName: 'io.github.bytefolk/mem-mcp', + repository: { type: 'git', url: 'git+https://github.com/bytefolk/mem.git' }, + bin: { 'mem-mcp': './mem-mcp' }, +}); +const server = () => ({ version: '0.1.2', name: 'mem-mcp', mcpName: 'io.github.bytefolk/mem-mcp' }); +const release = () => ({ id: 123, tag_name: tag, draft: false, prerelease: false, + published_at: '2026-09-09T23:00:00Z', html_url: `https://github.com/bytefolk/mem/releases/tag/${tag}`, + assets: [...ASSETS, 'mem-mcp-checksums.txt'].map(name => ({ name, size: 1, state: 'uploaded' })) }); +const before = () => ({ name: PACKAGE, versions: { '0.1.2-rc.0': { name: PACKAGE, version: '0.1.2-rc.0' } }, + 'dist-tags': { next: '0.1.2-rc.0' } }); +const integrity = 'sha512-' + Buffer.alloc(64, 1).toString('base64'); +const after = () => ({ ...before(), 'dist-tags': { next: '0.1.2' }, versions: { + ...before().versions, '0.1.2': { ...pkg(), bin: { 'mem-mcp': 'mem-mcp' }, + _npmUser: { trustedPublisher: { id: 'github', oidcConfigId: 'oidc:fixture' } }, + dist: { integrity, tarball: `${REGISTRY}/@bytefolk/mem-mcp/-/mem-mcp-0.1.2.tgz`, + signatures: [{ keyid: 'SHA256:fixture', sig: 'fixture' }], + attestations: { url: `${REGISTRY}/-/npm/v1/attestations/@bytefolk%2fmem-mcp@0.1.2`, + provenance: { predicateType: 'https://slsa.dev/provenance/v1' } } } } } }); + +test('accepts exact hosted tag context, current human proof, package, assets and registry readback', () => { + checkContext(tag, env(), '24.13.0', '11.15.0'); + checkProof(proof(), tag, commit, now); + checkPackage(pkg(), server(), tag); + checkRelease(release(), tag); + checkRegistryBefore(before(), tag); + checkRegistryAfter(after(), before(), tag, integrity, proof()); +}); + +for (const bad of ['', 'v01.2.3', '0.1.2', 'v0.1.2-rc.0', 'v0.1.2+build', 'v0.1.2\n', 'v0.1.2;id', '--help']) { + test(`rejects unsafe/nonstable tag ${JSON.stringify(bad)}`, () => { + assert.throws(() => checkContext(bad, env(), '24.13.0', '11.15.0')); + }); +} +for (const [field, value] of [ + ['GITHUB_REPOSITORY', 'attacker/mem'], ['GITHUB_EVENT_NAME', 'pull_request'], + ['GITHUB_REF', 'refs/heads/main'], ['GITHUB_SHA', ''], ['GITHUB_ACTIONS', 'false'], + ['RUNNER_ENVIRONMENT', 'self-hosted'], ['ACTIONS_ID_TOKEN_REQUEST_TOKEN', ''], + ['GITHUB_WORKFLOW_REF', 'bytefolk/mem/.github/workflows/other.yml@refs/tags/v0.1.2'], + ['NPM_TOKEN', 'fixture'], ['NODE_AUTH_TOKEN', 'fixture'], ['npm_config_provenance', 'false'], +]) { + test(`rejects wrong context or credential/config injection: ${field}`, () => { + assert.throws(() => checkContext(tag, { ...env(), [field]: value }, '24.13.0', '11.15.0')); + }); +} +for (const [node, npm] of [['22.0.0', '11.15.0'], ['24.13.0', '11.6.2'], ['24.13.0', '11.15.0-rc.1']]) { + test(`rejects unsupported tools ${node}/${npm}`, () => assert.throws(() => checkContext(tag, env(), node, npm))); +} +for (const [field, value] of [ + ['organizationControlVerified', false], ['packageAccessVerified', false], ['twoFactorVerified', false], + ['publisherVerified', false], ['allowPublish', false], ['publisherId', ''], + ['environment', 'production'], ['workflow', 'release.yml'], ['repository', 'someone/mem'], + ['package', '@fullstack-ai-infra/mem-mcp'], ['channel', 'latest'], ['tag', 'v0.1.3'], + ['commit', 'b'.repeat(40)], ['approvedBy', ''], ['evidence', ''], + ['expiresAt', '2026-09-09T23:59:00Z'], ['verifiedAt', '2026-09-11T00:00:00Z'], + ['expiresAt', '2027-01-01T00:00:00Z'], +]) { + test(`fails closed on missing/wrong/stale owner proof: ${field}`, () => { + assert.throws(() => checkProof({ ...proof(), [field]: value }, tag, commit, now)); + }); +} +test('no owner proof is a blocker', () => assert.throws(() => checkProof(null, tag, commit, now))); +for (const mutate of [ + p => { p.name = '@fullstack-ai-infra/mem-mcp'; }, p => { p.version = '0.1.1'; }, + p => { p.private = true; }, p => { p.repository.url = 'git+https://github.com/attacker/mem.git'; }, + p => { p.publishConfig = { tag: 'latest' }; }, p => { p.publishConfig = { registry: 'https://example.invalid' }; }, + p => { p.scripts = { prepublishOnly: 'dangerous' }; }, p => { p.dependencies = { surprise: '*' }; }, +]) { + test(`rejects unsafe package: ${mutate}`, () => { const p = pkg(); mutate(p); assert.throws(() => checkPackage(p, server(), tag)); }); +} +for (const mutate of [ + r => { r.draft = true; }, r => { r.prerelease = true; }, r => { r.tag_name = 'v0.1.1'; }, + r => { r.assets.pop(); }, r => { r.assets.push(r.assets[0]); }, r => { r.assets[0].size = 0; }, + r => { r.assets[0].name = '../outside'; }, r => { r.assets[0].state = 'new'; }, +]) { + test(`rejects incomplete or mismatched Release: ${mutate}`, () => { const r = release(); mutate(r); assert.throws(() => checkRelease(r, tag)); }); +} +test('registry 404/error-shaped or duplicate stable version is never publish permission', () => { + for (const data of [null, {}, { error: 'Not found' }, after()]) assert.throws(() => checkRegistryBefore(data, tag)); +}); +for (const mutate of [ + r => { r['dist-tags'].latest = '0.1.2'; }, r => { r['dist-tags'].next = '0.1.2-rc.0'; }, + r => { r.versions['0.1.2'].dist.integrity = 'bad'; }, + r => { delete r.versions['0.1.2'].dist.attestations; }, + r => { delete r.versions['0.1.2'].dist.signatures; }, + r => { r.versions['0.1.2']._npmUser.trustedPublisher.oidcConfigId = 'oidc:other'; }, + r => { r.versions['0.1.2'].repository.url = 'https://github.com/attacker/mem'; }, +]) { + test(`readback failure blocks success: ${mutate}`, () => { const r = after(); mutate(r); assert.throws(() => checkRegistryAfter(r, before(), tag, integrity, proof())); }); +} + +function fixture(t) { + const root = mkdtempSync(join(tmpdir(), 'mem-npm-release-test-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + const repo = join(root, 'repo'); + mkdirSync(join(repo, 'npm'), { recursive: true }); + writeFileSync(join(repo, 'npm/package.json'), JSON.stringify(pkg())); + writeFileSync(join(repo, 'npm/server.json'), JSON.stringify(server())); + const directory = join(root, 'out'); + const calls = []; + let registryReads = 0; + let published = false; + const r = release(); + const run = (command, args, cwd, descriptor) => { + calls.push([command, ...args]); + if (command === 'git') { + if (args[0] === 'cat-file') return 'tag'; + if (args[0] === 'rev-parse') return commit; + return ''; + } + if (command === 'bash') return ''; + if (command === 'gh' && args[0] === 'api') return JSON.stringify(r); + if (command === 'gh' && args[0] === 'release') { + const assetDir = args[args.indexOf('--dir') + 1]; + let manifest = ''; + for (const name of ASSETS) { + const body = Buffer.from(`fixture ${name}`); + writeFileSync(join(assetDir, name), body); + r.assets.find(a => a.name === name).size = body.length; + manifest += `${createHash('sha256').update(body).digest('hex')} ${name}\n`; + } + writeFileSync(join(assetDir, 'mem-mcp-checksums.txt'), manifest); + r.assets.at(-1).size = Buffer.byteLength(manifest); + return ''; + } + if (command === 'go') { + let bytes; + if (descriptor === undefined) bytes = readFileSync(args.at(-1)); + else { + assert.equal(args.at(-1), '/proc/self/fd/3'); + bytes = Buffer.alloc(fstatSync(descriptor).size); + assert.equal(readSync(descriptor, bytes, 0, bytes.length, 0), bytes.length); + } + const [, name, revision = commit] = bytes.toString().split(' '); + const [, , os, arch] = name.replace('.exe', '').split('-'); + return `\tpath\tgithub.com/PeterGuy326/mem/server/cmd/mem-mcp\n\tbuild\tGOOS=${os}\n\tbuild\tGOARCH=${arch}\n\tbuild\tvcs.revision=${revision}\n\tbuild\tvcs.modified=false\n`; + } + if (command === 'npm') { + if (args[0] === '--version') return '11.15.0'; + if (args[0] === 'pack') { + const body = Buffer.from('test tarball'); + writeFileSync(join(directory, 'bytefolk-mem-mcp-0.1.2.tgz'), body); + return JSON.stringify([{ name: PACKAGE, version: '0.1.2', filename: 'bytefolk-mem-mcp-0.1.2.tgz', + integrity: 'sha512-' + createHash('sha512').update(body).digest('base64'), + files: ['LICENSE', 'README.md', 'install.js', 'mem-mcp', 'package.json', 'platforms.js'].map(path => ({ path })) }]); + } + if (args[0] === 'publish') { published = true; return ''; } + if (args[0] === 'install' || args[0] === 'audit') return ''; + } + throw Error(`unexpected command: ${command} ${args.join(' ')}, cwd=${cwd}`); + }; + const getJSON = async () => { + registryReads++; + if (!published) return before(); + const data = after(); + data.versions['0.1.2'].dist.integrity = 'sha512-' + createHash('sha512').update('test tarball').digest('base64'); + return data; + }; + return { repo, directory, env: { ...env(), NPM_RELEASE_PROOF: JSON.stringify(proof()) }, + now: () => now, nodeVersion: '24.13.0', run, getJSON, calls, r, + readCount: () => registryReads }; +} + +test('asset read holds one descriptor across path replacement and always closes it', t => { + const f = fixture(t); + const file = join(f.repo, 'asset'); + writeFileSync(file, 'original'); + let descriptor; + const bytes = readReleaseFile(file, 8, fd => { + descriptor = fd; + renameSync(file, file + '.saved'); + writeFileSync(file, 'replaced'); + return readFileSync(fd); + }); + assert.equal(bytes.toString(), 'original'); + assert.equal(readFileSync(file, 'utf8'), 'replaced'); + assert.throws(() => fstatSync(descriptor), /EBADF/); +}); + +test('asset descriptor rejects symlinks, directories and wrong sizes', t => { + const f = fixture(t); + const file = join(f.repo, 'asset'); + writeFileSync(file, 'original'); + symlinkSync(file, file + '.link'); + assert.throws(() => readReleaseFile(file + '.link', 8)); + assert.throws(() => readReleaseFile(f.repo, 8), /regular file/); + assert.throws(() => readReleaseFile(file, 7), /size mismatch/); + let descriptor; + assert.throws(() => readReleaseFile(file, 8, fd => { descriptor = fd; throw Error('read failed'); }), /read failed/); + assert.throws(() => fstatSync(descriptor), /EBADF/); +}); + +test('real child metadata transport keeps the verified descriptor across path replacement', t => { + const f = fixture(t); + const file = join(f.repo, 'asset'); + writeFileSync(file, 'original'); + readReleaseFile(file, 8, readFileSync, fd => { + renameSync(file, file + '.saved'); + writeFileSync(file, 'replaced'); + const path = process.platform === 'linux' ? '/proc/self/fd/3' : '/dev/fd/3'; + const child = `const fs = require('node:fs'); const fd = fs.openSync(process.argv[1], 'r'); + const bytes = Buffer.alloc(fs.fstatSync(fd).size); fs.readSync(fd, bytes, 0, bytes.length, 0); + fs.closeSync(fd); process.stdout.write(bytes);`; + assert.equal(execFileSync(process.execPath, ['-e', child, path], { stdio: commandStdio(fd) }).toString(), 'original'); + }); +}); + +for (const replacedIndex of [0, 1]) { + test(`metadata and checksum cannot validate different objects: asset ${replacedIndex}`, t => { + const f = fixture(t); + const assets = join(f.directory, 'assets'); + mkdirSync(assets, { recursive: true }); + f.run('gh', ['release', 'download', tag, '--dir', assets]); + const name = ASSETS[replacedIndex]; + const path = join(assets, name); + const wrong = Buffer.from(`fixture ${name} ${'b'.repeat(40)}`); + writeFileSync(path, wrong); + f.r.assets.find(a => a.name === name).size = wrong.length; + const manifestPath = join(assets, 'mem-mcp-checksums.txt'); + const manifest = readFileSync(manifestPath, 'utf8').split('\n').map(line => + line.endsWith(` ${name}`) ? `${createHash('sha256').update(wrong).digest('hex')} ${name}` : line).join('\n'); + writeFileSync(manifestPath, manifest); + let replaced = false; + const run = (command, args, cwd, descriptor) => { + if (command === 'go' && !replaced) { + replaced = true; + renameSync(path, path + '.saved'); + writeFileSync(path, `fixture ${name} ${commit}`); + } + return f.run(command, args, cwd, descriptor); + }; + assert.throws(() => checkAssets(assets, f.r, commit, run), /checksum mismatch|build metadata mismatch/); + assert.equal(replaced, true); + }); +} + +test('receipt records verified local facts, not a registry-supplied payload', async t => { + const f = fixture(t); + const original = f.getJSON; + f.getJSON = async () => { + const data = await original(); + if (data.versions['0.1.2']) data.versions['0.1.2'].dist.attestations.url = `${REGISTRY}/-/npm/v1/attestations/server-controlled-marker`; + return data; + }; + const receipt = await runRelease(tag, f); + assert.equal(receipt.registryMetadata, `${REGISTRY}/@bytefolk%2fmem-mcp`); + assert.equal(receipt.attestations, 'verified by npm audit signatures'); + assert.ok(!readFileSync(join(f.directory, 'receipt.json'), 'utf8').includes('server-controlled-marker')); +}); + +test('full fixture publishes the checked tarball once to next then verifies signatures; no latest mutation', async t => { + const f = fixture(t); + await runRelease(tag, f); + const writes = f.calls.filter(c => c[0] === 'npm' && c[1] === 'publish'); + assert.equal(writes.length, 1); + assert.deepEqual(writes[0].slice(3), ['--tag', 'next', '--access', 'public', '--provenance', '--ignore-scripts', `--registry=${REGISTRY}`]); + assert.ok(f.calls.some(c => c.join(' ') === `npm audit signatures --registry=${REGISTRY}`)); + assert.ok(f.readCount() >= 3); + assert.ok(!f.calls.some(c => c.includes('dist-tag') || c.includes('deprecate'))); + assert.ok(readFileSync(join(f.directory, 'receipt.json'), 'utf8').includes('next')); +}); + +for (const failure of ['proof', 'source', 'registry', 'assets', 'pack', 'recheck', 'publish', 'readback', 'audit']) { + test(`full fixture stops at ${failure}; no publish retry or promotion`, async t => { + const f = fixture(t); + const original = f.run; + let sourceChecks = 0; + if (failure === 'proof') delete f.env.NPM_RELEASE_PROOF; + if (failure === 'registry' || failure === 'readback') { + const get = f.getJSON; + f.getJSON = async () => { if (failure === 'registry' || f.calls.some(c => c[1] === 'publish')) throw Error('fixture unavailable'); return get(); }; + } + f.run = (command, args, cwd, descriptor) => { + if (command === 'git' && args[0] === 'fetch') { + sourceChecks++; + if (failure === 'source' || (failure === 'recheck' && sourceChecks > 1)) throw Error('source moved'); + } + if ((failure === 'pack' && args[0] === 'pack') || (failure === 'publish' && args[0] === 'publish') || + (failure === 'audit' && args[0] === 'audit')) { f.calls.push([command, ...args]); throw Error('fixture failure'); } + const result = original(command, args, cwd, descriptor); + if (failure === 'assets' && command === 'gh' && args[0] === 'release') writeFileSync(join(f.directory, 'assets', ASSETS[0]), 'tampered'); + return result; + }; + await assert.rejects(runRelease(tag, f)); + const count = f.calls.filter(c => c[0] === 'npm' && c[1] === 'publish').length; + assert.equal(count, ['publish', 'readback', 'audit'].includes(failure) ? 1 : 0); + assert.ok(!f.calls.some(c => c.includes('dist-tag'))); + }); +} + +test('checksum parser rejects duplicate, traversal, missing and mismatched rows', t => { + const f = fixture(t); + mkdirSync(join(f.directory, 'assets'), { recursive: true }); + f.run('gh', ['release', 'download', tag, '--dir', join(f.directory, 'assets')]); + const path = join(f.directory, 'assets/mem-mcp-checksums.txt'); + const good = readFileSync(path, 'utf8'); + checkAssets(join(f.directory, 'assets'), f.r, commit, f.run); + for (const bad of [good + good.split('\n')[0] + '\n', good.replace(ASSETS[0], '../outside'), good.split('\n').slice(1).join('\n'), good.replace(/^[a-f0-9]/, 'z')]) { + writeFileSync(path, bad); + assert.throws(() => checkAssets(join(f.directory, 'assets'), f.r, commit, f.run)); + } +}); + +for (const changed of ['download_count', 'digest']) { + test(`final Release readback handles changing ${changed}`, async t => { + const f = fixture(t); + const original = f.run; + let releaseReads = 0; + f.run = (command, args, cwd, descriptor) => { + if (command === 'gh' && args[0] === 'api' && ++releaseReads === 3) { + f.r.assets[0][changed] = changed === 'digest' ? 'sha256:' + 'b'.repeat(64) : 17; + } + return original(command, args, cwd, descriptor); + }; + if (changed === 'digest') { + await assert.rejects(runRelease(tag, f), /Release changed/); + assert.ok(!f.calls.some(c => c[0] === 'npm' && c[1] === 'publish')); + } else { + await runRelease(tag, f); + } + }); +} diff --git a/scripts/test_nginx_security_headers.sh b/scripts/test_nginx_security_headers.sh new file mode 100755 index 0000000..35ab9b9 --- /dev/null +++ b/scripts/test_nginx_security_headers.sh @@ -0,0 +1,248 @@ +#!/usr/bin/env bash +# Security response-header contract for the web reverse proxy. +# +# nginx is the single authority for X-Content-Type-Options, X-Frame-Options and +# Referrer-Policy; the API owns Content-Security-Policy, X-XSS-Protection and +# Content-Disposition. This runs a real nginx because both failure modes are +# runtime semantics that reading the config cannot prove: add_header in a nested +# block silently voids the inherited set, and an upstream header survives the +# proxy unless it is explicitly hidden. +# +# Usage: scripts/test_nginx_security_headers.sh [path-to-nginx-binary] +set -euo pipefail + +repo_root=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd) +nginx_bin=${1:-${NGINX_BIN:-}} + +if [[ -n "$nginx_bin" && ! -x "$nginx_bin" ]]; then + # An explicitly requested binary that is not there must not become a green + # run: a caller that pins NGINX_BIN is asserting that the check executed. + printf 'ERROR: requested nginx binary is not executable: %s\n' "$nginx_bin" >&2 + exit 1 +fi +if [[ -z "$nginx_bin" ]]; then + nginx_bin=$(command -v nginx || true) +fi +if [[ -z "$nginx_bin" ]]; then + printf 'SKIP: no nginx binary found (pass one as $1 or set NGINX_BIN)\n' + exit 0 +fi +for tool in curl envsubst python3; do + command -v "$tool" >/dev/null 2>&1 || { + printf 'ERROR: %s is required to drive the proxy\n' "$tool" >&2 + exit 1 + } +done + +port=${PORT:-18080} +upstream_port=${UPSTREAM_PORT:-18081} +# 3 proxy-owned headers on five surfaces, + the 3 API-owned headers required on +# /v1/ and the 2 required absent on the other four, + the not-found status guard, +# + the /assets/ Cache-Control guard. +expected_checks=28 +work=$(mktemp -d) +upstream_pid='' +nginx_pid='' +failures=0 +checks=0 + +cleanup() { + [[ -n "$nginx_pid" ]] && kill "$nginx_pid" 2>/dev/null || true + [[ -n "$upstream_pid" ]] && kill "$upstream_pid" 2>/dev/null || true + wait 2>/dev/null || true + rm -rf "$work" +} +trap cleanup EXIT HUP INT TERM + +fail() { printf ' not ok - %s\n' "$*" >&2; failures=$((failures + 1)); checks=$((checks + 1)); } +pass() { printf ' ok - %s\n' "$*"; checks=$((checks + 1)); } + +# --- render the shipped template the way the container entrypoint does --- +prefix=$work +mkdir -p "$prefix/conf" "$prefix/logs" "$prefix/run" "$prefix/html/assets" \ + "$prefix/tmp/client" "$prefix/tmp/proxy" "$prefix/tmp/fastcgi" \ + "$prefix/tmp/uwsgi" "$prefix/tmp/scgi" +printf 'console.log(1)\n' >"$prefix/html/assets/app.js" +printf 'mem\n' >"$prefix/html/index.html" + +export MEMD_UPSTREAM="http://127.0.0.1:${upstream_port}" +export MEM_MAX_BODY_SIZE=256m +export NGINX_ENVSUBST_FILTER='^(MEMD_UPSTREAM|MEM_MAX_BODY_SIZE)$' +rendered=$work/default.conf +envsubst '$MEMD_UPSTREAM $MEM_MAX_BODY_SIZE' \ + <"$repo_root/web/nginx/default.conf.template" >"$rendered" +if grep -Eq '\$\{[A-Z_]+\}' "$rendered"; then + printf 'the template left a variable unexpanded:\n' >&2 + grep -Eo '\$\{[A-Z_]+\}' "$rendered" | sort -u >&2 + exit 1 +fi + +mime_candidates=( + "$(cd -- "$(dirname -- "$nginx_bin")/.." && pwd)/conf/mime.types" + /etc/nginx/mime.types +) +mime_types='' +for candidate in "${mime_candidates[@]}"; do + if [[ -f "$candidate" ]]; then mime_types=$candidate; break; fi +done +if [[ -z "$mime_types" ]]; then + printf 'no mime.types found near %s\n' "$nginx_bin" >&2 + exit 1 +fi + +python3 - "$rendered" "$work/conf/nginx.conf" "$prefix" "$port" "$mime_types" <<'PY' +import sys + +conf_path, out_path, prefix, port, mime_types = sys.argv[1:6] +with open(conf_path, encoding="utf-8") as handle: + server = handle.read() +server = server.replace("listen 8080;", "listen 127.0.0.1:%s;" % port) +server = server.replace("root /usr/share/nginx/html;", "root %s/html;" % prefix) +if "listen 127.0.0.1:%s;" % port not in server or ("%s/html" % prefix) not in server: + raise SystemExit("the shipped server block drifted: could not rebind it for the harness") + +with open(out_path, "w", encoding="utf-8") as handle: + handle.write(""" +worker_processes 1; +error_log {prefix}/logs/error.log warn; +pid {prefix}/run/nginx.pid; +daemon off; + +events {{ worker_connections 64; }} + +http {{ + include {mime_types}; + default_type application/octet-stream; + access_log {prefix}/logs/access.log; + client_body_temp_path {prefix}/tmp/client; + proxy_temp_path {prefix}/tmp/proxy; + fastcgi_temp_path {prefix}/tmp/fastcgi; + uwsgi_temp_path {prefix}/tmp/uwsgi; + scgi_temp_path {prefix}/tmp/scgi; + +{server} +}} +""".format(prefix=prefix, server=server.rstrip(), mime_types=mime_types)) +PY + +# --- fake upstream answering exactly like the Go API middleware does --- +cat >"$work/upstream.py" <<'PY' +import http.server +import os + +# Mirrors securityHeadersMiddleware (server/internal/api/util.go) plus the +# per-response Content-Disposition that the download handlers set. If the API's +# header set changes, this fixture must change with it. +class Handler(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_GET(self): # noqa: N802 + body = b'{"ok":true}' + self.send_response(200) + self.send_header("X-Content-Type-Options", "nosniff") + self.send_header("X-Frame-Options", "DENY") + self.send_header("Referrer-Policy", "no-referrer") + self.send_header("Content-Security-Policy", "default-src 'none'") + self.send_header("X-XSS-Protection", "0") + self.send_header("Content-Disposition", 'attachment; filename="note.txt"') + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + +http.server.HTTPServer(("127.0.0.1", int(os.environ["UPSTREAM_PORT"])), Handler).serve_forever() +PY +UPSTREAM_PORT=$upstream_port python3 "$work/upstream.py" & +upstream_pid=$! + +"$nginx_bin" -p "$prefix" -c "$work/conf/nginx.conf" & +nginx_pid=$! + +ready='' +for _ in $(seq 1 50); do + if curl -fsS -o /dev/null "http://127.0.0.1:${port}/v1/ping" 2>/dev/null; then ready=yes; break; fi + sleep 0.2 +done +if [[ -z "$ready" ]]; then + printf 'nginx never became ready; error log:\n' >&2 + cat "$prefix/logs/error.log" >&2 || true + exit 1 +fi + +header_values() { + # No -f: a 404 or a 502 has to be probeable too, because `always` is what + # keeps these headers on an error response. + curl -sS -D - -o /dev/null "$1" | tr -d '\r' | + awk -v h="$2" ' + /^$/ { exit } + tolower($0) ~ "^" tolower(h) ":" { sub(/^[^:]*:[ \t]?/, ""); print }' +} + +check_single() { + local url=$1 header=$2 want=$3 got count + got=$(header_values "$url" "$header") + count=$(printf '%s' "$got" | grep -c . || true) + if [[ $count -ne 1 ]]; then + fail "${url##*/}: $header appears $count times, want exactly 1 ($(printf '%s' "$got" | tr '\n' '|'))" + elif [[ $got != "$want" ]]; then + fail "${url##*/}: $header = $got, want $want" + else + pass "${url##*/}: $header: $got" + fi +} + +check_absent() { + local url=$1 header=$2 count + count=$(header_values "$url" "$header" | grep -c . || true) + if [[ $count -ne 0 ]]; then + fail "${url##*/}: $header present, want absent (nginx must not set what the API owns)" + else + pass "${url##*/}: $header absent" + fi +} + +for path in /index.html /assets/app.js /assets/does-not-exist.js /v1/ping /healthz; do + printf '\n== %s ==\n' "$path" + url="http://127.0.0.1:${port}${path}" + check_single "$url" X-Content-Type-Options nosniff + check_single "$url" X-Frame-Options DENY + check_single "$url" Referrer-Policy no-referrer + if [[ $path == /v1/* ]]; then + check_single "$url" Content-Security-Policy "default-src 'none'" + check_single "$url" X-XSS-Protection 0 + check_single "$url" Content-Disposition 'attachment; filename="note.txt"' + else + check_absent "$url" Content-Security-Policy + check_absent "$url" Content-Disposition + fi +done + +printf '\n== the not-found surface really was a not-found ==\n' +status=$(curl -sS -o /dev/null -w '%{http_code}' "http://127.0.0.1:${port}/assets/does-not-exist.js") +if [[ $status == 404 ]]; then + pass "/assets/does-not-exist.js: HTTP $status" +else + fail "/assets/does-not-exist.js: HTTP $status, want 404 -- the header assertions above would then be measuring a success response, not the always flag" +fi + +printf '\n== /assets/ caching is not collateral damage ==\n' +got=$(header_values "http://127.0.0.1:${port}/assets/app.js" Cache-Control) +if [[ $got == *immutable* ]]; then + pass "/assets/app.js: Cache-Control: $got" +else + fail "/assets/app.js: Cache-Control = ${got:-}, want it to still say immutable" +fi + +printf '\n' +if [[ $checks -ne $expected_checks ]]; then + printf '%d security-header assertions ran, expected %d\n' "$checks" "$expected_checks" >&2 + exit 1 +fi +if [[ $failures -ne 0 ]]; then + printf '%d of %d security-header assertions failed\n' "$failures" "$checks" >&2 + exit 1 +fi +printf 'all %d security-header assertions passed\n' "$checks" diff --git a/scripts/test_release_checksum_output_safety.sh b/scripts/test_release_checksum_output_safety.sh new file mode 100755 index 0000000..8392c16 --- /dev/null +++ b/scripts/test_release_checksum_output_safety.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +generator="${1:-${repo_root}/scripts/generate_release_checksums.sh}" +tmp_dir="$(mktemp -d "${TMPDIR:-/tmp}/mem-checksum-output.XXXXXX")" +trap 'rm -rf -- "${tmp_dir}"' EXIT + +die() { + printf 'FAIL: %s\n' "$*" >&2 + exit 1 +} + +assets=( + mem-mcp-darwin-amd64 + mem-mcp-darwin-arm64 + mem-mcp-linux-amd64 + mem-mcp-linux-arm64 + mem-mcp-windows-amd64.exe + mem-mcp-windows-arm64.exe + memd-darwin-amd64 + memd-darwin-arm64 + memd-linux-amd64 + memd-linux-arm64 + mem-migrate-darwin-amd64 + mem-migrate-darwin-arm64 + mem-migrate-linux-amd64 + mem-migrate-linux-arm64 + mem-healthcheck-darwin-amd64 + mem-healthcheck-darwin-arm64 + mem-healthcheck-linux-amd64 + mem-healthcheck-linux-arm64 + mem-darwin-amd64 + mem-darwin-arm64 + mem-linux-amd64 + mem-linux-arm64 +) +commit=1111111111111111111111111111111111111111 + +prepare() { + case_root="${tmp_dir}/${1} with spaces" + asset_dir="${case_root}/assets with spaces" + outside="${case_root}/outside with spaces" + mkdir -p -- "${asset_dir}" "${outside}" + for asset in "${assets[@]}"; do + printf 'test payload for %s\n' "${asset}" > "${asset_dir}/${asset}" + done + printf 'external data must remain unchanged\n' > "${outside}/keep.txt" + output="${asset_dir}/mem-mcp-checksums.txt" +} + +snapshot_files() { + find "${case_root}" -type f -exec sha256sum {} \; | LC_ALL=C sort +} + +for kind in symlink-directory symlink-file dangling-symlink directory regular-file; do + prepare "${kind}" + case "${kind}" in + symlink-directory) ln -s -- "${outside}" "${output}" ;; + symlink-file) ln -s -- "${outside}/keep.txt" "${output}" ;; + dangling-symlink) ln -s -- "${outside}/not-created.txt" "${output}" ;; + directory) mkdir -- "${output}" ;; + regular-file) printf 'existing manifest must remain unchanged\n' > "${output}" ;; + esac + before="$(snapshot_files)" + status=0 + bash "${generator}" v0.1.1 "${commit}" "${asset_dir}" > "${tmp_dir}/result.log" 2>&1 || status=$? + # Check side effects before status: the original directory-symlink bug may + # return failure only after mv has already written outside the asset tree. + [[ "$(snapshot_files)" == "${before}" ]] || + die "${kind}: output publication changed existing data or created an unexpected file" + [[ "${status}" -ne 0 ]] || die "${kind}: existing output was accepted" + grep -Fq 'checksum output path already exists' "${tmp_dir}/result.log" || + die "${kind}: missing output-path diagnostic" + case "${kind}" in + symlink-directory) [[ -L "${output}" && "$(readlink "${output}")" == "${outside}" ]] ;; + symlink-file) [[ -L "${output}" && "$(readlink "${output}")" == "${outside}/keep.txt" ]] ;; + dangling-symlink) [[ -L "${output}" && "$(readlink "${output}")" == "${outside}/not-created.txt" ]] ;; + directory) [[ -d "${output}" && ! -L "${output}" ]] ;; + regular-file) [[ -f "${output}" && ! -L "${output}" ]] ;; + esac || die "${kind}: existing output path was replaced" + printf 'PASS: %s rejected without changing external data or output path\n' "${kind}" +done + +# Simulate an output symlink appearing while checksum generation is in progress. +# The second absence check must reject it and cleanup only our own staging file. +prepare output-created-during-hashing +fake_bin="${case_root}/hash tools" +mkdir -p -- "${fake_bin}" +printf '%s\n' '#!/usr/bin/env bash' 'set -euo pipefail' \ + "\"\${REAL_SHA256SUM}\" \"\$@\"" \ + "ln -s -- \"\${OUTPUT_TARGET}\" \"\${OUTPUT_MANIFEST}\"" > "${fake_bin}/sha256sum" +chmod +x "${fake_bin}/sha256sum" +before="$(snapshot_files)" +status=0 +REAL_SHA256SUM="$(command -v sha256sum)" OUTPUT_TARGET="${outside}" OUTPUT_MANIFEST="${output}" \ + PATH="${fake_bin}:${PATH}" bash "${generator}" v0.1.1 "${commit}" "${asset_dir}" \ + > "${tmp_dir}/result.log" 2>&1 || status=$? +[[ "${status}" -ne 0 ]] || die 'late output symlink was accepted' +[[ "$(snapshot_files)" == "${before}" ]] || die 'late output symlink leaked staging data or changed external files' +[[ -L "${output}" && "$(readlink "${output}")" == "${outside}" ]] || die 'late output symlink was replaced' +grep -Fq 'checksum output path already exists' "${tmp_dir}/result.log" || die 'late output symlink lacks diagnostic' +printf 'PASS: late output symlink rejected and private staging file cleaned up\n' + +# The mktemp template is not a predictable staging filename. A pre-existing +# template-shaped symlink must remain untouched while a fresh manifest works. +prepare unique-temporary-file +ln -s -- "${outside}/keep.txt" "${asset_dir}/.mem-mcp-checksums.XXXXXX" +before="$(sha256sum "${outside}/keep.txt")" +bash "${generator}" v0.1.1 "${commit}" "${asset_dir}" >/dev/null +[[ "$(sha256sum "${outside}/keep.txt")" == "${before}" ]] || die 'temporary output overwrote external data' +[[ -L "${asset_dir}/.mem-mcp-checksums.XXXXXX" ]] || die 'temporary symlink was replaced' +[[ -f "${output}" && ! -L "${output}" ]] || die 'fresh manifest was not created' +( + cd -- "${asset_dir}" + sha256sum --check --strict mem-mcp-checksums.txt >/dev/null +) +printf 'PASS: unpredictable temporary output preserves a pre-existing template-shaped symlink\n' diff --git a/scripts/test_release_guards.sh b/scripts/test_release_guards.sh index 4366a82..236cdd2 100755 --- a/scripts/test_release_guards.sh +++ b/scripts/test_release_guards.sh @@ -27,6 +27,7 @@ verify_manifest() { ( cd -- "${directory}" sha256sum --check --strict mem-mcp-checksums.txt + sha256sum --check --strict mem-checksums.txt ) } @@ -66,7 +67,7 @@ source_validator_line="$( die "release workflow must contain exactly one Release creation call" [[ "$(grep -Fc -- 'gh release edit' "${release_workflow}" || true)" == 1 ]] || die "release workflow must contain exactly one Release publication call" -grep -Fq -- 'needs: [preflight, build]' "${release_workflow}" || +grep -Fq -- 'needs: [preflight, build-mcp, build-server]' "${release_workflow}" || die "Release creation must depend on preflight and every build" grep -Fq -- '--verify-tag' "${release_workflow}" || die "Release creation must refuse an absent remote tag" @@ -74,8 +75,8 @@ grep -Eq -- '^[[:space:]]+--draft([[:space:]]|$)' "${release_workflow}" || die "Release assets must first upload to a draft" grep -Fq -- '--draft=false' "${release_workflow}" || die "the verified draft must be published explicitly" -grep -Fq -- '(.assets | length == 7)' "${release_workflow}" || - die "remote draft validation must require exactly seven assets" +grep -Fq -- '(.assets | length == 24)' "${release_workflow}" || + die "remote draft validation must require exactly 24 assets" grep -Fq -- '(.size > 0)' "${release_workflow}" || die "remote draft validation must reject empty assets" @@ -99,6 +100,51 @@ fi expect_failure "version mismatch" \ "${repo_root}/scripts/validate_release_version.sh" 999.999.999 +# Keep CHANGELOG mutations in a fixture tree. All other version surfaces remain +# the real checkout, so failures below must reach the comparison-link guard. +version_fixture="${tmp_dir}/version fixture" +mkdir -p -- "${version_fixture}/scripts" +cp -- "${repo_root}/scripts/validate_release_version.sh" "${version_fixture}/scripts/" +for surface in npm server worker web deploy docs; do + ln -s -- "${repo_root}/${surface}" "${version_fixture}/${surface}" +done +fixture_validator="${version_fixture}/scripts/validate_release_version.sh" +previous_version=0.0.1 + +write_changelog_fixture() { + local link="$1" + local include_previous="${2:-yes}" + { + printf '## [Unreleased]\n\n## [%s] - 2026-01-01\n\n' "${current_version}" + if [[ "${include_previous}" == yes ]]; then + printf '## [%s] - 2025-01-01\n\n' "${previous_version}" + fi + printf '[Unreleased]: https://github.com/bytefolk/mem/compare/v%s...HEAD\n' "${current_version}" + printf '[%s]: https://github.com/bytefolk/mem/%s\n' "${current_version}" "${link}" + } > "${version_fixture}/CHANGELOG.md" +} + +correct_compare="compare/v${previous_version}...${current_tag}" +write_changelog_fixture "${correct_compare}" +"${fixture_validator}" "${current_version}" >/dev/null +for wrong_base in v0.0.0 "${current_tag}" arbitrary; do + write_changelog_fixture "compare/${wrong_base}...${current_tag}" + expect_failure "wrong compare base ${wrong_base}" "${fixture_validator}" "${current_version}" +done +write_changelog_fixture "compare/v${previous_version}...v999.999.999" +expect_failure "wrong compare endpoint" "${fixture_validator}" "${current_version}" +write_changelog_fixture "${correct_compare}/extra" +expect_failure "compare link suffix" "${fixture_validator}" "${current_version}" +write_changelog_fixture "${correct_compare}" no +expect_failure "missing compare predecessor" "${fixture_validator}" "${current_version}" +write_changelog_fixture "releases/tag/${current_tag}" no +"${fixture_validator}" "${current_version}" >/dev/null +# A tag link is only legitimate when no release precedes this one. Once a +# predecessor exists, pointing at the tag page must not satisfy the guard. +write_changelog_fixture "releases/tag/${current_tag}" +expect_failure "tag link replaces the predecessor" "${fixture_validator}" "${current_version}" +printf 'PASS: compare links require the exact predecessor and endpoint; tag links are valid only without a predecessor\n' + notes_file="${tmp_dir}/release-notes.md" "${repo_root}/scripts/render_release_notes.sh" "${current_tag}" > "${notes_file}" [[ -s "${notes_file}" ]] || die "release notes are empty" @@ -190,9 +236,36 @@ expect_failure "annotated tag version mismatch" env \ FAKE_HEAD_COMMIT="${same_commit}" \ "${repo_root}/scripts/validate_release_source.sh" v999.999.999 -asset_dir="${tmp_dir}/assets" +asset_dir="${tmp_dir}/assets with spaces" mkdir -p -- "${asset_dir}" +# An empty set must fail with the intended diagnostic, including on Bash 3.2. +if "${repo_root}/scripts/generate_release_checksums.sh" \ + "${current_tag}" "${same_commit}" "${asset_dir}" > "${tmp_dir}/empty-assets.log" 2>&1; then + die "empty asset directory: command unexpectedly succeeded" +fi +grep -Fq -- 'actual: ' "${tmp_dir}/empty-assets.log" || + die "empty asset directory must report the missing set" +[[ ! -e "${asset_dir}/mem-mcp-checksums.txt" ]] || + die "empty asset directory must not produce a manifest" +[[ ! -e "${asset_dir}/mem-checksums.txt" ]] || + die "empty asset directory must not produce a server manifest" assets=( + memd-darwin-amd64 + memd-darwin-arm64 + memd-linux-amd64 + memd-linux-arm64 + mem-migrate-darwin-amd64 + mem-migrate-darwin-arm64 + mem-migrate-linux-amd64 + mem-migrate-linux-arm64 + mem-healthcheck-darwin-amd64 + mem-healthcheck-darwin-arm64 + mem-healthcheck-linux-amd64 + mem-healthcheck-linux-arm64 + mem-darwin-amd64 + mem-darwin-arm64 + mem-linux-amd64 + mem-linux-arm64 mem-mcp-darwin-amd64 mem-mcp-darwin-arm64 mem-mcp-linux-amd64 @@ -200,23 +273,44 @@ assets=( mem-mcp-windows-amd64.exe mem-mcp-windows-arm64.exe ) +empty_asset_dir="${tmp_dir}/empty assets" +mkdir -p -- "${empty_asset_dir}" +if empty_error="$("${repo_root}/scripts/generate_release_checksums.sh" \ + "${current_tag}" "${same_commit}" "${empty_asset_dir}" 2>&1)"; then + die "empty asset directory unexpectedly succeeded" +fi +[[ "${empty_error}" == *'release assets differ from the exact expected set'* ]] || + die "empty assets must fail explicitly, not with a Bash 3.2 unbound array error" for asset in "${assets[@]}"; do printf 'test payload for %s\n' "${asset}" > "${asset_dir}/${asset}" done +# Use the real find, basename and sha256sum here. Unlike the Bash-only compat +# suite, this must also catch GNU basename rejecting batched find -exec paths. "${repo_root}/scripts/generate_release_checksums.sh" \ "${current_tag}" "${same_commit}" "${asset_dir}" >/dev/null -manifest="${asset_dir}/mem-mcp-checksums.txt" -[[ "$(wc -l < "${manifest}")" == 6 ]] || die "checksum manifest must have six rows" +mcp_manifest="${asset_dir}/mem-mcp-checksums.txt" +server_manifest="${asset_dir}/mem-checksums.txt" +# BSD wc pads its count with blanks, so a line count must not come from wc -l. +[[ "$(grep -c '' "${mcp_manifest}")" == 6 ]] || die "mcp checksum manifest must have six rows" +[[ "$(grep -c '' "${server_manifest}")" == 16 ]] || die "server checksum manifest must have 16 rows" +actual_manifest_names="$( + sed -E 's/^[0-9a-f]{64} //' "${mcp_manifest}" "${server_manifest}" | + LC_ALL=C sort +)" +expected_manifest_names="$(printf '%s\n' "${assets[@]}" | LC_ALL=C sort)" +[[ "${actual_manifest_names}" == "${expected_manifest_names}" ]] || + die "portable asset enumeration lost or combined a basename" ( cd -- "${asset_dir}" - sha256sum --check --strict "$(basename -- "${manifest}")" >/dev/null + sha256sum --check --strict "$(basename -- "${mcp_manifest}")" >/dev/null + sha256sum --check --strict "$(basename -- "${server_manifest}")" >/dev/null ) printf 'tampered\n' >> "${asset_dir}/${assets[0]}" expect_failure "tampered asset" verify_manifest "${asset_dir}" -rm -f -- "${manifest}" "${asset_dir}/${assets[0]}" +rm -f -- "${mcp_manifest}" "${server_manifest}" "${asset_dir}/${assets[0]}" expect_failure "missing asset" \ "${repo_root}/scripts/generate_release_checksums.sh" \ "${current_tag}" "${same_commit}" "${asset_dir}" @@ -239,4 +333,7 @@ expect_failure "symlink asset" \ "${repo_root}/scripts/generate_release_checksums.sh" \ "${current_tag}" "${same_commit}" "${asset_dir}" +bash "${repo_root}/scripts/test_release_checksum_output_safety.sh" printf 'PASS: release source, notes, asset-set and checksum guards fail closed\n' + +node --test "${repo_root}/scripts/npm-release.test.mjs" diff --git a/scripts/test_release_helpers_compat.sh b/scripts/test_release_helpers_compat.sh index 0757d1f..0cb8c80 100755 --- a/scripts/test_release_helpers_compat.sh +++ b/scripts/test_release_helpers_compat.sh @@ -27,9 +27,25 @@ if ! ( exit 1 fi -asset_dir="${tmp_dir}/assets" +asset_dir="${tmp_dir}/assets with spaces" mkdir -p -- "${asset_dir}" assets=( + memd-darwin-amd64 + memd-darwin-arm64 + memd-linux-amd64 + memd-linux-arm64 + mem-migrate-darwin-amd64 + mem-migrate-darwin-arm64 + mem-migrate-linux-amd64 + mem-migrate-linux-arm64 + mem-healthcheck-darwin-amd64 + mem-healthcheck-darwin-arm64 + mem-healthcheck-linux-amd64 + mem-healthcheck-linux-arm64 + mem-darwin-amd64 + mem-darwin-arm64 + mem-linux-amd64 + mem-linux-arm64 mem-mcp-darwin-amd64 mem-mcp-darwin-arm64 mem-mcp-linux-amd64 @@ -75,6 +91,15 @@ if ! ( exit 1 fi -[[ "$(wc -l < "${asset_dir}/mem-mcp-checksums.txt")" == 6 ]] +# This suite is the one that claims independence from GNU find and coreutils, +# so it must not count lines with wc -l: BSD wc pads the count with blanks. +[[ "$(grep -c '' "${asset_dir}/mem-mcp-checksums.txt")" == 6 ]] || { + printf 'ERROR: mcp checksum manifest must have six rows\n' >&2 + exit 1 +} +[[ "$(grep -c '' "${asset_dir}/mem-checksums.txt")" == 16 ]] || { + printf 'ERROR: server checksum manifest must have 16 rows\n' >&2 + exit 1 +} printf 'PASS: release version and checksum helpers run without Bash 4-only collection builtins\n' diff --git a/scripts/test_win_audit_verify.mjs b/scripts/test_win_audit_verify.mjs new file mode 100644 index 0000000..0b7b5ae --- /dev/null +++ b/scripts/test_win_audit_verify.mjs @@ -0,0 +1,55 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +assert.equal(process.platform, "win32", "Run this process regression in the Windows CI job"); +const source = readFileSync(new URL("./win-audit-verify.bat", import.meta.url), "utf8"); + +function invoke(script, status) { + const directory = mkdtempSync(join(tmpdir(), "mem audit evidence ")); + try { + writeFileSync(join(directory, "verify.bat"), script); + // cmd.exe resolves the real .cmd fixture from its working directory. + // No registry, installed package, or user npm configuration is involved. + writeFileSync(join(directory, "npm.cmd"), + `@echo off\r\necho fixture npm audit status: ${status}\r\nexit /b ${status}\r\n`); + const result = spawnSync(process.env.ComSpec || "cmd.exe", ["/d", "/c", "verify.bat"], { + cwd: directory, + encoding: "utf8", + timeout: 15_000, + env: { ...process.env, AUDIT_RC: "" }, + }); + assert.ifError(result.error); + assert.match(result.stdout, new RegExp(`fixture npm audit status: ${status}`)); + return result; + } finally { + rmSync(directory, { recursive: true, force: true }); + } +} + +for (const status of [0, 7]) { + test(`prints completed evidence and preserves audit exit ${status}`, () => { + const result = invoke(source, status); + assert.equal(result.status, status); + assert.match(result.stdout, /\[win-audit-verify\] finished at/); + assert.match(result.stdout, new RegExp(`npm run audit exit code: ${status}`)); + }); +} + +test("negative control: omitting CALL loses the post-audit evidence", () => { + const broken = source.replace("call npm run audit", "npm run audit"); + assert.notEqual(broken, source); + const result = invoke(broken, 7); + assert.doesNotMatch(result.stdout, /\[win-audit-verify\] finished at/); +}); + +test("negative control: separate ENDLOCAL loses a nonzero saved exit status", () => { + const broken = source.replace("endlocal & exit /b %AUDIT_RC%", "endlocal\r\nexit /b %AUDIT_RC%"); + assert.notEqual(broken, source); + const result = invoke(broken, 7); + assert.match(result.stdout, /npm run audit exit code: 7/); + assert.equal(result.status, 0); +}); diff --git a/scripts/validate_release_version.sh b/scripts/validate_release_version.sh index b376c58..766664a 100755 --- a/scripts/validate_release_version.sh +++ b/scripts/validate_release_version.sh @@ -38,7 +38,7 @@ fi require_exact_line npm/package.json " \"version\": \"${version}\"," require_exact_line npm/server.json " \"version\": \"${version}\"," require_exact_line server/cmd/mem-mcp/main.go \ - $'\t\"version\": \"'"${version}"$'\", // synced with npm/@fullstack-ai-infra/mem-mcp version' + $'\t\"version\": \"'"${version}"$'\", // synced with npm/@bytefolk/mem-mcp version' require_exact_line worker/pyproject.toml "version = \"${version}\"" require_exact_line worker/mem_worker/__init__.py "__version__ = \"${version}\"" @@ -85,7 +85,7 @@ heading_count="$(grep -Fc -- "## [${version}] - " "${changelog}" || true)" [[ "${heading_count}" == 1 ]] || die "CHANGELOG.md: expected exactly one ${version} release heading" require_exact_line CHANGELOG.md \ - "[Unreleased]: https://github.com/fullstack-ai-infra/mem/compare/v${version}...HEAD" + "[Unreleased]: https://github.com/bytefolk/mem/compare/v${version}...HEAD" version_links=() while IFS= read -r version_link; do @@ -93,11 +93,33 @@ while IFS= read -r version_link; do done < <(grep -F -- "[${version}]: " "${changelog}" || true) [[ "${#version_links[@]}" == 1 ]] || die "CHANGELOG.md: expected exactly one [${version}] comparison link" -if [[ "${version_links[0]}" != \ - "[${version}]: https://github.com/fullstack-ai-infra/mem/releases/tag/v${version}" && - "${version_links[0]}" != \ - "[${version}]: https://github.com/fullstack-ai-infra/mem/compare/"*"...v${version}" ]]; then - die "CHANGELOG.md: [${version}] link must terminate at v${version}" +compare_base="$(awk ' + /^## \[[0-9]/ { + if (seen++) { + gsub(/^## \[/, "", $0) + gsub(/\].*/, "", $0) + print $0 + exit + } + } +' "${changelog}")" + +# Exactly one link form is correct, decided by whether a release precedes this +# one: with a predecessor the link must start at that release; without one (a +# first release such as 0.1.0) there is nothing to compare from, so the release +# keeps its tag link. Accepting the tag form in both cases would let a later +# release dodge the predecessor requirement. +if [[ -n "${compare_base}" ]]; then + expected_link="[${version}]: https://github.com/bytefolk/mem/compare/v${compare_base}...v${version}" +else + expected_link="[${version}]: https://github.com/bytefolk/mem/releases/tag/v${version}" +fi + +if [[ "${version_links[0]}" != "${expected_link}" ]]; then + if [[ -n "${compare_base}" ]]; then + die "CHANGELOG.md: [${version}] link must start at the preceding release v${compare_base}:"$'\n'" ${expected_link}" + fi + die "CHANGELOG.md: nothing precedes [${version}], so its link must be exactly:"$'\n'" ${expected_link}" fi printf 'PASS: all release version surfaces match %s\n' "${version}" diff --git a/scripts/verify.sh b/scripts/verify.sh index e1aa3c8..0c74b8f 100755 --- a/scripts/verify.sh +++ b/scripts/verify.sh @@ -4,7 +4,7 @@ set -euo pipefail REPO_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" MODE="${1:-unit}" -EXPECTED_MIGRATION_HEAD=23 +EXPECTED_MIGRATION_HEAD=25 MIGRATION_ROLLBACK_TARGET=11 MODEL_TEXT_CANONICAL_BASE=15 WORKSPACE_AI_PROFILE_BASE=16 @@ -117,6 +117,11 @@ run_web() { (cd "${REPO_ROOT}/web" && npm run test:transfer) } +run_web_headers() { + log "Web reverse-proxy security headers" + "${REPO_ROOT}/scripts/test_nginx_security_headers.sh" +} + validate_test_database() { [[ -n "${MEM_TEST_DB:-}" ]] \ || die "MEM_TEST_DB is required; run: make test-env-up" @@ -299,6 +304,16 @@ run_migration_round_trip() { MEM_TEST_TARGET_DB="$MEM_TEST_DB" testdb assert-unsafe-derived-text-scrubbed } +run_migration_sequence() { + log "Strict populated migration upgrades from released head 23 to $EXPECTED_MIGRATION_HEAD" + ( + cd "${REPO_ROOT}/server" + MEM_MIGRATION_SEQUENCE_TEST_DB="$MEM_TEST_DB" \ + go test -count=1 -v ./internal/db -run '^TestMigrationUpgradeSequence$' + ) + assert_migration_version "$EXPECTED_MIGRATION_HEAD" +} + run_migrations_up() { ( cd "${REPO_ROOT}/server" @@ -334,6 +349,7 @@ run_postgres_tests() { TestManagedAISettlementOutboxPostgres TestReleasedFileStageRetryPostgres TestDurableContextPostgres + TestTextANNFileSemanticsPostgres ) integration_log="$(mktemp "${TMPDIR:-/tmp}/mem-integration.XXXXXX")" @@ -344,7 +360,7 @@ run_postgres_tests() { MEM_TEST_DB="$MEM_TEST_DB" go test \ ${race_flag:+"$race_flag"} \ -v -count=1 -p 1 -timeout 20m \ - -run '^(TestMemoryPostgres|TestHandoffPostgres|TestWorkspaceTransferPostgres|TestWorkspaceTransferMergeConservativePostgres|TestHandoffCrossAgentHTTPIntegration|TestRelocateHTTPPostgres|TestMemoryPathLifecycleIntegration|TestWorkspacePathLockingIntegration|TestFilePathLockingIntegration|TestAnnotationDecisionIntegration|TestIndexerEnrichmentIntegration|TestRecomputePerson|TestManagedEmbeddingEntitlementPostgres|TestManagedSearchReplayPostgres|TestManagedEmbeddingHTTPAuthorizationPostgres|TestAIProfilePostgres|TestIndexGenerationPostgres|TestManagedAISettlementOutboxPostgres|TestReleasedFileStageRetryPostgres|TestDurableContextPostgres)$' \ + -run '^(TestMemoryPostgres|TestHandoffPostgres|TestWorkspaceTransferPostgres|TestWorkspaceTransferMergeConservativePostgres|TestHandoffCrossAgentHTTPIntegration|TestRelocateHTTPPostgres|TestMemoryPathLifecycleIntegration|TestWorkspacePathLockingIntegration|TestFilePathLockingIntegration|TestAnnotationDecisionIntegration|TestIndexerEnrichmentIntegration|TestRecomputePerson|TestManagedEmbeddingEntitlementPostgres|TestManagedSearchReplayPostgres|TestManagedEmbeddingHTTPAuthorizationPostgres|TestAIProfilePostgres|TestIndexGenerationPostgres|TestManagedAISettlementOutboxPostgres|TestReleasedFileStageRetryPostgres|TestDurableContextPostgres|TestTextANNFileSemanticsPostgres)$' \ ./internal/memory \ ./internal/handoff \ ./internal/workspacetransfer \ @@ -380,10 +396,22 @@ run_postgres_tests() { run_integration() { validate_test_database + with_fresh_test_database migration_sequence run_migration_sequence with_fresh_test_database migration run_migration_round_trip + with_fresh_test_database hnsw_migration run_hnsw_migration with_fresh_test_database integration run_postgres_integration } +run_hnsw_migration() { + log "Populated HNSW migration, rollback, ingest, dimension rejection and EXPLAIN" + ( + cd "${REPO_ROOT}/server" + MEM_HNSW_TEST_DB="$MEM_TEST_DB" go test -v -count=1 \ + -run '^TestHNSWMigrationPostgres$' ./internal/db + ) + log "Planner EXPLAIN is recorded by TestHNSWMigrationPostgres (psql URI script is manual)" +} + run_integration_race() { validate_test_database with_fresh_test_database integration_race run_postgres_integration_race @@ -423,6 +451,7 @@ case "$MODE" in run_server run_worker run_web + run_web_headers ;; race) run_race ;; integration) run_integration ;; @@ -431,6 +460,7 @@ case "$MODE" in run_server run_worker run_web + run_web_headers run_race run_integration run_integration_race diff --git a/scripts/verify_hnsw_indexes.sh b/scripts/verify_hnsw_indexes.sh new file mode 100755 index 0000000..0eb0cac --- /dev/null +++ b/scripts/verify_hnsw_indexes.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Read-only planner verification for shipping text and visual query shapes. +# Requires a populated disposable database; does not force planner settings. +set -euo pipefail +trap 'echo "ERROR: HNSW verification aborted on an execution error; assertions are incomplete" >&2' ERR +DB_URL="${1:?Usage: $0 }" +CORPUS_USER="${2:?Supply the user UUID that owns the populated corpus}" +[[ "$CORPUS_USER" =~ ^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$ ]] || { + echo 'ERROR: corpus user must be a UUID' >&2; exit 1; +} +sql() { psql -X -A -t -v ON_ERROR_STOP=1 "$DB_URL" -c "$1"; } +db_name="$(sql 'SELECT current_database()')" +[[ "$db_name" == *_test ]] || { echo 'ERROR: database must end in _test' >&2; exit 1; } +pass=0 +fail=0 +for kind in text visual face; do + index="idx_embeddings_${kind}_embedding_hnsw" + valid="$(sql "SELECT count(*) FROM pg_index i + JOIN pg_class c ON c.oid = i.indexrelid JOIN pg_am a ON a.oid = c.relam + WHERE i.indrelid = 'embeddings_${kind}'::regclass + AND c.relname = '${index}' AND a.amname = 'hnsw' AND i.indisvalid + AND pg_get_indexdef(i.indexrelid) LIKE '%vector_cosine_ops%'")" + rows="$(sql "SELECT count(*) FROM embeddings_${kind} e JOIN files f ON f.id=e.file_id + WHERE f.user_id='${CORPUS_USER}'::uuid AND e.embedding IS NOT NULL")" + if [[ "$valid" == 1 && "$rows" -gt 0 ]]; then + echo "PASS: ${index} is valid; corpus contains ${rows} non-null vectors" + pass=$((pass + 1)) + else + echo "FAIL: ${index}: valid=${valid}, corpus vectors=${rows}" + fail=$((fail + 1)) + fi +done +assert_plan() { + local route="$1" query="$2" plan + plan="$(sql "EXPLAIN (ANALYZE, BUFFERS) ${query}")" + echo "$plan" + if grep -q "idx_embeddings_${route}_embedding_hnsw" <<<"$plan"; then + echo "PASS: shipping ${route} query uses HNSW" + pass=$((pass + 1)) + else + echo "FAIL: shipping ${route} query does not use HNSW" + fail=$((fail + 1)) + fi +} +# Match queryTextDistanceOrder: ANN CTE then file join. DISTINCT ON is fallback. +assert_plan text "WITH nearest AS ( + SELECT e.id, e.file_id, e.embedding <=> array_fill(0.1::real, ARRAY[768])::vector AS dist + FROM embeddings_text e + ORDER BY e.embedding <=> array_fill(0.1::real, ARRAY[768])::vector ASC + LIMIT 10 +) +SELECT e.id, f.id FROM nearest e JOIN files f ON f.id=e.file_id + WHERE f.user_id='${CORPUS_USER}'::uuid ORDER BY e.dist ASC" +assert_plan visual "SELECT e.file_id + FROM embeddings_visual e + ORDER BY e.embedding <=> array_fill(0.1::real, ARRAY[512])::vector ASC LIMIT 10" +echo "Results: ${pass} passed, ${fail} failed" +[[ "$fail" -eq 0 ]] diff --git a/scripts/win-audit-verify.bat b/scripts/win-audit-verify.bat new file mode 100644 index 0000000..e899083 --- /dev/null +++ b/scripts/win-audit-verify.bat @@ -0,0 +1,41 @@ +@echo off +REM win-audit-verify.bat — Verify audit-retry.mjs works on real Windows. +REM Must be run from a real Windows terminal (cmd.exe), NOT WSL. +REM Captures: commit, npm_execpath, full audit output, exit code. + +setlocal enabledelayedexpansion + +echo === win-audit-verify === +echo. + +REM 1. Show which commit is being tested +echo --- git head --- +git rev-parse HEAD +git log -1 --format=^"%%h %%s^" +echo. + +REM 2. Show Node version +echo --- node --- +node --version +echo. + +REM 3. Prove npm_execpath is set by npm run, and that a direct node.exe +REM invocation does NOT have it (the failure mode from the review). +echo --- npm_execpath probe via node -e (should be EMPTY) --- +node -e "console.log('npm_execpath=' + (process.env.npm_execpath || '(unset)'))" +echo. + +REM 4. Run the actual audit via npm run — this is the path that supplies npm_execpath. +echo --- npm run audit (full output) --- +echo [win-audit-verify] starting npm run audit at %DATE% %TIME% +call npm run audit +set AUDIT_RC=!ERRORLEVEL! +echo [win-audit-verify] finished at %DATE% %TIME% +echo. + +REM 5. Report exit code +echo --- result --- +echo [win-audit-verify] npm run audit exit code: !AUDIT_RC! +echo. + +endlocal & exit /b %AUDIT_RC% diff --git a/server/Dockerfile b/server/Dockerfile index 41a3ed7..2cf902b 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -9,7 +9,13 @@ ARG GOPROXY=https://proxy.golang.org,direct RUN GOPROXY="${GOPROXY}" go mod download COPY . . ARG VERSION=dev -RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w -X github.com/PeterGuy326/mem/server/internal/api.Version=${VERSION}" \ +ARG REVISION=unknown +ARG CONTRACT_VERSION=durable-context.v1 +RUN CGO_ENABLED=0 go build -trimpath \ + -ldflags="-s -w \ + -X github.com/PeterGuy326/mem/server/internal/api.Version=${VERSION} \ + -X github.com/PeterGuy326/mem/server/internal/api.Revision=${REVISION} \ + -X github.com/PeterGuy326/mem/server/internal/api.ContractVersion=${CONTRACT_VERSION}" \ -o /out/memd ./cmd/memd && \ CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" \ -o /out/mem-migrate ./cmd/mem-migrate && \ diff --git a/server/cmd/mem-mcp/main.go b/server/cmd/mem-mcp/main.go index e976d30..0d7d886 100644 --- a/server/cmd/mem-mcp/main.go +++ b/server/cmd/mem-mcp/main.go @@ -50,7 +50,7 @@ const protocolVersion = "2024-11-05" // serverInfo is what we report back in the `initialize` handshake. var serverInfo = map[string]any{ "name": "mem-mcp", - "version": "0.1.1", // synced with npm/@fullstack-ai-infra/mem-mcp version + "version": "0.1.2", // synced with npm/@bytefolk/mem-mcp version } func main() { diff --git a/server/cmd/mem-mcp/server_test.go b/server/cmd/mem-mcp/server_test.go index ba317b0..8ca3146 100644 --- a/server/cmd/mem-mcp/server_test.go +++ b/server/cmd/mem-mcp/server_test.go @@ -162,6 +162,74 @@ func TestMCP_ToolsCallRoundTrip(t *testing.T) { } } +func TestMCP_LexicalSearchSchemaAndForwarding(t *testing.T) { + requests := make(chan map[string]any, 1) + fake := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/v1/search" { + t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) + } + var body map[string]any + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Errorf("decode search: %v", err) + } + requests <- body + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"results":[]}`)) + })) + defer fake.Close() + reg := tools.New() + if err := builtin.RegisterAll(reg, apiclient.New(fake.URL, "test-token")); err != nil { + t.Fatal(err) + } + srv, buf := newTestServer(reg) + in := strings.NewReader(`{"jsonrpc":"2.0","id":1,"method":"tools/list"}` + "\n" + + `{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"mem_search","arguments":{"query":"budget","route":"lexical","scope":"/Work","type":"text","limit":3}}}` + "\n") + if err := srv.serve(in); err != nil { + t.Fatal(err) + } + responses := readResponses(t, buf) + if len(responses) != 2 { + t.Fatalf("want list and call responses, got %d", len(responses)) + } + t.Run("exported schema advertises all routes", func(t *testing.T) { + for _, item := range responses[0]["result"].(map[string]any)["tools"].([]any) { + tool := item.(map[string]any) + if tool["name"] != "mem_search" { + continue + } + schema := tool["inputSchema"].(map[string]any) + route := schema["properties"].(map[string]any)["route"].(map[string]any) + enum := route["enum"].([]any) + want := map[string]bool{"text": true, "visual": true, "auto": true, "lexical": true} + for _, value := range enum { + delete(want, value.(string)) + } + if len(enum) != 4 || len(want) != 0 { + t.Fatalf("mem_search route enum = %v; missing %v", enum, want) + } + return + } + t.Fatal("mem_search missing from tools/list") + }) + t.Run("lexical call forwards route and filters", func(t *testing.T) { + if responses[1]["error"] != nil { + t.Fatalf("RPC error: %v", responses[1]["error"]) + } + if result := responses[1]["result"].(map[string]any); result["isError"] != false { + t.Fatalf("tool error: %v", result) + } + select { + case body := <-requests: + if body["query"] != "budget" || body["route"] != "lexical" || + body["scope"] != "/Work" || body["type"] != "text" || body["limit"] != float64(3) { + t.Fatalf("forwarded search = %#v", body) + } + default: + t.Fatal("lexical request was not forwarded") + } + }) +} + func TestMCP_ToolErrorSurfacedInContent(t *testing.T) { // memd returns 404 fake := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { diff --git a/server/cmd/mem/client.go b/server/cmd/mem/client.go index cda2fd4..ee8625c 100644 --- a/server/cmd/mem/client.go +++ b/server/cmd/mem/client.go @@ -24,6 +24,25 @@ func newCliError(code int, msg, hint string) *cliError { return &cliError{code: code, msg: msg, hint: hint} } +// notLoggedInHint is the credential guidance that always applies. +const notLoggedInHint = "run `mem auth login` first" + +// firstRunDeployHint names the documented deployment path rather than a +// host-specific install recipe, so a machine that has never been configured is +// not sent off to build the bare-metal stack by hand. +const firstRunDeployHint = "no server configured yet — the documented path is deploy/compose, see docs/DEPLOYMENT.md" + +// errNotLoggedIn is the one fail-closed auth error for commands that need a +// credential. When no config file exists at all, the run is a first run: the +// hint additionally names the documented deployment path, because telling +// somebody to log in against a server that does not exist yet is not guidance. +func errNotLoggedIn() error { + if configFileExists() { + return newCliError(3, "not logged in", notLoggedInHint) + } + return newCliError(3, "not logged in", notLoggedInHint+"; "+firstRunDeployHint) +} + // fromAPIError maps an *apiclient.APIError to a *cliError with the SPEC §7.1 // exit code. Any other error is returned unchanged. func fromAPIError(err error) error { diff --git a/server/cmd/mem/cmds_auth.go b/server/cmd/mem/cmds_auth.go index 8631a17..5377d02 100644 --- a/server/cmd/mem/cmds_auth.go +++ b/server/cmd/mem/cmds_auth.go @@ -132,7 +132,7 @@ func newAuthStatusCmd() *cobra.Command { return err } if cfg.Token == "" { - return newCliError(3, "not logged in", "run `mem auth login` first") + return errNotLoggedIn() } var capabilities struct { @@ -241,7 +241,7 @@ func newTokenCreateCmd() *cobra.Command { return err } if cfg.Token == "" { - return newCliError(3, "not logged in", "run `mem auth login` first") + return errNotLoggedIn() } scopeList := splitCommas(scopes) body := map[string]any{ diff --git a/server/cmd/mem/cmds_auth_test.go b/server/cmd/mem/cmds_auth_test.go index 551cf49..46a3865 100644 --- a/server/cmd/mem/cmds_auth_test.go +++ b/server/cmd/mem/cmds_auth_test.go @@ -162,8 +162,17 @@ func TestAuthStatusWithoutTokenReturnsAuthExitCode(t *testing.T) { if !errors.As(err, &cliErr) { t.Fatalf("error type = %T, want *cliError", err) } - if cliErr.code != 3 || cliErr.hint != "run `mem auth login` first" { - t.Fatalf("cli error = %#v", cliErr) + // #112 REQ-002 changed this hint's text for a host with no config file at + // all, so the old exact-equality assertion is intentionally widened: the + // login step must stay, and the documented deployment path must now appear. + if cliErr.code != 3 { + t.Fatalf("cli error code = %d, want 3 (%#v)", cliErr.code, cliErr) + } + if !strings.HasPrefix(cliErr.hint, "run `mem auth login` first") { + t.Errorf("hint = %q, want it to keep the login step", cliErr.hint) + } + if !strings.Contains(cliErr.hint, "deploy/compose") || !strings.Contains(cliErr.hint, "docs/DEPLOYMENT.md") { + t.Errorf("hint = %q, want first-run guidance naming the documented path", cliErr.hint) } } diff --git a/server/cmd/mem/cmds_context.go b/server/cmd/mem/cmds_context.go index 45eaaa7..02cc2ce 100644 --- a/server/cmd/mem/cmds_context.go +++ b/server/cmd/mem/cmds_context.go @@ -82,7 +82,7 @@ Examples: return err } if cfg.Token == "" { - return newCliError(3, "not logged in", "run `mem auth login` first") + return errNotLoggedIn() } body := map[string]any{"query": strings.Join(args, " ")} if scope != "" { diff --git a/server/cmd/mem/cmds_doctor.go b/server/cmd/mem/cmds_doctor.go new file mode 100644 index 0000000..e7f1566 --- /dev/null +++ b/server/cmd/mem/cmds_doctor.go @@ -0,0 +1,377 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "os" + "strings" + "time" + + "github.com/PeterGuy326/mem/server/internal/apiclient" + "github.com/PeterGuy326/mem/server/internal/redact" + "github.com/spf13/cobra" +) + +// mem doctor is a read-only diagnosis surface (issue #112). It exists because a +// first-time user's most common failure is a missing prerequisite they cannot +// name: nothing listening at the configured URL, no credential, no workspace. +// Before this command the only signal was a per-command error. +// +// Contract, in the order the checks run: +// +// server_reachability GET /healthz with no credential +// credential is a token configured at all (no request) +// workspace GET /v1/capabilities +// version_skew GET /v1/version +// +// REQ-003 keeps this strictly diagnostic: every request is a GET, nothing is +// created, no dependency is installed, no Docker or compose command is issued. +// URLs pass through internal/redact on the way out: userinfo is redacted, query +// parameter values are blanked, and a URL that cannot be proven credential-free +// is withheld whole. + +// doctorContract and doctorSchemaVersion follow the repo convention of naming a +// machine-readable surface and versioning it, mirroring docs/schemas. +const ( + doctorContract = "mem.doctor" + doctorSchemaVersion = 1 +) + +// Statuses are a closed set. "skipped" is explicit: a check that could not run +// because an earlier one failed says so, instead of reporting an OK it did not +// earn or a failure it did not observe. +const ( + doctorOK = "ok" + doctorWarn = "warn" + doctorFail = "fail" + doctorSkipped = "skipped" +) + +// exit codes, per SPEC §7.1: 0 ok · 2 not_found · 3 auth · 4 plan/quota · +// 5 provider/timeout. +const ( + exitOK = 0 + exitNotFound = 2 + exitAuth = 3 + exitPlanQuota = 4 + exitProvider = 5 +) + +type doctorCheck struct { + Name string `json:"name"` + Status string `json:"status"` + // ExitCode is this finding's contribution to the process exit status. + // Advisory findings contribute 0. + ExitCode int `json:"exit_code"` + Detail string `json:"detail"` + Hint string `json:"hint,omitempty"` +} + +type doctorReport struct { + Contract string `json:"contract"` + SchemaVersion int `json:"schema_version"` + Server string `json:"server"` + CLIVersion string `json:"cli_version"` + ServerVersion string `json:"server_version,omitempty"` + ExitCode int `json:"exit_code"` + Checks []doctorCheck `json:"checks"` +} + +func newDoctorCmd() *cobra.Command { + var timeout time.Duration + cmd := &cobra.Command{ + Use: "doctor", + Short: "Diagnose local configuration and server connectivity (read-only)", + Long: `Report why the CLI cannot talk to a working mem server. + +doctor issues only GET requests and writes nothing: no token, no file, no +container and no configuration. It checks, in order, reachability of the +configured server URL, whether a credential exists, which workspace the server +resolved for that credential, and CLI/server version skew. Each finding carries +the SPEC §7.1 exit code it contributes (0 ok · 2 not_found · 3 auth · +4 plan/quota · 5 provider/timeout); the process exits with the first failing +check's code. + +A check that an earlier failure made impossible is reported as "skipped" rather +than guessed. + +Example: + mem doctor + mem doctor --format json + mem doctor --server http://localhost:8787 --timeout 2s`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + format, err := rememberOutputFormat(cmd) + if err != nil { + return err + } + report, err := runDoctor(cmd, timeout) + if err != nil { + return err + } + if format == "json" { + enc := json.NewEncoder(cmd.OutOrStdout()) + enc.SetIndent("", " ") + if err := enc.Encode(report); err != nil { + return err + } + } else { + printDoctorReport(cmd, report) + } + if report.ExitCode != exitOK { + f := report.firstFailed() + return newCliError(report.ExitCode, "doctor: "+f.Name+" failed", f.Detail) + } + return nil + }, + } + cmd.Flags().DurationVar(&timeout, "timeout", 5*time.Second, "per-request budget for the read-only probes") + return cmd +} + +func (r doctorReport) firstFailed() doctorCheck { + for _, c := range r.Checks { + if c.Status == doctorFail { + return c + } + } + return doctorCheck{Name: "doctor", Detail: "a check failed"} +} + +func runDoctor(cmd *cobra.Command, timeout time.Duration) (doctorReport, error) { + if timeout <= 0 { + return doctorReport{}, newCliError(1, "--timeout must be positive", "") + } + cfg, err := resolveConfig("") + if err != nil { + return doctorReport{}, err + } + report := doctorReport{ + Contract: doctorContract, + SchemaVersion: doctorSchemaVersion, + Server: redactURL(cfg.Server), + CLIVersion: cliVersion, + } + + reachCtx, cancel := context.WithTimeout(cmd.Context(), timeout) + reach := probeReachability(reachCtx, cfg.Server) + cancel() + report.Checks = append(report.Checks, reach) + + cred := probeCredential(cfg) + report.Checks = append(report.Checks, cred) + + // The remaining checks need a live, authenticated connection. Reporting a + // fabricated result for them would be the exact failure mode this command + // exists to remove. + var ws, ver doctorCheck + switch { + case reach.Status == doctorFail: + ws, ver = skippedCheck("workspace", reach.Name), skippedCheck("version_skew", reach.Name) + case cred.Status == doctorFail: + ws, ver = skippedCheck("workspace", cred.Name), skippedCheck("version_skew", cred.Name) + default: + wsCtx, wsCancel := context.WithTimeout(cmd.Context(), timeout) + ws = probeWorkspace(wsCtx, cfg) + wsCancel() + + verCtx, verCancel := context.WithTimeout(cmd.Context(), timeout) + ver = probeVersion(verCtx, cfg.Server, &report) + verCancel() + } + report.Checks = append(report.Checks, ws, ver) + + for _, c := range report.Checks { + if c.Status == doctorFail { + report.ExitCode = c.ExitCode + break + } + } + return report, nil +} + +// skippedCheck records a check that an earlier failure made impossible, and +// names the blocker so the text report stays actionable without the JSON. +func skippedCheck(name, blockedBy string) doctorCheck { + return doctorCheck{ + Name: name, + Status: doctorSkipped, + Detail: "not evaluated: " + blockedBy + " is failing", + } +} + +func probeReachability(ctx context.Context, server string) doctorCheck { + check := doctorCheck{Name: "server_reachability"} + // An unauthenticated probe: a 401 here would otherwise be read as "the + // server is down" by a user whose only problem is a bad token. + c := apiclient.New(server, "") + var resp struct { + OK bool `json:"ok"` + } + if err := c.DoJSON(ctx, http.MethodGet, "/healthz", nil, &resp); err != nil { + check.Status, check.ExitCode, check.Detail, check.Hint = classifyProbe(err) + return check + } + if !resp.OK { + check.Status = doctorFail + check.ExitCode = exitProvider + check.Detail = "healthz answered without ok:true" + check.Hint = deployPathHint() + return check + } + check.Status = doctorOK + check.Detail = "healthz ok at " + redactURL(server) + return check +} + +func probeCredential(cfg *cliConfig) doctorCheck { + check := doctorCheck{Name: "credential"} + if cfg.Token == "" { + check.Status = doctorFail + check.ExitCode = exitAuth + check.Detail = "no token configured" + check.Hint = notLoggedInHint + if !configFileExists() { + check.Hint += "; " + firstRunDeployHint + } + return check + } + // The value never leaves this function: only its origin is reported. + check.Status = doctorOK + check.Detail = "token present (from " + credentialSource() + ")" + return check +} + +// credentialSource names where the token came from without printing it. +func credentialSource() string { + if strings.TrimSpace(os.Getenv("MEM_TOKEN")) != "" { + return "$MEM_TOKEN" + } + return "config file" +} + +func probeWorkspace(ctx context.Context, cfg *cliConfig) doctorCheck { + check := doctorCheck{Name: "workspace"} + c := apiclient.New(cfg.Server, cfg.Token).WithWorkspace(cfg.Workspace) + var resp struct { + Workspace struct { + ID string `json:"id"` + Name string `json:"name"` + Role string `json:"role"` + } `json:"workspace"` + } + if err := c.DoJSON(ctx, http.MethodGet, "/v1/capabilities", nil, &resp); err != nil { + check.Status, check.ExitCode, check.Detail, check.Hint = classifyProbe(err) + return check + } + if resp.Workspace.ID == "" { + check.Status = doctorFail + check.ExitCode = exitNotFound + check.Detail = "server resolved no workspace for this credential" + check.Hint = "select one with `mem auth login` or --workspace " + return check + } + check.Status = doctorOK + if cfg.Workspace == "" { + check.Detail = fmt.Sprintf( + "server-resolved workspace %s (%s), role %s; none configured locally, using the server default", + resp.Workspace.Name, resp.Workspace.ID, resp.Workspace.Role, + ) + return check + } + check.Detail = fmt.Sprintf("workspace %s (%s), role %s", resp.Workspace.Name, resp.Workspace.ID, resp.Workspace.Role) + return check +} + +func probeVersion(ctx context.Context, server string, report *doctorReport) doctorCheck { + check := doctorCheck{Name: "version_skew"} + c := apiclient.New(server, "") + var resp struct { + Version string `json:"version"` + } + if err := c.DoJSON(ctx, http.MethodGet, "/v1/version", nil, &resp); err != nil { + check.Status, check.ExitCode, check.Detail, check.Hint = classifyProbe(err) + return check + } + report.ServerVersion = resp.Version + switch { + case resp.Version == "": + check.Status = doctorWarn + check.Detail = "server reported no version" + case cliVersion == "" || cliVersion == devCLIVersion: + // Honest limit, not a pass: release builds do not inject a CLI version + // yet, so there is nothing to compare against. + check.Status = doctorWarn + check.Detail = fmt.Sprintf( + "skew not computable: this CLI build reports %q (no version injected at build time); server reports %s", + cliVersion, resp.Version, + ) + check.Hint = "compare `mem version` against the release notes for the images you deployed" + case resp.Version == cliVersion: + check.Status = doctorOK + check.Detail = "CLI and server both report " + cliVersion + default: + check.Status = doctorWarn + check.Detail = fmt.Sprintf("CLI reports %s, server reports %s", cliVersion, resp.Version) + check.Hint = "upgrade the CLI or redeploy the server images so the two agree" + } + return check +} + +// classifyProbe turns a probe failure into the finding fields. The classification +// is shared with no other surface on purpose: ingest has a failure-code +// vocabulary for cycles, while this one maps to SPEC §7.1 process exit codes. +func classifyProbe(err error) (status string, code int, detail, hint string) { + var ae *apiclient.APIError + if errors.As(err, &ae) { + switch ae.Kind() { + case apiclient.KindAuth: + return doctorFail, exitAuth, fmt.Sprintf("server rejected the request (HTTP %d)", ae.StatusCode), notLoggedInHint + case apiclient.KindNotFound: + return doctorFail, exitNotFound, fmt.Sprintf("no mem server at this URL (HTTP %d)", ae.StatusCode), deployPathHint() + case apiclient.KindPlan, apiclient.KindQuota: + return doctorFail, exitPlanQuota, fmt.Sprintf("server refused for plan or quota (HTTP %d)", ae.StatusCode), "" + } + return doctorFail, exitProvider, fmt.Sprintf("server error (HTTP %d): %s", ae.StatusCode, ae.Message), deployPathHint() + } + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { + return doctorFail, exitProvider, "probe timed out", "raise --timeout, or check that the server is not behind a stalled proxy" + } + return doctorFail, exitProvider, "cannot reach the configured server: " + sanitizeProbeError(err), deployPathHint() +} + +// deployPathHint points at the container path the docs recommend, instead of a +// host-specific dependency recipe. +func deployPathHint() string { + return "start the documented container path: deploy/compose, see docs/DEPLOYMENT.md" +} + +// redactURL gates a configured URL on its way into a report an operator +// will paste into an issue. The policy lives in internal/redact so the CLI, the +// API client and memd share one implementation: a URL that can be positively +// proven to be a credential-free transport URL is echoed with its userinfo +// replaced, and one that cannot is withheld whole rather than scrubbed. +func redactURL(raw string) string { + return redact.URL(raw, redact.APIURLs) +} + +// sanitizeProbeError renders a probe failure without letting the configured URL +// out, including the shapes url.Parse reports as neither an error nor userinfo. +func sanitizeProbeError(err error) string { + return redact.TransportError(err, redact.APIURLs) +} + +func printDoctorReport(cmd *cobra.Command, r doctorReport) { + out := cmd.OutOrStdout() + fmt.Fprintf(out, "mem doctor (%s v%d)\n", r.Contract, r.SchemaVersion) + fmt.Fprintf(out, "server: %s\n", r.Server) + for _, c := range r.Checks { + fmt.Fprintf(out, "%-20s %-8s %s\n", c.Name, c.Status, c.Detail) + if c.Hint != "" { + fmt.Fprintf(out, "%-20s hint: %s\n", "", c.Hint) + } + } +} diff --git a/server/cmd/mem/cmds_doctor_test.go b/server/cmd/mem/cmds_doctor_test.go new file mode 100644 index 0000000..b9743db --- /dev/null +++ b/server/cmd/mem/cmds_doctor_test.go @@ -0,0 +1,748 @@ +package main + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "github.com/PeterGuy326/mem/server/internal/redact" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" +) + +// doctorStub answers the three endpoints mem doctor probes, and records every +// request. An unexpected method or path fails the test, which is how AC-002 +// ("performs no write request of any kind") is enforced. +type doctorStub struct { + mu sync.Mutex + requests []string + healthz func(http.ResponseWriter) + caps func(http.ResponseWriter) + version func(http.ResponseWriter) +} + +func newDoctorStub() *doctorStub { + s := &doctorStub{} + s.healthz = func(w http.ResponseWriter) { + _, _ = w.Write([]byte(`{"ok":true}`)) + } + s.caps = func(w http.ResponseWriter) { + _, _ = w.Write([]byte(`{"workspace":{"id":"11111111-1111-1111-1111-111111111111","name":"Personal","role":"owner"}}`)) + } + s.version = func(w http.ResponseWriter) { + _, _ = w.Write([]byte(`{"version":"0.1.0"}`)) + } + return s +} + +func (s *doctorStub) server(t *testing.T) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + raw, _ := io.ReadAll(r.Body) + if len(raw) > 0 { + t.Errorf("%s %s carried a request body: %s", r.Method, r.URL.Path, raw) + } + s.mu.Lock() + s.requests = append(s.requests, r.Method+" "+r.URL.Path) + s.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + switch { + case r.Method == http.MethodGet && r.URL.Path == "/healthz": + s.healthz(w) + case r.Method == http.MethodGet && r.URL.Path == "/v1/capabilities": + s.caps(w) + case r.Method == http.MethodGet && r.URL.Path == "/v1/version": + s.version(w) + default: + t.Errorf("doctor made an unexpected request: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusNotImplemented) + } + })) +} + +func (s *doctorStub) seen() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.requests...) +} + +// configureDoctor points the CLI at server with the given credential state. +// writeConfig controls whether a config file exists on disk at all, which is the +// first-run distinction REQ-002 turns on. +func configureDoctor(t *testing.T, server, token string, writeConfig bool) { + t.Helper() + dir := t.TempDir() + cfgPath := filepath.Join(dir, "config.yaml") + if writeConfig { + if err := os.WriteFile(cfgPath, []byte("server: "+server+"\n"), 0o600); err != nil { + t.Fatal(err) + } + } + t.Setenv("MEM_CONFIG", cfgPath) + t.Setenv("MEM_SERVER", server) + t.Setenv("MEM_WORKSPACE", "") + t.Setenv("MEM_TOKEN", token) +} + +// execDoctor runs `mem doctor` with args and returns what it printed on stdout, +// what it printed on stderr (cobra's own error and usage text), and the error. +// main.go merges the two, but the report and cobra's noise are different surfaces +// and the assertions need to tell them apart. +func execDoctor(t *testing.T, args ...string) (string, string, error) { + t.Helper() + var stdout, stderr bytes.Buffer + root := newRootCmd() + root.SetOut(&stdout) + root.SetErr(&stderr) + root.SetArgs(append([]string{"doctor"}, args...)) + err := root.Execute() + return stdout.String(), stderr.String(), err +} + +func decodeReport(t *testing.T, out string) doctorReport { + t.Helper() + // A failing command's output buffer also carries cobra's own "Error:" and + // usage block: cobra writes them via OutOrStderr, which is this same writer + // when a test routes output into a buffer. In production the report is on + // stdout and cobra's noise is on stderr. Decoding the first JSON value keeps + // the assertion about the report itself. + dec := json.NewDecoder(strings.NewReader(strings.TrimSpace(out))) + var rep doctorReport + if err := dec.Decode(&rep); err != nil { + t.Fatalf("decode doctor json: %v\n%s", err, out) + } + return rep +} + +func checkByName(t *testing.T, rep doctorReport, name string) doctorCheck { + t.Helper() + for _, c := range rep.Checks { + if c.Name == name { + return c + } + } + t.Fatalf("no check named %q in %+v", name, rep.Checks) + return doctorCheck{} +} + +func cliCode(t *testing.T, err error) int { + t.Helper() + var ce *cliError + if !errors.As(err, &ce) { + t.Fatalf("error = %#v, want *cliError", err) + } + return ce.code +} + +func TestDoctorHealthyReportsAllChecksAndExitsZero(t *testing.T) { + stub := newDoctorStub() + srv := stub.server(t) + defer srv.Close() + configureDoctor(t, srv.URL, "secret-token-value", true) + t.Cleanup(func() { cliVersion = devCLIVersion }) + cliVersion = "0.1.0" + + out, _, err := execDoctor(t, "--format", "json") + if err != nil { + t.Fatalf("healthy doctor returned %v\n%s", err, out) + } + rep := decodeReport(t, out) + if rep.ExitCode != exitOK { + t.Errorf("report exit_code = %d, want 0", rep.ExitCode) + } + want := []string{"server_reachability", "credential", "workspace", "version_skew"} + if len(rep.Checks) != len(want) { + t.Fatalf("checks = %d, want %d: %+v", len(rep.Checks), len(want), rep.Checks) + } + for i, name := range want { + if rep.Checks[i].Name != name { + t.Errorf("check %d = %q, want %q", i, rep.Checks[i].Name, name) + } + if rep.Checks[i].Status != doctorOK { + t.Errorf("%s status = %q (%s), want ok", name, rep.Checks[i].Status, rep.Checks[i].Detail) + } + } + // AC-002: exactly the three read probes, in order. + if got, wantReq := stub.seen(), []string{"GET /healthz", "GET /v1/capabilities", "GET /v1/version"}; strings.Join(got, ",") != strings.Join(wantReq, ",") { + t.Errorf("requests = %v, want %v", got, wantReq) + } + if strings.Contains(out, "secret-token-value") { + t.Errorf("report leaked the token value:\n%s", out) + } +} + +func TestDoctorUnreachableServer(t *testing.T) { + closed := httptest.NewServer(nil) + addr := closed.URL + closed.Close() + configureDoctor(t, addr, "tok", true) + + out, _, err := execDoctor(t, "--format", "json") + if err == nil { + t.Fatalf("doctor should exit non-zero for an unreachable server\n%s", out) + } + if code := cliCode(t, err); code != exitProvider { + t.Fatalf("exit code = %d, want %d", code, exitProvider) + } + rep := decodeReport(t, out) + c := checkByName(t, rep, "server_reachability") + if c.Status != doctorFail || c.ExitCode != exitProvider { + t.Errorf("reachability = %s/%d, want fail/%d", c.Status, c.ExitCode, exitProvider) + } + // The hint must name the documented container path, not a host recipe. + if !strings.Contains(c.Hint, "deploy/compose") || !strings.Contains(c.Hint, "docs/DEPLOYMENT.md") { + t.Errorf("hint = %q, want the documented deployment path", c.Hint) + } + for _, name := range []string{"workspace", "version_skew"} { + got := checkByName(t, rep, name) + if got.Status != doctorSkipped { + t.Errorf("%s = %s, want skipped", name, got.Status) + } + if !strings.Contains(got.Detail, "server_reachability") { + t.Errorf("%s detail = %q, want it to name the blocking check", name, got.Detail) + } + } +} + +func TestDoctorMissingCredential(t *testing.T) { + stub := newDoctorStub() + srv := stub.server(t) + defer srv.Close() + configureDoctor(t, srv.URL, "", false) // no config file: first run + + out, _, err := execDoctor(t, "--format", "json") + if err == nil { + t.Fatalf("doctor should exit non-zero without a credential\n%s", out) + } + if code := cliCode(t, err); code != exitAuth { + t.Fatalf("exit code = %d, want %d", code, exitAuth) + } + rep := decodeReport(t, out) + c := checkByName(t, rep, "credential") + if c.Status != doctorFail || c.ExitCode != exitAuth { + t.Errorf("credential = %s/%d, want fail/%d", c.Status, c.ExitCode, exitAuth) + } + if !strings.Contains(c.Hint, "mem auth login") { + t.Errorf("hint = %q, want it to name `mem auth login`", c.Hint) + } + if !strings.Contains(c.Hint, "deploy/compose") { + t.Errorf("hint = %q, want first-run guidance naming the documented path", c.Hint) + } + if got, wantReq := stub.seen(), "GET /healthz"; strings.Join(got, ",") != wantReq { + t.Errorf("requests = %v, want only the health probe", got) + } +} + +// A machine that already has a config is not a first run: it must not be told to +// deploy a stack it is evidently already talking to. +func TestDoctorMissingCredentialOnConfiguredHost(t *testing.T) { + stub := newDoctorStub() + srv := stub.server(t) + defer srv.Close() + configureDoctor(t, srv.URL, "", true) + + out, _, err := execDoctor(t, "--format", "json") + if err == nil { + t.Fatalf("want non-zero exit\n%s", out) + } + c := checkByName(t, decodeReport(t, out), "credential") + if !strings.Contains(c.Hint, "mem auth login") { + t.Errorf("hint = %q, want the login step", c.Hint) + } + if strings.Contains(c.Hint, "deploy/compose") { + t.Errorf("hint = %q, must not suggest deploying on an already-configured host", c.Hint) + } +} + +func TestDoctorNoWorkspaceSelected(t *testing.T) { + stub := newDoctorStub() + stub.caps = func(w http.ResponseWriter) { + _, _ = w.Write([]byte(`{"workspace":{"id":"","name":"","role":""}}`)) + } + srv := stub.server(t) + defer srv.Close() + configureDoctor(t, srv.URL, "tok", true) + + out, _, err := execDoctor(t, "--format", "json") + if err == nil { + t.Fatalf("doctor should exit non-zero when no workspace resolves\n%s", out) + } + if code := cliCode(t, err); code != exitNotFound { + t.Fatalf("exit code = %d, want %d", code, exitNotFound) + } + c := checkByName(t, decodeReport(t, out), "workspace") + if c.Status != doctorFail || c.ExitCode != exitNotFound { + t.Errorf("workspace = %s/%d, want fail/%d", c.Status, c.ExitCode, exitNotFound) + } +} + +func TestDoctorRejectedCredential(t *testing.T) { + stub := newDoctorStub() + stub.caps = func(w http.ResponseWriter) { + w.WriteHeader(http.StatusUnauthorized) + _, _ = w.Write([]byte(`{"error":"missing_bearer","code":"unauthorized"}`)) + } + srv := stub.server(t) + defer srv.Close() + configureDoctor(t, srv.URL, "expired-token", true) + + out, _, err := execDoctor(t, "--format", "json") + if err == nil { + t.Fatalf("doctor should exit non-zero on a rejected token\n%s", out) + } + if code := cliCode(t, err); code != exitAuth { + t.Fatalf("exit code = %d, want %d", code, exitAuth) + } + c := checkByName(t, decodeReport(t, out), "workspace") + if c.Status != doctorFail || c.ExitCode != exitAuth { + t.Errorf("workspace = %s/%d, want fail/%d", c.Status, c.ExitCode, exitAuth) + } +} + +// TestDoctorQuotaIsItsOwnCode pins the 4 (plan/quota) arm of the SPEC §7.1 map. +func TestDoctorQuotaIsItsOwnCode(t *testing.T) { + stub := newDoctorStub() + stub.caps = func(w http.ResponseWriter) { + w.WriteHeader(http.StatusTooManyRequests) + _, _ = w.Write([]byte(`{"error":"quota_exceeded","code":"quota"}`)) + } + srv := stub.server(t) + defer srv.Close() + configureDoctor(t, srv.URL, "tok", true) + + out, _, err := execDoctor(t, "--format", "json") + if err == nil { + t.Fatalf("want non-zero exit\n%s", out) + } + if code := cliCode(t, err); code != exitPlanQuota { + t.Fatalf("exit code = %d, want %d\n%s", code, exitPlanQuota, out) + } +} + +func TestDoctorVersionSkew(t *testing.T) { + stub := newDoctorStub() + srv := stub.server(t) + defer srv.Close() + configureDoctor(t, srv.URL, "tok", true) + t.Cleanup(func() { cliVersion = devCLIVersion }) + + // A dev build does not know its own version, so the check must say the + // comparison is impossible instead of claiming agreement. + cliVersion = devCLIVersion + out, _, err := execDoctor(t, "--format", "json") + if err != nil { + t.Fatalf("an advisory skew must not fail the run: %v\n%s", err, out) + } + c := checkByName(t, decodeReport(t, out), "version_skew") + if c.Status != doctorWarn || !strings.Contains(c.Detail, "not computable") { + t.Errorf("dev-build skew = %s (%s), want warn / not computable", c.Status, c.Detail) + } + + cliVersion = "0.0.9" + out, _, err = execDoctor(t, "--format", "json") + if err != nil { + t.Fatalf("skew should stay advisory: %v\n%s", err, out) + } + rep := decodeReport(t, out) + c = checkByName(t, rep, "version_skew") + if c.Status != doctorWarn || !strings.Contains(c.Detail, "0.0.9") || !strings.Contains(c.Detail, "0.1.0") { + t.Errorf("skew = %s (%s), want both versions named", c.Status, c.Detail) + } + if c.ExitCode != exitOK { + t.Errorf("skew exit contribution = %d, want 0 (advisory)", c.ExitCode) + } + if rep.ServerVersion != "0.1.0" { + t.Errorf("server_version = %q, want 0.1.0", rep.ServerVersion) + } + + cliVersion = "0.1.0" + out, _, _ = execDoctor(t, "--format", "json") + if c = checkByName(t, decodeReport(t, out), "version_skew"); c.Status != doctorOK { + t.Errorf("matching skew = %s (%s), want ok", c.Status, c.Detail) + } +} + +func TestDoctorNeverPrintsSecretValues(t *testing.T) { + const token = "sup3r-s3cret-token" + stub := newDoctorStub() + stub.caps = func(w http.ResponseWriter) { + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"error":"workspace_forbidden","code":"forbidden"}`)) + } + srv := stub.server(t) + defer srv.Close() + + dir := t.TempDir() + cfg := filepath.Join(dir, "config.yaml") + body := fmt.Sprintf("server: %s\nemail: ops@corp\ntoken: %s\nworkspace: w-1\n", srv.URL, token) + if err := os.WriteFile(cfg, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("MEM_CONFIG", cfg) + t.Setenv("MEM_SERVER", srv.URL) + t.Setenv("MEM_TOKEN", token) + t.Setenv("MEM_WORKSPACE", "") + + for _, format := range []string{"text", "json"} { + stdout, stderr, _ := execDoctor(t, "--format", format) + if stdout == "" { + t.Fatalf("%s run produced no output", format) + } + for label, out := range map[string]string{"stdout": stdout, "stderr": stderr} { + if strings.Contains(out, token) { + t.Errorf("%s %s leaks the token value:\n%s", format, label, out) + } + } + } +} + +func TestRedactURLStripsUserinfo(t *testing.T) { + const secret = "dsn-p4ssw0rd" + got := redactURL("http://admin:" + secret + "@mem.internal:8787") + if strings.Contains(got, secret) { + t.Errorf("redactURL = %q, still carries the password", got) + } + if !strings.Contains(got, "REDACTED@mem.internal:8787") { + t.Errorf("redactURL = %q, want the userinfo replaced and the host kept", got) + } + got = redactURL("http://admin:" + secret + "@ho st.example.com:8787") + if strings.Contains(got, secret) { + t.Errorf("redactURL = %q, still carries the malformed-password value", got) + } + if got != redact.Placeholder && !strings.Contains(got, "REDACTED@ho st.example.com:8787") { + t.Errorf("redactURL = %q, want userinfo replaced, or the whole value withheld "+ + "once it can no longer be proven a transport URL", got) + } + plain := "http://localhost:8787" + if redactURL(plain) != plain { + t.Errorf("redactURL(%q) = %q, want it unchanged", plain, redactURL(plain)) + } +} + +func TestDoctorMalformedServerURLDoesNotLeakCredentials(t *testing.T) { + const secret = "malformed-psswd" + malformed := "http://admin:" + secret + "@ho st.example.com:1/healthz" + configureDoctor(t, malformed, "tok", true) + + for _, format := range []string{"json", "text"} { + stdout, stderr, err := execDoctor(t, "--format", format) + if err == nil { + t.Fatalf("doctor should fail with malformed server URL in %s format\n%s", format, stdout) + } + if strings.Contains(stdout, secret) { + t.Errorf("stdout leaked credential for %s: %s", format, stdout) + } + if strings.Contains(stderr, secret) { + t.Errorf("stderr leaked credential for %s: %s", format, stderr) + } + if strings.Contains(stdout, malformed) { + t.Errorf("stdout should not carry raw malformed URL in %s: %s", format, stdout) + } + if format == "json" { + rep := decodeReport(t, stdout) + reach := checkByName(t, rep, "server_reachability") + if strings.Contains(reach.Detail, secret) { + t.Errorf("server_reachability detail leaked secret: %s", reach.Detail) + } + if strings.Contains(reach.Detail, malformed) { + t.Errorf("server_reachability detail leaked raw URL: %s", reach.Detail) + } + if !strings.Contains(reach.Detail, redact.UserMarker) && + !strings.Contains(reach.Detail, redact.Placeholder) { + t.Errorf("server_reachability detail should redact or withhold credentials: %s", reach.Detail) + } + } + } +} + +// TestDoctorSchemelessServerURLDoesNotLeakCredentials covers the shape the +// previous scrubber could not see: url.Parse succeeds on it and reports no +// userinfo, because "admin" is read as the scheme and the credential lands in +// Opaque. A gate that keys on User != nil echoes it verbatim. +func TestDoctorSchemelessServerURLDoesNotLeakCredentials(t *testing.T) { + const secret = "schemeless-psswd" + schemeless := "admin:" + secret + "@mem.invalid:8787" + configureDoctor(t, schemeless, "tok", true) + + for _, format := range []string{"json", "text"} { + stdout, stderr, err := execDoctor(t, "--format", format) + if err == nil { + t.Fatalf("doctor should fail against an unreachable schemeless URL in %s format\n%s", format, stdout) + } + if strings.Contains(stdout, secret) || strings.Contains(stderr, secret) { + t.Errorf("%s output leaked the credential: stdout=%s stderr=%s", format, stdout, stderr) + } + if strings.Contains(stdout, schemeless) || strings.Contains(stderr, schemeless) { + t.Errorf("%s output echoed the raw schemeless URL: stdout=%s stderr=%s", format, stdout, stderr) + } + if format == "json" { + rep := decodeReport(t, stdout) + if strings.Contains(rep.Server, secret) { + t.Errorf("report server field leaked the credential: %s", rep.Server) + } + reach := checkByName(t, rep, "server_reachability") + if strings.Contains(reach.Detail, secret) || strings.Contains(reach.Hint, secret) { + t.Errorf("detail/hint leaked the credential: detail=%s hint=%s", reach.Detail, reach.Hint) + } + } + } +} + +func TestDoctorTextOutputIsAFixedOrderedList(t *testing.T) { + stub := newDoctorStub() + srv := stub.server(t) + defer srv.Close() + configureDoctor(t, srv.URL, "tok", true) + t.Cleanup(func() { cliVersion = devCLIVersion }) + + out, _, err := execDoctor(t) + if err != nil { + t.Fatalf("healthy doctor returned %v\n%s", err, out) + } + names := []string{"server_reachability", "credential", "workspace", "version_skew"} + prev := -1 + for _, n := range names { + at := strings.Index(out, n) + if at < 0 { + t.Fatalf("text output missing %q:\n%s", n, out) + } + if at < prev { + t.Errorf("check %q printed out of order:\n%s", n, out) + } + prev = at + } + if !strings.Contains(out, "mem doctor (mem.doctor v1)") { + t.Errorf("text output missing the contract header:\n%s", out) + } + if strings.Contains(out, `"checks"`) { + t.Errorf("text output contains JSON:\n%s", out) + } +} + +func TestDoctorRejectsBadFlags(t *testing.T) { + dir := t.TempDir() + t.Setenv("MEM_CONFIG", filepath.Join(dir, "missing.yaml")) + t.Setenv("MEM_TOKEN", "tok") + + if _, _, err := execDoctor(t, "--format", "yaml"); err == nil { + t.Error("--format yaml should be rejected") + } + if _, _, err := execDoctor(t, "--timeout", "0s"); err == nil { + t.Error("--timeout 0s should be rejected") + } +} + +// doctorSchema mirrors the parts of docs/schemas/mem-doctor.v1.schema.json that +// this test can enforce without a draft-2020-12 evaluator: required keys, the +// closed enums, key admission (additionalProperties:false) and the fixed check +// order. +type doctorSchema struct { + Required []string `json:"required"` + Properties map[string]doctorSchemaNode `json:"properties"` + Defs map[string]doctorSchemaNode `json:"$defs"` +} + +type doctorSchemaNode struct { + Type string `json:"type"` + Enum []json.RawMessage `json:"enum"` + Const json.RawMessage `json:"const"` + Required []string `json:"required"` + Properties map[string]doctorSchemaNode `json:"properties"` + PrefixItems []json.RawMessage `json:"prefixItems"` +} + +func loadDoctorSchema(t *testing.T) doctorSchema { + t.Helper() + b, err := os.ReadFile(filepath.Join("..", "..", "..", "docs", "schemas", "mem-doctor.v1.schema.json")) + if err != nil { + t.Fatal(err) + } + var s doctorSchema + if err := json.Unmarshal(b, &s); err != nil { + t.Fatalf("checked-in schema is not parseable: %v", err) + } + if len(s.Required) == 0 || len(s.Properties) == 0 { + t.Fatalf("schema did not declare required keys or properties: %s", b) + } + if s.Defs["check"].Type != "object" { + t.Fatalf("schema missing $defs.check object: %s", b) + } + return s +} + +// TestDoctorJSONMatchesCheckedInSchema is AC-003. +func TestDoctorJSONMatchesCheckedInSchema(t *testing.T) { + stub := newDoctorStub() + srv := stub.server(t) + defer srv.Close() + configureDoctor(t, srv.URL, "tok", true) + t.Cleanup(func() { cliVersion = devCLIVersion }) + cliVersion = "0.1.0" + + out, _, err := execDoctor(t, "--format", "json") + if err != nil { + t.Fatalf("healthy doctor returned %v\n%s", err, out) + } + schema := loadDoctorSchema(t) + validateDoctorDoc(t, schema, []byte(out)) + + // httptest assigns an ephemeral port, which the report echoes in two places + // (server and the reachability detail). The golden pins everything except + // that, so drift in shape, order, wording or codes still fails loudly. + normalized := strings.ReplaceAll(out, srv.URL, "http://127.0.0.1:PORT") + + golden := filepath.Join("testdata", "doctor_healthy.golden.json") + if os.Getenv("MEM_UPDATE_GOLDEN") != "" { + if err := os.MkdirAll(filepath.Dir(golden), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(golden, []byte(normalized), 0o600); err != nil { + t.Fatal(err) + } + } + want, err := os.ReadFile(golden) + if err != nil { + t.Fatalf("read golden: %v (create it with MEM_UPDATE_GOLDEN=1 go test ./cmd/mem/ -run TestDoctorJSON)", err) + } + if strings.TrimSpace(string(want)) != strings.TrimSpace(normalized) { + t.Errorf("doctor json drifted from %s\n--- want ---\n%s\n--- got ---\n%s", golden, want, normalized) + } +} + +func validateDoctorDoc(t *testing.T, s doctorSchema, doc []byte) { + t.Helper() + var obj map[string]json.RawMessage + if err := json.Unmarshal(doc, &obj); err != nil { + t.Fatalf("report is not a JSON object: %v", err) + } + for _, req := range s.Required { + if _, ok := obj[req]; !ok { + t.Errorf("report missing required key %q", req) + } + } + for key := range obj { + if _, ok := s.Properties[key]; !ok { + t.Errorf("report has key %q, which the schema forbids (additionalProperties:false)", key) + } + } + for _, key := range []string{"contract", "schema_version"} { + if !nodeAllows(s.Properties[key], obj[key]) { + t.Errorf("%s = %s, outside the schema's const", key, obj[key]) + } + } + if !nodeAllows(s.Properties["exit_code"], obj["exit_code"]) { + t.Errorf("exit_code = %s, outside the SPEC 7.1 set", obj["exit_code"]) + } + + var checks []map[string]json.RawMessage + if err := json.Unmarshal(obj["checks"], &checks); err != nil { + t.Fatalf("checks is not an array: %v", err) + } + if len(checks) != len(s.Properties["checks"].PrefixItems) { + t.Fatalf("checks length = %d, want %d", len(checks), len(s.Properties["checks"].PrefixItems)) + } + def := s.Defs["check"] + for i, c := range checks { + for _, req := range def.Required { + if _, ok := c[req]; !ok { + t.Errorf("checks[%d] missing required key %q", i, req) + } + } + for key := range c { + if _, ok := def.Properties[key]; !ok { + t.Errorf("checks[%d] has key %q the schema forbids", i, key) + } + } + for _, field := range []string{"name", "status", "exit_code"} { + if !nodeAllows(def.Properties[field], c[field]) { + t.Errorf("checks[%d].%s = %s, outside the schema's closed enum", i, field, c[field]) + } + } + // prefixItems pins the order, so a reordered report fails here. + var slot struct { + AllOf []struct { + Properties map[string]doctorSchemaNode `json:"properties"` + } `json:"allOf"` + } + if err := json.Unmarshal(s.Properties["checks"].PrefixItems[i], &slot); err != nil { + t.Fatalf("prefixItems[%d] unreadable: %v", i, err) + } + for _, sub := range slot.AllOf { + if want, ok := sub.Properties["name"]; ok && !nodeAllows(want, c["name"]) { + t.Errorf("checks[%d].name = %s, want %s (order is part of the contract)", i, c["name"], want.Enum) + } + } + } +} + +// nodeAllows reports whether raw satisfies a leaf schema doctorSchemaNode that constrains by +// const or enum. A leaf with neither declares no value constraint. +func nodeAllows(n doctorSchemaNode, raw json.RawMessage) bool { + value := strings.TrimSpace(string(raw)) + if len(n.Const) > 0 { + return strings.TrimSpace(string(n.Const)) == value + } + if len(n.Enum) > 0 { + for _, e := range n.Enum { + if strings.TrimSpace(string(e)) == value { + return true + } + } + return false + } + return true +} + +// TestNotLoggedInGuidance is REQ-002 on an existing command surface: the hint +// that used to stop at `mem auth login` must additionally name the documented +// deployment path, but only when no credential exists at all. +func TestNotLoggedInGuidance(t *testing.T) { + dir := t.TempDir() + missing := filepath.Join(dir, "missing.yaml") + existing := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(existing, []byte("server: http://127.0.0.1:1\n"), 0o600); err != nil { + t.Fatal(err) + } + + cases := []struct { + name string + cfgPath string + want []string + deny string + }{ + {name: "first run", cfgPath: missing, want: []string{"mem auth login", "deploy/compose", "docs/DEPLOYMENT.md"}}, + {name: "configured but logged out", cfgPath: existing, want: []string{"mem auth login"}, deny: "deploy/compose"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("MEM_CONFIG", tc.cfgPath) + t.Setenv("MEM_TOKEN", "") + t.Setenv("MEM_SERVER", "") + var out bytes.Buffer + root := newRootCmd() + root.SetOut(&out) + root.SetErr(&out) + root.SetArgs([]string{"search", "fy27 recruiting"}) + err := root.Execute() + var code int + if code = cliCode(t, err); code != exitAuth { + t.Fatalf("exit code = %d, want %d", code, exitAuth) + } + ce := err.(*cliError) + for _, want := range tc.want { + if !strings.Contains(ce.hint, want) { + t.Errorf("hint = %q, want it to name %q", ce.hint, want) + } + } + if tc.deny != "" && strings.Contains(ce.hint, tc.deny) { + t.Errorf("hint = %q, must not suggest deploying where a config exists", ce.hint) + } + }) + } +} diff --git a/server/cmd/mem/cmds_face.go b/server/cmd/mem/cmds_face.go index 4bed56c..fcb53ee 100644 --- a/server/cmd/mem/cmds_face.go +++ b/server/cmd/mem/cmds_face.go @@ -41,7 +41,7 @@ func newFaceListCmd() *cobra.Command { return err } if cfg.Token == "" { - return newCliError(3, "not logged in", "run `mem auth login` first") + return errNotLoggedIn() } c := newHTTPClient(cfg) var resp faceListResp diff --git a/server/cmd/mem/cmds_file.go b/server/cmd/mem/cmds_file.go index 37f30a1..fc2a8a5 100644 --- a/server/cmd/mem/cmds_file.go +++ b/server/cmd/mem/cmds_file.go @@ -42,7 +42,7 @@ func newPutCmd() *cobra.Command { return err } if cfg.Token == "" { - return newCliError(3, "not logged in", "run `mem auth login` first") + return errNotLoggedIn() } c := newHTTPClient(cfg) sourceMetadata, err := cliSourceMetadata( @@ -491,15 +491,18 @@ func newVersionCmd() *cobra.Command { Use: "version", Short: "Print client + server version", RunE: func(cmd *cobra.Command, args []string) error { - fmt.Println("mem CLI dev") + fmt.Printf("mem CLI %s\n", cliVersion) cfg, _ := resolveConfig("") if cfg != nil && cfg.Server != "" { c := newHTTPClient(cfg) var resp struct { - Version string `json:"version"` + Version string `json:"version"` + Revision string `json:"revision"` + Contract string `json:"contract"` } if err := c.doJSON(http.MethodGet, "/v1/version", nil, &resp); err == nil { - fmt.Printf("server: %s (%s)\n", resp.Version, cfg.Server) + fmt.Printf("server: %s (revision %s, contract %s, %s)\n", + resp.Version, resp.Revision, resp.Contract, redactURL(cfg.Server)) } } return nil diff --git a/server/cmd/mem/cmds_file_annotations.go b/server/cmd/mem/cmds_file_annotations.go index c27c3ea..d22e646 100644 --- a/server/cmd/mem/cmds_file_annotations.go +++ b/server/cmd/mem/cmds_file_annotations.go @@ -82,7 +82,7 @@ func configuredFileAnnotationClient() (*apiclient.Client, error) { return nil, err } if cfg.Token == "" { - return nil, newCliError(3, "not logged in", "run `mem auth login` first") + return nil, errNotLoggedIn() } return newHTTPClient(cfg).api, nil } diff --git a/server/cmd/mem/cmds_folder.go b/server/cmd/mem/cmds_folder.go index 08d2517..8bc138d 100644 --- a/server/cmd/mem/cmds_folder.go +++ b/server/cmd/mem/cmds_folder.go @@ -22,7 +22,7 @@ func newMkdirCmd() *cobra.Command { return err } if cfg.Token == "" { - return newCliError(3, "not logged in", "run `mem auth login` first") + return errNotLoggedIn() } c := newHTTPClient(cfg) var resp map[string]any diff --git a/server/cmd/mem/cmds_handoff.go b/server/cmd/mem/cmds_handoff.go index dc6133f..b094244 100644 --- a/server/cmd/mem/cmds_handoff.go +++ b/server/cmd/mem/cmds_handoff.go @@ -58,7 +58,7 @@ Examples: return err } if cfg.Token == "" { - return newCliError(3, "not logged in", "run `mem auth login` first") + return errNotLoggedIn() } raw, err := newHTTPClient(cfg).api.Checkpoint( commandContext(cmd), @@ -265,7 +265,7 @@ Examples: return err } if cfg.Token == "" { - return newCliError(3, "not logged in", "run `mem auth login` first") + return errNotLoggedIn() } raw, err := newHTTPClient(cfg).api.Resume( commandContext(cmd), diff --git a/server/cmd/mem/cmds_ingest.go b/server/cmd/mem/cmds_ingest.go index 2ed5b20..8af9425 100644 --- a/server/cmd/mem/cmds_ingest.go +++ b/server/cmd/mem/cmds_ingest.go @@ -9,10 +9,10 @@ import ( "net/http" "os" "path/filepath" - "sort" "strings" "github.com/PeterGuy326/mem/server/internal/apiclient" + "github.com/PeterGuy326/mem/server/internal/ingest" "github.com/spf13/cobra" ) @@ -112,30 +112,14 @@ func cliStateRoot() string { return filepath.Join(home, ".mem") } -// expandTranscriptGlob recursively collects *.jsonl transcripts under base and -// sorts the results, so ingestion order is deterministic across runs and -// checkpoints are stable. (Go's filepath.Glob does not treat ** as recursive, so -// we walk the tree explicitly.) +// expandTranscriptGlob collects the transcripts to offer for ingestion. The walk +// and its ordering belong to the shared core; this wrapper keeps the CLI's +// exit-code contract for an unwalkable root. func expandTranscriptGlob(base string) ([]string, error) { - // A bare base that names a single existing file (not a directory) is - // accepted as a one-off transcript. - if fi, e := os.Stat(base); e == nil && !fi.IsDir() { - return []string{base}, nil - } - var paths []string - err := filepath.WalkDir(base, func(p string, d os.DirEntry, err error) error { - if err != nil { - return nil // skip unreadable entries rather than failing the walk - } - if !d.IsDir() && strings.EqualFold(filepath.Ext(p), ".jsonl") { - paths = append(paths, p) - } - return nil - }) + paths, err := ingest.Walk(base, ingest.HasJSONLExtension) if err != nil { return nil, newCliError(1, fmt.Sprintf("walk %s: %v", base, err), "") } - sort.Strings(paths) return paths, nil } @@ -144,6 +128,13 @@ func runIngestQoder(cmd *cobra.Command, o ingestOptions) error { if base == "" { return newCliError(1, "cannot determine session store root", "set --root or $HOME") } + // Walk, the checkpoint key and the project/session split must all see one + // identity. Left as the caller spelled it, a relative --root would make two + // working directories that each hold sessions/p.jsonl share a checkpoint. + base, err := ingest.CanonicalRoot(base) + if err != nil { + return newCliError(1, err.Error(), "set --root to an accessible path") + } paths, err := expandTranscriptGlob(base) if err != nil { return err @@ -158,109 +149,93 @@ func runIngestQoder(cmd *cobra.Command, o ingestOptions) error { return err } if !o.dryRun && cfg.Token == "" { - return newCliError(3, "not logged in", "run `mem auth login` first") + return errNotLoggedIn() } client := newHTTPClient(cfg) - stateDir := o.checkpointDir() + warn := func(format string, args ...any) { + fmt.Fprintf(cmd.ErrOrStderr(), format, args...) + } - var ( - files = 0 - memories int // written this run - replayed int // server-reported idempotent replays - unparseable int // parsed lines yielded no ingestible text - remaining = o.limit + report, err := ingest.Run( + context.Background(), + paths, + ingest.Options{ + StateDir: o.checkpointDir(), + DryRun: o.dryRun, + Limit: o.limit, + Log: warn, + }, + o.parseTranscript(base), + o.uploadMemory(client, warn), ) + if err != nil { + // Classify runs inside the core on the typed error, so the exit-code + // mapping belongs here, at the boundary that owns it. + return fromAPIError(err) + } + + fmt.Fprintf(cmd.OutOrStdout(), + "qoder ingest: %d file(s), %d memory written, %d server-replay, %d unparseable line%s\n", + report.Scanned, report.Ingested, report.Deduped, report.Unparseable, + ingestModeNote(o.dryRun)) + return nil +} - for _, abs := range paths { - files++ - cp := loadQoderCheckpoint(stateDir, abs) - turns, skipped, perr := parseQoderTranscript(abs, cp.LastLine) - if perr != nil { - return perr +// parseTranscript adapts the Qoder transcript reader to the core's ParseFunc: +// every turn becomes a unit carrying its own request body and stable key. +func (o ingestOptions) parseTranscript(base string) ingest.ParseFunc { + return func(abs string, skipBefore int) ([]ingest.Unit, int, error) { + turns, skipped, err := parseQoderTranscript(abs, skipBefore) + if err != nil { + return nil, skipped, err } - unparseable += skipped project, session := splitTranscriptPath(base, abs) - - newLast := cp.LastLine + units := make([]ingest.Unit, 0, len(turns)) for _, turn := range turns { - if o.limit > 0 && remaining <= 0 { - break - } - // parseQoderTranscript already skips <= cp.LastLine, so this - // guard is a belt-and-suspenders check against anomalies. - if turn.Line <= newLast { - continue - } - - key := ingestIdempotencyKey(abs, turn.Line) - body := ingestMemoryBody(o, abs, project, session, turn) - - if o.dryRun { - memories++ - if remaining > 0 { - remaining-- - } - continue - } - - var resp map[string]any - // Use the raw client to detect 409 (Idempotency-Key conflict) - // without wrapping into cliError, which would lose the kind. - err := client.api.DoJSONWithHeaders( - context.Background(), - http.MethodPost, - "/v1/memories", - body, - &resp, - map[string]string{"Idempotency-Key": key}, - ) - if err != nil { - // 409 Idempotency-Key conflict: the file was rewritten with - // different content at the same line — skip the remainder of - // this file and continue with others rather than aborting the - // whole run. The checkpoint is NOT advanced for this file, so - // the operator can investigate and retry. - var ae *apiclient.APIError - if errors.As(err, &ae) && ae.Kind() == apiclient.KindConflict { - fmt.Fprintf(cmd.ErrOrStderr(), - "warn: %s line %d: idempotency conflict (file rewritten?); skipping remaining lines in %s\n", - abs, turn.Line, filepath.Base(abs)) - break - } - return fromAPIError(err) - } - if r, _ := resp["replayed"].(bool); r { - replayed++ - } else { - memories++ - } - if remaining > 0 { - remaining-- - } - newLast = turn.Line + units = append(units, ingest.Unit{ + Line: turn.Line, + Body: ingestMemoryBody(o, abs, project, session, turn), + IdempotencyKey: ingestIdempotencyKey(abs, turn.Line), + }) } + return units, skipped, nil + } +} - if !o.dryRun { - if size, mtime, serr := fileState(abs); serr == nil { - if err := saveQoderCheckpoint(stateDir, qoderCheckpoint{ - Abs: abs, - Size: size, - ModTime: mtime, - LastLine: newLast, - }); err != nil { - fmt.Fprintf(cmd.ErrOrStderr(), "warn: save checkpoint for %s: %v\n", abs, err) - } +// uploadMemory posts one unit through the standard memories endpoint. Errors +// stay typed on purpose: the core's Classify dispatches on *apiclient.APIError +// for both the per-file degradation decision and the failure code in the report, +// and the SPEC §7.1 exit-code mapping is applied by the calling command. +func (o ingestOptions) uploadMemory(client *httpClient, warn func(string, ...any)) ingest.UploadFunc { + return func(ctx context.Context, abs string, u ingest.Unit) (ingest.Outcome, error) { + var resp map[string]any + err := client.api.DoJSONWithHeaders( + ctx, + http.MethodPost, + "/v1/memories", + u.Body, + &resp, + map[string]string{"Idempotency-Key": u.IdempotencyKey}, + ) + if err != nil { + var ae *apiclient.APIError + if errors.As(err, &ae) && ae.Kind() == apiclient.KindConflict { + // The file was rewritten with different content at the same + // line, so every later line keeps colliding on its stable key. + warn("warn: %s line %d: idempotency conflict (file rewritten?); skipping remaining lines in %s\n", + abs, u.Line, filepath.Base(abs)) + return ingest.Outcome{}, fmt.Errorf("%w: %s:%d", ingest.ErrDegradeFile, abs, u.Line) } + return ingest.Outcome{}, err } + if r, _ := resp["replayed"].(bool); r { + return ingest.Outcome{Deduplicated: true}, nil + } + return ingest.Outcome{}, nil } - - fmt.Fprintf(cmd.OutOrStdout(), - "qoder ingest: %d file(s), %d memory written, %d server-replay, %d unparseable line%s\n", - files, memories, replayed, unparseable, - ingestModeNote(o.dryRun)) - return nil } +// ingestModeNote is the stdout suffix that separates a plan from a write. func ingestModeNote(dryRun bool) string { if dryRun { return " (dry-run: no writes)" diff --git a/server/cmd/mem/cmds_ingest_test.go b/server/cmd/mem/cmds_ingest_test.go index 3f720f5..2a4bea0 100644 --- a/server/cmd/mem/cmds_ingest_test.go +++ b/server/cmd/mem/cmds_ingest_test.go @@ -2,17 +2,23 @@ package main import ( "bytes" + "context" "encoding/json" "errors" + "fmt" "io" "net/http" "net/http/httptest" "os" "path/filepath" + "runtime" "strings" "sync" "sync/atomic" "testing" + + "github.com/PeterGuy326/mem/server/internal/apiclient" + "github.com/PeterGuy326/mem/server/internal/ingest" ) // writeTranscript writes a small valid JSONL transcript containing three @@ -36,7 +42,11 @@ func writeTranscript(t *testing.T, dir, name string) string { if err := os.WriteFile(p, []byte(body+"\n"), 0o644); err != nil { t.Fatal(err) } - return p + canonical, err := ingest.CanonicalRoot(p) + if err != nil { + t.Fatal(err) + } + return canonical } func TestParseQoderTranscript(t *testing.T) { @@ -376,11 +386,11 @@ func TestIngestQoder409ConflictDegradesPerFile(t *testing.T) { if !strings.Contains(stdout.String(), "idempotency conflict") { t.Fatalf("expected conflict warning, got stdout = %q", stdout.String()) } - cp := loadQoderCheckpoint(cpDir, abs1) + cp := ingest.LoadCursor(cpDir, abs1) if cp.LastLine != 1 { t.Fatalf("project-a checkpoint LastLine = %d, want 1 (line 2 failed)", cp.LastLine) } - cp2 := loadQoderCheckpoint(cpDir, abs2) + cp2 := ingest.LoadCursor(cpDir, abs2) if cp2.LastLine != 3 { t.Fatalf("project-b checkpoint LastLine = %d, want 3", cp2.LastLine) } @@ -418,7 +428,7 @@ func TestIngestQoderLimitCheckpoint(t *testing.T) { if requests.Load() != 2 { t.Fatalf("first run requests = %d, want 2", requests.Load()) } - cp := loadQoderCheckpoint(cpDir, abs) + cp := ingest.LoadCursor(cpDir, abs) if cp.LastLine != 2 { t.Fatalf("after limit checkpoint LastLine = %d, want 2", cp.LastLine) } @@ -433,8 +443,300 @@ func TestIngestQoderLimitCheckpoint(t *testing.T) { if requests.Load() != 1 { t.Fatalf("second run requests = %d, want 1", requests.Load()) } - cp2 := loadQoderCheckpoint(cpDir, abs) + cp2 := ingest.LoadCursor(cpDir, abs) if cp2.LastLine != 3 { t.Fatalf("after second run LastLine = %d, want 3", cp2.LastLine) } } + +// TestIngestQoderUploadErrorsStayTyped covers the adapter's error contract: the +// core derives the report's failure code from the error it is handed, so +// translating it inside uploadMemory would report every API failure as a network +// failure. +func TestIngestQoderUploadErrorsStayTyped(t *testing.T) { + cases := []struct { + name string + status int + // apiError expects a *apiclient.APIError with this status to survive. + apiError bool + degrade bool + want ingest.Code + unreachable bool + }{ + {name: "400", status: http.StatusBadRequest, apiError: true, want: ingest.CodeUploadRejected}, + {name: "401", status: http.StatusUnauthorized, apiError: true, want: ingest.CodeAuth}, + {name: "402", status: http.StatusPaymentRequired, apiError: true, want: ingest.CodePlanQuota}, + {name: "403", status: http.StatusForbidden, apiError: true, want: ingest.CodeAuth}, + {name: "409", status: http.StatusConflict, degrade: true, want: ingest.CodeUploadRejected}, + {name: "429", status: http.StatusTooManyRequests, apiError: true, want: ingest.CodePlanQuota}, + {name: "502", status: http.StatusBadGateway, apiError: true, want: ingest.CodeProviderTimeout}, + {name: "503", status: http.StatusServiceUnavailable, apiError: true, want: ingest.CodeProviderTimeout}, + {name: "504", status: http.StatusGatewayTimeout, apiError: true, want: ingest.CodeProviderTimeout}, + {name: "network", unreachable: true, want: ingest.CodeNetwork}, + } + for _, tc := range cases { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(tc.status) + _, _ = fmt.Fprintf(w, `{"error":"%d","hint":"h"}`, tc.status) + })) + url := srv.URL + if tc.unreachable { + srv.Close() + } else { + defer srv.Close() + } + + client := newHTTPClient(&cliConfig{Server: url, Token: "tok"}) + upload := (ingestOptions{}).uploadMemory(client, func(string, ...any) {}) + _, err := upload(context.Background(), "/store/p/s.jsonl", ingest.Unit{ + Line: 2, Body: map[string]any{"kind": "observation", "content": "c"}, + }) + if err == nil { + t.Fatalf("%s: want an error", tc.name) + } + + var ae *apiclient.APIError + switch { + case tc.apiError && !errors.As(err, &ae): + t.Errorf("%s: err = %v, want the typed APIError to survive the adapter", tc.name, err) + case tc.apiError && ae.StatusCode != tc.status: + t.Errorf("%s: APIError.StatusCode = %d, want %d", tc.name, ae.StatusCode, tc.status) + case tc.degrade && !errors.Is(err, ingest.ErrDegradeFile): + t.Errorf("%s: err = %v, want per-file degradation", tc.name, err) + case tc.unreachable && errors.As(err, &ae): + t.Errorf("%s: err = %v, want a transport error", tc.name, err) + } + if got := ingest.Classify(err); got != tc.want { + t.Errorf("%s: Classify = %q, want %q", tc.name, got, tc.want) + } + } +} + +// TestIngestQoderReadFailuresClassify covers the other side of a run: a source +// that cannot be read must reach the core as the OS error it is, so the report +// names the read state instead of claiming a transport failure. +func TestIngestQoderReadFailuresClassify(t *testing.T) { + dir := t.TempDir() + parse := ingestOptions{}.parseTranscript(dir) + + run := func(path string, wantCode ingest.Code) { + t.Helper() + report, err := ingest.Run(context.Background(), []string{path}, + ingest.Options{StateDir: filepath.Join(dir, "state")}, + parse, + func(context.Context, string, ingest.Unit) (ingest.Outcome, error) { + t.Error("upload must not be reached for an unreadable source") + return ingest.Outcome{}, nil + }) + if err == nil { + t.Fatalf("%s: run succeeded", path) + } + if got := ingest.Classify(err); got != wantCode { + t.Errorf("%s: Classify(%v) = %q, want %q", path, err, got, wantCode) + } + if report.Failures[wantCode] != 1 { + t.Errorf("%s: report tally = %+v, want one %q", path, report.Failures, wantCode) + } + } + + run(filepath.Join(dir, "gone", "p.jsonl"), ingest.CodeRootMissing) + + if runtime.GOOS == "windows" || os.Geteuid() == 0 { + t.Skip("mode bits do not deny a read here: the permission case needs a POSIX non-root user") + } + denied := writeTranscript(t, dir, "denied/s.jsonl") + if err := os.Chmod(denied, 0o000); err != nil { + t.Fatal(err) + } + defer func() { _ = os.Chmod(denied, 0o600) }() + run(denied, ingest.CodeReadDenied) +} + +// TestIngestQoderMapsExitCodesAtTheBoundary pins the SPEC §7.1 codes that the +// command owns, which is where the APIError is allowed to become a cliError. +func TestIngestQoderMapsExitCodesAtTheBoundary(t *testing.T) { + cases := []struct { + status int + want int + }{ + {http.StatusBadRequest, 1}, + {http.StatusUnauthorized, 3}, + {http.StatusForbidden, 3}, + {http.StatusPaymentRequired, 4}, + {http.StatusTooManyRequests, 4}, + {http.StatusBadGateway, 5}, + {http.StatusServiceUnavailable, 5}, + {http.StatusGatewayTimeout, 5}, + {http.StatusInternalServerError, 1}, + } + for _, tc := range cases { + dir := t.TempDir() + transcriptDir := filepath.Join(dir, "store") + writeTranscript(t, transcriptDir, "p/s.jsonl") + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(tc.status) + _, _ = fmt.Fprintf(w, `{"error":"%d","hint":"h"}`, tc.status) + })) + + t.Setenv("MEM_CONFIG", filepath.Join(dir, "missing.yaml")) + t.Setenv("MEM_SERVER", srv.URL) + t.Setenv("MEM_TOKEN", "tok") + t.Setenv("MEM_STATE_DIR", filepath.Join(dir, "state")) + + root := newRootCmd() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"ingest", "qoder", "--root", transcriptDir}) + err := root.Execute() + srv.Close() + + var ce *cliError + if !errors.As(err, &ce) { + t.Fatalf("status %d: err = %v, want a cliError", tc.status, err) + } + if ce.code != tc.want { + t.Errorf("status %d: exit code = %d, want %d", tc.status, ce.code, tc.want) + } + } +} + +// TestIngestQoderRelativeRootKeepsSeparateCheckpoints is the cross-working- +// directory fixture: two directories each holding an identical store/p/s.jsonl +// and sharing one checkpoint directory must not look already-ingested to the +// second run. +func TestIngestQoderRelativeRootKeepsSeparateCheckpoints(t *testing.T) { + var requests atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"memory":{"id":"m-1"},"replayed":false}`)) + })) + defer srv.Close() + + base := t.TempDir() + dirA := filepath.Join(base, "a") + dirB := filepath.Join(base, "b") + for _, dir := range []string{dirA, dirB} { + writeTranscript(t, filepath.Join(dir, "store"), "p/s.jsonl") + } + + t.Setenv("MEM_CONFIG", filepath.Join(base, "missing.yaml")) + t.Setenv("MEM_SERVER", srv.URL) + t.Setenv("MEM_TOKEN", "tok") + t.Setenv("MEM_STATE_DIR", filepath.Join(base, "state")) + + run := func() int { + t.Helper() + requests.Store(0) + root := newRootCmd() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"ingest", "qoder", "--root", "store"}) + if err := root.Execute(); err != nil { + t.Fatal(err) + } + return int(requests.Load()) + } + + t.Chdir(dirA) + if got := run(); got != 3 { + t.Fatalf("first run in directory A posted %d, want 3", got) + } + + // Same relative spelling, different files: the cursor must not be shared. + t.Chdir(dirB) + if got := run(); got != 3 { + t.Fatalf("run in directory B posted %d, want 3 (both stores collided on one checkpoint)", got) + } + + // Directory A is still complete under its own identity. + t.Chdir(dirA) + if got := run(); got != 0 { + t.Fatalf("re-run in directory A posted %d, want 0", got) + } + // And the transcript the server was told about is an absolute path, so + // provenance does not depend on where the CLI happened to run. + absB, err := ingest.CanonicalRoot(filepath.Join(dirB, "store", "p", "s.jsonl")) + if err != nil { + t.Fatal(err) + } + if got := ingest.LoadCursor(filepath.Join(base, "state", "ingest", "qoder"), absB).LastLine; got != 3 { + t.Fatalf("directory B checkpoint LastLine = %d, want 3 keyed by %s", got, absB) + } +} + +// TestIngestQoderRelativeRootKeepsProjectSplit pins the other half of root +// canonicalization: the base used to derive a memory's project and session must +// be spelled the same way as the paths the walk returned. +func TestIngestQoderRelativeRootKeepsProjectSplit(t *testing.T) { + dir := t.TempDir() + abs, err := ingest.CanonicalRoot(filepath.Join(dir, "store", "campus-2027", "sessions", "recruit-s3e0a.jsonl")) + if err != nil { + t.Fatal(err) + } + writeTranscript(t, filepath.Join(dir, "store"), "campus-2027/sessions/recruit-s3e0a.jsonl") + + var ( + mu sync.Mutex + paths []string + refs []string + keys []string + posted int + ) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + raw, _ := io.ReadAll(r.Body) + var b map[string]any + _ = json.Unmarshal(raw, &b) + src, _ := b["source"].(map[string]any) + ref, _ := src["ref"].(string) + mu.Lock() + paths = append(paths, fmt.Sprint(b["path"])) + refs = append(refs, ref) + keys = append(keys, r.Header.Get("Idempotency-Key")) + posted++ + mu.Unlock() + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"memory":{"id":"m-1"},"replayed":false}`)) + })) + defer srv.Close() + + t.Setenv("MEM_CONFIG", filepath.Join(dir, "missing.yaml")) + t.Setenv("MEM_SERVER", srv.URL) + t.Setenv("MEM_TOKEN", "tok") + t.Setenv("MEM_STATE_DIR", filepath.Join(dir, "state")) + + t.Chdir(dir) + root := newRootCmd() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"ingest", "qoder", "--root", "store"}) + if err := root.Execute(); err != nil { + t.Fatal(err) + } + + mu.Lock() + defer mu.Unlock() + if posted != 3 { + t.Fatalf("posted %d memories, want 3", posted) + } + for i := range paths { + if !strings.HasPrefix(paths[i], "/AgentTranscripts/campus-2027/recruit-s3e0a") { + t.Errorf("path[%d] = %q, want the project taken from the store root", i, paths[i]) + } + if refs[i] != abs { + t.Errorf("source.ref[%d] = %q, want the canonical path %q", i, refs[i], abs) + } + if !strings.HasPrefix(keys[i], "qoder:") { + t.Errorf("key[%d] = %q", i, keys[i]) + } + } + // The key is derived from the same canonical identity, so it cannot change + // when the same store is reached from another working directory. + wantKey := ingestIdempotencyKey(abs, 2) + if keys[1] != wantKey { + t.Errorf("key[1] = %q, want %q", keys[1], wantKey) + } +} diff --git a/server/cmd/mem/cmds_memory.go b/server/cmd/mem/cmds_memory.go index b842ef5..1c43a93 100644 --- a/server/cmd/mem/cmds_memory.go +++ b/server/cmd/mem/cmds_memory.go @@ -123,7 +123,7 @@ cursor and bounded memory summaries for scripts.`, return err } if cfg.Token == "" { - return newCliError(3, "not logged in", "run `mem auth login` first") + return errNotLoggedIn() } options := apiclient.MemoryListOptions{ @@ -393,7 +393,7 @@ func configuredMemoryClient() (*apiclient.Client, error) { return nil, err } if cfg.Token == "" { - return nil, newCliError(3, "not logged in", "run `mem auth login` first") + return nil, errNotLoggedIn() } return newHTTPClient(cfg).api, nil } diff --git a/server/cmd/mem/cmds_model.go b/server/cmd/mem/cmds_model.go index 9ff8237..84c1997 100644 --- a/server/cmd/mem/cmds_model.go +++ b/server/cmd/mem/cmds_model.go @@ -425,7 +425,7 @@ func activateLocalModelProfile( return providerSetResp{}, err } if cfg.Token == "" { - return providerSetResp{}, newCliError(3, "not logged in", "run `mem auth login` first") + return providerSetResp{}, errNotLoggedIn() } var response providerSetResp client := newHTTPClient(cfg) diff --git a/server/cmd/mem/cmds_profile.go b/server/cmd/mem/cmds_profile.go index a65ce6e..9092e14 100644 --- a/server/cmd/mem/cmds_profile.go +++ b/server/cmd/mem/cmds_profile.go @@ -143,7 +143,7 @@ func configuredWorkspaceAIProfileClient() (*httpClient, error) { return nil, err } if cfg.Token == "" { - return nil, newCliError(3, "not logged in", "run `mem auth login` first") + return nil, errNotLoggedIn() } return newHTTPClient(cfg), nil } diff --git a/server/cmd/mem/cmds_provider.go b/server/cmd/mem/cmds_provider.go index 322dac3..d31e3e2 100644 --- a/server/cmd/mem/cmds_provider.go +++ b/server/cmd/mem/cmds_provider.go @@ -56,7 +56,7 @@ func newProviderListCmd() *cobra.Command { return err } if cfg.Token == "" { - return newCliError(3, "not logged in", "run `mem auth login` first") + return errNotLoggedIn() } c := newHTTPClient(cfg) var resp providerListResp @@ -116,7 +116,7 @@ vectors cannot silently enter different spaces.`, return err } if cfg.Token == "" { - return newCliError(3, "not logged in", "run `mem auth login` first") + return errNotLoggedIn() } c := newHTTPClient(cfg) kind := args[0] @@ -164,7 +164,7 @@ historical provider identity was not recorded.`, return err } if cfg.Token == "" { - return newCliError(3, "not logged in", "run `mem auth login` first") + return errNotLoggedIn() } var resp struct { Provider string `json:"provider"` @@ -201,7 +201,7 @@ func newProviderTestCmd() *cobra.Command { return err } if cfg.Token == "" { - return newCliError(3, "not logged in", "run `mem auth login` first") + return errNotLoggedIn() } c := newHTTPClient(cfg) kind := args[0] diff --git a/server/cmd/mem/cmds_related.go b/server/cmd/mem/cmds_related.go index 4146a31..818c28c 100644 --- a/server/cmd/mem/cmds_related.go +++ b/server/cmd/mem/cmds_related.go @@ -82,7 +82,7 @@ Relation types currently supported: return err } if cfg.Token == "" { - return newCliError(3, "not logged in", "run `mem auth login` first") + return errNotLoggedIn() } c := newHTTPClient(cfg) path := "/v1/files/" + args[0] + "/related" @@ -161,7 +161,7 @@ outgoing rows before recomputing.`, return err } if cfg.Token == "" { - return newCliError(3, "not logged in", "run `mem auth login` first") + return errNotLoggedIn() } c := newHTTPClient(cfg) body := rebuildReq{FileID: file} diff --git a/server/cmd/mem/cmds_remember.go b/server/cmd/mem/cmds_remember.go index 345dadc..e736920 100644 --- a/server/cmd/mem/cmds_remember.go +++ b/server/cmd/mem/cmds_remember.go @@ -115,7 +115,7 @@ Examples: return err } if cfg.Token == "" { - return newCliError(3, "not logged in", "run `mem auth login` first") + return errNotLoggedIn() } var resp map[string]any diff --git a/server/cmd/mem/cmds_search.go b/server/cmd/mem/cmds_search.go index ac866df..d686449 100644 --- a/server/cmd/mem/cmds_search.go +++ b/server/cmd/mem/cmds_search.go @@ -56,7 +56,7 @@ func newSearchCmd() *cobra.Command { return err } if cfg.Token == "" { - return newCliError(3, "not logged in", "run `mem auth login` first") + return errNotLoggedIn() } c := newHTTPClient(cfg) @@ -101,7 +101,7 @@ func newSearchCmd() *cobra.Command { }, } cmd.Flags().StringVar(&typ, "type", "", "mime prefix filter: image|text|application|audio|video") - cmd.Flags().StringVar(&route, "route", "", "search route: text|visual|auto (default auto)") + cmd.Flags().StringVar(&route, "route", "", "search route: text|visual|auto|lexical (default auto)") cmd.Flags().StringVar(&since, "since", "", "YYYY-MM-DD inclusive lower bound on timeline_at") cmd.Flags().StringVar(&until, "until", "", "YYYY-MM-DD inclusive upper bound on timeline_at") cmd.Flags().IntVar(&limit, "limit", 0, "max results (default 10, max 100)") diff --git a/server/cmd/mem/cmds_timeline.go b/server/cmd/mem/cmds_timeline.go index 1edb1f3..eef049a 100644 --- a/server/cmd/mem/cmds_timeline.go +++ b/server/cmd/mem/cmds_timeline.go @@ -43,7 +43,7 @@ func newTimelineCmd() *cobra.Command { return err } if cfg.Token == "" { - return newCliError(3, "not logged in", "run `mem auth login` first") + return errNotLoggedIn() } c := newHTTPClient(cfg) var resp timelineResp diff --git a/server/cmd/mem/config.go b/server/cmd/mem/config.go index 38d75d3..7388795 100644 --- a/server/cmd/mem/config.go +++ b/server/cmd/mem/config.go @@ -91,3 +91,16 @@ func resolveConfig(serverOverride string) (*cliConfig, error) { } return c, nil } + +// configFileExists reports whether a CLI config file is present on disk. +// loadConfig deliberately succeeds without one, so this is the only signal that +// separates "never configured" from "configured, but not logged in" — the +// distinction first-run guidance has to get right. +func configFileExists() bool { + p, err := configPath() + if err != nil { + return false + } + _, err = os.Stat(p) + return err == nil +} diff --git a/server/cmd/mem/main.go b/server/cmd/mem/main.go index 2205c6d..e882e90 100644 --- a/server/cmd/mem/main.go +++ b/server/cmd/mem/main.go @@ -20,6 +20,17 @@ var ( cliWorkspaceOverride string ) +// devCLIVersion is the placeholder this build reports when no version was +// injected. +const devCLIVersion = "dev" + +// cliVersion is the CLI's reported version. Nothing injects it yet: +// .github/workflows/release.yml builds with `-s -w` only, so release binaries +// currently report devCLIVersion, and `mem doctor`'s version_skew check reports +// "skew not computable" instead of inventing a comparison. Wiring this up is a +// release-side change (GOFLAGS/-ldflags=-X main.cliVersion=…), not a CLI one. +var cliVersion = devCLIVersion + func main() { root := newRootCmd() ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) @@ -85,6 +96,7 @@ func newRootCmd() *cobra.Command { root.AddCommand(newModelCmd()) root.AddCommand(newTimelineCmd()) root.AddCommand(newWorkspaceCmd()) + root.AddCommand(newDoctorCmd()) root.AddCommand(newVersionCmd()) return root } diff --git a/server/cmd/mem/qoder_checkpoint.go b/server/cmd/mem/qoder_checkpoint.go deleted file mode 100644 index a7fd069..0000000 --- a/server/cmd/mem/qoder_checkpoint.go +++ /dev/null @@ -1,89 +0,0 @@ -package main - -import ( - "crypto/sha1" - "encoding/hex" - "encoding/json" - "fmt" - "os" - "path/filepath" -) - -// qoderCheckpoint is the persisted per-transcript cursor that makes `mem ingest -// qoder` incremental: it records how many leading lines were already ingested so -// a re-run processes only newly appended messages. Combined with the stable -// Idempotency-Key per line, re-runs are both fast and idempotent. -type qoderCheckpoint struct { - Abs string `json:"abs"` // absolute path of the transcript - Size int64 `json:"size"` // file size at write time (diagnostic) - ModTime string `json:"mtime"` // file mtime at write time (diagnostic) - LastLine int `json:"last_line"` // highest 1-based line already ingested -} - -// qoderCheckpointPath returns the state-dir-relative path for a transcript's -// cursor, keyed by a content-stable hash of its absolute path. -func qoderCheckpointPath(stateDir, abs string) string { - sum := sha1.Sum([]byte(abs)) - return filepath.Join(stateDir, hex.EncodeToString(sum[:])+".json") -} - -// loadQoderCheckpoint reads a transcript cursor. A missing or malformed cursor -// yields the zero value (LastLine 0), meaning "nothing ingested yet" — never a -// hard error, so a corrupt cursor cannot block ingest. -// -// If the on-disk file is now smaller than when the checkpoint was written, the -// file was truncated and rewritten — reset LastLine so re-ingestion does not -// skip the new content at formerly-ingested line numbers. -func loadQoderCheckpoint(stateDir, abs string) qoderCheckpoint { - var cp qoderCheckpoint - p := qoderCheckpointPath(stateDir, abs) - b, err := os.ReadFile(p) - if err != nil { - return cp - } - if err := json.Unmarshal(b, &cp); err != nil { - return qoderCheckpoint{Abs: abs} - } - if cp.Abs == "" { - cp.Abs = abs - } - // Detect truncation: if the file was rewritten and is now smaller, reset - // the cursor so the new content at formerly-ingested line numbers is not - // silently skipped. - if cp.Size > 0 { - if fi, err := os.Stat(abs); err == nil && fi.Size() < cp.Size { - cp.LastLine = 0 - } - } - return cp -} - -// saveQoderCheckpoint atomically persists a transcript cursor. Errors are -// returned (callers may warn without failing the whole ingest). -func saveQoderCheckpoint(stateDir string, cp qoderCheckpoint) error { - p := qoderCheckpointPath(stateDir, cp.Abs) - if err := os.MkdirAll(filepath.Dir(p), 0o700); err != nil { - return fmt.Errorf("create checkpoint dir: %w", err) - } - b, err := json.Marshal(cp) - if err != nil { - return fmt.Errorf("encode checkpoint: %w", err) - } - tmp := p + ".tmp" - if err := os.WriteFile(tmp, b, 0o600); err != nil { - return fmt.Errorf("write checkpoint: %w", err) - } - if err := os.Rename(tmp, p); err != nil { - return fmt.Errorf("commit checkpoint: %w", err) - } - return nil -} - -// fileState returns the size and mtime of a transcript (for diagnostics). -func fileState(abs string) (size int64, mtime string, err error) { - fi, err := os.Stat(abs) - if err != nil { - return 0, "", err - } - return fi.Size(), fi.ModTime().UTC().Format("2006-01-02T15:04:05Z07:00"), nil -} diff --git a/server/cmd/mem/testdata/doctor_healthy.golden.json b/server/cmd/mem/testdata/doctor_healthy.golden.json new file mode 100644 index 0000000..7fcef97 --- /dev/null +++ b/server/cmd/mem/testdata/doctor_healthy.golden.json @@ -0,0 +1,34 @@ +{ + "contract": "mem.doctor", + "schema_version": 1, + "server": "http://127.0.0.1:PORT", + "cli_version": "0.1.0", + "server_version": "0.1.0", + "exit_code": 0, + "checks": [ + { + "name": "server_reachability", + "status": "ok", + "exit_code": 0, + "detail": "healthz ok at http://127.0.0.1:PORT" + }, + { + "name": "credential", + "status": "ok", + "exit_code": 0, + "detail": "token present (from $MEM_TOKEN)" + }, + { + "name": "workspace", + "status": "ok", + "exit_code": 0, + "detail": "server-resolved workspace Personal (11111111-1111-1111-1111-111111111111), role owner; none configured locally, using the server default" + }, + { + "name": "version_skew", + "status": "ok", + "exit_code": 0, + "detail": "CLI and server both report 0.1.0" + } + ] +} diff --git a/server/cmd/memd/main.go b/server/cmd/memd/main.go index 37cb47d..2cf20cc 100644 --- a/server/cmd/memd/main.go +++ b/server/cmd/memd/main.go @@ -8,7 +8,6 @@ import ( "fmt" "log/slog" "net/http" - "net/url" "os" "os/signal" "path/filepath" @@ -34,6 +33,7 @@ import ( "github.com/PeterGuy326/mem/server/internal/memory" "github.com/PeterGuy326/mem/server/internal/provider" "github.com/PeterGuy326/mem/server/internal/queue" + "github.com/PeterGuy326/mem/server/internal/redact" "github.com/PeterGuy326/mem/server/internal/relator" "github.com/PeterGuy326/mem/server/internal/search" "github.com/PeterGuy326/mem/server/internal/storage" @@ -45,7 +45,9 @@ import ( func main() { if err := run(); err != nil { - slog.Error("memd fatal", "err", err) + // run() wraps third-party errors that embed the configured DSN verbatim, + // and slog renders an error value as its text. + slog.Error("memd fatal", "err", redact.Text(err.Error(), redact.StoreURLs)) os.Exit(1) } } @@ -70,6 +72,8 @@ func run() error { "s3_endpoint", cfg.S3Endpoint, "s3_bucket", cfg.S3Bucket, "version", api.Version, + "revision", api.Revision, + "contract", api.ContractVersion, "workspace_bundle_max_bytes", cfg.WorkspaceBundleMaxBytes, "workspace_transfer_max_concurrent", cfg.WorkspaceTransferMaxConcurrent, "workspace_transfer_timeout", cfg.WorkspaceTransferTimeout, @@ -117,7 +121,10 @@ func run() error { "providers", cfg.ManagedEmbeddingProviders, ) } - folderSvc := folder.New(database.Pool) + folderSvc := folder.New(database.Pool, + folder.WithStore(store), + folder.WithLogger(logger), + ) fileSvc := file.New(database.Pool, store, folderSvc) memorySvc := memory.New(database.Pool) durableContextSvc := durablecontext.New(database.Pool, memorySvc) @@ -444,13 +451,9 @@ func redactDSN(s string) string { return redactURLCredentials(s) } +// redactURLCredentials gates the two DSNs this process logs. The store schemes +// are allowed here and nowhere else, so an API-shaped egress can never echo a +// database URL by accident. func redactURLCredentials(raw string) string { - parsed, err := url.Parse(raw) - if err != nil || parsed.User == nil { - return raw - } - if _, hasPassword := parsed.User.Password(); !hasPassword { - return raw - } - return parsed.Redacted() + return redact.URL(raw, redact.StoreURLs) } diff --git a/server/cmd/memd/main_test.go b/server/cmd/memd/main_test.go index f36a0fb..eeaff4a 100644 --- a/server/cmd/memd/main_test.go +++ b/server/cmd/memd/main_test.go @@ -6,6 +6,7 @@ import ( "strings" "testing" + "github.com/PeterGuy326/mem/server/internal/redact" "github.com/PeterGuy326/mem/server/internal/workspacebundle" ) @@ -15,14 +16,36 @@ func TestRedactURLCredentials(t *testing.T) { tests := []struct { name string raw string + want string }{ { name: "postgres", raw: "postgres://mem:database-secret@postgres:5432/mem?sslmode=disable", + want: "postgres://REDACTED@postgres:5432/mem?sslmode=REDACTED", }, { name: "redis", raw: "redis://:redis-secret@redis:6379/0", + want: "redis://REDACTED@redis:6379/0", + }, + { + // The old scrubber echoed this raw because it only masked values with a + // password; the gate treats any userinfo as a credential. + name: "username only", + raw: "postgres://mem@postgres:5432/mem?sslmode=disable", + want: "postgres://REDACTED@postgres:5432/mem?sslmode=REDACTED", + }, + { + // url.Parse reads the scheme as "redis" and parks the rest in Opaque, + // so a u.User check never fires. + name: "redis without a transport scheme", + raw: "redis:redis-secret@redis:6379/0", + want: redact.Placeholder, + }, + { + name: "postgres that fails to parse", + raw: "postgres://mem:database-secret@post gres:5432/mem", + want: redact.Placeholder, }, } for _, test := range tests { @@ -30,23 +53,22 @@ func TestRedactURLCredentials(t *testing.T) { t.Run(test.name, func(t *testing.T) { t.Parallel() got := redactURLCredentials(test.raw) + if got != test.want { + t.Fatalf("redactURLCredentials(%q) = %q, want %q", test.raw, got, test.want) + } if strings.Contains(got, "secret") { t.Fatalf("credentials leaked from %q: %q", test.raw, got) } - if !strings.Contains(got, "@") { - t.Fatalf("redacted URL lost its endpoint: %q", got) - } }) } } -func TestRedactURLCredentialsLeavesPasswordlessValuesAlone(t *testing.T) { +func TestRedactURLCredentialsLeavesCredentialFreeValuesAlone(t *testing.T) { t.Parallel() for _, raw := range []string{ "redis://redis:6379/0", - "redis:6379", - "://not-a-url", + "http://mem.internal:8787", } { if got := redactURLCredentials(raw); got != raw { t.Fatalf("redactURLCredentials(%q) = %q", raw, got) @@ -54,6 +76,23 @@ func TestRedactURLCredentialsLeavesPasswordlessValuesAlone(t *testing.T) { } } +// TestRedactURLCredentialsWithholdsWhatItCannotVerify pins the fail-closed half: +// these shapes carry no visible password today, but the gate cannot prove that +// from a parse it either failed or had to attribute to an unknown scheme, so +// logging them at all would be guessing. +func TestRedactURLCredentialsWithholdsWhatItCannotVerify(t *testing.T) { + t.Parallel() + + for _, raw := range []string{ + "redis:6379", + "://not-a-url", + } { + if got := redactURLCredentials(raw); got != redact.Placeholder { + t.Fatalf("redactURLCredentials(%q) = %q, want %q", raw, got, redact.Placeholder) + } + } +} + func TestWorkspaceTransferBundleLimitsAreConservativeAndConsistent(t *testing.T) { defaults := workspacebundle.DefaultLimits() limits := workspaceTransferBundleLimits() diff --git a/server/go.mod b/server/go.mod index a81d46a..f9cd20f 100644 --- a/server/go.mod +++ b/server/go.mod @@ -10,10 +10,10 @@ require ( github.com/minio/minio-go/v7 v7.0.77 github.com/pressly/goose/v3 v3.22.1 github.com/spf13/cobra v1.8.1 - golang.org/x/crypto v0.54.0 + golang.org/x/crypto v0.55.0 golang.org/x/sys v0.47.0 golang.org/x/term v0.45.0 - google.golang.org/grpc v1.82.1 + google.golang.org/grpc v1.83.2 google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 ) @@ -40,9 +40,9 @@ require ( github.com/spf13/cast v1.10.0 // indirect github.com/spf13/pflag v1.0.9 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/net v0.57.0 // indirect + golang.org/x/net v0.58.0 // indirect golang.org/x/sync v0.22.0 // indirect - golang.org/x/text v0.40.0 // indirect + golang.org/x/text v0.41.0 // indirect golang.org/x/time v0.14.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect ) diff --git a/server/go.sum b/server/go.sum index 345d1a6..b86b253 100644 --- a/server/go.sum +++ b/server/go.sum @@ -94,40 +94,40 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= -go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= -go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= -go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= -go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= -go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= -go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= -go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= -go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= -go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= -golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= -golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= -golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= -golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= -golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= -google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/server/internal/api/api.go b/server/internal/api/api.go index e8c6d07..d7edd63 100644 --- a/server/internal/api/api.go +++ b/server/internal/api/api.go @@ -50,8 +50,21 @@ import ( "github.com/PeterGuy326/mem/server/internal/workspacetransfer" ) -// Version is overridden by ldflags at release-build time. -var Version = "dev" +// Version coordinate set for client/server preflight (mem#151). +// All three are overridden by ldflags at build time: +// +// -X github.com/PeterGuy326/mem/server/internal/api.Version=${SEMVER} +// -X github.com/PeterGuy326/mem/server/internal/api.Revision=${GIT_COMMIT} +// -X github.com/PeterGuy326/mem/server/internal/api.ContractVersion=${CONTRACT} +// +// Version is the semver release tag (e.g. "0.1.1"). +// Revision is the 40-hex git commit the binary was built from. +// ContractVersion is the durable-context wire contract the server speaks. +var ( + Version = "dev" + Revision = "unknown" + ContractVersion = "durable-context.v1" +) // MemoryService is the write/read port used by HTTP handlers. Keeping the // handlers behind an interface makes authorization and error mapping testable @@ -209,7 +222,11 @@ func (s *Server) Router() http.Handler { }) r.Get("/readyz", s.handleReadiness) r.Get("/v1/version", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, http.StatusOK, map[string]any{"version": Version}) + writeJSON(w, http.StatusOK, map[string]any{ + "version": Version, + "revision": Revision, + "contract": ContractVersion, + }) }) // Public auth @@ -1334,9 +1351,9 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) { return } switch req.Route { - case "", search.RouteAuto, search.RouteText, search.RouteVisual: + case "", search.RouteAuto, search.RouteText, search.RouteVisual, search.RouteLexical: default: - writeError(w, http.StatusBadRequest, "bad_route", "route must be auto, text, or visual") + writeError(w, http.StatusBadRequest, "bad_route", "route must be auto, text, visual, or lexical") return } scope, err := pathx.Normalize(req.Scope) diff --git a/server/internal/api/api_test.go b/server/internal/api/api_test.go index a81782d..a38032f 100644 --- a/server/internal/api/api_test.go +++ b/server/internal/api/api_test.go @@ -1,6 +1,7 @@ package api import ( + "encoding/json" "log/slog" "net/http" "net/http/httptest" @@ -79,3 +80,41 @@ func TestCORSDisabledByDefault(t *testing.T) { t.Fatalf("CORS should be off when unconfigured, got Allow-Origin %q", got) } } + +func TestVersionEndpointExposesAllCoordinates(t *testing.T) { + prev_version := Version + prev_revision := Revision + prev_contract := ContractVersion + t.Cleanup(func() { + Version = prev_version + Revision = prev_revision + ContractVersion = prev_contract + }) + Version = "0.2.0" + Revision = "abcdef0123456789abcdef0123456789abcdef01" + ContractVersion = "durable-context.v1" + + s := &Server{Auth: auth.New(nil), Log: slog.Default()} + h := s.Router() + + req := httptest.NewRequest(http.MethodGet, "/v1/version", nil) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String()) + } + var got map[string]string + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if got["version"] != "0.2.0" { + t.Errorf("version = %q, want %q", got["version"], "0.2.0") + } + if got["revision"] != "abcdef0123456789abcdef0123456789abcdef01" { + t.Errorf("revision = %q, want 40-hex commit", got["revision"]) + } + if got["contract"] != "durable-context.v1" { + t.Errorf("contract = %q, want %q", got["contract"], "durable-context.v1") + } +} diff --git a/server/internal/api/managed_embeddings.go b/server/internal/api/managed_embeddings.go index a4e54f3..d9cd6dc 100644 --- a/server/internal/api/managed_embeddings.go +++ b/server/internal/api/managed_embeddings.go @@ -161,8 +161,9 @@ func (s *Server) managedSearcher( s.Search == nil { return nil, nil, entitlement.ErrEntitlementUnavailable } - // A visual-only query does not invoke the managed text embedding provider. - if query.Route == search.RouteVisual { + // A visual-only or lexical query does not invoke the managed text embedding + // provider. + if query.Route == search.RouteVisual || query.Route == search.RouteLexical { return s.Search, nil, nil } spec, err := s.Search.EmbeddingSpec(r.Context(), query.UserID) diff --git a/server/internal/api/managed_embeddings_test.go b/server/internal/api/managed_embeddings_test.go index 46dfd5b..508d538 100644 --- a/server/internal/api/managed_embeddings_test.go +++ b/server/internal/api/managed_embeddings_test.go @@ -573,3 +573,26 @@ func TestReadinessIsDeploymentModeAwareAndPlanIndependent(t *testing.T) { } }) } + +func TestLexicalSearchBypassesManagedEmbeddingReservation(t *testing.T) { + searchFake := &managedSearchFake{spec: "openai:text-embedding-3-small"} + usageFake := &managedEntitlementFake{reserveErr: errors.New("must not reserve lexical search")} + server := &Server{ + Search: searchFake, DeploymentMode: "saas", + ManagedEmbeddingProvider: searchFake.spec, Entitlements: usageFake, + } + // There is deliberately no paid plan, idempotency key, or model context. + request := httptest.NewRequest(http.MethodPost, "/v1/search", nil) + query := search.Query{UserID: uuid.New(), Route: search.RouteLexical, Text: "notes"} + searcher, executor, err := server.managedSearcher(request, "search.query", nil, query) + if err != nil || executor != nil || searcher != searchFake { + t.Fatalf("lexical dispatch: searcher=%T executor=%v err=%v", searcher, executor, err) + } + if _, err := searcher.Search(request.Context(), query); err != nil { + t.Fatal(err) + } + if searchFake.searchCalls != 1 || searchFake.embeddingCalls != 0 || usageFake.reserveCalls != 0 { + t.Fatalf("lexical dispatch invoked model policy: search=%d embedding=%d reserve=%d", + searchFake.searchCalls, searchFake.embeddingCalls, usageFake.reserveCalls) + } +} diff --git a/server/internal/apiclient/apiclient.go b/server/internal/apiclient/apiclient.go index 38dfabd..df01390 100644 --- a/server/internal/apiclient/apiclient.go +++ b/server/internal/apiclient/apiclient.go @@ -12,6 +12,8 @@ import ( "strconv" "strings" "time" + + "github.com/PeterGuy326/mem/server/internal/redact" ) const sourceMetadataHeader = "X-Mem-Source-Metadata" @@ -99,7 +101,7 @@ func (c *Client) DoJSONWithHeaders(ctx context.Context, method, path string, bod } req, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, rdr) if err != nil { - return err + return requestBuildError(method, c.baseURL+path, err) } if body != nil { req.Header.Set("Content-Type", "application/json") @@ -110,7 +112,7 @@ func (c *Client) DoJSONWithHeaders(ctx context.Context, method, path string, bod c.attachAuth(req) resp, err := c.hc.Do(req) if err != nil { - return err + return gateTransportError(err) } defer resp.Body.Close() return decode(resp, out) @@ -169,13 +171,13 @@ func (c *Client) UploadMultipartWithSourceMetadata(ctx context.Context, name, mi req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/v1/files", pr) if err != nil { - return err + return requestBuildError(http.MethodPost, c.baseURL+"/v1/files", err) } req.Header.Set("Content-Type", mw.FormDataContentType()) c.attachAuth(req) resp, err := c.hc.Do(req) if err != nil { - return err + return gateTransportError(err) } defer resp.Body.Close() if werr := <-errCh; werr != nil { @@ -214,9 +216,10 @@ func (c *Client) UploadStreamWithSourceMetadata(ctx context.Context, name, mimeT if err != nil { return err } - req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/v1/files?"+q.Encode(), body) + target := c.baseURL + "/v1/files?" + q.Encode() + req, err := http.NewRequestWithContext(ctx, http.MethodPost, target, body) if err != nil { - return err + return requestBuildError(http.MethodPost, target, err) } if sourceJSON != "" { req.Header.Set(sourceMetadataHeader, sourceJSON) @@ -230,7 +233,7 @@ func (c *Client) UploadStreamWithSourceMetadata(ctx context.Context, name, mimeT c.attachAuth(req) resp, err := c.hc.Do(req) if err != nil { - return err + return gateTransportError(err) } defer resp.Body.Close() return decode(resp, out) @@ -239,14 +242,15 @@ func (c *Client) UploadStreamWithSourceMetadata(ctx context.Context, name, mimeT // DownloadStream returns a streaming reader for GET /v1/files/{id}/content. // Callers MUST close the returned ReadCloser. func (c *Client) DownloadStream(ctx context.Context, fileID string) (io.ReadCloser, string, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+"/v1/files/"+fileID+"/content", nil) + dlTarget := c.baseURL + "/v1/files/" + fileID + "/content" + req, err := http.NewRequestWithContext(ctx, http.MethodGet, dlTarget, nil) if err != nil { - return nil, "", err + return nil, "", requestBuildError(http.MethodGet, dlTarget, err) } c.attachAuth(req) resp, err := c.hc.Do(req) if err != nil { - return nil, "", err + return nil, "", gateTransportError(err) } if resp.StatusCode >= 400 { defer resp.Body.Close() @@ -268,6 +272,51 @@ func marshalSourceMetadata(sourceMetadata *FileSourceMetadata) (string, error) { return string(raw), nil } +// requestBuildError and gateTransportError are the two halves of one rule: the +// configured base URL can carry credentials, and both http.NewRequestWithContext +// and http.Client.Do put that URL into the error they return. Go masks the +// password there but not the username, and masks nothing for a value it parses +// as an opaque scheme, so the URL goes through the shared gate instead. +// +// The wrappers keep the cause reachable through Unwrap so callers can still +// classify a timeout with errors.Is after the text has been rewritten. +func requestBuildError(method, target string, err error) error { + return &redactErr{ + cause: err, + message: fmt.Sprintf("%s %s: %s", method, redact.URL(target, redact.APIURLs), redact.Text(err.Error(), redact.APIURLs)), + } +} + +// newRequest is the construction site for the requests that are issued outside +// the DoJSON and Upload helpers, so a base URL that cannot be parsed fails +// through the same gate here as it does there. +func (c *Client) newRequest(ctx context.Context, method, target string, body io.Reader) (*http.Request, error) { + req, err := http.NewRequestWithContext(ctx, method, target, body) + if err != nil { + return nil, requestBuildError(method, target, err) + } + return req, nil +} + +func gateTransportError(err error) error { + if err == nil { + return nil + } + return &redactErr{ + cause: err, + message: redact.TransportError(err, redact.APIURLs), + } +} + +type redactErr struct { + cause error + message string +} + +func (e *redactErr) Error() string { return e.message } + +func (e *redactErr) Unwrap() error { return e.cause } + func (c *Client) attachAuth(req *http.Request) { if c.token != "" { req.Header.Set("Authorization", "Bearer "+c.token) diff --git a/server/internal/apiclient/apiclient_test.go b/server/internal/apiclient/apiclient_test.go index 0d61000..def3d8a 100644 --- a/server/internal/apiclient/apiclient_test.go +++ b/server/internal/apiclient/apiclient_test.go @@ -155,3 +155,63 @@ func TestUploadStreamWithSourceMetadata(t *testing.T) { t.Fatal("source_metadata leaked into the request URL") } } + +func TestRequestBuildErrorRedactsCredentialedURL(t *testing.T) { + const secret = "bad-token-xyz" + err := New("http://admin:"+secret+"@ho st.example.com:1", "token").DoJSON( + context.Background(), + http.MethodGet, + "/v1/test", + nil, + nil, + ) + if err == nil { + t.Fatal("expected request construction to fail for malformed URL") + } + if strings.Contains(err.Error(), secret) { + t.Fatalf("request-build error leaked credential: %v", err) + } + if !strings.Contains(err.Error(), "REDACTED") { + t.Fatalf("request-build error should redact credentials: %v", err) + } +} + +// A URL whose scheme is really a username parses, so request construction +// succeeds and the failure comes from the transport instead. Every entry point +// has its own http.Client.Do site, so each needs its own case: covering request +// construction does not cover request execution. +func TestTransportErrorRedactsCredentialedURL(t *testing.T) { + const secret = "bad-token-xyz" + schemeless := "admin:" + secret + "@mem.invalid:8787" + + cases := []struct { + name string + call func(*Client) error + }{ + {"DoJSON", func(c *Client) error { + return c.DoJSON(context.Background(), http.MethodGet, "/v1/test", nil, nil) + }}, + {"UploadMultipart", func(c *Client) error { + return c.UploadMultipart(context.Background(), "f.txt", "text/plain", "", strings.NewReader("x"), nil, nil) + }}, + {"UploadStream", func(c *Client) error { + return c.UploadStream(context.Background(), "f.txt", "text/plain", "", 1, nil, strings.NewReader("x"), nil) + }}, + {"DownloadStream", func(c *Client) error { + _, _, err := c.DownloadStream(context.Background(), "file-1") + return err + }}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := tc.call(New(schemeless, "token")) + if err == nil { + t.Fatal("expected the transport to fail for a schemeless base URL") + } + if strings.Contains(err.Error(), secret) { + t.Errorf("%s transport error leaked credential: %v", tc.name, err) + } + }) + } +} diff --git a/server/internal/apiclient/workspace_transfer.go b/server/internal/apiclient/workspace_transfer.go index 6273a97..1c64dcc 100644 --- a/server/internal/apiclient/workspace_transfer.go +++ b/server/internal/apiclient/workspace_transfer.go @@ -70,7 +70,7 @@ type WorkspaceImportConflict struct { // in memory. The server does not publish response headers until its complete // archive has been built and validated. func (c *Client) ExportWorkspace(ctx context.Context) (*WorkspaceBundleDownload, error) { - req, err := http.NewRequestWithContext( + req, err := c.newRequest( ctx, http.MethodGet, c.baseURL+"/v1/workspaces/current/export", @@ -133,7 +133,7 @@ func (c *Client) ImportWorkspace( return nil, fmt.Errorf("workspace bundle size must be -1 or non-negative") } query := url.Values{"mode": []string{mode}} - req, err := http.NewRequestWithContext( + req, err := c.newRequest( ctx, http.MethodPost, c.baseURL+"/v1/workspaces/current/import?"+query.Encode(), diff --git a/server/internal/db/hnsw_migration_test.go b/server/internal/db/hnsw_migration_test.go new file mode 100644 index 0000000..4bdc846 --- /dev/null +++ b/server/internal/db/hnsw_migration_test.go @@ -0,0 +1,172 @@ +package db + +import ( + "context" + "database/sql" + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/pressly/goose/v3" +) + +const hnswCorpusUser = "00000000-0000-0000-0000-000000000173" + +// Populated 24→25→24→25, ingest after index build, wrong-dimension rejection, +// and EXPLAIN of the shipping cosine-order text/visual shapes. +func TestHNSWMigrationPostgres(t *testing.T) { + dsn := os.Getenv("MEM_HNSW_TEST_DB") + if dsn == "" { + t.Skip("MEM_HNSW_TEST_DB not set; requires a fresh owned test database") + } + cfg, err := pgx.ParseConfig(dsn) + if err != nil { + t.Fatal(err) + } + if !strings.HasSuffix(cfg.Database, "_test") { + t.Fatalf("refusing non-test database %q", cfg.Database) + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + db, err := sql.Open("pgx", dsn) + if err != nil { + t.Fatal(err) + } + defer db.Close() + var history sql.NullString + if err := db.QueryRowContext(ctx, "SELECT to_regclass('goose_db_version')::text").Scan(&history); err != nil { + t.Fatal(err) + } + if history.Valid { + t.Fatal("refusing existing migration history; provide a new owned test database") + } + goose.SetBaseFS(migrationsFS) + if err := goose.SetDialect("postgres"); err != nil { + t.Fatal(err) + } + if err := goose.UpToContext(ctx, db, "migrations", 24); err != nil { + t.Fatal(err) + } + exec := func(query string) { + t.Helper() + if _, err := db.ExecContext(ctx, query); err != nil { + t.Fatal(err) + } + } + exec(`INSERT INTO users(id,email,password_hash) + VALUES ('` + hnswCorpusUser + `','hnsw@example.test','test'); + INSERT INTO files(id,user_id,name,path,size,sha256,mime,storage_key) + SELECT md5(i::text)::uuid,'` + hnswCorpusUser + `', + 'fixture-' || i, '/hnsw', 0, 'sha-' || i, 'text/plain', 'fixture-' || i + FROM generate_series(1,2000) i`) + for _, kind := range []string{"text", "visual", "face"} { + dim, extraCols, extraValues := 512, "", "" + if kind == "text" { + dim, extraCols, extraValues = 768, ",chunk_index,chunk_text", ",0,'fixture'" + } + exec(fmt.Sprintf(`INSERT INTO embeddings_%s(file_id,embedding%s) + SELECT id,array_fill(0.1::real,ARRAY[%d])::vector%s FROM files`, kind, extraCols, dim, extraValues)) + } + assertState := func(wantIndexes, wantRows int) { + t.Helper() + for _, kind := range []string{"text", "visual", "face"} { + var indexes, rows int + if err := db.QueryRowContext(ctx, `SELECT count(*) FROM pg_index i + JOIN pg_class c ON c.oid=i.indexrelid JOIN pg_am a ON a.oid=c.relam + WHERE i.indrelid=($1::text)::regclass AND c.relname=$2 AND a.amname='hnsw' + AND i.indisvalid AND pg_get_indexdef(i.indexrelid) LIKE '%vector_cosine_ops%'`, + "embeddings_"+kind, "idx_embeddings_"+kind+"_embedding_hnsw").Scan(&indexes); err != nil { + t.Fatal(err) + } + if err := db.QueryRowContext(ctx, "SELECT count(*) FROM embeddings_"+kind+" WHERE embedding IS NOT NULL").Scan(&rows); err != nil { + t.Fatal(err) + } + if indexes != wantIndexes || rows != wantRows { + t.Fatalf("%s: valid cosine indexes=%d want=%d, preserved vectors=%d want=%d", kind, indexes, wantIndexes, rows, wantRows) + } + } + } + assertState(0, 2000) + if err := goose.UpToContext(ctx, db, "migrations", 25); err != nil { + t.Fatal(err) + } + assertState(1, 2000) + if err := goose.DownToContext(ctx, db, "migrations", 24); err != nil { + t.Fatal(err) + } + assertState(0, 2000) + if err := goose.UpToContext(ctx, db, "migrations", 25); err != nil { + t.Fatal(err) + } + assertState(1, 2000) + exec(`INSERT INTO files(id,user_id,name,path,size,sha256,mime,storage_key) + VALUES (md5('2001')::uuid,'` + hnswCorpusUser + `', + 'after-index','/hnsw',0,'sha-2001','text/plain','after-index')`) + for _, kind := range []string{"text", "visual", "face"} { + dim, extraCols, extraValues := 512, "", "" + if kind == "text" { + dim, extraCols, extraValues = 768, ",chunk_index,chunk_text", ",0,'after-index'" + } + exec(fmt.Sprintf(`INSERT INTO embeddings_%s(file_id,embedding%s) + VALUES (md5('2001')::uuid,array_fill(0.2::real,ARRAY[%d])::vector%s)`, kind, extraCols, dim, extraValues)) + _, err := db.ExecContext(ctx, fmt.Sprintf(`UPDATE embeddings_%s + SET embedding=array_fill(0.1::real,ARRAY[%d])::vector WHERE file_id=md5('2001')::uuid`, kind, dim-1)) + if err == nil || !strings.Contains(strings.ToLower(err.Error()), "dimension") { + t.Fatalf("%s: wrong dimensionality must fail, got %v", kind, err) + } + } + assertState(1, 2001) + if _, err := db.ExecContext(ctx, `ANALYZE embeddings_text; ANALYZE embeddings_visual; ANALYZE files`); err != nil { + t.Fatal(err) + } + assertIndexScan(t, ctx, db, "idx_embeddings_text_embedding_hnsw", ` + WITH nearest AS ( + SELECT e.id, e.file_id, e.embedding <=> array_fill(0.1::real, ARRAY[768])::vector AS dist + FROM embeddings_text e + ORDER BY e.embedding <=> array_fill(0.1::real, ARRAY[768])::vector ASC + LIMIT 10 + ) + SELECT e.id, f.id + FROM nearest e + JOIN files f ON f.id = e.file_id + WHERE f.user_id = '`+hnswCorpusUser+`'::uuid + ORDER BY e.dist ASC`) + assertIndexScan(t, ctx, db, "idx_embeddings_visual_embedding_hnsw", ` + SELECT e.file_id + FROM embeddings_visual e + ORDER BY e.embedding <=> array_fill(0.1::real, ARRAY[512])::vector ASC + LIMIT 10`) + if err := (&DB{url: dsn}).Migrate(ctx); err != nil { + t.Fatal(err) + } + t.Log("PASS: populated 24->25->24->25, valid cosine HNSW, ingest, dimension rejection, text/visual EXPLAIN uses HNSW") +} + +func assertIndexScan(t *testing.T, ctx context.Context, db *sql.DB, index, query string) { + t.Helper() + rows, err := db.QueryContext(ctx, "EXPLAIN (ANALYZE, BUFFERS) "+query) + if err != nil { + t.Fatalf("EXPLAIN %s: %v", index, err) + } + defer rows.Close() + var plan strings.Builder + for rows.Next() { + var line string + if err := rows.Scan(&line); err != nil { + t.Fatal(err) + } + plan.WriteString(line) + plan.WriteByte('\n') + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + text := plan.String() + t.Logf("EXPLAIN %s:\n%s", index, text) + if !strings.Contains(text, index) { + t.Fatalf("shipping plan did not use %s:\n%s", index, text) + } +} diff --git a/server/internal/db/migration_sequence_test.go b/server/internal/db/migration_sequence_test.go new file mode 100644 index 0000000..5908cf3 --- /dev/null +++ b/server/internal/db/migration_sequence_test.go @@ -0,0 +1,132 @@ +package db + +import ( + "context" + "database/sql" + "errors" + "os" + "strconv" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/pressly/goose/v3" +) + +func contiguousMigrationHead(t *testing.T) int { + t.Helper() + entries, err := migrationsFS.ReadDir("migrations") + if err != nil { + t.Fatal(err) + } + for i, entry := range entries { + prefix, _, ok := strings.Cut(entry.Name(), "_") + version, err := strconv.Atoi(prefix) + if !ok || err != nil || version != i+1 { + t.Fatalf("migration sequence gap: want %04d, got %q; include predecessors before deployment", i+1, entry.Name()) + } + } + return len(entries) +} + +func TestMigrationFilesContiguous(t *testing.T) { + contiguousMigrationHead(t) +} + +// The runner supplies a NEW, owned database, distinct from the shared +// MEM_TEST_DB integration fixture. Never roll back or renumber deployed DDL. +func TestMigrationUpgradeSequence(t *testing.T) { + head := contiguousMigrationHead(t) + dsn := os.Getenv("MEM_MIGRATION_SEQUENCE_TEST_DB") + if dsn == "" { + t.Skip("MEM_MIGRATION_SEQUENCE_TEST_DB not set; requires a fresh owned test database") + } + cfg, err := pgx.ParseConfig(dsn) + if err != nil { + t.Fatal(err) + } + if !strings.HasSuffix(cfg.Database, "_test") { + t.Fatalf("refusing non-test database %q", cfg.Database) + } + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + sqldb, err := sql.Open("pgx", dsn) + if err != nil { + t.Fatal(err) + } + defer sqldb.Close() + var existing sql.NullString + if err := sqldb.QueryRowContext(ctx, "SELECT to_regclass('goose_db_version')::text").Scan(&existing); err != nil { + t.Fatal(err) + } + if existing.Valid { + t.Fatal("sequence regression requires a fresh database; refusing an existing migration history") + } + goose.SetBaseFS(migrationsFS) + if err := goose.SetDialect("postgres"); err != nil { + t.Fatal(err) + } + var userID, fileID uuid.UUID + for version := 23; version <= head; version++ { + // Match production's strict Goose behavior: no AllowMissing option. + if err := goose.UpToContext(ctx, sqldb, "migrations", int64(version)); err != nil { + t.Fatalf("upgrade to %d: %v", version, err) + } + actual, err := goose.GetDBVersionContext(ctx, sqldb) + if err != nil || actual != int64(version) { + t.Fatalf("migration head = %d, want %d, err=%v", actual, version, err) + } + var applied int + if err := sqldb.QueryRowContext(ctx, "SELECT count(DISTINCT version_id) FROM goose_db_version WHERE is_applied AND version_id BETWEEN 1 AND $1", version).Scan(&applied); err != nil || applied != version { + t.Fatalf("applied history has %d of %d predecessors, err=%v", applied, version, err) + } + if version == 23 { + if err := sqldb.QueryRowContext(ctx, "INSERT INTO users(email,password_hash) VALUES($1,'test') RETURNING id", uuid.NewString()+"@example.test").Scan(&userID); err != nil { + t.Fatal(err) + } + if err := sqldb.QueryRowContext(ctx, "INSERT INTO files(user_id,name,path,size,sha256,mime,storage_key) VALUES($1,'migration-sequence.txt','/fixture',0,'fixture','text/plain','test://sequence') RETURNING id", userID).Scan(&fileID); err != nil { + t.Fatal(err) + } + if _, err := sqldb.ExecContext(ctx, "INSERT INTO embeddings_text(file_id,chunk_index,chunk_text,embedding) SELECT $1,0,'populated duplicate',array_fill(0.1::real,ARRAY[768])::vector FROM generate_series(1,2)", fileID); err != nil { + t.Fatal(err) + } + } + if version >= 24 { + var lexical bool + if err := sqldb.QueryRowContext(ctx, "SELECT search_tsv @@ plainto_tsquery('simple','migration-sequence.txt') FROM files WHERE id=$1", fileID).Scan(&lexical); err != nil || !lexical { + t.Fatalf("populated lexical backfill: %v, err=%v", lexical, err) + } + } + if version >= 25 { + var indexes int + if err := sqldb.QueryRowContext(ctx, `SELECT count(*) FROM pg_index i JOIN pg_class c ON c.oid=i.indexrelid JOIN pg_am am ON am.oid=c.relam + WHERE i.indisvalid AND am.amname='hnsw' AND c.relname IN + ('idx_embeddings_text_embedding_hnsw','idx_embeddings_visual_embedding_hnsw','idx_embeddings_face_embedding_hnsw')`).Scan(&indexes); err != nil || indexes != 3 { + t.Fatalf("valid HNSW indexes=%d, err=%v", indexes, err) + } + } + var chunks int + wantChunks := 2 + if version >= 26 { + wantChunks = 1 + } + if err := sqldb.QueryRowContext(ctx, "SELECT count(*) FROM embeddings_text WHERE file_id=$1", fileID).Scan(&chunks); err != nil || chunks != wantChunks { + t.Fatalf("preserved chunks=%d, want=%d, err=%v", chunks, wantChunks, err) + } + t.Logf("PASS: strict Goose upgrade to %d; complete history and populated data preserved", version) + } + if head >= 26 { + _, err := sqldb.ExecContext(ctx, "INSERT INTO embeddings_text(file_id,chunk_index,chunk_text) VALUES($1,0,'duplicate')", fileID) + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || pgErr.Code != "23505" { + t.Fatalf("expected duplicate rejection 23505, got %v", err) + } + } + // The real startup path must accept the upgraded history unchanged. + if err := (&DB{url: dsn}).Migrate(ctx); err != nil { + t.Fatalf("production startup after sequential upgrade: %v", err) + } +} diff --git a/server/internal/db/migrations/0001_init.sql b/server/internal/db/migrations/0001_init.sql index a5f9bf2..e6a6328 100644 --- a/server/internal/db/migrations/0001_init.sql +++ b/server/internal/db/migrations/0001_init.sql @@ -106,8 +106,8 @@ CREATE TABLE IF NOT EXISTS embeddings_text ( embedding vector(768) ); CREATE INDEX IF NOT EXISTS idx_embeddings_text_file ON embeddings_text (file_id); --- HNSW index will be added by worker once we settle on a model dimension. Kept off here --- because pgvector requires the table to have data of consistent dim before building. +-- Cosine HNSW for embeddings_text/visual/face is created in migration 0025 +-- once vector(768)/vector(512) dimensions are fixed. -- +goose StatementEnd -- +goose StatementBegin diff --git a/server/internal/db/migrations/0019_versioned_index_generations.sql b/server/internal/db/migrations/0019_versioned_index_generations.sql index bd173c8..31a06d9 100644 --- a/server/internal/db/migrations/0019_versioned_index_generations.sql +++ b/server/internal/db/migrations/0019_versioned_index_generations.sql @@ -264,8 +264,10 @@ CREATE INDEX idx_index_generation_targets_file_hash -- +goose StatementBegin -- `vector` intentionally has no table-wide dimension. Every row is validated -- against its immutable generation.output_dimension by the canonical service. --- Future ANN indexes must be route/dimension-specific expression or partition --- indexes; silently padding or truncating vectors is never allowed. +-- ANN indexes on this undimensioned table must be route/dimension-specific +-- expression or partition indexes; silently padding or truncating vectors is +-- never allowed. Legacy embeddings_text/visual/face tables are indexed by +-- migration 0025; this table waits for the generation executor. CREATE TABLE index_generation_vectors ( generation_id uuid NOT NULL REFERENCES index_generations(id) ON DELETE CASCADE, workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, diff --git a/server/internal/db/migrations/0024_files_lexical_search.sql b/server/internal/db/migrations/0024_files_lexical_search.sql new file mode 100644 index 0000000..f6a46c2 --- /dev/null +++ b/server/internal/db/migrations/0024_files_lexical_search.sql @@ -0,0 +1,26 @@ +-- +goose Up +-- Model-free lexical lane for the file corpus. Mirrors the FTS + trigram +-- shape already established for memories (0008) so that filename (name column) +-- substring search works without an embedding worker. + +-- +goose StatementBegin +ALTER TABLE files + ADD COLUMN IF NOT EXISTS search_tsv tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', coalesce(name, '')) + ) STORED; +-- +goose StatementEnd + +-- +goose StatementBegin +CREATE INDEX IF NOT EXISTS idx_files_search_tsv + ON files USING gin (search_tsv); +-- +goose StatementEnd + +-- +goose StatementBegin +CREATE INDEX IF NOT EXISTS idx_files_name_trgm + ON files USING gin (lower(name) gin_trgm_ops); +-- +goose StatementEnd + +-- +goose Down +DROP INDEX IF EXISTS idx_files_name_trgm; +DROP INDEX IF EXISTS idx_files_search_tsv; +ALTER TABLE files DROP COLUMN IF EXISTS search_tsv; diff --git a/server/internal/db/migrations/0025_ann_hnsw_indexes.sql b/server/internal/db/migrations/0025_ann_hnsw_indexes.sql new file mode 100644 index 0000000..fdfb03d --- /dev/null +++ b/server/internal/db/migrations/0025_ann_hnsw_indexes.sql @@ -0,0 +1,47 @@ +-- +goose Up +-- Transactional startup migration: CREATE INDEX blocks writes on these tables +-- until commit. Schedule a maintenance window for a populated deployment. +-- +-- Operator class vector_cosine_ops matches the <=> operator used by search +-- and relator. pgvector defaults m=16, ef_construction=64; tune only after +-- representative recall/build/ingest measurements. +-- +-- Wrong-dimension vectors fail at INSERT/UPDATE against the fixed +-- vector(768)/vector(512) columns, before index maintenance. NULL embeddings +-- are allowed by the table DDL and are not present in a cosine HNSW index. +-- This file does not use CREATE INDEX CONCURRENTLY: a failed concurrent build +-- leaves an INVALID index that IF NOT EXISTS will skip. +-- +-- index_generation_vectors is intentionally not indexed (undimensioned; no +-- generation executor yet). Face clustering is still in-process; the face +-- index is DDL for a future SQL kNN, not a measured face-query speedup. +-- +-- Refs: https://github.com/bytefolk/mem/issues/173 + +-- +goose StatementBegin +CREATE INDEX IF NOT EXISTS idx_embeddings_text_embedding_hnsw + ON embeddings_text USING hnsw (embedding vector_cosine_ops); +-- +goose StatementEnd + +-- +goose StatementBegin +CREATE INDEX IF NOT EXISTS idx_embeddings_visual_embedding_hnsw + ON embeddings_visual USING hnsw (embedding vector_cosine_ops); +-- +goose StatementEnd + +-- +goose StatementBegin +CREATE INDEX IF NOT EXISTS idx_embeddings_face_embedding_hnsw + ON embeddings_face USING hnsw (embedding vector_cosine_ops); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP INDEX IF EXISTS idx_embeddings_face_embedding_hnsw; +-- +goose StatementEnd + +-- +goose StatementBegin +DROP INDEX IF EXISTS idx_embeddings_visual_embedding_hnsw; +-- +goose StatementEnd + +-- +goose StatementBegin +DROP INDEX IF EXISTS idx_embeddings_text_embedding_hnsw; +-- +goose StatementEnd diff --git a/server/internal/durablememory/contract_test.go b/server/internal/durablememory/contract_test.go new file mode 100644 index 0000000..9c5bd56 --- /dev/null +++ b/server/internal/durablememory/contract_test.go @@ -0,0 +1,474 @@ +package durablememory + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/google/uuid" +) + +const ( + exampleWorkspaceID = "11111111-1111-4111-8111-111111111111" + exampleMemoryID = "22222222-2222-4222-8222-222222222222" + exampleGrantID = "33333333-3333-4333-8333-333333333333" + examplePrincipal = "position.repo-owner" + examplePosition = "repo-owner" + exampleScope = "/workspaces/44444444-4444-4444-8444-444444444444/positions/repo-owner" + exampleSourceDig = "sha256:bee60ba20052b2969621c28f7378297c3b38cfa4eba66b22ff0df381447b2f8c" + examplePermDig = "sha256:98056de97087164dd9e0f5235cba6019d9576230faa1e37b104e735e5b5729a6" + exampleText = "Search APIs must match title OR body and return matchField." + exampleTextDig = "sha256:bfa1e9bb9bacbabafb44c2c89ee7a108e81e10628c53a645af5b49eff081166f" +) + +func TestDecodeRejectsFreeStringScope(t *testing.T) { + raw := []byte(`{ + "contract": "durable-memory.v1", + "memory_id": "` + exampleMemoryID + `", + "kind": "project_decision", + "scope": "/workspaces/ws_1/positions/repo-owner", + "text": "Search APIs must match title OR body.", + "trust": "untrusted", + "authority": "none" + }`) + _, err := DecodeRecord(raw) + if err == nil { + t.Fatal("free-string scope must fail closed") + } + if !errors.Is(err, ErrMalformed) { + t.Fatalf("want ErrMalformed, got %v", err) + } +} + +func TestPermissionDigestBindsWorkspacePrincipalAndScope(t *testing.T) { + rec := validRecord(t) + if rec.Grant.PermissionDigest != PermissionDigest(rec.Binding, rec.Grant.Mode, rec.Grant.GrantVersion) { + t.Fatal("valid record permission_digest drifted from canonical tuple") + } + body := mustJSON(t, rec) + var loose map[string]any + if err := json.Unmarshal(body, &loose); err != nil { + t.Fatal(err) + } + grant, _ := loose["grant"].(map[string]any) + grant["permission_digest"] = exampleTextDig + raw, err := json.Marshal(loose) + if err != nil { + t.Fatal(err) + } + if _, err := DecodeRecord(raw); err == nil { + t.Fatal("permission_digest that does not bind the grant tuple must fail") + } +} + +func TestDecodeRequiresPrincipalGrantBinding(t *testing.T) { + rec := validRecord(t) + body := mustJSON(t, rec) + var loose map[string]any + if err := json.Unmarshal(body, &loose); err != nil { + t.Fatal(err) + } + delete(loose, "binding") + raw, err := json.Marshal(loose) + if err != nil { + t.Fatal(err) + } + if _, err := DecodeRecord(raw); err == nil { + t.Fatal("record without binding must be rejected") + } + + loose = map[string]any{} + if err := json.Unmarshal(mustJSON(t, rec), &loose); err != nil { + t.Fatal(err) + } + delete(loose, "grant") + raw, err = json.Marshal(loose) + if err != nil { + t.Fatal(err) + } + if _, err := DecodeRecord(raw); err == nil { + t.Fatal("record without grant must be rejected") + } +} + +func TestExampleRoundTripMatchesCheckedInSchemaShape(t *testing.T) { + raw, err := os.ReadFile(examplePath(t)) + if err != nil { + t.Fatal(err) + } + rec, err := DecodeRecord(raw) + if err != nil { + t.Fatalf("example must decode: %v", err) + } + if rec.Contract != ContractVersion { + t.Fatalf("contract = %q", rec.Contract) + } + if rec.Binding.Principal != examplePrincipal { + t.Fatalf("principal = %q", rec.Binding.Principal) + } + if rec.Binding.MemoryScope != exampleScope { + t.Fatalf("memory_scope = %q", rec.Binding.MemoryScope) + } + if rec.Grant.RevokedAt != nil { + t.Fatal("example grant must be unrevoked") + } + if rec.Grant.PermissionDigest != examplePermDig { + t.Fatalf("permission_digest = %q", rec.Grant.PermissionDigest) + } +} + +func TestRecallEligibility(t *testing.T) { + now := time.Date(2026, 9, 18, 12, 0, 0, 0, time.UTC) + caller := RecallCaller{ + WorkspaceID: exampleWorkspaceID, + Principal: examplePrincipal, + MemoryScope: exampleScope, + At: now, + } + + t.Run("active granted in-scope is eligible", func(t *testing.T) { + rec := validRecord(t) + got := EvaluateRecall(rec, caller) + if !got.Eligible || got.OmitReason != "" { + t.Fatalf("got %+v", got) + } + }) + + t.Run("expired unpinned is not eligible", func(t *testing.T) { + rec := validRecord(t) + expired := now.Add(-time.Hour) + rec.ExpiresAt = &expired + got := EvaluateRecall(rec, caller) + if got.Eligible || got.OmitReason != OmitExpired { + t.Fatalf("got %+v", got) + } + if PhysicalDeleteImplied(rec, caller) { + t.Fatal("TTL must not imply physical delete of the source log") + } + }) + + t.Run("pin preserves TTL eligibility but not permission", func(t *testing.T) { + rec := validRecord(t) + expired := now.Add(-time.Hour) + rec.ExpiresAt = &expired + rec.Pinned = true + got := EvaluateRecall(rec, caller) + if !got.Eligible { + t.Fatalf("pinned expired record should remain eligible: %+v", got) + } + + foreign := caller + foreign.Principal = "position.other-agent" + denied := EvaluateRecall(rec, foreign) + if denied.Eligible || denied.OmitReason != OmitOutOfScope { + t.Fatalf("pin must not enlarge permission: %+v", denied) + } + }) + + t.Run("revoked grant is not eligible even when pinned", func(t *testing.T) { + rec := validRecord(t) + rec.Pinned = true + revoked := now.Add(-time.Minute) + rec.Grant.RevokedAt = &revoked + got := EvaluateRecall(rec, caller) + if got.Eligible || got.OmitReason != OmitRevoked { + t.Fatalf("got %+v", got) + } + }) + + t.Run("superseded archived forgotten malformed", func(t *testing.T) { + cases := []struct { + name string + mutate func(*Record) + want string + }{ + {"superseded", func(r *Record) { r.Lifecycle = LifecycleSuperseded }, OmitSuperseded}, + {"archived", func(r *Record) { r.Lifecycle = LifecycleArchived }, OmitSuperseded}, + {"forgotten", func(r *Record) { + r.Lifecycle = LifecycleForgotten + r.Text = "" + r.Digest = ContentDigest("") + }, OmitForgotten}, + {"malformed digest", func(r *Record) { r.Digest = "sha256:ab" }, OmitMalformed}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rec := validRecord(t) + tc.mutate(&rec) + got := EvaluateRecall(rec, caller) + if got.Eligible || got.OmitReason != tc.want { + t.Fatalf("got %+v want omit %s", got, tc.want) + } + }) + } + }) + + t.Run("out of scope is caller mismatch not record mutation", func(t *testing.T) { + rec := validRecord(t) + cases := []struct { + name string + caller RecallCaller + }{ + {"cross principal", func() RecallCaller { c := caller; c.Principal = "position.other-agent"; return c }()}, + {"cross workspace", func() RecallCaller { + c := caller + c.WorkspaceID = "55555555-5555-4555-8555-555555555555" + return c + }()}, + {"scope mismatch", func() RecallCaller { + c := caller + c.MemoryScope = "/workspaces/44444444-4444-4444-8444-444444444444/positions/other-agent" + return c + }()}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := EvaluateRecall(rec, tc.caller) + if got.Eligible || got.OmitReason != OmitOutOfScope { + t.Fatalf("got %+v", got) + } + }) + } + }) +} + +func TestExactReadback(t *testing.T) { + stored := validRecord(t) + same := stored + if err := ExactReadback(stored, same); err != nil { + t.Fatalf("identical records must read back: %v", err) + } + + tweaked := stored + tweaked.Text = stored.Text + " (drift)" + if err := ExactReadback(stored, tweaked); !errors.Is(err, ErrReadbackMismatch) { + t.Fatalf("want ErrReadbackMismatch, got %v", err) + } + + tweaked = stored + tweaked.StateVersion++ + if err := ExactReadback(stored, tweaked); !errors.Is(err, ErrReadbackMismatch) { + t.Fatalf("state_version drift must fail: %v", err) + } +} + +func TestForgetIsPermissionedAndNeverLocalFake(t *testing.T) { + rec := validRecord(t) + + denied := EvaluateForget(rec, ForgetActor{ + WorkspaceID: exampleWorkspaceID, + Principal: examplePrincipal, + MemoryScope: exampleScope, + TokenScopes: []string{"read", "write"}, + WorkspaceRoleAllowsDelete: true, + }) + if denied.Authorized || denied.LocalFake || denied.ErrorCode != ErrorForgetDenied { + t.Fatalf("read/write grant must not forget: %+v", denied) + } + + foreign := EvaluateForget(rec, ForgetActor{ + WorkspaceID: exampleWorkspaceID, + Principal: "position.other-agent", + MemoryScope: exampleScope, + TokenScopes: []string{"read", "write", "delete"}, + WorkspaceRoleAllowsDelete: true, + }) + if foreign.Authorized || foreign.LocalFake || foreign.ErrorCode != ErrorForgetDenied { + t.Fatalf("cross-principal forget must fail visibly: %+v", foreign) + } + + ok := EvaluateForget(rec, ForgetActor{ + WorkspaceID: exampleWorkspaceID, + Principal: examplePrincipal, + MemoryScope: exampleScope, + TokenScopes: []string{"read", "write", "delete"}, + WorkspaceRoleAllowsDelete: true, + }) + if !ok.Authorized || ok.LocalFake || ok.Deleted || ok.ErrorCode != "" { + t.Fatalf("authorized forget is a server intent, not a local delete: %+v", ok) + } +} + +func TestReceiptSurfacesGrantAndRevocation(t *testing.T) { + now := time.Date(2026, 9, 18, 12, 0, 0, 0, time.UTC) + caller := RecallCaller{ + WorkspaceID: exampleWorkspaceID, + Principal: examplePrincipal, + MemoryScope: exampleScope, + At: now, + } + + rec := validRecord(t) + receipt := BuildReceipt(rec, caller) + if receipt.Contract != ContractVersion { + t.Fatalf("contract = %q", receipt.Contract) + } + if !receipt.Eligible { + t.Fatalf("eligible receipt: %+v", receipt) + } + if receipt.Grant.Status != GrantStatusActive { + t.Fatalf("grant status = %q", receipt.Grant.Status) + } + if receipt.Grant.GrantID != rec.Grant.GrantID { + t.Fatalf("grant id missing from receipt") + } + if receipt.Grant.PermissionDigest != rec.Grant.PermissionDigest { + t.Fatal("permission digest must enter the receipt") + } + if receipt.Readback == nil || receipt.Readback.MemoryID != rec.MemoryID { + t.Fatal("eligible receipt must carry exact readback") + } + + revoked := now.Add(-time.Second) + rec.Grant.RevokedAt = &revoked + receipt = BuildReceipt(rec, caller) + if receipt.Eligible || receipt.OmitReason != OmitRevoked { + t.Fatalf("revoked receipt: %+v", receipt) + } + if receipt.Grant.Status != GrantStatusRevoked { + t.Fatalf("revocation must be visible on receipt, got %q", receipt.Grant.Status) + } + if receipt.Readback != nil { + t.Fatal("ineligible recall must not return payload readback") + } +} + +func TestReceiptHidesOutOfScopeMetadata(t *testing.T) { + now := time.Date(2026, 9, 18, 12, 0, 0, 0, time.UTC) + rec := validRecord(t) + foreign := RecallCaller{ + WorkspaceID: exampleWorkspaceID, + Principal: "position.other-agent", + MemoryScope: exampleScope, + At: now, + } + receipt := BuildReceipt(rec, foreign) + if receipt.Eligible || receipt.MemoryID != uuid.Nil || receipt.Locator != "" || receipt.Grant.GrantID != uuid.Nil { + t.Fatalf("out-of-scope receipt must not disclose the foreign record: %+v", receipt) + } +} + +func TestGrantModeIsReadOnly(t *testing.T) { + rec := validRecord(t) + body := mustJSON(t, rec) + var loose map[string]any + if err := json.Unmarshal(body, &loose); err != nil { + t.Fatal(err) + } + grant, _ := loose["grant"].(map[string]any) + grant["mode"] = "delete" + grant["permission_digest"] = PermissionDigest(rec.Binding, "delete", rec.Grant.GrantVersion) + raw, err := json.Marshal(loose) + if err != nil { + t.Fatal(err) + } + if _, err := DecodeRecord(raw); err == nil { + t.Fatal("grant.mode other than read must fail; forget uses token delete scope") + } +} + +func TestCheckedInSchemaForbidsScopeProperty(t *testing.T) { + raw, err := os.ReadFile(schemaPath(t)) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(raw), `"scope"`) { + t.Fatal("durable-memory.v1 schema must not declare a free scope string") + } + required := []string{`"binding"`, `"grant"`, `"principal"`, `"memory_scope"`, `"workspace_id"`, `"permission_digest"`, `"revoked_at"`} + for _, key := range required { + if !strings.Contains(string(raw), key) { + t.Fatalf("schema missing %s", key) + } + } +} + +func validRecord(t *testing.T) Record { + t.Helper() + created := time.Date(2026, 9, 17, 12, 0, 0, 0, time.UTC) + expires := time.Date(2026, 12, 17, 12, 0, 0, 0, time.UTC) + conf := 0.7 + rec := Record{ + Contract: ContractVersion, + MemoryID: uuid.MustParse(exampleMemoryID), + Kind: KindProjectDecision, + Binding: Binding{ + WorkspaceID: uuid.MustParse(exampleWorkspaceID), + PositionID: examplePosition, + Principal: examplePrincipal, + MemoryScope: exampleScope, + }, + Grant: Grant{ + GrantID: uuid.MustParse(exampleGrantID), + Mode: GrantModeRead, + GrantVersion: 1, + CapabilityGrant: CapabilityGrantRef{ + SchemaVersion: CapabilityGrantV1, + Server: "mem", + }, + GrantedAt: created, + }, + Source: Source{ + Kind: SourceKindSegment, + ID: "seg_01", + Digest: exampleSourceDig, + }, + Citations: []string{"turn:t33"}, + Producer: Producer{AgentID: examplePrincipal, SessionID: "sess_1", TaskID: "task_1"}, + EventAt: created.Add(-2 * time.Minute), + CreatedAt: created, + ExpiresAt: &expires, + Importance: "high", + Confidence: &conf, + StateVersion: 1, + Pinned: false, + Lifecycle: LifecycleActive, + Trust: TrustUntrusted, + Authority: AuthorityNone, + Text: exampleText, + Digest: ContentDigest(exampleText), + } + rec.Grant.PermissionDigest = PermissionDigest(rec.Binding, rec.Grant.Mode, rec.Grant.GrantVersion) + return rec +} + +func mustJSON(t *testing.T, rec Record) []byte { + t.Helper() + raw, err := json.Marshal(rec) + if err != nil { + t.Fatal(err) + } + return raw +} + +func repoRoot(t *testing.T) string { + t.Helper() + dir, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + for { + if _, err := os.Stat(filepath.Join(dir, "docs", "schemas")); err == nil { + return dir + } + parent := filepath.Dir(dir) + if parent == dir { + t.Fatal("repository root not found") + } + dir = parent + } +} + +func examplePath(t *testing.T) string { + t.Helper() + return filepath.Join(repoRoot(t), "docs", "examples", "durable-memory.v1.example.json") +} + +func schemaPath(t *testing.T) string { + t.Helper() + return filepath.Join(repoRoot(t), "docs", "schemas", "durable-memory.v1.schema.json") +} diff --git a/server/internal/durablememory/decode.go b/server/internal/durablememory/decode.go new file mode 100644 index 0000000..0024f51 --- /dev/null +++ b/server/internal/durablememory/decode.go @@ -0,0 +1,257 @@ +package durablememory + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "regexp" + "strings" + "time" + "unicode/utf8" + + "github.com/PeterGuy326/mem/server/internal/pathx" + "github.com/google/uuid" +) + +const ( + maxTextRunes = 16384 + maxCitationRunes = 2048 + maxCitations = 32 + maxIDRunes = 256 + maxScopeRunes = 1024 +) + +var ( + principalRE = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]{0,127}$`) + positionRE = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]{0,118}$`) + sha256RE = regexp.MustCompile(`^sha256:[a-f0-9]{64}$`) + memoryScopeRE = regexp.MustCompile(`^/workspaces/[^/]+/positions/[a-z0-9][a-z0-9._-]{0,118}$`) + importanceSet = map[string]struct{}{"low": {}, "normal": {}, "high": {}} + kindSet = map[string]struct{}{ + KindProjectDecision: {}, + KindUserPreference: {}, + KindReusableWorkflow: {}, + KindNegativeSignal: {}, + KindActiveTaskState: {}, + KindComplianceRetain: {}, + KindObservation: {}, + KindDecision: {}, + KindPreference: {}, + KindTaskState: {}, + KindFact: {}, + KindNote: {}, + KindArtifact: {}, + } + lifecycleSet = map[string]struct{}{ + LifecycleActive: {}, + LifecycleArchived: {}, + LifecycleSuperseded: {}, + LifecycleExpired: {}, + LifecycleForgotten: {}, + } + grantModeSet = map[string]struct{}{ + GrantModeRead: {}, + } + sourceKindSet = map[string]struct{}{ + SourceKindSegment: {}, + SourceKindMemory: {}, + SourceKindArtifact: {}, + } +) + +// DecodeRecord strictly decodes one durable-memory.v1 object. Unknown fields +// (including a free-string "scope") fail closed as malformed. +func DecodeRecord(raw []byte) (Record, error) { + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + var rec Record + if err := decoder.Decode(&rec); err != nil { + return Record{}, malformed("decode: %v", err) + } + var trailing any + if err := decoder.Decode(&trailing); err != io.EOF { + if err == nil { + return Record{}, malformed("must contain exactly one JSON value") + } + return Record{}, malformed("decode: %v", err) + } + if err := validateRecord(rec); err != nil { + return Record{}, err + } + return rec, nil +} + +func validateRecord(rec Record) error { + if rec.Contract != ContractVersion { + if rec.Contract == "" { + return malformed("contract is required") + } + return fmt.Errorf("%w: %s", ErrUnsupportedContract, rec.Contract) + } + if rec.MemoryID == uuid.Nil { + return malformed("memory_id is required") + } + if _, ok := kindSet[rec.Kind]; !ok { + return malformed("kind is not a durable-memory.v1 kind") + } + if err := validateBinding(rec.Binding); err != nil { + return err + } + if err := validateGrant(rec.Binding, rec.Grant); err != nil { + return err + } + if _, ok := sourceKindSet[rec.Source.Kind]; !ok { + return malformed("source.kind is invalid") + } + if strings.TrimSpace(rec.Source.ID) == "" || utf8.RuneCountInString(rec.Source.ID) > maxIDRunes { + return malformed("source.id is required") + } + if !sha256RE.MatchString(rec.Source.Digest) { + return malformed("source.digest must be sha256:<64 hex>") + } + if rec.Citations == nil { + return malformed("citations is required") + } + if len(rec.Citations) > maxCitations { + return malformed("too many citations") + } + for _, citation := range rec.Citations { + if strings.TrimSpace(citation) == "" || utf8.RuneCountInString(citation) > maxCitationRunes { + return malformed("citation is invalid") + } + } + if !principalRE.MatchString(strings.TrimSpace(rec.Producer.AgentID)) { + return malformed("producer.agent_id must be a principal") + } + if rec.CreatedAt.IsZero() || rec.EventAt.IsZero() { + return malformed("event_at and created_at are required") + } + if rec.ExpiresAt != nil && rec.ExpiresAt.IsZero() { + return malformed("expires_at must be a real timestamp when set") + } + if rec.StateVersion < 1 { + return malformed("state_version must be >= 1") + } + if _, ok := lifecycleSet[rec.Lifecycle]; !ok { + return malformed("lifecycle is invalid") + } + if rec.Trust != TrustUntrusted { + return malformed("trust must be untrusted") + } + if rec.Authority != AuthorityNone { + return malformed("authority must be none") + } + if rec.Lifecycle == LifecycleForgotten { + if rec.Text != "" { + return malformed("forgotten records must redact text") + } + } else if strings.TrimSpace(rec.Text) == "" || utf8.RuneCountInString(rec.Text) > maxTextRunes { + return malformed("text is required") + } + if rec.Digest != ContentDigest(rec.Text) { + return malformed("digest must be sha256 of text UTF-8 bytes") + } + if rec.Importance != "" { + if _, ok := importanceSet[rec.Importance]; !ok { + return malformed("importance is invalid") + } + } + if rec.Confidence != nil && (*rec.Confidence < 0 || *rec.Confidence > 1) { + return malformed("confidence must be in [0,1]") + } + return nil +} + +func validateBinding(b Binding) error { + if b.WorkspaceID == uuid.Nil { + return malformed("binding.workspace_id is required") + } + if !positionRE.MatchString(b.PositionID) { + return malformed("binding.position_id is invalid") + } + if !principalRE.MatchString(b.Principal) { + return malformed("binding.principal is invalid") + } + if b.Principal != "position."+b.PositionID { + return malformed("binding.principal must equal position.") + } + scope := strings.TrimSpace(b.MemoryScope) + if scope == "" || scope == pathx.Root { + return malformed("binding.memory_scope must be a non-root virtual path") + } + normalized, err := pathx.Normalize(scope) + if err != nil { + return malformed("binding.memory_scope: %v", err) + } + if normalized != scope { + return malformed("binding.memory_scope must already be canonical") + } + if utf8.RuneCountInString(scope) > maxScopeRunes { + return malformed("binding.memory_scope exceeds %d characters", maxScopeRunes) + } + if !memoryScopeRE.MatchString(scope) || !strings.HasSuffix(scope, "/positions/"+b.PositionID) { + return malformed("binding.memory_scope must bind /workspaces//positions/") + } + return nil +} + +func validateGrant(b Binding, g Grant) error { + if g.GrantID == uuid.Nil { + return malformed("grant.grant_id is required") + } + if _, ok := grantModeSet[g.Mode]; !ok { + return malformed("grant.mode is invalid") + } + if g.GrantVersion < 1 { + return malformed("grant.grant_version must be >= 1") + } + want := PermissionDigest(b, g.Mode, g.GrantVersion) + if g.PermissionDigest != want { + return malformed("grant.permission_digest must bind workspace, principal, memory_scope, mode, and grant_version") + } + if g.CapabilityGrant.SchemaVersion != CapabilityGrantV1 { + return malformed("grant.capability_grant.schema_version must be capability-grant.v1") + } + if g.CapabilityGrant.Server != "mem" { + return malformed("grant.capability_grant.server must be mem") + } + if g.GrantedAt.IsZero() { + return malformed("grant.granted_at is required") + } + if g.RevokedAt != nil && g.RevokedAt.Before(g.GrantedAt) { + return malformed("grant.revoked_at cannot precede granted_at") + } + return nil +} + +func malformed(format string, args ...any) error { + return fmt.Errorf("%w: %s", ErrMalformed, fmt.Sprintf(format, args...)) +} + +// ContentDigest is SHA-256 of the envelope text (UTF-8). Forgotten +// tombstones digest the empty string. +func ContentDigest(text string) string { + sum := sha256.Sum256([]byte(text)) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +// PermissionDigest is the SHA-256 of the canonical grant tuple. It is what +// receipts compare when detecting grant drift. +func PermissionDigest(b Binding, mode string, grantVersion int64) string { + payload := fmt.Sprintf( + "durable-memory.v1/permission\nworkspace_id=%s\nprincipal=%s\nmemory_scope=%s\nmode=%s\ngrant_version=%d\n", + b.WorkspaceID, b.Principal, b.MemoryScope, mode, grantVersion, + ) + sum := sha256.Sum256([]byte(payload)) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +func grantStatus(g Grant, at time.Time) string { + if g.RevokedAt != nil && !g.RevokedAt.After(at) { + return GrantStatusRevoked + } + return GrantStatusActive +} diff --git a/server/internal/durablememory/eligibility.go b/server/internal/durablememory/eligibility.go new file mode 100644 index 0000000..b3753dd --- /dev/null +++ b/server/internal/durablememory/eligibility.go @@ -0,0 +1,174 @@ +package durablememory + +import ( + "fmt" + "strings" + + "github.com/google/uuid" +) + +// EvaluateRecall decides whether one record may be injected. Pin may preserve +// TTL eligibility; it never enlarges workspace, principal, scope, or grant. +func EvaluateRecall(rec Record, caller RecallCaller) Eligibility { + if err := validateRecord(rec); err != nil { + return Eligibility{OmitReason: OmitMalformed} + } + if !sameBinding(rec.Binding, caller) { + return Eligibility{OmitReason: OmitOutOfScope} + } + at := caller.At + if grantStatus(rec.Grant, at) == GrantStatusRevoked { + return Eligibility{OmitReason: OmitRevoked} + } + switch rec.Lifecycle { + case LifecycleForgotten: + return Eligibility{OmitReason: OmitForgotten} + case LifecycleArchived, LifecycleSuperseded: + return Eligibility{OmitReason: OmitSuperseded} + } + if rec.ExpiresAt != nil && !rec.ExpiresAt.After(at) && !rec.Pinned { + return Eligibility{OmitReason: OmitExpired} + } + if rec.Lifecycle == LifecycleExpired && !rec.Pinned { + return Eligibility{OmitReason: OmitExpired} + } + return Eligibility{Eligible: true} +} + +// PhysicalDeleteImplied reports whether TTL/expiry authorizes destroying the +// source log. It is always false: expiry is recall eligibility only. +func PhysicalDeleteImplied(rec Record, caller RecallCaller) bool { + _ = rec + _ = caller + return false +} + +// ExactReadback compares decoded envelope fields after the same validateRecord +// gate used on decode. It is not RFC 8785 JSON canonicalization. +func ExactReadback(stored, observed Record) error { + if err := validateRecord(stored); err != nil { + return fmt.Errorf("%w: stored: %v", ErrReadbackMismatch, err) + } + if err := validateRecord(observed); err != nil { + return fmt.Errorf("%w: observed: %v", ErrReadbackMismatch, err) + } + if stored.MemoryID != observed.MemoryID || + stored.Kind != observed.Kind || + stored.Binding != observed.Binding || + stored.Grant.GrantID != observed.Grant.GrantID || + stored.Grant.Mode != observed.Grant.Mode || + stored.Grant.GrantVersion != observed.Grant.GrantVersion || + stored.Grant.PermissionDigest != observed.Grant.PermissionDigest || + stored.Source != observed.Source || + stored.Producer != observed.Producer || + stored.Text != observed.Text || + stored.Digest != observed.Digest || + stored.StateVersion != observed.StateVersion || + stored.Pinned != observed.Pinned || + stored.Lifecycle != observed.Lifecycle { + return ErrReadbackMismatch + } + if len(stored.Citations) != len(observed.Citations) { + return ErrReadbackMismatch + } + for i := range stored.Citations { + if stored.Citations[i] != observed.Citations[i] { + return ErrReadbackMismatch + } + } + return nil +} + +// EvaluateForget is permissioned. Failure is a visible error code. This +// contract evaluator never reports a local fake delete as success. +func EvaluateForget(rec Record, actor ForgetActor) ForgetDecision { + denied := ForgetDecision{ErrorCode: ErrorForgetDenied} + if err := validateRecord(rec); err != nil { + return denied + } + if !sameBinding(rec.Binding, RecallCaller{ + WorkspaceID: actor.WorkspaceID, + Principal: actor.Principal, + MemoryScope: actor.MemoryScope, + }) { + return denied + } + if !actor.WorkspaceRoleAllowsDelete || !hasScope(actor.TokenScopes, GrantModeDelete) { + return denied + } + return ForgetDecision{Authorized: true} +} + +// BuildReceipt projects grant/revocation into the readback envelope. +// Out-of-scope and malformed probes are indistinguishable from absence. +func BuildReceipt(rec Record, caller RecallCaller) RecallReceipt { + elig := EvaluateRecall(rec, caller) + if elig.OmitReason == OmitOutOfScope || elig.OmitReason == OmitMalformed { + return RecallReceipt{Contract: ContractVersion, Eligible: false} + } + at := caller.At + receipt := RecallReceipt{ + Contract: ContractVersion, + MemoryID: rec.MemoryID, + Locator: locator(rec.MemoryID, rec.StateVersion), + StateVersion: rec.StateVersion, + Eligible: elig.Eligible, + OmitReason: elig.OmitReason, + Pinned: rec.Pinned, + Grant: ReceiptGrant{ + GrantID: rec.Grant.GrantID, + GrantVersion: rec.Grant.GrantVersion, + Mode: rec.Grant.Mode, + Status: grantStatus(rec.Grant, at), + PermissionDigest: rec.Grant.PermissionDigest, + RevokedAt: rec.Grant.RevokedAt, + }, + } + if elig.Eligible { + receipt.Readback = cloneRecord(rec) + } + return receipt +} + +func cloneRecord(rec Record) *Record { + clone := rec + if rec.ExpiresAt != nil { + expires := *rec.ExpiresAt + clone.ExpiresAt = &expires + } + if rec.Confidence != nil { + conf := *rec.Confidence + clone.Confidence = &conf + } + if rec.Grant.RevokedAt != nil { + revoked := *rec.Grant.RevokedAt + clone.Grant.RevokedAt = &revoked + } + if rec.Citations != nil { + clone.Citations = append([]string(nil), rec.Citations...) + } + return &clone +} + +func locator(memoryID uuid.UUID, stateVersion int64) string { + return fmt.Sprintf("mem://memories/%s@%d", memoryID, stateVersion) +} + +func sameBinding(b Binding, caller RecallCaller) bool { + if b.WorkspaceID.String() != strings.TrimSpace(caller.WorkspaceID) { + return false + } + if b.Principal != strings.TrimSpace(caller.Principal) { + return false + } + return b.MemoryScope == strings.TrimSpace(caller.MemoryScope) +} + +func hasScope(scopes []string, want string) bool { + for _, scope := range scopes { + if scope == want { + return true + } + } + return false +} diff --git a/server/internal/durablememory/types.go b/server/internal/durablememory/types.go new file mode 100644 index 0000000..2ef4f14 --- /dev/null +++ b/server/internal/durablememory/types.go @@ -0,0 +1,192 @@ +// Package durablememory owns the additive durable-memory.v1 contract. +// +// This package is the version-pinned envelope for derived RoleWeave/mem +// records: principal binding, grant/revocation, expiry, pin, forget, and +// exact readback. It does not persist rows, expose HTTP, or replace the +// canonical /v1/memories control plane. Runtime wiring waits for Gate D0. +package durablememory + +import ( + "errors" + "time" + + "github.com/google/uuid" +) + +const ( + ContractVersion = "durable-memory.v1" + CapabilityGrantV1 = "capability-grant.v1" + + KindProjectDecision = "project_decision" + KindUserPreference = "user_preference" + KindReusableWorkflow = "reusable_workflow" + KindNegativeSignal = "negative_signal" + KindActiveTaskState = "active_task_state" + KindComplianceRetain = "compliance_retained" + KindObservation = "observation" + KindDecision = "decision" + KindPreference = "preference" + KindTaskState = "task_state" + KindFact = "fact" + KindNote = "note" + KindArtifact = "artifact" + + LifecycleActive = "active" + LifecycleArchived = "archived" + LifecycleSuperseded = "superseded" + LifecycleExpired = "expired" + LifecycleForgotten = "forgotten" + + GrantModeRead = "read" + GrantModeWrite = "write" + GrantModeDelete = "delete" + + GrantStatusActive = "active" + GrantStatusRevoked = "revoked" + + TrustUntrusted = "untrusted" + AuthorityNone = "none" + + SourceKindSegment = "segment" + SourceKindMemory = "memory" + SourceKindArtifact = "artifact" + + OmitExpired = "expired" + OmitRevoked = "revoked" + OmitMalformed = "malformed" + OmitSuperseded = "superseded" + OmitForgotten = "forgotten" + OmitOutOfScope = "out_of_scope" + + ErrorForgetDenied = "forget_denied" +) + +var ( + ErrMalformed = errors.New("durable memory record malformed") + ErrReadbackMismatch = errors.New("durable memory readback mismatch") + ErrUnsupportedContract = errors.New("durable memory contract unsupported") +) + +// Record is one derived durable-memory.v1 occurrence. Scope is not a free +// string: callers must bind workspace, position principal, memory scope, and +// a grant/revocation tuple. +type Record struct { + Contract string `json:"contract"` + MemoryID uuid.UUID `json:"memory_id"` + Kind string `json:"kind"` + Binding Binding `json:"binding"` + Grant Grant `json:"grant"` + Source Source `json:"source"` + Citations []string `json:"citations"` + Producer Producer `json:"producer"` + EventAt time.Time `json:"event_at"` + CreatedAt time.Time `json:"created_at"` + ExpiresAt *time.Time `json:"expires_at"` + Importance string `json:"importance,omitempty"` + Confidence *float64 `json:"confidence,omitempty"` + StateVersion int64 `json:"state_version"` + Pinned bool `json:"pinned"` + Lifecycle string `json:"lifecycle"` + Trust string `json:"trust"` + Authority string `json:"authority"` + Text string `json:"text"` + Digest string `json:"digest"` +} + +// Binding is the fail-closed isolation key. Cross-principal default deny. +type Binding struct { + WorkspaceID uuid.UUID `json:"workspace_id"` + PositionID string `json:"position_id"` + Principal string `json:"principal"` + MemoryScope string `json:"memory_scope"` +} + +// Grant reuses a durable-context.v1 grant id. Mode is always read. Forget uses +// the mem delete token scope, not this field. Revocation is first-class so +// in-scope receipts can show why recall was denied. +type Grant struct { + GrantID uuid.UUID `json:"grant_id"` + Mode string `json:"mode"` + GrantVersion int64 `json:"grant_version"` + PermissionDigest string `json:"permission_digest"` + CapabilityGrant CapabilityGrantRef `json:"capability_grant"` + GrantedAt time.Time `json:"granted_at"` + RevokedAt *time.Time `json:"revoked_at"` +} + +// CapabilityGrantRef is a normative pointer at capability-grant.v1, not a +// second authorization implementation. +type CapabilityGrantRef struct { + SchemaVersion string `json:"schema_version"` + Server string `json:"server"` +} + +// Source identifies the parent segment or artifact. Digests are full sha256. +type Source struct { + Kind string `json:"kind"` + ID string `json:"id"` + Digest string `json:"digest"` +} + +// Producer is the writing Agent identity. It is evidence, not a permission. +type Producer struct { + AgentID string `json:"agent_id"` + SessionID string `json:"session_id,omitempty"` + TaskID string `json:"task_id,omitempty"` +} + +// RecallCaller is the principal asking to resume one record. +type RecallCaller struct { + WorkspaceID string + Principal string + MemoryScope string + At time.Time +} + +// Eligibility is the recall decision. OmitReason is empty when Eligible. +type Eligibility struct { + Eligible bool + OmitReason string +} + +// ForgetActor is the permissioned caller of an explicit forget. A pin or +// read grant never authorizes this operation. +type ForgetActor struct { + WorkspaceID string + Principal string + MemoryScope string + TokenScopes []string + WorkspaceRoleAllowsDelete bool +} + +// ForgetDecision is never a local fake delete. Authorized means the caller +// may invoke mem's permissioned forget API; Deleted stays false here. +type ForgetDecision struct { + Authorized bool + Deleted bool + LocalFake bool + ErrorCode string +} + +// ReceiptGrant is how grant and revocation enter readback/receipt. +type ReceiptGrant struct { + GrantID uuid.UUID `json:"grant_id"` + GrantVersion int64 `json:"grant_version"` + Mode string `json:"mode"` + Status string `json:"status"` + PermissionDigest string `json:"permission_digest"` + RevokedAt *time.Time `json:"revoked_at,omitempty"` +} + +// RecallReceipt is the exact-readback envelope for one record. +type RecallReceipt struct { + Contract string `json:"contract"` + MemoryID uuid.UUID `json:"memory_id"` + Locator string `json:"locator"` + StateVersion int64 `json:"state_version"` + Eligible bool `json:"eligible"` + OmitReason string `json:"omit_reason,omitempty"` + Grant ReceiptGrant `json:"grant"` + Pinned bool `json:"pinned"` + Readback *Record `json:"readback"` +} diff --git a/server/internal/face/face.go b/server/internal/face/face.go index 146ba99..af157e6 100644 --- a/server/internal/face/face.go +++ b/server/internal/face/face.go @@ -10,9 +10,10 @@ // to that entity. Otherwise we create a new entity (unnamed). // 4. Insert embeddings_face (file_id, entity_id, bbox, embedding). // -// This is intentionally O(n) per insert — fine for a personal drive up to -// thousands of faces. For larger corpora swap in pgvector HNSW + offline -// re-clustering. +// Clustering is intentionally O(n) per insert (in-process centroid distance). +// Migration 0025 adds a cosine HNSW index on embeddings_face; assignCluster +// does not query it yet. A future SQL kNN plus offline re-clustering can use +// that index without changing the 512-d insightface space. package face import ( diff --git a/server/internal/folder/delete_integration_test.go b/server/internal/folder/delete_integration_test.go new file mode 100644 index 0000000..7ab8d6c --- /dev/null +++ b/server/internal/folder/delete_integration_test.go @@ -0,0 +1,362 @@ +package folder + +import ( + "bytes" + "context" + "errors" + "io" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" + + memdb "github.com/PeterGuy326/mem/server/internal/db" +) + +// trackingObjectStore records which keys are deleted so tests can assert that +// recursive folder delete cleans up blobs. +type trackingObjectStore struct { + mu sync.Mutex + objects map[string]bool + deleted []string +} + +func newTrackingObjectStore() *trackingObjectStore { + return &trackingObjectStore{objects: make(map[string]bool)} +} + +func (s *trackingObjectStore) Put(_ context.Context, key string, _ io.Reader, _ int64, _ string) error { + s.mu.Lock() + defer s.mu.Unlock() + s.objects[key] = true + return nil +} + +func (s *trackingObjectStore) Get(_ context.Context, key string) (io.ReadCloser, error) { + s.mu.Lock() + defer s.mu.Unlock() + if !s.objects[key] { + return nil, &objectNotFoundError{key: key} + } + return io.NopCloser(bytes.NewReader(nil)), nil +} + +func (s *trackingObjectStore) Delete(_ context.Context, key string) error { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.objects, key) + s.deleted = append(s.deleted, key) + return nil +} + +func (s *trackingObjectStore) has(key string) bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.objects[key] +} + +func (s *trackingObjectStore) deleteCount() int { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.deleted) +} + +type objectNotFoundError struct{ key string } + +func (e *objectNotFoundError) Error() string { return "object not found: " + e.key } + +// TestRecursiveDeleteCleansUpBlobs verifies that recursive folder delete +// removes objects from the store after the DB rows are deleted. +// +// MEM_TEST_DB=postgres://mem:mem@localhost:5432/mem_test?sslmode=disable \ +// go test ./internal/folder -run TestRecursiveDeleteCleansUpBlobs +func TestRecursiveDeleteCleansUpBlobs(t *testing.T) { + dsn := os.Getenv("MEM_TEST_DB") + if dsn == "" { + t.Skip("MEM_TEST_DB not set; skipping DB integration test") + } + config, err := pgxpool.ParseConfig(dsn) + if err != nil { + t.Fatalf("parse MEM_TEST_DB: %v", err) + } + if !strings.HasSuffix(config.ConnConfig.Database, "_test") { + t.Fatalf( + "refusing to modify non-test database %q; MEM_TEST_DB must end in _test", + config.ConnConfig.Database, + ) + } + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + database, err := memdb.Open(ctx, dsn) + if err != nil { + t.Fatalf("open test database: %v", err) + } + t.Cleanup(database.Close) + if err := database.Migrate(ctx); err != nil { + t.Fatalf("migrate test database: %v", err) + } + + userID, _ := createFolderDeleteTenant(t, ctx, database.Pool) + store := newTrackingObjectStore() + service := New(database.Pool, WithStore(store)) + + // Create a folder hierarchy with files. + if _, err := service.Create(ctx, userID, "/Project/Sub"); err != nil { + t.Fatalf("create folders: %v", err) + } + + // Insert files with storage keys that the store tracks. + fileKeys := []string{ + "users/" + userID.String() + "/" + uuid.NewString() + "/a.txt", + "users/" + userID.String() + "/" + uuid.NewString() + "/b.txt", + "users/" + userID.String() + "/" + uuid.NewString() + "/c.txt", + } + for i, key := range fileKeys { + if err := store.Put(ctx, key, nil, 0, "text/plain"); err != nil { + t.Fatalf("put object: %v", err) + } + paths := []string{"/Project", "/Project", "/Project/Sub"} + names := []string{"a.txt", "b.txt", "c.txt"} + if _, err := database.Pool.Exec(ctx, ` + INSERT INTO files (id, user_id, name, path, size, sha256, mime, storage_key, index_status) + VALUES ($1, $2, $3, $4, 0, $5, 'text/plain', $6, 'ready') + `, uuid.New(), userID, names[i], paths[i], strings.Repeat("x", 64), key); err != nil { + t.Fatalf("insert file: %v", err) + } + } + + // Verify objects exist before delete. + for _, key := range fileKeys { + if !store.has(key) { + t.Fatalf("object %s should exist before delete", key) + } + } + + // Recursive delete. + if err := service.Delete(ctx, userID, "/Project", true); err != nil { + t.Fatalf("recursive delete: %v", err) + } + + // Verify all objects were deleted from the store. + for _, key := range fileKeys { + if store.has(key) { + t.Errorf("object %s should have been deleted from store", key) + } + } + if store.deleteCount() != len(fileKeys) { + t.Errorf("delete calls = %d, want %d", store.deleteCount(), len(fileKeys)) + } + + // Verify DB rows are gone. + var fileCount int + if err := database.Pool.QueryRow(ctx, ` + SELECT count(*) FROM files WHERE user_id = $1 + `, userID).Scan(&fileCount); err != nil { + t.Fatalf("count files: %v", err) + } + if fileCount != 0 { + t.Errorf("files remaining = %d, want 0", fileCount) + } + + var folderCount int + if err := database.Pool.QueryRow(ctx, ` + SELECT count(*) FROM folders WHERE user_id = $1 + `, userID).Scan(&folderCount); err != nil { + t.Fatalf("count folders: %v", err) + } + if folderCount != 0 { + t.Errorf("folders remaining = %d, want 0", folderCount) + } +} + +// TestRecursiveDeleteWithoutStore verifies that recursive delete works without +// a store configured (backward compatibility — DB rows are deleted but blobs +// are not cleaned up). +func TestRecursiveDeleteWithoutStore(t *testing.T) { + dsn := os.Getenv("MEM_TEST_DB") + if dsn == "" { + t.Skip("MEM_TEST_DB not set; skipping DB integration test") + } + config, err := pgxpool.ParseConfig(dsn) + if err != nil { + t.Fatalf("parse MEM_TEST_DB: %v", err) + } + if !strings.HasSuffix(config.ConnConfig.Database, "_test") { + t.Fatalf( + "refusing to modify non-test database %q; MEM_TEST_DB must end in _test", + config.ConnConfig.Database, + ) + } + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + database, err := memdb.Open(ctx, dsn) + if err != nil { + t.Fatalf("open test database: %v", err) + } + t.Cleanup(database.Close) + if err := database.Migrate(ctx); err != nil { + t.Fatalf("migrate test database: %v", err) + } + + userID, _ := createFolderDeleteTenant(t, ctx, database.Pool) + service := New(database.Pool) // no store + + if _, err := service.Create(ctx, userID, "/Orphan"); err != nil { + t.Fatalf("create folder: %v", err) + } + key := "users/" + userID.String() + "/" + uuid.NewString() + "/orphan.txt" + if _, err := database.Pool.Exec(ctx, ` + INSERT INTO files (id, user_id, name, path, size, sha256, mime, storage_key, index_status) + VALUES ($1, $2, 'orphan.txt', '/Orphan', 0, $3, 'text/plain', $4, 'ready') + `, uuid.New(), userID, strings.Repeat("y", 64), key); err != nil { + t.Fatalf("insert file: %v", err) + } + + // Delete should succeed even without a store. + if err := service.Delete(ctx, userID, "/Orphan", true); err != nil { + t.Fatalf("recursive delete without store: %v", err) + } + + // DB row should be gone. + var fileCount int + if err := database.Pool.QueryRow(ctx, ` + SELECT count(*) FROM files WHERE user_id = $1 + `, userID).Scan(&fileCount); err != nil { + t.Fatalf("count files: %v", err) + } + if fileCount != 0 { + t.Errorf("files remaining = %d, want 0", fileCount) + } +} + +// TestRecursiveDeleteBlocksWhenMemoryCitesFileElsewhere is the #210 review +// regression: a memory living at /Work/task that cites a file under /Photos +// must block recursive delete of /Photos. Otherwise ON DELETE SET NULL plus +// blob cleanup would destroy the cited object while the memory stays active. +func TestRecursiveDeleteBlocksWhenMemoryCitesFileElsewhere(t *testing.T) { + dsn := os.Getenv("MEM_TEST_DB") + if dsn == "" { + t.Skip("MEM_TEST_DB not set; skipping DB integration test") + } + config, err := pgxpool.ParseConfig(dsn) + if err != nil { + t.Fatalf("parse MEM_TEST_DB: %v", err) + } + if !strings.HasSuffix(config.ConnConfig.Database, "_test") { + t.Fatalf( + "refusing to modify non-test database %q; MEM_TEST_DB must end in _test", + config.ConnConfig.Database, + ) + } + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + database, err := memdb.Open(ctx, dsn) + if err != nil { + t.Fatalf("open test database: %v", err) + } + t.Cleanup(database.Close) + if err := database.Migrate(ctx); err != nil { + t.Fatalf("migrate test database: %v", err) + } + + userID, workspaceID := createFolderDeleteTenant(t, ctx, database.Pool) + + store := newTrackingObjectStore() + service := New(database.Pool, WithStore(store)) + if _, err := service.Create(ctx, userID, "/Photos"); err != nil { + t.Fatalf("create /Photos: %v", err) + } + if _, err := service.Create(ctx, userID, "/Work"); err != nil { + t.Fatalf("create /Work: %v", err) + } + photos, err := service.Get(ctx, userID, "/Photos") + if err != nil { + t.Fatalf("get /Photos: %v", err) + } + + fileID := uuid.New() + key := "users/" + userID.String() + "/" + fileID.String() + "/cited.txt" + if err := store.Put(ctx, key, nil, 0, "text/plain"); err != nil { + t.Fatalf("put object: %v", err) + } + sha := strings.Repeat("ab", 32) + if _, err := database.Pool.Exec(ctx, ` + INSERT INTO files (id, user_id, folder_id, name, path, size, sha256, mime, storage_key, index_status) + VALUES ($1, $2, $3, 'cited.txt', '/Photos', 0, $4, 'text/plain', $5, 'ready') + `, fileID, userID, photos.ID, sha, key); err != nil { + t.Fatalf("insert cited file: %v", err) + } + if _, err := database.Pool.Exec(ctx, ` + INSERT INTO memories ( + workspace_id, kind, content, path, source_type, + source_file_id, source_file_sha256, + idempotency_key_sha256, request_sha256, content_sha256, + lifecycle_status + ) VALUES ( + $1, 'note', 'cites a photo', '/Work/task', 'agent', + $2, $3, + $4, $5, $6, + 'active' + ) + `, workspaceID, fileID, sha, sha, sha, sha); err != nil { + t.Fatalf("insert citing memory: %v", err) + } + + if err := service.Delete(ctx, userID, "/Photos", true); !errors.Is(err, ErrContainsMemories) { + t.Fatalf("recursive delete with cross-path citation = %v, want ErrContainsMemories", err) + } + if !store.has(key) { + t.Fatal("cited blob was deleted despite the blocking memory") + } + var fileCount int + if err := database.Pool.QueryRow(ctx, ` + SELECT count(*) FROM files WHERE id = $1 + `, fileID).Scan(&fileCount); err != nil { + t.Fatalf("count cited file: %v", err) + } + if fileCount != 1 { + t.Fatalf("cited file remaining = %d, want 1", fileCount) + } + if _, err := service.Get(ctx, userID, "/Photos"); err != nil { + t.Fatalf("/Photos changed despite blocked recursive delete: %v", err) + } +} + +func createFolderDeleteTenant(t *testing.T, ctx context.Context, pool *pgxpool.Pool) (uuid.UUID, uuid.UUID) { + t.Helper() + var userID uuid.UUID + if err := pool.QueryRow(ctx, ` + INSERT INTO users (email, password_hash) + VALUES ($1, 'folder-delete-test') + RETURNING id + `, "folder-delete-"+uuid.NewString()+"@example.com").Scan(&userID); err != nil { + t.Fatalf("create user: %v", err) + } + t.Cleanup(func() { + cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, _ = pool.Exec(cleanupCtx, `DELETE FROM users WHERE id = $1`, userID) + }) + var workspaceID uuid.UUID + if err := pool.QueryRow(ctx, ` + INSERT INTO workspaces (name, resource_owner_user_id) + VALUES ('folder-delete', $1) + RETURNING id + `, userID).Scan(&workspaceID); err != nil { + t.Fatalf("create workspace: %v", err) + } + if _, err := pool.Exec(ctx, ` + INSERT INTO workspace_memberships (workspace_id, user_id, role) + VALUES ($1, $2, 'owner') + `, workspaceID, userID); err != nil { + t.Fatalf("create workspace membership: %v", err) + } + return userID, workspaceID +} diff --git a/server/internal/folder/folder.go b/server/internal/folder/folder.go index 0dd0469..1c77a89 100644 --- a/server/internal/folder/folder.go +++ b/server/internal/folder/folder.go @@ -13,6 +13,7 @@ import ( "context" "errors" "fmt" + "log/slog" "strings" "time" @@ -58,13 +59,40 @@ type Node struct { Children []*Node `json:"children,omitempty"` } +// ObjectStore is the subset of storage.Store that folder needs for blob cleanup. +type ObjectStore interface { + Delete(context.Context, string) error +} + // Service is the folder service. type Service struct { - pool *pgxpool.Pool + pool *pgxpool.Pool + store ObjectStore + logger *slog.Logger +} + +// Option configures a folder Service. +type Option func(*Service) + +// WithStore sets the object store used for blob cleanup on recursive delete. +// If not set, recursive delete removes DB rows but leaves orphan blobs. +func WithStore(store ObjectStore) Option { + return func(s *Service) { s.store = store } +} + +// WithLogger sets the logger for reporting best-effort blob cleanup failures. +func WithLogger(logger *slog.Logger) Option { + return func(s *Service) { s.logger = logger } } // New constructs a folder Service. -func New(pool *pgxpool.Pool) *Service { return &Service{pool: pool} } +func New(pool *pgxpool.Pool, opts ...Option) *Service { + s := &Service{pool: pool} + for _, opt := range opts { + opt(s) + } + return s +} // Sentinel errors. var ( @@ -586,9 +614,10 @@ func rewritePrefixTx(ctx context.Context, tx pgx.Tx, userID, srcID uuid.UUID, ol // // - recursive=false (default): folder must be empty (no subfolders, no files) // or ErrNotEmpty is returned. -// - recursive=true: subfolders + files are deleted from the DB. S3 cleanup -// is TODO — for now we only purge the DB rows; orphan blobs will be -// reaped by a future garbage-collection pass. +// - recursive=true: subfolders + files are deleted from the DB, and their +// blobs are removed from object storage on a best-effort basis after the +// transaction commits. A failed blob removal does not roll back the DB +// delete; orphaned keys are logged so the operator can see them. func (s *Service) Delete(ctx context.Context, userID uuid.UUID, path string, recursive bool) error { norm, err := pathx.Normalize(path) if err != nil { @@ -597,7 +626,8 @@ func (s *Service) Delete(ctx context.Context, userID uuid.UUID, path string, rec if norm == pathx.Root { return ErrRootOp } - return s.withPathMutationTx(ctx, userID, func(tx pgx.Tx) error { + var orphanKeys []string + err = s.withPathMutationTx(ctx, userID, func(tx pgx.Tx) error { src, err := selectFolderByPathTx(ctx, tx, userID, norm) if err != nil { return err @@ -618,7 +648,7 @@ func (s *Service) Delete(ctx context.Context, userID uuid.UUID, path string, rec if containsTaskState { return ErrContainsTaskState } - containsMemories, err := containsMemoriesTx(ctx, tx, userID, src.Path, true) + containsMemories, err := containsMemoriesTx(ctx, tx, userID, src.ID, src.Path, true) if err != nil { return fmt.Errorf("check recursive delete memories: %w", err) } @@ -628,6 +658,14 @@ func (s *Service) Delete(ctx context.Context, userID uuid.UUID, path string, rec // explicit forget operation first. return ErrContainsMemories } + // Collect storage keys before deleting rows so we can clean up + // blobs after the transaction commits. Keys are per-row by + // construction (see 0013_file_content_identity.sql), so deleting + // each key cannot destroy another row's bytes. + orphanKeys, err = collectStorageKeysTx(ctx, tx, userID, src.ID, src.Path) + if err != nil { + return fmt.Errorf("collect storage keys for recursive delete: %w", err) + } // Hard delete: remove all descendant files first (FKs cascade // from folders → files would only NULL out folder_id, so we have // to delete files explicitly). @@ -648,6 +686,64 @@ func (s *Service) Delete(ctx context.Context, userID uuid.UUID, path string, rec } return nil }) + if err != nil { + return err + } + // Blob cleanup happens after the transaction commits: a failed object + // delete must not roll back the user's folder delete. This matches the + // best-effort shape of file.Delete. + if len(orphanKeys) > 0 && s.store != nil { + s.cleanupBlobs(ctx, orphanKeys) + } + return nil +} + +// collectStorageKeysTx returns the storage_key values for all files that will +// be deleted by a recursive folder delete. This must be called before the +// DELETE FROM files so the keys are available for post-commit blob cleanup. +func collectStorageKeysTx(ctx context.Context, tx pgx.Tx, userID uuid.UUID, folderID uuid.UUID, folderPath string) ([]string, error) { + rows, err := tx.Query(ctx, + `SELECT storage_key FROM files + WHERE user_id = $1 + AND (folder_id = $2 OR path = $3 + OR left(path, length($3) + 1) = $3 || '/') + AND storage_key <> ''`, + userID, folderID, folderPath) + if err != nil { + return nil, err + } + defer rows.Close() + var keys []string + for rows.Next() { + var key string + if err := rows.Scan(&key); err != nil { + return nil, err + } + keys = append(keys, key) + } + return keys, rows.Err() +} + +// cleanupBlobs removes objects from the store on a best-effort basis. Failures +// are logged so the operator can see which keys remain; they do not propagate +// to the caller because the DB rows are already gone. +func (s *Service) cleanupBlobs(ctx context.Context, keys []string) { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + for _, key := range keys { + if err := s.store.Delete(cleanupCtx, key); err != nil { + if s.logger != nil { + msg := "folder delete: blob cleanup failed" + if cleanupCtx.Err() != nil { + msg = "folder delete: blob cleanup stopped; 30s shared budget exhausted" + } + s.logger.Warn(msg, + "storage_key", key, + "error", err, + ) + } + } + } } func isEmptyTx(ctx context.Context, tx pgx.Tx, userID, folderID uuid.UUID, path string) (bool, error) { @@ -675,7 +771,7 @@ func isEmptyTx(ctx context.Context, tx pgx.Tx, userID, folderID uuid.UUID, path if containsTaskState { return false, nil } - containsMemories, err := containsMemoriesTx(ctx, tx, userID, path, false) + containsMemories, err := containsMemoriesTx(ctx, tx, userID, folderID, path, false) if err != nil { return false, fmt.Errorf("check folder memories: %w", err) } @@ -683,8 +779,15 @@ func isEmptyTx(ctx context.Context, tx pgx.Tx, userID, folderID uuid.UUID, path } // containsMemoriesTx reports whether a resource owner's workspace contains an -// active or archived memory at path. When recursive is true, descendants are -// included with a literal segment-boundary comparison. +// active or archived memory that would be harmed by deleting this folder. +// When recursive is true, descendants are included with a literal +// segment-boundary comparison, and memories that live elsewhere but reference +// a file in this tree via source_file_id also block the delete. +// +// That second check matters once recursive delete removes blobs: ON DELETE +// SET NULL would otherwise silently drop the citation while this function +// physically destroys the object, leaving an active memory pointing at +// bytes that no longer exist. // // Forgotten/tombstoned rows intentionally do not block folder deletion: an // explicit memory lifecycle transition has already happened for those rows. @@ -692,6 +795,7 @@ func containsMemoriesTx( ctx context.Context, tx pgx.Tx, userID uuid.UUID, + folderID uuid.UUID, path string, recursive bool, ) (bool, error) { @@ -704,9 +808,21 @@ func containsMemoriesTx( JOIN workspaces AS w ON w.id = m.workspace_id WHERE w.resource_owner_user_id = $1 AND m.lifecycle_status IN ('active', 'archived') - AND (m.path = $2 OR left(m.path, length($2) + 1) = $2 || '/') + AND ( + m.path = $2 + OR left(m.path, length($2) + 1) = $2 || '/' + OR EXISTS ( + SELECT 1 + FROM files AS f + WHERE f.id = m.source_file_id + AND f.user_id = $1 + AND (f.folder_id = $3 + OR f.path = $2 + OR left(f.path, length($2) + 1) = $2 || '/') + ) + ) )`, - userID, path).Scan(&exists) + userID, path, folderID).Scan(&exists) return exists, err } err := tx.QueryRow(ctx, diff --git a/server/internal/folder/folder_test.go b/server/internal/folder/folder_test.go index dcb9fc9..27ad925 100644 --- a/server/internal/folder/folder_test.go +++ b/server/internal/folder/folder_test.go @@ -211,6 +211,7 @@ func TestMemoryPathLifecycleIntegration(t *testing.T) { workspace_id uuid NOT NULL, path text NOT NULL, lifecycle_status text NOT NULL, + source_file_id uuid, updated_at timestamptz NOT NULL DEFAULT now() ) ON COMMIT PRESERVE ROWS `); err != nil { diff --git a/server/internal/ingest/cursor_lock.go b/server/internal/ingest/cursor_lock.go new file mode 100644 index 0000000..6619d2b --- /dev/null +++ b/server/internal/ingest/cursor_lock.go @@ -0,0 +1,60 @@ +package ingest + +import ( + "fmt" + "os" + "time" +) + +const ( + cursorLockWait = 5 * time.Second + cursorLockRetry = 10 * time.Millisecond +) + +// cursorLock holds an advisory lock on one cursor sidecar. The sidecar +// deliberately remains on disk after release: unlinking a locked file can +// create a second inode that another process locks independently. The OS +// releases the advisory lock when this descriptor, or its owning process, exits. +type cursorLock struct { + file *os.File +} + +func acquireCursorLock(cursorPath string) (*cursorLock, error) { + return acquireCursorLockWithTimeout(cursorPath, cursorLockWait) +} + +func acquireCursorLockWithTimeout(cursorPath string, timeout time.Duration) (*cursorLock, error) { + file, err := os.OpenFile(cursorPath+".lock", os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil, fmt.Errorf("open lock file: %w", err) + } + deadline := time.Now().Add(timeout) + for { + if err := tryLockCursorFile(file); err == nil { + return &cursorLock{file: file}, nil + } else if !isCursorLockBusy(err) { + _ = file.Close() + return nil, fmt.Errorf("acquire OS lock: %w", err) + } else if !time.Now().Before(deadline) { + _ = file.Close() + return nil, fmt.Errorf("acquire OS lock: timed out after %s: %w", timeout, err) + } + time.Sleep(cursorLockRetry) + } +} + +func (l *cursorLock) release() error { + if l == nil || l.file == nil { + return nil + } + unlockErr := unlockCursorFile(l.file) + closeErr := l.file.Close() + l.file = nil + if unlockErr != nil { + return fmt.Errorf("unlock OS lock: %w", unlockErr) + } + if closeErr != nil { + return fmt.Errorf("close lock file: %w", closeErr) + } + return nil +} diff --git a/server/internal/ingest/cursor_lock_aix.go b/server/internal/ingest/cursor_lock_aix.go new file mode 100644 index 0000000..b5bd033 --- /dev/null +++ b/server/internal/ingest/cursor_lock_aix.go @@ -0,0 +1,30 @@ +//go:build aix + +package ingest + +import ( + "errors" + "os" + + "golang.org/x/sys/unix" +) + +// AIX does not expose flock(2) through x/sys, so use the non-blocking fcntl +// record lock equivalent for the first byte of the persistent sidecar inode. +func tryLockCursorFile(file *os.File) error { + return unix.FcntlFlock(file.Fd(), unix.F_SETLK, &unix.Flock_t{ + Type: unix.F_WRLCK, + Len: 1, + }) +} + +func isCursorLockBusy(err error) bool { + return errors.Is(err, unix.EAGAIN) || errors.Is(err, unix.EACCES) +} + +func unlockCursorFile(file *os.File) error { + return unix.FcntlFlock(file.Fd(), unix.F_SETLK, &unix.Flock_t{ + Type: unix.F_UNLCK, + Len: 1, + }) +} diff --git a/server/internal/ingest/cursor_lock_other.go b/server/internal/ingest/cursor_lock_other.go new file mode 100644 index 0000000..c542b67 --- /dev/null +++ b/server/internal/ingest/cursor_lock_other.go @@ -0,0 +1,20 @@ +//go:build !(aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris || windows) + +package ingest + +import ( + "fmt" + "os" +) + +func tryLockCursorFile(_ *os.File) error { + return fmt.Errorf("cursor locks are not supported on this operating system") +} + +func isCursorLockBusy(_ error) bool { + return false +} + +func unlockCursorFile(_ *os.File) error { + return nil +} diff --git a/server/internal/ingest/cursor_lock_test.go b/server/internal/ingest/cursor_lock_test.go new file mode 100644 index 0000000..356d30b --- /dev/null +++ b/server/internal/ingest/cursor_lock_test.go @@ -0,0 +1,90 @@ +package ingest + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "runtime" + "testing" + "time" +) + +const cursorLockHelperEnv = "MEM_CURSOR_LOCK_HELPER" + +func TestCursorLockHelperProcess(t *testing.T) { + if os.Getenv(cursorLockHelperEnv) != "1" { + return + } + path := os.Getenv("MEM_CURSOR_LOCK_PATH") + ready := os.Getenv("MEM_CURSOR_LOCK_READY") + release := os.Getenv("MEM_CURSOR_LOCK_RELEASE") + lock, err := acquireCursorLock(path) + if err != nil { + os.Stderr.WriteString("cursor lock helper: " + err.Error() + "\n") + os.Exit(2) + } + defer runtime.KeepAlive(lock) + if err := os.WriteFile(ready, []byte("ready\n"), 0o600); err != nil { + os.Stderr.WriteString("cursor lock helper: write ready: " + err.Error() + "\n") + os.Exit(2) + } + for { + if _, err := os.Stat(release); err == nil { + os.Exit(0) + } else if !os.IsNotExist(err) { + os.Stderr.WriteString("cursor lock helper: inspect release: " + err.Error() + "\n") + os.Exit(2) + } + time.Sleep(5 * time.Millisecond) + } +} + +func TestAcquireCursorLockTimesOut(t *testing.T) { + if runtime.GOOS != "darwin" && runtime.GOOS != "linux" && runtime.GOOS != "windows" { + t.Skip("cursor locks are unsupported on this operating system") + } + dir := t.TempDir() + path := filepath.Join(dir, "cursor.json") + ready := filepath.Join(dir, "ready") + release := filepath.Join(dir, "release") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + t.Cleanup(cancel) + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestCursorLockHelperProcess$") + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + cmd.Env = append(os.Environ(), + cursorLockHelperEnv+"=1", + "MEM_CURSOR_LOCK_PATH="+path, + "MEM_CURSOR_LOCK_READY="+ready, + "MEM_CURSOR_LOCK_RELEASE="+release, + ) + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = os.WriteFile(release, []byte("release\n"), 0o600) + _ = cmd.Wait() + }) + + deadline := time.Now().Add(3 * time.Second) + for { + if _, err := os.Stat(ready); err == nil { + break + } else if !os.IsNotExist(err) { + t.Fatal(err) + } + if time.Now().After(deadline) { + t.Fatal("timed out waiting for helper to hold the cursor lock") + } + time.Sleep(5 * time.Millisecond) + } + + started := time.Now() + if _, err := acquireCursorLockWithTimeout(path, 25*time.Millisecond); err == nil { + t.Fatal("second cursor lock unexpectedly acquired") + } else if elapsed := time.Since(started); elapsed > time.Second { + t.Fatalf("lock timeout took too long: %s", elapsed) + } +} diff --git a/server/internal/ingest/cursor_lock_unix.go b/server/internal/ingest/cursor_lock_unix.go new file mode 100644 index 0000000..71e2e5a --- /dev/null +++ b/server/internal/ingest/cursor_lock_unix.go @@ -0,0 +1,22 @@ +//go:build darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris + +package ingest + +import ( + "errors" + "os" + + "golang.org/x/sys/unix" +) + +func tryLockCursorFile(file *os.File) error { + return unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB) +} + +func isCursorLockBusy(err error) bool { + return errors.Is(err, unix.EAGAIN) || errors.Is(err, unix.EWOULDBLOCK) +} + +func unlockCursorFile(file *os.File) error { + return unix.Flock(int(file.Fd()), unix.LOCK_UN) +} diff --git a/server/internal/ingest/cursor_lock_windows.go b/server/internal/ingest/cursor_lock_windows.go new file mode 100644 index 0000000..2503e43 --- /dev/null +++ b/server/internal/ingest/cursor_lock_windows.go @@ -0,0 +1,31 @@ +//go:build windows + +package ingest + +import ( + "errors" + "os" + + "golang.org/x/sys/windows" +) + +// Lock a one-byte range. Windows releases a LockFileEx lock when the owning +// process or file handle exits, matching the Unix advisory-lock lifecycle. +func tryLockCursorFile(file *os.File) error { + return windows.LockFileEx( + windows.Handle(file.Fd()), + windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, + 0, + 1, + 0, + &windows.Overlapped{}, + ) +} + +func isCursorLockBusy(err error) bool { + return errors.Is(err, windows.ERROR_LOCK_VIOLATION) +} + +func unlockCursorFile(file *os.File) error { + return windows.UnlockFileEx(windows.Handle(file.Fd()), 0, 1, 0, &windows.Overlapped{}) +} diff --git a/server/internal/ingest/ingest.go b/server/internal/ingest/ingest.go new file mode 100644 index 0000000..e852629 --- /dev/null +++ b/server/internal/ingest/ingest.go @@ -0,0 +1,497 @@ +// Package ingest owns the mechanics that every local→mem ingestion connector +// would otherwise re-implement: a deterministic recursive walk, a per-file +// incremental cursor store, the change decision, a closed failure-code +// vocabulary, and cycle report aggregation. +// +// The package is deliberately unaware of any particular input format, of HTTP, +// and of the command surface. A connector supplies two functions: +// +// Parse turns one local file into ordered Units, given the leading line +// count already ingested, and reports how many lines it could not use. +// Upload persists one Unit and reports whether the server replayed it. Its +// error is one that Classify understands. +// +// Call sites stay thin: `mem ingest qoder` today wires a transcript parser and +// a /v1/memories POST to Run, and a future `mem put --watch` wires a different +// source and sink to the same Run, so cursor layout, change detection and the +// report vocabulary are written once. +// +// Run returns a Report; printing it is the caller's job, which is why nothing +// here touches cobra or an io.Writer directly. Diagnostics go through +// Options.Log when the caller wants them. +// +// Contract notes that matter for new call sites: +// +// - The change decision is size-based, not content-hashed. A cursor records +// the file size at write time, and a file that has since become smaller is +// treated as rewritten so its cursor resets. Nothing compares content, so a +// same-size in-place edit is not detected here; adding a content gate is a +// decision to make, not an implementation detail of a call site. +// - Cursors are keyed by the canonical absolute path (see CanonicalRoot, +// CursorPath), which Walk establishes for every path it returns. Keying on +// path-plus-device identity would invalidate existing on-disk cursors. +// - --dry-run neither writes a request nor advances a cursor. Callers must +// not "optimize" by saving a cursor after a dry run. +package ingest + +import ( + "context" + "crypto/sha1" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/PeterGuy326/mem/server/internal/apiclient" +) + +// Code is the closed set of failure classifications a run may report. Names are +// shared vocabulary: report consumers (CLI text, watch daemon, JSON output) +// must not invent per-call-site aliases for the same condition. +type Code string + +const ( + // CodeAuth: the server rejected our credentials (401/403). + CodeAuth Code = "auth" + // CodePlanQuota: plan or quota blocked the write (402/429). + CodePlanQuota Code = "plan_quota" + // CodeProviderTimeout: an upstream provider stage failed or timed out + // (502/503/504). + CodeProviderTimeout Code = "provider_timeout" + // CodeNetwork: the request never reached the server. + CodeNetwork Code = "network" + // CodeReadDenied: a local path could not be read. + CodeReadDenied Code = "read_denied" + // CodeUploadRejected: the server refused this specific unit (409 on a + // stable idempotency key, or a rejected payload). + CodeUploadRejected Code = "upload_rejected" + // CodeRootMissing: the configured source root does not exist. + CodeRootMissing Code = "root_missing" + // CodeStateCorrupt: a cursor could not be decoded, so it was treated as + // "nothing ingested yet" rather than blocking the run. + CodeStateCorrupt Code = "state_corrupt" +) + +// ErrDegradeFile lets an UploadFunc say "stop this file, keep the run going". +// A rewritten file can conflict with its own stable per-line keys forever, so +// aborting the whole cycle would let one bad file block every other source. +// The cursor for that file is not advanced, which keeps a retry meaningful. +var ErrDegradeFile = errors.New("ingest: degrade file") + +// Unit is one ingestible item produced by a connector's Parse function. +type Unit struct { + // Line is the 1-based position in the source file that this unit came + // from. Run records it in the cursor as the high-water mark. + Line int + // Body is the request payload, opaque to this package. + Body any + // IdempotencyKey is the connector's stable retry key for this unit. + IdempotencyKey string +} + +// ParseFunc converts one local file into the units that have not been ingested +// yet. skipBefore is the cursor's high-water mark: units at or below it must +// not be returned. The second result counts lines that were readable but +// produced no unit (malformed, empty, or out of scope for the format). +type ParseFunc func(abs string, skipBefore int) ([]Unit, int, error) + +// Outcome is what an Upload call reports back about one unit. +type Outcome struct { + // Deduplicated marks a server-reported idempotent replay: the memory + // already existed for this key, so nothing new was written. Counting + // replays as ingested would overstate a re-run. + Deduplicated bool +} + +// UploadFunc persists one unit and reports whether the server treated it as a +// replay. Return an error wrapping ErrDegradeFile to skip the rest of the +// current file, or any other error to end the run. +type UploadFunc func(ctx context.Context, abs string, u Unit) (Outcome, error) + +// Cursor is the persisted per-file checkpoint. The field order and JSON names +// are the on-disk format: changing either would strand cursors that existing +// users already have. +type Cursor struct { + Abs string `json:"abs"` + Size int64 `json:"size"` + ModTime string `json:"mtime"` + LastLine int `json:"last_line"` + + // Corrupt is set in memory when a stored cursor failed to decode. It is + // never persisted. + Corrupt bool `json:"-"` +} + +// Options configures one run. +type Options struct { + // StateDir holds the cursors. Required. + StateDir string + // DryRun plans only: no Upload call, no cursor write. + DryRun bool + // Limit stops ingesting units after this many (0 = no limit). + Limit int + // Log receives diagnostics. Nil discards them. + Log func(format string, args ...any) +} + +// Report aggregates one cycle. Run populates Scanned, Ingested, Deduped, +// Changed, Failed and Unparseable. Unchanged and LocalGone are observations a +// caller makes, not states Run detects: comparing content is out of scope here +// (see the size-based contract note above) and Run never deletes a cursor, so +// both stay zero unless a watcher fills them. They exist so a watcher and a +// one-shot importer report the same names. +type Report struct { + Scanned int // files walked and offered to Parse + Ingested int // units persisted by this run (or planned, in dry-run) + Deduped int // units the server reported as replays + Unchanged int // reserved: files observed as already ingested + Changed int // files that had at least one unit accepted + LocalGone int // reserved: cursor records whose file disappeared + Failed int // files degraded rather than aborted + Unparseable int // readable lines that yielded no unit + Failures map[Code]int // per-code tally, including cursor degradation +} + +// Add folds another report into this one, for callers that run several batches +// and report once. +func (r *Report) Add(other Report) { + r.Scanned += other.Scanned + r.Ingested += other.Ingested + r.Deduped += other.Deduped + r.Unchanged += other.Unchanged + r.Changed += other.Changed + r.LocalGone += other.LocalGone + r.Failed += other.Failed + r.Unparseable += other.Unparseable + for code, n := range other.Failures { + if r.Failures == nil { + r.Failures = map[Code]int{} + } + r.Failures[code] += n + } +} + +func (r *Report) fail(code Code) { + if r.Failures == nil { + r.Failures = map[Code]int{} + } + r.Failures[code]++ +} + +// CanonicalRoot turns any caller-supplied path into the identity that cursors +// and idempotency keys are derived from. Without it a relative --root makes two +// different working directories that each contain the same relative source path +// collide on one cursor, so the second run sees an up-to-date checkpoint and +// silently ingests nothing. +// +// Symlinks are resolved so the same store reached by two spellings shares one +// identity. A root that does not exist yet keeps its absolute form rather than +// erroring, which is what Walk's documented "missing root yields no paths, no +// error" behavior needs. +func CanonicalRoot(p string) (string, error) { + abs, err := filepath.Abs(p) + if err != nil { + return "", fmt.Errorf("resolve %s: %w", p, err) + } + if resolved, err := filepath.EvalSymlinks(abs); err == nil { + return resolved, nil + } + return abs, nil +} + +// Walk collects candidate files under base in lexical order, so cursor +// high-water marks mean the same thing from one run to the next. Go's +// filepath.Glob does not treat ** as recursive, hence the explicit walk. +// Unreadable entries are skipped rather than failing the walk: a session store +// routinely contains directories the caller cannot enter. +// +// base is canonicalized, so every returned path is absolute and cursor identity +// cannot depend on the caller's working directory. +// +// A base that does not exist yields no paths and no error, which is what the +// existing connector does with it (it reports "no transcripts matched"). +// Whether a missing root is worth reporting is therefore the call site's +// decision; CodeRootMissing exists for the sites that report it, such as a +// watch daemon that must not sit idle on a typo'd path. +// +// accept is called with each candidate path; a nil accept takes every file. +func Walk(base string, accept func(abs string) bool) ([]string, error) { + root, err := CanonicalRoot(base) + if err != nil { + return nil, err + } + if fi, err := os.Stat(root); err == nil && !fi.IsDir() { + return []string{root}, nil + } + var paths []string + err = filepath.WalkDir(root, func(p string, d os.DirEntry, err error) error { + if err != nil { + return nil + } + if d.IsDir() { + return nil + } + if accept == nil || accept(p) { + paths = append(paths, p) + } + return nil + }) + if err != nil { + return nil, err + } + sort.Strings(paths) + return paths, nil +} + +// CursorPath returns the cursor file for one source path, keyed by a hash of +// its canonical absolute path (see CanonicalRoot) so any source layout is safe +// to store in one directory. +func CursorPath(stateDir, abs string) string { + sum := sha1.Sum([]byte(abs)) + return filepath.Join(stateDir, hex.EncodeToString(sum[:])+".json") +} + +// LoadCursor reads a cursor. A missing or undecodable cursor yields LastLine 0, +// meaning "nothing ingested yet", and is never a hard error: broken state must +// not block ingestion. Load reports that case through Cursor.Corrupt instead. +// +// A file that is now smaller than when its cursor was written was truncated +// and rewritten, so LastLine resets; otherwise new content at already-ingested +// line numbers would be skipped forever. +func LoadCursor(stateDir, abs string) Cursor { + var cp Cursor + b, err := os.ReadFile(CursorPath(stateDir, abs)) + if err != nil { + return cp + } + if err := json.Unmarshal(b, &cp); err != nil { + return Cursor{Abs: abs, Corrupt: true} + } + if cp.Abs == "" { + cp.Abs = abs + } + if cp.Size > 0 { + if fi, err := os.Stat(abs); err == nil && fi.Size() < cp.Size { + cp.LastLine = 0 + } + } + return cp +} + +// SaveCursor writes a cursor through a temporary file and an atomic rename, so +// an interrupted run cannot leave a half-written checkpoint behind. Each save +// gets its own temporary name: two runs may reach the same cursor concurrently +// and a shared name would interleave their writes before the rename. +// +// A save never rewinds a checkpoint that is already further along. A stored +// cursor with a smaller Size is the truncation signal LoadCursor resets on, so +// that case must still write; anything else that is ahead stays. +// +// The per-cursor OS-backed lock covers the read/merge/write sequence so +// independent ingest processes cannot move LastLine backwards or share a +// staging path. Errors are returned (callers may warn without failing the +// whole ingest). +func SaveCursor(stateDir string, cp Cursor) (err error) { + p := CursorPath(stateDir, cp.Abs) + dir := filepath.Dir(p) + if err := os.MkdirAll(dir, 0o700); err != nil { + return fmt.Errorf("create checkpoint dir: %w", err) + } + lock, err := acquireCursorLock(p) + if err != nil { + return fmt.Errorf("lock cursor: %w", err) + } + defer func() { + if releaseErr := lock.release(); err == nil && releaseErr != nil { + err = fmt.Errorf("release cursor lock: %w", releaseErr) + } + }() + + return saveCursorLocked(stateDir, p, cp) +} + +// saveCursorLocked commits cp while the caller owns p's cursor lock. Keeping +// this small inner operation separate lets the lock span the current-cursor +// read as well as the atomic replacement. +func saveCursorLocked(stateDir, p string, cp Cursor) error { + current := LoadCursor(stateDir, cp.Abs) + if current.LastLine > cp.LastLine && current.Size <= cp.Size { + return nil + } + b, err := json.Marshal(cp) + if err != nil { + return fmt.Errorf("encode checkpoint: %w", err) + } + tmp, err := os.CreateTemp(filepath.Dir(p), ".cursor-*.tmp") + if err != nil { + return fmt.Errorf("create checkpoint: %w", err) + } + tmpName := tmp.Name() + defer func() { + _ = tmp.Close() + _ = os.Remove(tmpName) + }() + if err := tmp.Chmod(0o600); err != nil { + return fmt.Errorf("secure checkpoint staging file: %w", err) + } + if _, err := tmp.Write(b); err != nil { + return fmt.Errorf("write checkpoint: %w", err) + } + if err := tmp.Sync(); err != nil { + return fmt.Errorf("sync checkpoint staging file: %w", err) + } + if err := tmp.Close(); err != nil { + return fmt.Errorf("close checkpoint: %w", err) + } + if err := os.Rename(tmpName, p); err != nil { + return fmt.Errorf("commit checkpoint: %w", err) + } + return nil +} + +// FileState returns the size and UTC mtime recorded alongside a cursor. Both +// are diagnostics: only size participates in the change decision. +func FileState(abs string) (size int64, mtime string, err error) { + fi, err := os.Stat(abs) + if err != nil { + return 0, "", err + } + return fi.Size(), fi.ModTime().UTC().Format("2006-01-02T15:04:05Z07:00"), nil +} + +// Classify maps an Upload or Parse failure onto the shared vocabulary. Callers +// use it to report a code without re-deriving it from statuses, and exit-code +// mapping stays in the command surface that owns it. +func Classify(err error) Code { + if err == nil { + return "" + } + var ae *apiclient.APIError + if errors.As(err, &ae) { + switch ae.Kind() { + case apiclient.KindAuth: + return CodeAuth + case apiclient.KindPlan, apiclient.KindQuota: + return CodePlanQuota + case apiclient.KindProvider, apiclient.KindTimeout: + return CodeProviderTimeout + case apiclient.KindConflict, apiclient.KindBadInput: + return CodeUploadRejected + } + } + switch { + case errors.Is(err, ErrDegradeFile): + return CodeUploadRejected + // The local-file cases must come before any net.Error-shaped probe: + // syscall.Errno implements Timeout and Temporary, so the *fs.PathError a + // failed open or stat returns satisfies net.Error, and treating it as a + // transport failure would hide which local path was unreadable or gone. + case errors.Is(err, os.ErrPermission): + return CodeReadDenied + case errors.Is(err, os.ErrNotExist): + return CodeRootMissing + } + // Anything else is a request that never reached the server, including a + // transport error and a deadline exceeded mid-call. + return CodeNetwork +} + +// Run walks the given paths, parses each against its cursor, uploads units +// through upload, and persists cursors for the files it actually wrote. +// +// A parse or unit error other than ErrDegradeFile ends the run: the report +// tallies its classified code and the error is returned as-is, so the caller +// keeps its own error surface. An ErrDegradeFile error ends the current file +// only; its cursor stays put and the report records the failure. +func Run(ctx context.Context, paths []string, opts Options, parse ParseFunc, upload UploadFunc) (Report, error) { + var report Report + remaining := opts.Limit + + for _, abs := range paths { + report.Scanned++ + + cp := LoadCursor(opts.StateDir, abs) + if cp.Corrupt { + report.fail(CodeStateCorrupt) + } + units, unparseable, err := parse(abs, cp.LastLine) + report.Unparseable += unparseable + if err != nil { + report.fail(Classify(err)) + return report, err + } + + newLast := cp.LastLine + moved := false + for _, u := range units { + if opts.Limit > 0 && remaining <= 0 { + break + } + // Parse already skips at or below the cursor; this guard keeps an + // inconsistent parser from rewinding a cursor. + if u.Line <= newLast { + continue + } + + if opts.DryRun { + report.Ingested++ + if remaining > 0 { + remaining-- + } + continue + } + + outcome, err := upload(ctx, abs, u) + if err != nil { + if errors.Is(err, ErrDegradeFile) { + report.Failed++ + report.fail(CodeUploadRejected) + break + } + report.fail(Classify(err)) + return report, err + } + if outcome.Deduplicated { + report.Deduped++ + } else { + report.Ingested++ + } + if remaining > 0 { + remaining-- + } + newLast = u.Line + moved = true + } + if moved { + report.Changed++ + } + + if opts.DryRun { + continue + } + if size, mtime, err := FileState(abs); err == nil { + if err := SaveCursor(opts.StateDir, Cursor{ + Abs: abs, + Size: size, + ModTime: mtime, + LastLine: newLast, + }); err != nil { + if opts.Log != nil { + opts.Log("warn: save checkpoint for %s: %v\n", abs, err) + } + report.fail(CodeStateCorrupt) + } + } + } + return report, nil +} + +// HasJSONLExtension reports whether a path looks like a JSON-lines file, +// ignoring case. It is the accept predicate the transcript connectors use. +func HasJSONLExtension(p string) bool { + return strings.EqualFold(filepath.Ext(p), ".jsonl") +} diff --git a/server/internal/ingest/ingest_test.go b/server/internal/ingest/ingest_test.go new file mode 100644 index 0000000..3cbaaeb --- /dev/null +++ b/server/internal/ingest/ingest_test.go @@ -0,0 +1,463 @@ +package ingest + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "testing" + + "github.com/PeterGuy326/mem/server/internal/apiclient" +) + +func writeSource(t *testing.T, dir, name, content string) string { + t.Helper() + p := filepath.Join(dir, name) + if err := os.MkdirAll(filepath.Dir(p), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + return p +} + +// unitsForLines builds a parser that yields one unit per line above the cursor. +func unitsForLines(lines int) ParseFunc { + return func(abs string, skipBefore int) ([]Unit, int, error) { + var units []Unit + for line := skipBefore + 1; line <= lines; line++ { + units = append(units, Unit{Line: line, Body: fmt.Sprintf("%s#%d", filepath.Base(abs), line)}) + } + return units, 0, nil + } +} + +func TestRunDryRunWritesNothing(t *testing.T) { + src := t.TempDir() + states := t.TempDir() + abs := writeSource(t, src, "a.jsonl", "x") + + var uploads int + report, err := Run(context.Background(), []string{abs}, Options{StateDir: states, DryRun: true}, + unitsForLines(3), + func(context.Context, string, Unit) (Outcome, error) { + uploads++ + return Outcome{}, errors.New("dry-run must not upload") + }) + if err != nil { + t.Fatal(err) + } + if uploads != 0 { + t.Fatalf("uploads = %d, want 0 (dry-run performed writes)", uploads) + } + if report.Ingested != 3 || report.Scanned != 1 { + t.Fatalf("report = %+v, want 3 planned units in 1 scanned file", report) + } + entries, err := os.ReadDir(states) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Fatalf("dry-run left cursor state behind: %v", entries) + } +} + +func TestRunRespectsLimitAndReportsDedupSeparately(t *testing.T) { + src := t.TempDir() + states := t.TempDir() + abs := writeSource(t, src, "a.jsonl", "x") + + var uploaded []int + report, err := Run(context.Background(), []string{abs}, Options{StateDir: states, Limit: 2}, + unitsForLines(5), + func(_ context.Context, _ string, u Unit) (Outcome, error) { + uploaded = append(uploaded, u.Line) + // The second unit is a server-reported replay. + return Outcome{Deduplicated: u.Line == 2}, nil + }) + if err != nil { + t.Fatal(err) + } + if len(uploaded) != 2 || uploaded[0] != 1 || uploaded[1] != 2 { + t.Fatalf("uploaded lines = %v, want [1 2]", uploaded) + } + if report.Ingested != 1 || report.Deduped != 1 { + t.Fatalf("report = %+v, want 1 ingested and 1 deduped", report) + } + cp := LoadCursor(states, abs) + if cp.LastLine != 2 { + t.Fatalf("cursor LastLine = %d, want 2", cp.LastLine) + } +} + +func TestRunDegradesOneFileAndContinuesOthers(t *testing.T) { + src := t.TempDir() + states := t.TempDir() + a := writeSource(t, src, "a.jsonl", "x") + b := writeSource(t, src, "b.jsonl", "x") + + // Seed a's cursor at line 1 so the conflict starts from line 2. + if err := SaveCursor(states, Cursor{Abs: a, Size: 1, ModTime: "2026-01-01T00:00:00Z", LastLine: 1}); err != nil { + t.Fatal(err) + } + + var uploaded []string + report, err := Run(context.Background(), []string{a, b}, Options{StateDir: states}, + unitsForLines(3), + func(_ context.Context, abs string, u Unit) (Outcome, error) { + uploaded = append(uploaded, fmt.Sprintf("%s:%d", filepath.Base(abs), u.Line)) + if abs == a { + return Outcome{}, fmt.Errorf("%w: %s:%d", ErrDegradeFile, abs, u.Line) + } + return Outcome{}, nil + }) + if err != nil { + t.Fatalf("a degraded file must not abort the run: %v", err) + } + want := []string{"a.jsonl:2", "b.jsonl:1", "b.jsonl:2", "b.jsonl:3"} + if strings.Join(uploaded, ",") != strings.Join(want, ",") { + t.Fatalf("uploads = %v, want %v", uploaded, want) + } + if report.Failed != 1 || report.Failures[CodeUploadRejected] != 1 { + t.Fatalf("report = %+v, want one upload_rejected failure", report) + } + if got := LoadCursor(states, a).LastLine; got != 1 { + t.Fatalf("degraded file cursor LastLine = %d, want 1 (retry must stay meaningful)", got) + } + if got := LoadCursor(states, b).LastLine; got != 3 { + t.Fatalf("healthy file cursor LastLine = %d, want 3", got) + } +} + +func TestRunAbortsOnUnclassifiedUploadError(t *testing.T) { + src := t.TempDir() + states := t.TempDir() + abs := writeSource(t, src, "a.jsonl", "x") + + authErr := &apiclient.APIError{StatusCode: http.StatusUnauthorized} + _, err := Run(context.Background(), []string{abs}, Options{StateDir: states}, + unitsForLines(3), + func(context.Context, string, Unit) (Outcome, error) { + return Outcome{}, authErr + }) + if !errors.Is(err, authErr) { + t.Fatalf("err = %v, want the caller's error returned unchanged", err) + } + if _, err := os.Stat(CursorPath(states, abs)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("aborted run must not advance the cursor, stat err = %v", err) + } +} + +func TestCorruptCursorDegradesWithoutBlocking(t *testing.T) { + src := t.TempDir() + states := t.TempDir() + abs := writeSource(t, src, "a.jsonl", "x") + + p := CursorPath(states, abs) + if err := os.WriteFile(p, []byte("{not json"), 0o600); err != nil { + t.Fatal(err) + } + cp := LoadCursor(states, abs) + if !cp.Corrupt || cp.LastLine != 0 { + t.Fatalf("cursor = %+v, want Corrupt with a reset high-water mark", cp) + } + + report, err := Run(context.Background(), []string{abs}, Options{StateDir: states}, + unitsForLines(2), + func(context.Context, string, Unit) (Outcome, error) { return Outcome{}, nil }) + if err != nil { + t.Fatalf("a corrupt cursor must not block the run: %v", err) + } + if report.Failures[CodeStateCorrupt] != 1 { + t.Fatalf("report = %+v, want one state_corrupt failure", report) + } + if report.Ingested != 2 { + t.Fatalf("report.Ingested = %d, want 2 (re-planned from line 1)", report.Ingested) + } +} + +func TestLoadCursorResetsWhenFileShrank(t *testing.T) { + src := t.TempDir() + states := t.TempDir() + abs := writeSource(t, src, "a.jsonl", strings.Repeat("x", 20)) + + if err := SaveCursor(states, Cursor{Abs: abs, Size: 999, ModTime: "2026-01-01T00:00:00Z", LastLine: 7}); err != nil { + t.Fatal(err) + } + if got := LoadCursor(states, abs).LastLine; got != 0 { + t.Fatalf("LastLine = %d, want 0 after the file shrank below the recorded size", got) + } + + // Growing the file keeps the high-water mark. + if err := SaveCursor(states, Cursor{Abs: abs, Size: 1, ModTime: "2026-01-01T00:00:00Z", LastLine: 7}); err != nil { + t.Fatal(err) + } + if got := LoadCursor(states, abs).LastLine; got != 7 { + t.Fatalf("LastLine = %d, want 7 while the file only grew", got) + } +} + +// TestCursorOnDiskFormat pins the persisted layout: existing users' cursors +// must stay readable, so the key names and their order are part of the contract. +func TestCursorOnDiskFormat(t *testing.T) { + states := t.TempDir() + abs := filepath.Join(t.TempDir(), "a.jsonl") + want := `{"abs":"` + abs + `","size":12,"mtime":"2026-08-30T06:14:01Z","last_line":3}` + + if err := SaveCursor(states, Cursor{Abs: abs, Size: 12, ModTime: "2026-08-30T06:14:01Z", LastLine: 3}); err != nil { + t.Fatal(err) + } + b, err := os.ReadFile(CursorPath(states, abs)) + if err != nil { + t.Fatal(err) + } + if string(b) != want { + t.Fatalf("cursor bytes =\n%s\nwant\n%s", b, want) + } + fi, err := os.Stat(CursorPath(states, abs)) + if err != nil { + t.Fatal(err) + } + if perm := fi.Mode().Perm(); perm != 0o600 { + t.Fatalf("cursor mode = %o, want 600", perm) + } +} + +// TestWalkCanonicalizesRelativeBase pins the identity rule: every path Walk +// returns is absolute and symlink-resolved, so a caller-supplied relative root +// cannot make two working directories share one cursor. +func TestWalkCanonicalizesRelativeBase(t *testing.T) { + store := t.TempDir() + writeSource(t, store, "sessions/a.jsonl", "x") + absBase, err := CanonicalRoot(store) + if err != nil { + t.Fatal(err) + } + + t.Chdir(store) + got, err := Walk("sessions", HasJSONLExtension) + if err != nil { + t.Fatal(err) + } + want := filepath.Join(absBase, "sessions", "a.jsonl") + if len(got) != 1 || got[0] != want { + t.Fatalf("walk returned %v, want [%s]", got, want) + } + + // A same-named source in another working directory is a different identity, + // so the two never share a cursor file. + storeB := t.TempDir() + writeSource(t, storeB, "sessions/a.jsonl", "x") + t.Chdir(storeB) + second, err := Walk("sessions", HasJSONLExtension) + if err != nil { + t.Fatal(err) + } + if len(second) != 1 || second[0] == want { + t.Fatalf("relative bases from two directories collided on %v", second) + } + if CursorPath("states", want) == CursorPath("states", second[0]) { + t.Fatalf("cursor keys collide for %s and %s", want, second[0]) + } + + // A root that does not exist yet stays absolute instead of erroring, which + // is what the missing-root contract above needs. + absent := filepath.Join(store, "nope", "deep") + resolved, err := CanonicalRoot(absent) + if err != nil { + t.Fatal(err) + } + if !filepath.IsAbs(resolved) { + t.Fatalf("CanonicalRoot of an absent path = %q, want absolute", resolved) + } + t.Chdir(store) + if paths, err := Walk("nope/deep", HasJSONLExtension); err != nil || len(paths) != 0 { + t.Fatalf("missing root: paths = %v, err = %v; want none, no error", paths, err) + } +} + +func TestSaveCursorKeepsCommittedProgressAndLeavesNoTempFile(t *testing.T) { + states := t.TempDir() + abs := filepath.Join(t.TempDir(), "a.jsonl") + + // A run that read the file earlier must not rewind one that finished first. + if err := SaveCursor(states, Cursor{Abs: abs, Size: 200, ModTime: "2026-08-30T06:14:01Z", LastLine: 10}); err != nil { + t.Fatal(err) + } + if err := SaveCursor(states, Cursor{Abs: abs, Size: 200, ModTime: "2026-08-30T06:14:02Z", LastLine: 7}); err != nil { + t.Fatal(err) + } + if got := LoadCursor(states, abs); got.LastLine != 10 || got.Size != 200 { + t.Fatalf("cursor = %+v, want the committed line 10 kept", got) + } + + // A rewrite that shrank the file is the one case allowed to rewind. + if err := SaveCursor(states, Cursor{Abs: abs, Size: 40, ModTime: "2026-08-30T06:14:03Z", LastLine: 2}); err != nil { + t.Fatal(err) + } + if got := LoadCursor(states, abs).LastLine; got != 2 { + t.Fatalf("LastLine = %d, want 2 after the source shrank", got) + } + + entries, err := os.ReadDir(states) + if err != nil { + t.Fatal(err) + } + if len(entries) != 2 { + t.Fatalf("state dir = %v, want cursor file and lock sidecar", entries) + } + var hasJSON, hasLock bool + for _, e := range entries { + if strings.HasSuffix(e.Name(), ".json") { + hasJSON = true + } + if strings.HasSuffix(e.Name(), ".json.lock") { + hasLock = true + } + } + if !hasJSON || !hasLock { + t.Fatalf("state dir = %v, want .json cursor and .json.lock sidecar", entries) + } +} + +// TestSaveCursorDoesNotStageInASharedSlot pins the temporary naming. Reusing +// .tmp gives every process writing that cursor the same staging file, +// so one run's write can land inside another's rename. +func TestSaveCursorDoesNotStageInASharedSlot(t *testing.T) { + states := t.TempDir() + abs := filepath.Join(t.TempDir(), "a.jsonl") + leftover := CursorPath(states, abs) + ".tmp" + if err := os.WriteFile(leftover, []byte("another run's staging file"), 0o600); err != nil { + t.Fatal(err) + } + + if err := SaveCursor(states, Cursor{Abs: abs, Size: 20, ModTime: "2026-08-30T06:14:01Z", LastLine: 4}); err != nil { + t.Fatal(err) + } + if got := LoadCursor(states, abs).LastLine; got != 4 { + t.Fatalf("LastLine = %d, want 4", got) + } + if b, err := os.ReadFile(leftover); err != nil || string(b) != "another run's staging file" { + t.Fatalf("save consumed the shared staging file: %q, err %v", b, err) + } +} + +func TestConcurrentSaveCursorPublishesWholeCursors(t *testing.T) { + states := t.TempDir() + abs := filepath.Join(t.TempDir(), "a.jsonl") + + var wg sync.WaitGroup + for i := 1; i <= 8; i++ { + wg.Add(1) + go func(line int) { + defer wg.Done() + if err := SaveCursor(states, Cursor{Abs: abs, Size: 100, ModTime: "2026-08-30T06:14:01Z", LastLine: line}); err != nil { + t.Error(err) + } + }(i) + } + wg.Wait() + + b, err := os.ReadFile(CursorPath(states, abs)) + if err != nil { + t.Fatal(err) + } + var cp Cursor + if err := json.Unmarshal(b, &cp); err != nil { + t.Fatalf("cursor published half-written: %v (%s)", err, b) + } + if cp.LastLine < 1 || cp.LastLine > 8 { + t.Fatalf("cursor = %+v", cp) + } + if entries, err := os.ReadDir(states); err != nil || len(entries) != 2 { + t.Fatalf("state dir = %v, err = %v; want cursor file and lock sidecar", entries, err) + } +} + +func TestWalkIsDeterministicAndSkipsUnreadable(t *testing.T) { + root := t.TempDir() + for _, name := range []string{"c.jsonl", "nested/b.jsonl", "nested/deep/a.jsonl", "ignore.txt"} { + writeSource(t, root, name, "x") + } + got, err := Walk(root, HasJSONLExtension) + if err != nil { + t.Fatal(err) + } + if len(got) != 3 { + t.Fatalf("walk returned %d paths, want 3 (*.jsonl only): %v", len(got), got) + } + if !sort.StringsAreSorted(got) { + t.Fatalf("walk order must be lexical for stable cursors: %v", got) + } + + // A base naming one existing file is accepted as a one-off source. + one := got[0] + single, err := Walk(one, HasJSONLExtension) + if err != nil { + t.Fatal(err) + } + if len(single) != 1 || single[0] != one { + t.Fatalf("single-file base returned %v, want [%s]", single, one) + } + + // A missing root is not an error today: the connector reports "no + // transcripts matched" instead. Pin that, because surfacing it as a + // failure here would change `mem ingest qoder`'s observable behaviour. + missing, err := Walk(filepath.Join(root, "nope"), HasJSONLExtension) + if err != nil || len(missing) != 0 { + t.Fatalf("missing root: paths = %v, err = %v; want none, no error", missing, err) + } +} + +func TestClassifyCoversSharedCodes(t *testing.T) { + cases := []struct { + err error + want Code + }{ + {&apiclient.APIError{StatusCode: http.StatusForbidden}, CodeAuth}, + {&apiclient.APIError{StatusCode: http.StatusPaymentRequired}, CodePlanQuota}, + {&apiclient.APIError{StatusCode: http.StatusTooManyRequests}, CodePlanQuota}, + {&apiclient.APIError{StatusCode: http.StatusServiceUnavailable}, CodeProviderTimeout}, + {&apiclient.APIError{StatusCode: http.StatusGatewayTimeout}, CodeProviderTimeout}, + {&apiclient.APIError{StatusCode: http.StatusConflict}, CodeUploadRejected}, + {fmt.Errorf("wrapped: %w", ErrDegradeFile), CodeUploadRejected}, + {&os.PathError{Op: "open", Err: os.ErrPermission}, CodeReadDenied}, + {&os.PathError{Op: "stat", Err: os.ErrNotExist}, CodeRootMissing}, + {nil, ""}, + } + for _, tc := range cases { + if got := Classify(tc.err); got != tc.want { + t.Errorf("Classify(%v) = %q, want %q", tc.err, got, tc.want) + } + } + + // The rows above build PathErrors around sentinel errors. A real failed open + // carries a syscall.Errno instead, and Errno implements net.Error, so only + // this shape reproduces a file error being reported as a transport failure. + if _, err := os.Open(filepath.Join(t.TempDir(), "absent.jsonl")); err != nil { + if got := Classify(fmt.Errorf("open transcript: %w", err)); got != CodeRootMissing { + t.Errorf("Classify(real open failure) = %q, want %q", got, CodeRootMissing) + } + } else { + t.Fatal("opening a file that does not exist succeeded") + } +} + +func TestReportAddAggregatesAcrossCycles(t *testing.T) { + var total Report + total.Add(Report{Scanned: 2, Ingested: 5, Deduped: 1, Failed: 1, Failures: map[Code]int{CodeUploadRejected: 1}}) + total.Add(Report{Scanned: 1, Ingested: 2, Unchanged: 1, Failures: map[Code]int{CodeStateCorrupt: 1}}) + if total.Scanned != 3 || total.Ingested != 7 || total.Deduped != 1 || total.Failed != 1 { + t.Fatalf("total = %+v", total) + } + if total.Failures[CodeUploadRejected] != 1 || total.Failures[CodeStateCorrupt] != 1 { + t.Fatalf("failure tally = %+v", total.Failures) + } +} diff --git a/server/internal/redact/redact.go b/server/internal/redact/redact.go new file mode 100644 index 0000000..c324715 --- /dev/null +++ b/server/internal/redact/redact.go @@ -0,0 +1,182 @@ +// Package redact gates URLs on their way out of the process. +// +// The gate is fail-closed on purpose. A configured URL can carry a password in +// shapes that url.Parse does not report as userinfo: "admin:pw@host" parses as +// Scheme="admin", Opaque="pw@host", User=nil. So "the parser found no userinfo" +// is not evidence that a value is credential-free, and any implementation that +// gates on u.User == nil echoes the credential unchanged. A value the gate +// cannot positively prove safe is withheld whole. +// +// Scrubbing a message that already contains the URL is not a substitute: Go +// renders url.Error with %q, so a quote inside a password arrives escaped and a +// scanner that pairs quotes mis-pairs and replaces nothing. Callers hand text +// here instead and accept withholding when a piece cannot be verified. +package redact + +import ( + "errors" + "net/url" + "strings" +) + +// Placeholder replaces a value the gate cannot prove credential-free. It is a +// fixed token so an operator can tell withholding apart from a real host. +// Deliberately free of '<', '>' and '&': encoding/json escapes those, so an +// angle-bracketed marker would render differently in text and JSON output. +const Placeholder = "[withheld]" + +// UserMarker replaces the userinfo of a URL, and the value of every query +// parameter, when the rest of the URL is safe to echo. It uses only unreserved +// characters because url.User("***") would percent-encode the asterisks and +// url.Values.Encode() would do the same to a marked-up query value. +const UserMarker = "REDACTED" + +// APIURLs are the schemes a client base URL may legitimately use. +var APIURLs = []string{"http", "https"} + +// StoreURLs are the schemes memd logs: the database DSN and the queue DSN. +var StoreURLs = []string{"http", "https", "postgres", "postgresql", "redis", "rediss", "redis+unix", "unix"} + +// URL returns raw with userinfo replaced by UserMarker, or Placeholder when the +// value cannot be proven credential-free. +func URL(raw string, allowed []string) string { + safe, ok := rewrite(raw, allowed) + if !ok { + return Placeholder + } + return safe +} + +// Text renders diagnostic text — an error message, typically — so that no +// URL-shaped token inside it can carry userinfo out. A single unverifiable +// token withholds the entire message rather than trimming that token, because a +// delimiter inside a credential splits the text into pieces that no longer look +// like a URL, and the piece without the "@" is exactly the half that leaked. +// +// Query and fragment values are part of the same problem: pgx honours +// postgres://host/db?password=x as the real password, so a value that parses as +// a clean URL is not thereby proven credential-free. +func Text(msg string, allowed []string) string { + out := msg + for _, token := range urlTokens(msg) { + safe, ok := rewrite(token, allowed) + if !ok { + return Placeholder + } + if safe != token { + out = strings.Replace(out, token, safe, 1) + } + } + return out +} + +// TransportError renders an error returned by http.Client.Do. The URL travels +// through the gate rather than through Go's own masking, which strips the +// password but leaves the username, and the wrapped cause is kept so the +// message still names what failed. +func TransportError(err error, allowed []string) string { + if err == nil { + return "" + } + var ue *url.Error + if errors.As(err, &ue) && ue.Err != nil { + return Text(ue.Op, allowed) + " " + URL(ue.URL, allowed) + ": " + Text(ue.Err.Error(), allowed) + } + return Text(err.Error(), allowed) +} + +// rewrite reports whether raw is a URL we can prove carries no credential, and +// returns the form that is safe to echo. +func rewrite(raw string, allowed []string) (string, bool) { + if raw == "" { + return "", true + } + parsed, err := url.Parse(raw) + if err != nil { + return "", false + } + // Credentials hide in Opaque precisely when the scheme is really userinfo, + // and an empty or unknown scheme means we are not looking at a transport URL + // we can reason about. + if parsed.Opaque != "" || !allowedScheme(parsed.Scheme, allowed) { + return "", false + } + if parsed.User != nil { + parsed.User = url.User(UserMarker) + } + // A credential can travel as a connection parameter, and pgx honours + // postgres://host/db?password=… as the real password. Blanking only the keys + // that look secret would claim that we can prove some other value is not a + // credential, which is the claim this package refuses to make, so every query + // value goes and only the parameter names survive. A fragment has no name to + // keep, so it withholds the URL. + // + // ponytail: that costs ?sslmode=disable its value in memd's startup log. If + // it costs someone a debugging minute, keep an allowlist of parameters that + // cannot carry a secret and fail every unknown one to the marker. + if parsed.RawQuery != "" { + q, err := url.ParseQuery(parsed.RawQuery) + if err != nil { + return "", false + } + blanked := make(url.Values, len(q)) + for key := range q { + blanked[key] = []string{UserMarker} + } + parsed.RawQuery = blanked.Encode() + } + if parsed.Fragment != "" { + return "", false + } + // What leaves the process is the re-serialised form, so verify that instead + // of trusting the first parse: String() can rebuild something different from + // the input, and an "@" surviving into the host means userinfo was never in + // the field we stripped. + rendered := parsed.String() + back, err := url.Parse(rendered) + if err != nil || back.Opaque != "" || back.Host != parsed.Host || + !strings.EqualFold(back.Scheme, parsed.Scheme) || strings.Contains(back.Host, "@") { + return "", false + } + if back.User != nil { + if _, hasPassword := back.User.Password(); hasPassword { + return "", false + } + if back.User.Username() != UserMarker { + return "", false + } + } + return rendered, true +} + +func allowedScheme(scheme string, allowed []string) bool { + for _, want := range allowed { + if strings.EqualFold(scheme, want) { + return true + } + } + return false +} + +// urlTokens returns the whitespace- and quote-delimited runs of msg that look +// like they could carry a host or userinfo. Splitting on delimiters is safe +// because any run produced this way still holds either the "@" or the "://" +// that marked the original as credential-shaped, unless the original held +// neither and was never a URL at all. +func urlTokens(msg string) []string { + var tokens []string + for _, token := range strings.FieldsFunc(msg, isDelimiter) { + if strings.Contains(token, "@") || strings.Contains(token, "://") { + tokens = append(tokens, token) + } + } + return tokens +} + +func isDelimiter(r rune) bool { + switch r { + case ' ', '\t', '\n', '\r', '"', '\'', '`', '(', ')', '[', ']', '{', '}', '<', '>', ',': + return true + } + return false +} diff --git a/server/internal/redact/redact_test.go b/server/internal/redact/redact_test.go new file mode 100644 index 0000000..1f6392b --- /dev/null +++ b/server/internal/redact/redact_test.go @@ -0,0 +1,291 @@ +package redact + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/url" + "strings" + "testing" +) + +// secret marks every fixture below. No case may let it reach the returned +// string, and the tests fail closed on the marker rather than on a specific +// redaction shape. +const secret = "s3ntinel-p4ssw0rd" + +func TestURLWithholdsShapesTheParserCannotAttribute(t *testing.T) { + cases := []struct { + name string + raw string + allowed []string + want string + }{ + { + name: "userinfo on a recognised scheme", + raw: "http://admin:" + secret + "@mem.internal:8787", + allowed: APIURLs, + want: "http://REDACTED@mem.internal:8787", + }, + { + name: "username only", + raw: "http://admin@mem.internal:8787", + allowed: APIURLs, + want: "http://REDACTED@mem.internal:8787", + }, + { + name: "no credential at all is echoed unchanged", + raw: "http://localhost:8787", + allowed: APIURLs, + want: "http://localhost:8787", + }, + { + name: "empty value has nothing to leak", + raw: "", + allowed: APIURLs, + want: "", + }, + { + name: "store scheme is allowed for a DSN egress", + raw: "postgres://mem:" + secret + "@localhost:5432/mem?sslmode=disable", + allowed: StoreURLs, + want: "postgres://REDACTED@localhost:5432/mem?sslmode=REDACTED", + }, + // The shape this package exists for: url.Parse succeeds, User is nil and + // the whole credential sits in Opaque, so a u.User != nil gate misses it. + { + name: "no scheme, credential in Opaque", + raw: "admin:" + secret + "@mem.internal:8787", + allowed: APIURLs, + want: Placeholder, + }, + { + name: "scheme the egress does not use", + raw: "gopher://" + secret + "@mem.internal:8787", + allowed: APIURLs, + want: Placeholder, + }, + { + name: "store scheme on an API egress", + raw: "postgres://mem:" + secret + "@localhost:5432/mem", + allowed: APIURLs, + want: Placeholder, + }, + { + name: "parse failure with a space in the host", + raw: "http://admin:" + secret + "@ho st.example.com:8787", + allowed: APIURLs, + want: Placeholder, + }, + { + name: "parse failure with a bad percent escape", + raw: "http://admin:" + secret + "@mem.internal:%zz", + allowed: APIURLs, + want: Placeholder, + }, + { + name: "parse failure with a space in the password", + raw: "http://admin:" + secret + " x@mem.internal:8787", + allowed: APIURLs, + want: Placeholder, + }, + { + name: "parse failure ending in an escape sign", + raw: "http://admin:" + secret + "@%", + allowed: APIURLs, + want: Placeholder, + }, + { + name: "out-of-range port still parses, so userinfo is stripped", + raw: "http://admin:" + secret + "@mem.internal:99999999", + allowed: APIURLs, + want: "http://REDACTED@mem.internal:99999999", + }, + { + name: "non-numeric port does not parse", + raw: "http://admin:" + secret + "@mem.internal:notaport", + allowed: APIURLs, + want: Placeholder, + }, + { + // No scheme is not a recognised transport scheme, so the adjudicated + // rule withholds it even though the credential did land in User. + name: "scheme-relative value has no scheme to check", + raw: "//admin:" + secret + "@mem.internal:8787", + allowed: APIURLs, + want: Placeholder, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := URL(tc.raw, tc.allowed) + if got != tc.want { + t.Errorf("URL(%q) = %q, want %q", tc.raw, got, tc.want) + } + if strings.Contains(got, secret) { + t.Errorf("URL(%q) = %q, leaks the sentinel", tc.raw, got) + } + }) + } +} + +func TestTextWithholdsWhenAnyCredentialShapedTokenIsUnverifiable(t *testing.T) { + cases := []struct { + name string + msg string + want string + }{ + { + name: "plain transport cause survives", + msg: `dial tcp: lookup mem.internal: no such host`, + want: `dial tcp: lookup mem.internal: no such host`, + }, + { + name: "well-formed credential URL is redacted in place", + msg: `Get "http://admin:` + secret + `@mem.internal:8787/healthz": dial tcp refused`, + want: `Get "http://REDACTED@mem.internal:8787/healthz": dial tcp refused`, + }, + { + name: "no-scheme shape withholds the whole line", + msg: `Get "admin:` + secret + `@mem.internal:8787": unsupported protocol scheme "admin"`, + want: Placeholder, + }, + { + name: "quote-escaped password withholds the whole line", + msg: `Get "http://admin:` + secret + `\"@mem.internal:8787": context deadline exceeded`, + want: Placeholder, + }, + { + name: "space-split password withholds the whole line", + msg: `Get "http://admin:` + secret + ` x@mem.internal:8787": dial tcp`, + want: Placeholder, + }, + { + name: "at sign in a path is not a credential", + msg: `GET https://mem.internal/v1/files/report%40mem.internal: permission denied`, + want: `GET https://mem.internal/v1/files/report%40mem.internal: permission denied`, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := Text(tc.msg, APIURLs) + if got != tc.want { + t.Errorf("Text(%q) = %q, want %q", tc.msg, got, tc.want) + } + if strings.Contains(got, secret) { + t.Errorf("Text(%q) = %q, leaks the sentinel", tc.msg, got) + } + }) + } +} + +// TestTextGatesAStoreDsnEmbeddedInAForeignError covers the shape that reaches +// memd's fatal log: asynq puts the whole DSN into its parse error and +// queue.NewClient wraps that verbatim. +func TestTextGatesAStoreDsnEmbeddedInAForeignError(t *testing.T) { + msg := `queue: parse redis url: asynq: could not parse redis uri: ` + + `parse "redis://:` + secret + `@ho st:6379/0": invalid character " " in host name` + + got := Text(msg, StoreURLs) + if strings.Contains(got, secret) { + t.Errorf("Text(%q) = %q, leaks the sentinel", msg, got) + } +} + +// TestQueryAndFragmentCredentialsAreWithheld pins the shape the userinfo gate +// used to miss: pgx honours postgres://host/db?password=… as a real password, so +// a URL that parses cleanly with User == nil is not thereby proven safe. Names +// of parameters survive so a log line still says which settings are on; no value +// does, and a bare secret in a fragment withholds the URL. +func TestQueryAndFragmentCredentialsAreWithheld(t *testing.T) { + cases := []struct { + raw string + want string + }{ + { + raw: "redis://queue.internal:6379/0?password=" + secret, + want: "redis://queue.internal:6379/0?password=REDACTED", + }, + { + raw: "postgres://mem@db.internal:5432/mem?sslmode=require&password=" + secret, + want: "postgres://REDACTED@db.internal:5432/mem?password=REDACTED&sslmode=REDACTED", + }, + { + raw: "redis://queue.internal:6379/0#" + secret, + want: Placeholder, + }, + } + + for _, tc := range cases { + if got := URL(tc.raw, StoreURLs); got != tc.want { + t.Errorf("URL(%q) = %q, want %q", tc.raw, got, tc.want) + } + if got := Text(tc.raw, StoreURLs); strings.Contains(got, secret) { + t.Errorf("Text(%q) = %q, leaks the sentinel", tc.raw, got) + } + } +} + +func TestTransportErrorNamesTheFailureWithoutEchoingCredentials(t *testing.T) { + sentinel := "http://admin:" + secret + "@mem.internal:8787" + cases := []struct { + name string + err error + }{ + { + name: "url error carrying parsed userinfo", + err: &url.Error{Op: "Get", URL: sentinel, Err: errors.New("dial tcp: connection refused")}, + }, + { + name: "url error whose URL is not a transport scheme", + err: &url.Error{Op: "Get", Err: errors.New("unsupported protocol scheme \"admin\""), URL: "admin:" + secret + "@mem.internal:8787"}, + }, + { + name: "bare error mentioning the URL in prose", + err: fmt.Errorf("proxy returned 407 for %s", sentinel), + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := TransportError(tc.err, APIURLs) + if strings.Contains(got, secret) { + t.Errorf("TransportError(%v) = %q, leaks the sentinel", tc.err, got) + } + if got == "" { + t.Errorf("TransportError(%v) = %q, want the failure to stay diagnosable", tc.err, got) + } + }) + } + + if got := TransportError(nil, APIURLs); got != "" { + t.Errorf("TransportError(nil) = %q, want empty", got) + } +} + +// TestTransportErrorKeepsErrorIdentity pins that rendering does not replace the +// chain a caller classifies on: doctor maps a timeout to exit 5 via errors.Is on +// context.DeadlineExceeded, and that has to keep working after the text changes. +func TestTransportErrorKeepsErrorIdentity(t *testing.T) { + sentinel := "http://admin:" + secret + "@mem.internal:8787" + client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return nil, &url.Error{Op: "Get", URL: sentinel, Err: context.DeadlineExceeded} + })} + _, err := client.Get(sentinel) + if err == nil { + t.Fatal("expected a transport failure") + } + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("transport error lost its cause: %v", err) + } + if rendered := TransportError(err, APIURLs); strings.Contains(rendered, secret) { + t.Errorf("TransportError = %q, leaks the sentinel", rendered) + } +} + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) } diff --git a/server/internal/relator/relator.go b/server/internal/relator/relator.go index 30d009b..e61f868 100644 --- a/server/internal/relator/relator.go +++ b/server/internal/relator/relator.go @@ -178,8 +178,9 @@ func (s *Service) fileMeta(ctx context.Context, id uuid.UUID) (userID uuid.UUID, // recomputeText finds the top-K text-embedding nearest neighbors for srcID // (within the same user) and rewrites file_relations rows of type same_topic. // -// Strategy: take the first chunk of src as the seed; ANN against ALL chunks -// of OTHER files, DISTINCT ON dst file (best chunk wins). +// Strategy: take the first chunk of src as the seed; walk cosine-ordered +// chunks of OTHER files (HNSW-compatible) and fall back to DISTINCT ON when +// a bounded scan underfills after per-file deduplication. func (s *Service) recomputeText(ctx context.Context, srcID, userID uuid.UUID, topK int) error { tx, err := s.pool.Begin(ctx) if err != nil { @@ -194,47 +195,22 @@ func (s *Service) recomputeText(ctx context.Context, srcID, userID uuid.UUID, to return fmt.Errorf("clear: %w", err) } - rows, err := tx.Query(ctx, ` - WITH seed AS ( - SELECT embedding FROM embeddings_text - WHERE file_id = $1 AND chunk_index = 0 - LIMIT 1 - ) - SELECT DISTINCT ON (e.file_id) - e.file_id, - (1 - (e.embedding <=> (SELECT embedding FROM seed)))::real AS score - FROM embeddings_text e - JOIN files f ON f.id = e.file_id - WHERE f.user_id = $2 - AND e.file_id != $1 - AND (SELECT embedding FROM seed) IS NOT NULL - ORDER BY e.file_id, e.embedding <=> (SELECT embedding FROM seed) ASC - LIMIT $3 - `, srcID, userID, topK) + neighbors, err := textNeighbors(ctx, tx, srcID, userID, topK) if err != nil { return fmt.Errorf("knn: %w", err) } - defer rows.Close() batch := &pgx.Batch{} count := 0 - for rows.Next() { - var dstID uuid.UUID - var score float32 - if err := rows.Scan(&dstID, &score); err != nil { - return fmt.Errorf("scan: %w", err) - } + for _, n := range neighbors { batch.Queue(` INSERT INTO file_relations (src_id, dst_id, type, score, computed_at) VALUES ($1, $2, $3, $4, now()) ON CONFLICT (src_id, dst_id, type) DO UPDATE SET score = EXCLUDED.score, computed_at = EXCLUDED.computed_at - `, srcID, dstID, TypeSameTopic, score) + `, srcID, n.fileID, TypeSameTopic, n.score) count++ } - if err := rows.Err(); err != nil { - return err - } if count > 0 { br := tx.SendBatch(ctx, batch) for i := 0; i < count; i++ { @@ -250,6 +226,158 @@ func (s *Service) recomputeText(ctx context.Context, srcID, userID uuid.UUID, to return tx.Commit(ctx) } +type textNeighbor struct { + fileID uuid.UUID + score float32 +} + +// textNeighbors preserves best-chunk-per-file top-K. It walks cosine-order +// candidates (HNSW-compatible) and falls back to exact DISTINCT ON when a +// bounded scan underfills after per-file deduplication. +func textNeighbors(ctx context.Context, tx pgx.Tx, srcID, userID uuid.UUID, topK int) ([]textNeighbor, error) { + if topK <= 0 { + return nil, nil + } + selected := make([]uuid.UUID, 0, topK) + seen := make(map[uuid.UUID]struct{}, topK) + out := make([]textNeighbor, 0, topK) + for round := 0; round <= topK && len(out) < topK; round++ { + remaining := topK - len(out) + batch, err := queryTextNeighborsDistanceOrder(ctx, tx, srcID, userID, selected, remaining) + if err != nil { + return nil, err + } + added := 0 + for _, n := range batch { + if _, ok := seen[n.fileID]; ok { + continue + } + seen[n.fileID] = struct{}{} + selected = append(selected, n.fileID) + out = append(out, n) + added++ + if len(out) >= topK { + break + } + } + if added == 0 { + rest, err := queryTextNeighborsExact(ctx, tx, srcID, userID, selected, remaining) + if err != nil { + return nil, err + } + out = append(out, rest...) + break + } + } + if len(out) > topK { + out = out[:topK] + } + return out, nil +} + +func queryTextNeighborsDistanceOrder( + ctx context.Context, + tx pgx.Tx, + srcID, userID uuid.UUID, + selected []uuid.UUID, + limit int, +) ([]textNeighbor, error) { + var seed string + err := tx.QueryRow(ctx, ` + SELECT embedding::text FROM embeddings_text + WHERE file_id = $1 AND chunk_index = 0 + LIMIT 1 + `, srcID).Scan(&seed) + if err == pgx.ErrNoRows { + return nil, nil + } + if err != nil { + return nil, err + } + excludeSQL := "TRUE" + args := []any{seed, srcID, userID, limit} + if len(selected) > 0 { + args = append(args, selected) + excludeSQL = fmt.Sprintf("NOT (e.file_id = ANY($%d::uuid[]))", len(args)) + } + rows, err := tx.Query(ctx, fmt.Sprintf(` + WITH nearest AS ( + SELECT e.file_id, e.embedding <=> $1::vector AS dist + FROM embeddings_text e + WHERE e.file_id != $2 + AND %s + ORDER BY e.embedding <=> $1::vector ASC + LIMIT $4 + ) + SELECT n.file_id, (1 - n.dist)::real AS score + FROM nearest n + JOIN files f ON f.id = n.file_id + WHERE f.user_id = $3 + ORDER BY n.dist ASC + `, excludeSQL), args...) + if err != nil { + return nil, err + } + defer rows.Close() + return scanTextNeighbors(rows) +} + +func queryTextNeighborsExact( + ctx context.Context, + tx pgx.Tx, + srcID, userID uuid.UUID, + selected []uuid.UUID, + limit int, +) ([]textNeighbor, error) { + excludeSQL := "TRUE" + args := []any{srcID, userID, limit} + if len(selected) > 0 { + args = append(args, selected) + excludeSQL = fmt.Sprintf("NOT (e.file_id = ANY($%d::uuid[]))", len(args)) + } + rows, err := tx.Query(ctx, fmt.Sprintf(` + WITH seed AS ( + SELECT embedding FROM embeddings_text + WHERE file_id = $1 AND chunk_index = 0 + LIMIT 1 + ) + SELECT file_id, score FROM ( + SELECT DISTINCT ON (e.file_id) + e.file_id, + (1 - (e.embedding <=> (SELECT embedding FROM seed)))::real AS score + FROM embeddings_text e + JOIN files f ON f.id = e.file_id + WHERE f.user_id = $2 + AND e.file_id != $1 + AND (SELECT embedding FROM seed) IS NOT NULL + AND %s + ORDER BY e.file_id, e.embedding <=> (SELECT embedding FROM seed) ASC + ) hits + ORDER BY score DESC + LIMIT $3 + `, excludeSQL), args...) + if err != nil { + return nil, err + } + defer rows.Close() + return scanTextNeighbors(rows) +} + +func scanTextNeighbors(rows pgx.Rows) ([]textNeighbor, error) { + out := make([]textNeighbor, 0, 8) + for rows.Next() { + var n textNeighbor + if err := rows.Scan(&n.fileID, &n.score); err != nil { + return nil, fmt.Errorf("scan: %w", err) + } + out = append(out, n) + } + if err := rows.Err(); err != nil { + return nil, err + } + return out, nil +} + func (s *Service) recomputeVisual(ctx context.Context, srcID, userID uuid.UUID, topK int) error { tx, err := s.pool.Begin(ctx) if err != nil { diff --git a/server/internal/search/hnsw_semantics_test.go b/server/internal/search/hnsw_semantics_test.go new file mode 100644 index 0000000..2f69ccc --- /dev/null +++ b/server/internal/search/hnsw_semantics_test.go @@ -0,0 +1,116 @@ +package search + +import ( + "context" + "os" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" + + memdb "github.com/PeterGuy326/mem/server/internal/db" +) + +// A chunk candidate budget must not become a file result budget or bypass +// owner/path/MIME/time scope. One file holding 101 nearest chunks must still +// yield k distinct eligible files, using HNSW continuation plus exact fallback. +func TestTextANNFileSemanticsPostgres(t *testing.T) { + dsn := os.Getenv("MEM_TEST_DB") + if dsn == "" { + t.Skip("MEM_TEST_DB not set; skipping text ANN PostgreSQL regression") + } + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + t.Fatal(err) + } + if !strings.HasSuffix(cfg.ConnConfig.Database, "_test") { + t.Fatalf("refusing non-test database %q", cfg.ConnConfig.Database) + } + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + database, err := memdb.Open(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer database.Close() + if err := database.Migrate(ctx); err != nil { + t.Fatal(err) + } + owner, other := uuid.New(), uuid.New() + for _, id := range []uuid.UUID{owner, other} { + if _, err := database.Pool.Exec(ctx, "INSERT INTO users(id,email,password_hash) VALUES($1,$2,'test')", id, id.String()+"@example.test"); err != nil { + t.Fatal(err) + } + } + defer func() { + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cleanupCancel() + if _, err := database.Pool.Exec(cleanupCtx, "DELETE FROM users WHERE id=ANY($1::uuid[])", []uuid.UUID{owner, other}); err != nil { + t.Errorf("cleanup: %v", err) + } + }() + now := time.Now().UTC().Truncate(time.Second) + since, until := now.Add(-time.Hour), now.Add(time.Hour) + vec := make([]float32, textEmbeddingSchemaDim) + vec[0] = 1 + addFile := func(user uuid.UUID, path, mime string, at time.Time, chunks int, far bool) uuid.UUID { + t.Helper() + id := uuid.New() + _, err := database.Pool.Exec(ctx, `INSERT INTO files(id,user_id,name,path,size,sha256,mime,storage_key,created_at,timeline_at) + VALUES($1,$2,'fixture',$3,0,$4,$5,$6,$7,$7)`, id, user, path, id.String(), mime, id.String(), at) + if err != nil { + t.Fatal(err) + } + for chunk := 0; chunk < chunks; chunk++ { + v := make([]float32, textEmbeddingSchemaDim) + if far { + v[1] = 1 + } else { + v[0], v[1] = 1, float32(chunk)*0.001 + } + _, err := database.Pool.Exec(ctx, `INSERT INTO embeddings_text(file_id,chunk_index,chunk_text,embedding) + VALUES($1,$2,'source chunk',$3::vector)`, id, chunk, vectorLiteral(v)) + if err != nil { + t.Fatal(err) + } + } + return id + } + best := addFile(owner, "/Work_%/Docs", "text/plain", now, 101, false) + eligible := map[uuid.UUID]bool{best: true} + for i := 0; i < 19; i++ { + eligible[addFile(owner, "/Work_%/Docs", "application/pdf", now, 1, true)] = true + } + addFile(other, "/Work_%/Docs", "text/plain", now, 1, false) + addFile(owner, "/Work_AB/Docs", "text/plain", now, 1, false) + addFile(owner, "/Work_%/Private", "text/plain", now, 1, false) + addFile(owner, "/Work_%/Docs", "image/png", now, 1, false) + addFile(owner, "/Work_%/Docs", "text/plain", since.Add(-time.Second), 1, false) + addFile(owner, "/Work_%/Docs", "text/plain", until.Add(time.Second), 1, false) + service := New(database.Pool, nil) + q := Query{UserID: owner, Limit: 10, PathPrefix: "/Work_%", AllowedPaths: []string{"/Work_%/Docs"}, Type: "doc", Since: &since, Until: &until, SnippetChars: 200} + hits, err := service.runTextANN(ctx, q, vec) + if err != nil { + t.Fatal(err) + } + if len(hits) != 10 { + t.Fatalf("got %d files, want 10 despite 101 nearest chunks belonging to one file", len(hits)) + } + seen := map[uuid.UUID]bool{} + for _, hit := range hits { + if !eligible[hit.FileID] || seen[hit.FileID] { + t.Fatalf("out-of-scope or duplicate file: %+v", hit) + } + seen[hit.FileID] = true + } + if hits[0].FileID != best || hits[0].ChunkIndex != 0 || hits[0].Score != 1 { + t.Fatalf("best chunk was not preserved: %+v", hits[0]) + } + q.AllowedPaths = []string{""} + hits, err = service.runTextANN(ctx, q, vec) + if err != nil || len(hits) != 0 { + t.Fatalf("invalid allow-list must fail closed: hits=%v err=%v", hits, err) + } +} diff --git a/server/internal/search/lexical_test.go b/server/internal/search/lexical_test.go new file mode 100644 index 0000000..1453459 --- /dev/null +++ b/server/internal/search/lexical_test.go @@ -0,0 +1,244 @@ +package search + +import ( + "context" + "os" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" + + memdb "github.com/PeterGuy326/mem/server/internal/db" +) + +func TestLexicalSearchWithoutWorker(t *testing.T) { + dsn := os.Getenv("MEM_TEST_DB") + if dsn == "" { + t.Skip("MEM_TEST_DB not set; skipping lexical search PostgreSQL test") + } + config, err := pgxpool.ParseConfig(dsn) + if err != nil { + t.Fatalf("parse MEM_TEST_DB: %v", err) + } + if !strings.HasSuffix(config.ConnConfig.Database, "_test") { + t.Fatalf( + "refusing to modify non-test database %q; MEM_TEST_DB must end in _test", + config.ConnConfig.Database, + ) + } + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + database, err := memdb.Open(ctx, dsn) + if err != nil { + t.Fatal(err) + } + t.Cleanup(database.Close) + if err := database.Migrate(ctx); err != nil { + t.Fatal(err) + } + + userID := uuid.New() + if _, err := database.Pool.Exec(ctx, ` + INSERT INTO users (id, email, password_hash) + VALUES ($1, $2, 'test-only') + `, userID, "lexical-"+uuid.NewString()+"@example.test"); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cleanupCancel() + database.Pool.Exec(cleanupCtx, `DELETE FROM users WHERE id = $1`, userID) + }) + + now := time.Now().UTC().Truncate(time.Second) + files := []struct { + name string + path string + mime string + }{ + {"quarterly_report.pdf", "/Work/Reports", "application/pdf"}, + {"meeting_notes.md", "/Work/Notes", "text/markdown"}, + {"meeting_notes.md", "/Work/NotesExtra", "text/markdown"}, + {"literal_scope.txt", "/Work/100%_done", "text/plain"}, + {"literal_scope.txt", "/Work/100XXdone", "text/plain"}, + {"photo_beach.jpg", "/Photos", "image/jpeg"}, + {"年度总结.docx", "/Work", "application/vnd.openxmlformats-officedocument.wordprocessingml.document"}, + } + for _, f := range files { + if _, err := database.Pool.Exec(ctx, ` + INSERT INTO files ( + user_id, name, path, size, sha256, mime, storage_key, + index_status, created_at, updated_at + ) VALUES ( + $1, $2, $3, 1, $4, $5, $6, + 'ready', $7, $7 + ) + `, userID, f.name, f.path, strings.Repeat("b", 64), f.mime, + "test/"+uuid.NewString(), now); err != nil { + t.Fatal(err) + } + } + + // Service with nil worker — the key precondition for this test. + service := New(database.Pool, nil) + + t.Run("lexical route works without worker", func(t *testing.T) { + hits, err := service.Search(ctx, Query{ + UserID: userID, + Text: "report", + Route: RouteLexical, + Limit: 10, + }) + if err != nil { + t.Fatalf("lexical search failed: %v", err) + } + if len(hits) == 0 { + t.Fatal("lexical search returned no results for 'report'") + } + found := false + for _, h := range hits { + if h.Name == "quarterly_report.pdf" { + found = true + if h.Source != RouteLexical { + t.Errorf("hit source = %q, want %q", h.Source, RouteLexical) + } + break + } + } + if !found { + t.Errorf("expected quarterly_report.pdf in results, got %+v", hits) + } + }) + + t.Run("text route fails without worker", func(t *testing.T) { + _, err := service.Search(ctx, Query{ + UserID: userID, + Text: "report", + Route: RouteText, + Limit: 10, + }) + if err == nil { + t.Fatal("text route should fail without worker") + } + if !strings.Contains(err.Error(), "worker not configured") { + t.Fatalf("unexpected error: %v", err) + } + }) + + t.Run("auto route fails without worker", func(t *testing.T) { + _, err := service.Search(ctx, Query{ + UserID: userID, + Text: "report", + Route: RouteAuto, + Limit: 10, + }) + if err == nil { + t.Fatal("auto route should fail without worker") + } + if !strings.Contains(err.Error(), "worker not configured") { + t.Fatalf("unexpected error: %v", err) + } + }) + + t.Run("substring match for CJK filename", func(t *testing.T) { + hits, err := service.Search(ctx, Query{ + UserID: userID, + Text: "总结", + Route: RouteLexical, + Limit: 10, + }) + if err != nil { + t.Fatalf("lexical CJK search failed: %v", err) + } + if len(hits) == 0 { + t.Fatal("lexical search returned no results for CJK query '总结'") + } + }) + + t.Run("path filter applies to lexical", func(t *testing.T) { + hits, err := service.Search(ctx, Query{ + UserID: userID, + Text: "notes", + Route: RouteLexical, + PathPrefix: "/Work/Notes", + Limit: 10, + }) + if err != nil { + t.Fatalf("lexical path-filtered search failed: %v", err) + } + if len(hits) != 1 || hits[0].Name != "meeting_notes.md" || hits[0].Path != "/Work/Notes" { + t.Fatalf("path filter should return only the matching subtree: %+v", hits) + } + }) + t.Run("trigram typo retains weak matches without similarity prefilter", func(t *testing.T) { + hits, err := service.Search(ctx, Query{UserID: userID, Route: RouteLexical, Text: "quaterly", Limit: 10}) + if err != nil { + t.Fatal(err) + } + for _, hit := range hits { + if hit.Name == "quarterly_report.pdf" { + if hit.Score < 0.20 || hit.Score >= 0.70 { + t.Fatalf("typo should be ranked in the trigram tier: %+v", hit) + } + return + } + } + t.Fatalf("trigram-only typo was dropped: %+v", hits) + }) + + t.Run("literal authorized subtree", func(t *testing.T) { + hits, err := service.Search(ctx, Query{ + UserID: userID, Route: RouteLexical, Text: "literal_scope", Limit: 10, + AllowedPaths: []string{"/Work/100%_done"}, + }) + if err != nil { + t.Fatal(err) + } + if len(hits) != 1 || hits[0].Path != "/Work/100%_done" { + t.Fatalf("literal authorization scope leaked or lost results: %+v", hits) + } + }) + + t.Run("MIME and time filters", func(t *testing.T) { + hits, err := service.Search(ctx, Query{UserID: userID, Route: RouteLexical, Text: "beach", Type: "image"}) + if err != nil || len(hits) != 1 || hits[0].Name != "photo_beach.jpg" { + t.Fatalf("image filter: hits=%+v err=%v", hits, err) + } + after := now.Add(time.Second) + before := now.Add(-time.Second) + for _, q := range []Query{ + {Type: "audio"}, {Since: &after}, {Until: &before}, + {AllowedPaths: []string{"/Private"}}, + {AllowedPaths: []string{""}}, + } { + q.UserID, q.Route, q.Text = userID, RouteLexical, "beach" + hits, err := service.Search(ctx, q) + if err != nil || len(hits) != 0 { + t.Fatalf("filter %+v: hits=%+v err=%v", q, hits, err) + } + } + }) + + t.Run("other owner cannot retrieve files", func(t *testing.T) { + hits, err := service.Search(ctx, Query{UserID: uuid.New(), Route: RouteLexical, Text: "quarterly_report"}) + if err != nil || len(hits) != 0 { + t.Fatalf("other-owner search: hits=%+v err=%v", hits, err) + } + }) + +} + +// Query validation belongs to the exported service entry point and must happen +// before any database or worker access, including the model-free route. +func TestLexicalSearchRejectsEmptyQuery(t *testing.T) { + for _, text := range []string{"", " ", "\n\t"} { + _, err := New(nil, nil).Search(context.Background(), Query{ + UserID: uuid.New(), Route: RouteLexical, Text: text, + }) + if err == nil || !strings.Contains(err.Error(), "query is empty") { + t.Fatalf("empty query %q returned %v", text, err) + } + } +} diff --git a/server/internal/search/search.go b/server/internal/search/search.go index c6ff4d0..899a635 100644 --- a/server/internal/search/search.go +++ b/server/internal/search/search.go @@ -149,10 +149,12 @@ var ErrReplayReferenceUnavailable = errors.New("managed embedding replay referen // "text" -> ANN over embeddings_text (Ollama / OpenAI text embedder) // "visual" -> ANN over embeddings_visual via CLIP text encoder // "auto" -> both routes in parallel, merged + deduped by file_id (default) +// "lexical" -> model-free FTS + trigram over files.name (no worker needed) const ( - RouteText = "text" - RouteVisual = "visual" - RouteAuto = "auto" + RouteText = "text" + RouteVisual = "visual" + RouteAuto = "auto" + RouteLexical = "lexical" ) // Hit is one search result. @@ -217,8 +219,10 @@ func (s *Service) Search(ctx context.Context, q Query) ([]Hit, error) { if q.SnippetChars > 16_000 { q.SnippetChars = 16_000 } - if s.worker == nil || !s.worker.Enabled() { - return nil, fmt.Errorf("search disabled: worker not configured") + if q.Route != RouteLexical { + if s.worker == nil || !s.worker.Enabled() { + return nil, fmt.Errorf("search disabled: worker not configured") + } } switch q.Route { @@ -228,8 +232,10 @@ func (s *Service) Search(ctx context.Context, q Query) ([]Hit, error) { return s.searchVisual(ctx, q, text) case "", RouteAuto: return s.searchAuto(ctx, q, text) + case RouteLexical: + return s.searchLexical(ctx, q, text) default: - return nil, fmt.Errorf("unknown route %q (expected text|visual|auto)", q.Route) + return nil, fmt.Errorf("unknown route %q (expected text|visual|auto|lexical)", q.Route) } } @@ -634,23 +640,134 @@ func (s *Service) mergeAutoResults(q Query, tr, vr autoResult) ([]Hit, error) { return out, nil } -// runTextANN issues the text-route SQL and scans results. +// runTextANN returns the k files whose best chunk is nearest the query. +// +// The previous DISTINCT ON (f.id) ORDER BY f.id, distance shape cannot use a +// cosine HNSW index. The shipping path now walks globally ordered chunks +// (HNSW-compatible ORDER BY distance LIMIT n), keeps the first sighting of +// each file (that chunk is the file's best), and excludes selected files on +// the next round. If a bounded approximate scan underfills — one file owning +// many near chunks, or post-filters emptying the HNSW candidate list — the +// original exact DISTINCT ON query fills the remaining slots. func (s *Service) runTextANN(ctx context.Context, q Query, vec []float32) ([]Hit, error) { + const maxTextANNLimit = 100 + if q.Limit <= 0 { + q.Limit = 10 + } + if q.Limit > maxTextANNLimit { + q.Limit = maxTextANNLimit + } args := []any{vectorLiteral(vec), q.UserID} where := []string{"f.user_id = $2"} args, where = appendPathFilters(args, where, q.PathPrefix, q.AllowedPaths) args, where = appendMIMEFilter(args, where, q.Type) - if q.Since != nil { - args = append(args, *q.Since) - where = append(where, fmt.Sprintf("COALESCE(f.timeline_at, f.created_at) >= $%d", len(args))) + args, where = appendTimeFilters(args, where, q.Since, q.Until) + + // Constant caps: CodeQL still treats a sanitized q.Limit as user-controlled. + selected := make([]uuid.UUID, 0, maxTextANNLimit) + seen := make(map[uuid.UUID]struct{}, maxTextANNLimit) + out := make([]Hit, 0, maxTextANNLimit) + for round := 0; round <= q.Limit && len(out) < q.Limit; round++ { + remaining := q.Limit - len(out) + batch, err := s.queryTextDistanceOrder(ctx, q, args, where, selected, remaining) + if err != nil { + return nil, err + } + added := 0 + for _, h := range batch { + if _, ok := seen[h.FileID]; ok { + continue + } + seen[h.FileID] = struct{}{} + selected = append(selected, h.FileID) + out = append(out, h) + added++ + if len(out) >= q.Limit { + break + } + } + if added == 0 { + rest, err := s.queryTextExactRemaining(ctx, q, args, where, selected, remaining) + if err != nil { + return nil, err + } + out = append(out, rest...) + break + } } - if q.Until != nil { - args = append(args, *q.Until) - where = append(where, fmt.Sprintf("COALESCE(f.timeline_at, f.created_at) <= $%d", len(args))) + sortHitsByScoreDesc(out) + if len(out) > q.Limit { + out = out[:q.Limit] } - args = append(args, q.Limit) - limitIdx := len(args) + return out, nil +} + +func cloneArgs(args []any) []any { + out := make([]any, len(args)) + copy(out, args) + return out +} + +func excludeFileIDs(selected []uuid.UUID) []uuid.UUID { + if selected == nil { + return []uuid.UUID{} + } + return selected +} +func (s *Service) queryTextDistanceOrder( + ctx context.Context, + q Query, + args []any, + where []string, + selected []uuid.UUID, + limit int, +) ([]Hit, error) { + queryArgs := cloneArgs(args) + excludeSQL := "TRUE" + if len(selected) > 0 { + queryArgs = append(queryArgs, selected) + excludeSQL = fmt.Sprintf("NOT (e.file_id = ANY($%d::uuid[]))", len(queryArgs)) + } + queryArgs = append(queryArgs, limit) + limitIdx := len(queryArgs) + // ANN first so the planner can use HNSW; file filters apply after. + sql := fmt.Sprintf(` + WITH nearest AS ( + SELECT e.id, e.file_id, e.chunk_index, e.chunk_text, + e.embedding <=> $1::vector AS dist + FROM embeddings_text e + WHERE %s + ORDER BY e.embedding <=> $1::vector ASC + LIMIT $%d + ) + SELECT e.id::text, f.id, f.name, f.path, f.mime, f.sha256, + e.chunk_index, (1 - e.dist) AS score, e.chunk_text, f.summary, + f.timeline_at, f.created_at + FROM nearest e + JOIN files f ON f.id = e.file_id + WHERE %s + ORDER BY e.dist ASC + `, excludeSQL, limitIdx, strings.Join(where, " AND ")) + return s.scanHits(ctx, sql, queryArgs, RouteText, q.SnippetChars) +} + +func (s *Service) queryTextExactRemaining( + ctx context.Context, + q Query, + args []any, + where []string, + selected []uuid.UUID, + limit int, +) ([]Hit, error) { + queryArgs := cloneArgs(args) + excludeSQL := "TRUE" + if len(selected) > 0 { + queryArgs = append(queryArgs, selected) + excludeSQL = fmt.Sprintf("NOT (e.file_id = ANY($%d::uuid[]))", len(queryArgs)) + } + queryArgs = append(queryArgs, limit) + limitIdx := len(queryArgs) sql := fmt.Sprintf(` SELECT evidence_id, file_id, name, path, mime, content_sha256, chunk_index, score, snippet, summary, timeline_at, created_at @@ -670,14 +787,14 @@ func (s *Service) runTextANN(ctx context.Context, q Query, vec []float32) ([]Hit f.created_at AS created_at FROM embeddings_text e JOIN files f ON f.id = e.file_id - WHERE %s - ORDER BY f.id, e.embedding <=> $1::vector ASC + WHERE %s + AND %s + ORDER BY f.id, e.embedding <=> $1::vector ASC ) hits ORDER BY score DESC LIMIT $%d - `, strings.Join(where, " AND "), limitIdx) - - return s.scanHits(ctx, sql, args, RouteText, q.SnippetChars) + `, strings.Join(where, " AND "), excludeSQL, limitIdx) + return s.scanHits(ctx, sql, queryArgs, RouteText, q.SnippetChars) } // runVisualANN issues the visual-route SQL. @@ -715,6 +832,77 @@ func (s *Service) runVisualANN(ctx context.Context, q Query, vec []float32) ([]H return s.scanHits(ctx, sql, args, RouteVisual, q.SnippetChars) } +// searchLexical is the model-free file recall lane. It uses the same +// three-tier shape as memory Recall (name substring → FTS → trigram) so a +// deployment with no worker can still find files by name. +func (s *Service) searchLexical(ctx context.Context, q Query, text string) ([]Hit, error) { + // Search rejects empty queries before dispatch. Keep the numeric trigram + // threshold here: a pg_trgm % prefilter uses a different similarity measure + // and would discard valid word_similarity matches. This query scores the + // filtered file set; index-backed candidate pruning is a separate change. + args := []any{q.UserID} + where := []string{"f.user_id = $1"} + args, where = appendPathFilters(args, where, q.PathPrefix, q.AllowedPaths) + args, where = appendMIMEFilter(args, where, q.Type) + if q.Since != nil { + args = append(args, *q.Since) + where = append(where, fmt.Sprintf("COALESCE(f.timeline_at, f.created_at) >= $%d", len(args))) + } + if q.Until != nil { + args = append(args, *q.Until) + where = append(where, fmt.Sprintf("COALESCE(f.timeline_at, f.created_at) <= $%d", len(args))) + } + args = append(args, text) + textArg := len(args) + args = append(args, q.Limit) + limitArg := len(args) + + sql := fmt.Sprintf(` + WITH candidates AS ( + SELECT f.id AS file_id, f.name, f.path, f.mime, f.sha256, + f.summary, f.timeline_at, f.created_at, + strpos(lower(f.name), lower($%d)) > 0 AS name_contains, + f.search_tsv @@ plainto_tsquery('simple', $%d) AS fts_match, + ts_rank_cd( + f.search_tsv, + plainto_tsquery('simple', $%d) + )::double precision AS fts_rank, + word_similarity( + lower($%d), + lower(f.name) + )::double precision AS trigram_score + FROM files f + WHERE %s + ), + ranked AS ( + SELECT candidates.*, + CASE + WHEN name_contains THEN 1.0::double precision + WHEN fts_match THEN LEAST( + 0.949::double precision, + 0.70::double precision + 0.24::double precision * fts_rank + ) + ELSE LEAST( + 0.699::double precision, + 0.20::double precision + 0.49::double precision * trigram_score + ) + END AS score + FROM candidates + WHERE name_contains + OR fts_match + OR trigram_score >= 0.12 + ) + SELECT 'lexical:' || r.file_id::text, r.file_id, r.name, r.path, r.mime, + r.sha256, -1, r.score::real, r.name, r.summary, + r.timeline_at, r.created_at + FROM ranked r + ORDER BY r.score DESC, r.created_at DESC, r.file_id + LIMIT $%d + `, textArg, textArg, textArg, textArg, strings.Join(where, " AND "), limitArg) + + return s.scanHits(ctx, sql, args, RouteLexical, q.SnippetChars) +} + // scanHits is the common cursor → []Hit loop. Tags every hit with its source route. func (s *Service) scanHits(ctx context.Context, sql string, args []any, route string, snippetChars int) ([]Hit, error) { rows, err := s.pool.Query(ctx, sql, args...) diff --git a/server/internal/tools/builtin/builtin.go b/server/internal/tools/builtin/builtin.go index 9c760e0..552347f 100644 --- a/server/internal/tools/builtin/builtin.go +++ b/server/internal/tools/builtin/builtin.go @@ -475,7 +475,7 @@ func registerSearch(reg *tools.Registry, c *apiclient.Client) error { "query": {Type: "string", Description: "Free-form natural-language query, e.g. \"2012 photos with Xiao Ming\""}, "scope": {Type: "string", Description: "Optional virtual-folder scope, e.g. /Projects/mem"}, "type": {Type: "string", Description: "MIME prefix filter: image|text|application|audio|video"}, - "route": {Type: "string", Description: "Search route: text|visual|auto (default auto fuses both)", Enum: []string{"text", "visual", "auto"}}, + "route": {Type: "string", Description: "Search route: text|visual|auto|lexical (default auto fuses text and visual; lexical needs no model)", Enum: []string{"text", "visual", "auto", "lexical"}}, "since": {Type: "string", Description: "YYYY-MM-DD lower bound on timeline_at"}, "until": {Type: "string", Description: "YYYY-MM-DD upper bound on timeline_at"}, "limit": {Type: "integer", Description: "Max results (default 10, max 100)", Default: 10}, diff --git a/web/audit-retry.mjs b/web/audit-retry.mjs new file mode 100644 index 0000000..923acfc --- /dev/null +++ b/web/audit-retry.mjs @@ -0,0 +1,138 @@ +import { spawnSync } from "node:child_process"; +import { setTimeout as sleep } from "node:timers/promises"; +import { pathToFileURL } from "node:url"; + +const MAX_ATTEMPTS = 3; +const BACKOFF_MS = 10_000; +// At most six 60-second attempts and four 10-second backoffs across both thresholds. +const ATTEMPT_TIMEOUT_MS = 60_000; + +const NETWORK_PATTERNS = [ + "network timeout", + "503 Service Unavailable", + "ECONNRESET", + "ETIMEDOUT", +]; + +const VULNERABILITY_PATTERNS = [ + "found \\d+ vulnerabilit(?:y|ies)", + "npm audit report", + "vulnerabilities found", +]; + +const COMMANDS = [ + { + label: "production dependencies (moderate threshold)", + args: ["audit", "--omit=dev", "--audit-level=moderate", "--fetch-timeout=45000"], + }, + { + label: "all dependencies (high threshold)", + args: ["audit", "--audit-level=high", "--fetch-timeout=45000"], + }, +]; + +function isNetworkError(stderr) { + return NETWORK_PATTERNS.some((p) => stderr.toLowerCase().includes(p.toLowerCase())); +} + +function isVulnerabilityReport(stdout) { + return VULNERABILITY_PATTERNS.some((p) => new RegExp(p, "i").test(stdout)); +} + +async function runWithRetry(label, args, { spawn, wait, npmExecPath, stdout, stderr }) { + let result; + const finish = () => { + stdout.write(result.stdout ?? ""); + stderr.write(result.stderr ?? ""); + if (result.error) { + stderr.write(`[audit-retry] ${result.error.code ?? "spawn error"}: ${result.error.message}\n`); + } + if (result.signal) { + stderr.write(`[audit-retry] terminated by ${result.signal}.\n`); + } + return Number.isInteger(result.status) && result.status > 0 && result.status <= 255 ? result.status : 1; + }; + + for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + const ts = new Date().toISOString(); + stderr.write( + `[audit-retry] ${ts} — ${label} (attempt ${attempt}/${MAX_ATTEMPTS})\n` + ); + + // npm run supplies the CLI path, including on Windows where npm is a .cmd + // shim that cannot be launched directly with shell-free spawnSync. + result = spawn(npmExecPath ? process.execPath : "npm", npmExecPath ? [npmExecPath, ...args] : args, { + encoding: "utf8", + stdio: ["inherit", "pipe", "pipe"], + timeout: ATTEMPT_TIMEOUT_MS, + killSignal: "SIGKILL", + }); + + // An interrupted or unstarted audit is never a valid audit result, even + // when its partial output happens to mention a transient network error. + if (result.error || result.signal || !Number.isInteger(result.status) || result.status < 0 || result.status > 255) { + stderr.write(`[audit-retry] ${label} did not complete — not retrying.\n`); + return finish(); + } + + if (result.status === 0) { + stdout.write(result.stdout ?? ""); + stderr.write(result.stderr ?? ""); + stderr.write(`[audit-retry] ${label} passed.\n`); + return 0; + } + + const output = `${result.stdout ?? ""}\n${result.stderr ?? ""}`; + + if (isVulnerabilityReport(output)) { + stderr.write( + `[audit-retry] ${label} found real vulnerabilities — not retrying.\n` + ); + return finish(); + } + + if (isNetworkError(output)) { + if (attempt < MAX_ATTEMPTS) { + stderr.write( + `[audit-retry] ${label} hit a network error — will retry.\n` + + // An attempt that self-heals would otherwise leave no trace, and + // stdout stays reserved for the final audit report. + `[audit-retry] ${label} attempt ${attempt}/${MAX_ATTEMPTS} output:\n${output.trimEnd()}\n` + ); + await wait(BACKOFF_MS); + continue; + } + stderr.write(`[audit-retry] ${label} exhausted ${MAX_ATTEMPTS} attempts.\n`); + return finish(); + } + + stderr.write( + `[audit-retry] ${label} failed with unrecognized error — not retrying.\n` + ); + return finish(); + } +} + +export async function runAudits({ + spawn = spawnSync, + wait = sleep, + npmExecPath = process.env.npm_execpath, + platform = process.platform, + stdout = process.stdout, + stderr = process.stderr, +} = {}) { + if (platform === "win32" && !npmExecPath) { + stderr.write("[audit-retry] Run npm run audit so npm supplies its CLI path on Windows.\n"); + return 1; + } + + for (const cmd of COMMANDS) { + const code = await runWithRetry(cmd.label, cmd.args, { spawn, wait, npmExecPath, stdout, stderr }); + if (code !== 0) return code; + } + return 0; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + process.exitCode = await runAudits(); +} diff --git a/web/audit-retry.test.mjs b/web/audit-retry.test.mjs new file mode 100644 index 0000000..1a4c560 --- /dev/null +++ b/web/audit-retry.test.mjs @@ -0,0 +1,309 @@ +// @vitest-environment node +import { spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { runAudits } from "./audit-retry.mjs"; + +const PASS = { status: 0, stdout: "", stderr: "" }; +const PROD_ARGS = ["audit", "--omit=dev", "--audit-level=moderate", "--fetch-timeout=45000"]; +const ALL_ARGS = ["audit", "--audit-level=high", "--fetch-timeout=45000"]; + +function harness(results, overrides = {}) { + let stdout = ""; + let stderr = ""; + const spawn = vi.fn(() => { + const result = results.shift(); + if (!result) throw new Error("Unexpected audit attempt"); + return result; + }); + const wait = vi.fn(async () => {}); + return { + spawn, + wait, + output: () => ({ stdout, stderr }), + run: () => runAudits({ + spawn, + wait, + npmExecPath: "/npm with spaces/npm-cli.js", + stdout: { write: (text) => { stdout += text; } }, + stderr: { write: (text) => { stderr += text; } }, + ...overrides, + }), + }; +} + +describe("audit retry policy", () => { + it("preserves successful audit reports, including findings below the threshold", async () => { + const production = { status: 0, stdout: "found 0 vulnerabilities\n", stderr: "production notice\n" }; + const development = { status: 0, stdout: "# npm audit report\n1 moderate severity vulnerability\n", stderr: "development notice\n" }; + const test = harness([production, development]); + expect(await test.run()).toBe(0); + expect(test.output().stdout).toBe(production.stdout + development.stdout); + expect(test.output().stderr).toContain(production.stderr); + expect(test.output().stderr).toContain(development.stderr); + expect(test.spawn).toHaveBeenCalledTimes(2); + expect(test.wait).not.toHaveBeenCalled(); + }); + + it("requires both unchanged thresholds to pass, using Node and the npm CLI path", async () => { + const test = harness([PASS, PASS], { platform: "win32" }); + expect(await test.run()).toBe(0); + expect(test.spawn.mock.calls.map(([command, args]) => [command, args])).toEqual([ + [process.execPath, ["/npm with spaces/npm-cli.js", ...PROD_ARGS]], + [process.execPath, ["/npm with spaces/npm-cli.js", ...ALL_ARGS]], + ]); + for (const [, args, options] of test.spawn.mock.calls) { + expect(options).toMatchObject({ timeout: 60_000, killSignal: "SIGKILL" }); + expect(options.shell).toBeUndefined(); + const fetchTimeout = args.filter(a => a.startsWith("--fetch-timeout=")).map(a => Number(a.split("=")[1])); + for (const ms of fetchTimeout) expect(ms).toBeLessThan(options.timeout); + } + expect(test.wait).not.toHaveBeenCalled(); + }); + + it("keeps direct Node invocation available on Unix", async () => { + const test = harness([PASS, PASS], { npmExecPath: "", platform: "linux" }); + expect(await test.run()).toBe(0); + expect(test.spawn.mock.calls[0].slice(0, 2)).toEqual(["npm", PROD_ARGS]); + }); + + it("fails with an actionable message when direct invocation lacks npm on Windows", async () => { + const test = harness([], { npmExecPath: "", platform: "win32" }); + expect(await test.run()).toBe(1); + expect(test.output().stderr).toContain("Run npm run audit"); + expect(test.spawn).not.toHaveBeenCalled(); + }); + + it.each(["network timeout", "503 Service Unavailable", "econnreset", "ETIMEDOUT"])( + "retries a recognized transient failure: %s", async (message) => { + const test = harness([{ status: 1, stderr: message }, PASS, PASS]); + expect(await test.run()).toBe(0); + expect(test.spawn).toHaveBeenCalledTimes(3); + expect(test.wait.mock.calls).toEqual([[10_000]]); + }, + ); + + it("recognizes transient failures on stdout", async () => { + const test = harness([{ status: 1, stdout: "ECONNRESET" }, PASS, PASS]); + expect(await test.run()).toBe(0); + expect(test.wait).toHaveBeenCalledTimes(1); + }); + + it("echoes the transcript of every retried attempt, on stderr only", async () => { + const transient = { status: 1, stdout: "registry notice\n", stderr: "network timeout at .../security/advisories/bulk\n" }; + const test = harness([transient, PASS, PASS]); + expect(await test.run()).toBe(0); + expect(test.output().stderr).toContain("attempt 1/3 output:"); + expect(test.output().stderr).toContain(transient.stdout); + expect(test.output().stderr).toContain(transient.stderr); + expect(test.output().stdout).not.toContain("registry notice"); + }); + + it("caps retries at three, skips the final backoff, and retains final diagnostics", async () => { + const failure = { status: 7, stdout: "final stdout\n", stderr: "503 Service Unavailable: final detail\n" }; + const test = harness([failure, failure, failure]); + expect(await test.run()).toBe(7); + expect(test.spawn).toHaveBeenCalledTimes(3); + expect(test.wait.mock.calls).toEqual([[10_000], [10_000]]); + expect(test.output().stdout).toBe(failure.stdout); + expect(test.output().stderr).toContain(failure.stderr); + expect(test.output().stderr).toContain("exhausted 3 attempts"); + expect(test.output().stderr.match(/will retry/g)).toHaveLength(2); + }); + + it("gives the second threshold its own bounded retry budget", async () => { + const failure = { status: 1, stderr: "ETIMEDOUT" }; + const test = harness([failure, failure, PASS, failure, failure, PASS]); + expect(await test.run()).toBe(0); + expect(test.spawn).toHaveBeenCalledTimes(6); + expect(test.wait.mock.calls).toEqual(Array(4).fill([10_000])); + expect(test.spawn.mock.calls[3][1].slice(1)).toEqual(ALL_ARGS); + }); + + it.each(["stdout", "stderr"])("never retries vulnerabilities on %s, even with network text", async (stream) => { + const test = harness([{ status: 1, [stream]: "# npm audit report\nETIMEDOUT\n" }]); + expect(await test.run()).toBe(1); + expect(test.spawn).toHaveBeenCalledTimes(1); + expect(test.wait).not.toHaveBeenCalled(); + expect(test.output()[stream]).toContain("# npm audit report"); + }); + + it.each(["found 1 vulnerability", "found 2 vulnerabilities", "vulnerabilities found"])( + "prioritizes vulnerability summaries over network text: %s", async (message) => { + const test = harness([{ status: 1, stdout: message, stderr: "ECONNRESET" }]); + expect(await test.run()).toBe(1); + expect(test.spawn).toHaveBeenCalledTimes(1); + expect(test.wait).not.toHaveBeenCalled(); + }, + ); + + it.each(["E401 unauthorized", "invalid config", "fetch failed", "audit endpoint returned an error"])( + "fails unknown or non-transient errors without retry: %s", async (message) => { + const test = harness([{ status: 2, stdout: "diagnostic\n", stderr: message }]); + expect(await test.run()).toBe(2); + expect(test.wait).not.toHaveBeenCalled(); + expect(test.output().stdout).toBe("diagnostic\n"); + expect(test.output().stderr).toContain(message); + }, + ); + + it.each([ + ["missing executable", { status: null, error: Object.assign(new Error("npm missing"), { code: "ENOENT" }) }, "ENOENT"], + ["timeout", { status: null, error: Object.assign(new Error("timed out"), { code: "ETIMEDOUT" }) }, "timed out"], + ["signal", { status: null, signal: "SIGTERM" }, "SIGTERM"], + ["null status", { status: null }, "did not complete"], + ["missing status", {}, "did not complete"], + ["negative status", { status: -1 }, "did not complete"], + ["out of range status", { status: 256 }, "did not complete"], + ["buffer overflow", { status: null, error: Object.assign(new Error("output limit"), { code: "ENOBUFS" }) }, "ENOBUFS"], + ["error with zero status", { status: 0, error: new Error("incomplete") }, "incomplete"], + ["signal with zero status", { status: 0, signal: "SIGKILL" }, "SIGKILL"], + ])("fails closed for %s regardless of transient-looking output", async (_label, result, diagnostic) => { + const test = harness([{ ...result, stderr: "503 Service Unavailable" }]); + expect(await test.run()).toBe(1); + expect(test.spawn).toHaveBeenCalledTimes(1); + expect(test.wait).not.toHaveBeenCalled(); + expect(test.output().stderr).toContain(diagnostic); + expect(test.output().stderr).toContain("503 Service Unavailable"); + }); + + it("fails overall if the second threshold finds vulnerabilities", async () => { + const test = harness([PASS, { status: 1, stdout: "found 2 vulnerabilities" }]); + expect(await test.run()).toBe(1); + expect(test.spawn).toHaveBeenCalledTimes(2); + expect(test.wait).not.toHaveBeenCalled(); + }); + + it("vite.config.ts includes audit-retry.test.mjs in test collection", async () => { + const configPath = join(__dirname, "vite.config.ts"); + const config = readFileSync(configPath, "utf8"); + expect(config).toContain("audit-retry.test.mjs"); + }); + + it("ci.yml keeps the audit transcript pipe fail-closed and uploads it", async () => { + const ci = readFileSync(join(__dirname, "../.github/workflows/ci.yml"), "utf8"); + const steps = ci.split(/\n(?= - name: )/); + + // Without `shell: bash` the default `bash -e {0}` has no pipefail, so tee's + // exit status would replace the audit's and a failing gate would pass. + const transcriptSteps = steps.filter((step) => step.includes('tee "${RUNNER_TEMP}/')); + expect(transcriptSteps.map((step) => step.match(/- name: (.+)/)[1])).toEqual([ + "Audit dependencies", + "Run Windows audit evidence helper against the real registry", + ]); + for (const step of transcriptSteps) { + expect(step).toMatch(/^[ ]+shell: bash$/m); + } + + // Evidence is only worth collecting if it also survives a red step. + for (const artifact of ["web-audit-transcript", "win-audit-transcript"]) { + const upload = steps.find((step) => step.includes(`name: ${artifact}-\${{ github.sha }}`)); + expect(upload).toBeDefined(); + expect(upload).toContain("if: always() && !cancelled()"); + expect(upload).toContain(`path: \${{ runner.temp }}/${artifact}.txt`); + expect(upload).toContain("if-no-files-found: error"); + expect(upload).toContain("retention-days: 14"); + } + + expect(ci).not.toMatch(/continue-on-error|\|\| true/); + }); + + it("ci.yml executes win-audit-verify.bat on a Windows runner", async () => { + const ci = readFileSync(join(__dirname, "../.github/workflows/ci.yml"), "utf8"); + // The helper's own test only replays its source text against a stub + // npm.cmd; a real invocation has to exist somewhere or the batch file is + // dead code that the PR description advertises as evidence. + expect(ci).toMatch(/cmd\.exe \/\/d \/\/c "\.\.\\scripts\\win-audit-verify\.bat"/); + // Matching the invocation text alone was not enough: the first version of + // this step passed CI on a cmd.exe that never ran the helper. + expect(ci).toMatch(/\[win-audit-verify\\\] finished at/); + expect(ci).toMatch(/npm run audit exit code: 0/); + const job = ci + .split(/\n(?= [a-z][a-z0-9-]*:\n)/) + .find((entry) => entry.includes("win-audit-verify.bat")) + // Judge configuration, not prose: the job's comments explain this very + // pattern, so quoting them would make the assertions self-defeating. + .split("\n") + .filter((line) => !/^\s*#/.test(line)) + .join("\n"); + expect(job).toMatch(/^ runs-on: windows-/m); + // The check must not be reachable only via a matrix entry that can vanish. + expect(job).not.toMatch(/if: runner\.os == 'Windows'/); + }); +}); + +describe("audit CLI process behavior", () => { + const directories = []; + afterEach(() => { + for (const directory of directories.splice(0)) rmSync(directory, { recursive: true, force: true }); + }); + + function fixture(results) { + const directory = mkdtempSync(join(tmpdir(), "audit npm fixture ")); + directories.push(directory); + const cli = join(directory, "npm cli.cjs"); + const log = join(directory, "calls.json"); + writeFileSync(cli, ` + const fs = require('node:fs'); + const log = ${JSON.stringify(log)}; + const calls = fs.existsSync(log) ? JSON.parse(fs.readFileSync(log, 'utf8')) : []; + const result = ${JSON.stringify(results)}[calls.length]; + calls.push(process.argv.slice(2)); + fs.writeFileSync(log, JSON.stringify(calls)); + process.stdout.write(result.stdout || ''); + process.stderr.write(result.stderr || ''); + process.exitCode = result.status; + `); + return { cli, log }; + } + + function invoke(cli) { + return spawnSync(process.execPath, [fileURLToPath(new URL("./audit-retry.mjs", import.meta.url))], { + encoding: "utf8", + timeout: 5_000, + killSignal: "SIGKILL", + env: { ...process.env, npm_execpath: cli }, + }); + } + + it("executes a CLI path with spaces and exits zero only after both audits", () => { + const { cli, log } = fixture([PASS, PASS]); + const result = invoke(cli); + expect(result.error).toBeUndefined(); + expect(result.status).toBe(0); + expect(JSON.parse(readFileSync(log, "utf8"))).toEqual([PROD_ARGS, ALL_ARGS]); + }); + + it.each(["# npm audit report\n", "unknown audit error\n"])("returns failure and output for %s", (message) => { + const { cli, log } = fixture([{ status: 2, stdout: message, stderr: "detail\n" }]); + const result = invoke(cli); + expect(result.error).toBeUndefined(); + expect(result.status).toBe(2); + expect(result.stdout).toBe(message); + expect(result.stderr).toContain("detail\n"); + expect(JSON.parse(readFileSync(log, "utf8"))).toEqual([PROD_ARGS]); + }); + + it("exits nonzero when the npm CLI cannot be started", () => { + const { cli } = fixture([]); + rmSync(cli); + const result = invoke(cli); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("MODULE_NOT_FOUND"); + }); + + it("terminates a stalled process and fails closed at the attempt timeout", async () => { + const test = harness([], { + spawn: (_command, _args, options) => spawnSync(process.execPath, [ + "-e", "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);", + ], { ...options, timeout: 200 }), + }); + expect(await test.run()).toBe(1); + expect(test.wait).not.toHaveBeenCalled(); + expect(test.output().stderr).toContain("ETIMEDOUT"); + expect(test.output().stderr).toContain("SIGKILL"); + }); +}); diff --git a/web/design-system/LICENSE b/web/design-system/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/web/design-system/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/web/design-system/README.md b/web/design-system/README.md new file mode 100644 index 0000000..1c448bf --- /dev/null +++ b/web/design-system/README.md @@ -0,0 +1,37 @@ +# Shared design token bridge + +`tokens.json` is an unmodified snapshot of +[`bytefolk/design-system` at 910456901dda74da4d5b0320cd03d36ad18650b0](https://github.com/bytefolk/design-system/blob/910456901dda74da4d5b0320cd03d36ad18650b0/tokens/design-tokens.json). +The upstream Apache-2.0 license is included as `LICENSE`. + +The shared JSON owns the palette. Do not edit the snapshot or generated CSS by +hand. To update it, copy the JSON from a reviewed upstream commit, update the +revision and expected SHA-256 in `../scripts/design-tokens.mjs`, regenerate, and review both themes. +The CSS header records the commit and content SHA-256. + +```sh +cd web +npm run tokens:generate +npm run tokens:check +``` + +`tokens:check` runs before every Web build. The zero-dependency generator maps +the default profile to mem's existing RGB variable API, preserves alpha, and +uses shared font/shadow roles. Human actions are blue; `ai` is reserved for AI +affordances. The user's saved light/dark preference is retained. + +Upstream base hues include decorative contrast exceptions. For actual small +text, the adapter raises source alpha or adjusts an opaque source hue toward +the theme's foreground until it reaches 4.5:1 on all five source surfaces. +Semantic foregrounds also meet this ratio on up to 30% tinted state backgrounds. Solid normal/hover backgrounds +are derived from upstream primary/hover hues and paired with the upstream +primary foreground. Generation checks 334 text/background pairs; actual +browser acceptance additionally checks compositing and control states. + +mem currently uses React 19 and local primitives; the published shared facade +expects React 18. This bridge deliberately adds no runtime dependency or React +migration. The local `EmptyState` follows the shared `ui-empty-state` structure +and spacing; it can be replaced by a compatible shared release later. Page, +card, dialog and form content starts at the reading edge; numeric comparison +columns and trailing actions align to the end; buttons, badges, tabs and whole +empty panels center their content. Business routes and actions stay in mem. diff --git a/web/design-system/tokens.json b/web/design-system/tokens.json new file mode 100644 index 0000000..cccf5a2 --- /dev/null +++ b/web/design-system/tokens.json @@ -0,0 +1,483 @@ +{ + "$schema": "https://design-tokens.github.io/community-group/format/", + "name": "@fullstack-ai-infra/ui", + "description": "Ant Design aligned semantic design tokens v3 (light+dark). Values extracted from antd@5 defaultAlgorithm/darkAlgorithm on 2026-08-24 by Design Lead; AA deviations documented in docs/org-workbench-design-language.md. Single token source per ADR 0002: src/styles/tokens.css is generated via scripts/generate-tokens-css.mjs.", + "tokens": { + "color": { + "canvas": { + "$type": "color", + "$value": { + "light": "#f5f5f5", + "dark": "#000000" + } + }, + "canvas-subtle": { + "$type": "color", + "$value": { + "light": "#fafafa", + "dark": "#141414" + } + }, + "navigation": { + "$type": "color", + "$value": { + "light": "#f0f0f0", + "dark": "#1f1f1f" + } + }, + "navigation-hover": { + "$type": "color", + "$value": { + "light": "#e6f4ff", + "dark": "#15325b" + } + }, + "surface": { + "$type": "color", + "$value": { + "light": "#ffffff", + "dark": "#141414" + } + }, + "surface-raised": { + "$type": "color", + "$value": { + "light": "#ffffff", + "dark": "#1f1f1f" + } + }, + "surface-inset": { + "$type": "color", + "$value": { + "light": "#fafafa", + "dark": "#1d1d1d" + } + }, + "foreground": { + "$type": "color", + "$value": { + "light": "rgba(0, 0, 0, 0.88)", + "dark": "rgba(255, 255, 255, 0.85)" + } + }, + "foreground-muted": { + "$type": "color", + "$value": { + "light": "rgba(0, 0, 0, 0.65)", + "dark": "rgba(255, 255, 255, 0.65)" + } + }, + "foreground-subtle": { + "$type": "color", + "$value": { + "light": "rgba(0, 0, 0, 0.45)", + "dark": "rgba(255, 255, 255, 0.45)" + } + }, + "border": { + "$type": "color", + "$value": { + "light": "#d9d9d9", + "dark": "#424242" + } + }, + "border-strong": { + "$type": "color", + "$value": { + "light": "#bfbfbf", + "dark": "#595959" + } + }, + "primary": { + "$type": "color", + "$value": { + "light": "#1677ff", + "dark": "#1668dc" + } + }, + "primary-hover": { + "$type": "color", + "$value": { + "light": "#4096ff", + "dark": "#3c89e8" + } + }, + "primary-foreground": { + "$type": "color", + "$value": "#ffffff" + }, + "primary-soft": { + "$type": "color", + "$value": { + "light": "#e6f4ff", + "dark": "#15325b" + } + }, + "ai": { + "$type": "color", + "$value": { + "light": "#722ed1", + "dark": "#642ab5" + } + }, + "ai-strong": { + "$type": "color", + "$value": { + "light": "#531dab", + "dark": "#854eca" + } + }, + "ai-hover": { + "$type": "color", + "$value": { + "light": "#9254de", + "dark": "#854eca" + } + }, + "ai-foreground": { + "$type": "color", + "$value": "#ffffff" + }, + "ai-soft": { + "$type": "color", + "$value": { + "light": "#f9f0ff", + "dark": "#301c4d" + } + }, + "info": { + "$type": "color", + "$value": { + "light": "#1677ff", + "dark": "#1668dc" + } + }, + "info-soft": { + "$type": "color", + "$value": { + "light": "#e6f4ff", + "dark": "#111a2c" + } + }, + "success": { + "$type": "color", + "$value": { + "light": "#52c41a", + "dark": "#49aa19" + } + }, + "success-strong": { + "$type": "color", + "$value": { + "light": "#237804", + "dark": "#95de64" + } + }, + "success-soft": { + "$type": "color", + "$value": { + "light": "#f6ffed", + "dark": "#162312" + } + }, + "warning": { + "$type": "color", + "$value": { + "light": "#faad14", + "dark": "#d89614" + } + }, + "warning-strong": { + "$type": "color", + "$value": { + "light": "#ad4e00", + "dark": "#f8ce5b" + } + }, + "warning-soft": { + "$type": "color", + "$value": { + "light": "#fffbe6", + "dark": "#2b2111" + } + }, + "danger": { + "$type": "color", + "$value": { + "light": "#ff4d4f", + "dark": "#dc4446" + } + }, + "danger-strong": { + "$type": "color", + "$value": { + "light": "#cf1322", + "dark": "#e84749" + } + }, + "danger-soft": { + "$type": "color", + "$value": { + "light": "#fff2f0", + "dark": "#2c1618" + } + }, + "overlay": { + "$type": "color", + "$value": "rgba(0, 0, 0, 0.45)" + }, + "focus": { + "$type": "color", + "$value": { + "light": "#1677ff", + "dark": "#1668dc" + } + }, + "selection": { + "$type": "color", + "$value": { + "light": "#bae0ff", + "dark": "#15417e" + } + } + }, + "fontFamily": { + "font-sans": { + "$type": "fontFamily", + "$value": "-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, 'Noto Sans', sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol', 'Noto Color Emoji'" + }, + "font-mono": { + "$type": "fontFamily", + "$value": "'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, Courier, monospace" + } + }, + "fontSize": { + "text-xs": { + "$type": "fontSize", + "$value": "12px" + }, + "text-sm": { + "$type": "fontSize", + "$value": "14px" + }, + "text-base": { + "$type": "fontSize", + "$value": "16px" + }, + "text-lg": { + "$type": "fontSize", + "$value": "20px" + }, + "text-xl": { + "$type": "fontSize", + "$value": "24px" + }, + "text-2xl": { + "$type": "fontSize", + "$value": "30px" + } + }, + "lineHeight": { + "leading-tight": { + "$type": "lineHeight", + "$value": "1.3333" + }, + "leading-normal": { + "$type": "lineHeight", + "$value": "1.5714" + } + }, + "dimension": { + "space-1": { + "$type": "dimension", + "$value": "0.25rem" + }, + "space-2": { + "$type": "dimension", + "$value": "0.5rem" + }, + "space-3": { + "$type": "dimension", + "$value": "0.75rem" + }, + "space-4": { + "$type": "dimension", + "$value": "1rem" + }, + "space-5": { + "$type": "dimension", + "$value": "1.25rem" + }, + "space-6": { + "$type": "dimension", + "$value": "1.5rem" + }, + "space-8": { + "$type": "dimension", + "$value": "2rem" + }, + "space-10": { + "$type": "dimension", + "$value": "2.5rem" + }, + "space-12": { + "$type": "dimension", + "$value": "3rem" + }, + "rail-width": { + "$type": "dimension", + "$value": "4.5rem" + }, + "sidebar-width": { + "$type": "dimension", + "$value": "16rem" + }, + "sidebar-wide": { + "$type": "dimension", + "$value": "18rem" + }, + "topbar-height": { + "$type": "dimension", + "$value": "3.75rem" + } + }, + "borderRadius": { + "radius-sm": { + "$type": "borderRadius", + "$value": "4px" + }, + "radius-md": { + "$type": "borderRadius", + "$value": "6px" + }, + "radius-lg": { + "$type": "borderRadius", + "$value": "8px" + }, + "radius-xl": { + "$type": "borderRadius", + "$value": "8px" + }, + "radius-full": { + "$type": "borderRadius", + "$value": "9999px" + } + }, + "boxShadow": { + "shadow-sm": { + "$type": "boxShadow", + "$value": "0 1px 2px 0 rgba(0, 0, 0, 0.03), 0 1px 6px -1px rgba(0, 0, 0, 0.02), 0 2px 4px 0 rgba(0, 0, 0, 0.02)" + }, + "shadow-md": { + "$type": "boxShadow", + "$value": "0 6px 16px 0 rgba(0, 0, 0, 0.08), 0 3px 6px -4px rgba(0, 0, 0, 0.12), 0 9px 28px 8px rgba(0, 0, 0, 0.05)" + }, + "shadow-lg": { + "$type": "boxShadow", + "$value": "0 6px 16px 0 rgba(0, 0, 0, 0.08), 0 3px 6px -4px rgba(0, 0, 0, 0.12), 0 9px 28px 8px rgba(0, 0, 0, 0.05)" + } + }, + "duration": { + "duration-fast": { + "$type": "duration", + "$value": "0.1s" + }, + "duration-normal": { + "$type": "duration", + "$value": "0.2s" + } + }, + "timingFunction": { + "ease": { + "$type": "timingFunction", + "$value": "cubic-bezier(0.645, 0.045, 0.355, 1)" + } + } + }, + "profiles": { + "mint": { + "$description": "Compact high-contrast mint profile for dark-first workbenches. Product applications may label this profile independently.", + "tokens": { + "color": { + "canvas": { "$value": { "light": "#f6f8f7", "dark": "#0a0c10" } }, + "canvas-subtle": { "$value": { "light": "#eff3f1", "dark": "#101318" } }, + "navigation": { "$value": { "light": "#eff3f1", "dark": "#101318" } }, + "navigation-hover": { "$value": { "light": "#e5ebe8", "dark": "#181d22" } }, + "surface": { "$value": { "light": "#ffffff", "dark": "#14171d" } }, + "surface-raised": { "$value": { "light": "#ffffff", "dark": "#1a1e25" } }, + "surface-inset": { "$value": { "light": "#f1f4f2", "dark": "#101318" } }, + "foreground": { "$value": { "light": "#191e24", "dark": "#e9edf2" } }, + "foreground-muted": { "$value": { "light": "#57606b", "dark": "#a3abb8" } }, + "foreground-subtle": { "$value": { "light": "#636d78", "dark": "#7d8694" } }, + "border": { "$value": { "light": "#dbe2df", "dark": "#232830" } }, + "border-strong": { "$value": { "light": "#bfcac4", "dark": "#343b45" } }, + "primary": { "$value": { "light": "#0a6b4e", "dark": "#19d89b" } }, + "primary-hover": { "$value": { "light": "#085a42", "dark": "#2ee9a8" } }, + "primary-foreground": { "$value": { "light": "#ffffff", "dark": "#04150f" } }, + "primary-soft": { "$value": { "light": "#e2f7ef", "dark": "#0d2a22" } }, + "ai": { "$value": { "light": "#0a6b4e", "dark": "#19d89b" } }, + "ai-strong": { "$value": { "light": "#07563f", "dark": "#2ee9a8" } }, + "ai-hover": { "$value": { "light": "#085a42", "dark": "#2ee9a8" } }, + "ai-foreground": { "$value": { "light": "#ffffff", "dark": "#04150f" } }, + "ai-soft": { "$value": { "light": "#e2f7ef", "dark": "#0d2a22" } }, + "info": { "$value": { "light": "#1b7fb5", "dark": "#5cc8e8" } }, + "info-soft": { "$value": { "light": "#e6f4fb", "dark": "#12313b" } }, + "success": { "$value": { "light": "#266b4a", "dark": "#78c89c" } }, + "success-strong": { "$value": { "light": "#1e5739", "dark": "#a3ddb8" } }, + "success-soft": { "$value": { "light": "#eaf5ef", "dark": "#173526" } }, + "warning": { "$value": { "light": "#7d4f0f", "dark": "#f0b563" } }, + "warning-strong": { "$value": { "light": "#6b420b", "dark": "#f6cf95" } }, + "warning-soft": { "$value": { "light": "#fff4df", "dark": "#352816" } }, + "danger": { "$value": { "light": "#a83232", "dark": "#f08d8d" } }, + "danger-strong": { "$value": { "light": "#8f2525", "dark": "#ffb1b1" } }, + "danger-soft": { "$value": { "light": "#fff0f0", "dark": "#3b2226" } }, + "overlay": { + "$value": { "light": "rgba(20, 30, 26, 0.35)", "dark": "rgba(0, 0, 0, 0.65)" } + }, + "focus": { "$value": { "light": "#0a6b4e", "dark": "#19d89b" } }, + "selection": { "$value": { "light": "#e2f7ef", "dark": "#0d2a22" } } + }, + "fontFamily": { + "font-sans": { + "$value": "-apple-system, BlinkMacSystemFont, 'Segoe UI', 'PingFang SC', 'Hiragino Sans GB', 'Microsoft YaHei', sans-serif" + }, + "font-mono": { + "$value": "'JetBrains Mono', ui-monospace, 'SF Mono', Menlo, Consolas, monospace" + } + }, + "fontSize": { + "text-xs": { "$value": "11px" }, + "text-sm": { "$value": "12px" }, + "text-base": { "$value": "13px" }, + "text-lg": { "$value": "17px" }, + "text-xl": { "$value": "22px" }, + "text-2xl": { "$value": "28px" } + }, + "lineHeight": { + "leading-tight": { "$value": "1.3" }, + "leading-normal": { "$value": "1.6" } + }, + "borderRadius": { + "radius-sm": { "$value": "6px" }, + "radius-md": { "$value": "8px" }, + "radius-lg": { "$value": "13px" }, + "radius-xl": { "$value": "16px" } + }, + "boxShadow": { + "shadow-sm": { "$value": "0 1px 2px 0 rgba(20, 21, 27, 0.03)" }, + "shadow-md": { + "$value": "0 4px 16px 0 rgba(20, 21, 27, 0.07), 0 1px 4px 0 rgba(20, 21, 27, 0.03)" + }, + "shadow-lg": { "$value": "0 16px 48px 0 rgba(20, 21, 27, 0.14)" } + }, + "duration": { + "duration-fast": { "$value": "0.12s" }, + "duration-normal": { "$value": "0.16s" } + }, + "timingFunction": { + "ease": { "$value": "cubic-bezier(0.22, 0.61, 0.36, 1)" } + } + } + } + } +} diff --git a/web/index.html b/web/index.html index e0268bc..a83cc1f 100644 --- a/web/index.html +++ b/web/index.html @@ -3,9 +3,9 @@ - + - + mem diff --git a/web/localization-acceptance.mjs b/web/localization-acceptance.mjs index b322272..96687c1 100644 --- a/web/localization-acceptance.mjs +++ b/web/localization-acceptance.mjs @@ -173,7 +173,7 @@ try { console.log('✓ unknown managed-embedding errors use the selected locale'); await page.goto(`${baseURL}/search`, { waitUntil: 'domcontentloaded' }); - await page.getByRole('heading', { name: '搜索' }).waitFor(); + await page.getByRole('heading', { name: '搜索', level: 1, exact: true }).waitFor(); await page.getByRole('button', { name: '草地上的金毛' }).waitFor(); assert.equal(await page.evaluate(() => window.__documentLangAtInteractive), 'zh-CN'); diff --git a/web/nginx/default.conf.template b/web/nginx/default.conf.template index 48b4382..72de1db 100644 --- a/web/nginx/default.conf.template +++ b/web/nginx/default.conf.template @@ -7,9 +7,16 @@ server { index index.html; client_max_body_size ${MEM_MAX_BODY_SIZE}; + # nginx is the single authority for these three headers on every response + # that leaves this container, so the values below are the values a client + # sees. The API sets the same three itself for deployments that run memd + # without a proxy in front; /v1/ hides those copies so they cannot arrive + # alongside these ones. Content-Security-Policy, X-XSS-Protection and + # Content-Disposition stay the API's, because they depend on what the + # response actually is and nginx cannot know that. add_header X-Content-Type-Options "nosniff" always; - add_header Referrer-Policy "same-origin" always; add_header X-Frame-Options "DENY" always; + add_header Referrer-Policy "no-referrer" always; location = /healthz { access_log off; @@ -18,6 +25,9 @@ server { } location /v1/ { + proxy_hide_header X-Content-Type-Options; + proxy_hide_header X-Frame-Options; + proxy_hide_header Referrer-Policy; proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; @@ -32,6 +42,12 @@ server { location /assets/ { try_files $uri =404; expires 1y; + # An add_header in this block replaces the inherited set rather than + # adding to it, so the three headers above have to be restated here or + # every cached bundle ships without them. + add_header X-Content-Type-Options "nosniff" always; + add_header X-Frame-Options "DENY" always; + add_header Referrer-Policy "no-referrer" always; add_header Cache-Control "public, immutable"; } diff --git a/web/package-lock.json b/web/package-lock.json index 2bd86c8..ce35cc0 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -1,12 +1,12 @@ { "name": "mem-web", - "version": "0.1.1", + "version": "0.1.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "mem-web", - "version": "0.1.1", + "version": "0.1.2", "dependencies": { "@radix-ui/react-dialog": "^1.1.2", "@radix-ui/react-dropdown-menu": "^2.1.2", @@ -2161,16 +2161,16 @@ } }, "node_modules/@vitest/expect": { - "version": "4.1.10", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.10.tgz", - "integrity": "sha512-YsCn+qAk1GWjQOWFEsEcL2gNQ0zmVmQu3T03qP6UyjhtmdtwtbuI+DASn/7iQB3HGTXkdBwGddzxPlmiql5vlA==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.11.tgz", + "integrity": "sha512-VX2x5vNJXET47KAFzwERI+KRMtTTCSWTfSMKsW7JsUsXV4psq++e3DvZpuTDOpHcxytiDs6p2nhVb2tVDiiUYw==", "dev": true, "license": "MIT", "dependencies": { "@standard-schema/spec": "^1.1.0", "@types/chai": "^5.2.2", - "@vitest/spy": "4.1.10", - "@vitest/utils": "4.1.10", + "@vitest/spy": "4.1.11", + "@vitest/utils": "4.1.11", "chai": "^6.2.2", "tinyrainbow": "^3.1.0" }, @@ -2179,13 +2179,13 @@ } }, "node_modules/@vitest/mocker": { - "version": "4.1.10", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.10.tgz", - "integrity": "sha512-v0xaezt+DKEmKfaxg133ldzADrwLGd7Ze1MfQQTYfvs8OqZIwbxyxaYURivwV7sWy5fqn3rH5uOrSp07bp44Ow==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.11.tgz", + "integrity": "sha512-2XJVD55d1o5AZous5CCGKS74g/riOj9odEt2bQpCVZeblHyHdnMeFl4jl0XjU21stf4mbjUkew2eXQZt65g5CQ==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/spy": "4.1.10", + "@vitest/spy": "4.1.11", "estree-walker": "^3.0.3", "magic-string": "^0.30.21" }, @@ -2206,9 +2206,9 @@ } }, "node_modules/@vitest/pretty-format": { - "version": "4.1.10", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.10.tgz", - "integrity": "sha512-W1HsjSH4MXQ9YfmmhLAoIYf1HRfekQCGngeIgcei6MP5QQGWUe0gkopdZQaVCFO+JDJMrAJGwa5pRpNpvy4P8Q==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.11.tgz", + "integrity": "sha512-yiZzPbGTS9Sr/JpFl8zHrcIkAofNbFV6k21vIgQN/cY/oxZeXhJv5sc/MBJ5jFKWmWs+oJHw0UXLZjmf931+Vw==", "dev": true, "license": "MIT", "dependencies": { @@ -2219,13 +2219,13 @@ } }, "node_modules/@vitest/runner": { - "version": "4.1.10", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.10.tgz", - "integrity": "sha512-IKI6kpIH+LmpROplyLwBBaCfMgOZOMsygVa6BARD6ahA04VRuJSa6OaVG7kRvSEMD870Vd91rSSw0eegtWyLGg==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.11.tgz", + "integrity": "sha512-LztvUgdwMNJMIkj3hQnnxiC2Xy1zNxq928W/xhjCLaNCzqTZOudjwbQf6v9IntZGPw132i2Lq2rgTRZHD3JHNw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/utils": "4.1.10", + "@vitest/utils": "4.1.11", "pathe": "^2.0.3" }, "funding": { @@ -2233,14 +2233,14 @@ } }, "node_modules/@vitest/snapshot": { - "version": "4.1.10", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.10.tgz", - "integrity": "sha512-xRkfOT1qpTAi/Ti4Y1LtfRc3kEuqxGw59eN2jN9pRWMtS/XDevekhcFSqvQqjUNGksfjMJu3Y+oJ+4Ypn2OaJw==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.11.tgz", + "integrity": "sha512-pN7ikn1ON7h8ee4gIAp4AzyK+zBtJPzVbqOgu5LCEh4VaJVbPQcgYQYJIMGQPXVeJJq1fnfazis7a5pFNPahog==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.1.10", - "@vitest/utils": "4.1.10", + "@vitest/pretty-format": "4.1.11", + "@vitest/utils": "4.1.11", "magic-string": "^0.30.21", "pathe": "^2.0.3" }, @@ -2249,9 +2249,9 @@ } }, "node_modules/@vitest/spy": { - "version": "4.1.10", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.10.tgz", - "integrity": "sha512-PLf/Ugvoq5wO/b4rwYCR1h2PSIdXz7wnkQFMiUpLdtM7l6pqVFcQIBEHyT1+l+cj7mNwAfZHzqXqDyjvOuwbDw==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.11.tgz", + "integrity": "sha512-apNa/prQy2qCeywhnixOHPRCgGNhvg7T4Dapfl1GahLp/R+uhBm5cPyFoNVyqsNd2h1nJxL6BqqdIjiABL60YA==", "dev": true, "license": "MIT", "funding": { @@ -2259,13 +2259,13 @@ } }, "node_modules/@vitest/utils": { - "version": "4.1.10", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.10.tgz", - "integrity": "sha512-fy9am/HWxbaGt/Sawrp90vt6Y6jQwf1RX77cz3uwoJwJVMli/e1IEwRPnMNJ7vKfPTwo0diXifkpPvwH9v7nGA==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.11.tgz", + "integrity": "sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.1.10", + "@vitest/pretty-format": "4.1.11", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.1.0" }, @@ -2461,9 +2461,9 @@ "license": "MIT" }, "node_modules/baseline-browser-mapping": { - "version": "2.10.30", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.30.tgz", - "integrity": "sha512-xjOFN16Ha1+Rz4nFYKqHU/LSB+gx/Vi3yQLX7r7sAW+Wa+8hhF2h4pvqTrTMc8+WcDBEunnUurr46Jvv0jk3Vg==", + "version": "2.11.20", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.20.tgz", + "integrity": "sha512-H0ulySigv6icDJ1F7SjtdCD6PrhTpdYCmP0CactWy1+ekh0AFd0o1Wn5T8b+hnTmdBx19u9yhL6wvCylXMY7zw==", "dev": true, "license": "Apache-2.0", "bin": { @@ -2520,9 +2520,9 @@ } }, "node_modules/browserslist": { - "version": "4.28.2", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.2.tgz", - "integrity": "sha512-48xSriZYYg+8qXna9kwqjIVzuQxi+KYWp2+5nCYnYKPTr0LvD89Jqk2Or5ogxz0NUMfIjhh2lIUX/LyX9B4oIg==", + "version": "4.28.8", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.8.tgz", + "integrity": "sha512-V2NpofLblG64mfOtSgDhOJESZEGogzDMBv/q+W6oc4LXWP/q75eOXoOaaOu1EOadB9U4Bwx/e0yzbvwKH8zalA==", "dev": true, "funding": [ { @@ -2540,11 +2540,11 @@ ], "license": "MIT", "dependencies": { - "baseline-browser-mapping": "^2.10.12", - "caniuse-lite": "^1.0.30001782", - "electron-to-chromium": "^1.5.328", - "node-releases": "^2.0.36", - "update-browserslist-db": "^1.2.3" + "baseline-browser-mapping": "^2.11.12", + "caniuse-lite": "^1.0.30001809", + "electron-to-chromium": "^1.5.402", + "node-releases": "^2.0.53", + "update-browserslist-db": "^1.3.0" }, "bin": { "browserslist": "cli.js" @@ -2574,9 +2574,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001793", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001793.tgz", - "integrity": "sha512-iwSsYWaCOoh26cV8NwNRViHlrfUvYsHDfRVcbtmw0Kg6PJIZZXwMkj1442FYLBGkeUf1juAsU3DTfxW579mrPA==", + "version": "1.0.30001810", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001810.tgz", + "integrity": "sha512-TITQPUkaz+aVk5GL6NhOdwk1aEaNTSDPsGFWrTuhKGtjTF70jL/Oht2W4c6rXUe5fu7Ie19VIahAXHIIiWWNeg==", "dev": true, "funding": [ { @@ -2934,9 +2934,9 @@ "peer": true }, "node_modules/electron-to-chromium": { - "version": "1.5.357", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.357.tgz", - "integrity": "sha512-NHlTIQDK8fmVwHwuIzmXYEJ1Ewq3D9wDNc0cWXxDGysP6Pb21giwGNkxiTifyKy/4SoPuN5l6GLP1W9Sv7zB2g==", + "version": "1.5.420", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.420.tgz", + "integrity": "sha512-2yD6XreGusOfNV+dUcvipJEXc3n/n7fgr7996aszTG+YY5E4mqM4tOq/3uhP129cazL9YHbVWSpc79ePotWtPA==", "dev": true, "license": "ISC" }, @@ -3784,9 +3784,9 @@ "peer": true }, "node_modules/js-yaml": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.1.tgz", - "integrity": "sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", "dev": true, "funding": [ { @@ -4410,11 +4410,14 @@ "license": "MIT" }, "node_modules/node-releases": { - "version": "2.0.44", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.44.tgz", - "integrity": "sha512-5WUyunoPMsvvEhS8AxHtRzP+oA8UCkJ7YRxatWKjngndhDGLiqEVAQKWjFAiAiuL8zMRGzGSJxFnLetoa43qGQ==", + "version": "2.0.54", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.54.tgz", + "integrity": "sha512-YHs7BmmcsdAI5Ozuf8JZo6PT0mv2GIWC9vMfvUC3dp65M8hn7Ux8CPL+2oBI7juNuj9d0ndhTcznq2ODBps9cQ==", "dev": true, - "license": "MIT" + "license": "MIT", + "engines": { + "node": ">=18" + } }, "node_modules/normalize-path": { "version": "3.0.0", @@ -4844,9 +4847,9 @@ } }, "node_modules/postcss-selector-parser": { - "version": "6.1.2", - "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.2.tgz", - "integrity": "sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg==", + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.4.tgz", + "integrity": "sha512-bIoJLOmjCO1S9XdY/DcnR5hJxvrDir1PbGChrzXG3vw0/FOliy/fA3dmdhQ441kah4gKv+TwckGzex6wNS5cnQ==", "dev": true, "license": "MIT", "dependencies": { @@ -5824,9 +5827,9 @@ } }, "node_modules/update-browserslist-db": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", - "integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.3.2.tgz", + "integrity": "sha512-UQ+MSxlhRm1bzjhU+DcuXfjFO1FzNtqhK5+9Yvlp90ItDLk5vT932A0rFu619nf7RVS+Y/VeaUW1jaRDqZ8VJw==", "dev": true, "funding": [ { @@ -6006,19 +6009,19 @@ } }, "node_modules/vitest": { - "version": "4.1.10", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.10.tgz", - "integrity": "sha512-R9jUTe5S4Qb0HCd4TNqpC7oGcrMssMRGXLW80ubjWsW9VH5GF8y1Y0SFLY9AbqSk6nt0PnOx4H4WNJYZ13GUPw==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.11.tgz", + "integrity": "sha512-fhACrNXUidIbGSBr5FlbuBkO7VWC1ZyLl0DO4CU2DrQoAPxX84Ysxs+HeGQpii5lZWV1Q4gBZTTu49mF+A6Edw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/expect": "4.1.10", - "@vitest/mocker": "4.1.10", - "@vitest/pretty-format": "4.1.10", - "@vitest/runner": "4.1.10", - "@vitest/snapshot": "4.1.10", - "@vitest/spy": "4.1.10", - "@vitest/utils": "4.1.10", + "@vitest/expect": "4.1.11", + "@vitest/mocker": "4.1.11", + "@vitest/pretty-format": "4.1.11", + "@vitest/runner": "4.1.11", + "@vitest/snapshot": "4.1.11", + "@vitest/spy": "4.1.11", + "@vitest/utils": "4.1.11", "es-module-lexer": "^2.0.0", "expect-type": "^1.3.0", "magic-string": "^0.30.21", @@ -6046,12 +6049,12 @@ "@edge-runtime/vm": "*", "@opentelemetry/api": "^1.9.0", "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", - "@vitest/browser-playwright": "4.1.10", - "@vitest/browser-preview": "4.1.10", - "@vitest/browser-webdriverio": "4.1.10", - "@vitest/coverage-istanbul": "4.1.10", - "@vitest/coverage-v8": "4.1.10", - "@vitest/ui": "4.1.10", + "@vitest/browser-playwright": "4.1.11", + "@vitest/browser-preview": "4.1.11", + "@vitest/browser-webdriverio": "4.1.11", + "@vitest/coverage-istanbul": "4.1.11", + "@vitest/coverage-v8": "4.1.11", + "@vitest/ui": "4.1.11", "happy-dom": "*", "jsdom": "*", "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" diff --git a/web/package.json b/web/package.json index 6a66d95..2988971 100644 --- a/web/package.json +++ b/web/package.json @@ -1,14 +1,14 @@ { "name": "mem-web", "private": true, - "version": "0.1.1", + "version": "0.1.2", "type": "module", "description": "mem · Agent-Native AI 网盘 · Web UI (Phase 1)", "scripts": { "dev": "vite", "build": "tsc -b && vite build", "preview": "vite preview", - "audit": "npm audit --omit=dev --audit-level=moderate && npm audit --audit-level=high", + "audit": "node audit-retry.mjs", "lint": "eslint . --ext .ts,.tsx --max-warnings 0", "format": "prettier --write \"src/**/*.{ts,tsx,css}\"", "test:enrichment": "node enrichment-acceptance.mjs", @@ -21,7 +21,10 @@ "test": "vitest run", "test:watch": "vitest", "test:coverage": "vitest run --coverage", - "typecheck": "tsc -b --noEmit" + "typecheck": "tsc -b --noEmit", + "tokens:generate": "node scripts/design-tokens.mjs", + "tokens:check": "node scripts/design-tokens.mjs --check", + "prebuild": "npm run tokens:check" }, "dependencies": { "@radix-ui/react-dialog": "^1.1.2", diff --git a/web/scripts/design-tokens.mjs b/web/scripts/design-tokens.mjs new file mode 100644 index 0000000..4899c60 --- /dev/null +++ b/web/scripts/design-tokens.mjs @@ -0,0 +1,136 @@ +import { readFile, writeFile } from 'node:fs/promises'; +import { createHash } from 'node:crypto'; +import assert from 'node:assert/strict'; + +// The snapshot, not this compatibility adapter, owns palette values. +const revision = '910456901dda74da4d5b0320cd03d36ad18650b0'; +const root = new URL('../', import.meta.url); +const source = await readFile(new URL('design-system/tokens.json', root), 'utf8'); +const tokens = JSON.parse(source).tokens; +const digest = createHash('sha256').update(source).digest('hex'); +assert.equal( + digest, + '75e62b372f23083c2f35400ce434d40bf1be098375cea85b97352dcfedbf5766', + 'Pinned upstream snapshot changed: review a new revision and its hash', +); +const value = (group, name, mode) => { + const v = tokens[group][name].$value; + return typeof v === 'string' ? v : v[mode]; +}; +const color = (name, mode) => { + const s = value('color', name, mode); + if (s.startsWith('#')) + return [ + ...s + .slice(1) + .match(/../g) + .map((v) => parseInt(v, 16)), + 1, + ]; + const c = s.match(/[\d.]+/g).map(Number); + return c.length === 3 ? [...c, 1] : c; +}; +const mix = (a, b, weight) => + a + .slice(0, 3) + .map((n, i) => Math.round(n * weight + b[i] * (1 - weight))) + .concat(1); +const luminance = (c) => + c + .slice(0, 3) + .map((v) => v / 255) + .map((v) => (v <= 0.04045 ? v / 12.92 : ((v + 0.055) / 1.055) ** 2.4)) + .reduce((sum, v, i) => sum + v * [0.2126, 0.7152, 0.0722][i], 0); +const contrast = (fg, bg) => { + const a = luminance(mix(fg, bg, fg[3] ?? 1)), + b = luminance(bg); + return (Math.max(a, b) + 0.05) / (Math.min(a, b) + 0.05); +}; +let css = `/* GENERATED by scripts/design-tokens.mjs; DO NOT EDIT.\n * Source: bytefolk/design-system@${revision}/tokens/design-tokens.json\n * SHA-256: ${digest}; Apache-2.0 (web/design-system/LICENSE).\n */\n`; +const checks = []; +for (const mode of ['dark', 'light']) { + const surfaces = ['canvas', 'navigation', 'surface', 'surface-inset', 'surface-raised'].map((n) => + color(n, mode), + ); + const target = mode === 'dark' ? [255, 255, 255] : [0, 0, 0]; + // Preserve the upstream hue while making actual small text readable on every + // surface. Upstream base hues are sometimes intended only for borders/icons. + const readable = (base, tinted = false) => { + for (let step = 0; step <= 100; step++) { + const c = + base[3] < 1 + ? [...base.slice(0, 3), base[3] + ((1 - base[3]) * step) / 100] + : mix(base, target, 1 - step / 100); + if ( + surfaces.every( + (bg) => contrast(c, bg) >= 4.5 && (!tinted || contrast(c, mix(c, bg, 0.3)) >= 4.5), + ) + ) + return c; + } + throw new Error('No readable foreground found'); + }; + const vars = { + bg: color('canvas', mode), + 'bg-subtle': color('navigation', mode), + 'bg-panel': color('surface', mode), + 'bg-inset': color('surface-inset', mode), + fg: color('foreground', mode), + 'fg-muted': readable(color('foreground-muted', mode)), + 'fg-subtle': readable(color('foreground-muted', mode)), + border: color('border', mode), + 'border-strong': color('border-strong', mode), + accent: readable(color('primary', mode), true), + 'accent-hover': readable(color('primary-hover', mode), true), + 'accent-muted': color('primary-soft', mode), + 'accent-solid': mix(color('primary', mode), [0, 0, 0], 0.8), + 'accent-solid-hover': mix(color('primary-hover', mode), [0, 0, 0], 0.7), + 'accent-foreground': color('primary-foreground', mode), + ai: readable(color('ai', mode), true), + success: readable(color('success-strong', mode), true), + warn: readable(color('warning-strong', mode), true), + danger: readable(color('danger-strong', mode), true), + }; + for (const name of [ + 'fg', + 'fg-muted', + 'fg-subtle', + 'accent', + 'accent-hover', + 'ai', + 'success', + 'warn', + 'danger', + ]) { + for (const bg of surfaces) { + const ratio = contrast(vars[name], bg); + assert.ok(ratio >= 4.5, `${mode} ${name}: ${ratio}`); + checks.push(ratio); + } + } + // Badges, selected navigation and danger-button hover compose tinted surfaces. + for (const name of ['accent', 'accent-hover', 'ai', 'success', 'warn', 'danger']) { + for (const bg of surfaces) + for (const alpha of [0.05, 0.1, 0.2, 0.3]) { + const ratio = contrast(vars[name], mix(vars[name], bg, alpha)); + assert.ok(ratio >= 4.5, `${mode} ${name} tint ${alpha}: ${ratio}`); + checks.push(ratio); + } + } + for (const name of ['accent-solid', 'accent-solid-hover']) { + const ratio = contrast(vars['accent-foreground'], vars[name]); + assert.ok(ratio >= 4.5, `${mode} ${name}: ${ratio}`); + checks.push(ratio); + } + css += `${mode === 'dark' ? ':root, .dark' : '.light'} {\n`; + for (const [name, c] of Object.entries(vars)) + css += ` --${name}: ${c.slice(0, 3).join(' ')};\n --${name}-opacity: ${c[3]};\n`; + css += ` --shadow-soft: ${value('boxShadow', 'shadow-sm', mode)};\n --font-sans: ${value('fontFamily', 'font-sans', mode)};\n --font-mono: ${value('fontFamily', 'font-mono', mode)};\n color-scheme: ${mode};\n}\n`; +} +const output = new URL('src/styles/design-tokens.generated.css', root); +if (process.argv.includes('--check')) + assert.equal(await readFile(output, 'utf8'), css, 'Regenerate shared design tokens'); +else await writeFile(output, css); +console.log( + `Shared token bridge: ${checks.length} contrast pairs >= ${Math.min(...checks).toFixed(2)}:1; ${process.argv.includes('--check') ? 'current' : 'generated'}`, +); diff --git a/web/src/components/explorer/ContextMenu.tsx b/web/src/components/explorer/ContextMenu.tsx index d12b0d3..24722d7 100644 --- a/web/src/components/explorer/ContextMenu.tsx +++ b/web/src/components/explorer/ContextMenu.tsx @@ -102,7 +102,7 @@ function ContextMenuView({ state, onClose }: { state: ContextMenuState; onClose: requestAnimationFrame(() => item.onSelect()); }} className={cn( - 'flex w-full items-center gap-2 px-2.5 py-1.5 text-sm rounded-sm mx-1 my-0.5', + 'grid w-[calc(100%-0.5rem)] grid-cols-[minmax(0,1fr)_auto] items-center gap-2 px-2.5 py-1.5 text-left text-sm rounded-sm mx-1 my-0.5', 'transition-colors', item.disabled ? 'text-fg-subtle cursor-not-allowed' @@ -111,9 +111,9 @@ function ContextMenuView({ state, onClose }: { state: ContextMenuState; onClose: : 'text-fg-muted hover:bg-bg-inset hover:text-fg', )} > - {item.label} + {item.label} {item.shortcut && ( - {item.shortcut} + {item.shortcut} )} diff --git a/web/src/components/explorer/FileGrid.tsx b/web/src/components/explorer/FileGrid.tsx index 458aa1c..d465cf0 100644 --- a/web/src/components/explorer/FileGrid.tsx +++ b/web/src/components/explorer/FileGrid.tsx @@ -70,9 +70,10 @@ export function FileGrid(props: FileGridProps) { return (
{pendingNewFolder && ( -
+
(
@@ -206,7 +207,7 @@ function FolderCard({ onDragLeave={() => setDropHover(false)} onDrop={onDrop} className={cn( - 'flex flex-col items-center text-center gap-2 p-2 rounded-lg cursor-default select-none', + 'flex flex-col items-center text-left gap-2 p-2 rounded-lg cursor-default select-none', 'border transition-colors', selected ? 'border-accent/60 bg-accent/10' @@ -218,6 +219,7 @@ function FolderCard({ {renaming ? ( )} -
{tt('drive.itemsN', { n: folder.fileCount })}
+
{tt('drive.itemsN', { n: folder.fileCount })}
); } @@ -265,7 +267,7 @@ function FileCard({ onDoubleClick={onDoubleClick} onContextMenu={onContextMenu} className={cn( - 'flex flex-col items-center text-center gap-2 p-2 rounded-lg cursor-default select-none', + 'flex flex-col items-center text-left gap-2 p-2 rounded-lg cursor-default select-none', 'border transition-colors', selected ? 'border-accent/60 bg-accent/10' @@ -286,7 +288,7 @@ function FileCard({ {file.index_status !== 'done' && }
{renaming ? ( - + ) : (
{file.name} @@ -307,10 +309,12 @@ function KindIcon({ kind }: { kind: FileKind }) { function StatusOverlay({ status }: { status: IndexStatus }) { const text = tt(`status.${status}`); - const tone = status === 'failed' ? 'bg-danger/80' : 'bg-bg/70'; + const tone = status === 'failed' + ? 'border border-danger/30 bg-bg-panel text-danger' + : 'bg-bg/70 text-fg'; return (
{text}
diff --git a/web/src/components/explorer/FileList.tsx b/web/src/components/explorer/FileList.tsx index 71e1396..5aaf6a7 100644 --- a/web/src/components/explorer/FileList.tsx +++ b/web/src/components/explorer/FileList.tsx @@ -33,10 +33,10 @@ export interface FileListProps { export function FileList(props: FileListProps) { const { t } = useT(); return ( -
+
{t('drive.colName')}
-
{t('drive.colSize')}
+
{t('drive.colSize')}
{t('drive.colModified')}
{t('drive.colType')}
@@ -215,7 +215,7 @@ function FolderRow({
)}
-
{tt('drive.itemsN', { n: folder.fileCount })}
+
{tt('drive.itemsN', { n: folder.fileCount })}
—
{tt('drive.folder')}
@@ -271,7 +271,7 @@ function FileRow({
)}
-
{formatBytes(file.size)}
+
{formatBytes(file.size)}
{formatRelative(file.updated_at)}
{tt(`kind.${file.kind === 'pdf' ? 'doc' : file.kind}`)}
@@ -285,4 +285,3 @@ function KindIconSmall({ kind }: { kind: FileKind }) { if (kind === 'pdf' || kind === 'doc' || kind === 'text') return ; return ; } - diff --git a/web/src/components/layout/TopBar.tsx b/web/src/components/layout/TopBar.tsx index 97bd25b..066d336 100644 --- a/web/src/components/layout/TopBar.tsx +++ b/web/src/components/layout/TopBar.tsx @@ -186,7 +186,7 @@ export function TopBar({ children }: { children?: React.ReactNode }) { @@ -166,7 +166,7 @@ export function CreateRelationDialog({ setReason(event.target.value as MemoryForgetReason)} - className="h-9 rounded-md border border-border bg-bg-inset px-3 text-sm text-fg outline-none focus:border-accent/60" + className="h-9 rounded-md border border-border bg-bg-inset px-3 text-left [text-align-last:left] text-sm text-fg outline-none focus:border-accent/60" > {REASONS.map((candidate) => (