Merge remote-tracking branch 'origin/staging' into codex/slack-lists-canvas-blocks

# Conflicts:
#	apps/sim/tools/generated/tool-metadata.ts
#	apps/sim/tools/generated/tool-outputs.ts
#	apps/sim/tools/index.test.ts

Author: TheodoreSpeaks

.agents/skills/add-connector/SKILL.md

@@ -6,6 +6,12 @@ argument-hint: <service-name> [api-docs-url]
 
 # Add Connector Skill
 
+## Choose the connector runtime first
+
+For **Sim Search**, use the live provider workflow in [the federated Search developer guide](../../../apps/sim/lib/sim-search/live/README.md#adding-a-live-search-connector). Its browser-safe provider catalog owns provider IDs, API origins, credential aliases, and account modes; its typed runtime registry requires both search and read handlers. `ConnectorMeta` remains the owner of logos and setup fields. Member mode has no admin resource filters. Service mode requires independent live source verification and shared selectors. Do not implement a Search source by adding a crawler, embeddings, or a scheduled ACL build.
+
+The ingestion instructions below apply to **ordinary knowledge-base connectors** and the explicit legacy Search backend (`SIM_SEARCH_LIVE=false`). If a provider supports both, implement and test both runtimes; adding `search: true` to metadata alone does not implement federated search. Preserve indexing documentation and behavior for those KB/legacy callers.
+
 You are an expert at adding knowledge base connectors to Sim. A connector syncs documents from an external source (Confluence, Google Drive, Notion, etc.) into a knowledge base.
 
 ## Your Task

.agents/skills/validate-connector/SKILL.md

@@ -6,6 +6,12 @@ argument-hint: <service-name> [api-docs-url]
 
 # Validate Connector Skill
 
+## Identify the runtime under review
+
+For **Sim Search**, validate the [live provider registration and access pipeline](../../../apps/sim/lib/sim-search/live/README.md#adding-a-live-search-connector): catalog and metadata parity, both search/read handlers, current member grants, service-source restrictions, safe scoped references, pagination, provenance, and provider failure behavior. Test real localhost setup/search/read with authorized fixtures when available, and distinguish those results from mocked provider tests. Live Search must not enqueue content indexing or background ACL/directory builds; GitLab still computes request-time permissions or uses current CSV grants.
+
+The ingestion-specific checks below apply to ordinary workspace KB connectors and legacy Search selected with `SIM_SEARCH_LIVE=false`. Keep those checks for providers supporting both runtimes; do not require a live-only provider to implement content hashes, ingestion cursors, embeddings, or stored ACL snapshots.
+
 You are an expert auditor for Sim knowledge base connectors. Your job is to thoroughly validate that an existing connector is correct, complete, and follows all conventions.
 
 ## Your Task

.github/CONTRIBUTING.md

@@ -256,6 +256,8 @@ If you prefer not to use Docker. **All commands run from the repository root unl
 
    For ad-hoc schema iteration during development you can also use `bun run db:push` from `packages/db`, but `db:migrate` is the canonical command for staging and production. `db:push` reconciles directly to the current schema without running versioned migration guards. For disposable local/dev databases, `bun run db:push --force` accepts Drizzle's data-loss prompts, including column drops.
 
+   `db:push` treats added and removed columns, tables, and other schema objects as separate creations and deletions. It never infers a rename. For an intentional rename during local development, run `bun run db:push --interactive-renames` in a terminal and select the old object in Drizzle's chooser. This flag does not approve data loss; `--force` controls that separately. After schema reconciliation succeeds, the wrapper reconciles credential policies and OAuth providers, then backfills search vectors. A failure stops subsequent steps. Staging and production changes still use reviewed versioned migrations with expand/contract deployment steps.
+
 4. **Run the Development Servers:**
 
    ```bash

.github/actions/docker-build/action.yml

@@ -19,6 +19,10 @@ inputs:
   tags:
     description: Comma-separated list of tags to push.
     required: true
+  build-args:
+    description: Newline-separated Docker build arguments.
+    required: false
+    default: ''
   max-cache-size-mb:
     description: >-
       Layer cache to retain after this action prunes, in MB. Must stay above one
@@ -72,6 +76,7 @@ runs:
         platforms: ${{ inputs.platforms }}
         push: true
         tags: ${{ inputs.tags }}
+        build-args: ${{ inputs.build-args }}
         provenance: false
         sbom: false
 
@@ -177,5 +182,6 @@ runs:
         platforms: ${{ inputs.platforms }}
         push: true
         tags: ${{ inputs.tags }}
+        build-args: ${{ inputs.build-args }}
         provenance: false
         sbom: false

.github/workflows/ci.yml

@@ -103,7 +103,7 @@ jobs:
             echo "ℹ️ No comparable base commit; skipping desktop prerelease"
             exit 0
           fi
-          if git diff --name-only "$BEFORE" HEAD | grep -qE '^(apps/desktop/|packages/desktop-bridge/|packages/browser-protocol/)'; then
+          if git diff --name-only "$BEFORE" HEAD | grep -qE '^(apps/desktop/|packages/desktop-bridge/|packages/browser-protocol/|\.github/workflows/(ci|desktop-release)\.yml$)'; then
             echo "changed=true" >> "$GITHUB_OUTPUT"
             echo "✅ Desktop shell code changed"
           else
@@ -221,15 +221,18 @@ jobs:
           file: ${{ matrix.dockerfile }}
           platforms: linux/amd64
           tags: ${{ steps.login-ecr.outputs.registry }}/${{ steps.ecr-repo.outputs.name }}:${{ github.sha }}-dev
+          build-args: |
+            SIM_SEARCH_LIVE_DEFAULT=true
+            MSHIP_PLAN_MODE_DEFAULT=true
           max-cache-size-mb: ${{ matrix.cache_mb }}
 
-  # Build and upload tasks alongside tests and images. The unpromoted version
-  # cannot serve new runs; promote-images waits for it and successful migrations.
+  # Staging/production coordinate task releases with app traffic cutover.
+  # Dev tasks deploy independently in deploy-trigger-dev.yml.
   prepare-trigger:
     name: Prepare Trigger.dev
     if: >-
       github.event_name == 'push' &&
-      (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/dev')
+      (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging')
     runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }}
     timeout-minutes: 30
     outputs:
@@ -265,7 +268,6 @@ jobs:
           case "$GITHUB_REF" in
             refs/heads/main) TRIGGER_ENV=prod; TRIGGER_BRANCH='' ;;
             refs/heads/staging) TRIGGER_ENV=staging; TRIGGER_BRANCH='' ;;
-            refs/heads/dev) TRIGGER_ENV=preview; TRIGGER_BRANCH=dev-sim ;;
             *) echo "ERROR: unsupported Trigger release ref: $GITHUB_REF" >&2; exit 1 ;;
           esac
           echo "environment=$TRIGGER_ENV" >> "$GITHUB_OUTPUT"
@@ -426,8 +428,9 @@ jobs:
           tags: ${{ steps.meta.outputs.tags }}
           max-cache-size-mb: ${{ matrix.cache_mb }}
 
-  # Promote the sha-tagged ECR images once tests, migrations, and the Trigger
-  # upload pass. Pushing the ECR latest/staging tag is what triggers
+  # Promote the sha-tagged ECR images once their build and migrations pass.
+  # Staging/production also require the Trigger upload; dev never waits for it.
+  # Pushing the ECR latest/staging/dev tag is what triggers
   # CodePipeline, so this seconds-long manifest retag is the deploy gate —
   # the image builds themselves run in parallel with the tests. A single job
   # (not a matrix) so all four sha manifests are verified before any tag
@@ -438,9 +441,9 @@ jobs:
     # Explicit results: see migrate's comment.
     if: >-
       !cancelled() && github.event_name == 'push' &&
-      needs.prepare-trigger.result == 'success' &&
       (
         ((github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging') &&
+          needs.prepare-trigger.result == 'success' &&
           needs.migrate.result == 'success' &&
           needs.build-amd64.result == 'success') ||
         (github.ref == 'refs/heads/dev' &&
@@ -546,7 +549,7 @@ jobs:
             fi
           done
 
-  # Promote the parked Trigger.dev version after observing the ECS
+  # Staging/production: promote the parked Trigger.dev version after observing the ECS
   # traffic cutover (CodeDeploy AllowTraffic on every target). The image retag
   # triggers the ECS pipeline; this job correlates it via the digest + retag epoch
   # (rejecting a stale execution reusing the digest) and promotes at cutover.
@@ -560,14 +563,13 @@ jobs:
     if: >-
       !cancelled() &&
       github.event_name == 'push' &&
-      (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/dev') &&
+      (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging') &&
       needs.promote-images.result == 'success' &&
       needs.prepare-trigger.result == 'success' &&
       needs.promote-images.outputs.promoted == 'true'
     runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }}
-    # Leave setup/promotion headroom above the cutover poll (dev: 20 min;
-    # staging/prod: 70 min, including a deploy queued behind a long bake).
-    timeout-minutes: ${{ github.ref == 'refs/heads/dev' && 40 || 90 }}
+    # Leave setup/promotion headroom above the 70-minute cutover poll.
+    timeout-minutes: 90
     permissions:
       contents: read
       id-token: write
@@ -599,14 +601,13 @@ jobs:
         with:
           role-to-assume: ${{ github.ref == 'refs/heads/main' && secrets.AWS_ROLE_TO_ASSUME || github.ref == 'refs/heads/dev' && secrets.DEV_AWS_ROLE_TO_ASSUME || secrets.STAGING_AWS_ROLE_TO_ASSUME }}
           aws-region: ${{ github.ref == 'refs/heads/main' && secrets.AWS_REGION || github.ref == 'refs/heads/dev' && secrets.DEV_AWS_REGION || secrets.STAGING_AWS_REGION }}
-          # Match each environment's session budget; both outlast their polls.
-          role-duration-seconds: ${{ github.ref == 'refs/heads/dev' && 2400 || 5400 }}
+          role-duration-seconds: 5400
 
       # An unchanged tag may belong to a failed or still-running earlier deploy.
       # Verify its latest cutover rather than treating tag equality as success.
       - name: Wait for ECS traffic cutover
         env:
-          OVERALL_TIMEOUT: ${{ github.ref == 'refs/heads/dev' && 1200 || 4200 }}
+          OVERALL_TIMEOUT: 4200
           APP_IMAGE_CHANGED: ${{ needs.promote-images.outputs.app_image_changed }}
           DIGEST: ${{ needs.promote-images.outputs.app_image_digest }}
           PIPELINE: sim-${{ github.ref == 'refs/heads/main' && 'production' || github.ref == 'refs/heads/dev' && 'dev' || 'staging' }}-us-east-1-app-deployment
@@ -1311,24 +1312,30 @@ jobs:
     name: Prune Desktop Prereleases
     runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }}
     timeout-minutes: 5
-    needs: [publish-desktop-prerelease]
+    needs: [create-desktop-prerelease, publish-desktop-prerelease]
     permissions:
       contents: read
     env:
       GH_TOKEN: ${{ secrets.DESKTOP_RELEASE_TOKEN }}
       GH_REPO: simstudioai/sim-desktop-releases
+      CURRENT_TAG: ${{ needs.create-desktop-prerelease.outputs.version }}
     steps:
       - name: Delete stale prereleases
         run: |
+          set -euo pipefail
+          : "${CURRENT_TAG:?Current publication tag is required before pruning}"
           if [ -z "$GH_TOKEN" ]; then
             echo "::error::DESKTOP_RELEASE_TOKEN is required to prune desktop prereleases."
             exit 1
           fi
           if [ "$GITHUB_REF" = "refs/heads/dev" ]; then CHANNELS='(dev|alpha)'; else CHANNELS='(staging|beta)'; fi
-          gh release list --limit 100 --json tagName,isPrerelease,isDraft,createdAt \
-            --jq "[.[] | select(.isPrerelease and (.isDraft | not) and (.tagName | test(\"-${CHANNELS}\\\\.\")))] | sort_by(.createdAt) | reverse | .[5:] | .[].tagName" |
+          # createdAt follows the tag's commit: published releases in the release-only
+          # repository can all share it. Retain by publication time, with deterministic ties.
+          gh release list --limit 100 --json tagName,isPrerelease,isDraft,publishedAt \
+            --jq "[.[] | select(.isPrerelease and (.isDraft | not) and (.tagName | test(\"-${CHANNELS}\\\\.\")))] | sort_by(.publishedAt, .tagName) | reverse | .[5:] | .[].tagName" |
           while read -r TAG; do
             [ -n "$TAG" ] || continue
+            [ "$TAG" != "$CURRENT_TAG" ] || continue
             echo "Deleting stale prerelease $TAG"
             gh release delete "$TAG" --cleanup-tag --yes
           done

.github/workflows/deploy-trigger-dev.yml

@@ -0,0 +1,75 @@
+name: Deploy Dev Tasks
+
+# Independent of app CI: task packaging and deployment must never delay dev images.
+on:
+  push:
+    branches: [dev]
+
+permissions:
+  contents: read
+
+# Serialize promotions without cancelling an external deployment in flight.
+# Pending pushes coalesce to the newest run while the current run finishes.
+concurrency:
+  group: deploy-trigger-dev
+  cancel-in-progress: false
+
+jobs:
+  deploy:
+    name: Deploy Trigger.dev preview
+    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }}
+    timeout-minutes: 30
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
+        with:
+          bun-version: 1.4.1
+
+      - name: Cache Bun dependencies
+        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
+        with:
+          path: |
+            ~/.bun/install/cache
+            node_modules
+            **/node_modules
+          key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-bun-
+
+      - name: Install dependencies
+        run: bun install --frozen-lockfile --ignore-scripts
+
+      - name: Upload preview version
+        id: deploy
+        working-directory: ./apps/sim
+        env:
+          TRIGGER_ACCESS_TOKEN: ${{ secrets.TRIGGER_ACCESS_TOKEN }}
+          TRIGGER_PROJECT_ID: ${{ secrets.TRIGGER_PROJECT_ID }}
+        run: |
+          set -euo pipefail
+          : "${TRIGGER_ACCESS_TOKEN:?TRIGGER_ACCESS_TOKEN must be configured}"
+          : "${TRIGGER_PROJECT_ID:?TRIGGER_PROJECT_ID must be configured}"
+          bunx trigger.dev@4.5.16 deploy --env preview --branch dev-sim --skip-promotion
+
+      - name: Promote current dev preview
+        working-directory: ./apps/sim
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TRIGGER_ACCESS_TOKEN: ${{ secrets.TRIGGER_ACCESS_TOKEN }}
+          TRIGGER_PROJECT_ID: ${{ secrets.TRIGGER_PROJECT_ID }}
+          VERSION: ${{ steps.deploy.outputs.deploymentVersion }}
+        run: |
+          set -euo pipefail
+          if ! [[ "$VERSION" =~ ^[0-9]{8}\.[0-9]+$ ]]; then
+            echo "ERROR: Trigger.dev did not report a valid deploymentVersion output" >&2
+            exit 1
+          fi
+          CURRENT_SHA=$(gh api "repos/$GITHUB_REPOSITORY/git/ref/heads/dev" --jq '.object.sha')
+          if [ "$CURRENT_SHA" != "$GITHUB_SHA" ]; then
+            echo "::notice::Skipping superseded dev task version $VERSION"
+            exit 0
+          fi
+          bunx trigger.dev@4.5.16 promote "$VERSION" --env preview --branch dev-sim

.github/workflows/migrations.yml

@@ -64,6 +64,8 @@ jobs:
           MIGRATION_DATABASE_URL: ${{ inputs.environment == 'production' && secrets.MIGRATION_DATABASE_URL || inputs.environment == 'staging' && secrets.STAGING_MIGRATION_DATABASE_URL || '' }}
           ENVIRONMENT: ${{ inputs.environment }}
         run: |
+          set -euo pipefail
+
           if [ -z "$DATABASE_URL" ]; then
             echo "ERROR: no database URL secret resolved for environment '${ENVIRONMENT}'" >&2
             exit 1
@@ -73,16 +75,7 @@ jobs:
             echo "Dev environment — pushing schema directly (db:push)"
             # Dev deliberately forces direct schema reconciliation; staging and
             # production use guarded versioned migrations in the other branch.
-            # drizzle-kit push needs a TTY to resolve ambiguous renames (--force only
-            # covers data-loss). In CI it throws "Interactive prompts require a TTY
-            # terminal" but still exits 0, so the job goes green without applying the
-            # change. tee keeps the output live in the log; we then fail on drizzle's
-            # own TTY error. A genuine non-zero exit already fails via `set -e`.
-            bun run db:push --force < /dev/null 2>&1 | tee /tmp/db-push.log
-            if grep -q "Interactive prompts require a TTY terminal" /tmp/db-push.log; then
-              echo "ERROR: db:push needs an interactive rename decision; land it as a versioned migration instead of relying on push." >&2
-              exit 1
-            fi
+            SIM_DEV_DB_PUSH=1 bun run db:push --force < /dev/null
           else
             echo "Applying versioned migrations (db:migrate)"
             bun run ./scripts/migrate.ts

.github/workflows/test-build.yml

@@ -83,6 +83,12 @@ jobs:
       - name: Install dependencies
         run: bun install --frozen-lockfile --ignore-scripts
 
+      - name: Verify direct schema push compatibility
+        working-directory: packages/db
+        env:
+          DB_PUSH_TEST_DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/postgres
+        run: bunx vitest run scripts/push.postgres.test.ts
+
       - name: Provision a fresh database through the supported command
         working-directory: packages/db
         run: |
@@ -233,14 +239,16 @@ jobs:
           if-no-files-found: ignore
           retention-days: 7
 
-      - name: Verify durable provenance, concurrent memory writes, and attachment replay
+      - name: Verify durable provenance, concurrent memory writes, and browser download admission
         working-directory: apps/sim
         env:
+          BROWSER_FILE_TRANSFER_TEST_DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/sim_auth_scim
           TABLE_PROVENANCE_TEST_DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/sim_auth_scim
           MEMORY_PROVENANCE_TEST_DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/sim_auth_scim
           AGENT_MEMORY_TEST_DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/sim_auth_scim
         run: >-
           bunx vitest run
+          lib/mothership/async-runs/browser-download-claim.postgres.test.ts
           lib/table/rows/secret-provenance.postgres.test.ts
           lib/memory/message-provenance.postgres.test.ts
           lib/memory/conversation-store.postgres.test.ts
@@ -256,6 +264,7 @@ jobs:
           script-migrations/0016_backfill_search_vectors.postgres.test.ts
           script-migrations/0018_repair_workspace_file_content_revision.postgres.test.ts
           script-migrations/0019_tin_keyword_projection.postgres.test.ts
+          member-sync-status-migration.postgres.test.ts
 
       - name: Verify Search progress, pagination, and outbox scheduling in PostgreSQL
         working-directory: apps/sim