From cd3d94d47aec7fca3209cb7cc793d997aa2b25ae Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:13:09 -0700 Subject: [PATCH 1/3] test(staging-smoke): fresh no-grants user must see connectable tools outside the inactive fold Adds a Tier-1 check that guards the #2504 regression: logs in as a SEPARATE fresh, grant-free staging account and asserts at least one connectable tool (GitHub/Linear/...) renders as a visible row on /v2/connectors, not buried in .v2-connectors__inactive-fold. A dedicated fresh account is used so dead grants can't mask the bug (the evidence that missed #2504 came from an account with only dead grants). deploy-staging passes the STAGING_SMOKE_FRESH_EMAIL/PW secrets. Validated green 9/9 on 7fce31f7 (visible=[GitHub, Linear,Jira,Atlassian]); it fails when offers are folded. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01V7T2fgquLRaqaYiGLC6Dey --- .github/workflows/deploy-staging.yml | 2 ++ scripts/staging-smoke/README.md | 8 ++++-- scripts/staging-smoke/staging-smoke.mjs | 33 ++++++++++++++++++++++++- 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index 719ccdd136..4a818f54b5 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -204,6 +204,8 @@ jobs: SMOKE_API: https://api-staging.commonly.me SMOKE_EMAIL: ${{ secrets.STAGING_SMOKE_EMAIL }} SMOKE_PW: ${{ secrets.STAGING_SMOKE_PW }} + SMOKE_FRESH_EMAIL: ${{ secrets.STAGING_SMOKE_FRESH_EMAIL }} + SMOKE_FRESH_PW: ${{ secrets.STAGING_SMOKE_FRESH_PW }} run: | set -euo pipefail if [ -z "${SMOKE_EMAIL:-}" ] || [ -z "${SMOKE_PW:-}" ]; then diff --git a/scripts/staging-smoke/README.md b/scripts/staging-smoke/README.md index 076c2763c6..4778af52b1 100644 --- a/scripts/staging-smoke/README.md +++ b/scripts/staging-smoke/README.md @@ -5,8 +5,11 @@ after the staging rollout, and its pass/fail feeds the `staging-smoke` commit st `deploy-dev.yml` (prod) requires on the same SHA. Checks: `/api/health/ready` 200 · seeded non-admin login · `/v2` home + a pod + `/v2/connectors` -+ Bring-your-own render with no console errors · left rail no horizontal scroll at 1440 & 390px. -The hosted-agent-answers check is skipped until staging has an LLM key (`SMOKE_HOSTED=1` to enable). ++ Bring-your-own render with no console errors · left rail no horizontal scroll at 1440 & 390px · +**a FRESH no-grants user sees at least one connectable tool as a visible row on `/v2/connectors`, +not buried in `.v2-connectors__inactive-fold`** (guards the #2504 fold regression — never-connected +offers must not be hidden from new users). The hosted-agent-answers check is skipped until staging +has an LLM key (`SMOKE_HOSTED=1` to enable). ## Run locally ``` @@ -17,6 +20,7 @@ SMOKE_EMAIL=... SMOKE_PW=... npm run smoke ## Env - `SMOKE_BASE` (default https://staging.commonly.me), `SMOKE_API` (default https://api-staging.commonly.me) - `SMOKE_EMAIL` / `SMOKE_PW` — a **non-admin** staging test user (in CI: secrets `STAGING_SMOKE_EMAIL` / `STAGING_SMOKE_PW`; locally a 0600 file, never printed) +- `SMOKE_FRESH_EMAIL` / `SMOKE_FRESH_PW` — a **separate FRESH no-grants** account for the connectors-fold check, kept grant-free so dead grants can't mask the bug (in CI: secrets `STAGING_SMOKE_FRESH_EMAIL` / `STAGING_SMOKE_FRESH_PW`). If unset, that one check is skipped. - `SMOKE_HOSTED=1` — also check a hosted-agent reply (needs a staging LLM key) Exit 0 = all green; non-zero = a blocking check failed → prod promotion must not proceed. diff --git a/scripts/staging-smoke/staging-smoke.mjs b/scripts/staging-smoke/staging-smoke.mjs index 36f2495f2a..4d7222331b 100644 --- a/scripts/staging-smoke/staging-smoke.mjs +++ b/scripts/staging-smoke/staging-smoke.mjs @@ -119,7 +119,38 @@ async function main() { await page.close(); } - // 6) hosted agent answers (skip until staging has an LLM key) + // 6) connectors fold — guards the #2504 regression: a FRESH no-grants user must see at least one + // connectable tool as a visible row, NOT buried in .v2-connectors__inactive-fold. Uses a + // SEPARATE fresh account (kept grant-free) in its own context, so dead grants can't mask the bug. + const FRESH_EMAIL = process.env.SMOKE_FRESH_EMAIL, FRESH_PW = process.env.SMOKE_FRESH_PW; + if (FRESH_EMAIL && FRESH_PW) { + try { + const { status, body } = await api('/api/auth/login', { method:'POST', headers:{'Content-Type':'application/json'}, body: JSON.stringify({ email: FRESH_EMAIL, password: FRESH_PW }) }); + const ftoken = body?.token || body?.accessToken || null; + if (!ftoken) { rec('connectors: fresh user sees tools outside the fold', false, `fresh login failed (status ${status})`); } + else { + const fctx = await browser.newContext({ viewport: { width: 1440, height: 900 } }); + await fctx.addInitScript(t => { try { localStorage.setItem('token', t); } catch(e){} }, ftoken); + const page = await fctx.newPage(); + await page.goto(BASE+'/v2/connectors', { waitUntil:'networkidle', timeout: 30000 }); + await page.waitForTimeout(1800); + const r = await page.evaluate(() => { + const TOOLS = ['GitHub','Linear','Slack','Notion','Google','Jira','Atlassian','Gmail']; + const folds = [...document.querySelectorAll('.v2-connectors__inactive-fold')]; + const inFold = el => folds.some(f => f.contains(el)); + const all = [...document.querySelectorAll('.v2-connectors__main *, .v2-tools *')]; + const visible = TOOLS.filter(name => all.some(el => el.offsetParent !== null && !inFold(el) && (el.textContent||'').includes(name) && el.children.length <= 4)); + return { visible, foldCount: folds.length }; + }); + rec('connectors: fresh user sees tools outside the fold', r.visible.length >= 1, `visible=[${r.visible.join(',')}] inactive-folds=${r.foldCount}`); + await fctx.close(); + } + } catch (e) { rec('connectors: fresh user sees tools outside the fold', false, e.message.slice(0,100)); } + } else { + console.log('SKIP connectors fresh-user check — SMOKE_FRESH_EMAIL/PW not set'); + } + + // 7) hosted agent answers (skip until staging has an LLM key) if (CHECK_HOSTED) rec('hosted agent answers', false, 'TODO once staging LLM key is set'); else console.log('SKIP hosted agent answers — staging has no LLM key yet (SMOKE_HOSTED!=1)'); From 51bdb5815a6d8ae8fda78015ba391a1d7b44e171 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:27:16 -0700 Subject: [PATCH 2/3] test(staging-smoke): close two fail-open defects in the connectors-fold check (Otto) (1) Fail CLOSED on missing fresh creds: absent SMOKE_FRESH_EMAIL/PW now FAILS the check (was SKIP + exit 0, so the gate went green without running it). deploy-staging.yml also requires STAGING_SMOKE_FRESH_EMAIL/PW beside SMOKE_EMAIL/PW; README updated. (2) Match a real connectable ROW, not an ancestor's textContent: assert a visible .v2-connector-row that has a .v2-connector-row__action and names a known tool and is NOT contained by .v2-connectors__inactive-fold. (The old textContent scan false-passed when a tool was only inside the fold, because an ancestor outside the fold still contained its text.) Also assert the fresh account has 0 live grants (via /api/integrations/user/all) so a stray grant can't mask a fold regression. Validated on 7fce31f7: with creds GREEN 10/10 (0 grants; visible=[Linear,Jira], foldedRows=0); without fresh creds RED exit 1 (fail-closed). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01V7T2fgquLRaqaYiGLC6Dey --- .github/workflows/deploy-staging.yml | 4 +-- scripts/staging-smoke/README.md | 2 +- scripts/staging-smoke/staging-smoke.mjs | 40 ++++++++++++++++--------- 3 files changed, 29 insertions(+), 17 deletions(-) diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index 4a818f54b5..ca1911bee0 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -208,8 +208,8 @@ jobs: SMOKE_FRESH_PW: ${{ secrets.STAGING_SMOKE_FRESH_PW }} run: | set -euo pipefail - if [ -z "${SMOKE_EMAIL:-}" ] || [ -z "${SMOKE_PW:-}" ]; then - echo "::error title=Tier-1 smoke::STAGING_SMOKE_EMAIL / STAGING_SMOKE_PW repo secrets are not set" + if [ -z "${SMOKE_EMAIL:-}" ] || [ -z "${SMOKE_PW:-}" ] || [ -z "${SMOKE_FRESH_EMAIL:-}" ] || [ -z "${SMOKE_FRESH_PW:-}" ]; then + echo "::error title=Tier-1 smoke::STAGING_SMOKE_EMAIL/PW and STAGING_SMOKE_FRESH_EMAIL/PW repo secrets must all be set (the connectors-fold check fails closed without the fresh account)" exit 1 fi cd scripts/staging-smoke diff --git a/scripts/staging-smoke/README.md b/scripts/staging-smoke/README.md index 4778af52b1..d7ee55c560 100644 --- a/scripts/staging-smoke/README.md +++ b/scripts/staging-smoke/README.md @@ -20,7 +20,7 @@ SMOKE_EMAIL=... SMOKE_PW=... npm run smoke ## Env - `SMOKE_BASE` (default https://staging.commonly.me), `SMOKE_API` (default https://api-staging.commonly.me) - `SMOKE_EMAIL` / `SMOKE_PW` — a **non-admin** staging test user (in CI: secrets `STAGING_SMOKE_EMAIL` / `STAGING_SMOKE_PW`; locally a 0600 file, never printed) -- `SMOKE_FRESH_EMAIL` / `SMOKE_FRESH_PW` — a **separate FRESH no-grants** account for the connectors-fold check, kept grant-free so dead grants can't mask the bug (in CI: secrets `STAGING_SMOKE_FRESH_EMAIL` / `STAGING_SMOKE_FRESH_PW`). If unset, that one check is skipped. +- `SMOKE_FRESH_EMAIL` / `SMOKE_FRESH_PW` — a **separate FRESH no-grants** account for the connectors-fold check, kept grant-free so dead grants can't mask the bug (in CI: secrets `STAGING_SMOKE_FRESH_EMAIL` / `STAGING_SMOKE_FRESH_PW`, which `deploy-staging.yml` requires). **Required — if unset the check FAILS, never skips green.** The check first asserts the account has 0 live grants, then that at least one connectable tool renders as a real row outside the inactive fold. - `SMOKE_HOSTED=1` — also check a hosted-agent reply (needs a staging LLM key) Exit 0 = all green; non-zero = a blocking check failed → prod promotion must not proceed. diff --git a/scripts/staging-smoke/staging-smoke.mjs b/scripts/staging-smoke/staging-smoke.mjs index 4d7222331b..1f7b197d1d 100644 --- a/scripts/staging-smoke/staging-smoke.mjs +++ b/scripts/staging-smoke/staging-smoke.mjs @@ -119,16 +119,27 @@ async function main() { await page.close(); } - // 6) connectors fold — guards the #2504 regression: a FRESH no-grants user must see at least one - // connectable tool as a visible row, NOT buried in .v2-connectors__inactive-fold. Uses a - // SEPARATE fresh account (kept grant-free) in its own context, so dead grants can't mask the bug. + // 6) connectors fold — guards the #2504 regression. A FRESH, grant-free account must see at least + // one connectable TOOL as a real row outside the inactive fold. Fail-CLOSED: absent creds, a + // failed login, or a non-grant-free account all FAIL (never skip-green). We match an actual + // .v2-connector-row that has a .v2-connector-row__action (Connect) and is NOT contained by + // .v2-connectors__inactive-fold — NOT an ancestor's textContent (that false-passed when GitHub + // was only inside the fold). const FRESH_EMAIL = process.env.SMOKE_FRESH_EMAIL, FRESH_PW = process.env.SMOKE_FRESH_PW; - if (FRESH_EMAIL && FRESH_PW) { + if (!FRESH_EMAIL || !FRESH_PW) { + rec('connectors: fresh user sees a tool row outside the fold', false, 'SMOKE_FRESH_EMAIL/PW not set — fresh-account check could not run (fail-closed)'); + } else { try { const { status, body } = await api('/api/auth/login', { method:'POST', headers:{'Content-Type':'application/json'}, body: JSON.stringify({ email: FRESH_EMAIL, password: FRESH_PW }) }); const ftoken = body?.token || body?.accessToken || null; - if (!ftoken) { rec('connectors: fresh user sees tools outside the fold', false, `fresh login failed (status ${status})`); } + if (!ftoken) { rec('connectors: fresh user sees a tool row outside the fold', false, `fresh login failed (status ${status})`); } else { + // 6a) the account must be genuinely grant-free, else a stray grant could mask a fold regression + const g = await api('/api/integrations/user/all', { headers: { Authorization: `Bearer ${ftoken}` } }); + const arr = Array.isArray(g.body) ? g.body : null; + const live = (arr || []).filter(it => it && (it.connected === true || it.isConnected === true || ['connected','active','granted'].includes(it.status))); + rec('connectors: fresh account has 0 live grants', arr !== null && arr.length === 0, arr === null ? 'unexpected grants shape' : `integrations=${arr.length} connected=${live.length}`); + // 6b) at least one connectable TOOL row visible OUTSIDE the inactive fold const fctx = await browser.newContext({ viewport: { width: 1440, height: 900 } }); await fctx.addInitScript(t => { try { localStorage.setItem('token', t); } catch(e){} }, ftoken); const page = await fctx.newPage(); @@ -136,18 +147,19 @@ async function main() { await page.waitForTimeout(1800); const r = await page.evaluate(() => { const TOOLS = ['GitHub','Linear','Slack','Notion','Google','Jira','Atlassian','Gmail']; - const folds = [...document.querySelectorAll('.v2-connectors__inactive-fold')]; - const inFold = el => folds.some(f => f.contains(el)); - const all = [...document.querySelectorAll('.v2-connectors__main *, .v2-tools *')]; - const visible = TOOLS.filter(name => all.some(el => el.offsetParent !== null && !inFold(el) && (el.textContent||'').includes(name) && el.children.length <= 4)); - return { visible, foldCount: folds.length }; + const fold = document.querySelector('.v2-connectors__inactive-fold'); + const inFold = el => !!fold && fold.contains(el); + const rows = [...document.querySelectorAll('.v2-connector-row')]; + const visibleToolRows = rows + .filter(row => row.offsetParent !== null && !inFold(row) && row.querySelector('.v2-connector-row__action')) + .map(row => TOOLS.find(t => (row.textContent || '').includes(t))) + .filter(Boolean); + return { visibleToolRows, rowsTotal: rows.length, foldedRows: fold ? fold.querySelectorAll('.v2-connector-row').length : 0 }; }); - rec('connectors: fresh user sees tools outside the fold', r.visible.length >= 1, `visible=[${r.visible.join(',')}] inactive-folds=${r.foldCount}`); + rec('connectors: fresh user sees a tool row outside the fold', r.visibleToolRows.length >= 1, `visible=[${[...new Set(r.visibleToolRows)].join(',')}] rows=${r.rowsTotal} foldedRows=${r.foldedRows}`); await fctx.close(); } - } catch (e) { rec('connectors: fresh user sees tools outside the fold', false, e.message.slice(0,100)); } - } else { - console.log('SKIP connectors fresh-user check — SMOKE_FRESH_EMAIL/PW not set'); + } catch (e) { rec('connectors: fresh user sees a tool row outside the fold', false, e.message.slice(0,100)); } } // 7) hosted agent answers (skip until staging has an LLM key) From b49b6758e2362983a9156acf9f5f3a3bbcd291e8 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:38:49 -0700 Subject: [PATCH 3/3] test(staging-smoke): make the connectors-fold check exact (Otto round 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (1) 0-grants now checks TOOL grants at /api/pods/:podId/grants summed across the fresh account's pods (not the Integration list /api/integrations/user/all, which a live tool grant would slip past). (2) Assert res.ok on every grants fetch — a 503 with body [] no longer passes. (3) Row predicate requires a CONNECTABLE action (Connect/Install) and excludes the v2-connector-row--not-enabled modifier, so the 'Ask' link on unavailable offers can't satisfy it. Validated on 7fce31f7: with creds GREEN 10/10 (pods=0 tool-grants=0; visible=[Linear,Jira, Airtable,Sentry] foldedRows=0); without fresh creds RED exit 1 (fail-closed). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01V7T2fgquLRaqaYiGLC6Dey --- scripts/staging-smoke/staging-smoke.mjs | 40 ++++++++++++++++++------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/scripts/staging-smoke/staging-smoke.mjs b/scripts/staging-smoke/staging-smoke.mjs index 1f7b197d1d..3c3bcd97d9 100644 --- a/scripts/staging-smoke/staging-smoke.mjs +++ b/scripts/staging-smoke/staging-smoke.mjs @@ -134,26 +134,44 @@ async function main() { const ftoken = body?.token || body?.accessToken || null; if (!ftoken) { rec('connectors: fresh user sees a tool row outside the fold', false, `fresh login failed (status ${status})`); } else { - // 6a) the account must be genuinely grant-free, else a stray grant could mask a fold regression - const g = await api('/api/integrations/user/all', { headers: { Authorization: `Bearer ${ftoken}` } }); - const arr = Array.isArray(g.body) ? g.body : null; - const live = (arr || []).filter(it => it && (it.connected === true || it.isConnected === true || ['connected','active','granted'].includes(it.status))); - rec('connectors: fresh account has 0 live grants', arr !== null && arr.length === 0, arr === null ? 'unexpected grants shape' : `integrations=${arr.length} connected=${live.length}`); - // 6b) at least one connectable TOOL row visible OUTSIDE the inactive fold + // 6a) 0 live TOOL grants. Tool grants live per-pod at /api/pods/:podId/grants (NOT the + // Integration list), so sum .grants across the fresh account's pods. res.ok is asserted + // on every fetch — a 503 whose body happens to be [] must FAIL, not pass. + const pr = await api('/api/pods', { headers: { Authorization: `Bearer ${ftoken}` } }); + if (pr.status !== 200) { + rec('connectors: fresh account has 0 live grants', false, `GET /api/pods -> ${pr.status}`); + } else { + const pods = Array.isArray(pr.body) ? pr.body : (pr.body?.pods || []); + let totalGrants = 0, bad = null; + for (const pod of pods) { + const id = pod._id || pod.id; + const gr = await api(`/api/pods/${id}/grants`, { headers: { Authorization: `Bearer ${ftoken}` } }); + if (gr.status !== 200) { bad = `grants ${id} -> ${gr.status}`; break; } + totalGrants += Array.isArray(gr.body?.grants) ? gr.body.grants.length : 0; + } + rec('connectors: fresh account has 0 live grants', bad === null && totalGrants === 0, bad || `pods=${pods.length} tool-grants=${totalGrants}`); + } + // 6b) at least one connectable TOOL row visible OUTSIDE the inactive fold. A real + // .v2-connector-row (never ancestor text) whose action is Connect/Install (not "Ask" + // or "Add") and that is not the unavailable --not-enabled modifier. const fctx = await browser.newContext({ viewport: { width: 1440, height: 900 } }); await fctx.addInitScript(t => { try { localStorage.setItem('token', t); } catch(e){} }, ftoken); const page = await fctx.newPage(); await page.goto(BASE+'/v2/connectors', { waitUntil:'networkidle', timeout: 30000 }); await page.waitForTimeout(1800); const r = await page.evaluate(() => { - const TOOLS = ['GitHub','Linear','Slack','Notion','Google','Jira','Atlassian','Gmail']; + const TOOLS = ['GitHub','Linear','Slack','Notion','Google','Jira','Atlassian','Gmail','Airtable','Sentry']; const fold = document.querySelector('.v2-connectors__inactive-fold'); const inFold = el => !!fold && fold.contains(el); const rows = [...document.querySelectorAll('.v2-connector-row')]; - const visibleToolRows = rows - .filter(row => row.offsetParent !== null && !inFold(row) && row.querySelector('.v2-connector-row__action')) - .map(row => TOOLS.find(t => (row.textContent || '').includes(t))) - .filter(Boolean); + const visibleToolRows = rows.filter(row => { + if (row.offsetParent === null) return false; // visible + if (inFold(row)) return false; // not in the inactive fold + if (row.classList.contains('v2-connector-row--not-enabled')) return false; // exclude unavailable ("Ask") offers + const action = (row.querySelector('.v2-connector-row__action')?.textContent || '').trim(); + if (!/\b(connect|install)\b/i.test(action)) return false; // require a connectable action + return TOOLS.some(t => (row.textContent || '').includes(t)); // names a known tool + }).map(row => TOOLS.find(t => (row.textContent || '').includes(t))).filter(Boolean); return { visibleToolRows, rowsTotal: rows.length, foldedRows: fold ? fold.querySelectorAll('.v2-connector-row').length : 0 }; }); rec('connectors: fresh user sees a tool row outside the fold', r.visibleToolRows.length >= 1, `visible=[${[...new Set(r.visibleToolRows)].join(',')}] rows=${r.rowsTotal} foldedRows=${r.foldedRows}`);