diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c43095..736ba4e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,8 @@ on: schedule: - cron: '0 1 1-31/2 * *' +permissions: + contents: read jobs: diff --git a/.github/workflows/dockerhubpublish.yml b/.github/workflows/dockerhubpublish.yml index 0962b72..ff3398e 100644 --- a/.github/workflows/dockerhubpublish.yml +++ b/.github/workflows/dockerhubpublish.yml @@ -1,52 +1,73 @@ name: build&publishtoDockerHub -on: - push: - tags: - - '[0-9]+.[0-9]+.[0-9]+(.*)' +# Normally driven by graphistry/graphistry CI (.github/workflows/base.yml jobs +# push_gak_tag + trigger-gak-build): it tags this repo with the graphistry VERSION, +# then dispatches this workflow on master with cuda_versions '["13", "12"]' and +# polls the run (30 min budget) — so the workflow name and master ref must stay. +# Manual flow: tag on master, then dispatch from master; the image version comes +# from the latest tag (`git describe --tags`). +# (A tag-push trigger existed but its pattern never matched a real tag, and a tag +# path would let any tag creator reach the DockerHub token, so it was removed.) +on: workflow_dispatch: inputs: version: description: 'Version to bump to' required: false cuda_versions: - description: 'JSON array of CUDA versions' + description: 'JSON array of CUDA versions, matching graphistry-forge-base tag suffixes (e.g. "13", "12")' required: false - default: '["12.8", "11.8"]' + default: '["13", "12"]' workflow_call: +permissions: {} + jobs: docker-build-publish: + # Pushes public images (including :latest-), so only master may publish; + # dispatch from any other ref is skipped. The DockerHub token lives on the protected + # `dockerhub-publish` environment (master only + required reviewers; see DEVELOP.md). + if: github.ref == 'refs/heads/master' runs-on: ubuntu-latest-4-cores + environment: dockerhub-publish + permissions: + contents: read strategy: matrix: - CUDA_SHORT_VERSION: ${{ fromJSON(inputs.cuda_versions || '["12.8", "11.8"]') }} + CUDA_SHORT_VERSION: ${{ fromJSON(inputs.cuda_versions || '["13", "12"]') }} fail-fast: true steps: - name: checkout repo - uses: actions/checkout@v3 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 + persist-credentials: false - name: get most recent tag run: | - echo "RELEASE_VERSION=$(git describe --tags --abbrev=0)" >> $GITHUB_ENV - echo "CUDA_SHORT_VERSION=${{ matrix.CUDA_SHORT_VERSION }}" >> $GITHUB_ENV + # Values flow into $GITHUB_ENV and image tags: reject anything but N or N.N + # (a newline in the free-form dispatch input would inject env vars). + if ! [[ "$MATRIX_CUDA_SHORT_VERSION" =~ ^[0-9]+(\.[0-9]+)?$ ]]; then + echo "::error::Invalid CUDA version in cuda_versions input (expected N or N.N, e.g. 13 or 12.8)" + exit 1 + fi + echo "RELEASE_VERSION=$(git describe --tags --abbrev=0)" >> "$GITHUB_ENV" + echo "CUDA_SHORT_VERSION=${MATRIX_CUDA_SHORT_VERSION}" >> "$GITHUB_ENV" + env: + MATRIX_CUDA_SHORT_VERSION: ${{ matrix.CUDA_SHORT_VERSION }} - name: check env run: | - echo $RELEASE_VERSION - echo ${{ env.RELEASE_VERSION }} - echo $CUDA_SHORT_VERSION - echo ${{ env.CUDA_SHORT_VERSION }} + echo "$RELEASE_VERSION" + echo "$CUDA_SHORT_VERSION" - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -58,12 +79,12 @@ jobs: run: | cd src/docker \ - && GRAPHISTRY_FORGE_BASE_VERSION=v${{ env.RELEASE_VERSION }}-${{ env.CUDA_SHORT_VERSION }} docker compose -f docker-compose.yml build + && GRAPHISTRY_FORGE_BASE_VERSION="v${RELEASE_VERSION}-${CUDA_SHORT_VERSION}" docker compose -f docker-compose.yml build - name: tag the image run: | - docker tag graphistry/graph-app-kit-st:latest-${{ env.CUDA_SHORT_VERSION }} graphistry/graph-app-kit-st:v${{ env.RELEASE_VERSION }}-${{ env.CUDA_SHORT_VERSION }} + docker tag "graphistry/graph-app-kit-st:latest-${CUDA_SHORT_VERSION}" "graphistry/graph-app-kit-st:v${RELEASE_VERSION}-${CUDA_SHORT_VERSION}" - name: Publish Graph-App-Kit to DockerHub run: | - docker push graphistry/graph-app-kit-st:v${{ env.RELEASE_VERSION }}-${{ env.CUDA_SHORT_VERSION }} && docker push graphistry/graph-app-kit-st:latest-${{ env.CUDA_SHORT_VERSION }} + docker push "graphistry/graph-app-kit-st:v${RELEASE_VERSION}-${CUDA_SHORT_VERSION}" && docker push "graphistry/graph-app-kit-st:latest-${CUDA_SHORT_VERSION}" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 54c5a3b..7a79bb4 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,54 +1,75 @@ name: Publish +# Uploads the CloudFormation templates that customers deploy (README "Launch Stack") +# to the public bucket under templates/latest/. +# +# Only master may publish, and only through the protected `publish-templates` +# environment (required reviewers + deployment branch policy: master). AWS access +# is a short-lived GitHub OIDC session whose IAM role trusts only that environment +# and may only write templates/latest/*. Setup: DEVELOP.md "AWS Publish action". + on: #Merge push: branches: - master + paths: + - 'src/bootstraps/core/*.yml' + - 'src/bootstraps/neptune/*.yml' + - '.github/workflows/publish.yml' - #Label - pull_request: - types: [labeled] - - - #Enable UI-driven branch testing + #Manual republish (master only; other refs are skipped) workflow_dispatch: - #Test main bidaily @ 1a - schedule: - - cron: '0 1 1-31/2 * *' +permissions: {} jobs: - stub_mt: - name: Always-succeed step to prevent appearing as a failure - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - name: Do nothing - run: echo "Do nothing" - publish: - name: Upload to Amazon S3 - if: github.event.action != 'labeled' || github.event.label.name == 'publish' + name: Upload CloudFormation templates to S3 + if: github.ref == 'refs/heads/master' runs-on: ubuntu-latest + timeout-minutes: 15 + # Job-level and keyed by ref so a skipped non-master dispatch never joins + # (and cancels a pending run in) the master publish queue. + concurrency: + group: publish-templates-${{ github.ref }} + cancel-in-progress: false + environment: + name: publish-templates + url: https://graph-app-kit-repo-public.s3.us-east-2.amazonaws.com/templates/latest/core/graphistry.yml + permissions: + contents: read + id-token: write + env: + AWS_REGION: us-east-2 + S3_BUCKET: graph-app-kit-repo-public + AWS_ROLE_ARN: ${{ vars.AWS_S3_PUBLISH_ROLE_ARN }} steps: + - name: Require OIDC role configuration + run: | + if [ -z "$AWS_ROLE_ARN" ]; then + echo "::error::Environment variable AWS_S3_PUBLISH_ROLE_ARN is not set on the publish-templates environment" + exit 1 + fi + - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 + - name: Configure AWS credentials (OIDC) + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0 with: - aws-access-key-id: ${{ secrets.S3_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.S3_AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.S3_AWS_REGION_PUBLIC }} + role-to-assume: ${{ env.AWS_ROLE_ARN }} + role-session-name: graph-app-kit-publish-${{ github.run_id }} + aws-region: ${{ env.AWS_REGION }} - name: Copy CloudFormation templates run: | - aws s3 cp src/bootstraps/core/graphistry.yml "s3://${{ secrets.AWS_S3_BUCKET_PUBLIC}}/templates/latest/core/graphistry.yml" - aws s3 cp src/bootstraps/core/minimal.yml "s3://${{ secrets.AWS_S3_BUCKET_PUBLIC}}/templates/latest/core/minimal.yml" - aws s3 cp src/bootstraps/neptune/graphistry.yml "s3://${{ secrets.AWS_S3_BUCKET_PUBLIC}}/templates/latest/neptune/graphistry.yml" - aws s3 cp src/bootstraps/neptune/minimal.yml "s3://${{ secrets.AWS_S3_BUCKET_PUBLIC}}/templates/latest/neptune/minimal.yml" - + set -euo pipefail + for template in core/graphistry.yml core/minimal.yml neptune/graphistry.yml neptune/minimal.yml; do + aws s3 cp "src/bootstraps/${template}" "s3://${S3_BUCKET}/templates/latest/${template}" + done diff --git a/.github/workflows/scan.yml b/.github/workflows/scan.yml index 5f78e34..0ebaf69 100644 --- a/.github/workflows/scan.yml +++ b/.github/workflows/scan.yml @@ -7,6 +7,9 @@ on: schedule: - cron: '0 4 * * *' +permissions: + contents: read + jobs: cancel_outstanding: @@ -14,6 +17,8 @@ jobs: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip-security-scan') }} runs-on: ubuntu-latest timeout-minutes: 10 + permissions: + actions: write steps: - name: Cancel Previous Runs if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip-cancel') }} @@ -26,6 +31,9 @@ jobs: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip-security-scan') }} runs-on: ubuntu-latest-4-cores timeout-minutes: 30 + permissions: + contents: read + security-events: write env: COMPOSE_DOCKER_CLI_BUILD: 1 DOCKER_BUILDKIT: 1 @@ -89,6 +97,9 @@ jobs: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip-security-scan') }} runs-on: ubuntu-latest-4-cores timeout-minutes: 30 + permissions: + contents: read + security-events: write env: COMPOSE_DOCKER_CLI_BUILD: 1 DOCKER_BUILDKIT: 1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 61f51bd..46e707e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,18 @@ Extensions: See [projects page](https://github.com/graphistry/graph-app-kit/projects) and [open pull requests](https://github.com/graphistry/graph-app-kit/pulls) +### Security + +* Publish workflow: CloudFormation templates (public S3 `templates/latest/`) publish only from `master`, via the `publish-templates` environment (required reviewers) and a GitHub OIDC role scoped to `templates/latest/*`; the PR `publish` label and schedule no longer publish +* DockerHub workflow: publish only by dispatch from `master`, via the `dockerhub-publish` environment; drop the tag-push trigger (its pattern never matched); validate `cuda_versions` entries (`N` or `N.N`) and default them to `["13", "12"]` to match what graphistry CI dispatches and the current `graphistry-forge-base` tags +* Requires one-time repo/AWS setup before merge, plus post-merge cleanup (retire the old AWS key and DockerHub token, set the default token to read-only): see `DEVELOP.md` "One-time setup" + +### Infra + +* Publish workflows: `permissions: {}` with per-job grants, actions pinned to commit SHAs (`actions/checkout` v7.0.1, `aws-actions/configure-aws-credentials` v6.3.0, `docker/login-action` v4.6.0), `persist-credentials: false`, no `${{ }}` interpolation in `run:` scripts +* Template publish runs only when templates or the workflow change on `master` (or on manual dispatch) +* CI and Scan workflows declare explicit least-privilege `permissions:` so the repo default token can be set to read-only + ### Dev * Add agent skills under `.agents/skills/`: `plan` (file-based task plans) and `review` (multi-wave PR review), adapted for this repo's Python views, GitHub Actions publish workflows, and CloudFormation templates. Add `AGENTS.md`; gitignore `plans/` diff --git a/DEVELOP.md b/DEVELOP.md index f14c870..b1467d4 100644 --- a/DEVELOP.md +++ b/DEVELOP.md @@ -49,30 +49,93 @@ For faster AWS launches and Graphistry Enterprise, we: * Paste into `src/bootstraps/core,neptune/graphistry.yml` * Update `src/docker/docker-compose.yml::GRAPHISTRY_FORGE_BASE_VERSION` -## DockerHub Automated Builds +## DockerHub publish action -Managed via DockerHub automates tagged builds +`.github/workflows/dockerhubpublish.yml` builds and pushes `graphistry/graph-app-kit-st` as `v-` and `latest-`, where `` is a `graphistry-forge-base` tag suffix (currently `13` and `12`). -Ahead of time: +**Normal path: driven by graphistry CI.** graphistry/graphistry's `base.yml` (`push_gak_tag` + `trigger-gak-build` jobs, on a `ci`-labeled PR, dispatch, or `workflow_call`) tags this repo with the graphistry `VERSION`, then dispatches this workflow on `master` with `cuda_versions '["13", "12"]'` using the PAT in graphistry's `GRAPHISTRY_GHA_CI_GAK` secret, and polls the run for up to 30 minutes. graphistry's release then builds its `graph-app-kit-*` app images on top of the published `v-` image. Keep the workflow name `build&publishtoDockerHub`, the `workflow_dispatch` trigger, and master as the dispatch ref, or that chain breaks; anything that holds a dispatched run (for example a required reviewer) must clear within that 30-minute budget. + +**Manual path** (rebuild or fallback). Ahead of time: -* Ensure you've set `GRAPHISTRY_FORGE_BASE_VERSION` in the `Dockerfile` (not just the `docker-compose.yml`) * Merged into master +* The workflow builds on `graphistry/graphistry-forge-base:v-` (it overrides the `GRAPHISTRY_FORGE_BASE_VERSION` default from the `Dockerfile`/`docker-compose.yml`), so the release tag must match a forge-base tag that exists for every CUDA version you publish. Suffixed tags such as `2.39.12.1` only work if a matching base tag exists. Publish: -1. `git tag 2.39.12` - * Use a tag that corresponds to the Graphistry version, or some suffix (`2.39.12.1`) - * Note lack of `v` -2. `git push --tags` +1. `git tag 2.39.12 && git push origin 2.39.12` from master + * Use the tag of the Graphistry version you are building on + * Note lack of `v`: the image is published as `v2.39.12-` +2. In GitHub Actions, run `build&publishtoDockerHub` from **master** (optionally pass `cuda_versions`, e.g. `["13"]`; each value must be `N` or `N.N`). The version comes from the latest tag. +3. If the `dockerhub-publish` environment has required reviewers, one approves the run -DockerHub automatic builds will: -* publish as tag `v2.39.12-11.4`: note addition of `v` and `-11.4` -* publish as tag `latest` +Dispatches from any other branch or tag are skipped. Pushing a tag no longer publishes on its own. See [current tags](https://hub.docker.com/r/graphistry/graph-app-kit-st/tags) and available [base tags](https://hub.docker.com/r/graphistry/graphistry-forge-base/tags) ## AWS Publish action -* Docker rebuilds on merge to main -* Push main tag ('v1.2.3') for building named versions -* CloudFormation templates uploaded to S3 upon a PR being labeled "publish", merge-to-main, or explicit GHA call +`.github/workflows/publish.yml` uploads the CloudFormation templates (`src/bootstraps/{core,neptune}/{graphistry,minimal}.yml`) to `s3://graph-app-kit-repo-public/templates/latest/`. The README "Launch Stack" buttons point at that path, so it is what customers deploy. + +* Runs on merge to `master` when a template (or the workflow) changes, or by manual dispatch **from master**. Other refs are skipped, and PR labels no longer publish. +* Every run waits in the `publish-templates` environment for a required reviewer to approve. +* AWS access uses GitHub OIDC. No AWS keys are stored in GitHub. + +To test template changes before merge, upload your branch's template by hand to CloudFormation (see "Test / AWS" above). Don't publish it to `templates/latest/`. + +### One-time setup (repo admin + AWS admin) + +**Order matters.** GitHub auto-creates a missing environment with *no* protection the first time any workflow, on any branch, references it. So lock both environments down before the IAM role exists and before merging the workflow change. Merging also triggers a publish run immediately, because the change touches `publish.yml`. + +1. **Create and lock the GitHub environments** (Settings → Environments): + * `publish-templates`: required reviewers = release maintainers, enable "Prevent self-review", **uncheck** "Allow administrators to bypass configured protection rules"; deployment branches and tags = "Selected branches and tags" with a single **branch** rule `master` (no tag rules). + * `dockerhub-publish`: the same settings. + * Required reviewers must include at least two people. With self-review blocked, the person who merges or dispatches cannot approve their own run. +2. **Close the path to master that skips review** (Settings → Actions → General → Workflow permissions): + * **Uncheck** "Allow GitHub Actions to create and approve pull requests". Otherwise any writer's workflow can approve their own PR with `GITHUB_TOKEN` and merge to master without human review, which defeats the master-only gate. Consider also enabling "Require approval of the most recent reviewable push" and "Do not allow bypassing the above settings" on master. + * Any token stored as a repo-level secret can be read by a workflow on any branch. Delete repo-level secrets that no workflow in this repo references, or move them to a protected environment. +3. **AWS OIDC provider**: in the bucket's AWS account, add the IAM identity provider `token.actions.githubusercontent.com` with audience `sts.amazonaws.com` (skip this if it already exists). +4. **IAM role** (for example `graph-app-kit-publish-templates`): + * Trust policy, limited to this repo's `publish-templates` environment: + + ```json + { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": { "Federated": "arn:aws:iam:::oidc-provider/token.actions.githubusercontent.com" }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "token.actions.githubusercontent.com:aud": "sts.amazonaws.com", + "token.actions.githubusercontent.com:sub": "repo:graphistry/graph-app-kit:environment:publish-templates" + } + } + }] + } + ``` + + Optional hardening, so the role no longer depends only on the environment's settings: if your AWS account supports STS condition keys for GitHub-specific claims, also require `"token.actions.githubusercontent.com:ref": "refs/heads/master"` and `"token.actions.githubusercontent.com:job_workflow_ref": "graphistry/graph-app-kit/.github/workflows/publish.yml@refs/heads/master"`. Check that the IAM policy editor accepts these keys. A condition on a claim STS doesn't provide fails closed, which shows up in step 6 as an AssumeRole error. + * Permissions policy, limited to the template path: + + ```json + { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": "s3:PutObject", + "Resource": "arn:aws:s3:::graph-app-kit-repo-public/templates/latest/*" + }] + } + ``` + +5. **Wire up the environments**: + * `publish-templates`: add the environment variable `AWS_S3_PUBLISH_ROLE_ARN` = the role ARN from step 4. Until it is set, publish runs fail closed at "Require OIDC role configuration". + * `dockerhub-publish`: create a **new** DockerHub access token and store environment secrets `DOCKERHUB_USERNAME` and `DOCKERHUB_TOKEN` (the new token). Any branch could reach the old token before this change, so don't move it; replace it. +6. **Merge, then verify**: the merge (or a manual dispatch from master) starts Publish. Approve it, check that the four `templates/latest/{core,neptune}/{graphistry,minimal}.yml` objects have a new LastModified time, and confirm the job log shows the assumed role. Verify the DockerHub path at the next real release (tag first, then dispatch). A test dispatch republishes the latest tag's `v-` and `latest-` images from current master. +7. **Retire the old credentials** (once the template publish in step 6 succeeds; do not wait for a DockerHub release): + * In IAM, check the "last used" info for the access key behind `S3_AWS_ACCESS_KEY_ID` to make sure nothing else uses it. Then deactivate and delete it. + * Delete the repo-level secrets `S3_AWS_ACCESS_KEY_ID`, `S3_AWS_SECRET_ACCESS_KEY`, `S3_AWS_REGION_PUBLIC`, `AWS_S3_BUCKET_PUBLIC`, `DOCKERHUB_TOKEN` and `DOCKERHUB_USERNAME`. + * Set Settings → Actions → General → Workflow permissions to "Read repository contents and packages permissions". Do this only once every workflow on master declares its own `permissions:`, which is true after this change merges. + * Revoke the old DockerHub token. Environment secrets take precedence over repo secrets with the same name, so the next real release, run on the new `dockerhub-publish` token, is the check that it works. If it fails, fix the environment secrets. + +Templates are no longer republished automatically on a schedule. If `templates/latest/` drifts, for example after a manual bucket edit, dispatch Publish from master.