diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 000000000..2688d5e37 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,135 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: release + +on: + workflow_dispatch: + inputs: + tag: + description: 'Image tag (e.g. v1.2.3-rc1). Leave blank to auto-generate from branch+SHA.' + required: false + create_release: + description: 'Create a GitHub release' + type: boolean + default: false + +permissions: + contents: write + packages: write + +jobs: + release: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Validate and resolve tag + id: tag + run: | + TAG="${{ inputs.tag }}" + if [[ -z "${TAG}" ]]; then + BRANCH="${GITHUB_REF_NAME//\//-}" + SHA="$(git rev-parse --short HEAD)" + TAG="${BRANCH}-${SHA}" + fi + if [[ "${{ inputs.create_release }}" == "true" ]]; then + if [[ ! "${TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9._-]+)?$ ]]; then + echo "::error::Tag '${TAG}' must match vMAJOR.MINOR.PATCH[-prerelease] when creating a release (e.g. v1.2.3 or v1.2.3-rc1)" + exit 1 + fi + fi + echo "value=${TAG}" >> "$GITHUB_OUTPUT" + if [[ "${{ inputs.create_release }}" == "true" ]]; then + echo "tags=${TAG},latest" >> "$GITHUB_OUTPUT" + else + echo "tags=${TAG}" >> "$GITHUB_OUTPUT" + fi + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + + - name: Install ko + uses: ko-build/setup-ko@v0.7 + + - name: Install Helm + uses: azure/setup-helm@v4 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up QEMU (multi-arch) + uses: docker/setup-qemu-action@v3 + + - name: Build and push images + env: + # ghcr.io// — resolves correctly in forks + IMAGE_REPOSITORY: ghcr.io/${{ github.repository }} + IMAGE_TAGS: ${{ steps.tag.outputs.tags }} + run: | + set -o errexit -o nounset -o pipefail + + for component in ateapi atecontroller atelet ateom-gvisor podcertcontroller atenet; do + KO_DOCKER_REPO="${IMAGE_REPOSITORY}/${component}" \ + ./hack/run-tool.sh ko build \ + --tags "${IMAGE_TAGS}" \ + --platform linux/amd64,linux/arm64 \ + --bare \ + "./cmd/${component}" + done + + - name: Package and push Helm charts + if: inputs.create_release + env: + HELM_EXPERIMENTAL_OCI: "1" + CHART_REPOSITORY: oci://ghcr.io/kagent-dev/substrate/helm + run: | + set -o errexit -o nounset -o pipefail + + tag="${{ steps.tag.outputs.value }}" + chart_version="${tag#v}" + package_dir="${RUNNER_TEMP}/helm-packages" + mkdir -p "${package_dir}" + + echo "${{ secrets.GITHUB_TOKEN }}" \ + | helm registry login ghcr.io \ + --username "${{ github.actor }}" \ + --password-stdin + + helm package charts/substrate-crds \ + --destination "${package_dir}" \ + --version "${chart_version}" \ + --app-version "${tag}" + helm package charts/substrate \ + --destination "${package_dir}" \ + --version "${chart_version}" \ + --app-version "${tag}" + + helm push "${package_dir}/substrate-crds-${chart_version}.tgz" "${CHART_REPOSITORY}" + helm push "${package_dir}/substrate-${chart_version}.tgz" "${CHART_REPOSITORY}" + + - name: Create GitHub Release + if: inputs.create_release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ steps.tag.outputs.value }} + generate_release_notes: true diff --git a/Makefile b/Makefile index 2310656d7..d999abe88 100644 --- a/Makefile +++ b/Makefile @@ -41,8 +41,9 @@ build: build-images build-atectl .PHONY: build-images build-images: GOFLAGS='"-ldflags=$(LDFLAGS)"' \ - $(KO) build \ + $(KO) build --base-import-paths \ ./cmd/ateapi \ + ./cmd/atecontroller \ ./cmd/atelet \ ./cmd/podcertcontroller \ ./cmd/atenet @@ -91,3 +92,19 @@ verify: test .PHONY: clean clean: rm -rf $(BINDIR) + +# Render the substrate Helm chart into manifests/ate-install/ (mTLS mode, +# the historical default install). Run this whenever charts/substrate/ changes. +.PHONY: helm-template +helm-template: + @./hack/render-manifests.sh + +# Verify that manifests/ate-install/ matches the chart output. Used in CI. +.PHONY: verify-helm-template +verify-helm-template: + @./hack/render-manifests.sh --check + +# Verify that the CRD chart mirrors the generated CRDs. +.PHONY: verify-crd-chart +verify-crd-chart: + @./hack/verify/crd-chart.sh diff --git a/README.md b/README.md index d4d5245a8..32d8ce22c 100644 --- a/README.md +++ b/README.md @@ -103,10 +103,10 @@ To quickly set up the complete environment: 2. Run the following steps: ```shell # create cluster and local registry -hack/create-kind-cluster.sh +KIND_ENABLE_PODCERT=false hack/create-kind-cluster.sh -# install ate, valkey, rustfs -hack/install-ate-kind.sh --deploy-ate-system +# install ate, valkey, rustfs using Helm in JWT mode +hack/install-ate-kind-jwt.sh # install counter demo hack/install-ate-kind.sh --deploy-demo-counter @@ -126,6 +126,21 @@ kubectl port-forward -n ate-system svc/atenet-router 8000:80 curl -X POST -H "Host: my-counter-1.actors.resources.substrate.ate.dev" -i http://localhost:8000/ ``` +#### mTLS mode + +JWT mode is the default install path and does not require pod certificate +feature gates. To test the older mTLS path, create kind with the +`ClusterTrustBundle` / `PodCertificateRequest` feature gates enabled and use the +mTLS install helper. + +```shell +# create cluster WITH podcert feature gates +hack/create-kind-cluster.sh + +# install ate using the mTLS manifests path +hack/install-ate-kind.sh --deploy-ate-system +``` + ### GKE Quickstart (Development) 1. Create and configure your environment file: diff --git a/manifests/ate-install/ate-system-namespace.yaml b/charts/substrate-crds/Chart.yaml similarity index 65% rename from manifests/ate-install/ate-system-namespace.yaml rename to charts/substrate-crds/Chart.yaml index 4fa19da0a..a69dcee0e 100644 --- a/manifests/ate-install/ate-system-namespace.yaml +++ b/charts/substrate-crds/Chart.yaml @@ -12,7 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -apiVersion: v1 -kind: Namespace -metadata: - name: ate-system \ No newline at end of file +apiVersion: v2 +name: substrate-crds +description: Agent Substrate CustomResourceDefinitions. +type: application +version: 0.1.0 +appVersion: "0.1.0" +home: https://github.com/agent-substrate/substrate +sources: +- https://github.com/agent-substrate/substrate +keywords: +- agent +- actor +- substrate +- crds diff --git a/charts/substrate-crds/README.md b/charts/substrate-crds/README.md new file mode 100644 index 000000000..12fa31f0a --- /dev/null +++ b/charts/substrate-crds/README.md @@ -0,0 +1,13 @@ +# substrate-crds + +Helm chart for installing the Agent Substrate CRDs. + +Install this chart before installing the main `substrate` chart: + +```bash +helm upgrade --install substrate-crds ./charts/substrate-crds +helm upgrade --install substrate ./charts/substrate --namespace ate-system --create-namespace +``` + +The CRD YAMLs in `templates/` mirror `manifests/ate-install/generated/`. +Run `hack/verify/crd-chart.sh` to verify they are in sync. diff --git a/charts/substrate-crds/templates/ate.dev_actortemplates.yaml b/charts/substrate-crds/templates/ate.dev_actortemplates.yaml new file mode 100644 index 000000000..20db85e4d --- /dev/null +++ b/charts/substrate-crds/templates/ate.dev_actortemplates.yaml @@ -0,0 +1,468 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.1 + name: actortemplates.ate.dev +spec: + group: ate.dev + names: + kind: ActorTemplate + listKind: ActorTemplateList + plural: actortemplates + shortNames: + - actortemplate + singular: actortemplate + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.sandboxClass + name: Class + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec defines the desired state of ActorTemplate. This field + is immutable. + properties: + containers: + description: Containers is the workload definition. + items: + description: A single application container that you want to run + within a WorkerPool. + properties: + command: + description: Entrypoint array. Not executed within a shell. + items: + type: string + maxItems: 64 + type: array + x-kubernetes-list-type: atomic + env: + description: Environment variables to set in the worker replicas. + items: + description: |- + EnvVar represents an environment variable supplied to a container in an + ActorTemplate. It models only a subset of Kubernetes Pod env behavior: + literal values are not expanded with Kubernetes-style $(VAR) references, + envFrom is not supported, and valueFrom currently supports only secretKeyRef. + properties: + name: + description: |- + Name is the name of the environment variable. May be any printable ASCII + character except '='. + minLength: 1 + pattern: ^[ -<>-~]+$ + type: string + value: + description: |- + Variable value. Mutually exclusive with ValueFrom. + Value is the literal value of the environment variable. Unlike in + Kubernetes pods, this value is not interpolated, and $(VAR) + references are not expanded. + minLength: 0 + type: string + valueFrom: + description: |- + Source for the environment variable's value. Mutually exclusive with + Value. + maxProperties: 1 + minProperties: 1 + properties: + secretKeyRef: + description: Selects a key of a Secret in the ActorTemplate's + namespace. + properties: + key: + description: Key to select within the Secret. + minLength: 1 + pattern: ^[-._a-zA-Z0-9]+$ + type: string + name: + description: Name of the referent Secret. + maxLength: 253 + type: string + x-kubernetes-validations: + - message: Name must be a valid DNS subdomain + rule: '!format.dns1123Subdomain().validate(self).hasValue()' + optional: + description: Specify whether the Secret or its + key must be defined. + type: boolean + required: + - key + - name + type: object + type: object + required: + - name + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [value valueFrom] + must be set + rule: '[has(self.value),has(self.valueFrom)].filter(x,x==true).size() + == 1' + maxItems: 32 + type: array + image: + description: Image to use for the worker replicas. + type: string + x-kubernetes-validations: + - message: All images must be pinned (changing the image invalidates + snapshots) + rule: self.contains('@') + name: + description: Name of the container. + maxLength: 63 + type: string + x-kubernetes-validations: + - message: Name must be a valid DNS label + rule: '!format.dns1123Label().validate(self).hasValue()' + readyz: + description: |- + Readyz is an optional HTTP readiness probe. When set, the actor is not + considered ready (and Run/Restore RPCs do not return success) until the + container's HTTP endpoint returns 200. + properties: + httpGet: + description: HTTPGet specifies the HTTP request to perform + against the container. + properties: + path: + default: /readyz + description: |- + Path to access on the HTTP server. Defaults to "/readyz". + Must be a valid URL path starting with "/". Only characters permitted + by RFC 3986 path segments are accepted; percent-escapes must be a + literal "%" followed by exactly two hex digits. Query strings ("?") + and fragments ("#") must be omitted. + maxLength: 1024 + pattern: ^/([A-Za-z0-9\-._~!$&'()*+,;=:@/]|%[0-9A-Fa-f]{2})*$ + type: string + port: + description: Port to access on the container. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - port + type: object + required: + - httpGet + type: object + volumeMounts: + description: volumeMounts define the volumes to mount into this + container. + items: + description: VolumeMount describes a mounting of a Volume + within a actor. + properties: + mountPath: + description: |- + Path within the actor at which the volume should be mounted. Must be a + clean absolute Unix path: must start with '/', not be '/', and contain + no ':', '..', '.', '//', trailing '/', or control characters. + maxLength: 4096 + type: string + x-kubernetes-validations: + - message: 'MountPath must be a clean absolute Unix path: + must start with ''/'', not be ''/'', and contain no + '':'', ''..'', ''.'', ''//'', trailing ''/'', or control + characters' + rule: self.startsWith('/') && size(self) > 1 && !self.endsWith('/') + && !self.contains('//') && !self.contains(':') && + !self.matches('[\x00-\x1f\x7f]') && !self.matches('(^|/)[.][.]?(/|$)') + name: + description: This must match the Name of a Volume. + maxLength: 63 + type: string + x-kubernetes-validations: + - message: Name must be a valid DNS label + rule: '!format.dns1123Label().validate(self).hasValue()' + required: + - mountPath + - name + type: object + maxItems: 32 + type: array + required: + - image + - name + type: object + maxItems: 10 + type: array + pauseImage: + description: |- + PauseImage is the container to use as the root sandbox container. + + Typically, set it to [1] for on-gcp, and [2] for off-gcp + + - [1] gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da + - [2] registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4 + type: string + x-kubernetes-validations: + - message: All images must be pinned (changing the image invalidates + snapshots) + rule: self.contains('@') + sandboxClass: + default: gvisor + description: |- + SandboxClass selects the sandbox runtime family this template's actors run + on. Only worker pools whose SandboxClass matches are eligible. Snapshots are + not portable across classes, so this is a hard gate, AND'd with WorkerSelector + and the actor's worker_selector. Defaults to gvisor. + + + 1) How does someone discover what classes are available, or what they mean? + 2) How does someone define a new sandbox class? + 3) Does a class mean the specific type of sandbox tech or does it include some aspect of config (e.g. can we have 2 different classes which both use gVisor with different config, or 2 classes which use different microvms) + 4) How does the default get set and who sets it? + + See Also: WorkerPool SandboxClass + enum: + - gvisor + - microvm + type: string + snapshotsConfig: + description: Snapshots configuration for the actor. + properties: + location: + description: Location to store snapshots in. + minLength: 1 + type: string + onCommit: + default: Full + description: |- + OnCommit specifies what to include in the snapshot when a commit is requested. + If not provided, the "Full" behavior is used by default. + onCommit must be a subset of the onPause content. + + For example: + - if onPause is "Full", then onCommit can be "Full" or "Data". + - if onPause is "Data", then onCommit must be "Data". + enum: + - Full + - Data + type: string + onPause: + default: Full + description: |- + OnPause specifies what to include in the snapshot when the actor is paused. + If not provided, the "Full" behavior is used by default. + enum: + - Full + - Data + type: string + required: + - location + type: object + x-kubernetes-validations: + - message: onCommit must be a subset of onPause + rule: '(has(self.onPause) ? self.onPause : ''Full'') == ''Full'' + || (has(self.onCommit) ? self.onCommit : ''Full'') == (has(self.onPause) + ? self.onPause : ''Full'')' + volumes: + description: Volumes defines the volumes to mount into all containers + in the actor. + items: + properties: + durableDir: + description: |- + durableDir represents a durable directory on rootfs that persists across + resumes and participates in snapshots. + type: object + name: + description: name of the volume. + maxLength: 63 + type: string + x-kubernetes-validations: + - message: Name must be a valid DNS label + rule: '!format.dns1123Label().validate(self).hasValue()' + required: + - name + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [durableDir] must be set + rule: '[has(self.durableDir)].filter(x,x==true).size() == 1' + maxItems: 32 + type: array + workerSelector: + description: |- + WorkerSelector restricts which worker pools actors from this template may + use. The scheduler only considers pools whose labels match this selector. + If nil, all pools are eligible (subject to the actor's own worker_selector). + Acts as a gate: the actor's worker_selector can only narrow this set further, + never expand it. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - pauseImage + - snapshotsConfig + type: object + x-kubernetes-validations: + - message: Spec is immutable + rule: self == oldSelf + - message: At most one DurableDir-typed volume is supported per ActorTemplate + rule: '!has(self.volumes) || self.volumes.filter(v, has(v.durableDir)).size() + <= 1' + - message: A container may mount at most one DurableDir-typed volume + rule: '!has(self.containers) || self.containers.all(c, !has(c.volumeMounts) + || c.volumeMounts.filter(vm, has(self.volumes) && self.volumes.exists(v, + v.name == vm.name && has(v.durableDir))).size() <= 1)' + - message: DurableDir volumes are not supported when sandboxClass is 'microvm' + rule: '!has(self.sandboxClass) || self.sandboxClass != ''microvm'' || + !has(self.volumes) || !self.volumes.exists(v, has(v.durableDir))' + status: + description: status is the observed state of ActorTemplate + properties: + conditions: + description: conditions defines the status conditions array + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + goldenActorID: + type: string + goldenSnapshot: + type: string + phase: + description: Phase of the actor template. + type: string + takeGoldenSnapshotAt: + format: date-time + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml b/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml new file mode 100644 index 000000000..fa572d140 --- /dev/null +++ b/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml @@ -0,0 +1,127 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.1 + name: sandboxconfigs.ate.dev +spec: + group: ate.dev + names: + kind: SandboxConfig + listKind: SandboxConfigList + plural: sandboxconfigs + shortNames: + - sandboxconfig + singular: sandboxconfig + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.sandboxClass + name: Class + type: string + - jsonPath: .spec.default + name: Default + type: boolean + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + SandboxConfig is cluster-scoped configuration describing the sandbox binaries + for a sandbox runtime family. It is referenced (or defaulted) by WorkerPools + and decouples sandbox binary selection from ActorTemplate. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec defines the desired state of SandboxConfig + properties: + assets: + additionalProperties: + additionalProperties: + description: |- + AssetFile is one content-addressed file that atelet fetches for a sandbox + runtime (e.g. the gVisor runsc binary, or a micro-VM kernel/firmware/config). + properties: + sha256: + description: |- + SHA256 is the lower-case hex SHA256 of the asset. It both names the cached + file (preventing collisions) and verifies the download's integrity. + pattern: ^[a-f0-9]{64}$ + type: string + url: + description: |- + URL is where to download the asset from (e.g. a gs:// URL). It may be + fetched anonymously or with credentials depending on atelet's + configuration. + minLength: 1 + type: string + required: + - sha256 + - url + type: object + type: object + description: |- + Assets is the set of files atelet fetches for this runtime, keyed first by + architecture (GOARCH, e.g. "amd64", "arm64") and then by asset name. The + asset names are interpreted by the sandbox backend: gVisor expects a + "runsc" asset; a micro-VM backend expects several (e.g. "cloud-hypervisor", + "kata-kernel", "kata-image"). The schema is intentionally generic; + per-class requirements are enforced by a ValidatingAdmissionPolicy. + type: object + default: + description: |- + Default marks this SandboxConfig as the cluster-wide default for its + SandboxClass. A WorkerPool with no explicit SandboxConfigName resolves to + the default config for its SandboxClass. At most one default is expected + per SandboxClass. + type: boolean + sandboxClass: + default: gvisor + description: |- + SandboxClass is the sandbox runtime family this config applies to. A + WorkerPool only uses SandboxConfigs whose SandboxClass matches its own. + enum: + - gvisor + - microvm + type: string + required: + - sandboxClass + type: object + required: + - spec + type: object + served: true + storage: true + subresources: {} diff --git a/charts/substrate-crds/templates/ate.dev_workerpools.yaml b/charts/substrate-crds/templates/ate.dev_workerpools.yaml new file mode 100644 index 000000000..246af4b37 --- /dev/null +++ b/charts/substrate-crds/templates/ate.dev_workerpools.yaml @@ -0,0 +1,436 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.1 + name: workerpools.ate.dev +spec: + group: ate.dev + names: + kind: WorkerPool + listKind: WorkerPoolList + plural: workerpools + shortNames: + - workerpool + singular: workerpool + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.replicas + name: Desired + type: integer + - jsonPath: .status.replicas + name: Replicas + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: WorkerPool is the Schema for the workerpools API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec defines the desired state of WorkerPool + properties: + ateomImage: + description: AteomImage is the ateom container image to deploy as + workers. + minLength: 1 + type: string + replicas: + description: Replicas is the number of worker pods to run. + format: int32 + minimum: 0 + type: integer + sandboxClass: + default: gvisor + description: |- + SandboxClass selects the sandbox runtime family for this pool, which drives + the worker pod shape (KVM/vhost device mounts and node placement) and which + SandboxConfigs are eligible. The concrete binary is still selected by + AteomImage. Defaults to gvisor. + + See Also: TODOs in ActorTemplate SandboxClass + enum: + - gvisor + - microvm + type: string + sandboxConfigName: + description: |- + SandboxConfigName names a cluster-scoped SandboxConfig to use for fetching + sandbox binaries. It overrides the cluster-wide default SandboxConfig for + this pool's SandboxClass. The referenced config's SandboxClass must match + this pool's SandboxClass. If empty, the default SandboxConfig for the + SandboxClass is used. + type: string + template: + description: Template holds optional pod scheduling and resource settings + for worker pods. + properties: + nodeAffinity: + description: |- + NodeAffinity scheduling rules for the worker pods. Mapped to + spec.affinity.nodeAffinity on the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with the + corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. + The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + nodeSelector: + additionalProperties: + type: string + description: NodeSelector is a selector which must be true for + the pod to fit on a node. + type: object + priorityClassName: + description: PriorityClassName for the worker pods. + type: string + resources: + description: Resources are the compute resources allocated for + each worker pod. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + tolerations: + description: Tolerations for the worker pods. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + maxItems: 16 + type: array + x-kubernetes-list-type: atomic + type: object + required: + - ateomImage + - replicas + type: object + status: + description: status is the observed state of WorkerPool + properties: + replicas: + description: Replicas is the total number of worker pods. + format: int32 + minimum: 0 + type: integer + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + scale: + specReplicasPath: .spec.replicas + statusReplicasPath: .status.replicas + status: {} diff --git a/charts/substrate/Chart.yaml b/charts/substrate/Chart.yaml new file mode 100644 index 000000000..52bd74800 --- /dev/null +++ b/charts/substrate/Chart.yaml @@ -0,0 +1,27 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v2 +name: substrate +description: Agent Substrate — actor runtime, control plane, and data-plane router. +type: application +version: 0.1.0 +appVersion: "0.1.0" +home: https://github.com/agent-substrate/substrate +sources: +- https://github.com/agent-substrate/substrate +keywords: +- agent +- actor +- substrate diff --git a/charts/substrate/README.md b/charts/substrate/README.md new file mode 100644 index 000000000..2fe9bbbbb --- /dev/null +++ b/charts/substrate/README.md @@ -0,0 +1,71 @@ +# substrate + +Helm chart for installing Agent Substrate. + +## Install modes + +| Mode | Default? | Cluster requirements | Trade-off | +|------|----------|----------------------|-----------| +| `jwt` | yes | none beyond stock K8s | Server certs and session signing pools are generated by the chart; clients authenticate via projected ServiceAccount tokens. Valkey runs plaintext intra-cluster. | +| `mtls` | | feature gates `ClusterTrustBundle`, `ClusterTrustBundleProjection`, `PodCertificateRequest` + `certificates.k8s.io/v1beta1` API | Full in-cluster mTLS via the bundled `podcertcontroller`. | + +```bash +# CRDs +helm upgrade --install substrate-crds ./charts/substrate-crds + +# JWT mode (default; no off-by-default feature gates) +helm upgrade --install substrate ./charts/substrate + +# mTLS mode (requires off-by-default feature gates) +helm upgrade --install substrate ./charts/substrate \ + --set auth.mode=mtls +``` + +By default, component images are pulled from `ghcr.io/kagent-dev/substrate` +using the chart `appVersion` as the tag. Override `image.registry` and +`image.tag` to install from a different image repository or tag. + +## JWT-mode bootstrap + +JWT mode is standalone by default. The chart generates: + +- `Secret/ateapi-tls` +- `ConfigMap/ateapi-ca` +- `Secret/session-id-jwt-pool` +- `Secret/session-id-ca-pool` + +Existing generated data is reused on upgrade so key material does not rotate +during normal chart upgrades. Set `auth.jwt.bootstrap.enabled=false` to bring +your own resources with those names. + +## Render manifests without applying + +```bash +helm template substrate ./charts/substrate # jwt +helm template substrate ./charts/substrate --set auth.mode=mtls +``` + +`manifests/ate-install/` in the repo is the rendered mTLS output and is +regenerated by `make helm-template`. The separate `substrate-crds` chart +mirrors `manifests/ate-install/generated/`. + +## Values + +See `values.yaml` for the full set; the important keys: + +| Key | Default | Notes | +|-----|---------|-------| +| `auth.mode` | `jwt` | `jwt` or `mtls` | +| `auth.jwt.issuer` | `https://kubernetes.default.svc.cluster.local` | Override for managed clusters with provider-specific issuers | +| `auth.jwt.audience` | `api.ate-system.svc` | SA token audience | +| `auth.jwt.bootstrap.enabled` | `true` | Generate JWT TLS and session signing material | +| `auth.jwt.serverCertSecret` | `ateapi-tls` | Secret name | +| `auth.jwt.caBundleConfigMap` | `ateapi-ca` | ConfigMap name | +| `valkey.enabled` | `true` | Set false if you bring your own Redis/Valkey | +| `valkey.replicas` | `6` | StatefulSet size | +| `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots | +| `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` | +| `redis.clusterAddress` | `""` (in-cluster) | Override to use external Redis | +| `redis.useIAMAuth` | `false` | Google IAM auth | +| `atelet.gcpAuthForImagePulls` | `false` | Enable only when using GCP registry auth | +| `otel.endpoint` | `""` | Set to an OTLP endpoint to export traces/metrics | diff --git a/charts/substrate/templates/NOTES.txt b/charts/substrate/templates/NOTES.txt new file mode 100644 index 000000000..f736c32f7 --- /dev/null +++ b/charts/substrate/templates/NOTES.txt @@ -0,0 +1,21 @@ +substrate {{ .Chart.AppVersion }} installed in mode: {{ .Values.auth.mode }} + +{{ if eq .Values.auth.mode "mtls" -}} +NOTE: mtls mode REQUIRES the following Kubernetes feature gates to be enabled: + - ClusterTrustBundle + - ClusterTrustBundleProjection + - PodCertificateRequest +plus the v1beta1 certificates API. On vanilla clusters (kind, EKS, etc.) you +must enable these explicitly. To install without them, pick auth.mode=jwt. +{{- else }} +JWT mode is active. + +{{- if .Values.auth.jwt.bootstrap.enabled }} +JWT bootstrap resources are managed by this chart. Existing key material is +reused on upgrade. +{{- else }} +JWT bootstrap is disabled. Provide {{ .Values.auth.jwt.serverCertSecret }}, +{{ .Values.auth.jwt.caBundleConfigMap }}, session-id-jwt-pool, and +session-id-ca-pool before pods become healthy. +{{- end }} +{{- end }} diff --git a/charts/substrate/templates/_helpers.tpl b/charts/substrate/templates/_helpers.tpl new file mode 100644 index 000000000..2f8ecc8e6 --- /dev/null +++ b/charts/substrate/templates/_helpers.tpl @@ -0,0 +1,80 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +Qualified resource name for a chart component. + +Usage: + {{ include "substrate.fullname" (list "ate-api-server" .) }} + +When the release name equals the chart name (the canonical render in +hack/render-manifests.sh — `helm template substrate charts/substrate`), this +returns the bare component name, so the generated manifests/ate-install/ +files keep their historical names ("ate-api-server", "ate-controller", ...). + +Otherwise resources are prefixed with the release name in the standard Helm +style ("foo-ate-api-server", ...) so multiple releases coexist without +colliding. +*/}} +{{- define "substrate.fullname" -}} +{{- $name := index . 0 -}} +{{- $ctx := index . 1 -}} +{{- if eq $ctx.Release.Name $ctx.Chart.Name -}} +{{- $name -}} +{{- else -}} +{{- printf "%s-%s" $ctx.Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + +{{/* +Build an image reference for a substrate component binary. + +Usage: + {{ include "substrate.componentImage" (list "ateapi" .) }} + +Produces {image.registry}/{name}:{tag} where tag is resolved as: + 1. image.tag value, if set and not the sentinel "" + 2. .Chart.AppVersion, if image.tag is empty + 3. no tag (no colon) when image.tag is the sentinel "" + +The "" sentinel is used by hack/render-manifests.sh so that ko:// refs +are emitted without a tag, letting `ko resolve` supply the digest at build time. +*/}} +{{- define "substrate.componentImage" -}} +{{- $name := index . 0 -}} +{{- $ctx := index . 1 -}} +{{- $registry := $ctx.Values.image.registry -}} +{{- $tag := $ctx.Values.image.tag | default $ctx.Chart.AppVersion -}} +{{- if ne $tag "" -}} +{{- printf "%s/%s:%s" $registry $name $tag -}} +{{- else -}} +{{- printf "%s/%s" $registry $name -}} +{{- end -}} +{{- end -}} + +{{/* +Validate auth.mode at template time. +*/}} +{{- define "substrate.validateAuthMode" -}} +{{- if not (or (eq .Values.auth.mode "mtls") (eq .Values.auth.mode "jwt")) -}} +{{- fail (printf "auth.mode must be 'mtls' or 'jwt', got %q" .Values.auth.mode) -}} +{{- end -}} +{{- if eq .Values.auth.mode "jwt" -}} +{{- if not .Values.auth.jwt.issuer -}} +{{- fail "auth.jwt.issuer is required when auth.mode=jwt" -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/charts/substrate/templates/ate-api-server-envvars.yaml b/charts/substrate/templates/ate-api-server-envvars.yaml new file mode 100644 index 000000000..754ff847a --- /dev/null +++ b/charts/substrate/templates/ate-api-server-envvars.yaml @@ -0,0 +1,27 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.ateApiServerEnvVarsConfigMap }} + namespace: {{ .Release.Namespace }} +data: + ATE_API_REDIS_ADDRESS: {{ .Values.redis.clusterAddress | default (printf "%s.%s.svc:6379" (include "substrate.fullname" (list "valkey-cluster" .)) .Release.Namespace) | quote }} + ATE_API_REDIS_USE_IAM_AUTH: {{ .Values.redis.useIAMAuth | toString | quote }} + ATE_API_REDIS_TLS_SERVER_NAME: {{ .Values.redis.tlsServerName | quote }} + ATE_API_REDIS_CLIENT_CERT: {{ .Values.redis.clientCert | default "" | quote }} + ATE_API_K8SJWT_ISSUER: {{ .Values.auth.jwt.issuer | quote }} diff --git a/charts/substrate/templates/ate-api-server.yaml b/charts/substrate/templates/ate-api-server.yaml new file mode 100644 index 000000000..ded60b4c7 --- /dev/null +++ b/charts/substrate/templates/ate-api-server.yaml @@ -0,0 +1,236 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "substrate.fullname" (list "ate-api-server-role" .) }} +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "watch", "list"] +- apiGroups: ["ate.dev"] + resources: ["actortemplates", "workerpools", "sandboxconfigs"] + verbs: ["get", "watch", "list"] +# Secret reads for env source resolution are intentionally NOT granted +# cluster-wide here. Each demo / tenant is responsible for granting +# ate-api-server read access only to the specific Secrets referenced by its +# ActorTemplates (e.g. via a namespace-scoped Role + RoleBinding using +# resourceNames). +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "ate-api-server" .) }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "ate-api-server-binding" .) }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "ate-api-server" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "substrate.fullname" (list "ate-api-server-role" .) }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "substrate.fullname" (list "ate-api-server-deployment" .) }} + namespace: {{ .Release.Namespace }} +spec: + replicas: 1 + selector: + matchLabels: + app: ate-api-server + template: + metadata: + labels: + app: ate-api-server + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + spec: + serviceAccountName: {{ include "substrate.fullname" (list "ate-api-server" .) }} + {{- with (or .Values.ateApiServer.nodeSelector .Values.nodeSelector) }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- if eq .Values.auth.mode "jwt" }} + initContainers: + - name: assemble-cred-bundle + image: {{ .Values.images.busybox }} + command: + - sh + - -c + - cat /run/ateapi-tls-src/tls.crt /run/ateapi-tls-src/tls.key > /run/ateapi-tls/credential-bundle.pem + volumeMounts: + - { name: ateapi-tls-src, mountPath: /run/ateapi-tls-src, readOnly: true } + - { name: ateapi-tls, mountPath: /run/ateapi-tls } +{{- end }} + containers: + - name: ate-api-server + image: {{ include "substrate.componentImage" (list "ateapi" .) }} + args: + - "--grpc-listen-addr=0.0.0.0:443" +{{- if eq .Values.auth.mode "mtls" }} + - "--grpc-server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem" + - "--redis-cluster-address=@env" + - "--redis-ca-certs=/etc/valkey-ca/ca.crt" + - "--redis-use-iam-auth=@env" + - "--redis-tls-server-name=@env" + - "--redis-client-cert=@env" + - "--client-jwt-issuer=@env" + - "--client-jwt-audience={{ .Values.auth.jwt.audience }}" + - "--session-id-jwt-pool=/run/session-id-jwt-pool/pool.json" + - "--session-id-ca-pool=/run/session-id-ca-pool/pool.json" + - "--workerpool-ca-certs=/run/workerpool-ca-certs/trust-bundle.pem" +{{- else }} + - "--grpc-server-cred-bundle=/run/ateapi-tls/credential-bundle.pem" + - "--auth-mode=jwt" + - "--redis-cluster-address=@env" + - "--redis-no-tls=true" + - "--redis-use-iam-auth=@env" + - "--client-jwt-issuer={{ .Values.auth.jwt.issuer }}" + - "--client-jwt-audience={{ .Values.auth.jwt.audience }}" + - "--session-id-jwt-pool=/run/session-id-jwt-pool/pool.json" + - "--session-id-ca-pool=/run/session-id-ca-pool/pool.json" + - "--client-jwt-ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" +{{- end }} + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: OTEL_RESOURCE_ATTRIBUTES + value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),service.instance.id=$(POD_UID) +{{- if .Values.otel.endpoint }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .Values.otel.endpoint | quote }} +{{- end }} + envFrom: + - configMapRef: + name: {{ .Values.ateApiServerEnvVarsConfigMap }} + optional: true + volumeMounts: +{{- if eq .Values.auth.mode "mtls" }} + - { name: servicedns, mountPath: /run/servicedns.podcert.ate.dev } + - { name: session-id-jwt-pool, mountPath: /run/session-id-jwt-pool } + - { name: valkey-ca-certs, mountPath: /etc/valkey-ca, readOnly: true } + - { name: session-id-ca-pool, mountPath: /run/session-id-ca-pool, readOnly: true } + - { name: workerpool-ca-certs, mountPath: /run/workerpool-ca-certs, readOnly: true } +{{- else }} + - { name: ateapi-tls, mountPath: /run/ateapi-tls, readOnly: true } + - { name: session-id-jwt-pool, mountPath: /run/session-id-jwt-pool } + - { name: session-id-ca-pool, mountPath: /run/session-id-ca-pool, readOnly: true } +{{- end }} + ports: + - containerPort: 443 + - name: prometheus + containerPort: 9090 + readinessProbe: + httpGet: + path: /readyz + port: 9090 + initialDelaySeconds: 5 + periodSeconds: 2 + volumes: +{{- if eq .Values.auth.mode "mtls" }} + - name: servicedns + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - name: session-id-jwt-pool + projected: + sources: + - secret: + name: session-id-jwt-pool + items: + - { key: pool, path: pool.json } + - name: valkey-ca-certs + projected: + sources: + - secret: + name: valkey-ca-certs + items: + - { key: ca.crt, path: ca.crt } + - name: session-id-ca-pool + projected: + sources: + - secret: + name: session-id-ca-pool + items: + - { key: pool, path: pool.json } + - name: workerpool-ca-certs + projected: + sources: + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem +{{- else }} + - name: ateapi-tls-src + secret: + secretName: {{ .Values.auth.jwt.serverCertSecret }} + - name: ateapi-tls + emptyDir: {} + - name: session-id-jwt-pool + projected: + sources: + - secret: + name: session-id-jwt-pool + items: + - { key: pool, path: pool.json } + - name: session-id-ca-pool + projected: + sources: + - secret: + name: session-id-ca-pool + items: + - { key: pool, path: pool.json } +{{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "substrate.fullname" (list "api" .) }} + namespace: {{ .Release.Namespace }} +spec: + type: ClusterIP + selector: + app: ate-api-server + ports: + - name: grpc + protocol: TCP + port: 443 + targetPort: 443 diff --git a/charts/substrate/templates/ate-client.yaml b/charts/substrate/templates/ate-client.yaml new file mode 100644 index 000000000..dfd2fdab6 --- /dev/null +++ b/charts/substrate/templates/ate-client.yaml @@ -0,0 +1,23 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "ate-client" .) }} + namespace: {{ .Release.Namespace }} + labels: + apps: ate-client diff --git a/charts/substrate/templates/ate-controller.yaml b/charts/substrate/templates/ate-controller.yaml new file mode 100644 index 000000000..04347d75d --- /dev/null +++ b/charts/substrate/templates/ate-controller.yaml @@ -0,0 +1,112 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + namespace: {{ .Release.Namespace }} + labels: + apps: ate-controller +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + apiGroup: rbac.authorization.k8s.io +--- +kind: Service +apiVersion: v1 +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: ate-controller +spec: + selector: + app: ate-controller + ports: + - name: metrics + port: 8080 + targetPort: metrics + protocol: TCP +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + namespace: {{ .Release.Namespace }} +spec: + replicas: 1 + selector: + matchLabels: + app: ate-controller + template: + metadata: + labels: + app: ate-controller + spec: + serviceAccountName: {{ include "substrate.fullname" (list "ate-controller" .) }} + {{- with (or .Values.ateController.nodeSelector .Values.nodeSelector) }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: ate-controller + image: {{ include "substrate.componentImage" (list "atecontroller" .) }} + args: + # The atecontroller binary defaults --ateapi-conn-spec to + # dns:///api.ate-system.svc:443, which is correct only for the + # canonical render (release name "substrate" in namespace + # "ate-system"). Pass the chart-resolved Service so the controller + # dials the right backend when substrate is installed as a subchart. + - "--ateapi-conn-spec=dns:///{{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443" +{{- if eq .Values.auth.mode "jwt" }} + - "--ateapi-auth=jwt" + - "--ateapi-ca-file=/run/ateapi-ca/ca.crt" + - "--ateapi-server-name={{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc" + - "--ateapi-token-file=/var/run/secrets/tokens/ateapi/token" +{{- end }} + ports: + - name: metrics + containerPort: 8080 + protocol: TCP + - name: healthz + containerPort: 8081 + protocol: TCP +{{- if eq .Values.auth.mode "jwt" }} + volumeMounts: + - { name: ateapi-ca, mountPath: /run/ateapi-ca, readOnly: true } + - { name: ateapi-token, mountPath: /var/run/secrets/tokens/ateapi, readOnly: true } + volumes: + - name: ateapi-ca + configMap: + name: {{ .Values.auth.jwt.caBundleConfigMap }} + - name: ateapi-token + projected: + sources: + - serviceAccountToken: + audience: {{ .Values.auth.jwt.audience }} + expirationSeconds: 3600 + path: token +{{- end }} diff --git a/charts/substrate/templates/atelet.yaml b/charts/substrate/templates/atelet.yaml new file mode 100644 index 000000000..d548b1185 --- /dev/null +++ b/charts/substrate/templates/atelet.yaml @@ -0,0 +1,121 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# atelet — identical across auth modes (does not dial ateapi). +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "atelet" .) }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "substrate.fullname" (list "atelet-role" .) }} +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "watch", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "atelet-binding" .) }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "atelet" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "substrate.fullname" (list "atelet-role" .) }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "substrate.fullname" (list "atelet" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: atelet +spec: + selector: + matchLabels: + app: atelet + template: + metadata: + labels: + app: atelet + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + spec: + serviceAccountName: {{ include "substrate.fullname" (list "atelet" .) }} + {{- with (or .Values.atelet.nodeSelector .Values.nodeSelector) }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: atelet + image: {{ include "substrate.componentImage" (list "atelet" .) }} + args: + - --gcp-auth-for-image-pulls={{ .Values.atelet.gcpAuthForImagePulls }} +{{- with .Values.atelet.extraArgs }} +{{ toYaml . | indent 8 }} +{{- end }} + securityContext: + privileged: true + env: + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName +{{- if .Values.otel.endpoint }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .Values.otel.endpoint | quote }} +{{- end }} + - name: ATE_STORAGE_BACKEND + value: {{ .Values.atelet.storageBackend | quote }} +{{- if .Values.rustfs.enabled }} + - name: AWS_REGION + value: us-east-1 + - name: AWS_ENDPOINT_URL + value: http://{{ include "substrate.fullname" (list "rustfs" .) }}.{{ .Release.Namespace }}.svc:9000 + - name: AWS_S3_USE_PATH_STYLE + value: "true" + - name: AWS_ACCESS_KEY_ID + value: {{ .Values.rustfs.accessKey | quote }} + - name: AWS_SECRET_ACCESS_KEY + value: {{ .Values.rustfs.secretKey | quote }} +{{- end }} +{{- with .Values.atelet.extraEnv }} +{{ toYaml . | indent 8 }} +{{- end }} + ports: + - name: grpc + containerPort: 8085 + hostPort: 8085 + - name: prometheus + containerPort: 9090 + hostPort: 9090 + protocol: TCP + volumeMounts: + - name: run-ateom + mountPath: /var/lib/ateom-gvisor + volumes: + - name: run-ateom + hostPath: + path: /var/lib/ateom-gvisor + type: DirectoryOrCreate diff --git a/charts/substrate/templates/atenet-dns.yaml b/charts/substrate/templates/atenet-dns.yaml new file mode 100644 index 000000000..bb581a3a0 --- /dev/null +++ b/charts/substrate/templates/atenet-dns.yaml @@ -0,0 +1,191 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# atenet-dns — identical across auth modes (does not dial ateapi). +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "atenet-dns" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: dns +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "substrate.fullname" (list "atenet-dns" .) }} + namespace: {{ .Release.Namespace }} +rules: +- apiGroups: [""] + resources: ["services"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "substrate.fullname" (list "atenet-dns" .) }} + namespace: {{ .Release.Namespace }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "atenet-dns" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: {{ include "substrate.fullname" (list "atenet-dns" .) }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "substrate.fullname" (list "atenet-dns" .) }} + namespace: kube-system +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "substrate.fullname" (list "atenet-dns" .) }} + namespace: kube-system +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "atenet-dns" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: {{ include "substrate.fullname" (list "atenet-dns" .) }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "substrate.fullname" (list "dns" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: dns +spec: + replicas: 1 + selector: + matchLabels: + app: dns + template: + metadata: + labels: + app: dns + spec: + serviceAccountName: {{ include "substrate.fullname" (list "atenet-dns" .) }} + {{- with (or .Values.dns.nodeSelector .Values.nodeSelector) }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + shareProcessNamespace: true + initContainers: + - name: init-dns + image: {{ .Values.images.busybox }} + command: ["sh", "-c"] + args: + - | + cat <<'EOF' > /etc/coredns/Corefile + .:53 { + errors + health :8080 + ready :8181 + reload + } + EOF + volumeMounts: + - name: dns-config-volume + mountPath: /etc/coredns + containers: + - name: coredns + image: {{ .Values.images.coredns }} + imagePullPolicy: IfNotPresent + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: dns-config-volume + mountPath: /etc/coredns + ports: + - name: dns + containerPort: 53 + protocol: UDP + - name: dns-tcp + containerPort: 53 + protocol: TCP + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /ready + port: 8181 + scheme: HTTP + initialDelaySeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + - name: dns-controller + image: {{ include "substrate.componentImage" (list "atenet" .) }} + args: + - "dns" + - "--log-level=debug" + - "--interval=10s" + - "--corefile-path=/etc/coredns/Corefile" + # Pass the chart-resolved Service names so the controller looks up the + # correct objects when substrate is installed as a subchart. The + # system namespace is read from POD_NAMESPACE below. + - "--router-service-name={{ include "substrate.fullname" (list "atenet-router" .) }}" + - "--dns-service-name={{ include "substrate.fullname" (list "dns" .) }}" + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: dns-config-volume + mountPath: /etc/coredns + volumes: + - name: dns-config-volume + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "substrate.fullname" (list "dns" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: dns +spec: + selector: + app: dns + type: ClusterIP + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP diff --git a/charts/substrate/templates/atenet-router.yaml b/charts/substrate/templates/atenet-router.yaml new file mode 100644 index 000000000..bdee2e10d --- /dev/null +++ b/charts/substrate/templates/atenet-router.yaml @@ -0,0 +1,277 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "atenet-router" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: atenet-router +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "substrate.fullname" (list "atenet-router" .) }} +rules: +- apiGroups: + - "ate.dev" + resources: + - actortemplates + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "atenet-router" .) }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "atenet-router" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "substrate.fullname" (list "atenet-router" .) }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "substrate.fullname" (list "atenet-router-agentgateway-config" .) }} + namespace: {{ .Release.Namespace }} +data: + config.yaml: | + # yaml-language-server: $schema=https://agentgateway.dev/schema/config + config: + adminAddr: "127.0.0.1:15000" + readinessAddr: "0.0.0.0:15021" + statsAddr: "0.0.0.0:15020" + binds: + - port: 8080 + listeners: + - name: http + protocol: HTTP + routes: + - name: substrate-http + matches: + - path: + pathPrefix: / + policies: + extProc: + host: "127.0.0.1:50051" + failureMode: failClosed + processingOptions: + requestBodyMode: none + responseBodyMode: none + requestHeaderMode: send + responseHeaderMode: skip + requestTrailerMode: skip + responseTrailerMode: skip + backends: + - dynamic: {} + - port: 8443 + listeners: + - name: https + protocol: HTTPS + tls: +{{ if eq .Values.auth.mode "mtls" }} + cert: "/run/servicedns.podcert.ate.dev/cert.pem" + key: "/run/servicedns.podcert.ate.dev/key.pem" +{{ else }} + cert: "/run/agentgateway-tls/tls.crt" + key: "/run/agentgateway-tls/tls.key" +{{ end }} + routes: + - name: substrate-https + matches: + - path: + pathPrefix: / + policies: + extProc: + host: "127.0.0.1:50051" + failureMode: failClosed + processingOptions: + requestBodyMode: none + responseBodyMode: none + requestHeaderMode: send + responseHeaderMode: skip + requestTrailerMode: skip + responseTrailerMode: skip + backends: + - dynamic: {} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "substrate.fullname" (list "atenet-router" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: atenet-router +spec: + replicas: 1 + selector: + matchLabels: + app: atenet-router + template: + metadata: + labels: + app: atenet-router + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + spec: + serviceAccountName: {{ include "substrate.fullname" (list "atenet-router" .) }} + {{- with (or .Values.atenetRouter.nodeSelector .Values.nodeSelector) }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: atenet-router + image: {{ include "substrate.componentImage" (list "atenet" .) }} + args: + - "router" + - "--standalone" + - "--networking-mode=agentgateway" + - "--namespace={{ .Release.Namespace }}" + # Pass the chart-resolved router Service name so /statusz looks up the + # correct Service when substrate is installed as a subchart. + - "--router-service-name={{ include "substrate.fullname" (list "atenet-router" .) }}" + - "--port-http=8080" + - "--port-extproc=50051" + - "--extproc-address=127.0.0.1" + - "--ateapi-address={{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443" +{{- if eq .Values.auth.mode "jwt" }} + - "--ateapi-auth=jwt" + - "--ateapi-ca-file=/run/ateapi-ca/ca.crt" + - "--ateapi-server-name={{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc" + - "--ateapi-token-file=/var/run/secrets/tokens/ateapi/token" +{{- end }} + - "--status-port=4040" + - "--port-https=8443" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: OTEL_RESOURCE_ATTRIBUTES + value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),service.instance.id=$(POD_UID) +{{- if .Values.otel.endpoint }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .Values.otel.endpoint | quote }} +{{- end }} + ports: + - name: extproc + containerPort: 50051 + - name: status + containerPort: 4040 + - name: metrics + containerPort: 9090 +{{- if eq .Values.auth.mode "jwt" }} + volumeMounts: + - name: ateapi-ca + mountPath: /run/ateapi-ca + readOnly: true + - name: ateapi-token + mountPath: /var/run/secrets/tokens/ateapi + readOnly: true +{{- end }} + - name: agentgateway + image: {{ .Values.images.agentgateway }} + args: + - "-f" + - "/etc/agentgateway/config.yaml" + ports: + - name: http + containerPort: 8080 + - name: https + containerPort: 8443 + - name: readiness + containerPort: 15021 + - name: gw-metrics + containerPort: 15020 + volumeMounts: + - name: agentgateway-config + mountPath: /etc/agentgateway +{{- if eq .Values.auth.mode "mtls" }} + - name: "servicedns" + mountPath: "/run/servicedns.podcert.ate.dev" +{{- else }} + - name: agentgateway-tls + mountPath: /run/agentgateway-tls + readOnly: true +{{- end }} + readinessProbe: + httpGet: + path: /healthz/ready + port: readiness + periodSeconds: 10 + volumes: + - name: agentgateway-config + configMap: + name: {{ include "substrate.fullname" (list "atenet-router-agentgateway-config" .) }} +{{- if eq .Values.auth.mode "mtls" }} + - name: "servicedns" + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + certificateChainPath: cert.pem + keyPath: key.pem +{{- else }} + - name: agentgateway-tls + secret: + secretName: {{ .Values.auth.jwt.serverCertSecret }} + - name: ateapi-ca + configMap: + name: {{ .Values.auth.jwt.caBundleConfigMap }} + - name: ateapi-token + projected: + sources: + - serviceAccountToken: + audience: {{ .Values.auth.jwt.audience }} + expirationSeconds: 3600 + path: token +{{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "substrate.fullname" (list "atenet-router" .) }} + namespace: {{ .Release.Namespace }} +spec: + type: ClusterIP + selector: + app: atenet-router + ports: + - name: http + port: 80 + targetPort: 8080 + protocol: TCP + - name: https + port: 443 + targetPort: 8443 + protocol: TCP diff --git a/charts/substrate/templates/jwt-bootstrap.yaml b/charts/substrate/templates/jwt-bootstrap.yaml new file mode 100644 index 000000000..e3299fe08 --- /dev/null +++ b/charts/substrate/templates/jwt-bootstrap.yaml @@ -0,0 +1,73 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and (eq .Values.auth.mode "jwt") .Values.auth.jwt.bootstrap.enabled }} +{{- $apiName := include "substrate.fullname" (list "api" .) }} +{{- $routerName := include "substrate.fullname" (list "atenet-router" .) }} +{{- $apiHost := printf "%s.%s.svc" $apiName .Release.Namespace }} +{{- $ca := genCA (printf "%s-ca" $apiName) 3650 }} +{{- $serverCert := genSignedCert $apiHost nil (list $apiHost (printf "%s.%s.svc.cluster.local" $apiName .Release.Namespace) (printf "%s.%s.svc" $routerName .Release.Namespace)) 365 $ca }} +{{- $sessionJWTKey := genPrivateKey "ecdsa" }} +{{- $sessionCA := genCA "session-id-ca" 3650 }} +{{- if .Values.auth.jwt.bootstrap.serverCert.enabled }} +{{- $existingTLS := lookup "v1" "Secret" .Release.Namespace .Values.auth.jwt.serverCertSecret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.auth.jwt.serverCertSecret }} + namespace: {{ .Release.Namespace }} +type: kubernetes.io/tls +data: + tls.crt: {{ if $existingTLS }}{{ index $existingTLS.data "tls.crt" }}{{ else }}{{ $serverCert.Cert | b64enc }}{{ end }} + tls.key: {{ if $existingTLS }}{{ index $existingTLS.data "tls.key" }}{{ else }}{{ $serverCert.Key | b64enc }}{{ end }} +--- +{{- $existingCA := lookup "v1" "ConfigMap" .Release.Namespace .Values.auth.jwt.caBundleConfigMap }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.auth.jwt.caBundleConfigMap }} + namespace: {{ .Release.Namespace }} +data: + ca.crt: | +{{- if $existingCA }} +{{ index $existingCA.data "ca.crt" | nindent 4 }} +{{- else }} +{{ $ca.Cert | nindent 4 }} +{{- end }} +{{- end }} +{{- if .Values.auth.jwt.bootstrap.sessionPools.enabled }} +--- +{{- $existingJWTSecret := lookup "v1" "Secret" .Release.Namespace "session-id-jwt-pool" }} +apiVersion: v1 +kind: Secret +metadata: + name: session-id-jwt-pool + namespace: {{ .Release.Namespace }} +type: Opaque +data: + pool: {{ if $existingJWTSecret }}{{ index $existingJWTSecret.data "pool" }}{{ else }}{{ dict "Authorities" (list (dict "ID" "1" "Algorithm" "ES256" "SigningKeyPEM" $sessionJWTKey)) | toJson | b64enc }}{{ end }} +--- +{{- $existingCASecret := lookup "v1" "Secret" .Release.Namespace "session-id-ca-pool" }} +apiVersion: v1 +kind: Secret +metadata: + name: session-id-ca-pool + namespace: {{ .Release.Namespace }} +type: Opaque +data: + pool: {{ if $existingCASecret }}{{ index $existingCASecret.data "pool" }}{{ else }}{{ dict "CAs" (list (dict "ID" "1" "SigningKeyPEM" $sessionCA.Key "RootCertificatePEM" $sessionCA.Cert)) | toJson | b64enc }}{{ end }} +{{- end }} +{{- end }} diff --git a/charts/substrate/templates/jwt-oidc-rbac.yaml b/charts/substrate/templates/jwt-oidc-rbac.yaml new file mode 100644 index 000000000..a9fd499e9 --- /dev/null +++ b/charts/substrate/templates/jwt-oidc-rbac.yaml @@ -0,0 +1,42 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if eq .Values.auth.mode "jwt" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "substrate.fullname" (list "oidc-discovery-viewer" .) }} +rules: +- nonResourceURLs: + - /.well-known/openid-configuration + - /openid/v1/jwks + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "oidc-discovery-viewer" .) }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "ate-api-server" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "substrate.fullname" (list "oidc-discovery-viewer" .) }} + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/charts/substrate/templates/namespace.yaml b/charts/substrate/templates/namespace.yaml new file mode 100644 index 000000000..63401c00d --- /dev/null +++ b/charts/substrate/templates/namespace.yaml @@ -0,0 +1,23 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- include "substrate.validateAuthMode" . -}} +{{- if .Values.createNamespace }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Release.Namespace }} +{{- end }} diff --git a/charts/substrate/templates/pod-certificate-controller.yaml b/charts/substrate/templates/pod-certificate-controller.yaml new file mode 100644 index 000000000..3aaaa9df9 --- /dev/null +++ b/charts/substrate/templates/pod-certificate-controller.yaml @@ -0,0 +1,200 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if eq .Values.auth.mode "mtls" -}} +apiVersion: v1 +kind: Namespace +metadata: + name: podcertificate-controller-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "substrate.fullname" (list "podcert-ate-dev-signer" .) }} +rules: +# The service signer needs to be able to read services and pods. +- apiGroups: + - "" + resources: + - services + - pods + verbs: + - get + - list + - watch +- apiGroups: + - certificates.k8s.io + resources: + - podcertificaterequests + verbs: + - get + - list + - watch + - update +- apiGroups: + - certificates.k8s.io + resources: + - clustertrustbundles + verbs: + - create + - get + - list + - watch + - update + - delete +- apiGroups: + - certificates.k8s.io + resources: + - podcertificaterequests/status + verbs: + - update +- apiGroups: + - certificates.k8s.io + resources: + - signers + resourceNames: + - servicedns.podcert.ate.dev/* + - podidentity.podcert.ate.dev/* + verbs: + - sign + - attest +- apiGroups: + - events.k8s.io + resources: + - events + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "podcert-ate-dev-signer" .) }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "substrate.fullname" (list "podcert-ate-dev-signer" .) }} +subjects: +- kind: ServiceAccount + namespace: podcertificate-controller-system + name: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: podcertificate-controller-system + name: coordinator +rules: +- apiGroups: + - "coordination.k8s.io" + resources: + - "leases" + verbs: + - create + - get + - list + - watch + - update + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: podcertificate-controller-is-a-coordinator + namespace: podcertificate-controller-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: coordinator +subjects: +- kind: ServiceAccount + namespace: podcertificate-controller-system + name: default +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: podcertificate-controller + namespace: podcertificate-controller-system + labels: + app: podcertificate-controller +spec: + replicas: 1 + selector: + matchLabels: + app: podcertificate-controller + template: + metadata: + labels: + app: podcertificate-controller + spec: + containers: + - name: controller + image: {{ include "substrate.componentImage" (list "podcertcontroller" .) }} + args: + - --in-cluster=true + - --sharding-pod-namespace=$(POD_NAMESPACE) + - --sharding-pod-name=$(POD_NAME) + - --sharding-pod-uid=$(POD_UID) + - --sharding-application-name=podcertificate-controller + - --service-dns-ca-pool=/run/ca-state/service-dns-pool.json + - --pod-identity-ca-pool=/run/ca-state/pod-identity-pool.json + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + volumeMounts: + - name: "ca-state" + mountPath: "/run/ca-state" + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - ALL + readOnlyRootFilesystem: true + volumes: + - name: "ca-state" + projected: + sources: + - secret: + name: "service-dns-ca-pool" + items: + - key: "pool" + path: "service-dns-pool.json" + - secret: + name: "pod-identity-ca-pool" + items: + - key: "pool" + path: "pod-identity-pool.json" + dnsPolicy: Default + nodeSelector: + kubernetes.io/os: linux + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + serviceAccountName: default + terminationGracePeriodSeconds: 30 +{{- end }} diff --git a/manifests/ate-install/generated/role.yaml b/charts/substrate/templates/role.yaml similarity index 95% rename from manifests/ate-install/generated/role.yaml rename to charts/substrate/templates/role.yaml index 7341d28dd..8e3f7117d 100644 --- a/manifests/ate-install/generated/role.yaml +++ b/charts/substrate/templates/role.yaml @@ -16,7 +16,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: ate-controller + name: {{ include "substrate.fullname" (list "ate-controller" .) }} rules: - apiGroups: - "" diff --git a/charts/substrate/templates/rustfs.yaml b/charts/substrate/templates/rustfs.yaml new file mode 100644 index 000000000..6e71c16d5 --- /dev/null +++ b/charts/substrate/templates/rustfs.yaml @@ -0,0 +1,141 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.rustfs.enabled -}} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "substrate.fullname" (list "rustfs-data" .) }} + namespace: {{ .Release.Namespace }} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.rustfs.storageSize }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "substrate.fullname" (list "rustfs" .) }} + namespace: {{ .Release.Namespace }} +spec: + selector: + app: rustfs + ports: + - name: api + port: 9000 + targetPort: 9000 + - name: console + port: 9001 + targetPort: 9001 + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "substrate.fullname" (list "rustfs" .) }} + namespace: {{ .Release.Namespace }} +spec: + replicas: 1 + selector: + matchLabels: + app: rustfs + template: + metadata: + labels: + app: rustfs + spec: + {{- with (or .Values.rustfs.nodeSelector .Values.nodeSelector) }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + containers: + - name: rustfs + image: {{ .Values.images.rustfs }} + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9000 + name: api + - containerPort: 9001 + name: console + env: + - name: RUSTFS_ADDRESS + value: ":9000" + - name: RUSTFS_CONSOLE_ADDRESS + value: ":9001" + - name: RUSTFS_CONSOLE_ENABLE + value: "true" + - name: RUSTFS_VOLUMES + value: "/data" + - name: RUSTFS_ACCESS_KEY + value: {{ .Values.rustfs.accessKey | quote }} + - name: RUSTFS_SECRET_KEY + value: {{ .Values.rustfs.secretKey | quote }} + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + persistentVolumeClaim: + claimName: {{ include "substrate.fullname" (list "rustfs-data" .) }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "substrate.fullname" (list "rustfs-bucket-init" .) }} + namespace: {{ .Release.Namespace }} +spec: + backoffLimit: 10 + template: + spec: + restartPolicy: OnFailure + containers: + - name: create-bucket + image: {{ .Values.images.awsCli }} + env: + - name: AWS_ACCESS_KEY_ID + value: {{ .Values.rustfs.accessKey | quote }} + - name: AWS_SECRET_ACCESS_KEY + value: {{ .Values.rustfs.secretKey | quote }} + - name: AWS_REGION + value: us-east-1 + - name: AWS_ENDPOINT_URL + value: http://{{ include "substrate.fullname" (list "rustfs" .) }}.{{ .Release.Namespace }}.svc:9000 + command: + - /bin/sh + - -c + - | + set -e + for i in $(seq 1 60); do + if aws s3api head-bucket --bucket {{ .Values.rustfs.bucket }} 2>/dev/null; then + echo "bucket {{ .Values.rustfs.bucket }} already exists" + exit 0 + fi + if aws s3api create-bucket --bucket {{ .Values.rustfs.bucket }} 2>/dev/null; then + echo "bucket {{ .Values.rustfs.bucket }} created" + exit 0 + fi + echo "waiting for rustfs to become available... ($i/60)" + sleep 2 + done + echo "timed out waiting for rustfs" + exit 1 +{{- end }} diff --git a/charts/substrate/templates/sandboxconfig-gvisor.yaml b/charts/substrate/templates/sandboxconfig-gvisor.yaml new file mode 100644 index 000000000..3dc4e9d16 --- /dev/null +++ b/charts/substrate/templates/sandboxconfig-gvisor.yaml @@ -0,0 +1,37 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# Cluster-wide default SandboxConfig for the gVisor (runsc) sandbox class. A +# WorkerPool with sandboxClass gvisor (the default) and no explicit +# sandboxConfigName resolves to this. atelet fetches the runsc binary matching +# the worker node's architecture. To pin a different runsc, edit the assets +# below or create another SandboxConfig and name it from the WorkerPool. +apiVersion: ate.dev/v1alpha1 +kind: SandboxConfig +metadata: + name: gvisor-default +spec: + sandboxClass: gvisor + default: true + assets: + amd64: + runsc: + url: "gs://gvisor/releases/release/20260622/x86_64/runsc" + sha256: "f18a948bf9c8bbb54eb998549a3a8d719a1c7de2efbe8fdd2ff0ee5fecd06f19" + arm64: + runsc: + url: "gs://gvisor/releases/release/20260622/aarch64/runsc" + sha256: "62eee121f8c188e347c428acc96f111568ede3be37b906046b6f28bbe2cc40c0" diff --git a/charts/substrate/templates/sandboxconfig-validation.yaml b/charts/substrate/templates/sandboxconfig-validation.yaml new file mode 100644 index 000000000..48302d2fd --- /dev/null +++ b/charts/substrate/templates/sandboxconfig-validation.yaml @@ -0,0 +1,56 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# Per-sandbox-class asset requirements for SandboxConfig. The CRD schema is +# generic (any arch -> any asset name -> {url, sha256}); this policy enforces the +# requirements a given sandbox class actually needs, fail-closed at apply time. +# (url/sha256 being required and well-formed is enforced by the CRD schema.) +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: sandboxconfig-assets +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: ["ate.dev"] + apiVersions: ["v1alpha1"] + operations: ["CREATE", "UPDATE"] + resources: ["sandboxconfigs"] + validations: + # gVisor needs a "runsc" asset for every architecture it advertises. + - expression: >- + object.spec.sandboxClass != 'gvisor' || + (has(object.spec.assets) && size(object.spec.assets) > 0 && + object.spec.assets.all(arch, 'runsc' in object.spec.assets[arch])) + message: "a gvisor SandboxConfig must define a 'runsc' asset for every architecture under spec.assets" + # The micro-VM (cloud-hypervisor) runtime needs its asset set for every + # architecture it advertises. + - expression: >- + object.spec.sandboxClass != 'microvm' || + (has(object.spec.assets) && size(object.spec.assets) > 0 && + object.spec.assets.all(arch, + ['cloud-hypervisor', 'virtiofsd', 'kata-kernel', 'kata-image', 'kata-config'] + .all(name, name in object.spec.assets[arch]))) + message: "a microvm SandboxConfig must define cloud-hypervisor, virtiofsd, kata-kernel, kata-image, and kata-config assets for every architecture under spec.assets" +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: sandboxconfig-assets +spec: + policyName: sandboxconfig-assets + validationActions: ["Deny"] diff --git a/charts/substrate/templates/valkey.yaml b/charts/substrate/templates/valkey.yaml new file mode 100644 index 000000000..b8233eea9 --- /dev/null +++ b/charts/substrate/templates/valkey.yaml @@ -0,0 +1,253 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.valkey.enabled -}} +{{- $sts := include "substrate.fullname" (list "valkey-cluster" .) -}} +{{- $headless := include "substrate.fullname" (list "valkey-cluster-service" .) -}} +{{- $ns := .Release.Namespace -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "substrate.fullname" (list "valkey-config" .) }} + namespace: {{ .Release.Namespace }} +data: + valkey.conf: | +{{- if eq .Values.auth.mode "mtls" }} + # Enforce TLS and disable standard port + port 0 + tls-port 6379 + tls-cluster yes + tls-replication yes + + # Load certificates from projected volume + tls-cert-file /run/servicedns.podcert.ate.dev/credential-bundle.pem + tls-key-file /run/servicedns.podcert.ate.dev/credential-bundle.pem + tls-ca-cert-file /etc/valkey-ca/ca.crt + tls-auth-clients yes + + # Enable cluster mode +{{- else }} + # Plaintext: serve on the standard port, no TLS. + port 6379 + +{{- end }} + cluster-enabled yes + cluster-config-file nodes.conf + cluster-node-timeout 5000 + appendonly yes + protected-mode no +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $headless }} + namespace: {{ .Release.Namespace }} +spec: + clusterIP: None + selector: + app: valkey-cluster + ports: + - name: valkey + port: 6379 + targetPort: 6379 + - name: bus + port: 16379 + targetPort: 16379 +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $sts }} + namespace: {{ .Release.Namespace }} +spec: + selector: + app: valkey-cluster + ports: + - name: valkey + port: 6379 + targetPort: 6379 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ $sts }} + namespace: {{ .Release.Namespace }} +spec: + serviceName: {{ $headless }} + replicas: {{ .Values.valkey.replicas }} + podManagementPolicy: Parallel + selector: + matchLabels: + app: valkey-cluster + template: + metadata: + labels: + app: valkey-cluster + spec: + containers: + - name: valkey + image: {{ .Values.images.valkey }} + command: ["valkey-server", "/etc/valkey/valkey.conf"] + ports: + - name: valkey + containerPort: 6379 + - name: bus + containerPort: 16379 + volumeMounts: + - name: config + mountPath: /etc/valkey +{{- if eq .Values.auth.mode "mtls" }} + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + - name: valkey-ca-certs + mountPath: /etc/valkey-ca + readOnly: true +{{- end }} + - name: data + mountPath: /data + volumes: + - name: config + configMap: + name: {{ include "substrate.fullname" (list "valkey-config" .) }} +{{- if eq .Values.auth.mode "mtls" }} + - name: servicedns + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - name: valkey-ca-certs + projected: + sources: + - secret: + name: valkey-ca-certs + items: + - key: ca.crt + path: ca.crt +{{- end }} + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: {{ .Values.valkey.storageSize }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "substrate.fullname" (list "valkey-cluster-init" .) }} + namespace: {{ .Release.Namespace }} +spec: + template: + metadata: + labels: + app: valkey-cluster-init + spec: + restartPolicy: OnFailure + containers: + - name: init + image: {{ .Values.images.valkey }} +{{- if eq .Values.auth.mode "mtls" }} + volumeMounts: + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + - name: valkey-ca-certs + mountPath: /etc/valkey-ca + readOnly: true +{{- end }} + command: + - /bin/sh + - -c + - | + set -e + echo "Waiting for all Valkey pods to resolve..." + for i in 0 1 2 3 4 5; do + until getent hosts {{ $sts }}-${i}.{{ $headless }}.{{ $ns }}.svc >/dev/null 2>&1; do + echo "Waiting for {{ $sts }}-${i} DNS..." + sleep 2 + done + done + + echo "All pods resolved. Getting IPs..." + POD_IPS="" + for i in 0 1 2 3 4 5; do + ip=$(getent hosts {{ $sts }}-${i}.{{ $headless }}.{{ $ns }}.svc | awk '{print $1}') + POD_IPS="${POD_IPS} ${ip}:6379" + done + + echo "Checking if Valkey cluster is already initialized..." +{{- if eq .Values.auth.mode "mtls" }} + until valkey-cli --tls --cacert /etc/valkey-ca/ca.crt --cert /run/servicedns.podcert.ate.dev/credential-bundle.pem --key /run/servicedns.podcert.ate.dev/credential-bundle.pem -h {{ $sts }}-0.{{ $headless }}.{{ $ns }}.svc ping >/dev/null 2>&1; do + echo "Waiting for {{ $sts }}-0 to respond to ping..." + sleep 2 + done + + INIT_STATUS=$(valkey-cli --tls --cacert /etc/valkey-ca/ca.crt --cert /run/servicedns.podcert.ate.dev/credential-bundle.pem --key /run/servicedns.podcert.ate.dev/credential-bundle.pem -h {{ $sts }}-0.{{ $headless }}.{{ $ns }}.svc cluster info 2>/dev/null | grep cluster_state || true) + + if [ -z "${INIT_STATUS}" ] || ! echo "${INIT_STATUS}" | grep -q "cluster_state:ok"; then + echo "Initializing Valkey cluster..." + valkey-cli --tls \ + --cacert /etc/valkey-ca/ca.crt \ + --cert /run/servicedns.podcert.ate.dev/credential-bundle.pem \ + --key /run/servicedns.podcert.ate.dev/credential-bundle.pem \ + --cluster create ${POD_IPS} \ + --cluster-replicas 1 \ + --cluster-yes + echo "Cluster initialization complete!" + else + echo "Cluster already initialized." + fi +{{- else }} + until valkey-cli -h {{ $sts }}-0.{{ $headless }}.{{ $ns }}.svc -p 6379 ping >/dev/null 2>&1; do + echo "Waiting for {{ $sts }}-0 to respond to ping..." + sleep 2 + done + + INIT_STATUS=$(valkey-cli -h {{ $sts }}-0.{{ $headless }}.{{ $ns }}.svc -p 6379 cluster info 2>/dev/null | grep cluster_state || true) + + if [ -z "${INIT_STATUS}" ] || ! echo "${INIT_STATUS}" | grep -q "cluster_state:ok"; then + echo "Initializing Valkey cluster..." + valkey-cli \ + --cluster create ${POD_IPS} \ + --cluster-replicas 1 \ + --cluster-yes + echo "Cluster initialization complete!" + else + echo "Cluster already initialized." + fi +{{- end }} +{{- if eq .Values.auth.mode "mtls" }} + volumes: + - name: servicedns + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - name: valkey-ca-certs + projected: + sources: + - secret: + name: valkey-ca-certs + items: + - key: ca.crt + path: ca.crt +{{- end }} +{{- end }} diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml new file mode 100644 index 000000000..cc844605e --- /dev/null +++ b/charts/substrate/values.yaml @@ -0,0 +1,151 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for the substrate chart. +# +# The chart supports two installation modes via `auth.mode`: +# +# - "jwt" (default): No PodCertificateRequest / ClusterTrustBundle usage. Server +# certs and session signing pools are generated by the chart by default, +# and can be disabled when you want to provide your own key material. +# Clients authenticate to ateapi with a projected Kubernetes ServiceAccount +# token. Valkey runs plaintext. +# +# - "mtls": Server certs are issued by the in-cluster podcertcontroller via +# PodCertificateRequest + projected into pods via the ClusterTrustBundle / +# podCertificate projection sources. Valkey runs with full TLS + +# client-cert verification. REQUIRES the off-by-default Kubernetes feature +# gates: +# ClusterTrustBundle, ClusterTrustBundleProjection, PodCertificateRequest +# and the v1beta1 certificates API. + +auth: + mode: jwt # jwt | mtls + + jwt: + # OIDC issuer URL the cluster uses to mint SA tokens. The default matches + # stock kind/kubeadm-style clusters. Override this for managed clusters + # whose service account issuer is provider-specific. Examples: + # GKE: https://container.googleapis.com/v1/projects//locations//clusters/ + # kind: https://kubernetes.default.svc.cluster.local + # EKS: https://oidc.eks..amazonaws.com/id/ + issuer: https://kubernetes.default.svc.cluster.local + + # Audience SA tokens are minted for, and that ateapi expects. + audience: api.ate-system.svc + + bootstrap: + # Generate JWT-mode TLS and session-signing key material with Helm. + # Existing generated resources are reused on upgrade via lookup. + enabled: true + serverCert: + enabled: true + sessionPools: + enabled: true + + # Name of a kubernetes.io/tls Secret in the release namespace, with keys + # tls.crt and tls.key. Created by the chart when + # auth.jwt.bootstrap.serverCert.enabled=true. + serverCertSecret: ateapi-tls + + # Name of a ConfigMap in the release namespace with key "ca.crt" holding + # the CA(s) that signed serverCertSecret. Clients mount it to verify the + # ateapi server certificate. Created by the chart when + # auth.jwt.bootstrap.serverCert.enabled=true. + caBundleConfigMap: ateapi-ca + +# Set to true to have the chart create the release namespace. +# Off by default — most helm workflows expect the namespace to already exist +# (helm install -n --create-namespace). Enable for the generated +# manifests/ate-install/ install path (kubectl apply). +createNamespace: false + +# Global nodeSelector applied to the substrate workloads (ate-api-server, +# ate-controller, atenet-router, dns, rustfs, and the atelet DaemonSet). Each +# of those components also exposes its own `nodeSelector`; when set to a +# non-empty map it overrides this global default for that component. +nodeSelector: {} + +valkey: + enabled: true + replicas: 6 + storageSize: 1Gi + +rustfs: + enabled: true + storageSize: 1Gi + bucket: ate-snapshots + accessKey: rustfsadmin + secretKey: rustfsadmin + # Overrides the global .Values.nodeSelector for the rustfs deployment. + nodeSelector: {} + +# atelet daemonset overrides. Defaults use the in-cluster RustFS deployment for +# snapshots. Set rustfs.enabled=false and override these fields when using +# external storage. +# extraArgs / extraEnv are appended verbatim for installer-specific knobs +# (e.g. registry replacement for kind). +atelet: + gcpAuthForImagePulls: false + storageBackend: s3 + extraArgs: [] + extraEnv: [] + # Overrides the global .Values.nodeSelector for the atelet DaemonSet. + nodeSelector: {} + +# Per-component overrides for the control-plane deployments. Each nodeSelector, +# when set to a non-empty map, overrides the global .Values.nodeSelector for +# that component only. +ateApiServer: + nodeSelector: {} + +ateController: + nodeSelector: {} + +atenetRouter: + nodeSelector: {} + +dns: + nodeSelector: {} + +redis: + # Override the cluster address. Empty -> derived from valkey.enabled + # (defaults to "valkey-cluster.ate-system.svc:6379"). + clusterAddress: "" + # Google IAM auth (for managed Memorystore / cloud Valkey). + useIAMAuth: false + # Override TLS server name for Redis hostname verification (mtls mode). + tlsServerName: "" + # File path for Redis client TLS credential bundle (mtls mode). + clientCert: "" + +# Name of a ConfigMap in the release namespace that supplies per-environment +# overrides for ate-api-server (ATE_API_REDIS_*, ATE_API_K8SJWT_ISSUER, ...). +# Mounted via envFrom with optional=true. Created by the chart from these values. +ateApiServerEnvVarsConfigMap: ate-api-server-envvars + +otel: + endpoint: "" + +image: + registry: ghcr.io/kagent-dev/substrate + tag: "" + +images: + valkey: valkey/valkey:8.0 + rustfs: rustfs/rustfs:1.0.0-beta.3@sha256:378642b05b7dcb4849fb77ebe6aca4ced1c3f66e7e504247df95a5c9018d3358 + awsCli: amazon/aws-cli:2.17.0@sha256:643507c10ada7964ca6157b3d799f030b90577643da9955d319a77399ed80d73 + agentgateway: cr.agentgateway.dev/agentgateway:v1.3.0-alpha.1 + coredns: coredns/coredns:1.11.1 + busybox: busybox:1.36 diff --git a/cmd/ateapi/internal/controlapi/functional_test.go b/cmd/ateapi/internal/controlapi/functional_test.go index 6d5f6155e..cc575638d 100644 --- a/cmd/ateapi/internal/controlapi/functional_test.go +++ b/cmd/ateapi/internal/controlapi/functional_test.go @@ -29,6 +29,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" "github.com/agent-substrate/substrate/internal/ateinterceptors" "github.com/agent-substrate/substrate/internal/envtestbins" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/proto/ateletpb" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/client/clientset/versioned" @@ -236,6 +237,7 @@ type testContext struct { k8sClient kubernetes.Interface substrateClient versioned.Interface persistence *ateredis.Persistence + workerCache *workercache.Cache fakeAtelet *FakeAteletServer cleanup func() actorTemplateLister listersv1alpha1.ActorTemplateLister @@ -272,7 +274,7 @@ func setupTest(t *testing.T, ns string) *testContext { // 3. Initialize Informers workerFactory, workerInformer := WorkerPodInformer(k8sClient) - ateletFactory, ateletInformer := AteletInformer(k8sClient) + ateletFactory, ateletInformer := AteletInformer(k8sClient, installdefaults.SystemNamespace) substrateInformerFactory := externalversions.NewSharedInformerFactory(substrateClient, 0) actorTemplateLister := substrateInformerFactory.Api().V1alpha1().ActorTemplates().Lister() @@ -301,7 +303,7 @@ func setupTest(t *testing.T, ns string) *testContext { } dialer := NewAteletDialer(workerInformer.GetIndexer(), ateletInformer.GetIndexer()) - service := NewService(persistence, wc, actorTemplateLister, workerPoolLister, sandboxConfigLister, dialer, k8sClient) + service := NewService(persistence, wc, actorTemplateLister, workerPoolLister, sandboxConfigLister, dialer, k8sClient, 30*time.Second) // 5. Start REAL gRPC Server for ATE API grpcServer := grpc.NewServer(grpc.UnaryInterceptor(ateinterceptors.ServerUnaryInterceptor)) @@ -368,6 +370,7 @@ func setupTest(t *testing.T, ns string) *testContext { k8sClient: k8sClient, substrateClient: substrateClient, persistence: persistence, + workerCache: wc, fakeAtelet: fakeAtelet, cleanup: cleanup, actorTemplateLister: actorTemplateLister, @@ -582,13 +585,13 @@ func createWorkerPod(t *testing.T, tc *testContext, ns string, name string, node t.Fatalf("failed to update worker pod status: %v", err) } - // Wait for worker to be registered via API + // Wait for worker to be visible to the scheduler. err = wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) { - resp, err := tc.client.ListWorkers(ctx, &ateapipb.ListWorkersRequest{}) + workers, err := tc.workerCache.Workers() if err != nil { - return false, nil // Retry on API error + return false, nil } - for _, w := range resp.GetWorkers() { + for _, w := range workers { if w.GetWorkerNamespace() == ns && w.GetWorkerPod() == name { return true, nil } @@ -607,13 +610,13 @@ func deleteWorkerPod(t *testing.T, tc *testContext, ns string, name string) { t.Fatalf("failed to delete worker pod %s: %v", name, err) } - // Wait for worker to be removed from API + // Wait for worker to be removed from the scheduler. err = wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) { - resp, err := tc.client.ListWorkers(ctx, &ateapipb.ListWorkersRequest{}) + workers, err := tc.workerCache.Workers() if err != nil { - return false, nil // Retry on API error + return false, nil } - for _, w := range resp.GetWorkers() { + for _, w := range workers { if w.GetWorkerNamespace() == ns && w.GetWorkerPod() == name { return false, nil // Still there } diff --git a/cmd/ateapi/internal/controlapi/informer.go b/cmd/ateapi/internal/controlapi/informer.go index 1f082cdd0..fcaa6c3ec 100644 --- a/cmd/ateapi/internal/controlapi/informer.go +++ b/cmd/ateapi/internal/controlapi/informer.go @@ -25,15 +25,15 @@ import ( ) const ( - ateletNamespace = "ate-system" byNamespaceAndName = "by-namespace-and-name" byWorkerPool = "by-worker-pool" byNode = "by-node" workerPodLabel = "ate.dev/worker-pool" ) -// AteletInformer creates a SharedInformerFactory and SharedIndexInformer for Atelet pods. -func AteletInformer(kc kubernetes.Interface) (informers.SharedInformerFactory, cache.SharedIndexInformer) { +// AteletInformer creates a SharedInformerFactory and SharedIndexInformer for +// Atelet pods in the given namespace. +func AteletInformer(kc kubernetes.Interface, ateletNamespace string) (informers.SharedInformerFactory, cache.SharedIndexInformer) { factory := informers.NewSharedInformerFactoryWithOptions(kc, 0, informers.WithNamespace(ateletNamespace), informers.WithTweakListOptions(func(options *metav1.ListOptions) { diff --git a/cmd/ateapi/internal/controlapi/service.go b/cmd/ateapi/internal/controlapi/service.go index 7ec695b00..a13a33b4b 100644 --- a/cmd/ateapi/internal/controlapi/service.go +++ b/cmd/ateapi/internal/controlapi/service.go @@ -15,6 +15,8 @@ package controlapi import ( + "time" + "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" @@ -34,7 +36,8 @@ type Service struct { var _ ateapipb.ControlServer = (*Service)(nil) -// NewService creates a service. +// NewService creates a service. actorWorkflowDeadline bounds how long a single +// Resume/Suspend workflow can run end-to-end. func NewService( persistence store.Interface, workerCache *workercache.Cache, @@ -43,13 +46,14 @@ func NewService( sandboxConfigLister listersv1alpha1.SandboxConfigLister, dialer *AteletDialer, kubeClient kubernetes.Interface, + actorWorkflowDeadline time.Duration, ) *Service { s := &Service{ persistence: persistence, actorTemplateLister: actorTemplateLister, workerPoolLister: workerPoolLister, dialer: dialer, - actorWorkflow: NewActorWorkflow(persistence, workerCache, dialer, actorTemplateLister, workerPoolLister, sandboxConfigLister, kubeClient), + actorWorkflow: NewActorWorkflow(persistence, workerCache, dialer, actorTemplateLister, workerPoolLister, sandboxConfigLister, kubeClient, actorWorkflowDeadline), } return s diff --git a/cmd/ateapi/internal/controlapi/workflow.go b/cmd/ateapi/internal/controlapi/workflow.go index 25c4f36ac..7465914ef 100644 --- a/cmd/ateapi/internal/controlapi/workflow.go +++ b/cmd/ateapi/internal/controlapi/workflow.go @@ -18,6 +18,7 @@ import ( "context" "errors" "fmt" + "log/slog" "time" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" @@ -33,6 +34,18 @@ import ( "k8s.io/client-go/kubernetes" ) +// actorLockTTL is the Redis TTL on the per-actor workflow lock. It bounds how +// long a peer must wait to retry an actor after this process crashes mid-workflow. +const actorLockTTL = 30 * time.Second + +// actorLockHeartbeatInterval is how often the heartbeat refreshes the lock. +// Chosen so we get ~3 attempts before the TTL would otherwise lapse. +const actorLockHeartbeatInterval = actorLockTTL / 3 + +// errLostActorLock is the context cause set when the heartbeat can no longer +// keep the actor lock alive (peer stole it, or Redis returned an error). +var errLostActorLock = errors.New("lost actor lock during workflow") + // WorkflowStep represents a single, idempotent operation in a workflow graph. // Params is the immutable parameters used to start the workflow. // Context is the mutable context fetched or modified during execution. @@ -123,9 +136,14 @@ type ActorWorkflow struct { sandboxConfigLister listersv1alpha1.SandboxConfigLister kubeClient kubernetes.Interface secretCache *envSecretCache + // workflowDeadline is the maximum duration of a single Resume/Suspend + // workflow. The lock is kept alive across this duration by a heartbeat, + // independent of actorLockTTL. + workflowDeadline time.Duration } -// NewActorWorkflow creates a new ActorWorkflow. +// NewActorWorkflow creates a new ActorWorkflow. workflowDeadline bounds how +// long a single Resume/Suspend can run end-to-end. func NewActorWorkflow( store store.Interface, workerCache *workercache.Cache, @@ -134,6 +152,7 @@ func NewActorWorkflow( workerPoolLister listersv1alpha1.WorkerPoolLister, sandboxConfigLister listersv1alpha1.SandboxConfigLister, kubeClient kubernetes.Interface, + workflowDeadline time.Duration, ) *ActorWorkflow { return &ActorWorkflow{ store: store, @@ -144,6 +163,7 @@ func NewActorWorkflow( sandboxConfigLister: sandboxConfigLister, kubeClient: kubeClient, secretCache: newEnvSecretCache(envSecretCacheTTL), + workflowDeadline: workflowDeadline, } } @@ -156,9 +176,7 @@ func (w *ActorWorkflow) ResumeActor(ctx context.Context, atespace, id string, bo } state := &ResumeState{} - // Acquire lock and get the timeout context for the workflow - // Lock TTL is 30 seconds, with 2 seconds padding for workflow timeout - ctx, releaseLock, err := w.acquireActorLock(ctx, id, 30*time.Second, 2*time.Second) + ctx, releaseLock, err := w.acquireActorLock(ctx, id, actorLockTTL, actorLockHeartbeatInterval) if err != nil { return nil, err } @@ -186,9 +204,7 @@ func (w *ActorWorkflow) SuspendActor(ctx context.Context, atespace, id string) ( } state := &SuspendState{} - // Acquire lock and get the timeout context for the workflow - // Lock TTL is 30 seconds, with 2 seconds padding for workflow timeout - ctx, releaseLock, err := w.acquireActorLock(ctx, id, 30*time.Second, 2*time.Second) + ctx, releaseLock, err := w.acquireActorLock(ctx, id, actorLockTTL, actorLockHeartbeatInterval) if err != nil { return nil, err } @@ -216,9 +232,7 @@ func (w *ActorWorkflow) PauseActor(ctx context.Context, atespace, id string) (*a } state := &PauseState{} - // Acquire lock and get the timeout context for the workflow - // Lock TTL is 30 seconds, with 2 seconds padding for workflow timeout - ctx, releaseLock, err := w.acquireActorLock(ctx, id, 30*time.Second, 2*time.Second) + ctx, releaseLock, err := w.acquireActorLock(ctx, id, actorLockTTL, actorLockHeartbeatInterval) if err != nil { return nil, err } @@ -238,27 +252,71 @@ func (w *ActorWorkflow) PauseActor(ctx context.Context, atespace, id string) (*a return state.Actor, nil } -func (w *ActorWorkflow) acquireActorLock(ctx context.Context, id string, ttl time.Duration, padding time.Duration) (context.Context, func(), error) { +// acquireActorLock takes the per-actor workflow lock and returns a workflow +// context bounded by w.workflowDeadline. A background heartbeat keeps the lock +// alive — independent of lockTTL — for as long as the workflow runs. If the +// heartbeat fails (Redis error or another peer stole the lock) the returned +// context is cancelled with errLostActorLock as the cause, and in-flight steps +// will see ctx.Err() and unwind. The returned release function stops the +// heartbeat, waits for it to exit, then best-effort releases the lock. +func (w *ActorWorkflow) acquireActorLock(ctx context.Context, id string, lockTTL, heartbeatInterval time.Duration) (context.Context, func(), error) { lockKey := "lock:actor:" + id lockValue := uuid.New().String() - // Create a child context for the workflow that expires BEFORE the lock - workflowTimeout := ttl - padding - workflowCtx, cancel := context.WithTimeout(ctx, workflowTimeout) - - acquired, err := w.store.AcquireLock(workflowCtx, lockKey, lockValue, ttl) + acquired, err := w.store.AcquireLock(ctx, lockKey, lockValue, lockTTL) if err != nil { - cancel() return nil, nil, fmt.Errorf("while acquiring lock: %w", err) } if !acquired { - cancel() return nil, nil, status.Error(grpcCodes.Aborted, "another operation is in progress for this actor") } - return workflowCtx, func() { - cancel() + cancellableCtx, cancelCause := context.WithCancelCause(ctx) + workflowCtx, cancelDeadline := context.WithTimeout(cancellableCtx, w.workflowDeadline) + + heartbeatDone := make(chan struct{}) + go w.runLockHeartbeat(workflowCtx, lockKey, lockValue, id, lockTTL, heartbeatInterval, cancelCause, heartbeatDone) + + release := func() { + cancelDeadline() + cancelCause(context.Canceled) + <-heartbeatDone // Use context.Background() to ensure the lock is released even if the workflow context was canceled. w.store.ReleaseLock(context.Background(), lockKey, lockValue) //nolint:errcheck // best-effort release; the lock TTL is the safety net. - }, nil + } + return workflowCtx, release, nil +} + +// runLockHeartbeat refreshes the actor lock on a ticker until ctx is done. If +// a refresh fails or returns false (we no longer own the lock), it cancels the +// workflow context with errLostActorLock so workflow steps tear down promptly. +func (w *ActorWorkflow) runLockHeartbeat(ctx context.Context, lockKey, lockValue, actorID string, lockTTL, heartbeatInterval time.Duration, cancelCause context.CancelCauseFunc, done chan<- struct{}) { + defer close(done) + ticker := time.NewTicker(heartbeatInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + ok, err := w.store.RefreshLock(ctx, lockKey, lockValue, lockTTL) + if err != nil { + // If ctx was cancelled out from under us we're already tearing + // down — no need to set a misleading cause. + if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) { + slog.WarnContext(ctx, "Lock heartbeat failed; cancelling workflow", + slog.String("actor_id", actorID), + slog.String("err", err.Error())) + cancelCause(fmt.Errorf("%w: %w", errLostActorLock, err)) + } + return + } + if !ok { + slog.WarnContext(ctx, "Actor lock no longer owned; cancelling workflow", + slog.String("actor_id", actorID)) + cancelCause(errLostActorLock) + return + } + } + } } diff --git a/cmd/ateapi/internal/controlapi/workflow_test.go b/cmd/ateapi/internal/controlapi/workflow_test.go new file mode 100644 index 000000000..57d99e711 --- /dev/null +++ b/cmd/ateapi/internal/controlapi/workflow_test.go @@ -0,0 +1,124 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlapi + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/ateredis" + "github.com/alicebob/miniredis/v2" + "github.com/redis/go-redis/v9" +) + +func newLockTestWorkflow(t *testing.T) (*miniredis.Miniredis, *ActorWorkflow) { + t.Helper() + mr, err := miniredis.Run() + if err != nil { + t.Fatalf("miniredis.Run: %v", err) + } + t.Cleanup(mr.Close) + rdb := redis.NewClusterClient(&redis.ClusterOptions{Addrs: []string{mr.Addr()}}) + return mr, &ActorWorkflow{ + store: ateredis.NewPersistence(rdb), + workflowDeadline: 30 * time.Second, + } +} + +func TestAcquireActorLock_HeartbeatKeepsLockAlivePastTTL(t *testing.T) { + mr, w := newLockTestWorkflow(t) + + lockTTL := 150 * time.Millisecond + heartbeat := 40 * time.Millisecond + + ctx, release, err := w.acquireActorLock(context.Background(), "actor-1", lockTTL, heartbeat) + if err != nil { + t.Fatalf("acquireActorLock: %v", err) + } + defer release() + + // Wait through multiple TTLs. If the heartbeat is working the lock key + // must still be in Redis — its TTL is being PEXPIRE'd back to `lockTTL` + // every `heartbeat`. + time.Sleep(4 * lockTTL) + + if !mr.Exists("lock:actor:actor-1") { + t.Fatalf("lock key disappeared from Redis despite heartbeat; ctx err=%v cause=%v", ctx.Err(), context.Cause(ctx)) + } + if ctx.Err() != nil { + t.Fatalf("workflow ctx cancelled while heartbeat was healthy: err=%v cause=%v", ctx.Err(), context.Cause(ctx)) + } +} + +func TestAcquireActorLock_LostLockCancelsWorkflow(t *testing.T) { + mr, w := newLockTestWorkflow(t) + + lockTTL := 200 * time.Millisecond + heartbeat := 30 * time.Millisecond + + ctx, release, err := w.acquireActorLock(context.Background(), "actor-2", lockTTL, heartbeat) + if err != nil { + t.Fatalf("acquireActorLock: %v", err) + } + defer release() + + // Simulate a peer stealing the lock (or the TTL lapsing and someone else + // re-acquiring): wipe our key so the next heartbeat refresh's CAS fails. + mr.Del("lock:actor:actor-2") + + select { + case <-ctx.Done(): + case <-time.After(2 * time.Second): + t.Fatalf("workflow ctx was not cancelled after lock was lost") + } + + if cause := context.Cause(ctx); !errors.Is(cause, errLostActorLock) { + t.Errorf("context.Cause = %v, want errLostActorLock", cause) + } +} + +func TestAcquireActorLock_ReleaseRemovesLock(t *testing.T) { + mr, w := newLockTestWorkflow(t) + + _, release, err := w.acquireActorLock(context.Background(), "actor-3", 200*time.Millisecond, 60*time.Millisecond) + if err != nil { + t.Fatalf("acquireActorLock: %v", err) + } + + if !mr.Exists("lock:actor:actor-3") { + t.Fatalf("lock key not in Redis after acquire") + } + release() + if mr.Exists("lock:actor:actor-3") { + t.Errorf("lock key still in Redis after release") + } +} + +func TestAcquireActorLock_ConflictReturnsAborted(t *testing.T) { + _, w := newLockTestWorkflow(t) + + _, release, err := w.acquireActorLock(context.Background(), "actor-4", 5*time.Second, 1*time.Second) + if err != nil { + t.Fatalf("first acquireActorLock: %v", err) + } + defer release() + + _, _, err = w.acquireActorLock(context.Background(), "actor-4", 5*time.Second, 1*time.Second) + if err == nil { + t.Fatalf("expected second acquireActorLock to fail") + } +} diff --git a/cmd/ateapi/internal/credbundle/credbundle.go b/cmd/ateapi/internal/credbundle/credbundle.go index 58d86df76..281840adc 100644 --- a/cmd/ateapi/internal/credbundle/credbundle.go +++ b/cmd/ateapi/internal/credbundle/credbundle.go @@ -20,6 +20,7 @@ package credbundle import ( + "crypto" "crypto/tls" "crypto/x509" "encoding/pem" @@ -47,6 +48,7 @@ func Parse(bundlePath string) (*tls.Certificate, error) { } var leafKeyBytes []byte + var leafKeyBlockType string var chainBytes [][]byte for { @@ -59,8 +61,9 @@ func Parse(bundlePath string) (*tls.Certificate, error) { switch block.Type { case "CERTIFICATE": chainBytes = append(chainBytes, block.Bytes) - case "PRIVATE KEY": + case "PRIVATE KEY", "RSA PRIVATE KEY", "EC PRIVATE KEY": leafKeyBytes = block.Bytes + leafKeyBlockType = block.Type default: return nil, fmt.Errorf("unknown PEM block type %q", block.Type) } @@ -74,7 +77,7 @@ func Parse(bundlePath string) (*tls.Certificate, error) { return nil, fmt.Errorf("no CERTIFICATE blocks found") } - leafKey, err := x509.ParsePKCS8PrivateKey(leafKeyBytes) + leafKey, err := parsePrivateKey(leafKeyBlockType, leafKeyBytes) if err != nil { return nil, fmt.Errorf("while parsing private key: %w", err) } @@ -90,3 +93,16 @@ func Parse(bundlePath string) (*tls.Certificate, error) { PrivateKey: leafKey, }, nil } + +func parsePrivateKey(blockType string, keyBytes []byte) (crypto.PrivateKey, error) { + switch blockType { + case "PRIVATE KEY": + return x509.ParsePKCS8PrivateKey(keyBytes) + case "RSA PRIVATE KEY": + return x509.ParsePKCS1PrivateKey(keyBytes) + case "EC PRIVATE KEY": + return x509.ParseECPrivateKey(keyBytes) + default: + return nil, fmt.Errorf("unsupported private key block type %q", blockType) + } +} diff --git a/cmd/ateapi/internal/credbundle/credbundle_test.go b/cmd/ateapi/internal/credbundle/credbundle_test.go index 2b426eb6e..c435829fe 100644 --- a/cmd/ateapi/internal/credbundle/credbundle_test.go +++ b/cmd/ateapi/internal/credbundle/credbundle_test.go @@ -51,13 +51,13 @@ func TestParsePKCS8PrivateKeyBlock(t *testing.T) { } } -func TestParseRejectsNonPKCS8PrivateKeyBlock(t *testing.T) { +func TestParseRSAPrivateKeyBlock(t *testing.T) { certDER := generateCertificate(t) bundle := append(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}), pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(generateRSAKey(t))})...) bundlePath := writeBundle(t, bundle) - if _, err := Parse(bundlePath); err == nil { - t.Fatalf("Parse() error = nil, want unsupported private key block error") + if _, err := Parse(bundlePath); err != nil { + t.Fatalf("Parse() error = %v", err) } } diff --git a/cmd/ateapi/internal/sessionidentity/sessionidentity.go b/cmd/ateapi/internal/sessionidentity/sessionidentity.go index a9a581271..13ff37647 100644 --- a/cmd/ateapi/internal/sessionidentity/sessionidentity.go +++ b/cmd/ateapi/internal/sessionidentity/sessionidentity.go @@ -28,8 +28,8 @@ import ( "strings" "time" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/k8sjwt" "github.com/agent-substrate/substrate/cmd/ateapi/internal/sessionidjwt" + "github.com/agent-substrate/substrate/internal/k8sjwt" "github.com/agent-substrate/substrate/internal/localca" "github.com/agent-substrate/substrate/internal/localjwtauthority" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" diff --git a/cmd/ateapi/internal/store/ateredis/ateredis.go b/cmd/ateapi/internal/store/ateredis/ateredis.go index 2a27fa7c1..e4bad74dd 100644 --- a/cmd/ateapi/internal/store/ateredis/ateredis.go +++ b/cmd/ateapi/internal/store/ateredis/ateredis.go @@ -799,3 +799,23 @@ func (s *Persistence) ReleaseLock(ctx context.Context, key string, value string) } return nil } + +func (s *Persistence) RefreshLock(ctx context.Context, key string, value string, ttl time.Duration) (bool, error) { + var luaRefresh = redis.NewScript(` + if redis.call("get", KEYS[1]) == ARGV[1] then + return redis.call("pexpire", KEYS[1], ARGV[2]) + else + return 0 + end + `) + + res, err := luaRefresh.Run(ctx, s.rdb, []string{key}, value, ttl.Milliseconds()).Result() + if err != nil { + return false, fmt.Errorf("while refreshing lock for %q with value %q: %w", key, value, err) + } + n, ok := res.(int64) + if !ok { + return false, fmt.Errorf("while refreshing lock for %q: unexpected result type %T", key, res) + } + return n == 1, nil +} diff --git a/cmd/ateapi/internal/store/ateredis/ateredis_test.go b/cmd/ateapi/internal/store/ateredis/ateredis_test.go index 3d5355151..8d69e3488 100644 --- a/cmd/ateapi/internal/store/ateredis/ateredis_test.go +++ b/cmd/ateapi/internal/store/ateredis/ateredis_test.go @@ -807,6 +807,97 @@ func receiveEvent(t *testing.T, ch <-chan store.WorkerEvent) store.WorkerEvent { } } +func TestRefreshLock_ExtendsTTL(t *testing.T) { + mr, s, ctx := setupTest(t) + defer mr.Close() + + key := "test-lock" + value := "token-1" + otherValue := "token-2" + ttl := 10 * time.Second + + acquired, err := s.AcquireLock(ctx, key, value, ttl) + if err != nil { + t.Fatalf("AcquireLock failed: %v", err) + } + if !acquired { + t.Fatalf("expected lock to be acquired") + } + + // Advance to just before expiry, refresh, and verify another holder still + // can't acquire after the original TTL would have elapsed. + mr.FastForward(8 * time.Second) + refreshed, err := s.RefreshLock(ctx, key, value, ttl) + if err != nil { + t.Fatalf("RefreshLock failed: %v", err) + } + if !refreshed { + t.Fatalf("expected lock to be refreshed") + } + + mr.FastForward(5 * time.Second) + stolen, err := s.AcquireLock(ctx, key, otherValue, ttl) + if err != nil { + t.Fatalf("AcquireLock failed: %v", err) + } + if stolen { + t.Errorf("expected refreshed lock to still be held, but it was stolen") + } +} + +func TestRefreshLock_WrongValueReturnsFalse(t *testing.T) { + mr, s, ctx := setupTest(t) + defer mr.Close() + + key := "test-lock" + value := "token-1" + otherValue := "token-2" + ttl := 10 * time.Second + + acquired, err := s.AcquireLock(ctx, key, value, ttl) + if err != nil { + t.Fatalf("AcquireLock failed: %v", err) + } + if !acquired { + t.Fatalf("expected lock to be acquired") + } + + refreshed, err := s.RefreshLock(ctx, key, otherValue, ttl) + if err != nil { + t.Fatalf("RefreshLock failed: %v", err) + } + if refreshed { + t.Errorf("expected RefreshLock with wrong value to return false") + } +} + +func TestRefreshLock_AfterExpirationReturnsFalse(t *testing.T) { + mr, s, ctx := setupTest(t) + defer mr.Close() + + key := "test-lock" + value := "token-1" + ttl := 5 * time.Second + + acquired, err := s.AcquireLock(ctx, key, value, ttl) + if err != nil { + t.Fatalf("AcquireLock failed: %v", err) + } + if !acquired { + t.Fatalf("expected lock to be acquired") + } + + mr.FastForward(6 * time.Second) + + refreshed, err := s.RefreshLock(ctx, key, value, ttl) + if err != nil { + t.Fatalf("RefreshLock failed: %v", err) + } + if refreshed { + t.Errorf("expected RefreshLock after expiration to return false") + } +} + func TestAcquireLock_NonReentry(t *testing.T) { mr, s, ctx := setupTest(t) defer mr.Close() diff --git a/cmd/ateapi/internal/store/store.go b/cmd/ateapi/internal/store/store.go index 5de4bdc30..287df48a0 100644 --- a/cmd/ateapi/internal/store/store.go +++ b/cmd/ateapi/internal/store/store.go @@ -105,6 +105,12 @@ type Interface interface { // Returns an error only on database failure. ReleaseLock(ctx context.Context, key string, value string) error + // RefreshLock extends the TTL on a distributed lock iff the stored value still matches. + // Returns true if the TTL was extended. + // Returns false if we no longer own the lock (either expired and reclaimed, or never held). + // Returns an error only on database failure. + RefreshLock(ctx context.Context, key string, value string, ttl time.Duration) (bool, error) + // DebugClearAll drop all data from the database. Useful for debugging / local testing/ DebugClearAll(ctx context.Context) error } diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index c4b3724c4..7236c8819 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -29,12 +29,13 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/controlapi" "github.com/agent-substrate/substrate/cmd/ateapi/internal/credbundle" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/k8sjwt" "github.com/agent-substrate/substrate/cmd/ateapi/internal/sessionidentity" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/ateredis" "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" "github.com/agent-substrate/substrate/internal/ateapiauth" "github.com/agent-substrate/substrate/internal/ateinterceptors" + "github.com/agent-substrate/substrate/internal/installdefaults" + "github.com/agent-substrate/substrate/internal/k8sjwt" "github.com/agent-substrate/substrate/internal/serverboot" "github.com/agent-substrate/substrate/internal/version" "github.com/agent-substrate/substrate/pkg/client/clientset/versioned" @@ -62,6 +63,7 @@ var ( redisUseIAMAuth = pflag.String("redis-use-iam-auth", "true", "Whether to use Google IAM authentication for Redis/Valkey.") redisTLSServerName = pflag.String("redis-tls-server-name", "", "The ServerName to use for Redis TLS hostname verification.") redisClientCert = pflag.String("redis-client-cert", "", "The file containing client TLS certificate/key credential bundle for Redis/Valkey.") + redisNoTLS = pflag.Bool("redis-no-tls", false, "If true, connect to Redis/Valkey in plaintext.") clientJWTIssuer = pflag.String("client-jwt-issuer", "", "The expected issuer URL for client JWTs.") clientJWTAudience = pflag.String("client-jwt-audience", "", "The expected audience for client JWTs.") @@ -70,6 +72,8 @@ var ( sessionIDCAPoolFile = pflag.String("session-id-ca-pool", "", "The file that contains the CA pool for signing session JWTs") workerpoolCACerts = pflag.String("workerpool-ca-certs", "", "The file that contains the CA for verifying workerpool client certificates.") + actorWorkflowDeadline = pflag.Duration("actor-workflow-deadline", 5*time.Minute, "Maximum wall-clock duration of a single Resume/Suspend workflow. A heartbeat keeps the per-actor lock alive across this duration; raise it for slow image registries.") + showVersion = pflag.Bool("version", false, "Print version and exit.") authMode = pflag.String("auth-mode", "mtls", "Auth mode for incoming gRPC: mtls|jwt. 'mtls' (default) relies on transport-level mTLS for client identity. 'jwt' additionally requires a Kubernetes ServiceAccount Bearer token on every RPC. Substrate will drop support for JWT auth mode once the Pod Certificates feature is enabled by default in the minimum supported Kubernetes version.") clientJWTCAFile = pflag.String("client-jwt-ca-cert", ateapiauth.DefaultServiceAccountCAFile, "CA cert file used to verify TLS when fetching the OIDC discovery document and JWKS for JWT authentication. Defaults to the in-cluster service account CA.") @@ -134,8 +138,13 @@ func main() { workerPoolLister := ateFactory.Api().V1alpha1().WorkerPools().Lister() sandboxConfigLister := ateFactory.Api().V1alpha1().SandboxConfigs().Lister() + // atelet shares ateapi's namespace in every supported deployment topology, + // so we read it from Kubernetes' downward API rather than expose a flag. + ateletNamespace := installdefaults.NamespaceFromPodEnv() + slog.InfoContext(ctx, "Resolved atelet namespace", slog.String("atelet-namespace", ateletNamespace)) + workerPodInformerFactory, workerPodInformer := controlapi.WorkerPodInformer(clientset) - ateletPodInformerFactory, ateletPodInformer := controlapi.AteletInformer(clientset) + ateletPodInformerFactory, ateletPodInformer := controlapi.AteletInformer(clientset, ateletNamespace) syncer := controlapi.NewWorkerPoolSyncer(redisPersistence, workerPodInformer) syncer.Start(ctx) @@ -151,7 +160,7 @@ func main() { ateFactory.WaitForCacheSync(stopCh) dialer := controlapi.NewAteletDialer(workerPodInformer.GetIndexer(), ateletPodInformer.GetIndexer()) - sm := controlapi.NewService(redisPersistence, workerCache, actorTemplateLister, workerPoolLister, sandboxConfigLister, dialer, clientset) + sm := controlapi.NewService(redisPersistence, workerCache, actorTemplateLister, workerPoolLister, sandboxConfigLister, dialer, clientset, *actorWorkflowDeadline) jwtIssuerDiscoveryClient := buildK8sServiceAccountIssuerDiscoveryClient(ctx, *clientJWTCAFile, *clientJWTIssuer) if authModeParsed == ateapiauth.ModeJWT && jwtIssuerDiscoveryClient == nil { @@ -233,26 +242,31 @@ func logFlagValues(ctx context.Context) { slog.String("redis-use-iam-auth", *redisUseIAMAuth), slog.String("redis-tls-server-name", *redisTLSServerName), slog.String("redis-client-cert", *redisClientCert), + slog.Bool("redis-no-tls", *redisNoTLS), slog.String("client-jwt-issuer", *clientJWTIssuer), slog.String("client-jwt-audience", *clientJWTAudience), slog.String("session-id-jwt-pool", *sessionIDJWTPoolFile), slog.String("session-id-ca-pool", *sessionIDCAPoolFile), slog.String("workerpool-ca-certs", *workerpoolCACerts), slog.String("auth-mode", *authMode), + slog.Duration("actor-workflow-deadline", *actorWorkflowDeadline), ) } // connectRedis builds the Redis/Valkey TLS config, plumbs IAM auth if // requested, opens the cluster client, and pings with retries. func connectRedis(ctx context.Context) (*redis.ClusterClient, error) { - tlsConfig, err := buildRedisTLSConfig(ctx) - if err != nil { - return nil, err - } - clusterOpts := &redis.ClusterOptions{ - Addrs: []string{*redisClusterAddress}, - TLSConfig: tlsConfig, + Addrs: []string{*redisClusterAddress}, + } + if *redisNoTLS { + slog.InfoContext(ctx, "Connecting to Redis/Valkey without TLS") + } else { + tlsConfig, err := buildRedisTLSConfig(ctx) + if err != nil { + return nil, err + } + clusterOpts.TLSConfig = tlsConfig } if *redisUseIAMAuth != "false" { diff --git a/cmd/atecontroller/internal/controllers/gen.go b/cmd/atecontroller/internal/controllers/gen.go index e8b03d30f..d4a372df9 100644 --- a/cmd/atecontroller/internal/controllers/gen.go +++ b/cmd/atecontroller/internal/controllers/gen.go @@ -14,4 +14,4 @@ package controllers -//go:generate bash ../../../../hack/run-tool.sh controller-gen rbac:headerFile=../../../../hack/boilerplate/sh.txt,roleName=ate-controller paths="./..." output:rbac:artifacts:config=../../../../manifests/ate-install/generated/ +//go:generate bash ../../../../hack/gen-rbac.sh diff --git a/cmd/atenet/internal/dns.go b/cmd/atenet/internal/dns.go index 471e93679..5fdced39d 100644 --- a/cmd/atenet/internal/dns.go +++ b/cmd/atenet/internal/dns.go @@ -30,6 +30,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/config" "github.com/agent-substrate/substrate/cmd/atenet/internal/dns" + "github.com/agent-substrate/substrate/internal/installdefaults" ) type DnsConfig struct { @@ -37,6 +38,8 @@ type DnsConfig struct { Kubeconfig string ReconcileInterval time.Duration CorefilePath string + RouterServiceName string + DNSServiceName string } func NewDnsCmd() *cobra.Command { @@ -86,11 +89,20 @@ func NewDnsCmd() *cobra.Command { return fmt.Errorf("failed to initialize cluster client: %w", err) } + // atenet shares its namespace with atenet-router and substrate's + // CoreDNS in every supported deployment topology, so we read it + // from Kubernetes' downward API rather than expose a flag. + systemNamespace := installdefaults.NamespaceFromPodEnv() + slog.InfoContext(ctx, "Resolved system namespace", slog.String("system-namespace", systemNamespace)) + dnsController := &dns.Controller{ - Client: k8sClient, - Interval: cfg.ReconcileInterval, - CorefilePath: cfg.CorefilePath, - Reloader: dns.NewConfigReloader(), + Client: k8sClient, + Interval: cfg.ReconcileInterval, + CorefilePath: cfg.CorefilePath, + Reloader: dns.NewConfigReloader(), + SystemNamespace: systemNamespace, + RouterServiceName: cfg.RouterServiceName, + DNSServiceName: cfg.DNSServiceName, } slog.InfoContext(ctx, "Starting DNS Controller subsystem") @@ -102,6 +114,8 @@ func NewDnsCmd() *cobra.Command { cmd.Flags().StringVar(&cfg.Kubeconfig, "kubeconfig", "", "Absolute path to the kubeconfig configuration file") cmd.Flags().DurationVar(&cfg.ReconcileInterval, "interval", 10*time.Second, "Interval for reconciling DNS configurations") cmd.Flags().StringVar(&cfg.CorefilePath, "corefile-path", "/etc/coredns/Corefile", "Path to the local Corefile configuration on shared volume") + cmd.Flags().StringVar(&cfg.RouterServiceName, "router-service-name", installdefaults.RouterServiceName, "Service name of the atenet-router. Override when the deployment renames the Service.") + cmd.Flags().StringVar(&cfg.DNSServiceName, "dns-service-name", installdefaults.DNSServiceName, "Service name of substrate's CoreDNS. Override when the deployment renames the Service.") return cmd } diff --git a/cmd/atenet/internal/dns/dns.go b/cmd/atenet/internal/dns/dns.go index cf2db99b6..4cfaf34ef 100644 --- a/cmd/atenet/internal/dns/dns.go +++ b/cmd/atenet/internal/dns/dns.go @@ -33,18 +33,23 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -const ( - // serviceName is the name of the CoreDNS service. - serviceName = "dns" - systemNamespace = "ate-system" -) - // Controller manages the DNS configuration for the ATE. type Controller struct { Client client.Client Interval time.Duration CorefilePath string Reloader ConfigReloader + + // SystemNamespace is the namespace where atenet-router and the substrate + // CoreDNS Service live. Defaults to installdefaults.SystemNamespace. + SystemNamespace string + // RouterServiceName is the Service name of the atenet-router that the + // CoreDNS Corefile forwards actor traffic to. Defaults to + // installdefaults.RouterServiceName. + RouterServiceName string + // DNSServiceName is the Service name of substrate's CoreDNS. Defaults to + // installdefaults.DNSServiceName. + DNSServiceName string } // Run the DNS orchestration loop until ctx is canceled. @@ -71,14 +76,15 @@ func (c *Controller) Run(ctx context.Context) error { func (c *Controller) reconcile(ctx context.Context) error { slog.DebugContext(ctx, "Reconciling DNS orchestration configuration...") - // 1. Get the ClusterIP of atenet-router in ate-system namespace + // 1. Get the ClusterIP of the atenet-router Service in the substrate namespace. routerSvc := &corev1.Service{} - if err := c.Client.Get(ctx, types.NamespacedName{Name: "atenet-router", Namespace: systemNamespace}, routerSvc); err != nil { + if err := c.Client.Get(ctx, types.NamespacedName{Name: c.RouterServiceName, Namespace: c.SystemNamespace}, routerSvc); err != nil { if errors.IsNotFound(err) { - slog.WarnContext(ctx, "atenet-router service not found, skipping until it is available") + slog.WarnContext(ctx, "atenet-router service not found, skipping until it is available", + slog.String("name", c.RouterServiceName), slog.String("namespace", c.SystemNamespace)) return nil } - return fmt.Errorf("failed to get atenet-router service: %w", err) + return fmt.Errorf("failed to get atenet-router service %s/%s: %w", c.SystemNamespace, c.RouterServiceName, err) } routerIP := routerSvc.Spec.ClusterIP @@ -87,14 +93,15 @@ func (c *Controller) reconcile(ctx context.Context) error { return nil } - // 2. Get the ClusterIP of dns service in ate-system namespace + // 2. Get the ClusterIP of substrate's CoreDNS Service in the same namespace. dnsSvc := &corev1.Service{} - if err := c.Client.Get(ctx, types.NamespacedName{Name: serviceName, Namespace: systemNamespace}, dnsSvc); err != nil { + if err := c.Client.Get(ctx, types.NamespacedName{Name: c.DNSServiceName, Namespace: c.SystemNamespace}, dnsSvc); err != nil { if errors.IsNotFound(err) { - slog.WarnContext(ctx, "dns service not found, skipping until it is available") + slog.WarnContext(ctx, "dns service not found, skipping until it is available", + slog.String("name", c.DNSServiceName), slog.String("namespace", c.SystemNamespace)) return nil } - return fmt.Errorf("failed to get dns service: %w", err) + return fmt.Errorf("failed to get dns service %s/%s: %w", c.SystemNamespace, c.DNSServiceName, err) } dnsIP := dnsSvc.Spec.ClusterIP diff --git a/cmd/atenet/internal/dns/dns_test.go b/cmd/atenet/internal/dns/dns_test.go index 34116db28..bf27941e1 100644 --- a/cmd/atenet/internal/dns/dns_test.go +++ b/cmd/atenet/internal/dns/dns_test.go @@ -28,6 +28,8 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/agent-substrate/substrate/internal/installdefaults" ) type mockConfigReloader struct { @@ -94,10 +96,13 @@ func TestReconcile(t *testing.T) { reloader := &mockConfigReloader{} controller := &Controller{ - Client: client, - Interval: 1 * time.Second, - CorefilePath: corefilePath, - Reloader: reloader, + Client: client, + Interval: 1 * time.Second, + CorefilePath: corefilePath, + Reloader: reloader, + SystemNamespace: installdefaults.SystemNamespace, + RouterServiceName: installdefaults.RouterServiceName, + DNSServiceName: installdefaults.DNSServiceName, } // Run one reconciliation loop @@ -185,10 +190,13 @@ func TestReconcileKubeDNSNotFound(t *testing.T) { Build() controller := &Controller{ - Client: client, - Interval: 1 * time.Second, - CorefilePath: corefilePath, - Reloader: &mockConfigReloader{}, + Client: client, + Interval: 1 * time.Second, + CorefilePath: corefilePath, + Reloader: &mockConfigReloader{}, + SystemNamespace: installdefaults.SystemNamespace, + RouterServiceName: installdefaults.RouterServiceName, + DNSServiceName: installdefaults.DNSServiceName, } ctx := context.Background() diff --git a/cmd/atenet/internal/router.go b/cmd/atenet/internal/router.go index c58f31a7e..471351dc3 100644 --- a/cmd/atenet/internal/router.go +++ b/cmd/atenet/internal/router.go @@ -22,6 +22,7 @@ import ( "github.com/agent-substrate/substrate/cmd/atenet/internal/router" "github.com/agent-substrate/substrate/internal/ateapiauth" + "github.com/agent-substrate/substrate/internal/installdefaults" ) func NewRouterCmd() *cobra.Command { @@ -45,18 +46,22 @@ func NewRouterCmd() *cobra.Command { cmd.Flags().StringVar(&cfg.MetricsAddr, "metrics-listen-addr", ":9090", "Address and port the prometheus metrics server should listen on.") cmd.Flags().BoolVar(&cfg.Standalone, "standalone", false, "Run in standalone mode, bypassing creation of managed deployment and services in Kubernetes cluster") cmd.Flags().StringVar(&cfg.Namespace, "namespace", "default", "Target operations namespace") + cmd.Flags().StringVar(&cfg.RouterServiceName, "router-service-name", installdefaults.RouterServiceName, "Service name of this atenet-router in the operations namespace. Override when the deployment renames the Service.") cmd.Flags().StringVar(&cfg.Kubeconfig, "kubeconfig", "", "Absolute path to the kubeconfig configuration file") cmd.Flags().StringVar(&cfg.AteapiAddr, "ateapi-address", "api.ate-system.svc:443", "gRPC host address of the cluster ateapi Control instance") cmd.Flags().IntVar(&cfg.HttpPort, "port-http", 8080, "TCP port for workload traffic entering through the Envoy Router") cmd.Flags().IntVar(&cfg.XdsPort, "port-xds", 18000, "TCP port listening for the xDS dynamic Envoy connections") cmd.Flags().IntVar(&cfg.ExtprocPort, "port-extproc", 50051, "Listen port for the Envoy dynamic External Processing (ext_proc) server") cmd.Flags().StringVar(&cfg.ExtprocAddr, "extproc-address", "127.0.0.1", "Host IP or address of the Envoy External Processing (ext_proc) server") + cmd.Flags().StringVar(&cfg.NetworkingMode, "networking-mode", router.NetworkingModeAgentgateway, "Networking proxy mode: agentgateway or envoy") cmd.Flags().StringVar(&cfg.EnvoyImage, "envoy-image", "envoyproxy/envoy:v1.30-latest", "Image URI used for dynamically launched router instances") + cmd.Flags().StringVar(&cfg.AgentgatewayImage, "agentgateway-image", "cr.agentgateway.dev/agentgateway:v1.3.0-alpha.1", "Image URI used for Agentgateway router instances") cmd.Flags().StringVar(&cfg.TemplatesFile, "actor-templates-file", "", "Path to offline YAML configuration file listing ActorTemplates") cmd.Flags().IntVar(&cfg.StatusPort, "status-port", 4040, "Port to serve /statusz on (set <= 0 to disable serving status)") cmd.Flags().DurationVar(&cfg.HealthInterval, "health-interval", 1*time.Second, "Interval for checking health of dependent services") cmd.Flags().IntVar(&cfg.HttpsPort, "port-https", 8443, "TCP port for HTTPS workload traffic entering through the Envoy Router") - cmd.Flags().StringVar(&cfg.EnvoyCertPath, "envoy-cert-path", "", "Path to the Envoy certificate file (if empty, a self-signed cert will be generated for testing)") + cmd.Flags().StringVar(&cfg.TLSCertPath, "tls-cert-path", "", "Path to the proxy TLS certificate file") + cmd.Flags().StringVar(&cfg.TLSKeyPath, "tls-key-path", "", "Path to the proxy TLS private key file") cmd.Flags().StringVar(&cfg.OtlpCollectorAddress, "otlp-collector-address", "", "host:port of the OTLP gRPC collector that Envoy reports tracing spans to (empty disables Envoy tracing)") cmd.Flags().StringVar(&cfg.AteapiAuthMode, "ateapi-auth", "mtls", "Client auth to ateapi: mtls|jwt. 'mtls' (default) dials with insecure TLS and relies on pod-projected mTLS credentials for identity. 'jwt' verifies the server cert and sends a Bearer SA token.") cmd.Flags().StringVar(&cfg.AteapiCAFile, "ateapi-ca-file", ateapiauth.DefaultServiceAccountCAFile, "PEM file with CAs trusted to verify the ateapi server cert. Required for jwt.") diff --git a/cmd/atenet/internal/router/agentgateway.go b/cmd/atenet/internal/router/agentgateway.go new file mode 100644 index 000000000..af7fabe5a --- /dev/null +++ b/cmd/atenet/internal/router/agentgateway.go @@ -0,0 +1,151 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package router + +import ( + "context" + "fmt" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +type agentgatewayProvider struct { + cfg RouterConfig +} + +func (p agentgatewayProvider) Name() string { + return NetworkingModeAgentgateway +} + +func (p agentgatewayProvider) RequiresXDS() bool { + return false +} + +func (p agentgatewayProvider) ConfigMapData() map[string]string { + return map[string]string{"config.yaml": p.localConfig()} +} + +func (p agentgatewayProvider) Container() corev1.Container { + ports := []corev1.ContainerPort{ + {Name: "http", ContainerPort: int32(p.cfg.HttpPort)}, + {Name: "readiness", ContainerPort: 15021}, + {Name: "metrics", ContainerPort: 15020}, + } + if p.cfg.HttpsPort > 0 && tlsCertPath(p.cfg) != "" { + ports = append(ports, corev1.ContainerPort{Name: "https", ContainerPort: int32(p.cfg.HttpsPort)}) + } + + return corev1.Container{ + Name: "agentgateway", + Image: p.cfg.AgentgatewayImage, + Args: []string{"-f", "/etc/agentgateway/config.yaml"}, + Ports: ports, + VolumeMounts: []corev1.VolumeMount{ + {Name: "proxy-config", MountPath: "/etc/agentgateway"}, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz/ready", + Port: intstr.FromInt32(15021), + }, + }, + PeriodSeconds: 10, + }, + } +} + +func (p agentgatewayProvider) ServicePorts() []corev1.ServicePort { + ports := []corev1.ServicePort{ + {Name: "http", Port: int32(p.cfg.HttpPort), TargetPort: intstr.FromString("http")}, + } + if p.cfg.HttpsPort > 0 && tlsCertPath(p.cfg) != "" { + ports = append(ports, corev1.ServicePort{Name: "https", Port: int32(p.cfg.HttpsPort), TargetPort: intstr.FromString("https")}) + } + return ports +} + +func (p agentgatewayProvider) CheckReady(ctx context.Context) (bool, string) { + return checkHTTPReady(ctx, "http://127.0.0.1:15021/healthz/ready", "") +} + +func (p agentgatewayProvider) localConfig() string { + httpRoute := p.routeBlock("substrate-http") + config := fmt.Sprintf(`# yaml-language-server: $schema=https://agentgateway.dev/schema/config +config: + adminAddr: "127.0.0.1:15000" + readinessAddr: "0.0.0.0:15021" + statsAddr: "0.0.0.0:15020" +binds: +- port: %d + listeners: + - name: http + protocol: HTTP + routes: +%s`, p.cfg.HttpPort, indent(httpRoute, 4)) + + if p.cfg.HttpsPort > 0 && tlsCertPath(p.cfg) != "" { + cert := tlsCertPath(p.cfg) + key := tlsKeyPath(p.cfg) + config += fmt.Sprintf(`- port: %d + listeners: + - name: https + protocol: HTTPS + tls: + cert: %q + key: %q + routes: +%s`, p.cfg.HttpsPort, cert, key, indent(p.routeBlock("substrate-https"), 4)) + } + + return config +} + +func (p agentgatewayProvider) routeBlock(name string) string { + extprocHost := fmt.Sprintf("%s:%d", p.cfg.ExtprocAddr, p.cfg.ExtprocPort) + // processingOptions limit ext_proc to request headers only; agentgateway defaults + // break WebSocket upgrades because this server only handles headers. + return fmt.Sprintf(`- name: %s + matches: + - path: + pathPrefix: / + policies: + extProc: + host: %q + failureMode: failClosed + processingOptions: + requestBodyMode: none + responseBodyMode: none + requestHeaderMode: send + responseHeaderMode: skip + requestTrailerMode: skip + responseTrailerMode: skip + backends: + - dynamic: {} +`, name, extprocHost) +} + +func indent(s string, spaces int) string { + prefix := strings.Repeat(" ", spaces) + lines := strings.Split(s, "\n") + for i, line := range lines { + if line != "" { + lines[i] = prefix + line + } + } + return strings.Join(lines, "\n") +} diff --git a/cmd/atenet/internal/router/controller.go b/cmd/atenet/internal/router/controller.go index 1b0a1ed18..d9a31d7eb 100644 --- a/cmd/atenet/internal/router/controller.go +++ b/cmd/atenet/internal/router/controller.go @@ -31,9 +31,10 @@ type Controller struct { cfg RouterConfig xdsSrv *XdsServer extprocSrv *ExtProcServer + provider proxyProvider atStore atStore - envoyRunner *envoyrunner + proxyRunner *proxyrunner } func NewController( @@ -42,8 +43,11 @@ func NewController( cfg RouterConfig, xdsSrv *XdsServer, extprocSrv *ExtProcServer, + provider proxyProvider, ) *Controller { - xdsSrv.SetConfig(cfg.HttpPort, cfg.ExtprocPort, cfg.ExtprocAddr) + if xdsSrv != nil { + xdsSrv.SetConfig(cfg.HttpPort, cfg.ExtprocPort, cfg.ExtprocAddr) + } var store atStore if cfg.TemplatesFile != "" { @@ -58,9 +62,10 @@ func NewController( cfg: cfg, xdsSrv: xdsSrv, extprocSrv: extprocSrv, + provider: provider, atStore: store, - envoyRunner: newEnvoyRunner(k8sClient, cfg), + proxyRunner: newProxyRunner(k8sClient, cfg, provider), } } @@ -92,16 +97,18 @@ func (c *Controller) reconcile(ctx context.Context) error { return err } - if err := c.xdsSrv.UpdateSnapshot(); err != nil { - slog.ErrorContext(ctx, "xDS Configuration generation problem", slog.String("err", err.Error())) - return err + if c.provider.RequiresXDS() { + if err := c.xdsSrv.UpdateSnapshot(); err != nil { + slog.ErrorContext(ctx, "xDS Configuration generation problem", slog.String("err", err.Error())) + return err + } } if !c.cfg.Standalone && c.cfg.TemplatesFile == "" { - // Reconcile Envoy router Deployment and Kubernetes cluster entities - err := c.envoyRunner.reconcile(ctx) + // Reconcile router proxy Deployment and Kubernetes cluster entities. + err := c.proxyRunner.reconcile(ctx) if err != nil { - slog.ErrorContext(ctx, "Error during Envoy router reconciliation", slog.String("err", err.Error())) + slog.ErrorContext(ctx, "Error during router proxy reconciliation", slog.String("err", err.Error())) return err } } diff --git a/cmd/atenet/internal/router/dashboard.html b/cmd/atenet/internal/router/dashboard.html index 41f3d93af..025ffa9dc 100644 --- a/cmd/atenet/internal/router/dashboard.html +++ b/cmd/atenet/internal/router/dashboard.html @@ -252,12 +252,16 @@

atenet Router Status

Namespace Context {{ .Namespace }} +
+ Networking Mode + {{ .NetworkingMode }} +
Component Network Allocation
- Workload Port (Http Envoy) + Workload HTTP Port {{ .HttpPort }}
@@ -278,8 +282,8 @@

atenet Router Status

System Component Health Checks
- Envoy Health - {{ if .Health.Envoy.Healthy }} + Proxy Health ({{ .Health.Provider }}) + {{ if .Health.Proxy.Healthy }} Healthy {{ else }} Degraded @@ -287,11 +291,11 @@

atenet Router Status

Message / Err: - {{ .Health.Envoy.Message }} + {{ .Health.Proxy.Message }}
- Ok: {{ .Health.Envoy.SuccessCount }} | Err: {{ .Health.Envoy.FailureCount }} - LKG: {{ if not .Health.Envoy.LastSuccess.IsZero }}{{ .Health.Envoy.LastSuccess.Format "15:04:05" }}{{ else }}N/A{{ end }} + Ok: {{ .Health.Proxy.SuccessCount }} | Err: {{ .Health.Proxy.FailureCount }} + LKG: {{ if not .Health.Proxy.LastSuccess.IsZero }}{{ .Health.Proxy.LastSuccess.Format "15:04:05" }}{{ else }}N/A{{ end }}
diff --git a/cmd/atenet/internal/router/envoy.go b/cmd/atenet/internal/router/envoy.go new file mode 100644 index 000000000..88ec95365 --- /dev/null +++ b/cmd/atenet/internal/router/envoy.go @@ -0,0 +1,118 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package router + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +type envoyProvider struct { + cfg RouterConfig +} + +func (p envoyProvider) Name() string { + return NetworkingModeEnvoy +} + +func (p envoyProvider) RequiresXDS() bool { + return true +} + +func (p envoyProvider) ConfigMapData() map[string]string { + envoyYaml := fmt.Sprintf(`admin: + address: + socket_address: + address: 0.0.0.0 + port_value: 9901 + +node: + id: %s + cluster: substrate-router-cluster + +dynamic_resources: + lds_config: + resource_api_version: V3 + ads: {} + cds_config: + resource_api_version: V3 + ads: {} + ads_config: + api_type: GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: xds_cluster + +static_resources: + clusters: + - name: xds_cluster + connect_timeout: 0.25s + type: STRICT_DNS + lb_policy: ROUND_ROBIN + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + http2_protocol_options: {} + load_assignment: + cluster_name: xds_cluster + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: atenet-router + port_value: %d +`, NodeID, p.cfg.XdsPort) + + return map[string]string{"envoy.yaml": envoyYaml} +} + +func (p envoyProvider) Container() corev1.Container { + return corev1.Container{ + Name: "envoy", + Image: p.cfg.EnvoyImage, + Command: []string{ + "/usr/local/bin/envoy", + "-c", + "/etc/envoy/envoy.yaml", + "--component-log-level", + "upstream:debug,router:debug,ext_proc:debug", + }, + Ports: []corev1.ContainerPort{ + {Name: "http", ContainerPort: int32(p.cfg.HttpPort)}, + {Name: "https", ContainerPort: int32(p.cfg.HttpsPort)}, + {Name: "admin", ContainerPort: 9901}, + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: "proxy-config", MountPath: "/etc/envoy"}, + }, + } +} + +func (p envoyProvider) ServicePorts() []corev1.ServicePort { + return []corev1.ServicePort{ + {Name: "http", Port: int32(p.cfg.HttpPort), TargetPort: intstr.FromString("http")}, + {Name: "https", Port: int32(p.cfg.HttpsPort), TargetPort: intstr.FromString("https")}, + } +} + +func (p envoyProvider) CheckReady(ctx context.Context) (bool, string) { + return checkHTTPReady(ctx, "http://127.0.0.1:9901/ready", "LIVE") +} diff --git a/cmd/atenet/internal/router/envoyrunner.go b/cmd/atenet/internal/router/envoyrunner.go deleted file mode 100644 index df8765c62..000000000 --- a/cmd/atenet/internal/router/envoyrunner.go +++ /dev/null @@ -1,258 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package router - -import ( - "context" - "fmt" - "log/slog" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - k8serrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -const ( - EnvoyDeploymentName = "atenet-router-envoy" - EnvoyServiceName = "atenet-router-envoy" - EnvoyConfigMapName = "atenet-router-envoy-config" -) - -// envoyrunner manages the dynamic deployment and lifecycle of the underlying -// Envoy proxy instance running inside Kubernetes. -type envoyrunner struct { - k8sClient client.Client - cfg RouterConfig -} - -func newEnvoyRunner(k8sClient client.Client, cfg RouterConfig) *envoyrunner { - return &envoyrunner{ - k8sClient: k8sClient, - cfg: cfg, - } -} - -func (r *envoyrunner) reconcile(ctx context.Context) error { - if err := r.reconcileEnvoyConfigMap(ctx); err != nil { - return fmt.Errorf("failed configmap reconciliation: %w", err) - } - - if err := r.reconcileEnvoyDeployment(ctx); err != nil { - return fmt.Errorf("failed deployment reconciliation: %w", err) - } - - if err := r.reconcileEnvoyService(ctx); err != nil { - return fmt.Errorf("failed service reconciliation: %w", err) - } - - return nil -} - -func (r *envoyrunner) reconcileEnvoyConfigMap(ctx context.Context) error { - envoyYaml := fmt.Sprintf(`admin: - address: - socket_address: - address: 0.0.0.0 - port_value: 9901 - -node: - id: %s - cluster: substrate-router-cluster - -dynamic_resources: - lds_config: - resource_api_version: V3 - ads: {} - cds_config: - resource_api_version: V3 - ads: {} - ads_config: - api_type: GRPC - transport_api_version: V3 - grpc_services: - - envoy_grpc: - cluster_name: xds_cluster - -static_resources: - clusters: - - name: xds_cluster - connect_timeout: 0.25s - type: STRICT_DNS - lb_policy: ROUND_ROBIN - typed_extension_protocol_options: - envoy.extensions.upstreams.http.v3.HttpProtocolOptions: - "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions - explicit_http_config: - http2_protocol_options: {} - load_assignment: - cluster_name: xds_cluster - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: atenet-router - port_value: %d -`, NodeID, r.cfg.XdsPort) - - cm := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: EnvoyConfigMapName, - Namespace: r.cfg.Namespace, - }, - Data: map[string]string{ - "envoy.yaml": envoyYaml, - }, - } - - var existing corev1.ConfigMap - err := r.k8sClient.Get(ctx, client.ObjectKey{Namespace: r.cfg.Namespace, Name: EnvoyConfigMapName}, &existing) - if err != nil { - if k8serrors.IsNotFound(err) { - slog.InfoContext(ctx, "Creating Envoy bootstrap ConfigMap", - slog.String("namespace", r.cfg.Namespace), - slog.String("name", EnvoyConfigMapName)) - return r.k8sClient.Create(ctx, cm) - } - return err - } - - existing.Data = cm.Data - return r.k8sClient.Update(ctx, &existing) -} - -func (r *envoyrunner) reconcileEnvoyDeployment(ctx context.Context) error { - replicas := int32(1) - - dep := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: EnvoyDeploymentName, - Namespace: r.cfg.Namespace, - Labels: map[string]string{ - "app": "atenet-router-envoy", - }, - }, - Spec: appsv1.DeploymentSpec{ - Replicas: &replicas, - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "atenet-router-envoy", - }, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app": "atenet-router-envoy", - }, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "envoy", - Image: r.cfg.EnvoyImage, - Command: []string{ - "/usr/local/bin/envoy", - "-c", - "/etc/envoy/envoy.yaml", - "--component-log-level", - "upstream:debug,router:debug,ext_proc:debug", - }, - Ports: []corev1.ContainerPort{ - { - Name: "http", - ContainerPort: int32(r.cfg.HttpPort), - }, - { - Name: "admin", - ContainerPort: 9901, - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "envoy-config", - MountPath: "/etc/envoy", - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "envoy-config", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: EnvoyConfigMapName, - }, - }, - }, - }, - }, - }, - }, - }, - } - - var existing appsv1.Deployment - err := r.k8sClient.Get(ctx, client.ObjectKey{Namespace: r.cfg.Namespace, Name: EnvoyDeploymentName}, &existing) - if err != nil { - if k8serrors.IsNotFound(err) { - slog.InfoContext(ctx, "Creating managed Envoy router Deployment", slog.String("namespace", r.cfg.Namespace)) - return r.k8sClient.Create(ctx, dep) - } - return err - } - - existing.Spec = dep.Spec - return r.k8sClient.Update(ctx, &existing) -} - -func (r *envoyrunner) reconcileEnvoyService(ctx context.Context) error { - svc := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: EnvoyServiceName, - Namespace: r.cfg.Namespace, - }, - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - Selector: map[string]string{ - "app": "atenet-router-envoy", - }, - Ports: []corev1.ServicePort{ - { - Name: "http", - Port: int32(r.cfg.HttpPort), - TargetPort: intstr.FromString("http"), - }, - }, - }, - } - - var existing corev1.Service - err := r.k8sClient.Get(ctx, client.ObjectKey{Namespace: r.cfg.Namespace, Name: EnvoyServiceName}, &existing) - if err != nil { - if k8serrors.IsNotFound(err) { - slog.InfoContext(ctx, "Creating managed Envoy router ClusterIP service", slog.String("namespace", r.cfg.Namespace)) - return r.k8sClient.Create(ctx, svc) - } - return err - } - - existing.Spec.Ports = svc.Spec.Ports - existing.Spec.Selector = svc.Spec.Selector - return r.k8sClient.Update(ctx, &existing) -} diff --git a/cmd/atenet/internal/router/extproc.go b/cmd/atenet/internal/router/extproc.go index ac5b5d5b8..9dd4061ac 100644 --- a/cmd/atenet/internal/router/extproc.go +++ b/cmd/atenet/internal/router/extproc.go @@ -17,7 +17,6 @@ package router import ( "context" "errors" - "fmt" "io" "log/slog" "net" @@ -111,14 +110,7 @@ func (s *ExtProcServer) Process(stream extprocv3.ExternalProcessor_ProcessServer } default: - // No modification for other processing states, but log because this should - // not be called. - slog.Error("Unexpected request type", slog.String("reqType", fmt.Sprintf("%T", reqType))) - resp.Response = &extprocv3.ProcessingResponse_RequestHeaders{ - RequestHeaders: &extprocv3.HeadersResponse{ - Response: &extprocv3.CommonResponse{}, - }, - } + resp = continueExtProcResponse(req) } if err := stream.Send(resp); err != nil { @@ -187,6 +179,50 @@ func (s *ExtProcServer) handleRequestHeaders( }, metadata, targetAddr, tmplNs, tmplName, nil } +func continueExtProcResponse(req *extprocv3.ProcessingRequest) *extprocv3.ProcessingResponse { + common := &extprocv3.CommonResponse{ + Status: extprocv3.CommonResponse_CONTINUE, + } + switch req.Request.(type) { + case *extprocv3.ProcessingRequest_RequestBody: + return &extprocv3.ProcessingResponse{ + Response: &extprocv3.ProcessingResponse_RequestBody{ + RequestBody: &extprocv3.BodyResponse{Response: common}, + }, + } + case *extprocv3.ProcessingRequest_ResponseHeaders: + return &extprocv3.ProcessingResponse{ + Response: &extprocv3.ProcessingResponse_ResponseHeaders{ + ResponseHeaders: &extprocv3.HeadersResponse{Response: common}, + }, + } + case *extprocv3.ProcessingRequest_ResponseBody: + return &extprocv3.ProcessingResponse{ + Response: &extprocv3.ProcessingResponse_ResponseBody{ + ResponseBody: &extprocv3.BodyResponse{Response: common}, + }, + } + case *extprocv3.ProcessingRequest_RequestTrailers: + return &extprocv3.ProcessingResponse{ + Response: &extprocv3.ProcessingResponse_RequestTrailers{ + RequestTrailers: &extprocv3.TrailersResponse{}, + }, + } + case *extprocv3.ProcessingRequest_ResponseTrailers: + return &extprocv3.ProcessingResponse{ + Response: &extprocv3.ProcessingResponse_ResponseTrailers{ + ResponseTrailers: &extprocv3.TrailersResponse{}, + }, + } + default: + return &extprocv3.ProcessingResponse{ + Response: &extprocv3.ProcessingResponse_RequestHeaders{ + RequestHeaders: &extprocv3.HeadersResponse{Response: common}, + }, + } + } +} + func (s *ExtProcServer) recordRouteDuration(ctx context.Context, d time.Duration, tmplNs, tmplName, outcome string) { if s.routeDuration == nil { return diff --git a/cmd/atenet/internal/router/health.go b/cmd/atenet/internal/router/health.go index d9e1fddab..e8904caea 100644 --- a/cmd/atenet/internal/router/health.go +++ b/cmd/atenet/internal/router/health.go @@ -17,15 +17,11 @@ package router import ( "context" "fmt" - "io" "log/slog" - "net/http" - "strings" "sync" "time" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" - "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "k8s.io/client-go/kubernetes" ) @@ -39,9 +35,10 @@ type ComponentHealth struct { } type RouterHealthReport struct { - Envoy ComponentHealth `json:"envoy"` - K8sAPI ComponentHealth `json:"k8s_api"` - AteAPI ComponentHealth `json:"ate_api"` + Proxy ComponentHealth `json:"proxy"` + Provider string `json:"provider"` + K8sAPI ComponentHealth `json:"k8s_api"` + AteAPI ComponentHealth `json:"ate_api"` } // routerHealth periodically checks the dependent services of router to track health @@ -51,23 +48,23 @@ type routerHealth struct { report RouterHealthReport - interval time.Duration - clientset kubernetes.Interface - apiClient ateapipb.ControlClient - cfg RouterConfig - envoyClient *http.Client + interval time.Duration + clientset kubernetes.Interface + apiClient ateapipb.ControlClient + cfg RouterConfig + provider proxyProvider } -func newRouterHealth(interval time.Duration, clientset kubernetes.Interface, apiClient ateapipb.ControlClient, cfg RouterConfig) *routerHealth { +func newRouterHealth(interval time.Duration, clientset kubernetes.Interface, apiClient ateapipb.ControlClient, cfg RouterConfig, provider proxyProvider) *routerHealth { if interval <= 0 { interval = time.Second } return &routerHealth{ - interval: interval, - clientset: clientset, - apiClient: apiClient, - cfg: cfg, - envoyClient: &http.Client{Transport: otelhttp.NewTransport(http.DefaultTransport)}, + interval: interval, + clientset: clientset, + apiClient: apiClient, + cfg: cfg, + provider: provider, } } @@ -94,20 +91,25 @@ func (rh *routerHealth) check(ctx context.Context) { slog.InfoContext(ctx, "Checking health") - // 1. Check Envoy + // 1. Check configured proxy { - healthy, msg := rh.checkEnvoy(ctx) + providerName := "" + if rh.provider != nil { + providerName = rh.provider.Name() + rh.report.Provider = providerName + } + healthy, msg := rh.checkProxy(ctx) if healthy { - rh.report.Envoy.Healthy = true - rh.report.Envoy.Message = msg - rh.report.Envoy.LastSuccess = time.Now() - rh.report.Envoy.SuccessCount++ + rh.report.Proxy.Healthy = true + rh.report.Proxy.Message = msg + rh.report.Proxy.LastSuccess = time.Now() + rh.report.Proxy.SuccessCount++ } else { - rh.report.Envoy.Healthy = false - rh.report.Envoy.Message = msg - rh.report.Envoy.LastFailure = time.Now() - rh.report.Envoy.FailureCount++ - slog.ErrorContext(ctx, "Envoy health check failed", slog.String("msg", msg)) + rh.report.Proxy.Healthy = false + rh.report.Proxy.Message = msg + rh.report.Proxy.LastFailure = time.Now() + rh.report.Proxy.FailureCount++ + slog.ErrorContext(ctx, "Proxy health check failed", slog.String("provider", providerName), slog.String("msg", msg)) } } @@ -146,36 +148,11 @@ func (rh *routerHealth) check(ctx context.Context) { } } -func (rh *routerHealth) checkEnvoy(ctx context.Context) (bool, string) { - timeoutCtx, cancel := context.WithTimeout(ctx, 500*time.Millisecond) - defer cancel() - - req, err := http.NewRequestWithContext(timeoutCtx, "GET", "http://127.0.0.1:9901/ready", nil) - if err != nil { - return false, err.Error() - } - - resp, err := rh.envoyClient.Do(req) - if err != nil { - return false, err.Error() +func (rh *routerHealth) checkProxy(ctx context.Context) (bool, string) { + if rh.provider == nil { + return false, "No proxy provider" } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return false, fmt.Sprintf("unexpected status code %d", resp.StatusCode) - } - - bodyBytes, err := io.ReadAll(resp.Body) - if err != nil { - return false, err.Error() - } - - bodyStr := strings.TrimSpace(string(bodyBytes)) - if bodyStr != "LIVE" { - return false, fmt.Sprintf("expected LIVE but got %q", bodyStr) - } - - return true, "LIVE" + return rh.provider.CheckReady(ctx) } func (rh *routerHealth) checkK8s() (bool, string) { diff --git a/cmd/atenet/internal/router/provider.go b/cmd/atenet/internal/router/provider.go new file mode 100644 index 000000000..ecab5f8ae --- /dev/null +++ b/cmd/atenet/internal/router/provider.go @@ -0,0 +1,102 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package router + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" +) + +const ( + NetworkingModeEnvoy = "envoy" + NetworkingModeAgentgateway = "agentgateway" +) + +// proxyProvider owns proxy-specific configuration and runtime details. The +// Substrate router keeps actor lookup and ext_proc behavior shared above this +// provider boundary. +type proxyProvider interface { + Name() string + RequiresXDS() bool + ConfigMapData() map[string]string + Container() corev1.Container + ServicePorts() []corev1.ServicePort + CheckReady(ctx context.Context) (bool, string) +} + +func newProxyProvider(cfg RouterConfig) (proxyProvider, error) { + switch strings.ToLower(cfg.NetworkingMode) { + case "", NetworkingModeAgentgateway: + return agentgatewayProvider{cfg: cfg}, nil + case NetworkingModeEnvoy: + return envoyProvider{cfg: cfg}, nil + default: + return nil, fmt.Errorf("unsupported networking mode %q", cfg.NetworkingMode) + } +} + +func tlsCertPath(cfg RouterConfig) string { + return cfg.TLSCertPath +} + +func tlsKeyPath(cfg RouterConfig) string { + if cfg.TLSKeyPath != "" { + return cfg.TLSKeyPath + } + if cfg.TLSCertPath != "" { + return cfg.TLSCertPath + } + return "" +} + +func checkHTTPReady(ctx context.Context, url string, expectedBody string) (bool, string) { + timeoutCtx, cancel := context.WithTimeout(ctx, 500*time.Millisecond) + defer cancel() + + req, err := http.NewRequestWithContext(timeoutCtx, "GET", url, nil) + if err != nil { + return false, err.Error() + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return false, err.Error() + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return false, fmt.Sprintf("unexpected status code %d", resp.StatusCode) + } + + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return false, err.Error() + } + + bodyStr := strings.TrimSpace(string(bodyBytes)) + if expectedBody != "" && bodyStr != expectedBody { + return false, fmt.Sprintf("expected %s but got %q", expectedBody, bodyStr) + } + if bodyStr == "" { + return true, "ready" + } + return true, bodyStr +} diff --git a/cmd/atenet/internal/router/proxyrunner.go b/cmd/atenet/internal/router/proxyrunner.go new file mode 100644 index 000000000..d55913f1e --- /dev/null +++ b/cmd/atenet/internal/router/proxyrunner.go @@ -0,0 +1,183 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package router + +import ( + "context" + "fmt" + "log/slog" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + ProxyDeploymentName = "atenet-router-proxy" + ProxyServiceName = "atenet-router-proxy" + ProxyConfigMapName = "atenet-router-proxy-config" +) + +// proxyrunner manages the dynamic deployment and lifecycle of the configured +// networking proxy instance running inside Kubernetes. +type proxyrunner struct { + k8sClient client.Client + cfg RouterConfig + provider proxyProvider +} + +func newProxyRunner(k8sClient client.Client, cfg RouterConfig, provider proxyProvider) *proxyrunner { + return &proxyrunner{ + k8sClient: k8sClient, + cfg: cfg, + provider: provider, + } +} + +func (r *proxyrunner) reconcile(ctx context.Context) error { + if err := r.reconcileProxyConfigMap(ctx); err != nil { + return fmt.Errorf("failed configmap reconciliation: %w", err) + } + + if err := r.reconcileProxyDeployment(ctx); err != nil { + return fmt.Errorf("failed deployment reconciliation: %w", err) + } + + if err := r.reconcileProxyService(ctx); err != nil { + return fmt.Errorf("failed service reconciliation: %w", err) + } + + return nil +} + +func (r *proxyrunner) reconcileProxyConfigMap(ctx context.Context) error { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: ProxyConfigMapName, + Namespace: r.cfg.Namespace, + }, + Data: r.provider.ConfigMapData(), + } + + var existing corev1.ConfigMap + err := r.k8sClient.Get(ctx, client.ObjectKey{Namespace: r.cfg.Namespace, Name: ProxyConfigMapName}, &existing) + if err != nil { + if k8serrors.IsNotFound(err) { + slog.InfoContext(ctx, "Creating proxy ConfigMap", + slog.String("namespace", r.cfg.Namespace), + slog.String("name", ProxyConfigMapName), + slog.String("provider", r.provider.Name())) + return r.k8sClient.Create(ctx, cm) + } + return err + } + + existing.Data = cm.Data + return r.k8sClient.Update(ctx, &existing) +} + +func (r *proxyrunner) reconcileProxyDeployment(ctx context.Context) error { + replicas := int32(1) + labels := map[string]string{ + "app": "atenet-router-proxy", + "substrate.ate.dev/proxy": r.provider.Name(), + "substrate.ate.dev/router": "atenet-router", + } + + dep := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: ProxyDeploymentName, + Namespace: r.cfg.Namespace, + Labels: labels, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{r.provider.Container()}, + Volumes: []corev1.Volume{ + { + Name: "proxy-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: ProxyConfigMapName, + }, + }, + }, + }, + }, + }, + }, + }, + } + + var existing appsv1.Deployment + err := r.k8sClient.Get(ctx, client.ObjectKey{Namespace: r.cfg.Namespace, Name: ProxyDeploymentName}, &existing) + if err != nil { + if k8serrors.IsNotFound(err) { + slog.InfoContext(ctx, "Creating managed router proxy Deployment", + slog.String("namespace", r.cfg.Namespace), + slog.String("provider", r.provider.Name())) + return r.k8sClient.Create(ctx, dep) + } + return err + } + + existing.Spec = dep.Spec + return r.k8sClient.Update(ctx, &existing) +} + +func (r *proxyrunner) reconcileProxyService(ctx context.Context) error { + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: ProxyServiceName, + Namespace: r.cfg.Namespace, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + Selector: map[string]string{ + "app": "atenet-router-proxy", + "substrate.ate.dev/proxy": r.provider.Name(), + "substrate.ate.dev/router": "atenet-router", + }, + Ports: r.provider.ServicePorts(), + }, + } + + var existing corev1.Service + err := r.k8sClient.Get(ctx, client.ObjectKey{Namespace: r.cfg.Namespace, Name: ProxyServiceName}, &existing) + if err != nil { + if k8serrors.IsNotFound(err) { + slog.InfoContext(ctx, "Creating managed router proxy ClusterIP service", + slog.String("namespace", r.cfg.Namespace), + slog.String("provider", r.provider.Name())) + return r.k8sClient.Create(ctx, svc) + } + return err + } + + existing.Spec.Ports = svc.Spec.Ports + existing.Spec.Selector = svc.Spec.Selector + return r.k8sClient.Update(ctx, &existing) +} diff --git a/cmd/atenet/internal/router/router.go b/cmd/atenet/internal/router/router.go index 7ed77df73..b2932d8b6 100644 --- a/cmd/atenet/internal/router/router.go +++ b/cmd/atenet/internal/router/router.go @@ -64,22 +64,29 @@ func init() { // RouterConfig holds deployment setup and endpoint options for the router node instance. type RouterConfig struct { - Standalone bool - Namespace string - Kubeconfig string - AteapiAddr string - HttpPort int - XdsPort int - ExtprocPort int - ExtprocAddr string - EnvoyImage string - TemplatesFile string - StatusPort int - HealthInterval time.Duration - HttpsPort int - EnvoyCertPath string - LogLevel string - MetricsAddr string + Standalone bool + Namespace string + // RouterServiceName is the Service name of this atenet-router in the + // operations namespace, used by /statusz to look up its own ClusterIP. + // Defaults to installdefaults.RouterServiceName. + RouterServiceName string + Kubeconfig string + AteapiAddr string + HttpPort int + XdsPort int + ExtprocPort int + ExtprocAddr string + NetworkingMode string + EnvoyImage string + AgentgatewayImage string + TemplatesFile string + StatusPort int + HealthInterval time.Duration + HttpsPort int + TLSCertPath string + TLSKeyPath string + LogLevel string + MetricsAddr string // OtlpCollectorAddress is the host:port of the OTLP gRPC collector that // Envoy reports tracing spans to. Empty disables Envoy-side tracing. OtlpCollectorAddress string @@ -104,6 +111,10 @@ type RouterServer struct { } func NewRouterServer(cfg RouterConfig) (*RouterServer, error) { + if cfg.NetworkingMode == "" { + cfg.NetworkingMode = NetworkingModeAgentgateway + } + var k8sClient client.Client var clientset kubernetes.Interface @@ -221,6 +232,11 @@ func (s *RouterServer) Run(ctx context.Context) error { g, ctx := errgroup.WithContext(ctx) + provider, err := newProxyProvider(s.cfg) + if err != nil { + return err + } + xdsSrv := NewXdsServer(s.cfg.XdsPort) xdsSrv.SetConfig(s.cfg.HttpPort, s.cfg.ExtprocPort, s.cfg.ExtprocAddr) if err := xdsSrv.SetOtlpCollector(s.cfg.OtlpCollectorAddress); err != nil { @@ -228,16 +244,15 @@ func (s *RouterServer) Run(ctx context.Context) error { } var certContent, keyContent string - if s.cfg.EnvoyCertPath == "" { - slog.InfoContext(ctx, "No Envoy certificate path provided, generating self-signed certificate for testing") - var err error + if provider.RequiresXDS() && tlsCertPath(s.cfg) == "" { + slog.InfoContext(ctx, "No proxy TLS certificate path provided, generating self-signed certificate for testing") certContent, keyContent, err = generateSelfSignedCert() if err != nil { return fmt.Errorf("failed to generate self-signed cert: %w", err) } } - xdsSrv.SetTlsConfig(s.cfg.HttpsPort, s.cfg.EnvoyCertPath, certContent, keyContent) + xdsSrv.SetTlsConfig(s.cfg.HttpsPort, tlsCertPath(s.cfg), certContent, keyContent) if s.extprocSrv == nil { routeDuration, err := newRouteDurationHistogram() if err != nil { @@ -245,9 +260,9 @@ func (s *RouterServer) Run(ctx context.Context) error { } s.extprocSrv = NewExtProcServer(s.cfg.ExtprocPort, s.apiClient, routeDuration) } - ctrl := NewController(s.k8sClient, s.clientset, s.cfg, xdsSrv, s.extprocSrv) + ctrl := NewController(s.k8sClient, s.clientset, s.cfg, xdsSrv, s.extprocSrv, provider) - s.health = newRouterHealth(s.cfg.HealthInterval, s.clientset, s.apiClient, s.cfg) + s.health = newRouterHealth(s.cfg.HealthInterval, s.clientset, s.apiClient, s.cfg, provider) // Start Controller / Watcher g.Go(func() error { @@ -262,17 +277,19 @@ func (s *RouterServer) Run(ctx context.Context) error { return nil }) - // Start xDS Server - g.Go(func() error { - slog.InfoContext(ctx, "Starting Envoy xDS Server", slog.Int("port", s.cfg.XdsPort)) - lis, err := net.Listen("tcp", fmt.Sprintf(":%d", s.cfg.XdsPort)) - if err != nil { - return fmt.Errorf("failed to listen on port %d: %w", s.cfg.XdsPort, err) - } - defer lis.Close() + if provider.RequiresXDS() { + // Start xDS Server + g.Go(func() error { + slog.InfoContext(ctx, "Starting xDS Server", slog.Int("port", s.cfg.XdsPort), slog.String("provider", provider.Name())) + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", s.cfg.XdsPort)) + if err != nil { + return fmt.Errorf("failed to listen on port %d: %w", s.cfg.XdsPort, err) + } + defer lis.Close() - return xdsSrv.Serve(ctx, lis) - }) + return xdsSrv.Serve(ctx, lis) + }) + } // Start ExtProc Server g.Go(func() error { diff --git a/cmd/atenet/internal/router/status.go b/cmd/atenet/internal/router/status.go index eab4c8183..979c74cb6 100644 --- a/cmd/atenet/internal/router/status.go +++ b/cmd/atenet/internal/router/status.go @@ -138,6 +138,7 @@ type DashboardContext struct { BuildTag string `json:"build_tag"` RouterClusterIP string `json:"router_cluster_ip"` Namespace string `json:"namespace"` + NetworkingMode string `json:"networking_mode"` HttpPort int `json:"port_http"` XdsPort int `json:"port_xds"` ExtprocPort int `json:"port_extproc"` @@ -165,7 +166,7 @@ func (s *RouterServer) getRouterIP(ctx context.Context) string { return "Standalone Mode (No Cluster IP)" } - svc, err := s.clientset.CoreV1().Services(s.cfg.Namespace).Get(ctx, "atenet-router", metav1.GetOptions{}) + svc, err := s.clientset.CoreV1().Services(s.cfg.Namespace).Get(ctx, s.cfg.RouterServiceName, metav1.GetOptions{}) if err != nil { return fmt.Sprintf("Lookup Failed: %v", err) } @@ -248,6 +249,7 @@ func (s *RouterServer) handleStatusz(w http.ResponseWriter, req *http.Request) { BuildTag: buildInfo, RouterClusterIP: routerIP, Namespace: s.cfg.Namespace, + NetworkingMode: s.cfg.NetworkingMode, HttpPort: s.cfg.HttpPort, XdsPort: s.cfg.XdsPort, ExtprocPort: s.cfg.ExtprocPort, diff --git a/cmd/atenet/internal/router/xds.go b/cmd/atenet/internal/router/xds.go index e7e1602a9..588817898 100644 --- a/cmd/atenet/internal/router/xds.go +++ b/cmd/atenet/internal/router/xds.go @@ -66,6 +66,7 @@ const ( RouteName = "substrate_routes" ClusterName = "ate-cluster" OtlpClusterName = "otel_collector_cluster" + websocketUpgradeType = "websocket" ) // XdsServer implements an aggregated discovery service server for dynamic Envoy router nodes. @@ -374,6 +375,9 @@ func (x *XdsServer) buildRoutes() *routev3.RouteConfiguration { Cluster: "dynamic_forward_proxy_cluster", }, Timeout: durationpb.New(10 * time.Second), + UpgradeConfigs: []*routev3.RouteAction_UpgradeConfig{ + {UpgradeType: websocketUpgradeType}, + }, }, }, }, @@ -422,6 +426,10 @@ func (x *XdsServer) buildHcm(statPrefix string) *anypb.Any { StatPrefix: statPrefix, GenerateRequestId: &wrapperspb.BoolValue{Value: true}, Tracing: x.buildTracing(), + UpgradeConfigs: []*hcmv3.HttpConnectionManager_UpgradeConfig{ + {UpgradeType: websocketUpgradeType}, + }, + StreamIdleTimeout: durationpb.New(0), AccessLog: []*accesslogv3.AccessLog{ { Name: "envoy.access_loggers.stdout", diff --git a/cmd/atenet/internal/router/xds_test.go b/cmd/atenet/internal/router/xds_test.go index 92e347648..f6548f41a 100644 --- a/cmd/atenet/internal/router/xds_test.go +++ b/cmd/atenet/internal/router/xds_test.go @@ -24,6 +24,7 @@ import ( clusterv3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" listenerv3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" routev3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + hcmv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" cachev3 "github.com/envoyproxy/go-control-plane/pkg/cache/v3" resourcev3 "github.com/envoyproxy/go-control-plane/pkg/resource/v3" ) @@ -116,6 +117,10 @@ func TestXdsServer_UpdateSnapshot(t *testing.T) { if fallbackRoute.GetMatch().GetPrefix() != "/" { t.Errorf("Expected path mapping prefix '/', got '%s'", fallbackRoute.GetMatch().GetPrefix()) } + routeAction := fallbackRoute.GetRoute() + if len(routeAction.GetUpgradeConfigs()) != 1 || routeAction.GetUpgradeConfigs()[0].GetUpgradeType() != websocketUpgradeType { + t.Errorf("Expected route websocket upgrade config, got %+v", routeAction.GetUpgradeConfigs()) + } } // Verify listeners generated @@ -135,6 +140,15 @@ func TestXdsServer_UpdateSnapshot(t *testing.T) { if sa.GetAddress() != "0.0.0.0" { t.Errorf("Expected address '0.0.0.0', got %s", sa.GetAddress()) } + + hcmAny := l.GetFilterChains()[0].GetFilters()[0].GetTypedConfig() + hcm := &hcmv3.HttpConnectionManager{} + if err := hcmAny.UnmarshalTo(hcm); err != nil { + t.Fatalf("Failed to unmarshal HCM: %v", err) + } + if len(hcm.GetUpgradeConfigs()) != 1 || hcm.GetUpgradeConfigs()[0].GetUpgradeType() != websocketUpgradeType { + t.Errorf("Expected HCM websocket upgrade config, got %+v", hcm.GetUpgradeConfigs()) + } } } diff --git a/hack/create-kind-cluster.sh b/hack/create-kind-cluster.sh index 26548f2b4..e7dcbaba1 100755 --- a/hack/create-kind-cluster.sh +++ b/hack/create-kind-cluster.sh @@ -18,6 +18,12 @@ set -o errexit -o nounset -o pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kind}" +# Enable the off-by-default certificate feature gates required by the mTLS +# install path (cmd/podcertcontroller). On by default — the Quickstart's +# `hack/install-ate-kind.sh --deploy-ate-system` uses mTLS. Opt out +# (KIND_ENABLE_PODCERT=false) only when installing JWT-mode manifests, which +# do not require these gates. +KIND_ENABLE_PODCERT="${KIND_ENABLE_PODCERT:-true}" reg_name="kind-registry" reg_port="5001" @@ -57,7 +63,7 @@ else echo "/dev/kvm not available: micro-VM support disabled (gVisor still works)." fi -echo "Creating kind configuration for cluster '${KIND_CLUSTER_NAME}'..." +echo "Creating kind configuration for cluster '${KIND_CLUSTER_NAME}' (KIND_ENABLE_PODCERT=${KIND_ENABLE_PODCERT})..." cat < "${ROOT}/bin/kind-config.yaml" kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 @@ -73,6 +79,8 @@ if [ "${HAS_KVM}" = "1" ]; then containerPath: /dev/kvm EOF fi + +if [ "${KIND_ENABLE_PODCERT}" = "true" ]; then cat <> "${ROOT}/bin/kind-config.yaml" # cmd/podcertcontroller depends on ClusterTrustBundle & PodCertificateRequest. # They are not enabled by default as of Kubernetes v1.36 @@ -84,6 +92,7 @@ featureGates: runtimeConfig: "certificates.k8s.io/v1beta1": "true" EOF +fi echo "Deleting existing kind cluster '${KIND_CLUSTER_NAME}' if it exists..." "${ROOT}"/hack/kind.sh delete cluster --name "${KIND_CLUSTER_NAME}" || true diff --git a/hack/gen-rbac.sh b/hack/gen-rbac.sh new file mode 100755 index 000000000..baa22fa51 --- /dev/null +++ b/hack/gen-rbac.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generate the controller ClusterRole into the Helm chart and templatize its +# name so multi-release installs do not collide on a cluster-scoped resource. +# +# controller-gen emits a YAML file with a fixed `roleName=` value. We post- +# process that file to swap the static name for the chart's fullname helper, +# matching the convention used by every other resource in charts/substrate/. +# +# Invoked via `go generate ./cmd/atecontroller/internal/controllers/...`. +set -o errexit -o nounset -o pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +OUT="${ROOT}/charts/substrate/templates/role.yaml" + +bash "${ROOT}/hack/run-tool.sh" controller-gen \ + "rbac:headerFile=${ROOT}/hack/boilerplate/sh.txt,roleName=ate-controller" \ + paths="${ROOT}/cmd/atecontroller/internal/controllers/..." \ + "output:rbac:artifacts:config=${ROOT}/charts/substrate/templates/" + +# Templatize the ClusterRole name. controller-gen emits ` name: ate-controller` +# at column 0; the substitution is exact-match to stay robust. +sed -i 's|^ name: ate-controller$| name: {{ include "substrate.fullname" (list "ate-controller" .) }}|' "${OUT}" diff --git a/hack/install-ate-kind-jwt.sh b/hack/install-ate-kind-jwt.sh new file mode 100755 index 000000000..66e36fb25 --- /dev/null +++ b/hack/install-ate-kind-jwt.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Install Agent Substrate on a kind cluster in JWT auth mode. +# +# Unlike the mTLS install path (hack/install-ate-kind.sh), this works on a +# stock Kubernetes cluster — no ClusterTrustBundle / PodCertificateRequest +# feature gates required. Suitable for a kind cluster created with +# KIND_ENABLE_PODCERT=false hack/create-kind-cluster.sh. +# +# Steps: +# 1. Render the chart with auth.mode=jwt + kind-specific values, resolve +# ko:// image refs against a local registry, and apply. +# 2. Apply the kind-only OTel collector from manifests/ate-install/kind/. +set -o errexit -o nounset -o pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +NS="${NS:-ate-system}" +KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kind}" +KUBECTL_CONTEXT="${KUBECTL_CONTEXT:-}" +KO_DOCKER_REPO="${KO_DOCKER_REPO:-localhost:5001}" +KO_DEFAULTPLATFORMS="${KO_DEFAULTPLATFORMS:-linux/$(go env GOARCH)}" +reg_name="kind-registry" +reg_port="5001" + +export KO_DOCKER_REPO KO_DEFAULTPLATFORMS + +run_kubectl() { + kubectl ${KUBECTL_CONTEXT:+--context=${KUBECTL_CONTEXT}} "$@" +} + +run_helm() { + helm ${KUBECTL_CONTEXT:+--kube-context=${KUBECTL_CONTEXT}} "$@" +} + +log_step() { + echo -e "\033[1;36m[step]:\033[0m $1" +} + +ensure_namespace() { + log_step "ensure_namespace ${NS}" + run_kubectl create namespace "${NS}" --dry-run=client -o yaml | run_kubectl apply -f - +} + +ensure_kind_local_registry() { + log_step "ensure_kind_local_registry" + + if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" == "true" ]; then + if ! docker port "${reg_name}" | grep -q "${reg_port}"; then + echo "Registry exists but is not mapped to port ${reg_port}. Recreating..." + docker rm -f "${reg_name}" + fi + fi + + if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" != "true" ]; then + docker run \ + -d --restart=always \ + --label created-by=agent-substrate \ + -p "127.0.0.1:${reg_port}:5000" \ + -p "[::1]:${reg_port}:5000" \ + --network bridge --name "${reg_name}" \ + registry:3 + fi + + if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name}")" = "null" ]; then + docker network connect "kind" "${reg_name}" + fi + + local registry_dir="/etc/containerd/certs.d/localhost:${reg_port}" + local node + for node in $("${ROOT}"/hack/kind.sh get nodes --name "${KIND_CLUSTER_NAME}"); do + docker exec "${node}" mkdir -p "${registry_dir}" + cat <') + + # ko resolve replaces ko:// refs with built+pushed image refs. + echo "${rendered}" | bash "${ROOT}/hack/run-tool.sh" ko resolve -f - \ + | run_kubectl apply -f - +} + +apply_crds() { + log_step "apply_crds" + run_helm upgrade --install substrate-crds "${ROOT}/charts/substrate-crds" +} + +apply_sandbox_configs() { + log_step "apply_sandbox_configs" + run_kubectl apply -f "${ROOT}/manifests/ate-install/sandboxconfig-validation.yaml" + run_kubectl apply -f "${ROOT}/manifests/ate-install/sandboxconfig-gvisor.yaml" +} + +apply_kind_extras() { + log_step "apply_kind_extras (otel-collector)" + run_kubectl apply -f "${ROOT}/manifests/ate-install/kind/otel-collector.yaml" +} + +wait_rollouts() { + log_step "wait_rollouts" + run_kubectl -n "${NS}" rollout status deployment/ate-api-server-deployment --timeout=180s + run_kubectl -n "${NS}" rollout status deployment/ate-controller --timeout=180s + run_kubectl -n "${NS}" rollout status deployment/atenet-router --timeout=180s + run_kubectl -n "${NS}" rollout status daemonset/atelet --timeout=180s + run_kubectl -n "${NS}" rollout status statefulset/valkey-cluster --timeout=180s +} + +ensure_namespace +ensure_kind_local_registry +apply_crds +apply_sandbox_configs +apply_chart +apply_kind_extras +wait_rollouts + +echo "Substrate (JWT mode) installed in namespace ${NS}." diff --git a/hack/install-ate.sh b/hack/install-ate.sh index c4f0880f9..53457311c 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -38,6 +38,8 @@ fi # ATE_DEMOS is an array that registers the prefix name of the demo functions. ATE_DEMOS=() +ATE_INSTALL_ATENET_ROUTER="${ATE_INSTALL_ATENET_ROUTER:-agentgateway}" + # Include demos. source "${ROOT}"/hack/install-demo-counter.sh source "${ROOT}"/hack/install-demo-sandbox.sh @@ -61,6 +63,7 @@ function usage() { echo "Overall infrastructure (all infrastructure components):" echo "" echo " --deploy-ate-system Deploy core system (CRDs, atelet, apiserver)" + echo " --router=agentgateway Select atenet-router implementation (default: agentgateway)" echo " --delete-ate-system Delete core system" echo " --delete-all Delete core system and all registered demos" echo " --auth-mode=mtls|jwt Select ateapi auth mode for --deploy-ate-system (default: mtls)" @@ -135,6 +138,54 @@ ate_auth_mode() { esac } +set_atenet_router() { + case "$1" in + agentgateway) + ATE_INSTALL_ATENET_ROUTER="$1" + ;; + *) + echo "unsupported atenet router mode: $1 (only agentgateway is supported)" >&2 + exit 1 + ;; + esac +} + +atenet_router_manifest() { + case "${ATE_INSTALL_ATENET_ROUTER}" in + agentgateway) + echo "manifests/ate-install/atenet-router.yaml" + ;; + *) + echo "unsupported atenet router mode: ${ATE_INSTALL_ATENET_ROUTER} (only agentgateway is supported)" >&2 + exit 1 + ;; + esac +} + +ate_install_kustomize_base_dir() { + case "${ATE_INSTALL_ATENET_ROUTER}" in + agentgateway) + echo "manifests/ate-install/base" + ;; + *) + echo "unsupported atenet router mode: ${ATE_INSTALL_ATENET_ROUTER} (only agentgateway is supported)" >&2 + exit 1 + ;; + esac +} + +ate_install_kustomize_dir() { + case "${ATE_INSTALL_ATENET_ROUTER}" in + agentgateway) + echo "manifests/ate-install/kind" + ;; + *) + echo "unsupported atenet router mode: ${ATE_INSTALL_ATENET_ROUTER} (only agentgateway is supported)" >&2 + exit 1 + ;; + esac +} + render_ate_system_manifests() { local auth_mode="" auth_mode="$(ate_auth_mode)" @@ -149,11 +200,9 @@ render_ate_system_manifests() { fi if [[ "${ATE_INSTALL_KIND:-false}" == "true" ]]; then - # Build everything resolved with Kustomize for Kind - kubectl kustomize manifests/ate-install/kind --load-restrictor LoadRestrictionsNone | run_ko resolve -f - + kubectl kustomize "$(ate_install_kustomize_dir)" --load-restrictor LoadRestrictionsNone | run_ko resolve -f - else - # Build everything resolved with base manifests for GKE - run_ko resolve -f manifests/ate-install + kubectl kustomize "$(ate_install_kustomize_base_dir)" --load-restrictor LoadRestrictionsNone | run_ko resolve -f - fi } @@ -253,6 +302,7 @@ ensure_crds() { deploy_crds() { log_step "deploy_crds" run_ko apply -f manifests/ate-install/generated + run_kubectl apply -f manifests/ate-install/role.yaml } deploy_ate_system() { @@ -270,7 +320,7 @@ deploy_ate_system() { run_kubectl apply -f manifests/ate-install/sandboxconfig-gvisor.yaml # Ensure namespace exists - run_kubectl apply -f manifests/ate-install/ate-system-namespace.yaml \ + run_kubectl apply -f manifests/ate-install/namespace.yaml \ && run_kubectl wait --for=jsonpath='{.status.phase}'=Active namespace/ate-system --timeout=60s ensure_apiserver_prerequisites @@ -321,7 +371,7 @@ deploy_ate_apiserver() { ensure_crds # Ensure namespace exists - run_kubectl apply -f manifests/ate-install/ate-system-namespace.yaml \ + run_kubectl apply -f manifests/ate-install/namespace.yaml \ && run_kubectl wait --for=jsonpath='{.status.phase}'=Active namespace/ate-system --timeout=60s ensure_apiserver_prerequisites @@ -335,7 +385,7 @@ deploy_atelet() { ensure_crds # Ensure namespace exists - run_kubectl apply -f manifests/ate-install/ate-system-namespace.yaml \ + run_kubectl apply -f manifests/ate-install/namespace.yaml \ && run_kubectl wait --for=jsonpath='{.status.phase}'=Active namespace/ate-system --timeout=60s local manifest="" @@ -355,10 +405,10 @@ deploy_atenet() { ensure_crds # Ensure namespace exists - run_kubectl apply -f manifests/ate-install/ate-system-namespace.yaml \ + run_kubectl apply -f manifests/ate-install/namespace.yaml \ && run_kubectl wait --for=jsonpath='{.status.phase}'=Active namespace/ate-system --timeout=60s - run_ko apply -f manifests/ate-install/atenet-router.yaml + run_ko apply -f "$(atenet_router_manifest)" run_ko apply -f manifests/ate-install/atenet-dns.yaml run_kubectl rollout status deployment/atenet-router -n ate-system --timeout=120s run_kubectl rollout status deployment/atenet-dns -n ate-system --timeout=120s @@ -472,7 +522,7 @@ delete_ate_system() { delete_atenet() { log_step "delete_atenet" - run_kubectl delete --ignore-not-found -f manifests/ate-install/atenet-router.yaml + run_kubectl delete --ignore-not-found -f "$(atenet_router_manifest)" } deploy_benchmarks() { @@ -507,6 +557,9 @@ for arg in "$@"; do usage exit 0 ;; + --router=*) + set_atenet_router "${arg#--router=}" + ;; esac done @@ -560,6 +613,7 @@ while [[ "$#" -gt 0 ]]; do ;; --deploy-ate-system) deploy_ate_system ;; + --router=*) ;; --delete-ate-system) delete_ate_system ;; --delete-all) delete_all ;; diff --git a/hack/render-manifests.sh b/hack/render-manifests.sh new file mode 100755 index 000000000..bbe50befb --- /dev/null +++ b/hack/render-manifests.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Render the substrate Helm chart into manifests/ate-install/ (mTLS-mode +# install) — the canonical kubectl-apply install path. The chart at +# charts/substrate/ is the single source of truth; this script only renders. +# +# Usage: +# hack/render-manifests.sh # write into manifests/ate-install/ +# hack/render-manifests.sh --check # fail if rendered output differs +# +set -o errexit -o nounset -o pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +OUT_DIR="${ROOT}/manifests/ate-install" +CHART_DIR="${ROOT}/charts/substrate" +CHECK_MODE="false" + +if [ "${1:-}" = "--check" ]; then + CHECK_MODE="true" +fi + +if ! command -v helm >/dev/null 2>&1; then + echo "helm not found in PATH" >&2 + exit 1 +fi + +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT + +helm template substrate "${CHART_DIR}" \ + --namespace ate-system \ + --set auth.mode=mtls \ + --set createNamespace=true \ + --set image.registry=ko://github.com/agent-substrate/substrate/cmd \ + --set image.tag="" \ + > "${TMP_DIR}/all.yaml" + +# Split into per-source files so the directory structure mirrors the chart +# templates, making diffs friendlier. +python3 - "${TMP_DIR}/all.yaml" "${TMP_DIR}/out" <<'PY' +import os, re, sys, yaml +in_path, out_dir = sys.argv[1], sys.argv[2] +os.makedirs(out_dir, exist_ok=True) + +with open(in_path) as f: + raw = f.read() + +# Helm prepends a "# Source: /templates/" comment to each doc. +docs_by_source = {} +for doc in raw.split('\n---\n'): + m = re.search(r'#\s*Source:\s*\S+/templates/(\S+)', doc) + src = m.group(1) if m else "misc.yaml" + # Drop the leading "# Source:" line from the written file. + cleaned = re.sub(r'^\s*#\s*Source:.*\n', '', doc, count=1, flags=re.MULTILINE) + if not cleaned.strip(): + continue + docs_by_source.setdefault(src, []).append(cleaned.strip()) + +for src, docs in docs_by_source.items(): + header = ( + "# Copyright 2026 Google LLC\n" + "#\n" + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n" + "# you may not use this file except in compliance with the License.\n" + "# You may obtain a copy of the License at\n" + "#\n" + "# http://www.apache.org/licenses/LICENSE-2.0\n" + "#\n" + "# Unless required by applicable law or agreed to in writing, software\n" + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n" + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n" + "# See the License for the specific language governing permissions and\n" + "# limitations under the License.\n" + "\n" + "# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh.\n" + "# Run `make helm-template` to regenerate.\n" + "\n" + ) + with open(os.path.join(out_dir, src), "w") as out: + out.write(header) + out.write("\n---\n".join(docs)) + out.write("\n") +PY + +if [ "${CHECK_MODE}" = "true" ]; then + # Only compare top-level files; subdirs like generated/ and kind/ are not + # produced by the chart and live alongside it intentionally. + CHECK_TMP="$(mktemp -d)" + trap 'rm -rf "$TMP_DIR" "$CHECK_TMP"' EXIT + mkdir -p "${CHECK_TMP}/current" + find "${OUT_DIR}" -maxdepth 1 -type f -name '*.yaml' -exec cp {} "${CHECK_TMP}/current/" \; + if ! diff -ruN "${CHECK_TMP}/current" "${TMP_DIR}/out" >/dev/null 2>&1; then + echo "manifests/ate-install/ is out of date. Run: make helm-template" >&2 + diff -ruN "${CHECK_TMP}/current" "${TMP_DIR}/out" | head -60 >&2 || true + exit 1 + fi + echo "manifests/ate-install/ matches chart output." + exit 0 +fi + +# Replace contents (preserve kind/ and generated/ subdirs which are not chart output). +mkdir -p "${OUT_DIR}" +find "${OUT_DIR}" -maxdepth 1 -type f -name '*.yaml' -delete +cp "${TMP_DIR}/out/"*.yaml "${OUT_DIR}/" +rendered_count="$(find "${OUT_DIR}" -maxdepth 1 -type f -name '*.yaml' | wc -l | xargs)" +echo "Rendered ${rendered_count} manifest files into ${OUT_DIR}" diff --git a/hack/update/licenses.sh b/hack/update/licenses.sh index 7861e6beb..b7e3b9765 100755 --- a/hack/update/licenses.sh +++ b/hack/update/licenses.sh @@ -24,6 +24,14 @@ OUTDIR="LICENSES" # under $ROOT # Ensure the tool is built and up-to-date GO_LICENSES_BIN="$(bash "${ROOT}/hack/run-tool.sh" --print-bin-path go-licenses)" +# go-licenses runs in temporary verification worktrees that do not have enough +# VCS metadata for Go's build stamping. +if [[ -n "${GOFLAGS:-}" ]]; then + export GOFLAGS="${GOFLAGS} -buildvcs=false" +else + export GOFLAGS="-buildvcs=false" +fi + # Clean out previous licenses rm -rf "${OUTDIR}" mkdir -p "${OUTDIR}" diff --git a/hack/values-kind-jwt.yaml b/hack/values-kind-jwt.yaml new file mode 100644 index 000000000..bbe2e387a --- /dev/null +++ b/hack/values-kind-jwt.yaml @@ -0,0 +1,41 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Helm values for installing substrate on a kind cluster in JWT mode. +# Used by hack/install-ate-kind-jwt.sh — does NOT require the off-by-default +# certificate feature gates that the mTLS install path needs. + +auth: + mode: jwt + jwt: + # Kind's default API server issuer. + issuer: https://kubernetes.default.svc.cluster.local + audience: api.ate-system.svc + serverCertSecret: ateapi-tls + caBundleConfigMap: ateapi-ca + +createNamespace: false + +# In-cluster OTel collector deployed alongside via manifests/ate-install/kind/otel-collector.yaml +otel: + endpoint: http://opentelemetry-collector.otel-system.svc:4317 + +monitoring: + gkePodMonitoring: + enabled: false + +atelet: + gcpAuthForImagePulls: false + extraArgs: + - --localhost-registry-replacement=kind-registry:5000 diff --git a/hack/verify/crd-chart.sh b/hack/verify/crd-chart.sh new file mode 100755 index 000000000..0837a8473 --- /dev/null +++ b/hack/verify/crd-chart.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit -o nounset -o pipefail + +ROOT="$(git rev-parse --show-toplevel)" +cd "${ROOT}" + +GENERATED_DIR="manifests/ate-install/generated" +CHART_TEMPLATES_DIR="charts/substrate-crds/templates" + +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "${TMP_DIR}"' EXIT + +mkdir -p "${TMP_DIR}/generated" "${TMP_DIR}/chart" +cp "${GENERATED_DIR}/"*.yaml "${TMP_DIR}/generated/" +cp "${CHART_TEMPLATES_DIR}/"*.yaml "${TMP_DIR}/chart/" + +# The generated CRDs start with a leading document separator after the +# boilerplate header. In chart templates that separator renders as a +# comment-only YAML document, so the chart copies intentionally omit it. +for file in "${TMP_DIR}/generated/"*.yaml; do + awk 'BEGIN { removed = 0 } /^---$/ && removed == 0 { removed = 1; next } { print }' "${file}" > "${file}.tmp" + mv "${file}.tmp" "${file}" +done + +if ! diff -ruN "${TMP_DIR}/generated" "${TMP_DIR}/chart" >/dev/null 2>&1; then + echo "charts/substrate-crds/templates is out of sync with ${GENERATED_DIR}" >&2 + echo "Copy updated CRDs into charts/substrate-crds/templates." >&2 + diff -ruN "${TMP_DIR}/generated" "${TMP_DIR}/chart" | head -80 >&2 || true + exit 1 +fi + +echo "charts/substrate-crds/templates matches generated CRDs." diff --git a/internal/ateclient/builder.go b/internal/ateclient/builder.go index 4a47ba29d..5d01822b5 100644 --- a/internal/ateclient/builder.go +++ b/internal/ateclient/builder.go @@ -24,6 +24,7 @@ import ( "strings" "sync" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "go.opentelemetry.io/otel" @@ -132,22 +133,22 @@ func dialPortForward(ctx context.Context, kubeconfigPath, k8sContext string, tra return nil, fmt.Errorf("failed to create k8s client: %w", err) } - // Look up the 'api' Service to dynamically get its pod selector - svc, err := clientset.CoreV1().Services("ate-system").Get(ctx, "api", metav1.GetOptions{}) + // Look up the ateapi Service to dynamically get its pod selector. + svc, err := clientset.CoreV1().Services(installdefaults.SystemNamespace).Get(ctx, installdefaults.APIServiceName, metav1.GetOptions{}) if err != nil { - return nil, fmt.Errorf("failed to get api service: %w", err) + return nil, fmt.Errorf("failed to get ateapi service %s/%s: %w", installdefaults.SystemNamespace, installdefaults.APIServiceName, err) } selector := labels.SelectorFromSet(svc.Spec.Selector).String() // Find the pods backing the service - pods, err := clientset.CoreV1().Pods("ate-system").List(ctx, metav1.ListOptions{ + pods, err := clientset.CoreV1().Pods(installdefaults.SystemNamespace).List(ctx, metav1.ListOptions{ LabelSelector: selector, }) if err != nil { return nil, fmt.Errorf("failed to list ateapi pods: %w", err) } if len(pods.Items) == 0 { - return nil, fmt.Errorf("no ate-api-server pods found in ate-system namespace") + return nil, fmt.Errorf("no ate-api-server pods found in %q namespace", installdefaults.SystemNamespace) } targetPod := pods.Items[0] @@ -254,7 +255,7 @@ func jwtDialOptions(ctx context.Context, clientset *kubernetes.Clientset) ([]grp ExpirationSeconds: &expirationSeconds, }, } - token, err := clientset.CoreV1().ServiceAccounts("ate-system").CreateToken(ctx, "ate-client", tokenRequest, metav1.CreateOptions{}) + token, err := clientset.CoreV1().ServiceAccounts(installdefaults.SystemNamespace).CreateToken(ctx, "ate-client", tokenRequest, metav1.CreateOptions{}) if err != nil { return nil, fmt.Errorf("failed to request ateapi bearer token: %w", err) } @@ -267,7 +268,7 @@ func jwtDialOptions(ctx context.Context, clientset *kubernetes.Clientset) ([]grp func isJWTMode(ctx context.Context, clientset *kubernetes.Clientset) (bool, error) { // TODO: Replace deployment introspection with an explicit client-readable // config file once ateapi auth mode is part of install/runtime config. - deployment, err := clientset.AppsV1().Deployments("ate-system").Get(ctx, "ate-api-server-deployment", metav1.GetOptions{}) + deployment, err := clientset.AppsV1().Deployments(installdefaults.SystemNamespace).Get(ctx, "ate-api-server-deployment", metav1.GetOptions{}) if err != nil { return false, fmt.Errorf("failed to get ate-api-server deployment: %w", err) } diff --git a/internal/e2e/suites/demo/demo_test.go b/internal/e2e/suites/demo/demo_test.go index 32f079fc3..fadee68f3 100644 --- a/internal/e2e/suites/demo/demo_test.go +++ b/internal/e2e/suites/demo/demo_test.go @@ -20,6 +20,8 @@ import ( "io" "net/http" "os" + "regexp" + "strconv" "strings" "testing" "time" @@ -330,11 +332,7 @@ func pauseActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj * } waitForActorStatus(ctx, t, clients, actorID, ateapipb.Actor_STATUS_RUNNING) - resp, err := callActor(t, demoAtespace, actorID) - if err != nil { - t.Fatalf("failed to call actor: %v", err) - } - + resp := callActorUntilCountAtLeast(t, demoAtespace, actorID, 1) if isMicroVMEnvironment() { validateCounterResponse(t, resp, "after creation", 1, -1) } else { @@ -359,10 +357,7 @@ func pauseActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj * } waitForActorStatus(ctx, t, clients, actorID, ateapipb.Actor_STATUS_RUNNING) - resp, err = callActor(t, demoAtespace, actorID) - if err != nil { - t.Fatalf("failed to call actor again: %v", err) - } + resp = callActorUntilCountAtLeast(t, demoAtespace, actorID, 2) if isMicroVMEnvironment() { validateCounterResponse(t, resp, "after pause", 2, -1) } else { @@ -418,10 +413,7 @@ func suspendActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj } waitForActorStatus(ctx, t, clients, actorID, ateapipb.Actor_STATUS_RUNNING) - resp, err := callActor(t, demoAtespace, actorID) - if err != nil { - t.Fatalf("failed to call actor: %v", err) - } + resp := callActorUntilCountAtLeast(t, demoAtespace, actorID, 1) if isMicroVMEnvironment() { validateCounterResponse(t, resp, "after creation", 1, -1) } else { @@ -446,10 +438,7 @@ func suspendActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj } waitForActorStatus(ctx, t, clients, actorID, ateapipb.Actor_STATUS_RUNNING) - resp, err = callActor(t, demoAtespace, actorID) - if err != nil { - t.Fatalf("failed to call actor again: %v", err) - } + resp = callActorUntilCountAtLeast(t, demoAtespace, actorID, 2) if isMicroVMEnvironment() { validateCounterResponse(t, resp, "after suspend", 2, -1) } else { @@ -619,6 +608,51 @@ func waitForActorStatus(ctx context.Context, t *testing.T, clients *e2e.Clients, t.Fatalf("timed out waiting for actor %q to reach status %v", actorID, expectedStatus) } +var preservedCountRe = regexp.MustCompile(`preserved memory count: ([0-9]+)`) + +func callActorUntilCountAtLeast(t *testing.T, atespace, actorID string, minCount int) string { + t.Helper() + + var lastErr error + var lastResp string + deadline := time.Now().Add(20 * time.Second) + for time.Now().Before(deadline) { + resp, err := callActor(t, atespace, actorID) + if err != nil { + lastErr = err + } else { + lastResp = resp + count, err := preservedCount(resp) + if err != nil { + lastErr = err + } else if count >= minCount { + return resp + } else { + lastErr = fmt.Errorf("expected preserved memory count >= %d, got %d in response: %s", minCount, count, resp) + } + } + time.Sleep(500 * time.Millisecond) + } + + if lastResp != "" { + t.Fatalf("timed out calling actor %q; last response: %s; last error: %v", actorID, lastResp, lastErr) + } + t.Fatalf("timed out calling actor %q; last error: %v", actorID, lastErr) + return "" +} + +func preservedCount(resp string) (int, error) { + matches := preservedCountRe.FindStringSubmatch(resp) + if matches == nil { + return 0, fmt.Errorf("response does not include preserved memory count: %s", resp) + } + count, err := strconv.Atoi(matches[1]) + if err != nil { + return 0, fmt.Errorf("parse preserved memory count %q: %w", matches[1], err) + } + return count, nil +} + func callActor(t *testing.T, atespace, actorID string) (string, error) { t.Helper() diff --git a/internal/installdefaults/installdefaults.go b/internal/installdefaults/installdefaults.go new file mode 100644 index 000000000..8f47d84f0 --- /dev/null +++ b/internal/installdefaults/installdefaults.go @@ -0,0 +1,47 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package installdefaults holds the default namespace and Service names +// that match the canonical install layout in manifests/ate-install/. +// Binaries use these as flag defaults; deployments that diverge from +// the canonical layout pass actual values via the corresponding flags. +package installdefaults + +import "os" + +const ( + // SystemNamespace is the namespace where substrate's control-plane + // components and the atelet DaemonSet run. + SystemNamespace = "ate-system" + // APIServiceName is the Service name of ate-api-server. + APIServiceName = "api" + // RouterServiceName is the Service name of atenet-router. + RouterServiceName = "atenet-router" + // DNSServiceName is the Service name of substrate's CoreDNS. + DNSServiceName = "dns" + + // PodNamespaceEnv is the conventional env var name for the namespace + // a pod is running in, exposed via Kubernetes' downward API. + PodNamespaceEnv = "POD_NAMESPACE" +) + +// NamespaceFromPodEnv returns the namespace from the PodNamespaceEnv env +// var when set (typically populated via Kubernetes' downward API), and +// falls back to SystemNamespace for non-k8s invocations (tests, local dev). +func NamespaceFromPodEnv() string { + if ns := os.Getenv(PodNamespaceEnv); ns != "" { + return ns + } + return SystemNamespace +} diff --git a/cmd/ateapi/internal/k8sjwt/k8sjwt.go b/internal/k8sjwt/k8sjwt.go similarity index 100% rename from cmd/ateapi/internal/k8sjwt/k8sjwt.go rename to internal/k8sjwt/k8sjwt.go diff --git a/manifests/ate-install/ate-api-server-envvars.yaml b/manifests/ate-install/ate-api-server-envvars.yaml new file mode 100644 index 000000000..61cf00820 --- /dev/null +++ b/manifests/ate-install/ate-api-server-envvars.yaml @@ -0,0 +1,28 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: ate-api-server-envvars + namespace: ate-system +data: + ATE_API_REDIS_ADDRESS: "valkey-cluster.ate-system.svc:6379" + ATE_API_REDIS_USE_IAM_AUTH: "false" + ATE_API_REDIS_TLS_SERVER_NAME: "" + ATE_API_REDIS_CLIENT_CERT: "" + ATE_API_K8SJWT_ISSUER: "https://kubernetes.default.svc.cluster.local" diff --git a/manifests/ate-install/ate-api-server.yaml b/manifests/ate-install/ate-api-server.yaml index 0b7b0c0ee..75cad754a 100644 --- a/manifests/ate-install/ate-api-server.yaml +++ b/manifests/ate-install/ate-api-server.yaml @@ -1,19 +1,26 @@ -# Copyright 2026 Google LLC +# Copyright 2026 Google LLC # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. -# Define Permissions (Read-Only for Pods) +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ate-api-server + namespace: ate-system +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -25,23 +32,12 @@ rules: - apiGroups: ["ate.dev"] resources: ["actortemplates", "workerpools", "sandboxconfigs"] verbs: ["get", "watch", "list"] -- apiGroups: ["ate.dev"] - resources: ["workerpools"] - verbs: ["get", "watch", "list"] # Secret reads for env source resolution are intentionally NOT granted # cluster-wide here. Each demo / tenant is responsible for granting # ate-api-server read access only to the specific Secrets referenced by its # ActorTemplates (e.g. via a namespace-scoped Role + RoleBinding using # resourceNames). --- -# Create Service Account for Workload Identity -apiVersion: v1 -kind: ServiceAccount -metadata: - name: ate-api-server - namespace: ate-system ---- -# 4. Bind Identity to Permissions apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -55,7 +51,21 @@ roleRef: name: ate-api-server-role apiGroup: rbac.authorization.k8s.io --- -# 5. Deploy the API Server +apiVersion: v1 +kind: Service +metadata: + name: api + namespace: ate-system +spec: + type: ClusterIP + selector: + app: ate-api-server + ports: + - name: grpc + protocol: TCP + port: 443 + targetPort: 443 +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -79,18 +89,18 @@ spec: - name: ate-api-server image: ko://github.com/agent-substrate/substrate/cmd/ateapi args: - - "--grpc-listen-addr=0.0.0.0:443" - - "--grpc-server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem" - - --redis-cluster-address=@env - - --redis-ca-certs=/etc/valkey-ca/ca.crt - - --redis-use-iam-auth=@env - - --redis-tls-server-name=@env - - --redis-client-cert=@env - - --client-jwt-issuer=@env - - --client-jwt-audience=api.ate-system.svc - - --session-id-jwt-pool=/run/session-id-jwt-pool/pool.json - - --session-id-ca-pool=/run/session-id-ca-pool/pool.json - - --workerpool-ca-certs=/run/workerpool-ca-certs/trust-bundle.pem + - "--grpc-listen-addr=0.0.0.0:443" + - "--grpc-server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem" + - "--redis-cluster-address=@env" + - "--redis-ca-certs=/etc/valkey-ca/ca.crt" + - "--redis-use-iam-auth=@env" + - "--redis-tls-server-name=@env" + - "--redis-client-cert=@env" + - "--client-jwt-issuer=@env" + - "--client-jwt-audience=api.ate-system.svc" + - "--session-id-jwt-pool=/run/session-id-jwt-pool/pool.json" + - "--session-id-ca-pool=/run/session-id-ca-pool/pool.json" + - "--workerpool-ca-certs=/run/workerpool-ca-certs/trust-bundle.pem" env: - name: POD_NAME valueFrom: @@ -106,31 +116,16 @@ spec: fieldPath: metadata.uid - name: OTEL_RESOURCE_ATTRIBUTES value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),service.instance.id=$(POD_UID) - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: http://opentelemetry-collector.gke-managed-otel.svc.cluster.local:4317 - # Inject env vars from a ConfigMap created by each developer. This lets - # each developer customize their own redis address, etc, without having - # to edit this manifest, which can remain constant across all - # developers. envFrom: - configMapRef: name: ate-api-server-envvars optional: true volumeMounts: - - name: "servicedns" - mountPath: "/run/servicedns.podcert.ate.dev" - - name: "session-id-jwt-pool" - mountPath: "/run/session-id-jwt-pool" - # Note: See README.md for how to generate this secret. - - name: "valkey-ca-certs" - mountPath: "/etc/valkey-ca" - readOnly: true - - name: "session-id-ca-pool" - mountPath: "/run/session-id-ca-pool" - readOnly: true - - name: "workerpool-ca-certs" - mountPath: "/run/workerpool-ca-certs" - readOnly: true + - { name: servicedns, mountPath: /run/servicedns.podcert.ate.dev } + - { name: session-id-jwt-pool, mountPath: /run/session-id-jwt-pool } + - { name: valkey-ca-certs, mountPath: /etc/valkey-ca, readOnly: true } + - { name: session-id-ca-pool, mountPath: /run/session-id-ca-pool, readOnly: true } + - { name: workerpool-ca-certs, mountPath: /run/workerpool-ca-certs, readOnly: true } ports: - containerPort: 443 - name: prometheus @@ -142,38 +137,35 @@ spec: initialDelaySeconds: 5 periodSeconds: 2 volumes: - - name: "servicedns" + - name: servicedns projected: sources: - podCertificate: signerName: servicedns.podcert.ate.dev/identity keyType: ECDSAP256 credentialBundlePath: credential-bundle.pem - - name: "session-id-jwt-pool" + - name: session-id-jwt-pool projected: sources: - secret: - name: "session-id-jwt-pool" + name: session-id-jwt-pool items: - - key: "pool" - path: "pool.json" - - name: "valkey-ca-certs" + - { key: pool, path: pool.json } + - name: valkey-ca-certs projected: sources: - secret: - name: "valkey-ca-certs" + name: valkey-ca-certs items: - - key: "ca.crt" - path: "ca.crt" - - name: "session-id-ca-pool" + - { key: ca.crt, path: ca.crt } + - name: session-id-ca-pool projected: sources: - secret: - name: "session-id-ca-pool" + name: session-id-ca-pool items: - - key: "pool" - path: "pool.json" - - name: "workerpool-ca-certs" + - { key: pool, path: pool.json } + - name: workerpool-ca-certs projected: sources: - clusterTrustBundle: @@ -182,19 +174,3 @@ spec: matchLabels: podcert.ate.dev/canarying: live path: trust-bundle.pem ---- -# 6. Expose the Session Assigner -apiVersion: v1 -kind: Service -metadata: - name: api - namespace: ate-system -spec: - type: ClusterIP - selector: - app: ate-api-server - ports: - - name: grpc - protocol: TCP - port: 443 - targetPort: 443 diff --git a/manifests/ate-install/ate-client.yaml b/manifests/ate-install/ate-client.yaml new file mode 100644 index 000000000..e59bd53f8 --- /dev/null +++ b/manifests/ate-install/ate-client.yaml @@ -0,0 +1,24 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ate-client + namespace: ate-system + labels: + apps: ate-client diff --git a/manifests/ate-install/ate-controller.yaml b/manifests/ate-install/ate-controller.yaml index 65ee46075..82eafb9cb 100644 --- a/manifests/ate-install/ate-controller.yaml +++ b/manifests/ate-install/ate-controller.yaml @@ -1,32 +1,20 @@ -# Copyright 2026 Google LLC +# Copyright 2026 Google LLC # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -kind: Namespace -apiVersion: v1 -metadata: - name: ate-system +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: ate-client - namespace: ate-system - labels: - apps: ate-client - ---- apiVersion: v1 kind: ServiceAccount metadata: @@ -34,7 +22,6 @@ metadata: namespace: ate-system labels: apps: ate-controller - --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -48,9 +35,7 @@ roleRef: kind: ClusterRole name: ate-controller apiGroup: rbac.authorization.k8s.io - --- - kind: Service apiVersion: v1 metadata: @@ -66,9 +51,7 @@ spec: port: 8080 targetPort: metrics protocol: TCP - --- - kind: Deployment apiVersion: apps/v1 metadata: @@ -89,7 +72,12 @@ spec: - name: ate-controller image: ko://github.com/agent-substrate/substrate/cmd/atecontroller args: - - --ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem + # The atecontroller binary defaults --ateapi-conn-spec to + # dns:///api.ate-system.svc:443, which is correct only for the + # canonical render (release name "substrate" in namespace + # "ate-system"). Pass the chart-resolved Service so the controller + # dials the right backend when substrate is installed as a subchart. + - "--ateapi-conn-spec=dns:///api.ate-system.svc:443" ports: - name: metrics containerPort: 8080 diff --git a/manifests/ate-install/atelet.yaml b/manifests/ate-install/atelet.yaml index c9564c40d..f366d19bd 100644 --- a/manifests/ate-install/atelet.yaml +++ b/manifests/ate-install/atelet.yaml @@ -1,25 +1,27 @@ -# Copyright 2026 Google LLC +# Copyright 2026 Google LLC # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -# 1. Create Service Account +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. + +# atelet — identical across auth modes (does not dial ateapi). apiVersion: v1 kind: ServiceAccount metadata: name: atelet namespace: ate-system --- -# 2. Define Permissions apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -29,7 +31,6 @@ rules: resources: ["pods"] verbs: ["get", "watch", "list"] --- -# 3. Bind Identity to Permissions apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -43,7 +44,6 @@ roleRef: name: atelet-role apiGroup: rbac.authorization.k8s.io --- -# 4. Create DaemonSet apiVersion: apps/v1 kind: DaemonSet metadata: @@ -68,7 +68,7 @@ spec: - name: atelet image: ko://github.com/agent-substrate/substrate/cmd/atelet args: - - --gcp-auth-for-image-pulls=true + - --gcp-auth-for-image-pulls=false securityContext: privileged: true env: @@ -76,24 +76,18 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: OTEL_RESOURCE_ATTRIBUTES - value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),service.instance.id=$(POD_UID) - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: http://opentelemetry-collector.gke-managed-otel.svc.cluster.local:4317 - name: ATE_STORAGE_BACKEND - value: "gcs" + value: "s3" + - name: AWS_REGION + value: us-east-1 + - name: AWS_ENDPOINT_URL + value: http://rustfs.ate-system.svc:9000 + - name: AWS_S3_USE_PATH_STYLE + value: "true" + - name: AWS_ACCESS_KEY_ID + value: "rustfsadmin" + - name: AWS_SECRET_ACCESS_KEY + value: "rustfsadmin" ports: - name: grpc containerPort: 8085 diff --git a/manifests/ate-install/atenet-dns.yaml b/manifests/ate-install/atenet-dns.yaml index 46968a386..48507c880 100644 --- a/manifests/ate-install/atenet-dns.yaml +++ b/manifests/ate-install/atenet-dns.yaml @@ -1,17 +1,21 @@ -# Copyright 2026 Google LLC +# Copyright 2026 Google LLC # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. + +# atenet-dns — identical across auth modes (does not dial ateapi). apiVersion: v1 kind: ServiceAccount metadata: @@ -34,6 +38,16 @@ rules: verbs: ["get", "list", "watch", "create", "update", "patch"] --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: atenet-dns + namespace: kube-system +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: atenet-dns @@ -48,16 +62,6 @@ roleRef: apiGroup: rbac.authorization.k8s.io --- apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: atenet-dns - namespace: kube-system -rules: -- apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch", "create", "update", "patch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: atenet-dns @@ -71,6 +75,25 @@ roleRef: name: atenet-dns apiGroup: rbac.authorization.k8s.io --- +apiVersion: v1 +kind: Service +metadata: + name: dns + namespace: ate-system + labels: + app: dns +spec: + selector: + app: dns + type: ClusterIP + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -94,8 +117,6 @@ spec: - name: init-dns image: busybox:1.36 command: ["sh", "-c"] - # Initial core file is sufficient to start CoreDNS but does not contain - # any additional configuration. The controller will update the Corefile. args: - | cat <<'EOF' > /etc/coredns/Corefile @@ -149,28 +170,19 @@ spec: - "--log-level=debug" - "--interval=10s" - "--corefile-path=/etc/coredns/Corefile" + # Pass the chart-resolved Service names so the controller looks up the + # correct objects when substrate is installed as a subchart. The + # system namespace is read from POD_NAMESPACE below. + - "--router-service-name=atenet-router" + - "--dns-service-name=dns" + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace volumeMounts: - name: dns-config-volume mountPath: /etc/coredns volumes: - name: dns-config-volume emptyDir: {} ---- -apiVersion: v1 -kind: Service -metadata: - name: dns - namespace: ate-system - labels: - app: dns -spec: - selector: - app: dns - type: ClusterIP - ports: - - name: dns - port: 53 - protocol: UDP - - name: dns-tcp - port: 53 - protocol: TCP \ No newline at end of file diff --git a/manifests/ate-install/atenet-router-monitoring.yaml b/manifests/ate-install/atenet-router-monitoring.yaml deleted file mode 100644 index 6a42ac904..000000000 --- a/manifests/ate-install/atenet-router-monitoring.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Scrape the Envoy sidecar's admin /stats/prometheus endpoint so its end-to-end -# request-latency histogram (envoy_http_downstream_rq_time, milliseconds) reaches -# Google Managed Prometheus. This is E2E *context* for the per-stage latency -# dashboard, not an SLI we own (the SLI is the OTLP atenet.router.route.duration -# histogram). Envoy only speaks Prometheus, so it needs an explicit scrape; the -# admin port (9901) is already exposed by the envoy container above. -apiVersion: monitoring.googleapis.com/v1 -kind: PodMonitoring -metadata: - name: atenet-router-envoy - namespace: ate-system - labels: - app: atenet-router -spec: - selector: - matchLabels: - app: atenet-router - endpoints: - - port: admin - path: /stats/prometheus - interval: 30s diff --git a/manifests/ate-install/atenet-router.yaml b/manifests/ate-install/atenet-router.yaml index 967f22c29..e389432d7 100644 --- a/manifests/ate-install/atenet-router.yaml +++ b/manifests/ate-install/atenet-router.yaml @@ -1,16 +1,19 @@ -# Copyright 2026 Google LLC +# Copyright 2026 Google LLC # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. apiVersion: v1 kind: ServiceAccount @@ -20,6 +23,69 @@ metadata: labels: app: atenet-router --- +apiVersion: v1 +kind: ConfigMap +metadata: + name: atenet-router-agentgateway-config + namespace: ate-system +data: + config.yaml: | + # yaml-language-server: $schema=https://agentgateway.dev/schema/config + config: + adminAddr: "127.0.0.1:15000" + readinessAddr: "0.0.0.0:15021" + statsAddr: "0.0.0.0:15020" + binds: + - port: 8080 + listeners: + - name: http + protocol: HTTP + routes: + - name: substrate-http + matches: + - path: + pathPrefix: / + policies: + extProc: + host: "127.0.0.1:50051" + failureMode: failClosed + processingOptions: + requestBodyMode: none + responseBodyMode: none + requestHeaderMode: send + responseHeaderMode: skip + requestTrailerMode: skip + responseTrailerMode: skip + backends: + - dynamic: {} + - port: 8443 + listeners: + - name: https + protocol: HTTPS + tls: + + cert: "/run/servicedns.podcert.ate.dev/cert.pem" + key: "/run/servicedns.podcert.ate.dev/key.pem" + + routes: + - name: substrate-https + matches: + - path: + pathPrefix: / + policies: + extProc: + host: "127.0.0.1:50051" + failureMode: failClosed + processingOptions: + requestBodyMode: none + responseBodyMode: none + requestHeaderMode: send + responseHeaderMode: skip + requestTrailerMode: skip + responseTrailerMode: skip + backends: + - dynamic: {} +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -48,56 +114,23 @@ roleRef: apiGroup: rbac.authorization.k8s.io --- apiVersion: v1 -kind: ConfigMap +kind: Service metadata: - name: atenet-router-envoy-config + name: atenet-router namespace: ate-system -data: - envoy.yaml: | - admin: - address: - socket_address: - address: 0.0.0.0 - port_value: 9901 - - node: - id: substrate-envoy-node - cluster: substrate-router-cluster - - dynamic_resources: - lds_config: - resource_api_version: V3 - ads: {} - cds_config: - resource_api_version: V3 - ads: {} - ads_config: - api_type: GRPC - transport_api_version: V3 - grpc_services: - - envoy_grpc: - cluster_name: xds_cluster - - static_resources: - clusters: - - name: xds_cluster - connect_timeout: 0.25s - type: STRICT_DNS - lb_policy: ROUND_ROBIN - typed_extension_protocol_options: - envoy.extensions.upstreams.http.v3.HttpProtocolOptions: - "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions - explicit_http_config: - http2_protocol_options: {} - load_assignment: - cluster_name: xds_cluster - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: 127.0.0.1 - port_value: 18000 +spec: + type: ClusterIP + selector: + app: atenet-router + ports: + - name: http + port: 80 + targetPort: 8080 + protocol: TCP + - name: https + port: 443 + targetPort: 8443 + protocol: TCP --- apiVersion: apps/v1 kind: Deployment @@ -126,17 +159,17 @@ spec: args: - "router" - "--standalone" + - "--networking-mode=agentgateway" - "--namespace=ate-system" + # Pass the chart-resolved router Service name so /statusz looks up the + # correct Service when substrate is installed as a subchart. + - "--router-service-name=atenet-router" - "--port-http=8080" - - "--port-xds=18000" - "--port-extproc=50051" - "--extproc-address=127.0.0.1" - "--ateapi-address=api.ate-system.svc:443" - "--status-port=4040" - "--port-https=8443" - - "--envoy-cert-path=/run/servicedns.podcert.ate.dev/credential-bundle.pem" - - "--otlp-collector-address=opentelemetry-collector.gke-managed-otel.svc.cluster.local:4317" - - "--ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem" env: - name: POD_NAME valueFrom: @@ -152,64 +185,46 @@ spec: fieldPath: metadata.uid - name: OTEL_RESOURCE_ATTRIBUTES value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),service.instance.id=$(POD_UID) - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: http://opentelemetry-collector.gke-managed-otel.svc.cluster.local:4317 ports: - - name: xds - containerPort: 18000 - name: extproc containerPort: 50051 - name: status containerPort: 4040 - name: metrics containerPort: 9090 - - name: envoy - image: envoyproxy/envoy:v1.30-latest - command: - - "/usr/local/bin/envoy" - - "-c" - - "/etc/envoy/envoy.yaml" - - "--component-log-level" - - "upstream:debug,router:debug,ext_proc:debug" + - name: agentgateway + image: cr.agentgateway.dev/agentgateway:v1.3.0-alpha.1 + args: + - "-f" + - "/etc/agentgateway/config.yaml" ports: - name: http containerPort: 8080 - name: https containerPort: 8443 - - name: admin - containerPort: 9901 + - name: readiness + containerPort: 15021 + - name: gw-metrics + containerPort: 15020 volumeMounts: - - name: envoy-config - mountPath: /etc/envoy + - name: agentgateway-config + mountPath: /etc/agentgateway - name: "servicedns" mountPath: "/run/servicedns.podcert.ate.dev" + readinessProbe: + httpGet: + path: /healthz/ready + port: readiness + periodSeconds: 10 volumes: - - name: envoy-config + - name: agentgateway-config configMap: - name: atenet-router-envoy-config + name: atenet-router-agentgateway-config - name: "servicedns" projected: sources: - podCertificate: signerName: servicedns.podcert.ate.dev/identity keyType: ECDSAP256 - credentialBundlePath: credential-bundle.pem ---- -apiVersion: v1 -kind: Service -metadata: - name: atenet-router - namespace: ate-system -spec: - type: ClusterIP - selector: - app: atenet-router - ports: - - name: http - port: 80 - targetPort: 8080 - protocol: TCP - - name: https - port: 443 - targetPort: 8443 - protocol: TCP + certificateChainPath: cert.pem + keyPath: key.pem diff --git a/manifests/ate-install/base-agentgateway/kustomization.yaml b/manifests/ate-install/base-agentgateway/kustomization.yaml new file mode 100644 index 000000000..61cde5ace --- /dev/null +++ b/manifests/ate-install/base-agentgateway/kustomization.yaml @@ -0,0 +1,25 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../ate-api-server.yaml + - ../ate-controller.yaml + - ../atelet.yaml + - ../atenet-dns.yaml + - ../atenet-router.yaml + - ../valkey.yaml + - ../pod-certificate-controller.yaml diff --git a/manifests/ate-install/base/kustomization.yaml b/manifests/ate-install/base/kustomization.yaml new file mode 100644 index 000000000..61cde5ace --- /dev/null +++ b/manifests/ate-install/base/kustomization.yaml @@ -0,0 +1,25 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../ate-api-server.yaml + - ../ate-controller.yaml + - ../atelet.yaml + - ../atenet-dns.yaml + - ../atenet-router.yaml + - ../valkey.yaml + - ../pod-certificate-controller.yaml diff --git a/manifests/ate-install/jwt/kustomization.yaml b/manifests/ate-install/jwt/kustomization.yaml index 64179b823..296efc70f 100644 --- a/manifests/ate-install/jwt/kustomization.yaml +++ b/manifests/ate-install/jwt/kustomization.yaml @@ -16,6 +16,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: + - ../ate-client.yaml - ../ate-api-server.yaml - ../ate-controller.yaml - ../atelet.yaml @@ -44,11 +45,11 @@ patches: namespace: ate-system patch: |- - op: add - path: /spec/template/spec/containers/0/args/- - value: --ateapi-auth=jwt - - op: add - path: /spec/template/spec/containers/0/args/- - value: --ateapi-token-file=/run/ateapi-token/token + path: /spec/template/spec/containers/0/args + value: + - --ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem + - --ateapi-auth=jwt + - --ateapi-token-file=/run/ateapi-token/token - target: group: apps version: v1 @@ -56,6 +57,9 @@ patches: name: atenet-router namespace: ate-system patch: |- + - op: add + path: /spec/template/spec/containers/0/args/- + value: --ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem - op: add path: /spec/template/spec/containers/0/args/- value: --ateapi-auth=jwt diff --git a/manifests/ate-install/kind-agentgateway/kustomization.yaml b/manifests/ate-install/kind-agentgateway/kustomization.yaml new file mode 100644 index 000000000..b3f849937 --- /dev/null +++ b/manifests/ate-install/kind-agentgateway/kustomization.yaml @@ -0,0 +1,43 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../ate-api-server.yaml + - ../ate-controller.yaml + - ../kind/atelet + - ../atenet-dns.yaml + - ../atenet-router.yaml + - ../valkey.yaml + - ../pod-certificate-controller.yaml + - ../kind/rustfs.yaml + - ../kind/otel-collector.yaml + +patches: + - patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: ate-api-server-deployment + namespace: ate-system + spec: + template: + spec: + containers: + - name: ate-api-server + env: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: http://opentelemetry-collector.otel-system.svc:4317 diff --git a/manifests/ate-install/kind-jwt/kustomization.yaml b/manifests/ate-install/kind-jwt/kustomization.yaml index cd3d9ea1f..ec26cc4c9 100644 --- a/manifests/ate-install/kind-jwt/kustomization.yaml +++ b/manifests/ate-install/kind-jwt/kustomization.yaml @@ -16,6 +16,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: + - ../ate-client.yaml - ../kind patches: @@ -38,11 +39,11 @@ patches: namespace: ate-system patch: |- - op: add - path: /spec/template/spec/containers/0/args/- - value: --ateapi-auth=jwt - - op: add - path: /spec/template/spec/containers/0/args/- - value: --ateapi-token-file=/run/ateapi-token/token + path: /spec/template/spec/containers/0/args + value: + - --ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem + - --ateapi-auth=jwt + - --ateapi-token-file=/run/ateapi-token/token - target: group: apps version: v1 @@ -50,6 +51,9 @@ patches: name: atenet-router namespace: ate-system patch: |- + - op: add + path: /spec/template/spec/containers/0/args/- + value: --ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem - op: add path: /spec/template/spec/containers/0/args/- value: --ateapi-auth=jwt diff --git a/manifests/ate-install/kind/kustomization.yaml b/manifests/ate-install/kind/kustomization.yaml index 43683e075..26d5d170f 100644 --- a/manifests/ate-install/kind/kustomization.yaml +++ b/manifests/ate-install/kind/kustomization.yaml @@ -23,6 +23,7 @@ resources: - ../atenet-router.yaml - ../valkey.yaml - ../pod-certificate-controller.yaml + - ../role.yaml - rustfs.yaml - ./otel-collector.yaml - ./prometheus.yaml diff --git a/manifests/ate-install/namespace.yaml b/manifests/ate-install/namespace.yaml new file mode 100644 index 000000000..9b6e06c5a --- /dev/null +++ b/manifests/ate-install/namespace.yaml @@ -0,0 +1,22 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. + +--- +apiVersion: v1 +kind: Namespace +metadata: + name: ate-system diff --git a/manifests/ate-install/pod-certificate-controller.yaml b/manifests/ate-install/pod-certificate-controller.yaml index 17c7b6fd2..987c1974b 100644 --- a/manifests/ate-install/pod-certificate-controller.yaml +++ b/manifests/ate-install/pod-certificate-controller.yaml @@ -1,16 +1,19 @@ -# Copyright 2026 Google LLC +# Copyright 2026 Google LLC # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. apiVersion: v1 kind: Namespace diff --git a/manifests/ate-install/role.yaml b/manifests/ate-install/role.yaml new file mode 100644 index 000000000..6a3558617 --- /dev/null +++ b/manifests/ate-install/role.yaml @@ -0,0 +1,98 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ate-controller +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ate.dev + resources: + - actortemplates + - workerpools + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ate.dev + resources: + - actortemplates/finalizers + - workerpools/finalizers + verbs: + - update +- apiGroups: + - ate.dev + resources: + - actortemplates/status + - workerpools/status + verbs: + - get + - patch + - update +--- +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/manifests/ate-install/rustfs.yaml b/manifests/ate-install/rustfs.yaml new file mode 100644 index 000000000..d6be30812 --- /dev/null +++ b/manifests/ate-install/rustfs.yaml @@ -0,0 +1,136 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. + +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: rustfs-data + namespace: ate-system +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: rustfs + namespace: ate-system +spec: + selector: + app: rustfs + ports: + - name: api + port: 9000 + targetPort: 9000 + - name: console + port: 9001 + targetPort: 9001 + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rustfs + namespace: ate-system +spec: + replicas: 1 + selector: + matchLabels: + app: rustfs + template: + metadata: + labels: + app: rustfs + spec: + securityContext: + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + containers: + - name: rustfs + image: rustfs/rustfs:1.0.0-beta.3@sha256:378642b05b7dcb4849fb77ebe6aca4ced1c3f66e7e504247df95a5c9018d3358 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9000 + name: api + - containerPort: 9001 + name: console + env: + - name: RUSTFS_ADDRESS + value: ":9000" + - name: RUSTFS_CONSOLE_ADDRESS + value: ":9001" + - name: RUSTFS_CONSOLE_ENABLE + value: "true" + - name: RUSTFS_VOLUMES + value: "/data" + - name: RUSTFS_ACCESS_KEY + value: "rustfsadmin" + - name: RUSTFS_SECRET_KEY + value: "rustfsadmin" + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + persistentVolumeClaim: + claimName: rustfs-data +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: rustfs-bucket-init + namespace: ate-system +spec: + backoffLimit: 10 + template: + spec: + restartPolicy: OnFailure + containers: + - name: create-bucket + image: amazon/aws-cli:2.17.0@sha256:643507c10ada7964ca6157b3d799f030b90577643da9955d319a77399ed80d73 + env: + - name: AWS_ACCESS_KEY_ID + value: "rustfsadmin" + - name: AWS_SECRET_ACCESS_KEY + value: "rustfsadmin" + - name: AWS_REGION + value: us-east-1 + - name: AWS_ENDPOINT_URL + value: http://rustfs.ate-system.svc:9000 + command: + - /bin/sh + - -c + - | + set -e + for i in $(seq 1 60); do + if aws s3api head-bucket --bucket ate-snapshots 2>/dev/null; then + echo "bucket ate-snapshots already exists" + exit 0 + fi + if aws s3api create-bucket --bucket ate-snapshots 2>/dev/null; then + echo "bucket ate-snapshots created" + exit 0 + fi + echo "waiting for rustfs to become available... ($i/60)" + sleep 2 + done + echo "timed out waiting for rustfs" + exit 1 diff --git a/manifests/ate-install/sandboxconfig-gvisor.yaml b/manifests/ate-install/sandboxconfig-gvisor.yaml index 7c4ad881c..2caea1d75 100644 --- a/manifests/ate-install/sandboxconfig-gvisor.yaml +++ b/manifests/ate-install/sandboxconfig-gvisor.yaml @@ -1,16 +1,19 @@ -# Copyright 2026 Google LLC +# Copyright 2026 Google LLC # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. # Cluster-wide default SandboxConfig for the gVisor (runsc) sandbox class. A # WorkerPool with sandboxClass gvisor (the default) and no explicit diff --git a/manifests/ate-install/sandboxconfig-validation.yaml b/manifests/ate-install/sandboxconfig-validation.yaml index d350d10b4..30545a3b0 100644 --- a/manifests/ate-install/sandboxconfig-validation.yaml +++ b/manifests/ate-install/sandboxconfig-validation.yaml @@ -1,16 +1,19 @@ -# Copyright 2026 Google LLC +# Copyright 2026 Google LLC # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. # Per-sandbox-class asset requirements for SandboxConfig. The CRD schema is # generic (any arch -> any asset name -> {url, sha256}); this policy enforces the diff --git a/manifests/ate-install/valkey.yaml b/manifests/ate-install/valkey.yaml index 2e5fb0088..f36724382 100644 --- a/manifests/ate-install/valkey.yaml +++ b/manifests/ate-install/valkey.yaml @@ -1,16 +1,19 @@ -# Copyright 2026 Google LLC +# Copyright 2026 Google LLC # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. apiVersion: v1 kind: ConfigMap @@ -30,8 +33,6 @@ data: tls-key-file /run/servicedns.podcert.ate.dev/credential-bundle.pem tls-ca-cert-file /etc/valkey-ca/ca.crt tls-auth-clients yes - # Reload the cert every 12h. - tls-auto-reload-interval 43200 # Enable cluster mode cluster-enabled yes @@ -89,21 +90,8 @@ spec: spec: containers: - name: valkey - image: valkey/valkey:9.1@sha256:4963247afc4cd33c7d3b2d2816b9f7f8eeebab148d29056c2ca4d7cbc966f2d9 - command: - - /bin/sh - - -c - - | - set -e - POD_DNS="${HOSTNAME}.valkey-cluster-service.ate-system.svc" - cp /etc/valkey/valkey.conf /tmp/valkey.conf - { - echo "cluster-announce-hostname ${POD_DNS}" - echo "cluster-announce-tls-port 6379" - echo "cluster-announce-bus-port 16379" - echo "cluster-preferred-endpoint-type hostname" - } >> /tmp/valkey.conf - exec valkey-server /tmp/valkey.conf + image: valkey/valkey:8.0 + command: ["valkey-server", "/etc/valkey/valkey.conf"] ports: - name: valkey containerPort: 6379 @@ -161,7 +149,7 @@ spec: restartPolicy: OnFailure containers: - name: init - image: valkey/valkey:9.1@sha256:4963247afc4cd33c7d3b2d2816b9f7f8eeebab148d29056c2ca4d7cbc966f2d9 + image: valkey/valkey:8.0 volumeMounts: - name: servicedns mountPath: /run/servicedns.podcert.ate.dev @@ -181,10 +169,11 @@ spec: done done - echo "All pods resolved. Building stable node list..." - VALKEY_NODES="" + echo "All pods resolved. Getting IPs..." + POD_IPS="" for i in 0 1 2 3 4 5; do - VALKEY_NODES="${VALKEY_NODES} valkey-cluster-${i}.valkey-cluster-service.ate-system.svc:6379" + ip=$(getent hosts valkey-cluster-${i}.valkey-cluster-service.ate-system.svc | awk '{print $1}') + POD_IPS="${POD_IPS} ${ip}:6379" done echo "Checking if Valkey cluster is already initialized..." @@ -201,7 +190,7 @@ spec: --cacert /etc/valkey-ca/ca.crt \ --cert /run/servicedns.podcert.ate.dev/credential-bundle.pem \ --key /run/servicedns.podcert.ate.dev/credential-bundle.pem \ - --cluster create ${VALKEY_NODES} \ + --cluster create ${POD_IPS} \ --cluster-replicas 1 \ --cluster-yes echo "Cluster initialization complete!"