From a9cdb8a9ca8c06b228d661e842a10d684988dbee Mon Sep 17 00:00:00 2001 From: Yuriy Bezsonov Date: Mon, 12 Oct 2026 10:50:05 +0200 Subject: [PATCH 1/5] fix(java-performance): Platform pods off the app's node The platform (Prometheus, Grafana, Pyroscope, Alloy, sensor, Kyverno, boost controller, metrics-server) had no node selector and shared the app's 4-vCPU workshop node. The app then shared physical cores with it: baseline startup 18.5-19.7 s instead of ~16.5 s, steady CPU 0.22 instead of ~0.14, throttling 0.15 instead of ~0.07. - platform-placement.sh: every platform Deployment/StatefulSet to the Auto Mode general-purpose pool, marked do-not-disrupt; run by the template after stage E. - nodepool-pin.sh: workshop pool consolidates WhenEmpty (5m), so the app's node is not replaced when the app shrinks. --- .../deploy/java-performance/nodepool-pin.sh | 10 +++- .../java-performance/platform-placement.sh | 58 +++++++++++++++++++ infra/scripts/templates/java-performance.sh | 4 ++ 3 files changed, 70 insertions(+), 2 deletions(-) create mode 100755 infra/scripts/deploy/java-performance/platform-placement.sh diff --git a/infra/scripts/deploy/java-performance/nodepool-pin.sh b/infra/scripts/deploy/java-performance/nodepool-pin.sh index 5afa6b87..f4769746 100755 --- a/infra/scripts/deploy/java-performance/nodepool-pin.sh +++ b/infra/scripts/deploy/java-performance/nodepool-pin.sh @@ -9,6 +9,11 @@ # (e.g. m8azn at 5 GHz) and startup "improves" from hardware, not from the change. # Family, not type, keeps capacity flexible across sizes. # +# Consolidation: WhenEmpty only. The pool carries just the app (platform-placement.sh), and +# one m6a.xlarge fits every state of the session, rollout surge and boosted start included. +# WhenEmptyOrUnderutilized (the eks.sh default) would replace the node when the app shrinks +# and move the pod mid-measurement; WhenEmpty only removes a node nothing runs on. +# # Replaces the three scheduling requirements (cpu-manufacturer, instance-category, # instance-generation) with one (instance-family); capacity-type, arch, os, cpu and # memory requirements stay. Idempotent. Runs after setup/eks.sh. @@ -38,7 +43,8 @@ PATCH="$(kubectl get nodepool "${NODEPOOL}" -o json | jq -c --arg fam "${INSTANC "eks.amazonaws.com/instance-generation", "eks.amazonaws.com/instance-family") | not)) + [{"key":"eks.amazonaws.com/instance-family","operator":"In","values":[$fam]}] - | {spec:{template:{spec:{requirements:.}}}}')" + | {spec:{template:{spec:{requirements:.}}, + disruption:{consolidationPolicy:"WhenEmpty",consolidateAfter:"5m"}}}')" kubectl patch nodepool "${NODEPOOL}" --type merge -p "${PATCH}" >/dev/null \ - && log_success "NodePool ${NODEPOOL} pinned to ${INSTANCE_FAMILY} (existing nodes drain on consolidation; new nodes are ${INSTANCE_FAMILY})" \ + && log_success "NodePool ${NODEPOOL} pinned to ${INSTANCE_FAMILY}, consolidation WhenEmpty (existing nodes drain on consolidation; new nodes are ${INSTANCE_FAMILY})" \ || { log_error "NodePool patch failed"; exit 1; } diff --git a/infra/scripts/deploy/java-performance/platform-placement.sh b/infra/scripts/deploy/java-performance/platform-placement.sh new file mode 100755 index 00000000..ab07e4f5 --- /dev/null +++ b/infra/scripts/deploy/java-performance/platform-placement.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# ============================================================================= +# platform-placement.sh — keep the platform off the application's node (java-performance only). +# +# The app selects the `workshop` NodePool (karpenter.sh/nodepool: workshop). The platform +# workloads have no selector, so Karpenter places them wherever there is room, usually on +# the app's 4-vCPU node. There the app shares physical cores (SMT) with Prometheus, +# Pyroscope and Kyverno, and the session's measurements move with that load (startup and +# steady CPU 10–60 % higher in a test run). This script pins every platform Deployment and +# StatefulSet to the Auto Mode built-in `general-purpose` NodePool, so the `workshop` node +# carries only the app and the DaemonSets. The app manifest is unchanged. +# +# Namespaces: monitoring (Prometheus, Grafana, Pyroscope, Alloy, sensor), kyverno, +# kube-startup-cpu-boost-system, and metrics-server in kube-system. Pods move once, at +# bootstrap; Prometheus, Grafana and Pyroscope keep their data on their PVCs. The pods are +# marked do-not-disrupt, so consolidation does not move them later. +# Idempotent. Run after the platform installs (template stage E). A later `helm upgrade` +# of one of these releases drops the selector: run this script again afterwards. +# ============================================================================= +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/../../lib/common.sh" + +POOL="${PLATFORM_NODEPOOL:-general-purpose}" +kubectl get nodepool "${POOL}" >/dev/null 2>&1 || { log_error "NodePool ${POOL} not found (EKS Auto Mode built-in pools disabled?)"; exit 1; } + +# do-not-disrupt: the general-purpose pool consolidates underutilized nodes after 30 s. A +# consolidation during the session would move the sensor or Kyverno in the middle of a +# checklist or an admission; the annotation keeps their nodes until the pods go. +PATCH="{\"spec\":{\"template\":{\"metadata\":{\"annotations\":{\"karpenter.sh/do-not-disrupt\":\"true\"}},\"spec\":{\"nodeSelector\":{\"karpenter.sh/nodepool\":\"${POOL}\"}}}}}" +RC=0 +pin() { # pin NAMESPACE KIND/NAME... + local ns="$1"; shift + for w in "$@"; do + if kubectl -n "${ns}" patch "${w}" --type merge -p "${PATCH}" >/dev/null; then + log_info "${ns}/${w} → ${POOL}" + else + log_error "patch failed: ${ns}/${w}"; RC=1 + fi + done +} + +for ns in monitoring kyverno kube-startup-cpu-boost-system; do + kubectl get namespace "${ns}" >/dev/null 2>&1 || { log_warning "namespace ${ns} not found, skipped"; continue; } + mapfile -t W < <(kubectl -n "${ns}" get deployment,statefulset -o name) + [ "${#W[@]}" -gt 0 ] && pin "${ns}" "${W[@]}" +done +kubectl -n kube-system get deployment metrics-server >/dev/null 2>&1 && pin kube-system deployment/metrics-server + +for ns in monitoring kyverno kube-startup-cpu-boost-system; do + kubectl get namespace "${ns}" >/dev/null 2>&1 || continue + for w in $(kubectl -n "${ns}" get deployment,statefulset -o name); do + kubectl -n "${ns}" rollout status "${w}" --timeout=600s >/dev/null || { log_error "${ns}/${w} not ready on ${POOL}"; RC=1; } + done +done + +[ "${RC}" -eq 0 ] && log_success "Platform workloads on NodePool ${POOL}; the workshop node carries the app only" || exit 1 diff --git a/infra/scripts/templates/java-performance.sh b/infra/scripts/templates/java-performance.sh index a19d335f..1ad0f929 100755 --- a/infra/scripts/templates/java-performance.sh +++ b/infra/scripts/templates/java-performance.sh @@ -15,6 +15,7 @@ # --- wait for A --- # D app app-deploy (namespace, Pod Identity, manifests) } in # E platform perf-profiler install, startup-cpu-boost, perf-sensor } parallel +# placement platform Deployments/StatefulSets → general-purpose NodePool # F verify ALB answers, sensor measures, Claude wiring present # # Each stage script is idempotent and runnable on its own from the IDE. @@ -99,6 +100,9 @@ cat "$D_LOG"; cat "$E_LOG" [ "$D_RC" -eq 0 ] && log_success "$(stamp)Stage D done (app deployed)" || { log_error "$(stamp)Stage D FAILED"; exit 1; } [ "$E_RC" -eq 0 ] && log_success "$(stamp)Stage E done (profiler, boost, sensor)" || { log_error "$(stamp)Stage E FAILED"; exit 1; } +# --- platform placement: platform pods off the app's node ------------------------------ +run "Platform on the general-purpose NodePool" "$DEPLOY/platform-placement.sh" + # --- F: verify -------------------------------------------------------------------------- stage "F: verify" run "Environment verified" "$DEPLOY/verify.sh" From afb2975f248f193cccc822567afc8ecbfa0fec82 Mon Sep 17 00:00:00 2001 From: Yuriy Bezsonov Date: Mon, 12 Oct 2026 10:50:05 +0200 Subject: [PATCH 2/5] fix(monitoring): Pyroscope keeps its metastore on a PVC Pyroscope 2 writes blocks to S3 but keeps the metastore (the index of those blocks) on the data volume. On an emptyDir a rescheduled pod starts with an empty index and every earlier profile, still in S3, becomes unqueryable. A 10Gi PVC (default gp3 class), like Prometheus and Grafana. Shared by all workshops that install monitoring.sh. --- infra/scripts/setup/monitoring.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/infra/scripts/setup/monitoring.sh b/infra/scripts/setup/monitoring.sh index 4ed1412d..7c1ead67 100755 --- a/infra/scripts/setup/monitoring.sh +++ b/infra/scripts/setup/monitoring.sh @@ -174,8 +174,13 @@ pyroscope: prometheus.io/scrape: "true" prometheus.io/port: "4040" prometheus.io/path: /metrics + # The metastore (the index of the blocks in S3) lives on the data volume: on an emptyDir + # a rescheduled pod starts with an empty index and every earlier profile, still in S3, + # becomes unqueryable. A PVC (the default gp3 class from eks.sh) keeps it across moves, + # like Prometheus and Grafana. persistence: - enabled: false + enabled: true + size: 10Gi resources: requests: cpu: 200m From 1f61d11a3f96e0cb514f0fd4ecf68f192a22fd2c Mon Sep 17 00:00:00 2001 From: Yuriy Bezsonov Date: Mon, 12 Oct 2026 10:50:05 +0200 Subject: [PATCH 3/5] fix(java-performance): No warning badge on the Claude terminal tab The Claude Code VS Code extension asks to relaunch the first terminal Claude runs in and marks its tab. ide-layout.sh turns the environment-change indicator off; capture/ide.mjs runs Claude once in a throwaway terminal and takes the tools shot in a second one. --- .../deploy/java-performance/ide-layout.sh | 8 +++++-- infra/scripts/ws-test/capture/ide.mjs | 23 +++++++++++++++---- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/infra/scripts/deploy/java-performance/ide-layout.sh b/infra/scripts/deploy/java-performance/ide-layout.sh index 403d52ec..95272273 100755 --- a/infra/scripts/deploy/java-performance/ide-layout.sh +++ b/infra/scripts/deploy/java-performance/ide-layout.sh @@ -7,7 +7,10 @@ # # Writes ~/.code-editor-server/data/User/tasks.json (task "Shell", runOn folderOpen) and # sets "task.allowAutomaticTasks": "on" in the user settings (without it Code Editor asks -# before running the task). Runs as ec2-user after ide/code-editor.sh. Idempotent. +# before running the task) and "terminal.integrated.environmentChangesIndicator": "off" (the +# Claude Code extension asks to relaunch every terminal Claude runs in, which otherwise marks +# the tab with a warning badge for the rest of the session). Runs as ec2-user after +# ide/code-editor.sh. Idempotent. # ============================================================================= set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -18,7 +21,8 @@ SETTINGS="${USER_DIR}/settings.json" [ -f "${SETTINGS}" ] || { log_error "no Code Editor user settings at ${SETTINGS} (ide/code-editor.sh)"; exit 1; } tmp="$(mktemp)" -jq '. + {"workbench.startupEditor": "terminal", "task.allowAutomaticTasks": "on"}' "${SETTINGS}" > "${tmp}" +jq '. + {"workbench.startupEditor": "terminal", "task.allowAutomaticTasks": "on", + "terminal.integrated.environmentChangesIndicator": "off"}' "${SETTINGS}" > "${tmp}" mv "${tmp}" "${SETTINGS}" cat > "${USER_DIR}/tasks.json" <<'JSON' diff --git a/infra/scripts/ws-test/capture/ide.mjs b/infra/scripts/ws-test/capture/ide.mjs index 59e84208..13a6c720 100644 --- a/infra/scripts/ws-test/capture/ide.mjs +++ b/infra/scripts/ws-test/capture/ide.mjs @@ -1,10 +1,9 @@ // ide.mjs tools|scm OUT_DIR — Code Editor screenshots (run through ide.sh, which prepares // the IDE and sets IDE_URL and WS_ACCOUNT). // -// tools the IDE as it opens (deploy/java-performance/ide-layout.sh): Claude Code started -// in the editor-area terminal lists the MCP tools of perf-sensor and eks-mcp, the -// Shell terminal in the panel -> OUT_DIR/tools.png. Starts an interactive Claude: -// never while a ws-test runs. +// tools Claude Code in a fresh editor-area terminal lists the MCP tools of perf-sensor and +// eks-mcp, the Shell terminal in the panel -> OUT_DIR/tools.png. Starts interactive +// Claude sessions (two, see below): never while a ws-test runs. // open the IDE as it opens: Explorer, the editor-area terminal, the Shell terminal // -> OUT_DIR/open.png // scm the Source Control view with SCM_FILE's diff open (default deployment.yaml) @@ -18,8 +17,24 @@ const { browser, page } = await open(process.env.IDE_URL); if (what === 'open') { await page.waitForTimeout(6000); } else if (what === 'tools') { + // The first terminal that runs Claude Code activates its VS Code extension, which then + // asks to relaunch that terminal and marks its tab with a warning badge for the rest of + // the session. So: run Claude once in a throwaway terminal, close it, and take the shot in + // a second one, where the extension's variables are set at creation and no badge appears. + const newTerminal = () => + page.locator('.editor-group-container .editor-actions .codicon-plus').first().click(); await page.locator('.editor-group-container .tab', { hasText: 'zsh' }).first().click(); await page.waitForTimeout(800); + await newTerminal(); + await page.waitForTimeout(4000); + await typeInTerminal(page, 'claude'); + await page.waitForTimeout(18000); + await typeInTerminal(page, '/exit'); + await page.waitForTimeout(4000); + await typeInTerminal(page, 'exit'); // closes the throwaway terminal and its tab + await page.waitForTimeout(3000); + await newTerminal(); + await page.waitForTimeout(4000); await typeInTerminal(page, 'claude'); await page.waitForTimeout(15000); await typeInTerminal(page, 'List the MCP tools you can see from perf-sensor and eks-mcp, names only.'); From 6e1137b59ba5dd95927271340086f97912a756d2 Mon Sep 17 00:00:00 2001 From: Yuriy Bezsonov Date: Mon, 12 Oct 2026 10:50:05 +0200 Subject: [PATCH 4/5] fix(perf-sensor): The policy answer takes the section 5 format A run answered the enforce question with a stub (items covered, no Root cause, Evidence or Solution). The enforce entry now names the format and its contents, and section 5 says every answer, the policy one included, starts with Root cause. --- .../perf-sensor/skills/java-on-eks-optimization/SKILL.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/apps/perf-sensor/skills/java-on-eks-optimization/SKILL.md b/apps/perf-sensor/skills/java-on-eks-optimization/SKILL.md index 89d06abc..afdf370a 100644 --- a/apps/perf-sensor/skills/java-on-eks-optimization/SKILL.md +++ b/apps/perf-sensor/skills/java-on-eks-optimization/SKILL.md @@ -204,13 +204,16 @@ Read the file and pass the values through. The *why* for each: → `measure` (the items must pass on the current Deployment, or the policy rejects its next update: say which fail and fix them in the same change). Artifact: `k8s/policy.yaml` from `kyverno-policy.yaml`, `` filled, expressions verbatim. It covers items 1, 3, 9 - and 10; name the others as runtime items the sensor keeps scoring. Close with the step-5 - line. Reference: `kyverno-policy.md`. + and 10; name the others as runtime items the sensor keeps scoring. The answer takes the + section 5 format like every other: `Root cause` is that a regression reaches the cluster + before the next measurement, `Evidence` is `measure`'s values for items 1, 3, 9 and 10 on + the current Deployment. Reference: `kyverno-policy.md`. - **"how are we doing / score"** → run the `java-on-eks-checklist` skill. ## 5. Answer format -The answer starts with the first `Root cause` line: no preamble about the work you did. +Every answer, the policy one included, starts with the first `Root cause` line: no preamble +about the work you did, and no answer that only names the items it covered. ``` Root cause: From 86463a5b64d5829563b6cecc750c8f7c380f31af Mon Sep 17 00:00:00 2001 From: Yuriy Bezsonov Date: Mon, 12 Oct 2026 10:50:06 +0200 Subject: [PATCH 5/5] test(java-performance): Regenerate ws-test for the AOT step, startup lines and profile links Sample mapping for the image page (AOT question and build, CRaC question and build, profile link), README expected flips with the AOT step. --- infra/scripts/ws-test/README.md | 3 +- infra/scripts/ws-test/java-performance.sh | 130 +++++++++++++----- .../ws-test/samples/java-performance.py | 27 +++- 3 files changed, 118 insertions(+), 42 deletions(-) diff --git a/infra/scripts/ws-test/README.md b/infra/scripts/ws-test/README.md index 2e08dc8b..69c7a041 100644 --- a/infra/scripts/ws-test/README.md +++ b/infra/scripts/ws-test/README.md @@ -120,7 +120,8 @@ Numbers vary with the measured working set (memory `max(1.4 × peak, 1.5 × floo |---|---|---|---|---| | Measure the baseline | `checklist` | — | 5/12 | ❌ 2 3 4 6 8 10 11 (6: a 1-core request is several times the steady CPU) | | Improve the manifest | `sizeMemory`, `sizeCpu`, `startupLog` | `k8s/deployment.yaml`, `k8s/startup-cpu-boost.yaml` | 9/12 | 2 3 4 6 → ✅; 8 halves (≈ 15 s → ≈ 8 s); `jfr.container` effectiveCpuCount 1 with `cpuQuotaCores` 2.0; `cpu.max` 1 core after the resize | -| Improve the image | `startupLog`, `measure` | `pom.xml`, one Java file, `Dockerfile.crac`, `k8s/deployment.yaml` | 10/12 | 8 → ✅ (Restored, < 0.1 s); 3 4 5 stay ✅ from the checkpoint; 2 may sit near its bar (the restored pod's working set is lower under a limit sized for the cold one) | +| Improve the image, AOT | `startupLog`, `profileTop` | `Dockerfile.aot` | (no checklist) | startup line *Started* in about a third of the boosted cold start | +| Improve the image, CRaC | `startupLog`, `measure` | `pom.xml`, one Java file, `Dockerfile.crac`, `k8s/deployment.yaml` | 10/12 | 8 → ✅ (Restored, < 0.1 s); 3 4 5 stay ✅ from the checkpoint; 2 may sit near its bar (the restored pod's working set is lower under a limit sized for the cold one) | | Improve the code | `diagnoseBlocking`, `sizeMemory`, `sizeCpu` | `UnicornService.java`, `k8s/deployment.yaml` (sizes, `terminationGracePeriodSeconds`), `Dockerfile.crac` | 12/12 | 10 11 → ✅ (blocked 0, in-transaction 0; pool waits shown, not scored); memory steps down, new heap baked into the checkpoint, rebuild stated | | Enforce the practices | `measure` | `k8s/policy.yaml` | 12/12 | none; the policy matches `references/kyverno-policy.yaml`; the PolicyReport shows the running Deployment passing; an apply with `-Xmx` is denied with the item 3 message | diff --git a/infra/scripts/ws-test/java-performance.sh b/infra/scripts/ws-test/java-performance.sh index 4235e7e3..09f06d12 100755 --- a/infra/scripts/ws-test/java-performance.sh +++ b/infra/scripts/ws-test/java-performance.sh @@ -6,7 +6,7 @@ set -Eeuo pipefail WS_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${WS_SCRIPT_DIR}/runtime.sh" ws_begin_run 'Java performance optimization' "${WS_SCRIPT_DIR}/reports/java-performance" 5 "$@" -ws_set_executable_total 16 +ws_set_executable_total 20 ws_set_default_timeout 600 ws_source_environment '/etc/profile.d/workshop.sh' @@ -59,7 +59,7 @@ ws_end_page ws_begin_page 'Measure the baseline' 100 'baseline/index.en.md' -ws_run_block 1 'The injection policy' 'The injection policy' 25 42 'bash' '' <<'WS_TEST_BLOCK_100_1' +ws_run_block 1 'The injection policy' 'The injection policy' 25 48 'bash' '' <<'WS_TEST_BLOCK_100_1' # when the policy applies: a CEL condition on the pod's labels kubectl get mutatingpolicy inject-perf-profiler -o yaml | yq '.spec.matchConditions[0].expression' @@ -72,27 +72,43 @@ yq -i '.spec.template.metadata.labels."perf-profile/sidecar" = "true"' ~/environ kubectl -n unicorn-store-spring apply -f ~/environment/unicorn-store-spring/k8s/deployment.yaml kubectl -n unicorn-store-spring rollout status deploy/unicorn-store-spring --timeout=300s +# startup line of the new pod +kubectl -n unicorn-store-spring logs -c unicorn-store-spring $(kubectl -n unicorn-store-spring get pod \ + -l app=unicorn-store-spring --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}') \ + | grep -E "Started StoreApplication|restored JVM" + # the manifest lists one container, the running pod has two: admission added the sidecar echo "manifest: $(yq '.spec.template.spec.containers[].name' ~/environment/unicorn-store-spring/k8s/deployment.yaml | xargs)" -echo "pod: $(kubectl -n unicorn-store-spring get pod -l app=unicorn-store-spring -o jsonpath='{.items[0].spec.containers[*].name}')" +echo "pod: $(kubectl -n unicorn-store-spring get pod -l app=unicorn-store-spring \ + --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].spec.containers[*].name}')" git -C ~/environment/unicorn-store-spring add -A \ && git -C ~/environment/unicorn-store-spring commit -q -m "baseline: profiler attached" WS_TEST_BLOCK_100_1 -ws_skip_block 2 'The injection policy' 'The injection policy' 47 57 'text' '' 'copy action disabled' +ws_skip_block 2 'The injection policy' 'The injection policy' 53 64 'text' '' 'copy action disabled' -ws_run_block 3 'The sensor' 'The sensor' 70 71 'bash' '' <<'WS_TEST_BLOCK_100_3' +ws_run_block 3 'The sensor' 'The sensor' 77 78 'bash' '' <<'WS_TEST_BLOCK_100_3' kubectl -n monitoring get configmap perf-sensor-checklist-rules -o jsonpath='{.data.checklist-rules\.yaml}' \ | yq '.items[] | select(.id == 6)' WS_TEST_BLOCK_100_3 -ws_skip_block 4 'The sensor' 'The sensor' 76 88 'yaml' '' 'copy action disabled' +ws_skip_block 4 'The sensor' 'The sensor' 83 95 'yaml' '' 'copy action disabled' -ws_run_block 5 'Ask for the baseline' 'In the Shell terminal:' 109 109 'bash' 'non-interactive' <<'WS_TEST_BLOCK_100_5' +ws_run_block 5 'Ask for the baseline' 'In the Shell terminal:' 116 116 'bash' 'non-interactive' <<'WS_TEST_BLOCK_100_5' cd ~/environment && claude -c -p "How is unicorn-store-spring doing against best practices?" WS_TEST_BLOCK_100_5 +ws_run_block 6 'Where the start spends its CPU' 'In the Shell terminal, print a link to the CPU profile of the new pod'"'"'s first 90 seconds in Grafana:' 129 135 'bash' '' <<'WS_TEST_BLOCK_100_6' +POD=$(kubectl -n unicorn-store-spring get pod -l app=unicorn-store-spring \ + --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}') +START=$(date -d "$(kubectl -n unicorn-store-spring get pod $POD -o jsonpath='{.status.startTime}')" +%s) +GRAFANA_URL=$(kubectl -n monitoring get svc grafana -o jsonpath="{.status.loadBalancer.ingress[0].hostname}") +echo "http://${GRAFANA_URL}/a/grafana-pyroscope-app/explore?explorationType=flame-graph&var-dataSource=pyroscope\ +&var-serviceName=unicorn-store-spring&var-profileMetricId=process_cpu:cpu:nanoseconds:cpu:nanoseconds\ +&var-filters=pod%7C%3D%7C${POD}&from=${START}000&to=$((START + 90))000" +WS_TEST_BLOCK_100_6 + ws_end_page ws_begin_page 'Improve the manifest' 200 'improve-manifest/index.en.md' @@ -101,11 +117,11 @@ ws_run_block 1 'Improve the manifest' 'In the Shell terminal:' 23 23 'bash' 'non cd ~/environment && claude -c -p "Right-size the memory and CPU of unicorn-store-spring and make it start faster without changing the image." WS_TEST_BLOCK_200_1 -ws_skip_block 2 'Review the change' 'Review the change' 49 76 'text' '' 'copy action disabled' +ws_skip_block 2 'Review the change' 'Review the change' 49 74 'text' '' 'copy action disabled' -ws_skip_block 3 'Review the change' '- requests.cpu and ActiveProcessorCount=1 come from sizeCpu. limits.cpu stays at 1' 94 115 'yaml' '' 'copy action disabled' +ws_skip_block 3 'Review the change' '- resizePolicy with restartPolicy: NotRequired for CPU states that a CPU change needs no restart. It is the default on current Kubernetes, and the prerequisite for the resize the controller performs' 93 114 'yaml' '' 'copy action disabled' -ws_run_block 4 'Roll out, verify and score' 'In the Shell terminal:' 127 148 'bash' '' 900 <<'WS_TEST_BLOCK_200_4' +ws_run_block 4 'Roll out, verify and score' 'In the Shell terminal:' 126 149 'bash' '' 900 <<'WS_TEST_BLOCK_200_4' kubectl -n unicorn-store-spring apply -f ~/environment/unicorn-store-spring/k8s/startup-cpu-boost.yaml kubectl -n unicorn-store-spring apply -f ~/environment/unicorn-store-spring/k8s/deployment.yaml kubectl -n unicorn-store-spring rollout status deploy/unicorn-store-spring --timeout=300s @@ -115,8 +131,10 @@ kubectl -n unicorn-store-spring exec deploy/unicorn-store-spring -c unicorn-stor sh -c 'unset JAVA_TOOL_OPTIONS; jcmd $(jcmd -l | grep -v JCmd | cut -d" " -f1) VM.flags' \ | tr ' ' '\n' | grep -E 'MaxHeapSize|InitialHeapSize|UseSerialGC|ActiveProcessorCount' -# startup with the boost -curl -s localhost:8090/api/v1/startupLog/unicorn-store-spring | jq -r .line +# startup line of the new pod, with the boost +kubectl -n unicorn-store-spring logs -c unicorn-store-spring $(kubectl -n unicorn-store-spring get pod \ + -l app=unicorn-store-spring --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}') \ + | grep -E "Started StoreApplication|restored JVM" # what the JVM saw at start (the boosted quota) and what the cgroup allows now sleep 30 @@ -130,25 +148,52 @@ git -C ~/environment/unicorn-store-spring add -A \ curl -sN "localhost:8090/api/v1/checklist/unicorn-store-spring?format=text&wait=true" WS_TEST_BLOCK_200_4 -ws_skip_block 5 'Roll out, verify and score' 'Roll out, verify and score' 155 165 'text' '' 'copy action disabled' +ws_skip_block 5 'Roll out, verify and score' 'Roll out, verify and score' 156 166 'text' '' 'copy action disabled' -ws_skip_block 6 'The score' 'The score' 192 210 'text' '' 'copy action disabled' +ws_skip_block 6 'The score' 'The score' 193 211 'text' '' 'copy action disabled' ws_end_page ws_begin_page 'Improve the image' 300 'improve-image/index.en.md' -ws_run_block 1 'Improve the image' 'In the Shell terminal:' 23 23 'bash' 'non-interactive' <<'WS_TEST_BLOCK_300_1' -cd ~/environment && claude -c -p "How can unicorn-store-spring start in under a second?" +ws_run_block 1 'Without changing the code' 'In the Shell terminal:' 25 25 'bash' 'non-interactive' <<'WS_TEST_BLOCK_300_1' +cd ~/environment && claude -c -p "Make unicorn-store-spring start faster without changing its code." WS_TEST_BLOCK_300_1 -ws_skip_block 2 'Review the change' 'Review the change' 42 75 'text' '' 'copy action disabled' +ws_skip_block 2 'Review the change' 'Review the change' 43 57 'text' '' 'copy action disabled' + +ws_run_block 3 'Build, roll out and check the startup' 'In the Shell terminal:' 68 83 'bash' '' 1500 <<'WS_TEST_BLOCK_300_3' +# ≈ 2 min: builds the jar, trains and creates the cache; pushes :aot +~/environment/unicorn-store-spring/scripts/build.sh aot Dockerfile.aot + +# point the Deployment at the :aot tag and roll it out +yq -i '.spec.template.spec.containers[0].image |= sub(":[^:]+$"; ":aot")' \ + ~/environment/unicorn-store-spring/k8s/deployment.yaml +kubectl -n unicorn-store-spring apply -f ~/environment/unicorn-store-spring/k8s/deployment.yaml +kubectl -n unicorn-store-spring rollout status deploy/unicorn-store-spring --timeout=300s -ws_skip_block 3 'Review the change' '- k8s/deployment.yaml: JAVATOOLOPTIONS removed. A restored JVM keeps the flags of the checkpoint, and a GC flag in the environment stops the restore' 88 131 'dockerfile' '' 'copy action disabled' +# startup line of the new pod +kubectl -n unicorn-store-spring logs -c unicorn-store-spring $(kubectl -n unicorn-store-spring get pod \ + -l app=unicorn-store-spring --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}') \ + | grep -E "Started StoreApplication|restored JVM" + +git -C ~/environment/unicorn-store-spring add -A \ + && git -C ~/environment/unicorn-store-spring commit -q -m "aot cache" +WS_TEST_BLOCK_300_3 -ws_skip_block 4 'Review the change' '- Line 44: -XX:CRaCRestoreFrom restores instead of starting. There are no GC or heap flags: they came with the checkpoint' 144 174 'java' '' 'copy action disabled' +ws_skip_block 4 'Build, roll out and check the startup' 'Build, roll out and check the startup' 88 92 'text' '' 'copy action disabled' -ws_run_block 5 'Build, roll out, verify and score' 'In the Shell terminal. The build starts the application inside Docker, warms it up, checkpoints, and pushes the :crac tag:' 187 207 'bash' '' 1500 <<'WS_TEST_BLOCK_300_5' +ws_run_block 5 'With a code change' 'In the Shell terminal:' 113 113 'bash' 'non-interactive' <<'WS_TEST_BLOCK_300_5' +cd ~/environment && claude -c -p "Can unicorn-store-spring start even faster if its code may change?" +WS_TEST_BLOCK_300_5 + +ws_skip_block 6 'Review the change' 'Review the change' 132 168 'text' '' 'copy action disabled' + +ws_skip_block 7 'Review the change' '- k8s/deployment.yaml: JAVATOOLOPTIONS removed. A restored JVM keeps the flags of the checkpoint, and a GC flag in the environment stops the restore' 181 224 'dockerfile' '' 'copy action disabled' + +ws_skip_block 8 'Review the change' '- Line 44: -XX:CRaCRestoreFrom restores instead of starting. There are no GC or heap flags: they came with the checkpoint' 237 267 'java' '' 'copy action disabled' + +ws_run_block 9 'Build, roll out, verify and score' 'In the Shell terminal:' 280 302 'bash' '' 1500 <<'WS_TEST_BLOCK_300_9' # ≈ 3 min: builds, starts, warms up, checkpoints; pushes :crac ~/environment/unicorn-store-spring/scripts/build.sh crac Dockerfile.crac @@ -158,8 +203,10 @@ yq -i '.spec.template.spec.containers[0].image |= sub(":[^:]+$"; ":crac")' \ kubectl -n unicorn-store-spring apply -f ~/environment/unicorn-store-spring/k8s/deployment.yaml kubectl -n unicorn-store-spring rollout status deploy/unicorn-store-spring --timeout=300s -# startup line: "restored", well under a second -curl -s localhost:8090/api/v1/startupLog/unicorn-store-spring | jq -r .line +# startup line of the new pod +kubectl -n unicorn-store-spring logs -c unicorn-store-spring $(kubectl -n unicorn-store-spring get pod \ + -l app=unicorn-store-spring --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}') \ + | grep -E "Started StoreApplication|restored JVM" # heap flags came from the checkpoint, not from the Deployment kubectl -n unicorn-store-spring exec deploy/unicorn-store-spring -c unicorn-store-spring -- \ @@ -170,11 +217,21 @@ git -C ~/environment/unicorn-store-spring add -A \ && git -C ~/environment/unicorn-store-spring commit -q -m "crac" curl -sN "localhost:8090/api/v1/checklist/unicorn-store-spring?format=text&wait=true" -WS_TEST_BLOCK_300_5 +WS_TEST_BLOCK_300_9 + +ws_run_block 10 'Where the restored start spends its CPU' 'Print the same link for the restored pod:' 324 330 'bash' '' <<'WS_TEST_BLOCK_300_10' +POD=$(kubectl -n unicorn-store-spring get pod -l app=unicorn-store-spring \ + --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}') +START=$(date -d "$(kubectl -n unicorn-store-spring get pod $POD -o jsonpath='{.status.startTime}')" +%s) +GRAFANA_URL=$(kubectl -n monitoring get svc grafana -o jsonpath="{.status.loadBalancer.ingress[0].hostname}") +echo "http://${GRAFANA_URL}/a/grafana-pyroscope-app/explore?explorationType=flame-graph&var-dataSource=pyroscope\ +&var-serviceName=unicorn-store-spring&var-profileMetricId=process_cpu:cpu:nanoseconds:cpu:nanoseconds\ +&var-filters=pod%7C%3D%7C${POD}&from=${START}000&to=$((START + 90))000" +WS_TEST_BLOCK_300_10 -ws_skip_block 6 'Where the restored start spends its CPU' 'Where the restored start spends its CPU' 232 241 'text' '' 'copy action disabled' +ws_skip_block 11 'Where the restored start spends its CPU' 'Search for ClassLoader: 13% of the baseline pod'"'"'s window in this run, 1% after the restore. Search for CompileBroker: the JIT compiler threads still take about half of the window.' 339 348 'text' '' 'copy action disabled' -ws_skip_block 7 'The score' 'The score' 257 275 'text' '' 'copy action disabled' +ws_skip_block 12 'The score' 'The score' 364 382 'text' '' 'copy action disabled' ws_end_page @@ -184,15 +241,20 @@ ws_run_block 1 'Improve the code' 'In the Shell terminal:' 23 23 'bash' 'non-int cd ~/environment && claude -c -p "Why do some writes to unicorn-store-spring take seconds under load? Fix it, size the service again and fix item 10." WS_TEST_BLOCK_400_1 -ws_skip_block 2 'Review the change' 'Review the change' 41 75 'text' '' 'copy action disabled' +ws_skip_block 2 'Review the change' 'Review the change' 41 79 'text' '' 'copy action disabled' -ws_run_block 3 'Build, roll out, verify and score' 'In the Shell terminal. The image tag stays :crac with a new digest, and the Deployment changed, so apply rolls out both:' 96 110 'bash' '' 1500 <<'WS_TEST_BLOCK_400_3' +ws_run_block 3 'Build, roll out, verify and score' 'The image tag stays :crac with a new digest, and the Deployment changed, so apply rolls out both. In the Shell terminal:' 100 119 'bash' '' 1500 <<'WS_TEST_BLOCK_400_3' # code and heap flags changed: rebuild the checkpoint (≈ 2 min) ~/environment/unicorn-store-spring/scripts/build.sh crac Dockerfile.crac kubectl -n unicorn-store-spring apply -f ~/environment/unicorn-store-spring/k8s/deployment.yaml kubectl -n unicorn-store-spring rollout status deploy/unicorn-store-spring --timeout=300s +# startup line of the new pod +kubectl -n unicorn-store-spring logs -c unicorn-store-spring $(kubectl -n unicorn-store-spring get pod \ + -l app=unicorn-store-spring --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}') \ + | grep -E "Started StoreApplication|restored JVM" + # heap flags from the new checkpoint: 75 % / 50 % of the new limit kubectl -n unicorn-store-spring exec deploy/unicorn-store-spring -c unicorn-store-spring -- \ sh -c 'unset JAVA_TOOL_OPTIONS; jcmd $(jcmd -l | grep -v JCmd | cut -d" " -f1) VM.flags' \ @@ -204,9 +266,9 @@ git -C ~/environment/unicorn-store-spring add -A \ curl -sN "localhost:8090/api/v1/checklist/unicorn-store-spring?format=text&wait=true" WS_TEST_BLOCK_400_3 -ws_skip_block 4 'The pool and the database' 'The pool and the database' 129 135 'text' '' 'copy action disabled' +ws_skip_block 4 'The pool and the database' 'The pool and the database' 138 145 'text' '' 'copy action disabled' -ws_skip_block 5 'The score' 'The score' 151 169 'text' '' 'copy action disabled' +ws_skip_block 5 'The score' 'The score' 161 179 'text' '' 'copy action disabled' ws_end_page @@ -216,11 +278,11 @@ ws_run_block 1 'Enforce the practices' 'In the Shell terminal:' 23 23 'bash' 'no cd ~/environment && claude -c -p "Turn the checklist items that can be checked on the manifest into a Kyverno policy for unicorn-store-spring." WS_TEST_BLOCK_500_1 -ws_skip_block 2 'Review the policy' 'Review the policy' 44 65 'text' '' 'copy action disabled' +ws_skip_block 2 'Review the policy' 'Review the policy' 44 66 'text' '' 'copy action disabled' -ws_skip_block 3 'Review the policy' 'Two of the seven validations in k8s/policy.yaml:' 72 84 'yaml' '' 'copy action disabled' +ws_skip_block 3 'Review the policy' 'Two of the seven validations in k8s/policy.yaml:' 73 85 'yaml' '' 'copy action disabled' -ws_run_block 4 'Apply and test' 'In the Shell terminal:' 97 109 'bash' '' <<'WS_TEST_BLOCK_500_4' +ws_run_block 4 'Apply and test' 'In the Shell terminal:' 98 110 'bash' '' <<'WS_TEST_BLOCK_500_4' kubectl apply -f ~/environment/unicorn-store-spring/k8s/policy.yaml # the background scan reports the running Deployment @@ -236,13 +298,13 @@ git -C ~/environment/unicorn-store-spring add -A \ && git -C ~/environment/unicorn-store-spring commit -q -m "kyverno policy" WS_TEST_BLOCK_500_4 -ws_skip_block 5 'Apply and test' 'Apply and test' 114 118 'text' '' 'copy action disabled' +ws_skip_block 5 'Apply and test' 'Apply and test' 115 119 'text' '' 'copy action disabled' ws_end_page ws_begin_page 'Results' 650 'results/index.en.md' -ws_skip_block 1 'Choosing for your own workloads' 'Choosing for your own workloads' 29 48 '' '' 'informational block without language' +ws_skip_block 1 'Choosing for your own workloads' 'Choosing for your own workloads' 30 49 '' '' 'informational block without language' ws_end_page diff --git a/infra/scripts/ws-test/samples/java-performance.py b/infra/scripts/ws-test/samples/java-performance.py index 9e361451..bafa532b 100644 --- a/infra/scripts/ws-test/samples/java-performance.py +++ b/infra/scripts/ws-test/samples/java-performance.py @@ -2,11 +2,14 @@ setup: expand 0 = the environment check. baseline: expand 0 = the policy-view + attach block (rollout wait lines dropped), expand 2 = - the checklist answer as raw markdown. -improve-manifest, improve-image, improve-code: two blocks each, the Claude question and - the combined roll-out block. Expand 0 = Claude's answer wrapped at 110 columns, - expand 1 = the roll-out block up to "Pod is" (image and code: only the build, rollout, - startup and heap-flag lines), expand 2 = the checklist from "Score:". + the checklist answer as raw markdown. The profile-link block (block 3) has no expand. +improve-manifest, improve-code: two blocks each, the Claude question and the combined + roll-out block. Expand 0 = Claude's answer wrapped at 110 columns, expand 1 = the roll-out + block up to "Pod is" (code: only the build, rollout, startup and heap-flag lines), expand 2 = + the checklist from "Score:". +improve-image: the AOT question and its build block, the CRaC question and its build block, + then the profile link (no expand). Expands 0–4 = AOT answer, AOT build lines, CRaC answer, + CRaC build lines up to "Pod is", the checklist from "Score:". enforce: expand 0 = Claude's answer, expand 1 = the apply block. A block re-run by hand replaces its expand with that real output; this mapping then @@ -15,6 +18,7 @@ from refresh import drop, keep, since, until, wrap ROLLOUT_WAIT = r'^Waiting for deployment' +BUILD_LINES = ('[build] docker build', '[build] pushed', 'deployment', '20', '-XX') def refresh(s): @@ -24,16 +28,25 @@ def refresh(s): s.set_expand('baseline/index.en.md', 0, drop(b[0], ROLLOUT_WAIT)) s.set_expand('baseline/index.en.md', 2, b[2], raw=True) - for page in ('improve-manifest', 'improve-image', 'improve-code'): + for page in ('improve-manifest', 'improve-code'): md = page + '/index.en.md' question, run = s.blocks(md) s.set_expand(md, 0, wrap(question)) head = drop(until(run, 'Pod is'), ROLLOUT_WAIT) if page != 'improve-manifest': - head = keep(head, '[build] docker build', '[build] pushed', 'deployment', '20', '-XX') + head = keep(head, *BUILD_LINES) s.set_expand(md, 1, head) s.set_expand(md, 2, since(run, 'Score:')) + # improve-image: AOT question + build block, CRaC question + build block, profile link. + md = 'improve-image/index.en.md' + aot_q, aot_run, crac_q, crac_run = s.blocks(md)[:4] + s.set_expand(md, 0, wrap(aot_q)) + s.set_expand(md, 1, keep(drop(aot_run, ROLLOUT_WAIT), *BUILD_LINES)) + s.set_expand(md, 2, wrap(crac_q)) + s.set_expand(md, 3, keep(drop(until(crac_run, 'Pod is'), ROLLOUT_WAIT), *BUILD_LINES)) + s.set_expand(md, 4, since(crac_run, 'Score:')) + question, run = s.blocks('enforce/index.en.md') s.set_expand('enforce/index.en.md', 0, wrap(question)) s.set_expand('enforce/index.en.md', 1, run)