diff --git a/apps/perf-sensor/skills/java-on-eks-optimization/SKILL.md b/apps/perf-sensor/skills/java-on-eks-optimization/SKILL.md index 89d06abc..afdf370a 100644 --- a/apps/perf-sensor/skills/java-on-eks-optimization/SKILL.md +++ b/apps/perf-sensor/skills/java-on-eks-optimization/SKILL.md @@ -204,13 +204,16 @@ Read the file and pass the values through. The *why* for each: → `measure` (the items must pass on the current Deployment, or the policy rejects its next update: say which fail and fix them in the same change). Artifact: `k8s/policy.yaml` from `kyverno-policy.yaml`, `` filled, expressions verbatim. It covers items 1, 3, 9 - and 10; name the others as runtime items the sensor keeps scoring. Close with the step-5 - line. Reference: `kyverno-policy.md`. + and 10; name the others as runtime items the sensor keeps scoring. The answer takes the + section 5 format like every other: `Root cause` is that a regression reaches the cluster + before the next measurement, `Evidence` is `measure`'s values for items 1, 3, 9 and 10 on + the current Deployment. Reference: `kyverno-policy.md`. - **"how are we doing / score"** → run the `java-on-eks-checklist` skill. ## 5. Answer format -The answer starts with the first `Root cause` line: no preamble about the work you did. +Every answer, the policy one included, starts with the first `Root cause` line: no preamble +about the work you did, and no answer that only names the items it covered. ``` Root cause: diff --git a/infra/scripts/deploy/java-performance/ide-layout.sh b/infra/scripts/deploy/java-performance/ide-layout.sh index 403d52ec..95272273 100755 --- a/infra/scripts/deploy/java-performance/ide-layout.sh +++ b/infra/scripts/deploy/java-performance/ide-layout.sh @@ -7,7 +7,10 @@ # # Writes ~/.code-editor-server/data/User/tasks.json (task "Shell", runOn folderOpen) and # sets "task.allowAutomaticTasks": "on" in the user settings (without it Code Editor asks -# before running the task). Runs as ec2-user after ide/code-editor.sh. Idempotent. +# before running the task) and "terminal.integrated.environmentChangesIndicator": "off" (the +# Claude Code extension asks to relaunch every terminal Claude runs in, which otherwise marks +# the tab with a warning badge for the rest of the session). Runs as ec2-user after +# ide/code-editor.sh. Idempotent. # ============================================================================= set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -18,7 +21,8 @@ SETTINGS="${USER_DIR}/settings.json" [ -f "${SETTINGS}" ] || { log_error "no Code Editor user settings at ${SETTINGS} (ide/code-editor.sh)"; exit 1; } tmp="$(mktemp)" -jq '. + {"workbench.startupEditor": "terminal", "task.allowAutomaticTasks": "on"}' "${SETTINGS}" > "${tmp}" +jq '. + {"workbench.startupEditor": "terminal", "task.allowAutomaticTasks": "on", + "terminal.integrated.environmentChangesIndicator": "off"}' "${SETTINGS}" > "${tmp}" mv "${tmp}" "${SETTINGS}" cat > "${USER_DIR}/tasks.json" <<'JSON' diff --git a/infra/scripts/deploy/java-performance/nodepool-pin.sh b/infra/scripts/deploy/java-performance/nodepool-pin.sh index 5afa6b87..f4769746 100755 --- a/infra/scripts/deploy/java-performance/nodepool-pin.sh +++ b/infra/scripts/deploy/java-performance/nodepool-pin.sh @@ -9,6 +9,11 @@ # (e.g. m8azn at 5 GHz) and startup "improves" from hardware, not from the change. # Family, not type, keeps capacity flexible across sizes. # +# Consolidation: WhenEmpty only. The pool carries just the app (platform-placement.sh), and +# one m6a.xlarge fits every state of the session, rollout surge and boosted start included. +# WhenEmptyOrUnderutilized (the eks.sh default) would replace the node when the app shrinks +# and move the pod mid-measurement; WhenEmpty only removes a node nothing runs on. +# # Replaces the three scheduling requirements (cpu-manufacturer, instance-category, # instance-generation) with one (instance-family); capacity-type, arch, os, cpu and # memory requirements stay. Idempotent. Runs after setup/eks.sh. @@ -38,7 +43,8 @@ PATCH="$(kubectl get nodepool "${NODEPOOL}" -o json | jq -c --arg fam "${INSTANC "eks.amazonaws.com/instance-generation", "eks.amazonaws.com/instance-family") | not)) + [{"key":"eks.amazonaws.com/instance-family","operator":"In","values":[$fam]}] - | {spec:{template:{spec:{requirements:.}}}}')" + | {spec:{template:{spec:{requirements:.}}, + disruption:{consolidationPolicy:"WhenEmpty",consolidateAfter:"5m"}}}')" kubectl patch nodepool "${NODEPOOL}" --type merge -p "${PATCH}" >/dev/null \ - && log_success "NodePool ${NODEPOOL} pinned to ${INSTANCE_FAMILY} (existing nodes drain on consolidation; new nodes are ${INSTANCE_FAMILY})" \ + && log_success "NodePool ${NODEPOOL} pinned to ${INSTANCE_FAMILY}, consolidation WhenEmpty (existing nodes drain on consolidation; new nodes are ${INSTANCE_FAMILY})" \ || { log_error "NodePool patch failed"; exit 1; } diff --git a/infra/scripts/deploy/java-performance/platform-placement.sh b/infra/scripts/deploy/java-performance/platform-placement.sh new file mode 100755 index 00000000..ab07e4f5 --- /dev/null +++ b/infra/scripts/deploy/java-performance/platform-placement.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# ============================================================================= +# platform-placement.sh — keep the platform off the application's node (java-performance only). +# +# The app selects the `workshop` NodePool (karpenter.sh/nodepool: workshop). The platform +# workloads have no selector, so Karpenter places them wherever there is room, usually on +# the app's 4-vCPU node. There the app shares physical cores (SMT) with Prometheus, +# Pyroscope and Kyverno, and the session's measurements move with that load (startup and +# steady CPU 10–60 % higher in a test run). This script pins every platform Deployment and +# StatefulSet to the Auto Mode built-in `general-purpose` NodePool, so the `workshop` node +# carries only the app and the DaemonSets. The app manifest is unchanged. +# +# Namespaces: monitoring (Prometheus, Grafana, Pyroscope, Alloy, sensor), kyverno, +# kube-startup-cpu-boost-system, and metrics-server in kube-system. Pods move once, at +# bootstrap; Prometheus, Grafana and Pyroscope keep their data on their PVCs. The pods are +# marked do-not-disrupt, so consolidation does not move them later. +# Idempotent. Run after the platform installs (template stage E). A later `helm upgrade` +# of one of these releases drops the selector: run this script again afterwards. +# ============================================================================= +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/../../lib/common.sh" + +POOL="${PLATFORM_NODEPOOL:-general-purpose}" +kubectl get nodepool "${POOL}" >/dev/null 2>&1 || { log_error "NodePool ${POOL} not found (EKS Auto Mode built-in pools disabled?)"; exit 1; } + +# do-not-disrupt: the general-purpose pool consolidates underutilized nodes after 30 s. A +# consolidation during the session would move the sensor or Kyverno in the middle of a +# checklist or an admission; the annotation keeps their nodes until the pods go. +PATCH="{\"spec\":{\"template\":{\"metadata\":{\"annotations\":{\"karpenter.sh/do-not-disrupt\":\"true\"}},\"spec\":{\"nodeSelector\":{\"karpenter.sh/nodepool\":\"${POOL}\"}}}}}" +RC=0 +pin() { # pin NAMESPACE KIND/NAME... + local ns="$1"; shift + for w in "$@"; do + if kubectl -n "${ns}" patch "${w}" --type merge -p "${PATCH}" >/dev/null; then + log_info "${ns}/${w} → ${POOL}" + else + log_error "patch failed: ${ns}/${w}"; RC=1 + fi + done +} + +for ns in monitoring kyverno kube-startup-cpu-boost-system; do + kubectl get namespace "${ns}" >/dev/null 2>&1 || { log_warning "namespace ${ns} not found, skipped"; continue; } + mapfile -t W < <(kubectl -n "${ns}" get deployment,statefulset -o name) + [ "${#W[@]}" -gt 0 ] && pin "${ns}" "${W[@]}" +done +kubectl -n kube-system get deployment metrics-server >/dev/null 2>&1 && pin kube-system deployment/metrics-server + +for ns in monitoring kyverno kube-startup-cpu-boost-system; do + kubectl get namespace "${ns}" >/dev/null 2>&1 || continue + for w in $(kubectl -n "${ns}" get deployment,statefulset -o name); do + kubectl -n "${ns}" rollout status "${w}" --timeout=600s >/dev/null || { log_error "${ns}/${w} not ready on ${POOL}"; RC=1; } + done +done + +[ "${RC}" -eq 0 ] && log_success "Platform workloads on NodePool ${POOL}; the workshop node carries the app only" || exit 1 diff --git a/infra/scripts/setup/monitoring.sh b/infra/scripts/setup/monitoring.sh index 4ed1412d..7c1ead67 100755 --- a/infra/scripts/setup/monitoring.sh +++ b/infra/scripts/setup/monitoring.sh @@ -174,8 +174,13 @@ pyroscope: prometheus.io/scrape: "true" prometheus.io/port: "4040" prometheus.io/path: /metrics + # The metastore (the index of the blocks in S3) lives on the data volume: on an emptyDir + # a rescheduled pod starts with an empty index and every earlier profile, still in S3, + # becomes unqueryable. A PVC (the default gp3 class from eks.sh) keeps it across moves, + # like Prometheus and Grafana. persistence: - enabled: false + enabled: true + size: 10Gi resources: requests: cpu: 200m diff --git a/infra/scripts/templates/java-performance.sh b/infra/scripts/templates/java-performance.sh index a19d335f..1ad0f929 100755 --- a/infra/scripts/templates/java-performance.sh +++ b/infra/scripts/templates/java-performance.sh @@ -15,6 +15,7 @@ # --- wait for A --- # D app app-deploy (namespace, Pod Identity, manifests) } in # E platform perf-profiler install, startup-cpu-boost, perf-sensor } parallel +# placement platform Deployments/StatefulSets → general-purpose NodePool # F verify ALB answers, sensor measures, Claude wiring present # # Each stage script is idempotent and runnable on its own from the IDE. @@ -99,6 +100,9 @@ cat "$D_LOG"; cat "$E_LOG" [ "$D_RC" -eq 0 ] && log_success "$(stamp)Stage D done (app deployed)" || { log_error "$(stamp)Stage D FAILED"; exit 1; } [ "$E_RC" -eq 0 ] && log_success "$(stamp)Stage E done (profiler, boost, sensor)" || { log_error "$(stamp)Stage E FAILED"; exit 1; } +# --- platform placement: platform pods off the app's node ------------------------------ +run "Platform on the general-purpose NodePool" "$DEPLOY/platform-placement.sh" + # --- F: verify -------------------------------------------------------------------------- stage "F: verify" run "Environment verified" "$DEPLOY/verify.sh" diff --git a/infra/scripts/ws-test/README.md b/infra/scripts/ws-test/README.md index 2e08dc8b..69c7a041 100644 --- a/infra/scripts/ws-test/README.md +++ b/infra/scripts/ws-test/README.md @@ -120,7 +120,8 @@ Numbers vary with the measured working set (memory `max(1.4 × peak, 1.5 × floo |---|---|---|---|---| | Measure the baseline | `checklist` | — | 5/12 | ❌ 2 3 4 6 8 10 11 (6: a 1-core request is several times the steady CPU) | | Improve the manifest | `sizeMemory`, `sizeCpu`, `startupLog` | `k8s/deployment.yaml`, `k8s/startup-cpu-boost.yaml` | 9/12 | 2 3 4 6 → ✅; 8 halves (≈ 15 s → ≈ 8 s); `jfr.container` effectiveCpuCount 1 with `cpuQuotaCores` 2.0; `cpu.max` 1 core after the resize | -| Improve the image | `startupLog`, `measure` | `pom.xml`, one Java file, `Dockerfile.crac`, `k8s/deployment.yaml` | 10/12 | 8 → ✅ (Restored, < 0.1 s); 3 4 5 stay ✅ from the checkpoint; 2 may sit near its bar (the restored pod's working set is lower under a limit sized for the cold one) | +| Improve the image, AOT | `startupLog`, `profileTop` | `Dockerfile.aot` | (no checklist) | startup line *Started* in about a third of the boosted cold start | +| Improve the image, CRaC | `startupLog`, `measure` | `pom.xml`, one Java file, `Dockerfile.crac`, `k8s/deployment.yaml` | 10/12 | 8 → ✅ (Restored, < 0.1 s); 3 4 5 stay ✅ from the checkpoint; 2 may sit near its bar (the restored pod's working set is lower under a limit sized for the cold one) | | Improve the code | `diagnoseBlocking`, `sizeMemory`, `sizeCpu` | `UnicornService.java`, `k8s/deployment.yaml` (sizes, `terminationGracePeriodSeconds`), `Dockerfile.crac` | 12/12 | 10 11 → ✅ (blocked 0, in-transaction 0; pool waits shown, not scored); memory steps down, new heap baked into the checkpoint, rebuild stated | | Enforce the practices | `measure` | `k8s/policy.yaml` | 12/12 | none; the policy matches `references/kyverno-policy.yaml`; the PolicyReport shows the running Deployment passing; an apply with `-Xmx` is denied with the item 3 message | diff --git a/infra/scripts/ws-test/capture/ide.mjs b/infra/scripts/ws-test/capture/ide.mjs index 59e84208..13a6c720 100644 --- a/infra/scripts/ws-test/capture/ide.mjs +++ b/infra/scripts/ws-test/capture/ide.mjs @@ -1,10 +1,9 @@ // ide.mjs tools|scm OUT_DIR — Code Editor screenshots (run through ide.sh, which prepares // the IDE and sets IDE_URL and WS_ACCOUNT). // -// tools the IDE as it opens (deploy/java-performance/ide-layout.sh): Claude Code started -// in the editor-area terminal lists the MCP tools of perf-sensor and eks-mcp, the -// Shell terminal in the panel -> OUT_DIR/tools.png. Starts an interactive Claude: -// never while a ws-test runs. +// tools Claude Code in a fresh editor-area terminal lists the MCP tools of perf-sensor and +// eks-mcp, the Shell terminal in the panel -> OUT_DIR/tools.png. Starts interactive +// Claude sessions (two, see below): never while a ws-test runs. // open the IDE as it opens: Explorer, the editor-area terminal, the Shell terminal // -> OUT_DIR/open.png // scm the Source Control view with SCM_FILE's diff open (default deployment.yaml) @@ -18,8 +17,24 @@ const { browser, page } = await open(process.env.IDE_URL); if (what === 'open') { await page.waitForTimeout(6000); } else if (what === 'tools') { + // The first terminal that runs Claude Code activates its VS Code extension, which then + // asks to relaunch that terminal and marks its tab with a warning badge for the rest of + // the session. So: run Claude once in a throwaway terminal, close it, and take the shot in + // a second one, where the extension's variables are set at creation and no badge appears. + const newTerminal = () => + page.locator('.editor-group-container .editor-actions .codicon-plus').first().click(); await page.locator('.editor-group-container .tab', { hasText: 'zsh' }).first().click(); await page.waitForTimeout(800); + await newTerminal(); + await page.waitForTimeout(4000); + await typeInTerminal(page, 'claude'); + await page.waitForTimeout(18000); + await typeInTerminal(page, '/exit'); + await page.waitForTimeout(4000); + await typeInTerminal(page, 'exit'); // closes the throwaway terminal and its tab + await page.waitForTimeout(3000); + await newTerminal(); + await page.waitForTimeout(4000); await typeInTerminal(page, 'claude'); await page.waitForTimeout(15000); await typeInTerminal(page, 'List the MCP tools you can see from perf-sensor and eks-mcp, names only.'); diff --git a/infra/scripts/ws-test/java-performance.sh b/infra/scripts/ws-test/java-performance.sh index 4235e7e3..09f06d12 100755 --- a/infra/scripts/ws-test/java-performance.sh +++ b/infra/scripts/ws-test/java-performance.sh @@ -6,7 +6,7 @@ set -Eeuo pipefail WS_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${WS_SCRIPT_DIR}/runtime.sh" ws_begin_run 'Java performance optimization' "${WS_SCRIPT_DIR}/reports/java-performance" 5 "$@" -ws_set_executable_total 16 +ws_set_executable_total 20 ws_set_default_timeout 600 ws_source_environment '/etc/profile.d/workshop.sh' @@ -59,7 +59,7 @@ ws_end_page ws_begin_page 'Measure the baseline' 100 'baseline/index.en.md' -ws_run_block 1 'The injection policy' 'The injection policy' 25 42 'bash' '' <<'WS_TEST_BLOCK_100_1' +ws_run_block 1 'The injection policy' 'The injection policy' 25 48 'bash' '' <<'WS_TEST_BLOCK_100_1' # when the policy applies: a CEL condition on the pod's labels kubectl get mutatingpolicy inject-perf-profiler -o yaml | yq '.spec.matchConditions[0].expression' @@ -72,27 +72,43 @@ yq -i '.spec.template.metadata.labels."perf-profile/sidecar" = "true"' ~/environ kubectl -n unicorn-store-spring apply -f ~/environment/unicorn-store-spring/k8s/deployment.yaml kubectl -n unicorn-store-spring rollout status deploy/unicorn-store-spring --timeout=300s +# startup line of the new pod +kubectl -n unicorn-store-spring logs -c unicorn-store-spring $(kubectl -n unicorn-store-spring get pod \ + -l app=unicorn-store-spring --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}') \ + | grep -E "Started StoreApplication|restored JVM" + # the manifest lists one container, the running pod has two: admission added the sidecar echo "manifest: $(yq '.spec.template.spec.containers[].name' ~/environment/unicorn-store-spring/k8s/deployment.yaml | xargs)" -echo "pod: $(kubectl -n unicorn-store-spring get pod -l app=unicorn-store-spring -o jsonpath='{.items[0].spec.containers[*].name}')" +echo "pod: $(kubectl -n unicorn-store-spring get pod -l app=unicorn-store-spring \ + --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].spec.containers[*].name}')" git -C ~/environment/unicorn-store-spring add -A \ && git -C ~/environment/unicorn-store-spring commit -q -m "baseline: profiler attached" WS_TEST_BLOCK_100_1 -ws_skip_block 2 'The injection policy' 'The injection policy' 47 57 'text' '' 'copy action disabled' +ws_skip_block 2 'The injection policy' 'The injection policy' 53 64 'text' '' 'copy action disabled' -ws_run_block 3 'The sensor' 'The sensor' 70 71 'bash' '' <<'WS_TEST_BLOCK_100_3' +ws_run_block 3 'The sensor' 'The sensor' 77 78 'bash' '' <<'WS_TEST_BLOCK_100_3' kubectl -n monitoring get configmap perf-sensor-checklist-rules -o jsonpath='{.data.checklist-rules\.yaml}' \ | yq '.items[] | select(.id == 6)' WS_TEST_BLOCK_100_3 -ws_skip_block 4 'The sensor' 'The sensor' 76 88 'yaml' '' 'copy action disabled' +ws_skip_block 4 'The sensor' 'The sensor' 83 95 'yaml' '' 'copy action disabled' -ws_run_block 5 'Ask for the baseline' 'In the Shell terminal:' 109 109 'bash' 'non-interactive' <<'WS_TEST_BLOCK_100_5' +ws_run_block 5 'Ask for the baseline' 'In the Shell terminal:' 116 116 'bash' 'non-interactive' <<'WS_TEST_BLOCK_100_5' cd ~/environment && claude -c -p "How is unicorn-store-spring doing against best practices?" WS_TEST_BLOCK_100_5 +ws_run_block 6 'Where the start spends its CPU' 'In the Shell terminal, print a link to the CPU profile of the new pod'"'"'s first 90 seconds in Grafana:' 129 135 'bash' '' <<'WS_TEST_BLOCK_100_6' +POD=$(kubectl -n unicorn-store-spring get pod -l app=unicorn-store-spring \ + --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}') +START=$(date -d "$(kubectl -n unicorn-store-spring get pod $POD -o jsonpath='{.status.startTime}')" +%s) +GRAFANA_URL=$(kubectl -n monitoring get svc grafana -o jsonpath="{.status.loadBalancer.ingress[0].hostname}") +echo "http://${GRAFANA_URL}/a/grafana-pyroscope-app/explore?explorationType=flame-graph&var-dataSource=pyroscope\ +&var-serviceName=unicorn-store-spring&var-profileMetricId=process_cpu:cpu:nanoseconds:cpu:nanoseconds\ +&var-filters=pod%7C%3D%7C${POD}&from=${START}000&to=$((START + 90))000" +WS_TEST_BLOCK_100_6 + ws_end_page ws_begin_page 'Improve the manifest' 200 'improve-manifest/index.en.md' @@ -101,11 +117,11 @@ ws_run_block 1 'Improve the manifest' 'In the Shell terminal:' 23 23 'bash' 'non cd ~/environment && claude -c -p "Right-size the memory and CPU of unicorn-store-spring and make it start faster without changing the image." WS_TEST_BLOCK_200_1 -ws_skip_block 2 'Review the change' 'Review the change' 49 76 'text' '' 'copy action disabled' +ws_skip_block 2 'Review the change' 'Review the change' 49 74 'text' '' 'copy action disabled' -ws_skip_block 3 'Review the change' '- requests.cpu and ActiveProcessorCount=1 come from sizeCpu. limits.cpu stays at 1' 94 115 'yaml' '' 'copy action disabled' +ws_skip_block 3 'Review the change' '- resizePolicy with restartPolicy: NotRequired for CPU states that a CPU change needs no restart. It is the default on current Kubernetes, and the prerequisite for the resize the controller performs' 93 114 'yaml' '' 'copy action disabled' -ws_run_block 4 'Roll out, verify and score' 'In the Shell terminal:' 127 148 'bash' '' 900 <<'WS_TEST_BLOCK_200_4' +ws_run_block 4 'Roll out, verify and score' 'In the Shell terminal:' 126 149 'bash' '' 900 <<'WS_TEST_BLOCK_200_4' kubectl -n unicorn-store-spring apply -f ~/environment/unicorn-store-spring/k8s/startup-cpu-boost.yaml kubectl -n unicorn-store-spring apply -f ~/environment/unicorn-store-spring/k8s/deployment.yaml kubectl -n unicorn-store-spring rollout status deploy/unicorn-store-spring --timeout=300s @@ -115,8 +131,10 @@ kubectl -n unicorn-store-spring exec deploy/unicorn-store-spring -c unicorn-stor sh -c 'unset JAVA_TOOL_OPTIONS; jcmd $(jcmd -l | grep -v JCmd | cut -d" " -f1) VM.flags' \ | tr ' ' '\n' | grep -E 'MaxHeapSize|InitialHeapSize|UseSerialGC|ActiveProcessorCount' -# startup with the boost -curl -s localhost:8090/api/v1/startupLog/unicorn-store-spring | jq -r .line +# startup line of the new pod, with the boost +kubectl -n unicorn-store-spring logs -c unicorn-store-spring $(kubectl -n unicorn-store-spring get pod \ + -l app=unicorn-store-spring --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}') \ + | grep -E "Started StoreApplication|restored JVM" # what the JVM saw at start (the boosted quota) and what the cgroup allows now sleep 30 @@ -130,25 +148,52 @@ git -C ~/environment/unicorn-store-spring add -A \ curl -sN "localhost:8090/api/v1/checklist/unicorn-store-spring?format=text&wait=true" WS_TEST_BLOCK_200_4 -ws_skip_block 5 'Roll out, verify and score' 'Roll out, verify and score' 155 165 'text' '' 'copy action disabled' +ws_skip_block 5 'Roll out, verify and score' 'Roll out, verify and score' 156 166 'text' '' 'copy action disabled' -ws_skip_block 6 'The score' 'The score' 192 210 'text' '' 'copy action disabled' +ws_skip_block 6 'The score' 'The score' 193 211 'text' '' 'copy action disabled' ws_end_page ws_begin_page 'Improve the image' 300 'improve-image/index.en.md' -ws_run_block 1 'Improve the image' 'In the Shell terminal:' 23 23 'bash' 'non-interactive' <<'WS_TEST_BLOCK_300_1' -cd ~/environment && claude -c -p "How can unicorn-store-spring start in under a second?" +ws_run_block 1 'Without changing the code' 'In the Shell terminal:' 25 25 'bash' 'non-interactive' <<'WS_TEST_BLOCK_300_1' +cd ~/environment && claude -c -p "Make unicorn-store-spring start faster without changing its code." WS_TEST_BLOCK_300_1 -ws_skip_block 2 'Review the change' 'Review the change' 42 75 'text' '' 'copy action disabled' +ws_skip_block 2 'Review the change' 'Review the change' 43 57 'text' '' 'copy action disabled' + +ws_run_block 3 'Build, roll out and check the startup' 'In the Shell terminal:' 68 83 'bash' '' 1500 <<'WS_TEST_BLOCK_300_3' +# ≈ 2 min: builds the jar, trains and creates the cache; pushes :aot +~/environment/unicorn-store-spring/scripts/build.sh aot Dockerfile.aot + +# point the Deployment at the :aot tag and roll it out +yq -i '.spec.template.spec.containers[0].image |= sub(":[^:]+$"; ":aot")' \ + ~/environment/unicorn-store-spring/k8s/deployment.yaml +kubectl -n unicorn-store-spring apply -f ~/environment/unicorn-store-spring/k8s/deployment.yaml +kubectl -n unicorn-store-spring rollout status deploy/unicorn-store-spring --timeout=300s -ws_skip_block 3 'Review the change' '- k8s/deployment.yaml: JAVATOOLOPTIONS removed. A restored JVM keeps the flags of the checkpoint, and a GC flag in the environment stops the restore' 88 131 'dockerfile' '' 'copy action disabled' +# startup line of the new pod +kubectl -n unicorn-store-spring logs -c unicorn-store-spring $(kubectl -n unicorn-store-spring get pod \ + -l app=unicorn-store-spring --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}') \ + | grep -E "Started StoreApplication|restored JVM" + +git -C ~/environment/unicorn-store-spring add -A \ + && git -C ~/environment/unicorn-store-spring commit -q -m "aot cache" +WS_TEST_BLOCK_300_3 -ws_skip_block 4 'Review the change' '- Line 44: -XX:CRaCRestoreFrom restores instead of starting. There are no GC or heap flags: they came with the checkpoint' 144 174 'java' '' 'copy action disabled' +ws_skip_block 4 'Build, roll out and check the startup' 'Build, roll out and check the startup' 88 92 'text' '' 'copy action disabled' -ws_run_block 5 'Build, roll out, verify and score' 'In the Shell terminal. The build starts the application inside Docker, warms it up, checkpoints, and pushes the :crac tag:' 187 207 'bash' '' 1500 <<'WS_TEST_BLOCK_300_5' +ws_run_block 5 'With a code change' 'In the Shell terminal:' 113 113 'bash' 'non-interactive' <<'WS_TEST_BLOCK_300_5' +cd ~/environment && claude -c -p "Can unicorn-store-spring start even faster if its code may change?" +WS_TEST_BLOCK_300_5 + +ws_skip_block 6 'Review the change' 'Review the change' 132 168 'text' '' 'copy action disabled' + +ws_skip_block 7 'Review the change' '- k8s/deployment.yaml: JAVATOOLOPTIONS removed. A restored JVM keeps the flags of the checkpoint, and a GC flag in the environment stops the restore' 181 224 'dockerfile' '' 'copy action disabled' + +ws_skip_block 8 'Review the change' '- Line 44: -XX:CRaCRestoreFrom restores instead of starting. There are no GC or heap flags: they came with the checkpoint' 237 267 'java' '' 'copy action disabled' + +ws_run_block 9 'Build, roll out, verify and score' 'In the Shell terminal:' 280 302 'bash' '' 1500 <<'WS_TEST_BLOCK_300_9' # ≈ 3 min: builds, starts, warms up, checkpoints; pushes :crac ~/environment/unicorn-store-spring/scripts/build.sh crac Dockerfile.crac @@ -158,8 +203,10 @@ yq -i '.spec.template.spec.containers[0].image |= sub(":[^:]+$"; ":crac")' \ kubectl -n unicorn-store-spring apply -f ~/environment/unicorn-store-spring/k8s/deployment.yaml kubectl -n unicorn-store-spring rollout status deploy/unicorn-store-spring --timeout=300s -# startup line: "restored", well under a second -curl -s localhost:8090/api/v1/startupLog/unicorn-store-spring | jq -r .line +# startup line of the new pod +kubectl -n unicorn-store-spring logs -c unicorn-store-spring $(kubectl -n unicorn-store-spring get pod \ + -l app=unicorn-store-spring --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}') \ + | grep -E "Started StoreApplication|restored JVM" # heap flags came from the checkpoint, not from the Deployment kubectl -n unicorn-store-spring exec deploy/unicorn-store-spring -c unicorn-store-spring -- \ @@ -170,11 +217,21 @@ git -C ~/environment/unicorn-store-spring add -A \ && git -C ~/environment/unicorn-store-spring commit -q -m "crac" curl -sN "localhost:8090/api/v1/checklist/unicorn-store-spring?format=text&wait=true" -WS_TEST_BLOCK_300_5 +WS_TEST_BLOCK_300_9 + +ws_run_block 10 'Where the restored start spends its CPU' 'Print the same link for the restored pod:' 324 330 'bash' '' <<'WS_TEST_BLOCK_300_10' +POD=$(kubectl -n unicorn-store-spring get pod -l app=unicorn-store-spring \ + --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}') +START=$(date -d "$(kubectl -n unicorn-store-spring get pod $POD -o jsonpath='{.status.startTime}')" +%s) +GRAFANA_URL=$(kubectl -n monitoring get svc grafana -o jsonpath="{.status.loadBalancer.ingress[0].hostname}") +echo "http://${GRAFANA_URL}/a/grafana-pyroscope-app/explore?explorationType=flame-graph&var-dataSource=pyroscope\ +&var-serviceName=unicorn-store-spring&var-profileMetricId=process_cpu:cpu:nanoseconds:cpu:nanoseconds\ +&var-filters=pod%7C%3D%7C${POD}&from=${START}000&to=$((START + 90))000" +WS_TEST_BLOCK_300_10 -ws_skip_block 6 'Where the restored start spends its CPU' 'Where the restored start spends its CPU' 232 241 'text' '' 'copy action disabled' +ws_skip_block 11 'Where the restored start spends its CPU' 'Search for ClassLoader: 13% of the baseline pod'"'"'s window in this run, 1% after the restore. Search for CompileBroker: the JIT compiler threads still take about half of the window.' 339 348 'text' '' 'copy action disabled' -ws_skip_block 7 'The score' 'The score' 257 275 'text' '' 'copy action disabled' +ws_skip_block 12 'The score' 'The score' 364 382 'text' '' 'copy action disabled' ws_end_page @@ -184,15 +241,20 @@ ws_run_block 1 'Improve the code' 'In the Shell terminal:' 23 23 'bash' 'non-int cd ~/environment && claude -c -p "Why do some writes to unicorn-store-spring take seconds under load? Fix it, size the service again and fix item 10." WS_TEST_BLOCK_400_1 -ws_skip_block 2 'Review the change' 'Review the change' 41 75 'text' '' 'copy action disabled' +ws_skip_block 2 'Review the change' 'Review the change' 41 79 'text' '' 'copy action disabled' -ws_run_block 3 'Build, roll out, verify and score' 'In the Shell terminal. The image tag stays :crac with a new digest, and the Deployment changed, so apply rolls out both:' 96 110 'bash' '' 1500 <<'WS_TEST_BLOCK_400_3' +ws_run_block 3 'Build, roll out, verify and score' 'The image tag stays :crac with a new digest, and the Deployment changed, so apply rolls out both. In the Shell terminal:' 100 119 'bash' '' 1500 <<'WS_TEST_BLOCK_400_3' # code and heap flags changed: rebuild the checkpoint (≈ 2 min) ~/environment/unicorn-store-spring/scripts/build.sh crac Dockerfile.crac kubectl -n unicorn-store-spring apply -f ~/environment/unicorn-store-spring/k8s/deployment.yaml kubectl -n unicorn-store-spring rollout status deploy/unicorn-store-spring --timeout=300s +# startup line of the new pod +kubectl -n unicorn-store-spring logs -c unicorn-store-spring $(kubectl -n unicorn-store-spring get pod \ + -l app=unicorn-store-spring --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}') \ + | grep -E "Started StoreApplication|restored JVM" + # heap flags from the new checkpoint: 75 % / 50 % of the new limit kubectl -n unicorn-store-spring exec deploy/unicorn-store-spring -c unicorn-store-spring -- \ sh -c 'unset JAVA_TOOL_OPTIONS; jcmd $(jcmd -l | grep -v JCmd | cut -d" " -f1) VM.flags' \ @@ -204,9 +266,9 @@ git -C ~/environment/unicorn-store-spring add -A \ curl -sN "localhost:8090/api/v1/checklist/unicorn-store-spring?format=text&wait=true" WS_TEST_BLOCK_400_3 -ws_skip_block 4 'The pool and the database' 'The pool and the database' 129 135 'text' '' 'copy action disabled' +ws_skip_block 4 'The pool and the database' 'The pool and the database' 138 145 'text' '' 'copy action disabled' -ws_skip_block 5 'The score' 'The score' 151 169 'text' '' 'copy action disabled' +ws_skip_block 5 'The score' 'The score' 161 179 'text' '' 'copy action disabled' ws_end_page @@ -216,11 +278,11 @@ ws_run_block 1 'Enforce the practices' 'In the Shell terminal:' 23 23 'bash' 'no cd ~/environment && claude -c -p "Turn the checklist items that can be checked on the manifest into a Kyverno policy for unicorn-store-spring." WS_TEST_BLOCK_500_1 -ws_skip_block 2 'Review the policy' 'Review the policy' 44 65 'text' '' 'copy action disabled' +ws_skip_block 2 'Review the policy' 'Review the policy' 44 66 'text' '' 'copy action disabled' -ws_skip_block 3 'Review the policy' 'Two of the seven validations in k8s/policy.yaml:' 72 84 'yaml' '' 'copy action disabled' +ws_skip_block 3 'Review the policy' 'Two of the seven validations in k8s/policy.yaml:' 73 85 'yaml' '' 'copy action disabled' -ws_run_block 4 'Apply and test' 'In the Shell terminal:' 97 109 'bash' '' <<'WS_TEST_BLOCK_500_4' +ws_run_block 4 'Apply and test' 'In the Shell terminal:' 98 110 'bash' '' <<'WS_TEST_BLOCK_500_4' kubectl apply -f ~/environment/unicorn-store-spring/k8s/policy.yaml # the background scan reports the running Deployment @@ -236,13 +298,13 @@ git -C ~/environment/unicorn-store-spring add -A \ && git -C ~/environment/unicorn-store-spring commit -q -m "kyverno policy" WS_TEST_BLOCK_500_4 -ws_skip_block 5 'Apply and test' 'Apply and test' 114 118 'text' '' 'copy action disabled' +ws_skip_block 5 'Apply and test' 'Apply and test' 115 119 'text' '' 'copy action disabled' ws_end_page ws_begin_page 'Results' 650 'results/index.en.md' -ws_skip_block 1 'Choosing for your own workloads' 'Choosing for your own workloads' 29 48 '' '' 'informational block without language' +ws_skip_block 1 'Choosing for your own workloads' 'Choosing for your own workloads' 30 49 '' '' 'informational block without language' ws_end_page diff --git a/infra/scripts/ws-test/samples/java-performance.py b/infra/scripts/ws-test/samples/java-performance.py index 9e361451..bafa532b 100644 --- a/infra/scripts/ws-test/samples/java-performance.py +++ b/infra/scripts/ws-test/samples/java-performance.py @@ -2,11 +2,14 @@ setup: expand 0 = the environment check. baseline: expand 0 = the policy-view + attach block (rollout wait lines dropped), expand 2 = - the checklist answer as raw markdown. -improve-manifest, improve-image, improve-code: two blocks each, the Claude question and - the combined roll-out block. Expand 0 = Claude's answer wrapped at 110 columns, - expand 1 = the roll-out block up to "Pod is" (image and code: only the build, rollout, - startup and heap-flag lines), expand 2 = the checklist from "Score:". + the checklist answer as raw markdown. The profile-link block (block 3) has no expand. +improve-manifest, improve-code: two blocks each, the Claude question and the combined + roll-out block. Expand 0 = Claude's answer wrapped at 110 columns, expand 1 = the roll-out + block up to "Pod is" (code: only the build, rollout, startup and heap-flag lines), expand 2 = + the checklist from "Score:". +improve-image: the AOT question and its build block, the CRaC question and its build block, + then the profile link (no expand). Expands 0–4 = AOT answer, AOT build lines, CRaC answer, + CRaC build lines up to "Pod is", the checklist from "Score:". enforce: expand 0 = Claude's answer, expand 1 = the apply block. A block re-run by hand replaces its expand with that real output; this mapping then @@ -15,6 +18,7 @@ from refresh import drop, keep, since, until, wrap ROLLOUT_WAIT = r'^Waiting for deployment' +BUILD_LINES = ('[build] docker build', '[build] pushed', 'deployment', '20', '-XX') def refresh(s): @@ -24,16 +28,25 @@ def refresh(s): s.set_expand('baseline/index.en.md', 0, drop(b[0], ROLLOUT_WAIT)) s.set_expand('baseline/index.en.md', 2, b[2], raw=True) - for page in ('improve-manifest', 'improve-image', 'improve-code'): + for page in ('improve-manifest', 'improve-code'): md = page + '/index.en.md' question, run = s.blocks(md) s.set_expand(md, 0, wrap(question)) head = drop(until(run, 'Pod is'), ROLLOUT_WAIT) if page != 'improve-manifest': - head = keep(head, '[build] docker build', '[build] pushed', 'deployment', '20', '-XX') + head = keep(head, *BUILD_LINES) s.set_expand(md, 1, head) s.set_expand(md, 2, since(run, 'Score:')) + # improve-image: AOT question + build block, CRaC question + build block, profile link. + md = 'improve-image/index.en.md' + aot_q, aot_run, crac_q, crac_run = s.blocks(md)[:4] + s.set_expand(md, 0, wrap(aot_q)) + s.set_expand(md, 1, keep(drop(aot_run, ROLLOUT_WAIT), *BUILD_LINES)) + s.set_expand(md, 2, wrap(crac_q)) + s.set_expand(md, 3, keep(drop(until(crac_run, 'Pod is'), ROLLOUT_WAIT), *BUILD_LINES)) + s.set_expand(md, 4, since(crac_run, 'Score:')) + question, run = s.blocks('enforce/index.en.md') s.set_expand('enforce/index.en.md', 0, wrap(question)) s.set_expand('enforce/index.en.md', 1, run)