diff --git a/apps/unicorn-store-spring/scripts/build.sh b/apps/unicorn-store-spring/scripts/build.sh index c37cadb0..e16fc49d 100755 --- a/apps/unicorn-store-spring/scripts/build.sh +++ b/apps/unicorn-store-spring/scripts/build.sh @@ -60,6 +60,12 @@ docker build \ --secret id=db-password,env=DB_PASS \ "${EXTRA_ARGS[@]}" \ -t "${REPO}:${TAG}" -f "$DOCKERFILE" . -docker push "${REPO}:${TAG}" +# A push can stall without output (seen once: 20 min for two layers that usually take +# seconds). Bound each attempt and retry; layers already uploaded are not sent again. +for attempt in 1 2 3; do + if timeout 180 docker push "${REPO}:${TAG}"; then break; fi + [ "${attempt}" = 3 ] && { echo "[build] push failed 3 times" >&2; exit 1; } + echo "[build] push did not finish in 180 s, retrying (${attempt}/3)..." >&2 +done echo "[build] pushed ${REPO}:${TAG}" >&2 echo "${REPO}:${TAG}" diff --git a/infra/scripts/deploy/java-performance/platform-placement.sh b/infra/scripts/deploy/java-performance/platform-placement.sh index ab07e4f5..4571d696 100755 --- a/infra/scripts/deploy/java-performance/platform-placement.sh +++ b/infra/scripts/deploy/java-performance/platform-placement.sh @@ -6,15 +6,22 @@ # workloads have no selector, so Karpenter places them wherever there is room, usually on # the app's 4-vCPU node. There the app shares physical cores (SMT) with Prometheus, # Pyroscope and Kyverno, and the session's measurements move with that load (startup and -# steady CPU 10–60 % higher in a test run). This script pins every platform Deployment and -# StatefulSet to the Auto Mode built-in `general-purpose` NodePool, so the `workshop` node -# carries only the app and the DaemonSets. The app manifest is unchanged. +# steady CPU 10–60 % higher in a test run). This script creates a `platform` NodePool and +# pins every platform Deployment and StatefulSet to it, so the `workshop` node carries only +# the app and the DaemonSets. The app manifest is unchanged. +# +# The pool admits only instances with at least 4 vCPU and 8 GiB (c6a.xlarge is the cheapest +# match): the platform requests about 1.5 vCPU and 4.4 GiB, so one node holds all of it. On +# the built-in general-purpose pool Karpenter started a c6a.large per batch of pending pods +# and kept two (do-not-disrupt blocks the merge), at more cost than one c6a.xlarge. # # Namespaces: monitoring (Prometheus, Grafana, Pyroscope, Alloy, sensor), kyverno, # kube-startup-cpu-boost-system, and metrics-server in kube-system. Pods move once, at # bootstrap; Prometheus, Grafana and Pyroscope keep their data on their PVCs. The pods are -# marked do-not-disrupt, so consolidation does not move them later. -# Idempotent. Run after the platform installs (template stage E). A later `helm upgrade` +# marked do-not-disrupt and the pool consolidates only when empty, so nothing moves them later. +# Idempotent. `platform-placement.sh pool` creates only the NodePool (template stage B, so +# monitoring.sh installs onto it via MONITORING_NODEPOOL); without an argument it also pins +# the workloads (after stage E). A later `helm upgrade` # of one of these releases drops the selector: run this script again afterwards. # ============================================================================= set -uo pipefail @@ -22,12 +29,54 @@ set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${SCRIPT_DIR}/../../lib/common.sh" -POOL="${PLATFORM_NODEPOOL:-general-purpose}" -kubectl get nodepool "${POOL}" >/dev/null 2>&1 || { log_error "NodePool ${POOL} not found (EKS Auto Mode built-in pools disabled?)"; exit 1; } +POOL="${PLATFORM_NODEPOOL:-platform}" + +kubectl apply -f - </dev/null || { log_error "NodePool ${POOL} could not be created"; exit 1; } +apiVersion: karpenter.sh/v1 +kind: NodePool +metadata: + name: ${POOL} +spec: + template: + spec: + nodeClassRef: + group: eks.amazonaws.com + kind: NodeClass + name: default + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: eks.amazonaws.com/instance-category + operator: In + values: ["c", "m"] + - key: eks.amazonaws.com/instance-generation + operator: Gt + values: ["5"] + - key: eks.amazonaws.com/instance-cpu + operator: Gt + values: ["3"] + - key: eks.amazonaws.com/instance-memory + operator: Gt + values: ["8000"] + limits: + cpu: "16" + disruption: + consolidationPolicy: WhenEmpty + consolidateAfter: 5m +EOF +log_info "NodePool ${POOL}: at least 4 vCPU and 8 GiB, consolidation WhenEmpty" +# `pool`: only the NodePool (template stage B, before monitoring.sh installs onto it). +[ "${1:-}" = pool ] && exit 0 -# do-not-disrupt: the general-purpose pool consolidates underutilized nodes after 30 s. A -# consolidation during the session would move the sensor or Kyverno in the middle of a -# checklist or an admission; the annotation keeps their nodes until the pods go. +# do-not-disrupt: a consolidation during the session would move the sensor or Kyverno in +# the middle of a checklist or an admission; the annotation keeps their node until the pods go. PATCH="{\"spec\":{\"template\":{\"metadata\":{\"annotations\":{\"karpenter.sh/do-not-disrupt\":\"true\"}},\"spec\":{\"nodeSelector\":{\"karpenter.sh/nodepool\":\"${POOL}\"}}}}}" RC=0 pin() { # pin NAMESPACE KIND/NAME... diff --git a/infra/scripts/setup/monitoring.sh b/infra/scripts/setup/monitoring.sh index 7c1ead67..405b2943 100755 --- a/infra/scripts/setup/monitoring.sh +++ b/infra/scripts/setup/monitoring.sh @@ -38,6 +38,17 @@ DATASOURCE_FILE="grafana-datasource.yaml" GRAFANA_VALUES_FILE="grafana-values.yaml" WORK="" + +# Optional NodePool for the stack: MONITORING_NODEPOOL, set by a workshop template that keeps +# the platform off its app's node (java-performance). Applied at install, so the PVCs of +# Prometheus, Pyroscope and Grafana bind in that pool's zone from the start. Unset: no +# selector, the charts install as before. +PROM_POOL=(); PYRO_POOL=(); GRAFANA_POOL=() +if [ -n "${MONITORING_NODEPOOL:-}" ]; then + PROM_POOL=(--set-string "server.nodeSelector.karpenter\\.sh/nodepool=${MONITORING_NODEPOOL}") + PYRO_POOL=(--set-string "pyroscope.nodeSelector.karpenter\\.sh/nodepool=${MONITORING_NODEPOOL}") + GRAFANA_POOL=(--set-string "nodeSelector.karpenter\\.sh/nodepool=${MONITORING_NODEPOOL}") +fi cleanup() { rm -f "$VALUES_FILE" "$DATASOURCE_FILE" "$GRAFANA_VALUES_FILE" [[ -n "$WORK" ]] && rm -rf "$WORK" @@ -79,7 +90,7 @@ EOF log_info "Deploying Prometheus..." helm upgrade --install prometheus prometheus-community/prometheus \ - --namespace "$NAMESPACE" \ + --namespace "$NAMESPACE" ${PROM_POOL[@]+"${PROM_POOL[@]}"} \ --values "$VALUES_FILE" >>"$LOG" 2>&1 \ || { log_error "Prometheus helm install failed — last 40 lines of $LOG:"; tail -n 40 "$LOG"; exit 1; } @@ -203,7 +214,7 @@ pyroscope: EOF helm upgrade --install pyroscope grafana/pyroscope \ - --namespace "${NAMESPACE}" \ + --namespace "${NAMESPACE}" ${PYRO_POOL[@]+"${PYRO_POOL[@]}"} \ --values "${WORK}/pyroscope-values.yaml" >>"$LOG" 2>&1 \ || { log_error "Pyroscope helm install failed — last 40 lines of $LOG:"; tail -n 40 "$LOG"; exit 1; } log_info "Pyroscope install submitted (readiness checked after Grafana)" @@ -291,7 +302,7 @@ EOF log_info "Deploying Grafana..." helm upgrade --install grafana grafana-community/grafana \ - --namespace "$NAMESPACE" \ + --namespace "$NAMESPACE" ${GRAFANA_POOL[@]+"${GRAFANA_POOL[@]}"} \ --values "$GRAFANA_VALUES_FILE" >>"$LOG" 2>&1 \ || { log_error "Grafana helm install failed — last 40 lines of $LOG:"; tail -n 40 "$LOG"; exit 1; } diff --git a/infra/scripts/templates/java-performance.sh b/infra/scripts/templates/java-performance.sh index 1ad0f929..f6dbf75d 100755 --- a/infra/scripts/templates/java-performance.sh +++ b/infra/scripts/templates/java-performance.sh @@ -15,7 +15,7 @@ # --- wait for A --- # D app app-deploy (namespace, Pod Identity, manifests) } in # E platform perf-profiler install, startup-cpu-boost, perf-sensor } parallel -# placement platform Deployments/StatefulSets → general-purpose NodePool +# placement platform Deployments/StatefulSets → own platform NodePool # F verify ALB answers, sensor measures, Claude wiring present # # Each stage script is idempotent and runnable on its own from the IDE. @@ -65,10 +65,12 @@ A_PID=$! stage "B: EKS cluster" run "EKS cluster configured" "$SETUP/eks.sh" try "NodePool pinned to m6a" "$DEPLOY/nodepool-pin.sh" +run "Platform NodePool created" "$DEPLOY/platform-placement.sh" pool try "metrics-server installed" "$DEPLOY/metrics-server.sh" # --- C: monitoring --------------------------------------------------------------------- stage "C: monitoring (Prometheus, Pyroscope, Grafana)" +export MONITORING_NODEPOOL=platform # monitoring.sh installs the stack onto the platform pool run "Monitoring stack deployed" "$SETUP/monitoring.sh" # --- wait for A ------------------------------------------------------------------------- @@ -101,7 +103,7 @@ cat "$D_LOG"; cat "$E_LOG" [ "$E_RC" -eq 0 ] && log_success "$(stamp)Stage E done (profiler, boost, sensor)" || { log_error "$(stamp)Stage E FAILED"; exit 1; } # --- platform placement: platform pods off the app's node ------------------------------ -run "Platform on the general-purpose NodePool" "$DEPLOY/platform-placement.sh" +run "Platform on its own NodePool" "$DEPLOY/platform-placement.sh" # --- F: verify -------------------------------------------------------------------------- stage "F: verify" diff --git a/infra/scripts/ws-test/java-performance.sh b/infra/scripts/ws-test/java-performance.sh index 09f06d12..f5f8f4ea 100755 --- a/infra/scripts/ws-test/java-performance.sh +++ b/infra/scripts/ws-test/java-performance.sh @@ -6,7 +6,7 @@ set -Eeuo pipefail WS_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${WS_SCRIPT_DIR}/runtime.sh" ws_begin_run 'Java performance optimization' "${WS_SCRIPT_DIR}/reports/java-performance" 5 "$@" -ws_set_executable_total 20 +ws_set_executable_total 27 ws_set_default_timeout 600 ws_source_environment '/etc/profile.d/workshop.sh' @@ -59,14 +59,18 @@ ws_end_page ws_begin_page 'Measure the baseline' 100 'baseline/index.en.md' -ws_run_block 1 'The injection policy' 'The injection policy' 25 48 'bash' '' <<'WS_TEST_BLOCK_100_1' +ws_run_block 1 'The injection policy' 'The injection policy' 25 30 'bash' '' <<'WS_TEST_BLOCK_100_1' # when the policy applies: a CEL condition on the pod's labels kubectl get mutatingpolicy inject-perf-profiler -o yaml | yq '.spec.matchConditions[0].expression' # what it changes: the JSON patch paths it writes into the pod spec kubectl get mutatingpolicy inject-perf-profiler -o yaml \ | yq '.spec.mutations[0].jsonPatch.expression' | grep -oE '"/spec/[a-zA-Z0-9/-]+"' | sort -u +WS_TEST_BLOCK_100_1 + +ws_skip_block 2 'The injection policy' 'The injection policy' 35 35 'text' '' 'copy action disabled' +ws_run_block 3 'The injection policy' 'Opt the service in with one label, and compare the manifest with the pod:' 44 60 'bash' '' <<'WS_TEST_BLOCK_100_3' # opt in: one label on the pod template yq -i '.spec.template.metadata.labels."perf-profile/sidecar" = "true"' ~/environment/unicorn-store-spring/k8s/deployment.yaml kubectl -n unicorn-store-spring apply -f ~/environment/unicorn-store-spring/k8s/deployment.yaml @@ -84,22 +88,22 @@ echo "pod: $(kubectl -n unicorn-store-spring get pod -l app=unicorn-store-s git -C ~/environment/unicorn-store-spring add -A \ && git -C ~/environment/unicorn-store-spring commit -q -m "baseline: profiler attached" -WS_TEST_BLOCK_100_1 +WS_TEST_BLOCK_100_3 -ws_skip_block 2 'The injection policy' 'The injection policy' 53 64 'text' '' 'copy action disabled' +ws_skip_block 4 'The injection policy' 'The injection policy' 65 76 'text' '' 'copy action disabled' -ws_run_block 3 'The sensor' 'The sensor' 77 78 'bash' '' <<'WS_TEST_BLOCK_100_3' +ws_run_block 5 'The sensor' 'The sensor' 89 90 'bash' '' <<'WS_TEST_BLOCK_100_5' kubectl -n monitoring get configmap perf-sensor-checklist-rules -o jsonpath='{.data.checklist-rules\.yaml}' \ | yq '.items[] | select(.id == 6)' -WS_TEST_BLOCK_100_3 +WS_TEST_BLOCK_100_5 -ws_skip_block 4 'The sensor' 'The sensor' 83 95 'yaml' '' 'copy action disabled' +ws_skip_block 6 'The sensor' 'The sensor' 95 107 'yaml' '' 'copy action disabled' -ws_run_block 5 'Ask for the baseline' 'In the Shell terminal:' 116 116 'bash' 'non-interactive' <<'WS_TEST_BLOCK_100_5' +ws_run_block 7 'Ask for the baseline' 'In the Shell terminal:' 128 128 'bash' 'non-interactive' <<'WS_TEST_BLOCK_100_7' cd ~/environment && claude -c -p "How is unicorn-store-spring doing against best practices?" -WS_TEST_BLOCK_100_5 +WS_TEST_BLOCK_100_7 -ws_run_block 6 'Where the start spends its CPU' 'In the Shell terminal, print a link to the CPU profile of the new pod'"'"'s first 90 seconds in Grafana:' 129 135 'bash' '' <<'WS_TEST_BLOCK_100_6' +ws_run_block 8 'Where the start spends its CPU' 'In the Shell terminal, print a link to the CPU profile of the new pod'"'"'s first 90 seconds in Grafana:' 141 147 'bash' '' <<'WS_TEST_BLOCK_100_8' POD=$(kubectl -n unicorn-store-spring get pod -l app=unicorn-store-spring \ --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}') START=$(date -d "$(kubectl -n unicorn-store-spring get pod $POD -o jsonpath='{.status.startTime}')" +%s) @@ -107,7 +111,7 @@ GRAFANA_URL=$(kubectl -n monitoring get svc grafana -o jsonpath="{.status.loadBa echo "http://${GRAFANA_URL}/a/grafana-pyroscope-app/explore?explorationType=flame-graph&var-dataSource=pyroscope\ &var-serviceName=unicorn-store-spring&var-profileMetricId=process_cpu:cpu:nanoseconds:cpu:nanoseconds\ &var-filters=pod%7C%3D%7C${POD}&from=${START}000&to=$((START + 90))000" -WS_TEST_BLOCK_100_6 +WS_TEST_BLOCK_100_8 ws_end_page @@ -121,36 +125,32 @@ ws_skip_block 2 'Review the change' 'Review the change' 49 74 'text' '' 'copy ac ws_skip_block 3 'Review the change' '- resizePolicy with restartPolicy: NotRequired for CPU states that a CPU change needs no restart. It is the default on current Kubernetes, and the prerequisite for the resize the controller performs' 93 114 'yaml' '' 'copy action disabled' -ws_run_block 4 'Roll out, verify and score' 'In the Shell terminal:' 126 149 'bash' '' 900 <<'WS_TEST_BLOCK_200_4' +ws_run_block 4 'Roll out' 'In the Shell terminal:' 126 136 'bash' '' <<'WS_TEST_BLOCK_200_4' kubectl -n unicorn-store-spring apply -f ~/environment/unicorn-store-spring/k8s/startup-cpu-boost.yaml kubectl -n unicorn-store-spring apply -f ~/environment/unicorn-store-spring/k8s/deployment.yaml kubectl -n unicorn-store-spring rollout status deploy/unicorn-store-spring --timeout=300s -# the heap follows the limit: 75 % / 50 %, SerialGC, one processor -kubectl -n unicorn-store-spring exec deploy/unicorn-store-spring -c unicorn-store-spring -- \ - sh -c 'unset JAVA_TOOL_OPTIONS; jcmd $(jcmd -l | grep -v JCmd | cut -d" " -f1) VM.flags' \ - | tr ' ' '\n' | grep -E 'MaxHeapSize|InitialHeapSize|UseSerialGC|ActiveProcessorCount' - # startup line of the new pod, with the boost kubectl -n unicorn-store-spring logs -c unicorn-store-spring $(kubectl -n unicorn-store-spring get pod \ -l app=unicorn-store-spring --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}') \ | grep -E "Started StoreApplication|restored JVM" -# what the JVM saw at start (the boosted quota) and what the cgroup allows now -sleep 30 -curl -s localhost:8090/api/v1/measure/unicorn-store-spring | jq -c '.jfr.container' -kubectl -n unicorn-store-spring exec deploy/unicorn-store-spring -c unicorn-store-spring -- cat /sys/fs/cgroup/cpu.max - git -C ~/environment/unicorn-store-spring add -A \ && git -C ~/environment/unicorn-store-spring commit -q -m "right-size memory and cpu, startup cpu boost" +WS_TEST_BLOCK_200_4 + +ws_skip_block 5 'Roll out' 'Roll out' 141 141 'text' '' 'copy action disabled' +ws_run_block 6 'Score' 'In the Shell terminal:' 152 157 'bash' '' <<'WS_TEST_BLOCK_200_6' # score the new pod: waits until it has run under load for 120 s, then measures curl -sN "localhost:8090/api/v1/checklist/unicorn-store-spring?format=text&wait=true" -WS_TEST_BLOCK_200_4 -ws_skip_block 5 'Roll out, verify and score' 'Roll out, verify and score' 156 166 'text' '' 'copy action disabled' +# what the JVM read at start (the boosted quota) and what the cgroup allows now +curl -s localhost:8090/api/v1/measure/unicorn-store-spring | jq -c '.jfr.container' +kubectl -n unicorn-store-spring exec deploy/unicorn-store-spring -c unicorn-store-spring -- cat /sys/fs/cgroup/cpu.max +WS_TEST_BLOCK_200_6 -ws_skip_block 6 'The score' 'The score' 193 211 'text' '' 'copy action disabled' +ws_skip_block 7 'The score' 'The score' 183 201 'text' '' 'copy action disabled' ws_end_page @@ -162,10 +162,12 @@ WS_TEST_BLOCK_300_1 ws_skip_block 2 'Review the change' 'Review the change' 43 57 'text' '' 'copy action disabled' -ws_run_block 3 'Build, roll out and check the startup' 'In the Shell terminal:' 68 83 'bash' '' 1500 <<'WS_TEST_BLOCK_300_3' -# ≈ 2 min: builds the jar, trains and creates the cache; pushes :aot +ws_run_block 3 'Build' 'In the Shell terminal:' 68 69 'bash' '' 1500 <<'WS_TEST_BLOCK_300_3' +# ≈ 1 min: builds the jar, trains and creates the cache; pushes :aot ~/environment/unicorn-store-spring/scripts/build.sh aot Dockerfile.aot +WS_TEST_BLOCK_300_3 +ws_run_block 4 'Roll out' 'In the Shell terminal:' 77 89 'bash' '' <<'WS_TEST_BLOCK_300_4' # point the Deployment at the :aot tag and roll it out yq -i '.spec.template.spec.containers[0].image |= sub(":[^:]+$"; ":aot")' \ ~/environment/unicorn-store-spring/k8s/deployment.yaml @@ -179,24 +181,26 @@ kubectl -n unicorn-store-spring logs -c unicorn-store-spring $(kubectl -n unicor git -C ~/environment/unicorn-store-spring add -A \ && git -C ~/environment/unicorn-store-spring commit -q -m "aot cache" -WS_TEST_BLOCK_300_3 +WS_TEST_BLOCK_300_4 -ws_skip_block 4 'Build, roll out and check the startup' 'Build, roll out and check the startup' 88 92 'text' '' 'copy action disabled' +ws_skip_block 5 'Roll out' 'Roll out' 94 94 'text' '' 'copy action disabled' -ws_run_block 5 'With a code change' 'In the Shell terminal:' 113 113 'bash' 'non-interactive' <<'WS_TEST_BLOCK_300_5' +ws_run_block 6 'With a code change' 'In the Shell terminal:' 115 115 'bash' 'non-interactive' <<'WS_TEST_BLOCK_300_6' cd ~/environment && claude -c -p "Can unicorn-store-spring start even faster if its code may change?" -WS_TEST_BLOCK_300_5 +WS_TEST_BLOCK_300_6 -ws_skip_block 6 'Review the change' 'Review the change' 132 168 'text' '' 'copy action disabled' +ws_skip_block 7 'Review the change' 'Review the change' 134 170 'text' '' 'copy action disabled' -ws_skip_block 7 'Review the change' '- k8s/deployment.yaml: JAVATOOLOPTIONS removed. A restored JVM keeps the flags of the checkpoint, and a GC flag in the environment stops the restore' 181 224 'dockerfile' '' 'copy action disabled' +ws_skip_block 8 'Review the change' '- k8s/deployment.yaml: JAVATOOLOPTIONS removed. A restored JVM keeps the flags of the checkpoint, and a GC flag in the environment stops the restore' 183 226 'dockerfile' '' 'copy action disabled' -ws_skip_block 8 'Review the change' '- Line 44: -XX:CRaCRestoreFrom restores instead of starting. There are no GC or heap flags: they came with the checkpoint' 237 267 'java' '' 'copy action disabled' +ws_skip_block 9 'Review the change' '- Line 44: -XX:CRaCRestoreFrom restores instead of starting. There are no GC or heap flags: they came with the checkpoint' 239 269 'java' '' 'copy action disabled' -ws_run_block 9 'Build, roll out, verify and score' 'In the Shell terminal:' 280 302 'bash' '' 1500 <<'WS_TEST_BLOCK_300_9' -# ≈ 3 min: builds, starts, warms up, checkpoints; pushes :crac +ws_run_block 10 'Build' 'In the Shell terminal:' 282 283 'bash' '' 1500 <<'WS_TEST_BLOCK_300_10' +# ≈ 2 min: builds, starts, warms up, checkpoints; pushes :crac ~/environment/unicorn-store-spring/scripts/build.sh crac Dockerfile.crac +WS_TEST_BLOCK_300_10 +ws_run_block 11 'Roll out' 'In the Shell terminal:' 305 317 'bash' '' <<'WS_TEST_BLOCK_300_11' # point the Deployment at the :crac tag and roll it out yq -i '.spec.template.spec.containers[0].image |= sub(":[^:]+$"; ":crac")' \ ~/environment/unicorn-store-spring/k8s/deployment.yaml @@ -208,18 +212,17 @@ kubectl -n unicorn-store-spring logs -c unicorn-store-spring $(kubectl -n unicor -l app=unicorn-store-spring --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}') \ | grep -E "Started StoreApplication|restored JVM" -# heap flags came from the checkpoint, not from the Deployment -kubectl -n unicorn-store-spring exec deploy/unicorn-store-spring -c unicorn-store-spring -- \ - sh -c 'unset JAVA_TOOL_OPTIONS; jcmd $(jcmd -l | grep -v JCmd | cut -d" " -f1) VM.flags' \ - | tr ' ' '\n' | grep -E 'MaxHeapSize|InitialHeapSize|UseSerialGC|ActiveProcessorCount' - git -C ~/environment/unicorn-store-spring add -A \ && git -C ~/environment/unicorn-store-spring commit -q -m "crac" +WS_TEST_BLOCK_300_11 + +ws_skip_block 12 'Roll out' 'Roll out' 322 322 'text' '' 'copy action disabled' +ws_run_block 13 'Score' 'In the Shell terminal:' 333 333 'bash' '' <<'WS_TEST_BLOCK_300_13' curl -sN "localhost:8090/api/v1/checklist/unicorn-store-spring?format=text&wait=true" -WS_TEST_BLOCK_300_9 +WS_TEST_BLOCK_300_13 -ws_run_block 10 'Where the restored start spends its CPU' 'Print the same link for the restored pod:' 324 330 'bash' '' <<'WS_TEST_BLOCK_300_10' +ws_run_block 14 'Where the restored start spends its CPU' 'Print the same link for the restored pod:' 343 349 'bash' '' <<'WS_TEST_BLOCK_300_14' POD=$(kubectl -n unicorn-store-spring get pod -l app=unicorn-store-spring \ --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}') START=$(date -d "$(kubectl -n unicorn-store-spring get pod $POD -o jsonpath='{.status.startTime}')" +%s) @@ -227,11 +230,9 @@ GRAFANA_URL=$(kubectl -n monitoring get svc grafana -o jsonpath="{.status.loadBa echo "http://${GRAFANA_URL}/a/grafana-pyroscope-app/explore?explorationType=flame-graph&var-dataSource=pyroscope\ &var-serviceName=unicorn-store-spring&var-profileMetricId=process_cpu:cpu:nanoseconds:cpu:nanoseconds\ &var-filters=pod%7C%3D%7C${POD}&from=${START}000&to=$((START + 90))000" -WS_TEST_BLOCK_300_10 - -ws_skip_block 11 'Where the restored start spends its CPU' 'Search for ClassLoader: 13% of the baseline pod'"'"'s window in this run, 1% after the restore. Search for CompileBroker: the JIT compiler threads still take about half of the window.' 339 348 'text' '' 'copy action disabled' +WS_TEST_BLOCK_300_14 -ws_skip_block 12 'The score' 'The score' 364 382 'text' '' 'copy action disabled' +ws_skip_block 15 'The score' 'The score' 366 384 'text' '' 'copy action disabled' ws_end_page @@ -243,10 +244,12 @@ WS_TEST_BLOCK_400_1 ws_skip_block 2 'Review the change' 'Review the change' 41 79 'text' '' 'copy action disabled' -ws_run_block 3 'Build, roll out, verify and score' 'The image tag stays :crac with a new digest, and the Deployment changed, so apply rolls out both. In the Shell terminal:' 100 119 'bash' '' 1500 <<'WS_TEST_BLOCK_400_3' -# code and heap flags changed: rebuild the checkpoint (≈ 2 min) +ws_run_block 3 'Build' 'The code and the heap flags changed, so the checkpoint is built again. In the Shell terminal:' 100 101 'bash' '' 1500 <<'WS_TEST_BLOCK_400_3' +# ≈ 2 min: builds, starts, warms up, checkpoints; pushes :crac ~/environment/unicorn-store-spring/scripts/build.sh crac Dockerfile.crac +WS_TEST_BLOCK_400_3 +ws_run_block 4 'Roll out' 'The image tag stays :crac with a new digest, and the Deployment changed, so apply rolls out both. In the Shell terminal:' 119 128 'bash' '' <<'WS_TEST_BLOCK_400_4' kubectl -n unicorn-store-spring apply -f ~/environment/unicorn-store-spring/k8s/deployment.yaml kubectl -n unicorn-store-spring rollout status deploy/unicorn-store-spring --timeout=300s @@ -255,20 +258,17 @@ kubectl -n unicorn-store-spring logs -c unicorn-store-spring $(kubectl -n unicor -l app=unicorn-store-spring --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}') \ | grep -E "Started StoreApplication|restored JVM" -# heap flags from the new checkpoint: 75 % / 50 % of the new limit -kubectl -n unicorn-store-spring exec deploy/unicorn-store-spring -c unicorn-store-spring -- \ - sh -c 'unset JAVA_TOOL_OPTIONS; jcmd $(jcmd -l | grep -v JCmd | cut -d" " -f1) VM.flags' \ - | tr ' ' '\n' | grep -E 'MaxHeapSize|InitialHeapSize' - git -C ~/environment/unicorn-store-spring add -A \ && git -C ~/environment/unicorn-store-spring commit -q -m "non-blocking publish, size again, shutdown budget" +WS_TEST_BLOCK_400_4 -curl -sN "localhost:8090/api/v1/checklist/unicorn-store-spring?format=text&wait=true" -WS_TEST_BLOCK_400_3 +ws_skip_block 5 'Roll out' 'Roll out' 133 133 'text' '' 'copy action disabled' -ws_skip_block 4 'The pool and the database' 'The pool and the database' 138 145 'text' '' 'copy action disabled' +ws_run_block 6 'Score' 'In the Shell terminal:' 142 142 'bash' '' <<'WS_TEST_BLOCK_400_6' +curl -sN "localhost:8090/api/v1/checklist/unicorn-store-spring?format=text&wait=true" +WS_TEST_BLOCK_400_6 -ws_skip_block 5 'The score' 'The score' 161 179 'text' '' 'copy action disabled' +ws_skip_block 7 'The score' 'The score' 161 179 'text' '' 'copy action disabled' ws_end_page diff --git a/infra/scripts/ws-test/samples/java-performance.py b/infra/scripts/ws-test/samples/java-performance.py index bafa532b..29b5d0f0 100644 --- a/infra/scripts/ws-test/samples/java-performance.py +++ b/infra/scripts/ws-test/samples/java-performance.py @@ -1,51 +1,50 @@ """Sample mapping for the java-performance content (used by refresh.py). -setup: expand 0 = the environment check. -baseline: expand 0 = the policy-view + attach block (rollout wait lines dropped), expand 2 = - the checklist answer as raw markdown. The profile-link block (block 3) has no expand. -improve-manifest, improve-code: two blocks each, the Claude question and the combined - roll-out block. Expand 0 = Claude's answer wrapped at 110 columns, expand 1 = the roll-out - block up to "Pod is" (code: only the build, rollout, startup and heap-flag lines), expand 2 = - the checklist from "Score:". -improve-image: the AOT question and its build block, the CRaC question and its build block, - then the profile link (no expand). Expands 0–4 = AOT answer, AOT build lines, CRaC answer, - CRaC build lines up to "Pod is", the checklist from "Score:". -enforce: expand 0 = Claude's answer, expand 1 = the apply block. +Each change in a module is split into short steps, each with its own block: Build (image +and code modules), Roll out (apply, rollout, startup line, commit), Score (the checklist). +Build blocks have no expand; Roll out expands drop the rollout wait lines; Score expands +start at "Score:". See refresh() for the blocks and expands per page. A block re-run by hand replaces its expand with that real output; this mapping then overwrites it on the next refresh, so re-run the block or restore the expand from git. """ -from refresh import drop, keep, since, until, wrap +from refresh import drop, since, wrap ROLLOUT_WAIT = r'^Waiting for deployment' -BUILD_LINES = ('[build] docker build', '[build] pushed', 'deployment', '20', '-XX') def refresh(s): s.set_expand('setup/index.en.md', 0, s.blocks('setup/index.en.md')[0]) + # baseline: read the policy, opt in, the rule (expand 2 is the rule YAML, not refreshed), + # the checklist question, the profile link (no expand). b = s.blocks('baseline/index.en.md') - s.set_expand('baseline/index.en.md', 0, drop(b[0], ROLLOUT_WAIT)) - s.set_expand('baseline/index.en.md', 2, b[2], raw=True) - - for page in ('improve-manifest', 'improve-code'): - md = page + '/index.en.md' - question, run = s.blocks(md) - s.set_expand(md, 0, wrap(question)) - head = drop(until(run, 'Pod is'), ROLLOUT_WAIT) - if page != 'improve-manifest': - head = keep(head, *BUILD_LINES) - s.set_expand(md, 1, head) - s.set_expand(md, 2, since(run, 'Score:')) - - # improve-image: AOT question + build block, CRaC question + build block, profile link. + s.set_expand('baseline/index.en.md', 0, b[0]) + s.set_expand('baseline/index.en.md', 1, drop(b[1], ROLLOUT_WAIT)) + s.set_expand('baseline/index.en.md', 3, b[3], raw=True) + + # manifest: question, roll out, score. + md = 'improve-manifest/index.en.md' + question, rollout, score = s.blocks(md) + s.set_expand(md, 0, wrap(question)) + s.set_expand(md, 1, drop(rollout, ROLLOUT_WAIT)) + s.set_expand(md, 2, since(score, 'Score:')) + + # image: AOT question, build, roll out; CRaC question, build, roll out, score; profile link. md = 'improve-image/index.en.md' - aot_q, aot_run, crac_q, crac_run = s.blocks(md)[:4] + aot_q, _, aot_rollout, crac_q, _, crac_rollout, crac_score = s.blocks(md)[:7] s.set_expand(md, 0, wrap(aot_q)) - s.set_expand(md, 1, keep(drop(aot_run, ROLLOUT_WAIT), *BUILD_LINES)) + s.set_expand(md, 1, drop(aot_rollout, ROLLOUT_WAIT)) s.set_expand(md, 2, wrap(crac_q)) - s.set_expand(md, 3, keep(drop(until(crac_run, 'Pod is'), ROLLOUT_WAIT), *BUILD_LINES)) - s.set_expand(md, 4, since(crac_run, 'Score:')) + s.set_expand(md, 3, drop(crac_rollout, ROLLOUT_WAIT)) + s.set_expand(md, 4, since(crac_score, 'Score:')) + + # code: question, build, roll out, score. + md = 'improve-code/index.en.md' + question, _, rollout, score = s.blocks(md) + s.set_expand(md, 0, wrap(question)) + s.set_expand(md, 1, drop(rollout, ROLLOUT_WAIT)) + s.set_expand(md, 2, since(score, 'Score:')) question, run = s.blocks('enforce/index.en.md') s.set_expand('enforce/index.en.md', 0, wrap(question))