Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion apps/perf-sensor/skills/java-on-eks-optimization/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,9 @@ Read the file and pass the values through. The *why* for each:
## 5. Answer format

Every answer, the policy one included, starts with the first `Root cause` line: no preamble
about the work you did, and no answer that only names the items it covered.
about the work you did, and no answer that only names the items it covered. Write it once,
after the last tool call, as plain text: no Markdown headings or bold labels, and no line
before `Root cause` such as a note that an edit is in place.

```
Root cause: <one line, with tool + signal>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ public JfrFacts collect(String podIP, String podName) {
JfrFacts parse(Path file, String podName) throws IOException {
Instant first = null, last = null;
ContainerConfig container = null;
Instant containerAt = null;
String jvmArgs = null;
int gcCount = 0;
double gcMax = 0, gcTotal = 0;
Expand All @@ -126,11 +127,19 @@ JfrFacts parse(Path file, String podName) throws IOException {
if (last == null || t.isAfter(last)) last = t;
}
switch (e.getEventType().getName()) {
case "jdk.ContainerConfiguration" -> container = new ContainerConfig(
(int) e.getLong("effectiveCpuCount"),
quotaCores(e),
e.getLong("memoryLimit") > 0 ? e.getLong("memoryLimit") / (1024.0 * 1024.0) : null,
e.getString("containerType"));
// One event per chunk, read from the cgroup at the chunk's start; every
// ring dump starts a chunk. Keep the earliest: the container as the JVM saw
// it in its first seconds (during a startup boost), not at the last dump.
case "jdk.ContainerConfiguration" -> {
if (container == null || (t != null && containerAt != null && t.isBefore(containerAt))) {
containerAt = t;
container = new ContainerConfig(
(int) e.getLong("effectiveCpuCount"),
quotaCores(e),
e.getLong("memoryLimit") > 0 ? e.getLong("memoryLimit") / (1024.0 * 1024.0) : null,
e.getString("containerType"));
}
}
case "jdk.JVMInformation" -> {
// null when the JVM was started without -XX flags: keep "" so the caller can tell
// "event seen, no args" from "ring unavailable" (null).
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@
* @param pod pod the ring was dumped from
* @param recordingStart first event timestamp in the dump (ISO-8601)
* @param recordingEnd last event timestamp in the dump (ISO-8601)
* @param container jdk.ContainerConfiguration — limits as the JVM read them
* @param container jdk.ContainerConfiguration — limits as the JVM read them, from the
* earliest event in the ring (its first chunk, seconds after start)
* @param jvmArgs the JVM tuning flags of jdk.JVMInformation.jvmArguments (incl. ENTRYPOINT/JAVA_TOOL_OPTIONS), {@link JvmOptions}
* @param gc jdk.GCPhasePause aggregate
* @param pinned jdk.VirtualThreadPinned aggregate (synchronized blocks pinning carriers)
Expand Down
72 changes: 36 additions & 36 deletions infra/scripts/ws-test/java-performance.sh
Original file line number Diff line number Diff line change
Expand Up @@ -68,9 +68,9 @@ kubectl get mutatingpolicy inject-perf-profiler -o yaml \
| yq '.spec.mutations[0].jsonPatch.expression' | grep -oE '"/spec/[a-zA-Z0-9/-]+"' | sort -u
WS_TEST_BLOCK_100_1

ws_skip_block 2 'The injection policy' 'The injection policy' 35 35 'text' '' 'copy action disabled'
ws_skip_block 2 'The injection policy' 'The injection policy' 35 41 'text' '' 'copy action disabled'

ws_run_block 3 'The injection policy' 'Opt the service in with one label, and compare the manifest with the pod:' 44 60 'bash' '' <<'WS_TEST_BLOCK_100_3'
ws_run_block 3 'The injection policy' 'Opt the service in with one label, and compare the manifest with the pod:' 50 66 'bash' '' <<'WS_TEST_BLOCK_100_3'
# opt in: one label on the pod template
yq -i '.spec.template.metadata.labels."perf-profile/sidecar" = "true"' ~/environment/unicorn-store-spring/k8s/deployment.yaml
kubectl -n unicorn-store-spring apply -f ~/environment/unicorn-store-spring/k8s/deployment.yaml
Expand All @@ -90,20 +90,20 @@ git -C ~/environment/unicorn-store-spring add -A \
&& git -C ~/environment/unicorn-store-spring commit -q -m "baseline: profiler attached"
WS_TEST_BLOCK_100_3

ws_skip_block 4 'The injection policy' 'The injection policy' 65 76 'text' '' 'copy action disabled'
ws_skip_block 4 'The injection policy' 'The injection policy' 71 75 'text' '' 'copy action disabled'

ws_run_block 5 'The sensor' 'The sensor' 89 90 'bash' '' <<'WS_TEST_BLOCK_100_5'
ws_run_block 5 'The sensor' 'The sensor' 88 89 'bash' '' <<'WS_TEST_BLOCK_100_5'
kubectl -n monitoring get configmap perf-sensor-checklist-rules -o jsonpath='{.data.checklist-rules\.yaml}' \
| yq '.items[] | select(.id == 6)'
WS_TEST_BLOCK_100_5

ws_skip_block 6 'The sensor' 'The sensor' 95 107 'yaml' '' 'copy action disabled'
ws_skip_block 6 'The sensor' 'The sensor' 94 106 'yaml' '' 'copy action disabled'

ws_run_block 7 'Ask for the baseline' 'In the Shell terminal:' 128 128 'bash' 'non-interactive' <<'WS_TEST_BLOCK_100_7'
ws_run_block 7 'Ask for the baseline' 'In the Shell terminal:' 127 127 'bash' 'non-interactive' <<'WS_TEST_BLOCK_100_7'
cd ~/environment && claude -c -p "How is unicorn-store-spring doing against best practices?"
WS_TEST_BLOCK_100_7

ws_run_block 8 'Where the start spends its CPU' 'In the Shell terminal, print a link to the CPU profile of the new pod'"'"'s first 90 seconds in Grafana:' 141 147 'bash' '' <<'WS_TEST_BLOCK_100_8'
ws_run_block 8 'Where the start spends its CPU' 'In the Shell terminal, print a link to the CPU profile of the new pod'"'"'s first 90 seconds in Grafana:' 140 146 'bash' '' <<'WS_TEST_BLOCK_100_8'
POD=$(kubectl -n unicorn-store-spring get pod -l app=unicorn-store-spring \
--sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}')
START=$(date -d "$(kubectl -n unicorn-store-spring get pod $POD -o jsonpath='{.status.startTime}')" +%s)
Expand All @@ -121,11 +121,11 @@ ws_run_block 1 'Improve the manifest' 'In the Shell terminal:' 23 23 'bash' 'non
cd ~/environment && claude -c -p "Right-size the memory and CPU of unicorn-store-spring and make it start faster without changing the image."
WS_TEST_BLOCK_200_1

ws_skip_block 2 'Review the change' 'Review the change' 49 74 'text' '' 'copy action disabled'
ws_skip_block 2 'Review the change' 'Review the change' 49 78 'text' '' 'copy action disabled'

ws_skip_block 3 'Review the change' '- resizePolicy with restartPolicy: NotRequired for CPU states that a CPU change needs no restart. It is the default on current Kubernetes, and the prerequisite for the resize the controller performs' 93 114 'yaml' '' 'copy action disabled'
ws_skip_block 3 'Review the change' '- resizePolicy with restartPolicy: NotRequired for CPU states that a CPU change needs no restart. It is the default on current Kubernetes, and the prerequisite for the resize the controller performs' 97 118 'yaml' '' 'copy action disabled'

ws_run_block 4 'Roll out' 'In the Shell terminal:' 126 136 'bash' '' <<'WS_TEST_BLOCK_200_4'
ws_run_block 4 'Roll out' 'In the Shell terminal:' 130 140 'bash' '' <<'WS_TEST_BLOCK_200_4'
kubectl -n unicorn-store-spring apply -f ~/environment/unicorn-store-spring/k8s/startup-cpu-boost.yaml
kubectl -n unicorn-store-spring apply -f ~/environment/unicorn-store-spring/k8s/deployment.yaml
kubectl -n unicorn-store-spring rollout status deploy/unicorn-store-spring --timeout=300s
Expand All @@ -139,9 +139,9 @@ git -C ~/environment/unicorn-store-spring add -A \
&& git -C ~/environment/unicorn-store-spring commit -q -m "right-size memory and cpu, startup cpu boost"
WS_TEST_BLOCK_200_4

ws_skip_block 5 'Roll out' 'Roll out' 141 141 'text' '' 'copy action disabled'
ws_skip_block 5 'Roll out' 'Roll out' 145 148 'text' '' 'copy action disabled'

ws_run_block 6 'Score' 'In the Shell terminal:' 152 157 'bash' '' <<'WS_TEST_BLOCK_200_6'
ws_run_block 6 'Score' 'In the Shell terminal:' 159 164 'bash' '' <<'WS_TEST_BLOCK_200_6'
# score the new pod: waits until it has run under load for 120 s, then measures
curl -sN "localhost:8090/api/v1/checklist/unicorn-store-spring?format=text&wait=true"

Expand All @@ -150,7 +150,7 @@ curl -s localhost:8090/api/v1/measure/unicorn-store-spring | jq -c '.jfr.contain
kubectl -n unicorn-store-spring exec deploy/unicorn-store-spring -c unicorn-store-spring -- cat /sys/fs/cgroup/cpu.max
WS_TEST_BLOCK_200_6

ws_skip_block 7 'The score' 'The score' 183 201 'text' '' 'copy action disabled'
ws_skip_block 7 'The score' 'The score' 190 210 'text' '' 'copy action disabled'

ws_end_page

Expand All @@ -160,14 +160,14 @@ ws_run_block 1 'Without changing the code' 'In the Shell terminal:' 25 25 'bash'
cd ~/environment && claude -c -p "Make unicorn-store-spring start faster without changing its code."
WS_TEST_BLOCK_300_1

ws_skip_block 2 'Review the change' 'Review the change' 43 57 'text' '' 'copy action disabled'
ws_skip_block 2 'Review the change' 'Review the change' 43 61 'text' '' 'copy action disabled'

ws_run_block 3 'Build' 'In the Shell terminal:' 68 69 'bash' '' 1500 <<'WS_TEST_BLOCK_300_3'
ws_run_block 3 'Build' 'In the Shell terminal:' 72 73 'bash' '' 1500 <<'WS_TEST_BLOCK_300_3'
# ≈ 1 min: builds the jar, trains and creates the cache; pushes :aot
~/environment/unicorn-store-spring/scripts/build.sh aot Dockerfile.aot
WS_TEST_BLOCK_300_3

ws_run_block 4 'Roll out' 'In the Shell terminal:' 77 89 'bash' '' <<'WS_TEST_BLOCK_300_4'
ws_run_block 4 'Roll out' 'In the Shell terminal:' 81 93 'bash' '' <<'WS_TEST_BLOCK_300_4'
# point the Deployment at the :aot tag and roll it out
yq -i '.spec.template.spec.containers[0].image |= sub(":[^:]+$"; ":aot")' \
~/environment/unicorn-store-spring/k8s/deployment.yaml
Expand All @@ -183,24 +183,24 @@ git -C ~/environment/unicorn-store-spring add -A \
&& git -C ~/environment/unicorn-store-spring commit -q -m "aot cache"
WS_TEST_BLOCK_300_4

ws_skip_block 5 'Roll out' 'Roll out' 94 94 'text' '' 'copy action disabled'
ws_skip_block 5 'Roll out' 'Roll out' 98 100 'text' '' 'copy action disabled'

ws_run_block 6 'With a code change' 'In the Shell terminal:' 115 115 'bash' 'non-interactive' <<'WS_TEST_BLOCK_300_6'
ws_run_block 6 'With a code change' 'In the Shell terminal:' 121 121 'bash' 'non-interactive' <<'WS_TEST_BLOCK_300_6'
cd ~/environment && claude -c -p "Can unicorn-store-spring start even faster if its code may change?"
WS_TEST_BLOCK_300_6

ws_skip_block 7 'Review the change' 'Review the change' 134 170 'text' '' 'copy action disabled'
ws_skip_block 7 'Review the change' 'Review the change' 140 164 'text' '' 'copy action disabled'

ws_skip_block 8 'Review the change' '- k8s/deployment.yaml: JAVATOOLOPTIONS removed. A restored JVM keeps the flags of the checkpoint, and a GC flag in the environment stops the restore' 183 226 'dockerfile' '' 'copy action disabled'
ws_skip_block 8 'Review the change' '- k8s/deployment.yaml: JAVATOOLOPTIONS removed. A restored JVM keeps the flags of the checkpoint, and a GC flag in the environment stops the restore' 177 220 'dockerfile' '' 'copy action disabled'

ws_skip_block 9 'Review the change' '- Line 44: -XX:CRaCRestoreFrom restores instead of starting. There are no GC or heap flags: they came with the checkpoint' 239 269 'java' '' 'copy action disabled'
ws_skip_block 9 'Review the change' '- Line 44: -XX:CRaCRestoreFrom restores instead of starting. There are no GC or heap flags: they came with the checkpoint' 233 263 'java' '' 'copy action disabled'

ws_run_block 10 'Build' 'In the Shell terminal:' 282 283 'bash' '' 1500 <<'WS_TEST_BLOCK_300_10'
# ≈ 2 min: builds, starts, warms up, checkpoints; pushes :crac
ws_run_block 10 'Build' 'In the Shell terminal:' 276 277 'bash' '' 1500 <<'WS_TEST_BLOCK_300_10'
# ≈ 1 min: builds, starts, warms up, checkpoints; pushes :crac
~/environment/unicorn-store-spring/scripts/build.sh crac Dockerfile.crac
WS_TEST_BLOCK_300_10

ws_run_block 11 'Roll out' 'In the Shell terminal:' 305 317 'bash' '' <<'WS_TEST_BLOCK_300_11'
ws_run_block 11 'Roll out' 'In the Shell terminal:' 299 311 'bash' '' <<'WS_TEST_BLOCK_300_11'
# point the Deployment at the :crac tag and roll it out
yq -i '.spec.template.spec.containers[0].image |= sub(":[^:]+$"; ":crac")' \
~/environment/unicorn-store-spring/k8s/deployment.yaml
Expand All @@ -216,13 +216,13 @@ git -C ~/environment/unicorn-store-spring add -A \
&& git -C ~/environment/unicorn-store-spring commit -q -m "crac"
WS_TEST_BLOCK_300_11

ws_skip_block 12 'Roll out' 'Roll out' 322 322 'text' '' 'copy action disabled'
ws_skip_block 12 'Roll out' 'Roll out' 316 318 'text' '' 'copy action disabled'

ws_run_block 13 'Score' 'In the Shell terminal:' 333 333 'bash' '' <<'WS_TEST_BLOCK_300_13'
ws_run_block 13 'Score' 'In the Shell terminal:' 329 329 'bash' '' <<'WS_TEST_BLOCK_300_13'
curl -sN "localhost:8090/api/v1/checklist/unicorn-store-spring?format=text&wait=true"
WS_TEST_BLOCK_300_13

ws_run_block 14 'Where the restored start spends its CPU' 'Print the same link for the restored pod:' 343 349 'bash' '' <<'WS_TEST_BLOCK_300_14'
ws_run_block 14 'Where the restored start spends its CPU' 'Print the same link for the restored pod:' 339 345 'bash' '' <<'WS_TEST_BLOCK_300_14'
POD=$(kubectl -n unicorn-store-spring get pod -l app=unicorn-store-spring \
--sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}')
START=$(date -d "$(kubectl -n unicorn-store-spring get pod $POD -o jsonpath='{.status.startTime}')" +%s)
Expand All @@ -232,7 +232,7 @@ echo "http://${GRAFANA_URL}/a/grafana-pyroscope-app/explore?explorationType=flam
&var-filters=pod%7C%3D%7C${POD}&from=${START}000&to=$((START + 90))000"
WS_TEST_BLOCK_300_14

ws_skip_block 15 'The score' 'The score' 366 384 'text' '' 'copy action disabled'
ws_skip_block 15 'The score' 'The score' 362 380 'text' '' 'copy action disabled'

ws_end_page

Expand All @@ -245,7 +245,7 @@ WS_TEST_BLOCK_400_1
ws_skip_block 2 'Review the change' 'Review the change' 41 79 'text' '' 'copy action disabled'

ws_run_block 3 'Build' 'The code and the heap flags changed, so the checkpoint is built again. In the Shell terminal:' 100 101 'bash' '' 1500 <<'WS_TEST_BLOCK_400_3'
# ≈ 2 min: builds, starts, warms up, checkpoints; pushes :crac
# ≈ 1 min: builds, starts, warms up, checkpoints; pushes :crac
~/environment/unicorn-store-spring/scripts/build.sh crac Dockerfile.crac
WS_TEST_BLOCK_400_3

Expand All @@ -262,13 +262,13 @@ git -C ~/environment/unicorn-store-spring add -A \
&& git -C ~/environment/unicorn-store-spring commit -q -m "non-blocking publish, size again, shutdown budget"
WS_TEST_BLOCK_400_4

ws_skip_block 5 'Roll out' 'Roll out' 133 133 'text' '' 'copy action disabled'
ws_skip_block 5 'Roll out' 'Roll out' 133 135 'text' '' 'copy action disabled'

ws_run_block 6 'Score' 'In the Shell terminal:' 142 142 'bash' '' <<'WS_TEST_BLOCK_400_6'
ws_run_block 6 'Score' 'In the Shell terminal:' 144 144 'bash' '' <<'WS_TEST_BLOCK_400_6'
curl -sN "localhost:8090/api/v1/checklist/unicorn-store-spring?format=text&wait=true"
WS_TEST_BLOCK_400_6

ws_skip_block 7 'The score' 'The score' 161 179 'text' '' 'copy action disabled'
ws_skip_block 7 'The score' 'The score' 163 181 'text' '' 'copy action disabled'

ws_end_page

Expand All @@ -278,11 +278,11 @@ ws_run_block 1 'Enforce the practices' 'In the Shell terminal:' 23 23 'bash' 'no
cd ~/environment && claude -c -p "Turn the checklist items that can be checked on the manifest into a Kyverno policy for unicorn-store-spring."
WS_TEST_BLOCK_500_1

ws_skip_block 2 'Review the policy' 'Review the policy' 44 66 'text' '' 'copy action disabled'
ws_skip_block 2 'Review the policy' 'Review the policy' 44 63 'text' '' 'copy action disabled'

ws_skip_block 3 'Review the policy' 'Two of the seven validations in k8s/policy.yaml:' 73 85 'yaml' '' 'copy action disabled'
ws_skip_block 3 'Review the policy' 'Two of the seven validations in k8s/policy.yaml:' 70 82 'yaml' '' 'copy action disabled'

ws_run_block 4 'Apply and test' 'In the Shell terminal:' 98 110 'bash' '' <<'WS_TEST_BLOCK_500_4'
ws_run_block 4 'Apply and test' 'In the Shell terminal:' 95 107 'bash' '' <<'WS_TEST_BLOCK_500_4'
kubectl apply -f ~/environment/unicorn-store-spring/k8s/policy.yaml

# the background scan reports the running Deployment
Expand All @@ -298,7 +298,7 @@ git -C ~/environment/unicorn-store-spring add -A \
&& git -C ~/environment/unicorn-store-spring commit -q -m "kyverno policy"
WS_TEST_BLOCK_500_4

ws_skip_block 5 'Apply and test' 'Apply and test' 115 119 'text' '' 'copy action disabled'
ws_skip_block 5 'Apply and test' 'Apply and test' 112 116 'text' '' 'copy action disabled'

ws_end_page

Expand Down
Loading