Feat/perf five module

.gitignore

@@ -46,3 +46,5 @@ dependency-reduced-pom.xml
 infra/scripts/ws-test/reports/
 
 .env
+
+__pycache__/

AGENTS.md

@@ -0,0 +1,42 @@
+# AGENTS.md
+
+Sample apps and the infrastructure for the Java on AWS workshops. The workshop content lives
+in sibling repositories (one per workshop, listed in `infra/workshops.json`), checked out next
+to this one.
+
+## Layout
+
+- `apps/`: sample applications and workshop tools (`unicorn-store-spring`, the
+  `perf-sensor` and `perf-profiler` samples, ...). The Java apps build with Maven; tests
+  run with `mvn test` in the directory with the `pom.xml`.
+- `infra/cdk/`: one CDK app that generates a CloudFormation template per workshop
+  (`infra/workshops.json`); `infra/cfn/` holds the generated templates.
+- `infra/scripts/ide/`: IDE bootstrap. `infra/scripts/templates/<workshop>.sh`: the
+  workshop's post-deploy on the IDE. `infra/scripts/deploy/<workshop>/`: the per-workshop
+  installs, plus `reset-app.sh` where a workshop can be reset to its starting point.
+- `infra/scripts/ws-test/`: runs a workshop's content as a test, and the operator tools for
+  a live environment. Read [its README](infra/scripts/ws-test/README.md) before a test run.
+
+## Conventions
+
+- Work on a branch cut from `origin/main` (`git checkout --no-track -b <name> origin/main`).
+  Never commit on `main`; changes reach it as squash-merged pull requests.
+- Commit messages: `type(scope): Summary` (`feat`, `fix`, `docs`, `test`, `chore`; scope =
+  app, workshop or area, for example `perf-sensor`, `java-performance`, `ws-test`).
+- Generated files are regenerated, never edited: `infra/cfn/*.yaml`
+  (`cd infra && npm run gen`), the content repos' `static/` templates and IAM policies
+  (`npm run sync`), `infra/scripts/ws-test/<workshop>.sh` (`node infra/scripts/ws-test/generate.mjs <workshop>`).
+- The IDE clones `main` at bootstrap: a branch reaches a test environment only through
+  `infra/scripts/ws-test/remote/sync-branch.sh`.
+- Workshops are independent: a change for one workshop must not change another's template,
+  scripts or policies. Shared code (the CDK constructs, `infra/scripts/lib/`, the apps) is
+  changed with all users in mind. java-on-amazon-eks runs several java-performance deploy
+  scripts through one-line wrappers (same app, sensor and profiler); a change there reaches
+  both workshops.
+- Shell scripts start with a header comment that says what the script does, its arguments
+  and its side effects; they source `infra/scripts/lib/common.sh` for logging where they run
+  on the IDE.
+- No account IDs, instance IDs, bucket names or credentials in code or docs. Credentials
+  for a test environment come from a git-ignored file (`WS_ENV_FILE`).
+- Treat any AWS account as production unless shown otherwise: read before you change, and
+  ask before deleting.

apps/perf-profiler/DumpServer.java

@@ -4,8 +4,8 @@
 // without a collector or `kubectl exec`.
 //
 //   GET /dump?kind=threads  -> jcmd <pid> Thread.dump_to_file -format=json  (virtual threads
-//                              included) -> the JSON file; 503 if the JVM did not write it
-//   GET /dump?kind=heap     -> jcmd <pid> GC.heap_info + VM.flags (text)
+//                              included) -> the JSON file; 503 if jcmd failed or wrote no file
+//   GET /dump?kind=heap     -> jcmd <pid> GC.heap_info + VM.flags (text); 503 if jcmd failed
 //   GET /dump?kind=jfr      -> jcmd <pid> JFR.dump name=perf (the 10-min ring started by
 //                              profile-loop.sh) -> the .jfr file bytes (application/octet-stream)
 //   GET /healthz            -> ok
@@ -16,7 +16,10 @@
 //
 // Files the TARGET writes (thread dump JSON, on-demand JFR) go to PERF_DIR (/perf), the
 // dedicated emptyDir the inject policy mounts in both containers — read here at the same
-// path. Fallback when the volume is absent: the target's /tmp via /proc/<pid>/root.
+// path. Fallback when the volume is absent: the target's /tmp via /proc/<pid>/root. Every
+// request writes its own file and deletes it after reading, and a non-zero jcmd exit is a
+// 503: a failed dump never serves an earlier one. Requests run on a small pool, so a slow
+// dump does not hold up /healthz or the next request; each jcmd has a 25 s deadline.
 import com.sun.net.httpserver.HttpExchange;
 import com.sun.net.httpserver.HttpServer;
 
@@ -28,17 +31,31 @@
 import java.nio.file.Path;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Stream;
 
 public class DumpServer {
 
+    private static final long JCMD_TIMEOUT_SECONDS = 25;
+    private static final AtomicLong REQUESTS = new AtomicLong();
+
+    /** jcmd's exit code (-1: did not finish or did not start) and its combined output. */
+    record Jcmd(int exit, String output) {
+        boolean ok() {
+            return exit == 0;
+        }
+    }
+
     public static void main(String[] args) throws IOException {
         int port = args.length > 0 ? Integer.parseInt(args[0]) : 9100;
         var server = HttpServer.create(new InetSocketAddress(port), 0);
         server.createContext("/healthz", ex -> respond(ex, 200, "ok\n"));
         server.createContext("/dump", DumpServer::handleDump);
-        server.setExecutor(null);
+        // The default executor runs every handler on the dispatcher thread: one 25 s jcmd would
+        // block /healthz and every other request behind it.
+        server.setExecutor(Executors.newFixedThreadPool(4));
         System.out.println("[dump-server] listening on :" + port);
         server.start();
     }
@@ -75,7 +92,15 @@ private static void handleDump(HttpExchange ex) throws IOException {
                     return;
                 }
             }
-            case "heap" -> body = jcmd(pid, "GC.heap_info") + "\n" + jcmd(pid, "VM.flags");
+            case "heap" -> {
+                var heap = jcmd(pid, "GC.heap_info");
+                var flags = heap.ok() ? jcmd(pid, "VM.flags") : heap;
+                if (!flags.ok()) {
+                    respond(ex, 503, "jcmd failed (exit " + flags.exit() + "): " + flags.output());
+                    return;
+                }
+                body = heap.output() + "\n" + flags.output();
+            }
             case "jfr" -> {
                 jfrDump(ex, pid);
                 return;
@@ -96,11 +121,12 @@ private static void handleDump(HttpExchange ex) throws IOException {
      * output when there is no such recording.
      */
     private static void jfrDump(HttpExchange ex, long pid) throws IOException {
-        String path = perfDir() + "/perf-ondemand-" + pid + ".jfr";
-        String out = jcmd(pid, "JFR.dump", "name=perf", "filename=" + path);
+        String path = uniquePath("perf-ondemand-" + pid, ".jfr");
+        var out = jcmd(pid, "JFR.dump", "name=perf", "filename=" + path);
         var onSidecar = readable(pid, path);
-        if (!Files.isReadable(onSidecar)) {
-            respond(ex, 503, "JFR.dump produced no file: " + out);
+        if (!out.ok() || !Files.isReadable(onSidecar)) {
+            Files.deleteIfExists(onSidecar);
+            respond(ex, 503, "JFR.dump failed (exit " + out.exit() + "): " + out.output());
             return;
         }
         try {
@@ -155,40 +181,70 @@ private static boolean isJava(long pid) {
      * Full JSON thread dump (jcmd Thread.dump_to_file -format=json) — enumerates VIRTUAL
      * threads and their stacks, unlike Thread.print. jcmd writes the file in the TARGET's
      * filesystem; we read it back at the shared /perf path (or /proc/&lt;pid&gt;/root/&lt;path&gt;).
-     * Returns null when the JVM did not write the file: the caller answers 503 so the sensor
-     * reports UNKNOWN instead of parsing a non-JSON fallback as "no blocked threads".
+     * Returns null when jcmd failed or the JVM wrote no file: the caller answers 503 so the
+     * sensor reports UNKNOWN instead of parsing a non-JSON fallback as "no blocked threads".
+     * The file is this request's own and is deleted after reading.
      */
     private static String threadDumpJson(long pid) {
-        String path = perfDir() + "/perf-threads-" + pid + ".json";
-        jcmd(pid, "Thread.dump_to_file", "-overwrite", "-format=json", path);
+        String path = uniquePath("perf-threads-" + pid, ".json");
         try {
-            var onSidecar = readable(pid, path);
-            return Files.isReadable(onSidecar) ? Files.readString(onSidecar) : null;
+            var out = jcmd(pid, "Thread.dump_to_file", "-format=json", path);
+            var onSidecar = readable(pid, path);   // after jcmd: which path exists depends on the file
+            return out.ok() && Files.isReadable(onSidecar) ? Files.readString(onSidecar) : null;
         } catch (Exception e) {
             return null;
+        } finally {
+            for (var f : List.of(Path.of(path), Path.of("/proc/" + pid + "/root" + path))) {
+                try {
+                    Files.deleteIfExists(f);
+                } catch (IOException ignored) {
+                    // best-effort cleanup
+                }
+            }
         }
     }
 
-    /** Run jcmd against the target with JAVA_TOOL_OPTIONS nulled (per the gotcha). */
-    private static String jcmd(long pid, String... command) {
+    /** A file name in PERF_DIR no other request uses. */
+    private static String uniquePath(String prefix, String suffix) {
+        return perfDir() + "/" + prefix + "-" + System.nanoTime() + "-" + REQUESTS.incrementAndGet() + suffix;
+    }
+
+    /**
+     * Run jcmd against the target with JAVA_TOOL_OPTIONS nulled (per the gotcha). The output is
+     * drained on its own thread so the deadline holds even when jcmd keeps stdout open.
+     */
+    static Jcmd jcmd(long pid, String... command) {
+        Process proc = null;
         try {
             var full = new java.util.ArrayList<String>(List.of("jcmd", String.valueOf(pid)));
             full.addAll(List.of(command));
             var pb = new ProcessBuilder(full);
             pb.environment().remove("JAVA_TOOL_OPTIONS");
             pb.redirectErrorStream(true);
-            var proc = pb.start();
-            byte[] out;
-            try (InputStream in = proc.getInputStream()) {
-                out = in.readAllBytes();
-            }
-            if (!proc.waitFor(25, TimeUnit.SECONDS)) {
+            proc = pb.start();
+            var out = new java.io.ByteArrayOutputStream();
+            InputStream in = proc.getInputStream();
+            var drain = Thread.ofVirtual().start(() -> {
+                try (in) {
+                    in.transferTo(out);
+                } catch (IOException ignored) {
+                    // the process was destroyed
+                }
+            });
+            if (!proc.waitFor(JCMD_TIMEOUT_SECONDS, TimeUnit.SECONDS)) {
                 proc.destroyForcibly();
-                return "jcmd " + String.join(" ", command) + " timed out\n";
+                drain.join(1000);
+                return new Jcmd(-1, "jcmd " + String.join(" ", command) + " timed out after " + JCMD_TIMEOUT_SECONDS + " s\n");
             }
-            return new String(out, StandardCharsets.UTF_8);
+            drain.join(5000);
+            return new Jcmd(proc.exitValue(), out.toString(StandardCharsets.UTF_8));
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            if (proc != null) proc.destroyForcibly();
+            return new Jcmd(-1, "jcmd " + String.join(" ", command) + " interrupted\n");
         } catch (Exception e) {
-            return "jcmd " + String.join(" ", command) + " failed: " + e + "\n";
+            if (proc != null) proc.destroyForcibly();
+            return new Jcmd(-1, "jcmd " + String.join(" ", command) + " failed: " + e + "\n");
         }
     }
 

apps/perf-profiler/README.md

@@ -4,12 +4,14 @@ Privilege-free JVM profiler sidecar. Amazon Corretto 25 +
 [async-profiler](https://github.com/async-profiler/async-profiler), baked into one image
 (checksum-verified at build; no runtime download). Companion of
 [`apps/perf-sensor`](../perf-sensor/README.md), which reads what this sidecar exposes.
+A sample for the Java on AWS workshops, not a supported product.
 
 ## What each part does
 
 | Part | One sentence |
 |---|---|
-| `k8s/sidecar-inject-policy.yaml` (Kyverno `MutatingPolicy`) | On pod creation with label `perf-profile/sidecar: "true"`, sets `shareProcessNamespace`, adds a `/perf` emptyDir to the app container and the sidecar (plus `/tmp` for the sidecar), and adds the sidecar container. |
+| `k8s/sidecar-inject-policy.yaml` (Kyverno `MutatingPolicy`) | On pod creation with label `perf-profile/sidecar: "true"`, sets `shareProcessNamespace`, adds a `/perf` emptyDir to the app container and the sidecar (plus `/tmp` for the sidecar), and adds the sidecar container. Only for a supported pod (below) without a sidecar yet and with none of the names, mount path or port the injection adds already in use. |
+| `k8s/sidecar-optin-policy.yaml` (Kyverno `ValidatingPolicy`) | Denies an opt-in the sidecar cannot serve, at the apply of the Deployment (autogen covers the pod controllers), with the reason. |
 | `profile-loop.sh` (entrypoint) | Finds the app JVM in the shared PID namespace by `libjvm.so` in its memory map (works for a CRaC-restored process), attaches async-profiler (`ctimer` + wall, 10 ms), starts a 10-minute JFR ring (`jcmd JFR.start name=perf maxage=10m`) inside the app JVM, and every 15 s posts the rotated JFR file to Pyroscope under `service_name` = the pod's `app` label. |
 | `DumpServer.java` (port 9100) | `GET /dump?kind=threads` → the JSON thread dump (virtual threads included) or 503; `kind=heap` → `GC.heap_info` + `VM.flags`; `kind=jfr` → the JFR ring file; all via `jcmd` on the app JVM, files exchanged through `/perf`. |
 
@@ -22,7 +24,10 @@ The app image is never modified: one label on the pod template opts a workload i
   dynamic attach (`jcmd`, `asprof`) requires the attaching process to have the target's
   UID. No root.
 - **`SYS_PTRACE` added, everything else dropped**: what async-profiler's `ctimer` engine
-  and `/proc/<pid>/root` access need to reach a sibling process.
+  and `/proc/<pid>/root` access need to reach a sibling process. Neither the `baseline` nor
+  the `restricted` [Pod Security Standard](https://kubernetes.io/docs/concepts/security/pod-security-standards/)
+  allows adding `SYS_PTRACE`: an opted-in pod needs a namespace that enforces neither (the
+  workshop's app namespace has no Pod Security label) or an exemption.
 - `allowPrivilegeEscalation: false`, `readOnlyRootFilesystem: true`, `RuntimeDefault`
   seccomp, `shareProcessNamespace: true` on the pod; no `privileged`, no `hostPID`.
 - Requests equal limits (250m / 384Mi), so injecting the sidecar leaves a Guaranteed pod
@@ -33,10 +38,13 @@ The app image is never modified: one label on the pod template opts a workload i
 
 ## Deploy
 
-`infra/scripts/deploy/java-on-amazon-eks/perf-profiler.sh` (`build`, `install`, or both)
+`infra/scripts/deploy/java-performance/perf-profiler.sh` (`build`, `install`, or both; java-on-amazon-eks runs it through a wrapper)
 builds and pushes the image, installs Kyverno (chart pinned), and applies the policy with the image pinned by digest,
 the cluster name, the app UID (`APP_UID`, default 1000) and the Pyroscope URL substituted.
-Then opt a workload in:
+A supported workload: the **first container is the app**, named like the pod's `app` label
+(the container the sensor reads), with **`securityContext.runAsUser` set to `APP_UID`** on the
+container or the pod (attach needs the app's UID; the image's `USER` is not visible at
+admission). Then opt it in:
 
 ```yaml
 # <app>/k8s/deployment.yaml
@@ -62,4 +70,7 @@ rather than a blocked rollout; check the container list after the first apply.
   `ASYNC_PROFILER_VERSION` **and** `ASYNC_PROFILER_SHA256` in the Dockerfile.
 - `PERF_DIR` (default `/perf`) is where the app JVM writes and the sidecar reads; without
   the volume the sidecar falls back to the target's `/tmp` through `/proc/<pid>/root`.
-- Replaces the retired privileged `perf-collector` DaemonSet.
+- In the java-performance and java-on-amazon-eks workshops it replaces the
+  `perf-collector` DaemonSet. The java-on-aws workshop still installs perf-collector
+  (`infra/scripts/deploy/java-on-aws/6-perf-platform-eks.sh`, `infra/scripts/setup/perf-platform.sh`),
+  which runs privileged, as root and with `hostPID`.

apps/perf-profiler/k8s/sidecar-inject-policy.yaml

@@ -25,14 +25,16 @@
 # The profiler is a BAKED image (Amazon Corretto 25 + async-profiler, built from
 # apps/perf-profiler) — no runtime download and no init container. The placeholders
 # below (__PERF_PROFILER_IMAGE__ pinned by digest, __CLUSTER_NAME__, __APP_UID__,
-# __PYROSCOPE_URL__) are substituted by deploy/java-on-amazon-eks/perf-profiler.sh.
+# __PYROSCOPE_URL__) are substituted by deploy/java-performance/perf-profiler.sh.
 #
 # CEL authoring notes (both verified on-cluster):
 #   1. patchType: ApplyConfiguration CANNOT set atomic fields (env.valueFrom.fieldRef,
 #      securityContext.capabilities.add) -> use JSONPatch.
 #   2. CEL will not widen heterogeneous map/list literals in a JSONPatch value
 #      (it unifies value types eagerly) -> every value is wrapped in dyn(...).
 #
+# Opting in also sets shareProcessNamespace: true on the pod (the sidecar must see the JVM).
+#
 # Prereq: Kyverno installed cluster-wide (by perf-profiler.sh / the bootstrap).
 # failurePolicy: Ignore so a Kyverno outage never blocks app pods from scheduling.
 apiVersion: policies.kyverno.io/v1beta1
@@ -65,6 +67,28 @@ spec:
         has(object.metadata.labels) &&
         'perf-profile/sidecar' in object.metadata.labels &&
         object.metadata.labels['perf-profile/sidecar'] == 'true'
+    # The supported workload: the first container is the app, named like the pod's `app`
+    # label (the container the sensor reads), and runs as __APP_UID__ set explicitly on the
+    # container or the pod (attach needs the same UID; the image's USER is not visible at
+    # admission). The perf-profiler-opt-in ValidatingPolicy rejects other opt-ins with a message.
+    - name: supported-app-container
+      expression: >-
+        has(object.metadata.labels) && 'app' in object.metadata.labels
+        && object.spec.containers[0].name == object.metadata.labels['app']
+        && (has(object.spec.containers[0].securityContext) && has(object.spec.containers[0].securityContext.runAsUser)
+            ? object.spec.containers[0].securityContext.runAsUser == __APP_UID__
+            : has(object.spec.securityContext) && has(object.spec.securityContext.runAsUser)
+              && object.spec.securityContext.runAsUser == __APP_UID__)
+    # Idempotent and composable: no sidecar yet, and nothing the injection adds is in use (the
+    # two volume names, /perf in the app container, port 9100). A pod that fails this one
+    # starts without the sidecar: check the container list after the first apply.
+    - name: not-injected-no-conflict
+      expression: >-
+        !object.spec.containers.exists(c, c.name == 'perf-profiler'
+          || (has(c.ports) && c.ports.exists(p, p.containerPort == 9100)))
+        && (!has(object.spec.volumes) || !object.spec.volumes.exists(v, v.name == 'perf-scratch' || v.name == 'perf-tmp'))
+        && (!has(object.spec.containers[0].volumeMounts)
+            || !object.spec.containers[0].volumeMounts.exists(m, m.mountPath == '/perf'))
   mutations:
     - patchType: JSONPatch
       jsonPatch: