diff --git a/.github/justfile b/.github/justfile index 73d61a8e986..0bd5133c648 100644 --- a/.github/justfile +++ b/.github/justfile @@ -44,6 +44,7 @@ test: python3 bench/compare.test.py python3 bench/summary.test.py python3 bench/profile.test.py + python3 bench/fanout.test.py python3 sh/gh/gst-formula.test.py python3 sh/changed.test.py python3 sh/large-files.test.py diff --git a/.github/relay-cpu-bisect.json b/.github/relay-cpu-bisect.json new file mode 100644 index 00000000000..b97fc71bf25 --- /dev/null +++ b/.github/relay-cpu-bisect.json @@ -0,0 +1,5243 @@ +{ + "version": 2, + "metric": "relay instructions (user and kernel, all threads) per delivered frame over a 5 s counter window after 7 s warmup", + "markers": { + "learn_end": "inclusive share of fixed-period instruction samples whose stack matches resume::Reader>::learn_end", + "poll_forward_drop": "inclusive share of fixed-period instruction samples whose stack matches poll_forward_drop", + "cursor_probe": "inclusive share of fixed-period instruction samples whose stack matches model::track::Cursor>::probe", + "syscall_leaf": "inclusive share of fixed-period instruction samples whose stack matches ;syscall$", + "mutex_leaf": "inclusive share of fixed-period instruction samples whose stack matches ;(lock|unlock)$" + }, + "runs": [ + { + "artifact": "run.eVjz2w", + "harness_revision": "82194441a92bb89fbc69f3759ef336d676fad643", + "workload": "chat", + "runtime": "io-uring-workers", + "workers": 4, + "generator_revision": "82194441a92bb89fbc69f3759ef336d676fad643", + "instruction_period": 10000019, + "sampling": "concurrent with the counter window", + "relay_cpus": "4-7", + "generator_cpus": "0-3", + "cells": [ + { + "revision": "2d73eea67d083ecf18938f4fc7c4842118eb87cf", + "label": "base-2d73eea67d08", + "round": 1, + "kinstr_per_frame": 121.8, + "kcycles_per_frame": 224.8, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 4.81, + "samples": 1209, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 8.9, + "cursor_probe": 0.0, + "syscall_leaf": 23.2, + "mutex_leaf": 16.3 + } + }, + { + "revision": "2d73eea67d083ecf18938f4fc7c4842118eb87cf", + "label": "base-2d73eea67d08", + "round": 2, + "kinstr_per_frame": 117.8, + "kcycles_per_frame": 232.6, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 5.87, + "samples": 1167, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 6.9, + "cursor_probe": 0.0, + "syscall_leaf": 20.1, + "mutex_leaf": 16.5 + } + }, + { + "revision": "2d73eea67d083ecf18938f4fc7c4842118eb87cf", + "label": "base-2d73eea67d08", + "round": 3, + "kinstr_per_frame": 122.6, + "kcycles_per_frame": 235.2, + "recv_fps": 19999, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 4.2, + "samples": 1216, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 6.6, + "cursor_probe": 0.0, + "syscall_leaf": 21.7, + "mutex_leaf": 16.0 + } + }, + { + "revision": "32c1e9940e5519a2bdc7c853d84846f55195cb94", + "label": "base-32c1e9940e55", + "round": 1, + "kinstr_per_frame": 123.4, + "kcycles_per_frame": 221.0, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 5.81, + "samples": 1229, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 8.0, + "cursor_probe": 0.0, + "syscall_leaf": 23.6, + "mutex_leaf": 17.8 + } + }, + { + "revision": "32c1e9940e5519a2bdc7c853d84846f55195cb94", + "label": "base-32c1e9940e55", + "round": 2, + "kinstr_per_frame": 115.8, + "kcycles_per_frame": 220.7, + "recv_fps": 20003, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 3.96, + "samples": 1150, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 8.5, + "cursor_probe": 0.0, + "syscall_leaf": 22.8, + "mutex_leaf": 19.7 + } + }, + { + "revision": "32c1e9940e5519a2bdc7c853d84846f55195cb94", + "label": "base-32c1e9940e55", + "round": 3, + "kinstr_per_frame": 117.3, + "kcycles_per_frame": 223.7, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 3.13, + "samples": 1168, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 8.0, + "cursor_probe": 0.0, + "syscall_leaf": 21.8, + "mutex_leaf": 17.6 + } + }, + { + "revision": "cbab2a8922c53a4541f919b89a95c131628f0cb0", + "label": "base-cbab2a8922c5", + "round": 1, + "kinstr_per_frame": 129.2, + "kcycles_per_frame": 241.4, + "recv_fps": 19997, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 5.81, + "samples": 1282, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 8.7, + "cursor_probe": 0.0, + "syscall_leaf": 24.1, + "mutex_leaf": 16.3 + } + }, + { + "revision": "cbab2a8922c53a4541f919b89a95c131628f0cb0", + "label": "base-cbab2a8922c5", + "round": 2, + "kinstr_per_frame": 121.7, + "kcycles_per_frame": 233.1, + "recv_fps": 19997, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 4.88, + "samples": 1210, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 8.8, + "cursor_probe": 0.0, + "syscall_leaf": 20.6, + "mutex_leaf": 18.1 + } + }, + { + "revision": "cbab2a8922c53a4541f919b89a95c131628f0cb0", + "label": "base-cbab2a8922c5", + "round": 3, + "kinstr_per_frame": 124.0, + "kcycles_per_frame": 244.0, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 4.37, + "samples": 1233, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 9.3, + "cursor_probe": 0.0, + "syscall_leaf": 23.5, + "mutex_leaf": 16.7 + } + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "label": "base-e84ffb3de313", + "round": 1, + "kinstr_per_frame": 133.4, + "kcycles_per_frame": 270.7, + "recv_fps": 20003, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 4.41, + "samples": 1327, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 6.9, + "cursor_probe": 0.0, + "syscall_leaf": 22.0, + "mutex_leaf": 16.9 + } + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "label": "base-e84ffb3de313", + "round": 2, + "kinstr_per_frame": 132.9, + "kcycles_per_frame": 282.7, + "recv_fps": 20001, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 6.04, + "samples": 1320, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 7.6, + "cursor_probe": 0.0, + "syscall_leaf": 21.4, + "mutex_leaf": 15.4 + } + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "label": "base-e84ffb3de313", + "round": 3, + "kinstr_per_frame": 135.4, + "kcycles_per_frame": 269.0, + "recv_fps": 20002, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 4.05, + "samples": 1346, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 7.6, + "cursor_probe": 0.0, + "syscall_leaf": 21.1, + "mutex_leaf": 16.5 + } + }, + { + "revision": "fe722cec9f8432e0e46cb633cc5e79e59e7724cb", + "label": "base-fe722cec9f84", + "round": 1, + "kinstr_per_frame": 117.5, + "kcycles_per_frame": 230.1, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 6.52, + "samples": 1166, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 8.1, + "cursor_probe": 0.0, + "syscall_leaf": 22.4, + "mutex_leaf": 16.8 + } + }, + { + "revision": "fe722cec9f8432e0e46cb633cc5e79e59e7724cb", + "label": "base-fe722cec9f84", + "round": 2, + "kinstr_per_frame": 114.6, + "kcycles_per_frame": 222.0, + "recv_fps": 20003, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 4.66, + "samples": 1139, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 8.5, + "cursor_probe": 0.0, + "syscall_leaf": 20.9, + "mutex_leaf": 18.1 + } + }, + { + "revision": "fe722cec9f8432e0e46cb633cc5e79e59e7724cb", + "label": "base-fe722cec9f84", + "round": 3, + "kinstr_per_frame": 120.5, + "kcycles_per_frame": 220.7, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 3.06, + "samples": 1200, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 9.0, + "cursor_probe": 0.0, + "syscall_leaf": 25.0, + "mutex_leaf": 17.9 + } + }, + { + "revision": "82194441a92bb89fbc69f3759ef336d676fad643", + "label": "current", + "round": 1, + "kinstr_per_frame": 146.9, + "kcycles_per_frame": 248.6, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 3.8, + "samples": 1461, + "markers": { + "learn_end": 0.5, + "poll_forward_drop": 14.9, + "cursor_probe": 6.4, + "syscall_leaf": 17.2, + "mutex_leaf": 17.5 + } + }, + { + "revision": "82194441a92bb89fbc69f3759ef336d676fad643", + "label": "current", + "round": 2, + "kinstr_per_frame": 145.2, + "kcycles_per_frame": 278.8, + "recv_fps": 20004, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 5.49, + "samples": 1447, + "markers": { + "learn_end": 0.5, + "poll_forward_drop": 13.3, + "cursor_probe": 6.7, + "syscall_leaf": 16.6, + "mutex_leaf": 18.9 + } + }, + { + "revision": "82194441a92bb89fbc69f3759ef336d676fad643", + "label": "current", + "round": 3, + "kinstr_per_frame": 147.1, + "kcycles_per_frame": 270.2, + "recv_fps": 20001, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 3.47, + "samples": 1465, + "markers": { + "learn_end": 0.5, + "poll_forward_drop": 15.1, + "cursor_probe": 6.8, + "syscall_leaf": 18.3, + "mutex_leaf": 17.2 + } + } + ] + }, + { + "artifact": "run.2DucdO", + "harness_revision": "82194441a92bb89fbc69f3759ef336d676fad643", + "workload": "chat", + "runtime": "io-uring-workers", + "workers": 4, + "generator_revision": "82194441a92bb89fbc69f3759ef336d676fad643", + "instruction_period": 10000019, + "sampling": "concurrent with the counter window", + "relay_cpus": "4-7", + "generator_cpus": "0-3", + "cells": [ + { + "revision": "2d73eea67d083ecf18938f4fc7c4842118eb87cf", + "label": "base-2d73eea67d08", + "round": 1, + "kinstr_per_frame": 123.1, + "kcycles_per_frame": 235.4, + "recv_fps": 20001, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 3.56, + "samples": 1221, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 7.9, + "cursor_probe": 0.0, + "syscall_leaf": 22.4, + "mutex_leaf": 16.7 + } + }, + { + "revision": "2d73eea67d083ecf18938f4fc7c4842118eb87cf", + "label": "base-2d73eea67d08", + "round": 2, + "kinstr_per_frame": 125.3, + "kcycles_per_frame": 247.9, + "recv_fps": 19999, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 3.19, + "samples": 1248, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 8.4, + "cursor_probe": 0.0, + "syscall_leaf": 24.8, + "mutex_leaf": 15.3 + } + }, + { + "revision": "2d73eea67d083ecf18938f4fc7c4842118eb87cf", + "label": "base-2d73eea67d08", + "round": 3, + "kinstr_per_frame": 114.9, + "kcycles_per_frame": 297.8, + "recv_fps": 19920, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 9.31, + "samples": 1138, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 6.4, + "cursor_probe": 0.0, + "syscall_leaf": 16.5, + "mutex_leaf": 17.9 + } + }, + { + "revision": "41cff87192983aae80db976d0058661525daa8ed", + "label": "base-41cff8719298", + "round": 1, + "kinstr_per_frame": 115.5, + "kcycles_per_frame": 221.3, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 4.57, + "samples": 1148, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 6.4, + "cursor_probe": 0.0, + "syscall_leaf": 19.7, + "mutex_leaf": 16.6 + } + }, + { + "revision": "41cff87192983aae80db976d0058661525daa8ed", + "label": "base-41cff8719298", + "round": 2, + "kinstr_per_frame": 123.6, + "kcycles_per_frame": 225.9, + "recv_fps": 20003, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 2.99, + "samples": 1228, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 6.8, + "cursor_probe": 0.0, + "syscall_leaf": 23.7, + "mutex_leaf": 17.5 + } + }, + { + "revision": "41cff87192983aae80db976d0058661525daa8ed", + "label": "base-41cff8719298", + "round": 3, + "kinstr_per_frame": 142.6, + "kcycles_per_frame": 412.5, + "recv_fps": 12490, + "groups_lost": 25854, + "host_load_1m": null, + "host_load_1m_logged": 30.78, + "samples": 888, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 4.3, + "cursor_probe": 0.0, + "syscall_leaf": 16.1, + "mutex_leaf": 17.3 + } + }, + { + "revision": "72259ab3cc3d3ed53a9f3dad8ff37caeac310b83", + "label": "base-72259ab3cc3d", + "round": 1, + "kinstr_per_frame": 126.5, + "kcycles_per_frame": 244.6, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 3.19, + "samples": 1259, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 8.7, + "cursor_probe": 0.0, + "syscall_leaf": 24.7, + "mutex_leaf": 15.3 + } + }, + { + "revision": "72259ab3cc3d3ed53a9f3dad8ff37caeac310b83", + "label": "base-72259ab3cc3d", + "round": 2, + "kinstr_per_frame": 127.0, + "kcycles_per_frame": 233.8, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 3.9, + "samples": 1262, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 9.0, + "cursor_probe": 0.0, + "syscall_leaf": 24.6, + "mutex_leaf": 15.1 + } + }, + { + "revision": "72259ab3cc3d3ed53a9f3dad8ff37caeac310b83", + "label": "base-72259ab3cc3d", + "round": 3, + "kinstr_per_frame": 120.2, + "kcycles_per_frame": 238.3, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 9.69, + "samples": 1193, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 8.4, + "cursor_probe": 0.0, + "syscall_leaf": 21.8, + "mutex_leaf": 16.7 + } + }, + { + "revision": "756637200cb10943871f6c51171f33a340addb6e", + "label": "base-756637200cb1", + "round": 1, + "kinstr_per_frame": 138.4, + "kcycles_per_frame": 267.2, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 3.97, + "samples": 1378, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 7.3, + "cursor_probe": 0.0, + "syscall_leaf": 21.9, + "mutex_leaf": 15.1 + } + }, + { + "revision": "756637200cb10943871f6c51171f33a340addb6e", + "label": "base-756637200cb1", + "round": 2, + "kinstr_per_frame": 133.9, + "kcycles_per_frame": 256.6, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 3.41, + "samples": 1330, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 6.4, + "cursor_probe": 0.0, + "syscall_leaf": 18.4, + "mutex_leaf": 16.9 + } + }, + { + "revision": "756637200cb10943871f6c51171f33a340addb6e", + "label": "base-756637200cb1", + "round": 3, + "kinstr_per_frame": 133.7, + "kcycles_per_frame": 266.5, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 15.73, + "samples": 1331, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 7.2, + "cursor_probe": 0.0, + "syscall_leaf": 21.5, + "mutex_leaf": 15.2 + } + }, + { + "revision": "7648ce03908df33c168f7471787f3572e87eeddb", + "label": "base-7648ce03908d", + "round": 1, + "kinstr_per_frame": 121.2, + "kcycles_per_frame": 231.2, + "recv_fps": 19998, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 4.22, + "samples": 1204, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 7.1, + "cursor_probe": 0.0, + "syscall_leaf": 22.8, + "mutex_leaf": 17.4 + } + }, + { + "revision": "7648ce03908df33c168f7471787f3572e87eeddb", + "label": "base-7648ce03908d", + "round": 2, + "kinstr_per_frame": 126.1, + "kcycles_per_frame": 230.6, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 2.31, + "samples": 1255, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 8.0, + "cursor_probe": 0.0, + "syscall_leaf": 25.7, + "mutex_leaf": 15.9 + } + }, + { + "revision": "7648ce03908df33c168f7471787f3572e87eeddb", + "label": "base-7648ce03908d", + "round": 3, + "kinstr_per_frame": 156.9, + "kcycles_per_frame": 384.1, + "recv_fps": 13977, + "groups_lost": 35553, + "host_load_1m": null, + "host_load_1m_logged": 55.74, + "samples": 1087, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 4.5, + "cursor_probe": 0.0, + "syscall_leaf": 20.8, + "mutex_leaf": 17.7 + } + }, + { + "revision": "a9a85580a4488a3ad3739d8d130ebb0f66e80510", + "label": "base-a9a85580a448", + "round": 1, + "kinstr_per_frame": 132.3, + "kcycles_per_frame": 268.1, + "recv_fps": 20001, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 4.9, + "samples": 1316, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 7.6, + "cursor_probe": 0.0, + "syscall_leaf": 22.1, + "mutex_leaf": 17.3 + } + }, + { + "revision": "a9a85580a4488a3ad3739d8d130ebb0f66e80510", + "label": "base-a9a85580a448", + "round": 2, + "kinstr_per_frame": 133.6, + "kcycles_per_frame": 262.5, + "recv_fps": 19999, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 3.31, + "samples": 1330, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 8.3, + "cursor_probe": 0.0, + "syscall_leaf": 22.3, + "mutex_leaf": 16.7 + } + }, + { + "revision": "a9a85580a4488a3ad3739d8d130ebb0f66e80510", + "label": "base-a9a85580a448", + "round": 3, + "kinstr_per_frame": 124.0, + "kcycles_per_frame": 283.8, + "recv_fps": 19998, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 18.89, + "samples": 1237, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 7.7, + "cursor_probe": 0.0, + "syscall_leaf": 17.3, + "mutex_leaf": 17.8 + } + }, + { + "revision": "b4dfafac8879ffc4cb661c0d47200aeccea7a2ba", + "label": "base-b4dfafac8879", + "round": 1, + "kinstr_per_frame": 130.1, + "kcycles_per_frame": 243.7, + "recv_fps": 20004, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 4.07, + "samples": 1290, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 9.5, + "cursor_probe": 0.0, + "syscall_leaf": 22.7, + "mutex_leaf": 15.4 + } + }, + { + "revision": "b4dfafac8879ffc4cb661c0d47200aeccea7a2ba", + "label": "base-b4dfafac8879", + "round": 2, + "kinstr_per_frame": 128.5, + "kcycles_per_frame": 229.6, + "recv_fps": 20001, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 3.77, + "samples": 1278, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 7.5, + "cursor_probe": 0.0, + "syscall_leaf": 24.2, + "mutex_leaf": 14.9 + } + }, + { + "revision": "b4dfafac8879ffc4cb661c0d47200aeccea7a2ba", + "label": "base-b4dfafac8879", + "round": 3, + "kinstr_per_frame": 117.7, + "kcycles_per_frame": 282.4, + "recv_fps": 20001, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 11.64, + "samples": 1170, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 5.9, + "cursor_probe": 0.0, + "syscall_leaf": 18.4, + "mutex_leaf": 18.3 + } + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "label": "base-e84ffb3de313", + "round": 1, + "kinstr_per_frame": 136.9, + "kcycles_per_frame": 260.8, + "recv_fps": 20003, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 3.43, + "samples": 1363, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 9.6, + "cursor_probe": 0.0, + "syscall_leaf": 22.5, + "mutex_leaf": 16.7 + } + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "label": "base-e84ffb3de313", + "round": 2, + "kinstr_per_frame": 134.9, + "kcycles_per_frame": 262.6, + "recv_fps": 20001, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 3.26, + "samples": 1341, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 7.8, + "cursor_probe": 0.0, + "syscall_leaf": 23.3, + "mutex_leaf": 15.5 + } + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "label": "base-e84ffb3de313", + "round": 3, + "kinstr_per_frame": 134.9, + "kcycles_per_frame": 267.8, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 11.14, + "samples": 1342, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 8.5, + "cursor_probe": 0.0, + "syscall_leaf": 20.1, + "mutex_leaf": 17.2 + } + }, + { + "revision": "82194441a92bb89fbc69f3759ef336d676fad643", + "label": "current", + "round": 1, + "kinstr_per_frame": 147.4, + "kcycles_per_frame": 264.3, + "recv_fps": 20001, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 3.09, + "samples": 1466, + "markers": { + "learn_end": 0.4, + "poll_forward_drop": 15.8, + "cursor_probe": 8.0, + "syscall_leaf": 17.9, + "mutex_leaf": 17.9 + } + }, + { + "revision": "82194441a92bb89fbc69f3759ef336d676fad643", + "label": "current", + "round": 2, + "kinstr_per_frame": 148.0, + "kcycles_per_frame": 261.4, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 3.13, + "samples": 1474, + "markers": { + "learn_end": 0.1, + "poll_forward_drop": 14.1, + "cursor_probe": 7.7, + "syscall_leaf": 18.7, + "mutex_leaf": 19.3 + } + }, + { + "revision": "82194441a92bb89fbc69f3759ef336d676fad643", + "label": "current", + "round": 3, + "kinstr_per_frame": 148.0, + "kcycles_per_frame": 266.1, + "recv_fps": 19997, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 7.96, + "samples": 1472, + "markers": { + "learn_end": 0.6, + "poll_forward_drop": 15.4, + "cursor_probe": 8.4, + "syscall_leaf": 16.6, + "mutex_leaf": 18.9 + } + } + ] + }, + { + "artifact": "run.BIL6LP", + "harness_revision": "d935d13e8dba7d36e06f836a04984d940dbf0a57", + "workload": "chat", + "runtime": "io-uring-workers", + "workers": 4, + "generator_revision": "d935d13e8dba7d36e06f836a04984d940dbf0a57", + "instruction_period": 2000003, + "sampling": "concurrent with the counter window", + "relay_cpus": "4-7", + "generator_cpus": "0-3", + "cells": [ + { + "revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", + "label": "base-0c2a798de5c2", + "round": 1, + "kinstr_per_frame": 160.0, + "kcycles_per_frame": 331.8, + "recv_fps": 19997, + "groups_lost": 0, + "host_load_1m": 7.69, + "host_load_1m_logged": 8.5, + "samples": 7997, + "markers": { + "learn_end": 14.7, + "poll_forward_drop": 9.7, + "cursor_probe": 0.0, + "syscall_leaf": 24.1, + "mutex_leaf": 16.7 + } + }, + { + "revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", + "label": "base-0c2a798de5c2", + "round": 2, + "kinstr_per_frame": 166.2, + "kcycles_per_frame": 323.7, + "recv_fps": 20003, + "groups_lost": 0, + "host_load_1m": 2.31, + "host_load_1m_logged": 2.29, + "samples": 8320, + "markers": { + "learn_end": 15.5, + "poll_forward_drop": 11.3, + "cursor_probe": 0.0, + "syscall_leaf": 27.1, + "mutex_leaf": 16.2 + } + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 1, + "kinstr_per_frame": 173.9, + "kcycles_per_frame": 310.8, + "recv_fps": 19997, + "groups_lost": 0, + "host_load_1m": 3.01, + "host_load_1m_logged": 3.01, + "samples": 8696, + "markers": { + "learn_end": 0.9, + "poll_forward_drop": 20.9, + "cursor_probe": 6.9, + "syscall_leaf": 21.9, + "mutex_leaf": 21.4 + } + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 2, + "kinstr_per_frame": 178.0, + "kcycles_per_frame": 316.8, + "recv_fps": 19997, + "groups_lost": 0, + "host_load_1m": 2.89, + "host_load_1m_logged": 2.74, + "samples": 8898, + "markers": { + "learn_end": 1.0, + "poll_forward_drop": 20.9, + "cursor_probe": 7.4, + "syscall_leaf": 23.1, + "mutex_leaf": 21.8 + } + }, + { + "revision": "2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3", + "label": "base-2fff7ea7ae4e", + "round": 1, + "kinstr_per_frame": 209.5, + "kcycles_per_frame": 415.5, + "recv_fps": 19997, + "groups_lost": 0, + "host_load_1m": 2.45, + "host_load_1m_logged": 2.43, + "samples": 10473, + "markers": { + "learn_end": 16.6, + "poll_forward_drop": 23.4, + "cursor_probe": 7.3, + "syscall_leaf": 31.6, + "mutex_leaf": 17.2 + } + }, + { + "revision": "2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3", + "label": "base-2fff7ea7ae4e", + "round": 2, + "kinstr_per_frame": 213.5, + "kcycles_per_frame": 430.9, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 2.55, + "host_load_1m_logged": 2.51, + "samples": 10671, + "markers": { + "learn_end": 16.6, + "poll_forward_drop": 23.4, + "cursor_probe": 7.7, + "syscall_leaf": 32.4, + "mutex_leaf": 17.4 + } + }, + { + "revision": "49b5ec9b5630c54f2c5e99657b114535087b7da2", + "label": "base-49b5ec9b5630", + "round": 1, + "kinstr_per_frame": 211.8, + "kcycles_per_frame": 415.6, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 4.47, + "host_load_1m_logged": 4.47, + "samples": 10583, + "markers": { + "learn_end": 13.1, + "poll_forward_drop": 19.1, + "cursor_probe": 8.2, + "syscall_leaf": 31.8, + "mutex_leaf": 17.5 + } + }, + { + "revision": "49b5ec9b5630c54f2c5e99657b114535087b7da2", + "label": "base-49b5ec9b5630", + "round": 2, + "kinstr_per_frame": 211.3, + "kcycles_per_frame": 395.3, + "recv_fps": 20003, + "groups_lost": 0, + "host_load_1m": 1.87, + "host_load_1m_logged": 2.03, + "samples": 10560, + "markers": { + "learn_end": 13.2, + "poll_forward_drop": 19.3, + "cursor_probe": 8.0, + "syscall_leaf": 31.2, + "mutex_leaf": 18.0 + } + }, + { + "revision": "55f6ed229250d17fc99389237659906d011d8a94", + "label": "base-55f6ed229250", + "round": 1, + "kinstr_per_frame": 161.1, + "kcycles_per_frame": 313.9, + "recv_fps": 19998, + "groups_lost": 0, + "host_load_1m": 5.46, + "host_load_1m_logged": 5.32, + "samples": 8050, + "markers": { + "learn_end": 15.6, + "poll_forward_drop": 10.9, + "cursor_probe": 0.0, + "syscall_leaf": 26.5, + "mutex_leaf": 18.6 + } + }, + { + "revision": "55f6ed229250d17fc99389237659906d011d8a94", + "label": "base-55f6ed229250", + "round": 2, + "kinstr_per_frame": 165.5, + "kcycles_per_frame": 320.9, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 3.07, + "host_load_1m_logged": 2.99, + "samples": 8282, + "markers": { + "learn_end": 15.9, + "poll_forward_drop": 11.0, + "cursor_probe": 0.0, + "syscall_leaf": 28.6, + "mutex_leaf": 16.7 + } + }, + { + "revision": "991572d636d264c6189a3477849d2969e689f55c", + "label": "base-991572d636d2", + "round": 1, + "kinstr_per_frame": 206.0, + "kcycles_per_frame": 402.2, + "recv_fps": 20003, + "groups_lost": 0, + "host_load_1m": 3.6, + "host_load_1m_logged": 3.6, + "samples": 10295, + "markers": { + "learn_end": 15.4, + "poll_forward_drop": 19.7, + "cursor_probe": 8.0, + "syscall_leaf": 36.2, + "mutex_leaf": 16.3 + } + }, + { + "revision": "991572d636d264c6189a3477849d2969e689f55c", + "label": "base-991572d636d2", + "round": 2, + "kinstr_per_frame": 207.0, + "kcycles_per_frame": 402.5, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 2.61, + "host_load_1m_logged": 2.51, + "samples": 10344, + "markers": { + "learn_end": 16.0, + "poll_forward_drop": 19.6, + "cursor_probe": 7.0, + "syscall_leaf": 35.0, + "mutex_leaf": 16.5 + } + }, + { + "revision": "e45064e3ccb43a535adb949b7db9a5a0aeb32413", + "label": "base-e45064e3ccb4", + "round": 1, + "kinstr_per_frame": 158.5, + "kcycles_per_frame": 316.3, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 5.21, + "host_load_1m_logged": 5.97, + "samples": 7921, + "markers": { + "learn_end": 14.5, + "poll_forward_drop": 10.0, + "cursor_probe": 0.0, + "syscall_leaf": 24.8, + "mutex_leaf": 17.0 + } + }, + { + "revision": "e45064e3ccb43a535adb949b7db9a5a0aeb32413", + "label": "base-e45064e3ccb4", + "round": 2, + "kinstr_per_frame": 168.4, + "kcycles_per_frame": 329.1, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 2.27, + "host_load_1m_logged": 2.27, + "samples": 8424, + "markers": { + "learn_end": 16.3, + "poll_forward_drop": 11.4, + "cursor_probe": 0.0, + "syscall_leaf": 27.9, + "mutex_leaf": 16.2 + } + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "label": "base-e84ffb3de313", + "round": 1, + "kinstr_per_frame": 136.2, + "kcycles_per_frame": 277.1, + "recv_fps": 19998, + "groups_lost": 0, + "host_load_1m": 6.92, + "host_load_1m_logged": 7.17, + "samples": 6802, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 7.5, + "cursor_probe": 0.0, + "syscall_leaf": 21.4, + "mutex_leaf": 16.4 + } + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "label": "base-e84ffb3de313", + "round": 2, + "kinstr_per_frame": 133.8, + "kcycles_per_frame": 258.5, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 3.14, + "host_load_1m_logged": 3.21, + "samples": 6691, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 7.5, + "cursor_probe": 0.0, + "syscall_leaf": 21.1, + "mutex_leaf": 16.6 + } + }, + { + "revision": "f206893047dc2f1db7b3422622dbb505470d0c70", + "label": "base-f206893047dc", + "round": 1, + "kinstr_per_frame": 168.5, + "kcycles_per_frame": 338.2, + "recv_fps": 20004, + "groups_lost": 0, + "host_load_1m": 9.21, + "host_load_1m_logged": 9.23, + "samples": 8424, + "markers": { + "learn_end": 16.6, + "poll_forward_drop": 10.8, + "cursor_probe": 0.0, + "syscall_leaf": 29.4, + "mutex_leaf": 16.6 + } + }, + { + "revision": "f206893047dc2f1db7b3422622dbb505470d0c70", + "label": "base-f206893047dc", + "round": 2, + "kinstr_per_frame": 165.9, + "kcycles_per_frame": 325.0, + "recv_fps": 19997, + "groups_lost": 0, + "host_load_1m": 2.56, + "host_load_1m_logged": 2.4, + "samples": 8293, + "markers": { + "learn_end": 16.4, + "poll_forward_drop": 11.2, + "cursor_probe": 0.0, + "syscall_leaf": 27.2, + "mutex_leaf": 16.5 + } + }, + { + "revision": "d935d13e8dba7d36e06f836a04984d940dbf0a57", + "label": "current", + "round": 1, + "kinstr_per_frame": 147.6, + "kcycles_per_frame": 260.9, + "recv_fps": 19997, + "groups_lost": 0, + "host_load_1m": 2.72, + "host_load_1m_logged": 2.83, + "samples": 7372, + "markers": { + "learn_end": 0.4, + "poll_forward_drop": 14.5, + "cursor_probe": 7.3, + "syscall_leaf": 17.8, + "mutex_leaf": 19.0 + } + }, + { + "revision": "d935d13e8dba7d36e06f836a04984d940dbf0a57", + "label": "current", + "round": 2, + "kinstr_per_frame": 147.0, + "kcycles_per_frame": 251.2, + "recv_fps": 19999, + "groups_lost": 0, + "host_load_1m": 2.99, + "host_load_1m_logged": 3.16, + "samples": 7347, + "markers": { + "learn_end": 0.5, + "poll_forward_drop": 14.5, + "cursor_probe": 6.9, + "syscall_leaf": 17.5, + "mutex_leaf": 18.9 + } + } + ] + }, + { + "artifact": "run.1KRu4S", + "harness_revision": "a6cd6f91b6a0c79a81f2a012a2683bc77bd6c7d7", + "workload": "chat", + "runtime": "io-uring-workers", + "workers": 4, + "generator_revision": "21d6da9fc1", + "instruction_period": 2000003, + "sampling": "concurrent with the counter window", + "relay_cpus": "4-7", + "generator_cpus": "0-3", + "cells": [ + { + "revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", + "label": "base-0c2a798de5c2", + "round": 1, + "kinstr_per_frame": 165.1, + "kcycles_per_frame": 318.0, + "recv_fps": 19999, + "groups_lost": 0, + "host_load_1m": 2.58, + "host_load_1m_logged": 2.58, + "samples": 8253, + "markers": { + "learn_end": 15.7, + "poll_forward_drop": 10.8, + "cursor_probe": 0.0, + "syscall_leaf": 26.7, + "mutex_leaf": 16.7 + } + }, + { + "revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", + "label": "base-0c2a798de5c2", + "round": 2, + "kinstr_per_frame": 166.1, + "kcycles_per_frame": 322.2, + "recv_fps": 20001, + "groups_lost": 0, + "host_load_1m": 4.23, + "host_load_1m_logged": 3.99, + "samples": 8304, + "markers": { + "learn_end": 15.0, + "poll_forward_drop": 10.7, + "cursor_probe": 0.0, + "syscall_leaf": 27.0, + "mutex_leaf": 17.0 + } + }, + { + "revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", + "label": "base-0c2a798de5c2", + "round": 3, + "kinstr_per_frame": 162.1, + "kcycles_per_frame": 327.9, + "recv_fps": 19997, + "groups_lost": 0, + "host_load_1m": 7.69, + "host_load_1m_logged": 7.69, + "samples": 8100, + "markers": { + "learn_end": 14.9, + "poll_forward_drop": 10.9, + "cursor_probe": 0.0, + "syscall_leaf": 26.4, + "mutex_leaf": 17.0 + } + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 1, + "kinstr_per_frame": 175.2, + "kcycles_per_frame": 327.1, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 3.17, + "host_load_1m_logged": 2.99, + "samples": 8769, + "markers": { + "learn_end": 1.3, + "poll_forward_drop": 22.0, + "cursor_probe": 7.4, + "syscall_leaf": 22.8, + "mutex_leaf": 20.7 + } + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 2, + "kinstr_per_frame": 173.5, + "kcycles_per_frame": 322.5, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 3.03, + "host_load_1m_logged": 3.04, + "samples": 8677, + "markers": { + "learn_end": 1.2, + "poll_forward_drop": 21.4, + "cursor_probe": 7.7, + "syscall_leaf": 21.9, + "mutex_leaf": 21.8 + } + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 3, + "kinstr_per_frame": 125.6, + "kcycles_per_frame": 279.7, + "recv_fps": 20569, + "groups_lost": 32897, + "host_load_1m": 24.77, + "host_load_1m_logged": 24.77, + "samples": 6509, + "markers": { + "learn_end": 0.4, + "poll_forward_drop": 14.9, + "cursor_probe": 6.5, + "syscall_leaf": 12.4, + "mutex_leaf": 23.5 + } + }, + { + "revision": "2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3", + "label": "base-2fff7ea7ae4e", + "round": 1, + "kinstr_per_frame": 207.1, + "kcycles_per_frame": 412.6, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 2.9, + "host_load_1m_logged": 2.98, + "samples": 10354, + "markers": { + "learn_end": 16.7, + "poll_forward_drop": 22.5, + "cursor_probe": 7.0, + "syscall_leaf": 29.5, + "mutex_leaf": 17.7 + } + }, + { + "revision": "2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3", + "label": "base-2fff7ea7ae4e", + "round": 2, + "kinstr_per_frame": 202.8, + "kcycles_per_frame": 412.8, + "recv_fps": 20001, + "groups_lost": 0, + "host_load_1m": 3.67, + "host_load_1m_logged": 4.13, + "samples": 10136, + "markers": { + "learn_end": 16.3, + "poll_forward_drop": 22.0, + "cursor_probe": 6.7, + "syscall_leaf": 29.1, + "mutex_leaf": 18.1 + } + }, + { + "revision": "2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3", + "label": "base-2fff7ea7ae4e", + "round": 3, + "kinstr_per_frame": 189.9, + "kcycles_per_frame": 401.7, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 11.36, + "host_load_1m_logged": 11.56, + "samples": 9511, + "markers": { + "learn_end": 15.0, + "poll_forward_drop": 21.0, + "cursor_probe": 7.0, + "syscall_leaf": 24.8, + "mutex_leaf": 19.0 + } + }, + { + "revision": "55f6ed229250d17fc99389237659906d011d8a94", + "label": "base-55f6ed229250", + "round": 1, + "kinstr_per_frame": 171.1, + "kcycles_per_frame": 332.6, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 2.19, + "host_load_1m_logged": 2.06, + "samples": 8555, + "markers": { + "learn_end": 17.3, + "poll_forward_drop": 11.6, + "cursor_probe": 0.0, + "syscall_leaf": 30.6, + "mutex_leaf": 16.7 + } + }, + { + "revision": "55f6ed229250d17fc99389237659906d011d8a94", + "label": "base-55f6ed229250", + "round": 2, + "kinstr_per_frame": 167.3, + "kcycles_per_frame": 337.6, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 3.78, + "host_load_1m_logged": 3.72, + "samples": 8366, + "markers": { + "learn_end": 16.0, + "poll_forward_drop": 10.1, + "cursor_probe": 0.0, + "syscall_leaf": 28.8, + "mutex_leaf": 16.7 + } + }, + { + "revision": "55f6ed229250d17fc99389237659906d011d8a94", + "label": "base-55f6ed229250", + "round": 3, + "kinstr_per_frame": 169.2, + "kcycles_per_frame": 337.9, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 5.82, + "host_load_1m_logged": 6.05, + "samples": 8458, + "markers": { + "learn_end": 16.8, + "poll_forward_drop": 11.5, + "cursor_probe": 0.0, + "syscall_leaf": 30.4, + "mutex_leaf": 16.4 + } + }, + { + "revision": "991572d636d264c6189a3477849d2969e689f55c", + "label": "base-991572d636d2", + "round": 1, + "kinstr_per_frame": 202.7, + "kcycles_per_frame": 392.8, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 2.06, + "host_load_1m_logged": 2.15, + "samples": 10130, + "markers": { + "learn_end": 15.3, + "poll_forward_drop": 19.7, + "cursor_probe": 7.3, + "syscall_leaf": 33.7, + "mutex_leaf": 16.7 + } + }, + { + "revision": "991572d636d264c6189a3477849d2969e689f55c", + "label": "base-991572d636d2", + "round": 2, + "kinstr_per_frame": 195.1, + "kcycles_per_frame": 379.0, + "recv_fps": 20002, + "groups_lost": 0, + "host_load_1m": 4.73, + "host_load_1m_logged": 4.91, + "samples": 9752, + "markers": { + "learn_end": 15.2, + "poll_forward_drop": 18.3, + "cursor_probe": 6.9, + "syscall_leaf": 32.3, + "mutex_leaf": 16.6 + } + }, + { + "revision": "991572d636d264c6189a3477849d2969e689f55c", + "label": "base-991572d636d2", + "round": 3, + "kinstr_per_frame": 184.4, + "kcycles_per_frame": 365.8, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 6.87, + "host_load_1m_logged": 7.05, + "samples": 9215, + "markers": { + "learn_end": 14.6, + "poll_forward_drop": 18.3, + "cursor_probe": 7.1, + "syscall_leaf": 27.8, + "mutex_leaf": 18.1 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 1, + "kinstr_per_frame": 118.7, + "kcycles_per_frame": 212.2, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 2.22, + "host_load_1m_logged": 2.44, + "samples": 5942, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 6.8, + "cursor_probe": 0.0, + "syscall_leaf": 25.2, + "mutex_leaf": 16.1 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 2, + "kinstr_per_frame": 111.5, + "kcycles_per_frame": 222.3, + "recv_fps": 19999, + "groups_lost": 0, + "host_load_1m": 4.8, + "host_load_1m_logged": 5.2, + "samples": 5579, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 7.4, + "cursor_probe": 0.0, + "syscall_leaf": 22.4, + "mutex_leaf": 18.1 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 3, + "kinstr_per_frame": 109.8, + "kcycles_per_frame": 235.5, + "recv_fps": 20003, + "groups_lost": 0, + "host_load_1m": 21.62, + "host_load_1m_logged": 24.0, + "samples": 5484, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 6.9, + "cursor_probe": 0.0, + "syscall_leaf": 21.3, + "mutex_leaf": 17.3 + } + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "label": "base-e84ffb3de313", + "round": 1, + "kinstr_per_frame": 135.0, + "kcycles_per_frame": 266.2, + "recv_fps": 19997, + "groups_lost": 0, + "host_load_1m": 2.84, + "host_load_1m_logged": 3.41, + "samples": 6746, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 7.8, + "cursor_probe": 0.0, + "syscall_leaf": 22.2, + "mutex_leaf": 16.0 + } + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "label": "base-e84ffb3de313", + "round": 2, + "kinstr_per_frame": 136.9, + "kcycles_per_frame": 267.9, + "recv_fps": 19997, + "groups_lost": 0, + "host_load_1m": 4.93, + "host_load_1m_logged": 5.19, + "samples": 6846, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 8.0, + "cursor_probe": 0.0, + "syscall_leaf": 22.6, + "mutex_leaf": 16.1 + } + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "label": "base-e84ffb3de313", + "round": 3, + "kinstr_per_frame": 116.1, + "kcycles_per_frame": 310.9, + "recv_fps": 22018, + "groups_lost": 31280, + "host_load_1m": 27.7, + "host_load_1m_logged": 27.7, + "samples": 6404, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 4.8, + "cursor_probe": 0.0, + "syscall_leaf": 16.0, + "mutex_leaf": 14.5 + } + }, + { + "revision": "a6cd6f91b6a0c79a81f2a012a2683bc77bd6c7d7", + "label": "current", + "round": 1, + "kinstr_per_frame": 145.6, + "kcycles_per_frame": 272.3, + "recv_fps": 19997, + "groups_lost": 0, + "host_load_1m": 9.76, + "host_load_1m_logged": 11.0, + "samples": 7274, + "markers": { + "learn_end": 0.5, + "poll_forward_drop": 14.8, + "cursor_probe": 6.8, + "syscall_leaf": 17.3, + "mutex_leaf": 19.5 + } + }, + { + "revision": "a6cd6f91b6a0c79a81f2a012a2683bc77bd6c7d7", + "label": "current", + "round": 2, + "kinstr_per_frame": 146.5, + "kcycles_per_frame": 281.0, + "recv_fps": 19998, + "groups_lost": 0, + "host_load_1m": 3.64, + "host_load_1m_logged": 3.63, + "samples": 7315, + "markers": { + "learn_end": 0.5, + "poll_forward_drop": 14.3, + "cursor_probe": 6.9, + "syscall_leaf": 16.8, + "mutex_leaf": 19.6 + } + }, + { + "revision": "a6cd6f91b6a0c79a81f2a012a2683bc77bd6c7d7", + "label": "current", + "round": 3, + "kinstr_per_frame": 143.1, + "kcycles_per_frame": 313.0, + "recv_fps": 19304, + "groups_lost": 0, + "host_load_1m": 32.43, + "host_load_1m_logged": 32.92, + "samples": 6910, + "markers": { + "learn_end": 0.4, + "poll_forward_drop": 14.1, + "cursor_probe": 7.4, + "syscall_leaf": 14.0, + "mutex_leaf": 19.5 + } + } + ] + }, + { + "artifact": "run.qYPSii", + "harness_revision": "74f46e79b75bb2cbb4c5d4c4d75e390249ba22d5", + "workload": "chat", + "runtime": "io-uring-workers", + "workers": 4, + "generator_revision": "21d6da9fc1", + "instruction_period": 2000003, + "sampling": "after the counter window", + "relay_cpus": "4-7", + "generator_cpus": "0-3", + "cells": [ + { + "revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", + "label": "base-0c2a798de5c2", + "round": 1, + "kinstr_per_frame": 246.0, + "kcycles_per_frame": 599.4, + "recv_fps": 7525, + "groups_lost": 16750, + "host_load_1m": 45.62, + "host_load_1m_logged": 54.66, + "samples": 5208, + "markers": { + "learn_end": 8.2, + "poll_forward_drop": 11.1, + "cursor_probe": 0.0, + "syscall_leaf": 14.2, + "mutex_leaf": 15.9 + } + }, + { + "revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", + "label": "base-0c2a798de5c2", + "round": 2, + "kinstr_per_frame": 149.9, + "kcycles_per_frame": 304.2, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 19.15, + "host_load_1m_logged": 19.77, + "samples": 7163, + "markers": { + "learn_end": 12.1, + "poll_forward_drop": 8.9, + "cursor_probe": 0.0, + "syscall_leaf": 18.0, + "mutex_leaf": 20.2 + } + }, + { + "revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", + "label": "base-0c2a798de5c2", + "round": 3, + "kinstr_per_frame": 169.8, + "kcycles_per_frame": 325.1, + "recv_fps": 19999, + "groups_lost": 0, + "host_load_1m": 4.38, + "host_load_1m_logged": 4.75, + "samples": 8337, + "markers": { + "learn_end": 15.4, + "poll_forward_drop": 11.7, + "cursor_probe": 0.0, + "syscall_leaf": 27.5, + "mutex_leaf": 16.8 + } + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 1, + "kinstr_per_frame": 160.3, + "kcycles_per_frame": 305.2, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 17.94, + "host_load_1m_logged": 18.93, + "samples": 7895, + "markers": { + "learn_end": 0.9, + "poll_forward_drop": 19.7, + "cursor_probe": 7.6, + "syscall_leaf": 15.5, + "mutex_leaf": 23.8 + } + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 2, + "kinstr_per_frame": 167.9, + "kcycles_per_frame": 337.0, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 9.63, + "host_load_1m_logged": 9.43, + "samples": 8307, + "markers": { + "learn_end": 1.1, + "poll_forward_drop": 21.0, + "cursor_probe": 7.2, + "syscall_leaf": 19.2, + "mutex_leaf": 22.7 + } + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 3, + "kinstr_per_frame": 176.2, + "kcycles_per_frame": 328.6, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 3.1, + "host_load_1m_logged": 3.64, + "samples": 8758, + "markers": { + "learn_end": 1.3, + "poll_forward_drop": 21.2, + "cursor_probe": 7.3, + "syscall_leaf": 22.3, + "mutex_leaf": 20.7 + } + }, + { + "revision": "2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3", + "label": "base-2fff7ea7ae4e", + "round": 1, + "kinstr_per_frame": 169.1, + "kcycles_per_frame": 409.6, + "recv_fps": 17369, + "groups_lost": 5362, + "host_load_1m": 24.92, + "host_load_1m_logged": 23.96, + "samples": 9126, + "markers": { + "learn_end": 13.8, + "poll_forward_drop": 18.9, + "cursor_probe": 6.2, + "syscall_leaf": 17.4, + "mutex_leaf": 21.8 + } + }, + { + "revision": "2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3", + "label": "base-2fff7ea7ae4e", + "round": 2, + "kinstr_per_frame": 161.7, + "kcycles_per_frame": 349.3, + "recv_fps": 19820, + "groups_lost": 0, + "host_load_1m": 11.62, + "host_load_1m_logged": 10.84, + "samples": 8749, + "markers": { + "learn_end": 14.1, + "poll_forward_drop": 18.8, + "cursor_probe": 6.7, + "syscall_leaf": 19.0, + "mutex_leaf": 21.1 + } + }, + { + "revision": "2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3", + "label": "base-2fff7ea7ae4e", + "round": 3, + "kinstr_per_frame": 214.0, + "kcycles_per_frame": 415.4, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 2.35, + "host_load_1m_logged": 2.56, + "samples": 10709, + "markers": { + "learn_end": 17.0, + "poll_forward_drop": 24.0, + "cursor_probe": 7.6, + "syscall_leaf": 32.7, + "mutex_leaf": 16.9 + } + }, + { + "revision": "55f6ed229250d17fc99389237659906d011d8a94", + "label": "base-55f6ed229250", + "round": 1, + "kinstr_per_frame": 245.2, + "kcycles_per_frame": 655.0, + "recv_fps": 7433, + "groups_lost": 28655, + "host_load_1m": 53.37, + "host_load_1m_logged": 53.25, + "samples": 5224, + "markers": { + "learn_end": 7.2, + "poll_forward_drop": 8.3, + "cursor_probe": 0.0, + "syscall_leaf": 15.9, + "mutex_leaf": 16.9 + } + }, + { + "revision": "55f6ed229250d17fc99389237659906d011d8a94", + "label": "base-55f6ed229250", + "round": 2, + "kinstr_per_frame": 166.6, + "kcycles_per_frame": 340.8, + "recv_fps": 19997, + "groups_lost": 0, + "host_load_1m": 11.77, + "host_load_1m_logged": 12.27, + "samples": 8277, + "markers": { + "learn_end": 16.1, + "poll_forward_drop": 10.8, + "cursor_probe": 0.0, + "syscall_leaf": 27.6, + "mutex_leaf": 17.9 + } + }, + { + "revision": "55f6ed229250d17fc99389237659906d011d8a94", + "label": "base-55f6ed229250", + "round": 3, + "kinstr_per_frame": 174.6, + "kcycles_per_frame": 343.7, + "recv_fps": 19999, + "groups_lost": 0, + "host_load_1m": 3.23, + "host_load_1m_logged": 3.42, + "samples": 8758, + "markers": { + "learn_end": 17.5, + "poll_forward_drop": 11.7, + "cursor_probe": 0.0, + "syscall_leaf": 31.6, + "mutex_leaf": 16.3 + } + }, + { + "revision": "991572d636d264c6189a3477849d2969e689f55c", + "label": "base-991572d636d2", + "round": 1, + "kinstr_per_frame": 164.2, + "kcycles_per_frame": 331.5, + "recv_fps": 19825, + "groups_lost": 0, + "host_load_1m": 30.42, + "host_load_1m_logged": 31.94, + "samples": 8019, + "markers": { + "learn_end": 11.0, + "poll_forward_drop": 15.1, + "cursor_probe": 6.5, + "syscall_leaf": 17.6, + "mutex_leaf": 20.5 + } + }, + { + "revision": "991572d636d264c6189a3477849d2969e689f55c", + "label": "base-991572d636d2", + "round": 2, + "kinstr_per_frame": 181.6, + "kcycles_per_frame": 363.0, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 9.45, + "host_load_1m_logged": 9.45, + "samples": 9524, + "markers": { + "learn_end": 14.6, + "poll_forward_drop": 19.1, + "cursor_probe": 7.1, + "syscall_leaf": 30.3, + "mutex_leaf": 17.2 + } + }, + { + "revision": "991572d636d264c6189a3477849d2969e689f55c", + "label": "base-991572d636d2", + "round": 3, + "kinstr_per_frame": 204.7, + "kcycles_per_frame": 382.8, + "recv_fps": 20001, + "groups_lost": 0, + "host_load_1m": 2.13, + "host_load_1m_logged": 2.32, + "samples": 10294, + "markers": { + "learn_end": 15.5, + "poll_forward_drop": 20.1, + "cursor_probe": 7.7, + "syscall_leaf": 35.0, + "mutex_leaf": 16.6 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 1, + "kinstr_per_frame": 124.4, + "kcycles_per_frame": 287.4, + "recv_fps": 16559, + "groups_lost": 8554, + "host_load_1m": 21.13, + "host_load_1m_logged": 22.93, + "samples": 5763, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 5.3, + "cursor_probe": 0.0, + "syscall_leaf": 19.8, + "mutex_leaf": 16.7 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 2, + "kinstr_per_frame": 115.3, + "kcycles_per_frame": 219.5, + "recv_fps": 20001, + "groups_lost": 0, + "host_load_1m": 4.79, + "host_load_1m_logged": 5.05, + "samples": 5711, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 7.5, + "cursor_probe": 0.0, + "syscall_leaf": 23.9, + "mutex_leaf": 16.1 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 3, + "kinstr_per_frame": 117.7, + "kcycles_per_frame": 214.6, + "recv_fps": 19998, + "groups_lost": 0, + "host_load_1m": 2.38, + "host_load_1m_logged": 2.7, + "samples": 5823, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 7.1, + "cursor_probe": 0.0, + "syscall_leaf": 24.5, + "mutex_leaf": 16.0 + } + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "label": "base-e84ffb3de313", + "round": 1, + "kinstr_per_frame": 125.5, + "kcycles_per_frame": 294.0, + "recv_fps": 19998, + "groups_lost": 0, + "host_load_1m": 21.29, + "host_load_1m_logged": 22.36, + "samples": 6352, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 6.3, + "cursor_probe": 0.0, + "syscall_leaf": 16.8, + "mutex_leaf": 17.2 + } + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "label": "base-e84ffb3de313", + "round": 2, + "kinstr_per_frame": 128.2, + "kcycles_per_frame": 355.6, + "recv_fps": 19590, + "groups_lost": 0, + "host_load_1m": 24.24, + "host_load_1m_logged": 27.66, + "samples": 6356, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 5.0, + "cursor_probe": 0.0, + "syscall_leaf": 17.9, + "mutex_leaf": 15.0 + } + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "label": "base-e84ffb3de313", + "round": 3, + "kinstr_per_frame": 135.7, + "kcycles_per_frame": 264.2, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 2.24, + "host_load_1m_logged": 2.24, + "samples": 6774, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 7.6, + "cursor_probe": 0.0, + "syscall_leaf": 21.6, + "mutex_leaf": 16.2 + } + }, + { + "revision": "74f46e79b75bb2cbb4c5d4c4d75e390249ba22d5", + "label": "current", + "round": 1, + "kinstr_per_frame": 145.2, + "kcycles_per_frame": 278.0, + "recv_fps": 19999, + "groups_lost": 0, + "host_load_1m": 11.7, + "host_load_1m_logged": 11.76, + "samples": 7185, + "markers": { + "learn_end": 0.4, + "poll_forward_drop": 14.3, + "cursor_probe": 6.6, + "syscall_leaf": 15.5, + "mutex_leaf": 19.8 + } + }, + { + "revision": "74f46e79b75bb2cbb4c5d4c4d75e390249ba22d5", + "label": "current", + "round": 2, + "kinstr_per_frame": 147.0, + "kcycles_per_frame": 278.5, + "recv_fps": 20003, + "groups_lost": 0, + "host_load_1m": 6.81, + "host_load_1m_logged": 6.97, + "samples": 7385, + "markers": { + "learn_end": 0.6, + "poll_forward_drop": 15.5, + "cursor_probe": 7.1, + "syscall_leaf": 17.7, + "mutex_leaf": 18.0 + } + }, + { + "revision": "74f46e79b75bb2cbb4c5d4c4d75e390249ba22d5", + "label": "current", + "round": 3, + "kinstr_per_frame": 148.8, + "kcycles_per_frame": 269.2, + "recv_fps": 20001, + "groups_lost": 0, + "host_load_1m": 3.07, + "host_load_1m_logged": 3.07, + "samples": 7392, + "markers": { + "learn_end": 0.5, + "poll_forward_drop": 14.4, + "cursor_probe": 6.6, + "syscall_leaf": 17.7, + "mutex_leaf": 19.3 + } + } + ] + }, + { + "artifact": "run.16AJUb", + "harness_revision": "bd812a762c149a4345430b53530a597d261144b8", + "workload": "chat", + "runtime": "io-uring-workers", + "workers": 4, + "generator_revision": "21d6da9fc1", + "instruction_period": 2000003, + "sampling": "after the counter window", + "relay_cpus": "4-7", + "generator_cpus": "0-3", + "cells": [ + { + "revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", + "label": "base-0c2a798de5c2", + "round": 1, + "kinstr_per_frame": 158.5, + "kcycles_per_frame": 330.8, + "recv_fps": 20001, + "groups_lost": 0, + "host_load_1m": 13.91, + "host_load_1m_logged": 14.83, + "samples": 7839, + "markers": { + "learn_end": 14.9, + "poll_forward_drop": 10.7, + "cursor_probe": 0.0, + "syscall_leaf": 23.5, + "mutex_leaf": 17.1 + } + }, + { + "revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", + "label": "base-0c2a798de5c2", + "round": 2, + "kinstr_per_frame": 139.5, + "kcycles_per_frame": 330.1, + "recv_fps": 19753, + "groups_lost": 0, + "host_load_1m": 25.02, + "host_load_1m_logged": 25.76, + "samples": 6782, + "markers": { + "learn_end": 11.1, + "poll_forward_drop": 8.5, + "cursor_probe": 0.0, + "syscall_leaf": 15.1, + "mutex_leaf": 18.7 + } + }, + { + "revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", + "label": "base-0c2a798de5c2", + "round": 3, + "kinstr_per_frame": 140.8, + "kcycles_per_frame": 335.0, + "recv_fps": 19906, + "groups_lost": 0, + "host_load_1m": 21.62, + "host_load_1m_logged": 21.62, + "samples": 7065, + "markers": { + "learn_end": 12.0, + "poll_forward_drop": 9.0, + "cursor_probe": 0.0, + "syscall_leaf": 16.1, + "mutex_leaf": 19.6 + } + }, + { + "revision": "1e05129753c720d0b2318cedbab345d745227d2b", + "label": "base-1e05129753c7", + "round": 1, + "kinstr_per_frame": 216.4, + "kcycles_per_frame": 513.7, + "recv_fps": 9349, + "groups_lost": 22438, + "host_load_1m": 32.14, + "host_load_1m_logged": 32.14, + "samples": 5175, + "markers": { + "learn_end": 8.5, + "poll_forward_drop": 6.8, + "cursor_probe": 0.0, + "syscall_leaf": 13.7, + "mutex_leaf": 17.7 + } + }, + { + "revision": "1e05129753c720d0b2318cedbab345d745227d2b", + "label": "base-1e05129753c7", + "round": 2, + "kinstr_per_frame": 139.6, + "kcycles_per_frame": 328.5, + "recv_fps": 19696, + "groups_lost": 6849, + "host_load_1m": 29.39, + "host_load_1m_logged": 29.39, + "samples": 7166, + "markers": { + "learn_end": 12.1, + "poll_forward_drop": 8.4, + "cursor_probe": 0.0, + "syscall_leaf": 17.0, + "mutex_leaf": 19.3 + } + }, + { + "revision": "1e05129753c720d0b2318cedbab345d745227d2b", + "label": "base-1e05129753c7", + "round": 3, + "kinstr_per_frame": 150.4, + "kcycles_per_frame": 299.4, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 7.22, + "host_load_1m_logged": 7.12, + "samples": 7901, + "markers": { + "learn_end": 14.5, + "poll_forward_drop": 11.2, + "cursor_probe": 0.0, + "syscall_leaf": 24.7, + "mutex_leaf": 17.7 + } + }, + { + "revision": "45007f6fb14fb44adc02823a8035f4f39de8ab94", + "label": "base-45007f6fb14f", + "round": 1, + "kinstr_per_frame": 144.6, + "kcycles_per_frame": 353.6, + "recv_fps": 16712, + "groups_lost": 13701, + "host_load_1m": 23.05, + "host_load_1m_logged": 28.3, + "samples": 5098, + "markers": { + "learn_end": 9.1, + "poll_forward_drop": 6.8, + "cursor_probe": 0.0, + "syscall_leaf": 14.3, + "mutex_leaf": 18.0 + } + }, + { + "revision": "45007f6fb14fb44adc02823a8035f4f39de8ab94", + "label": "base-45007f6fb14f", + "round": 2, + "kinstr_per_frame": 156.9, + "kcycles_per_frame": 330.2, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 20.44, + "host_load_1m_logged": 22.17, + "samples": 7645, + "markers": { + "learn_end": 13.6, + "poll_forward_drop": 9.5, + "cursor_probe": 0.0, + "syscall_leaf": 21.8, + "mutex_leaf": 19.1 + } + }, + { + "revision": "45007f6fb14fb44adc02823a8035f4f39de8ab94", + "label": "base-45007f6fb14f", + "round": 3, + "kinstr_per_frame": 160.8, + "kcycles_per_frame": 329.1, + "recv_fps": 19998, + "groups_lost": 0, + "host_load_1m": 252.79, + "host_load_1m_logged": 254.35, + "samples": 7942, + "markers": { + "learn_end": 15.2, + "poll_forward_drop": 11.2, + "cursor_probe": 0.0, + "syscall_leaf": 25.0, + "mutex_leaf": 17.2 + } + }, + { + "revision": "87515b48c693457b7541db12c66836fd6fc6b1e8", + "label": "base-87515b48c693", + "round": 1, + "kinstr_per_frame": 141.7, + "kcycles_per_frame": 332.4, + "recv_fps": 19983, + "groups_lost": 0, + "host_load_1m": 31.75, + "host_load_1m_logged": 32.37, + "samples": 6666, + "markers": { + "learn_end": 11.0, + "poll_forward_drop": 8.6, + "cursor_probe": 0.0, + "syscall_leaf": 15.3, + "mutex_leaf": 20.4 + } + }, + { + "revision": "87515b48c693457b7541db12c66836fd6fc6b1e8", + "label": "base-87515b48c693", + "round": 2, + "kinstr_per_frame": 141.6, + "kcycles_per_frame": 303.1, + "recv_fps": 19765, + "groups_lost": 0, + "host_load_1m": 15.34, + "host_load_1m_logged": 16.9, + "samples": 6887, + "markers": { + "learn_end": 11.0, + "poll_forward_drop": 8.4, + "cursor_probe": 0.0, + "syscall_leaf": 14.7, + "mutex_leaf": 20.3 + } + }, + { + "revision": "87515b48c693457b7541db12c66836fd6fc6b1e8", + "label": "base-87515b48c693", + "round": 3, + "kinstr_per_frame": 169.8, + "kcycles_per_frame": 340.9, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 96.24, + "host_load_1m_logged": 112.61, + "samples": 8412, + "markers": { + "learn_end": 14.8, + "poll_forward_drop": 11.1, + "cursor_probe": 0.0, + "syscall_leaf": 28.2, + "mutex_leaf": 16.3 + } + }, + { + "revision": "8ee3c80894186483b3c0d8f7e84b37ab0f525390", + "label": "base-8ee3c8089418", + "round": 1, + "kinstr_per_frame": 141.9, + "kcycles_per_frame": 356.1, + "recv_fps": 15002, + "groups_lost": 38842, + "host_load_1m": 30.58, + "host_load_1m_logged": 31.49, + "samples": 6099, + "markers": { + "learn_end": 10.8, + "poll_forward_drop": 8.8, + "cursor_probe": 0.0, + "syscall_leaf": 12.7, + "mutex_leaf": 18.4 + } + }, + { + "revision": "8ee3c80894186483b3c0d8f7e84b37ab0f525390", + "label": "base-8ee3c8089418", + "round": 2, + "kinstr_per_frame": 152.6, + "kcycles_per_frame": 349.8, + "recv_fps": 20012, + "groups_lost": 0, + "host_load_1m": 29.14, + "host_load_1m_logged": 29.13, + "samples": 8262, + "markers": { + "learn_end": 15.4, + "poll_forward_drop": 11.5, + "cursor_probe": 0.0, + "syscall_leaf": 27.4, + "mutex_leaf": 16.7 + } + }, + { + "revision": "8ee3c80894186483b3c0d8f7e84b37ab0f525390", + "label": "base-8ee3c8089418", + "round": 3, + "kinstr_per_frame": 150.4, + "kcycles_per_frame": 303.5, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 10.23, + "host_load_1m_logged": 10.69, + "samples": 7586, + "markers": { + "learn_end": 13.8, + "poll_forward_drop": 9.4, + "cursor_probe": 0.0, + "syscall_leaf": 20.6, + "mutex_leaf": 17.5 + } + }, + { + "revision": "c3c88aec81350cece608599cef5c33322547d146", + "label": "base-c3c88aec8135", + "round": 1, + "kinstr_per_frame": 150.7, + "kcycles_per_frame": 322.3, + "recv_fps": 19997, + "groups_lost": 0, + "host_load_1m": 20.63, + "host_load_1m_logged": 22.23, + "samples": 7644, + "markers": { + "learn_end": 14.9, + "poll_forward_drop": 10.1, + "cursor_probe": 0.0, + "syscall_leaf": 22.2, + "mutex_leaf": 17.7 + } + }, + { + "revision": "c3c88aec81350cece608599cef5c33322547d146", + "label": "base-c3c88aec8135", + "round": 2, + "kinstr_per_frame": 123.9, + "kcycles_per_frame": 272.4, + "recv_fps": 22352, + "groups_lost": 10324, + "host_load_1m": 23.92, + "host_load_1m_logged": 24.14, + "samples": 6730, + "markers": { + "learn_end": 11.5, + "poll_forward_drop": 8.4, + "cursor_probe": 0.0, + "syscall_leaf": 15.8, + "mutex_leaf": 19.8 + } + }, + { + "revision": "c3c88aec81350cece608599cef5c33322547d146", + "label": "base-c3c88aec8135", + "round": 3, + "kinstr_per_frame": 160.4, + "kcycles_per_frame": 329.0, + "recv_fps": 20003, + "groups_lost": 0, + "host_load_1m": 36.05, + "host_load_1m_logged": 44.87, + "samples": 8046, + "markers": { + "learn_end": 15.6, + "poll_forward_drop": 10.2, + "cursor_probe": 0.0, + "syscall_leaf": 25.8, + "mutex_leaf": 16.7 + } + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "label": "base-e84ffb3de313", + "round": 1, + "kinstr_per_frame": 129.7, + "kcycles_per_frame": 275.4, + "recv_fps": 20002, + "groups_lost": 0, + "host_load_1m": 32.54, + "host_load_1m_logged": 32.54, + "samples": 6495, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 6.6, + "cursor_probe": 0.0, + "syscall_leaf": 19.1, + "mutex_leaf": 16.6 + } + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "label": "base-e84ffb3de313", + "round": 2, + "kinstr_per_frame": 131.0, + "kcycles_per_frame": 259.4, + "recv_fps": 19883, + "groups_lost": 0, + "host_load_1m": 15.61, + "host_load_1m_logged": 15.61, + "samples": 6344, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 6.3, + "cursor_probe": 0.0, + "syscall_leaf": 16.6, + "mutex_leaf": 17.1 + } + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "label": "base-e84ffb3de313", + "round": 3, + "kinstr_per_frame": 130.0, + "kcycles_per_frame": 267.5, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 9.84, + "host_load_1m_logged": 9.84, + "samples": 6231, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 5.7, + "cursor_probe": 0.0, + "syscall_leaf": 16.4, + "mutex_leaf": 17.0 + } + }, + { + "revision": "fd59eb18a50f6e0ac819e73d941a3832cbdac91f", + "label": "base-fd59eb18a50f", + "round": 1, + "kinstr_per_frame": 136.4, + "kcycles_per_frame": 357.7, + "recv_fps": 21611, + "groups_lost": 3490, + "host_load_1m": 20.4, + "host_load_1m_logged": 20.99, + "samples": 6836, + "markers": { + "learn_end": 10.5, + "poll_forward_drop": 8.2, + "cursor_probe": 0.0, + "syscall_leaf": 15.4, + "mutex_leaf": 18.7 + } + }, + { + "revision": "fd59eb18a50f6e0ac819e73d941a3832cbdac91f", + "label": "base-fd59eb18a50f", + "round": 2, + "kinstr_per_frame": 146.0, + "kcycles_per_frame": 364.8, + "recv_fps": 20004, + "groups_lost": 0, + "host_load_1m": 37.85, + "host_load_1m_logged": 37.14, + "samples": 7347, + "markers": { + "learn_end": 12.6, + "poll_forward_drop": 8.9, + "cursor_probe": 0.0, + "syscall_leaf": 17.4, + "mutex_leaf": 17.7 + } + }, + { + "revision": "fd59eb18a50f6e0ac819e73d941a3832cbdac91f", + "label": "base-fd59eb18a50f", + "round": 3, + "kinstr_per_frame": 159.4, + "kcycles_per_frame": 311.3, + "recv_fps": 20003, + "groups_lost": 0, + "host_load_1m": 7.45, + "host_load_1m_logged": 7.49, + "samples": 8229, + "markers": { + "learn_end": 14.9, + "poll_forward_drop": 10.2, + "cursor_probe": 0.0, + "syscall_leaf": 26.3, + "mutex_leaf": 16.3 + } + }, + { + "revision": "bd812a762c149a4345430b53530a597d261144b8", + "label": "current", + "round": 1, + "kinstr_per_frame": 144.6, + "kcycles_per_frame": 354.3, + "recv_fps": 19955, + "groups_lost": 0, + "host_load_1m": 15.03, + "host_load_1m_logged": 15.03, + "samples": 6179, + "markers": { + "learn_end": 0.3, + "poll_forward_drop": 12.2, + "cursor_probe": 6.4, + "syscall_leaf": 14.2, + "mutex_leaf": 19.0 + } + }, + { + "revision": "bd812a762c149a4345430b53530a597d261144b8", + "label": "current", + "round": 2, + "kinstr_per_frame": 141.5, + "kcycles_per_frame": 298.8, + "recv_fps": 20036, + "groups_lost": 0, + "host_load_1m": 29.44, + "host_load_1m_logged": 28.52, + "samples": 7167, + "markers": { + "learn_end": 0.3, + "poll_forward_drop": 13.6, + "cursor_probe": 7.4, + "syscall_leaf": 14.2, + "mutex_leaf": 20.2 + } + }, + { + "revision": "bd812a762c149a4345430b53530a597d261144b8", + "label": "current", + "round": 3, + "kinstr_per_frame": 142.7, + "kcycles_per_frame": 281.2, + "recv_fps": 19996, + "groups_lost": 0, + "host_load_1m": 16.58, + "host_load_1m_logged": 17.57, + "samples": 7166, + "markers": { + "learn_end": 0.3, + "poll_forward_drop": 14.8, + "cursor_probe": 6.7, + "syscall_leaf": 16.0, + "mutex_leaf": 18.4 + } + } + ] + }, + { + "artifact": "run.SkAD0k", + "harness_revision": "bd812a762c149a4345430b53530a597d261144b8", + "workload": "video", + "runtime": "io-uring-workers", + "workers": 4, + "generator_revision": "21d6da9fc1", + "instruction_period": 2000003, + "sampling": "after the counter window", + "relay_cpus": "4-7", + "generator_cpus": "0-3", + "cells": [ + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 1, + "kinstr_per_frame": 141.1, + "kcycles_per_frame": 330.6, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 5.42, + "host_load_1m_logged": 5.31, + "samples": 331, + "markers": { + "learn_end": 0.6, + "poll_forward_drop": 6.0, + "cursor_probe": 1.8, + "syscall_leaf": 35.6, + "mutex_leaf": 9.7 + } + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 2, + "kinstr_per_frame": 139.5, + "kcycles_per_frame": 336.0, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 7.27, + "host_load_1m_logged": 7.17, + "samples": 324, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 5.6, + "cursor_probe": 1.2, + "syscall_leaf": 33.6, + "mutex_leaf": 9.6 + } + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 3, + "kinstr_per_frame": 138.3, + "kcycles_per_frame": 339.8, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 7.29, + "host_load_1m_logged": 7.18, + "samples": 320, + "markers": { + "learn_end": 0.3, + "poll_forward_drop": 5.9, + "cursor_probe": 1.6, + "syscall_leaf": 32.8, + "mutex_leaf": 8.4 + } + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 4, + "kinstr_per_frame": 146.8, + "kcycles_per_frame": 455.4, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 26.0, + "host_load_1m_logged": 27.34, + "samples": 343, + "markers": { + "learn_end": 0.3, + "poll_forward_drop": 4.7, + "cursor_probe": 1.7, + "syscall_leaf": 37.6, + "mutex_leaf": 9.3 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 1, + "kinstr_per_frame": 109.6, + "kcycles_per_frame": 291.5, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 4.97, + "host_load_1m_logged": 5.13, + "samples": 260, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 1.5, + "cursor_probe": 0.0, + "syscall_leaf": 43.8, + "mutex_leaf": 9.6 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 2, + "kinstr_per_frame": 105.6, + "kcycles_per_frame": 273.9, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 6.76, + "host_load_1m_logged": 6.91, + "samples": 253, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 2.8, + "cursor_probe": 0.0, + "syscall_leaf": 42.7, + "mutex_leaf": 7.9 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 3, + "kinstr_per_frame": 109.9, + "kcycles_per_frame": 286.4, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 6.84, + "host_load_1m_logged": 7.46, + "samples": 256, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 1.2, + "cursor_probe": 0.0, + "syscall_leaf": 41.0, + "mutex_leaf": 7.4 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 4, + "kinstr_per_frame": 119.9, + "kcycles_per_frame": 437.1, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 12.82, + "host_load_1m_logged": 18.92, + "samples": 282, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 1.4, + "cursor_probe": 0.0, + "syscall_leaf": 42.6, + "mutex_leaf": 8.2 + } + }, + { + "revision": "bd812a762c149a4345430b53530a597d261144b8", + "label": "current", + "round": 1, + "kinstr_per_frame": 120.5, + "kcycles_per_frame": 278.0, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 7.16, + "host_load_1m_logged": 7.16, + "samples": 279, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 0.4, + "cursor_probe": 0.0, + "syscall_leaf": 35.1, + "mutex_leaf": 4.3 + } + }, + { + "revision": "bd812a762c149a4345430b53530a597d261144b8", + "label": "current", + "round": 2, + "kinstr_per_frame": 124.2, + "kcycles_per_frame": 302.8, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 7.5, + "host_load_1m_logged": 7.63, + "samples": 284, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 0.4, + "cursor_probe": 0.4, + "syscall_leaf": 39.8, + "mutex_leaf": 4.2 + } + }, + { + "revision": "bd812a762c149a4345430b53530a597d261144b8", + "label": "current", + "round": 3, + "kinstr_per_frame": 125.6, + "kcycles_per_frame": 285.4, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 7.8, + "host_load_1m_logged": 7.8, + "samples": 288, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 0.3, + "cursor_probe": 0.0, + "syscall_leaf": 38.9, + "mutex_leaf": 3.1 + } + }, + { + "revision": "bd812a762c149a4345430b53530a597d261144b8", + "label": "current", + "round": 4, + "kinstr_per_frame": 138.3, + "kcycles_per_frame": 364.1, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 39.8, + "host_load_1m_logged": 40.13, + "samples": 326, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 0.3, + "cursor_probe": 0.0, + "syscall_leaf": 46.3, + "mutex_leaf": 5.2 + } + } + ] + }, + { + "artifact": "run.IV47ua", + "harness_revision": "bd812a762c149a4345430b53530a597d261144b8", + "workload": "chat", + "runtime": "tokio-workers", + "workers": 4, + "generator_revision": "21d6da9fc1", + "instruction_period": 2000003, + "sampling": "after the counter window", + "relay_cpus": "4-7", + "generator_cpus": "0-3", + "cells": [ + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 1, + "kinstr_per_frame": 185.1, + "kcycles_per_frame": 366.0, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 13.99, + "host_load_1m_logged": 13.99, + "samples": 9276, + "markers": { + "learn_end": 0.9, + "poll_forward_drop": 17.1, + "cursor_probe": 6.4, + "syscall_leaf": 4.3, + "mutex_leaf": 24.8 + } + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 2, + "kinstr_per_frame": 189.6, + "kcycles_per_frame": 360.9, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 6.01, + "host_load_1m_logged": 6.21, + "samples": 9513, + "markers": { + "learn_end": 1.1, + "poll_forward_drop": 17.0, + "cursor_probe": 6.1, + "syscall_leaf": 6.5, + "mutex_leaf": 22.7 + } + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 3, + "kinstr_per_frame": 184.3, + "kcycles_per_frame": 361.2, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 7.76, + "host_load_1m_logged": 7.98, + "samples": 9265, + "markers": { + "learn_end": 1.0, + "poll_forward_drop": 17.1, + "cursor_probe": 6.2, + "syscall_leaf": 4.0, + "mutex_leaf": 24.1 + } + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 4, + "kinstr_per_frame": 187.1, + "kcycles_per_frame": 354.2, + "recv_fps": 19997, + "groups_lost": 0, + "host_load_1m": 6.29, + "host_load_1m_logged": 5.95, + "samples": 9465, + "markers": { + "learn_end": 0.8, + "poll_forward_drop": 18.0, + "cursor_probe": 6.8, + "syscall_leaf": 6.2, + "mutex_leaf": 23.3 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 1, + "kinstr_per_frame": 123.6, + "kcycles_per_frame": 273.2, + "recv_fps": 19998, + "groups_lost": 0, + "host_load_1m": 22.76, + "host_load_1m_logged": 22.7, + "samples": 6351, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 5.1, + "cursor_probe": 0.0, + "syscall_leaf": 0.3, + "mutex_leaf": 19.9 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 2, + "kinstr_per_frame": 131.1, + "kcycles_per_frame": 242.1, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 6.37, + "host_load_1m_logged": 7.05, + "samples": 6537, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 5.8, + "cursor_probe": 0.0, + "syscall_leaf": 2.8, + "mutex_leaf": 19.0 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 3, + "kinstr_per_frame": 131.1, + "kcycles_per_frame": 240.6, + "recv_fps": 20004, + "groups_lost": 0, + "host_load_1m": 5.3, + "host_load_1m_logged": 5.36, + "samples": 6642, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 6.9, + "cursor_probe": 0.0, + "syscall_leaf": 3.2, + "mutex_leaf": 20.7 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 4, + "kinstr_per_frame": 131.5, + "kcycles_per_frame": 241.5, + "recv_fps": 20003, + "groups_lost": 0, + "host_load_1m": 6.86, + "host_load_1m_logged": 7.27, + "samples": 6565, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 6.0, + "cursor_probe": 0.0, + "syscall_leaf": 2.6, + "mutex_leaf": 20.0 + } + }, + { + "revision": "bd812a762c149a4345430b53530a597d261144b8", + "label": "current", + "round": 1, + "kinstr_per_frame": 172.8, + "kcycles_per_frame": 324.4, + "recv_fps": 19996, + "groups_lost": 0, + "host_load_1m": 7.61, + "host_load_1m_logged": 8.28, + "samples": 8639, + "markers": { + "learn_end": 0.3, + "poll_forward_drop": 12.2, + "cursor_probe": 6.2, + "syscall_leaf": 1.6, + "mutex_leaf": 20.5 + } + }, + { + "revision": "bd812a762c149a4345430b53530a597d261144b8", + "label": "current", + "round": 2, + "kinstr_per_frame": 171.0, + "kcycles_per_frame": 362.7, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 11.35, + "host_load_1m_logged": 11.47, + "samples": 8545, + "markers": { + "learn_end": 0.3, + "poll_forward_drop": 12.4, + "cursor_probe": 6.3, + "syscall_leaf": 1.0, + "mutex_leaf": 20.1 + } + }, + { + "revision": "bd812a762c149a4345430b53530a597d261144b8", + "label": "current", + "round": 3, + "kinstr_per_frame": 172.2, + "kcycles_per_frame": 318.2, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 5.71, + "host_load_1m_logged": 5.76, + "samples": 8638, + "markers": { + "learn_end": 0.4, + "poll_forward_drop": 11.6, + "cursor_probe": 6.0, + "syscall_leaf": 1.9, + "mutex_leaf": 19.8 + } + }, + { + "revision": "bd812a762c149a4345430b53530a597d261144b8", + "label": "current", + "round": 4, + "kinstr_per_frame": 167.8, + "kcycles_per_frame": 370.0, + "recv_fps": 19845, + "groups_lost": 0, + "host_load_1m": 10.76, + "host_load_1m_logged": 10.88, + "samples": 8671, + "markers": { + "learn_end": 0.2, + "poll_forward_drop": 12.1, + "cursor_probe": 6.1, + "syscall_leaf": 1.3, + "mutex_leaf": 20.4 + } + } + ] + }, + { + "artifact": "run.Z04ENl", + "harness_revision": "bd812a762c149a4345430b53530a597d261144b8", + "workload": "video", + "runtime": "tokio-workers", + "workers": 4, + "generator_revision": "21d6da9fc1", + "instruction_period": 2000003, + "sampling": "after the counter window", + "relay_cpus": "4-7", + "generator_cpus": "0-3", + "cells": [ + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 1, + "kinstr_per_frame": 170.2, + "kcycles_per_frame": 535.3, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 25.25, + "host_load_1m_logged": 27.59, + "samples": 370, + "markers": { + "learn_end": 0.5, + "poll_forward_drop": 9.5, + "cursor_probe": 2.4, + "syscall_leaf": 0.0, + "mutex_leaf": 15.4 + } + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 2, + "kinstr_per_frame": 175.3, + "kcycles_per_frame": 483.3, + "recv_fps": 959, + "groups_lost": 0, + "host_load_1m": 30.01, + "host_load_1m_logged": 28.96, + "samples": 426, + "markers": { + "learn_end": 0.5, + "poll_forward_drop": 6.6, + "cursor_probe": 1.6, + "syscall_leaf": 0.0, + "mutex_leaf": 14.3 + } + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 3, + "kinstr_per_frame": 163.2, + "kcycles_per_frame": 519.2, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 33.02, + "host_load_1m_logged": 33.58, + "samples": 384, + "markers": { + "learn_end": 0.3, + "poll_forward_drop": 9.1, + "cursor_probe": 2.3, + "syscall_leaf": 0.0, + "mutex_leaf": 17.7 + } + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 4, + "kinstr_per_frame": 164.5, + "kcycles_per_frame": 386.2, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 39.42, + "host_load_1m_logged": 39.65, + "samples": 385, + "markers": { + "learn_end": 0.3, + "poll_forward_drop": 9.1, + "cursor_probe": 2.3, + "syscall_leaf": 0.3, + "mutex_leaf": 14.3 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 1, + "kinstr_per_frame": 148.2, + "kcycles_per_frame": 378.2, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 13.51, + "host_load_1m_logged": 17.53, + "samples": 357, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 2.5, + "cursor_probe": 0.0, + "syscall_leaf": 0.0, + "mutex_leaf": 12.0 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 2, + "kinstr_per_frame": 149.5, + "kcycles_per_frame": 302.2, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 18.81, + "host_load_1m_logged": 19.83, + "samples": 354, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 2.8, + "cursor_probe": 0.0, + "syscall_leaf": 0.0, + "mutex_leaf": 13.0 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 3, + "kinstr_per_frame": 163.9, + "kcycles_per_frame": 506.6, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 23.81, + "host_load_1m_logged": 26.39, + "samples": 383, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 3.1, + "cursor_probe": 0.0, + "syscall_leaf": 0.0, + "mutex_leaf": 12.5 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 4, + "kinstr_per_frame": 156.0, + "kcycles_per_frame": 556.9, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 39.7, + "host_load_1m_logged": 39.65, + "samples": 368, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 3.5, + "cursor_probe": 0.0, + "syscall_leaf": 0.0, + "mutex_leaf": 14.4 + } + }, + { + "revision": "bd812a762c149a4345430b53530a597d261144b8", + "label": "current", + "round": 1, + "kinstr_per_frame": 147.3, + "kcycles_per_frame": 372.5, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 28.85, + "host_load_1m_logged": 28.75, + "samples": 348, + "markers": { + "learn_end": 0.3, + "poll_forward_drop": 0.3, + "cursor_probe": 0.3, + "syscall_leaf": 0.0, + "mutex_leaf": 9.8 + } + }, + { + "revision": "bd812a762c149a4345430b53530a597d261144b8", + "label": "current", + "round": 2, + "kinstr_per_frame": 142.9, + "kcycles_per_frame": 322.9, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 23.15, + "host_load_1m_logged": 24.73, + "samples": 335, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 0.6, + "cursor_probe": 0.0, + "syscall_leaf": 0.0, + "mutex_leaf": 8.7 + } + }, + { + "revision": "bd812a762c149a4345430b53530a597d261144b8", + "label": "current", + "round": 3, + "kinstr_per_frame": 146.4, + "kcycles_per_frame": 428.6, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 17.83, + "host_load_1m_logged": 18.09, + "samples": 347, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 0.3, + "cursor_probe": 0.3, + "syscall_leaf": 0.0, + "mutex_leaf": 11.0 + } + }, + { + "revision": "bd812a762c149a4345430b53530a597d261144b8", + "label": "current", + "round": 4, + "kinstr_per_frame": 147.2, + "kcycles_per_frame": 330.8, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 24.93, + "host_load_1m_logged": 24.93, + "samples": 345, + "markers": { + "learn_end": 0.3, + "poll_forward_drop": 0.3, + "cursor_probe": 0.0, + "syscall_leaf": 0.0, + "mutex_leaf": 9.6 + } + } + ] + } + ], + "fanout_method": "just bench-fanout: user instructions per delivered group on one pinned thread, a run of many cycles less a run of zero, at the listed subscriber counts; model = the track model alone, session = publisher, relay origin, and viewers over the in-memory transport", + "fanout_runs": [ + { + "artifact": "hist", + "note": "model counts per revision with frame pointers forced (RUSTFLAGS=-C force-frame-pointers=yes); compare only within this list", + "frame_pointers": true, + "counts": [ + { + "revision": "dcd7d2cbb891dc1cf72c68295ba70fbd17a4ec15", + "subscribers_512": 4411.1, + "subscribers_64": 4540.9, + "subscribers_1": 12552.6 + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "subscribers_512": 4386.3, + "subscribers_64": 4512.5, + "subscribers_1": 12234.1 + }, + { + "revision": "fe722cec9f8432e0e46cb633cc5e79e59e7724cb", + "subscribers_512": 5313.0, + "subscribers_64": 5442.9, + "subscribers_1": 13455.1 + }, + { + "revision": "41cff87192983aae80db976d0058661525daa8ed", + "subscribers_512": 5612.2, + "subscribers_64": 5761.9, + "subscribers_1": 14591.6 + }, + { + "revision": "9477a589ccc5d0c3647df96788ebccd236523e19", + "subscribers_512": 5578.9, + "subscribers_64": 5723.1, + "subscribers_1": 14466.7 + }, + { + "revision": "2036225ed77a9cbea81d3e8c63f2912086a536b5", + "subscribers_512": 5604.1, + "subscribers_64": 5766.0, + "subscribers_1": 14607.6 + }, + { + "revision": "7648ce03908df33c168f7471787f3572e87eeddb", + "subscribers_512": 5787.2, + "subscribers_64": 5924.9, + "subscribers_1": 14778.8 + }, + { + "revision": "d44813cb703bf0d5366266ae8206f794856c69f8", + "subscribers_512": 5888.3, + "subscribers_64": 6041.8, + "subscribers_1": 15156.4 + }, + { + "revision": "b4dfafac8879ffc4cb661c0d47200aeccea7a2ba", + "subscribers_512": 5815.7, + "subscribers_64": 6041.8, + "subscribers_1": 15230.6 + }, + { + "revision": "28a8c9a7a665f5aef042a9a715e012bf17dac47a", + "subscribers_512": 5815.7, + "subscribers_64": 5969.6, + "subscribers_1": 15230.4 + }, + { + "revision": "9f9149812ce374bd66a14350d9a471f0150e691b", + "subscribers_512": 6016.2, + "subscribers_64": 6180.8, + "subscribers_1": 16033.6 + }, + { + "revision": "ce70844359e04f7d803111cd3763c21e7c85fdd5", + "subscribers_512": 6016.8, + "subscribers_64": 6180.5, + "subscribers_1": 16105.7 + }, + { + "revision": "a9a85580a4488a3ad3739d8d130ebb0f66e80510", + "subscribers_512": 6016.8, + "subscribers_64": 6108.5, + "subscribers_1": 16033.9 + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "subscribers_512": 5943.6, + "subscribers_64": 6108.5, + "subscribers_1": 16033.6 + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "subscribers_512": 12463.3, + "subscribers_64": 12670.9, + "subscribers_1": 26020.2 + }, + { + "revision": "a6f3d28216931c216d76b0ffd8e9ea2455741341", + "subscribers_512": 13681.1, + "subscribers_64": 13888.1, + "subscribers_1": 27299.5 + }, + { + "revision": "49e8eb0c6eb811521b5637a8d44a0947da294bc3", + "subscribers_512": 5170.5, + "subscribers_64": 5337.5, + "subscribers_1": 15445.4 + }, + { + "revision": "1bfbd89e1634ecd788c80f53aa85af1875a6fd97", + "subscribers_512": 5158.8, + "subscribers_64": 5320.8, + "subscribers_1": 15071.4 + }, + { + "revision": "1582fdc07d439b5a20a3a6caf8e3e865df23db72", + "subscribers_512": 5170.5, + "subscribers_64": 5337.5, + "subscribers_1": 15445.5 + }, + { + "revision": "318aa2f2e78e46d219dfca9b7d218e653137ecce", + "subscribers_512": 5166.8, + "subscribers_64": 5324.7, + "subscribers_1": 15111.8 + }, + { + "revision": "3d2530d74e7cbb23e9970d1ca0db6d898cece732", + "subscribers_512": 5166.8, + "subscribers_64": 5324.7, + "subscribers_1": 15111.8 + }, + { + "revision": "37b49d8042a6d8fc773045fd83e03621a4d52558", + "subscribers_512": 9342.4, + "subscribers_64": 9495.7, + "subscribers_1": 18970.1 + }, + { + "revision": "5b5fcf0116053b8c8d1fd957b30b5145bca03caa", + "subscribers_512": 9342.3, + "subscribers_64": 9495.2, + "subscribers_1": 18937.2 + }, + { + "revision": "fe08ce0f9ee4dd35566509b47a13308824f55890", + "subscribers_512": 9454.8, + "subscribers_64": 9637.0, + "subscribers_1": 21115.3 + }, + { + "revision": "c219c8db7755c5357eb9d47d873b6e9579a2be36", + "subscribers_512": 9693.7, + "subscribers_64": 9875.2, + "subscribers_1": 21298.0 + }, + { + "revision": "c764acebb90b5c102a8ea90207dd125ae4fd326b", + "subscribers_512": 9691.9, + "subscribers_64": 9874.3, + "subscribers_1": 21328.8 + }, + { + "revision": "3bbf2464a559ae453ac74013e969b803ea199a91", + "subscribers_512": 9777.2, + "subscribers_64": 9975.0, + "subscribers_1": 22322.1 + }, + { + "revision": "bdfa5b1d06394e05e30f167e297b45cc913af1f0", + "subscribers_512": 10151.3, + "subscribers_64": 10349.5, + "subscribers_1": 22700.9 + }, + { + "revision": "dffb2a43ba0f8da5a14bec35676f1d30ed978afd", + "subscribers_512": 10224.5, + "subscribers_64": 10417.9, + "subscribers_1": 22956.5 + }, + { + "revision": "267934b66dc097247fd878352bd8aae664ed873b", + "subscribers_512": 9927.4, + "subscribers_64": 10121.4, + "subscribers_1": 22684.1 + }, + { + "revision": "e01b0c0bdd4e0264b5e89ba0fe76c06d3b8caae7", + "subscribers_512": 9928.8, + "subscribers_64": 10133.1, + "subscribers_1": 22883.2 + }, + { + "revision": "145c61c62e10d764be6bf2e996591994495c2ba6", + "subscribers_512": 10081.0, + "subscribers_64": 10285.5, + "subscribers_1": 23084.6 + }, + { + "revision": "d44ee77b18b039fd8b72d410b82404ab8353db0e", + "subscribers_512": 10068.8, + "subscribers_64": 10273.5, + "subscribers_1": 23070.5 + }, + { + "revision": "14e9973dc8cd4c7fb932317f5dab7c3eb6bd58e5", + "subscribers_512": 11318.9, + "subscribers_64": 11517.4, + "subscribers_1": 24258.1 + }, + { + "revision": "3292489a46083d9ef75ac1ec83258fb040f59fba", + "subscribers_512": 12337.9, + "subscribers_64": 12542.6, + "subscribers_1": 25653.2 + }, + { + "revision": "7ad8d426e072a4b1f03483204ec2c9ae8d3d3ffb", + "subscribers_512": 12275.9, + "subscribers_64": 12480.5, + "subscribers_1": 25874.5 + }, + { + "revision": "64cba1337136c67ce77aaf6ae337f355c97a895e", + "subscribers_512": 12689.3, + "subscribers_64": 12872.9, + "subscribers_1": 26244.8 + }, + { + "revision": "ef2e423b93bfc189ed0305a7c6dee8453b590e5a", + "subscribers_512": 12302.0, + "subscribers_64": 12507.4, + "subscribers_1": 26015.0 + }, + { + "revision": "cc3c1961d849dd536dc203724bf6b6cca46f0d88", + "subscribers_512": 13213.4, + "subscribers_64": 13417.7, + "subscribers_1": 26900.8 + }, + { + "revision": "c9a43ad7b235932716fcb01879468f051a352e2b", + "subscribers_512": 13681.1, + "subscribers_64": 13888.0, + "subscribers_1": 27286.8 + } + ] + }, + { + "artifact": "run.vi0R1a", + "note": "model sweep of moq-net and kio commits in the large steps after e84ffb3de3", + "harness_revision": "44c0dc494ff3262de6f8bad38230f8ff09e6ebf1", + "frame_pointers": false, + "counts": [ + { + "label": "base-aff2e253a8ae", + "revision": "aff2e253a8aeb2de6f8de929ff11c5153ac58426", + "scenario": "model", + "subscribers": 512, + "per_group": 5010.427900300665 + }, + { + "label": "base-caff81eb7f64", + "revision": "caff81eb7f64fff28c554cf218ab393da182f55a", + "scenario": "model", + "subscribers": 512, + "per_group": 9274.04038890577 + }, + { + "label": "base-47bbe49ccd43", + "revision": "47bbe49ccd4324167274d9628d9d6f1d4d5a11aa", + "scenario": "model", + "subscribers": 512, + "per_group": 9274.040661783522 + }, + { + "label": "base-109235732c1f", + "revision": "109235732c1fec1a6e0284b3bd77d51fcc1074c2", + "scenario": "model", + "subscribers": 512, + "per_group": 9274.040714759787 + }, + { + "label": "base-5ad3652871f7", + "revision": "5ad3652871f7b62736e5cce0ca0338d2b162f3ee", + "scenario": "model", + "subscribers": 512, + "per_group": 9062.04057482248 + }, + { + "label": "base-2832aeb7b6c0", + "revision": "2832aeb7b6c06db26a26ac7437776a7a683d782e", + "scenario": "model", + "subscribers": 512, + "per_group": 9062.034914358368 + }, + { + "label": "base-fa59050f7064", + "revision": "fa59050f70642386bebcd7f0a043a2635dcc40d9", + "scenario": "model", + "subscribers": 512, + "per_group": 9062.533816850051 + }, + { + "label": "base-3a402c9b949c", + "revision": "3a402c9b949cf24822db1216473cbdf172d2e7bb", + "scenario": "model", + "subscribers": 512, + "per_group": 9062.533618938716 + }, + { + "label": "base-d4cf2f0ce097", + "revision": "d4cf2f0ce097a051acac3c62848e2b345ff8c7ab", + "scenario": "model", + "subscribers": 512, + "per_group": 9061.956926297018 + }, + { + "label": "base-e3a1cdf2328f", + "revision": "e3a1cdf2328f853254c467b2b3c9c7327753d31a", + "scenario": "model", + "subscribers": 512, + "per_group": 9813.631270190954 + }, + { + "label": "base-0c8b0924b579", + "revision": "0c8b0924b579da4c7fc34f9af40f866d0e4cd338", + "scenario": "model", + "subscribers": 512, + "per_group": 9813.631249200358 + }, + { + "label": "base-30f23c3f8af0", + "revision": "30f23c3f8af0b46eeebff687bfa45397ed70cf0a", + "scenario": "model", + "subscribers": 512, + "per_group": 9813.631152243795 + }, + { + "label": "base-deee150f8f64", + "revision": "deee150f8f6400c14632627601960a89fd8e2b26", + "scenario": "model", + "subscribers": 512, + "per_group": 10967.270665741748 + }, + { + "label": "base-69b44ddd2218", + "revision": "69b44ddd221819f93cb2bb0b8c8e724f5b7ae280", + "scenario": "model", + "subscribers": 512, + "per_group": 10967.270647749809 + }, + { + "label": "base-6cbb58f7903b", + "revision": "6cbb58f7903baa9eb98f9556ab811add85f760a5", + "scenario": "model", + "subscribers": 512, + "per_group": 10967.270635755182 + }, + { + "label": "base-17e00c8148d7", + "revision": "17e00c8148d76af6c1ee17a8efdb87c7e400bd2b", + "scenario": "model", + "subscribers": 512, + "per_group": 10967.270562787871 + }, + { + "label": "base-c1a141282003", + "revision": "c1a141282003e913c38f4b1953ba769c2030c884", + "scenario": "model", + "subscribers": 512, + "per_group": 10967.270676736822 + }, + { + "label": "base-d42abb2a8608", + "revision": "d42abb2a8608a43729c742feb550cc925c538d51", + "scenario": "model", + "subscribers": 512, + "per_group": 10967.27058977578 + }, + { + "label": "base-947c91d05b4b", + "revision": "947c91d05b4b12594e64b5f0ab84a28c1831a491", + "scenario": "model", + "subscribers": 512, + "per_group": 10985.500266880437 + }, + { + "label": "base-68fb455bac47", + "revision": "68fb455bac47023b19020266ac1e6d5618dd08bc", + "scenario": "model", + "subscribers": 512, + "per_group": 11189.308664718206 + }, + { + "label": "base-08e71cd3df56", + "revision": "08e71cd3df56d80d83f134844cf2147498b279fe", + "scenario": "model", + "subscribers": 512, + "per_group": 11189.303892856 + }, + { + "label": "base-32391c452792", + "revision": "32391c4527927c5e854b8139aebf2e2ae11f0714", + "scenario": "model", + "subscribers": 512, + "per_group": 11189.303752918693 + }, + { + "label": "base-664b1aad2e6f", + "revision": "664b1aad2e6ff7fe8ade4a57a272bcd5c0cc128b", + "scenario": "model", + "subscribers": 512, + "per_group": 11189.303716934814 + }, + { + "label": "base-81fe8a6631cf", + "revision": "81fe8a6631cf4c3402ab7e04a1dbf3ccb61ba264", + "scenario": "model", + "subscribers": 512, + "per_group": 11746.942647693833 + }, + { + "label": "base-bfccf91db866", + "revision": "bfccf91db8662fa8d36abd90da5cd54b07c36041", + "scenario": "model", + "subscribers": 512, + "per_group": 11746.942757644576 + }, + { + "label": "base-41118d073d47", + "revision": "41118d073d47e2aa3030dce40b8e93f48fa9ce27", + "scenario": "model", + "subscribers": 512, + "per_group": 11746.942336833099 + }, + { + "label": "base-33686a29725f", + "revision": "33686a29725f9785d85ab5926cc4a4f34eef5b9b", + "scenario": "model", + "subscribers": 512, + "per_group": 11746.947575486181 + }, + { + "label": "base-3c0dd8a2af6b", + "revision": "3c0dd8a2af6ba98c13965fa13c31c4e3185830ac", + "scenario": "model", + "subscribers": 512, + "per_group": 11959.76987609551 + }, + { + "label": "base-a039ca1be51f", + "revision": "a039ca1be51f28530b9acf519604039cbfd82504", + "scenario": "model", + "subscribers": 512, + "per_group": 11929.205388985734 + }, + { + "label": "base-e3e1cc884155", + "revision": "e3e1cc8841558a52c7811eb0fd8ebb0dcefb29cf", + "scenario": "model", + "subscribers": 512, + "per_group": 11929.205050137538 + }, + { + "label": "base-f1034b7cc973", + "revision": "f1034b7cc973009bb33abfd06404be1a1dea0266", + "scenario": "model", + "subscribers": 512, + "per_group": 12830.06617935165 + }, + { + "label": "base-d43ff4394baf", + "revision": "d43ff4394baff600c414ba2c6a2a02c230cb2c51", + "scenario": "model", + "subscribers": 512, + "per_group": 12834.068875143936 + }, + { + "label": "base-097dfc42257f", + "revision": "097dfc42257fc893dee5116006a265660fd71b01", + "scenario": "model", + "subscribers": 512, + "per_group": 12834.065782529427 + }, + { + "label": "base-5ab878666c31", + "revision": "5ab878666c312fc2a497a31656630efeb02ca55f", + "scenario": "model", + "subscribers": 512, + "per_group": 12784.511050049578 + }, + { + "label": "base-c68c93102534", + "revision": "c68c93102534b0f12382dcad20704fb57385bef1", + "scenario": "model", + "subscribers": 512, + "per_group": 13071.60937100179 + }, + { + "label": "base-be3918800076", + "revision": "be3918800076ea8685fc8e0468a097234516e3a9", + "scenario": "model", + "subscribers": 512, + "per_group": 13071.609004166134 + }, + { + "label": "base-1777a2d6f98d", + "revision": "1777a2d6f98d7b5e12787dbc0687a6a8cdc2ae38", + "scenario": "model", + "subscribers": 512, + "per_group": 13100.6231598244 + }, + { + "label": "base-ad3ca7784aa0", + "revision": "ad3ca7784aa00f7b8b93b8a8076304673c50e733", + "scenario": "model", + "subscribers": 512, + "per_group": 13100.492399405066 + }, + { + "label": "base-25ff6e97546f", + "revision": "25ff6e97546f59dcebde66f43b67ba55f590867c", + "scenario": "model", + "subscribers": 512, + "per_group": 13236.30172082907 + }, + { + "label": "base-651c30fee9d8", + "revision": "651c30fee9d88fd56f4079433dd3e1f98d77f21c", + "scenario": "model", + "subscribers": 512, + "per_group": 9534.986503046635 + }, + { + "label": "base-82f7159bb25c", + "revision": "82f7159bb25ca64a63d893296e97b83e1774fbea", + "scenario": "model", + "subscribers": 512, + "per_group": 9534.986494050665 + }, + { + "label": "base-767685e1ba0d", + "revision": "767685e1ba0dabfd921fe8439d7f55ccf65eeedb", + "scenario": "model", + "subscribers": 512, + "per_group": 9534.986662974987 + }, + { + "label": "base-e7ca859b5527", + "revision": "e7ca859b552748fbc97e588117fd1c2915258b6d", + "scenario": "model", + "subscribers": 512, + "per_group": 9535.916662335274 + }, + { + "label": "base-197bdae8eeeb", + "revision": "197bdae8eeeb32b325b6f5ad64aadf1477b0241e", + "scenario": "model", + "subscribers": 512, + "per_group": 9535.917204092566 + }, + { + "label": "base-6eff65506e81", + "revision": "6eff65506e817ca82c6ee3d408c55066cf811510", + "scenario": "model", + "subscribers": 512, + "per_group": 9535.917027171827 + }, + { + "label": "base-d6e175f8e15f", + "revision": "d6e175f8e15f0f9323e472b9746ca4346bfa7b35", + "scenario": "model", + "subscribers": 512, + "per_group": 9535.92292952757 + }, + { + "label": "base-7c7e1d1c0f31", + "revision": "7c7e1d1c0f31dccdb4adcc76acb9acf6515180e8", + "scenario": "model", + "subscribers": 512, + "per_group": 9535.922853561604 + }, + { + "label": "base-84d9b018b83d", + "revision": "84d9b018b83d880d901cfd84a9064b4923d284b5", + "scenario": "model", + "subscribers": 512, + "per_group": 9535.922934525333 + }, + { + "label": "current", + "revision": "44c0dc494ff3262de6f8bad38230f8ff09e6ebf1", + "scenario": "model", + "subscribers": 512, + "per_group": 10060.128887258508 + } + ] + }, + { + "artifact": "run.TmGdFy", + "note": "each culprit against its first parent", + "harness_revision": "b243c4046f472d0a547e401deb71036f1a5c94c3", + "frame_pointers": false, + "counts": [ + { + "label": "base-55f6ed229250", + "revision": "55f6ed229250d17fc99389237659906d011d8a94", + "scenario": "model", + "subscribers": 64, + "per_group": 5173.549239 + }, + { + "label": "base-55f6ed229250", + "revision": "55f6ed229250d17fc99389237659906d011d8a94", + "scenario": "model", + "subscribers": 512, + "per_group": 5015.741348875704 + }, + { + "label": "base-caff81eb7f64", + "revision": "caff81eb7f64fff28c554cf218ab393da182f55a", + "scenario": "model", + "subscribers": 64, + "per_group": 9437.337956 + }, + { + "label": "base-caff81eb7f64", + "revision": "caff81eb7f64fff28c554cf218ab393da182f55a", + "scenario": "model", + "subscribers": 512, + "per_group": 9279.352597036528 + }, + { + "label": "base-30f23c3f8af0", + "revision": "30f23c3f8af0b46eeebff687bfa45397ed70cf0a", + "scenario": "model", + "subscribers": 64, + "per_group": 10020.597976 + }, + { + "label": "base-30f23c3f8af0", + "revision": "30f23c3f8af0b46eeebff687bfa45397ed70cf0a", + "scenario": "model", + "subscribers": 512, + "per_group": 9818.072495522007 + }, + { + "label": "base-deee150f8f64", + "revision": "deee150f8f6400c14632627601960a89fd8e2b26", + "scenario": "model", + "subscribers": 64, + "per_group": 11171.813651 + }, + { + "label": "base-deee150f8f64", + "revision": "deee150f8f6400c14632627601960a89fd8e2b26", + "scenario": "model", + "subscribers": 512, + "per_group": 10972.215946256078 + }, + { + "label": "base-947c91d05b4b", + "revision": "947c91d05b4b12594e64b5f0ab84a28c1831a491", + "scenario": "model", + "subscribers": 64, + "per_group": 11190.094659 + }, + { + "label": "base-947c91d05b4b", + "revision": "947c91d05b4b12594e64b5f0ab84a28c1831a491", + "scenario": "model", + "subscribers": 512, + "per_group": 10990.25496077757 + }, + { + "label": "base-68fb455bac47", + "revision": "68fb455bac47023b19020266ac1e6d5618dd08bc", + "scenario": "model", + "subscribers": 64, + "per_group": 11394.092394 + }, + { + "label": "base-68fb455bac47", + "revision": "68fb455bac47023b19020266ac1e6d5618dd08bc", + "scenario": "model", + "subscribers": 512, + "per_group": 11194.250525764457 + }, + { + "label": "base-3f5939eb2ee3", + "revision": "3f5939eb2ee39f4ff9d52f6c061ff366110ced6a", + "scenario": "model", + "subscribers": 64, + "per_group": 11394.092954 + }, + { + "label": "base-3f5939eb2ee3", + "revision": "3f5939eb2ee39f4ff9d52f6c061ff366110ced6a", + "scenario": "model", + "subscribers": 512, + "per_group": 11194.25055874968 + }, + { + "label": "base-81fe8a6631cf", + "revision": "81fe8a6631cf4c3402ab7e04a1dbf3ccb61ba264", + "scenario": "model", + "subscribers": 64, + "per_group": 11957.435432 + }, + { + "label": "base-81fe8a6631cf", + "revision": "81fe8a6631cf4c3402ab7e04a1dbf3ccb61ba264", + "scenario": "model", + "subscribers": 512, + "per_group": 11751.995400060772 + }, + { + "label": "base-3d095a319cb6", + "revision": "3d095a319cb695e6812350cb09aef4ee712c247a", + "scenario": "model", + "subscribers": 64, + "per_group": 11957.435119 + }, + { + "label": "base-3d095a319cb6", + "revision": "3d095a319cb695e6812350cb09aef4ee712c247a", + "scenario": "model", + "subscribers": 512, + "per_group": 11751.995168164662 + }, + { + "label": "base-3c0dd8a2af6b", + "revision": "3c0dd8a2af6ba98c13965fa13c31c4e3185830ac", + "scenario": "model", + "subscribers": 64, + "per_group": 12171.447257 + }, + { + "label": "base-3c0dd8a2af6b", + "revision": "3c0dd8a2af6ba98c13965fa13c31c4e3185830ac", + "scenario": "model", + "subscribers": 512, + "per_group": 11965.12150656506 + }, + { + "label": "base-4eb9c23db5de", + "revision": "4eb9c23db5de2baa48ef838c357392fa6424fa2a", + "scenario": "model", + "subscribers": 64, + "per_group": 12117.255659 + }, + { + "label": "base-4eb9c23db5de", + "revision": "4eb9c23db5de2baa48ef838c357392fa6424fa2a", + "scenario": "model", + "subscribers": 512, + "per_group": 11911.038718654043 + }, + { + "label": "base-f1034b7cc973", + "revision": "f1034b7cc973009bb33abfd06404be1a1dea0266", + "scenario": "model", + "subscribers": 64, + "per_group": 12993.101633 + }, + { + "label": "base-f1034b7cc973", + "revision": "f1034b7cc973009bb33abfd06404be1a1dea0266", + "scenario": "model", + "subscribers": 512, + "per_group": 12786.68907129606 + }, + { + "label": "base-b7187a4559ed", + "revision": "b7187a4559ed2093b59c6a69b6058e10466cd37c", + "scenario": "model", + "subscribers": 64, + "per_group": 13044.710638 + }, + { + "label": "base-b7187a4559ed", + "revision": "b7187a4559ed2093b59c6a69b6058e10466cd37c", + "scenario": "model", + "subscribers": 512, + "per_group": 12835.771269471277 + }, + { + "label": "base-c68c93102534", + "revision": "c68c93102534b0f12382dcad20704fb57385bef1", + "scenario": "model", + "subscribers": 64, + "per_group": 13319.716747 + }, + { + "label": "base-c68c93102534", + "revision": "c68c93102534b0f12382dcad20704fb57385bef1", + "scenario": "model", + "subscribers": 512, + "per_group": 13110.767371217695 + }, + { + "label": "base-bd71f72b40d9", + "revision": "bd71f72b40d9a85001acc1b36ab40b6a5e1a6760", + "scenario": "model", + "subscribers": 64, + "per_group": 13348.304234 + }, + { + "label": "base-bd71f72b40d9", + "revision": "bd71f72b40d9a85001acc1b36ab40b6a5e1a6760", + "scenario": "model", + "subscribers": 512, + "per_group": 13139.74781497889 + }, + { + "label": "base-25ff6e97546f", + "revision": "25ff6e97546f59dcebde66f43b67ba55f590867c", + "scenario": "model", + "subscribers": 64, + "per_group": 13508.303632 + }, + { + "label": "base-25ff6e97546f", + "revision": "25ff6e97546f59dcebde66f43b67ba55f590867c", + "scenario": "model", + "subscribers": 512, + "per_group": 13299.747363181295 + }, + { + "label": "base-84d9b018b83d", + "revision": "84d9b018b83d880d901cfd84a9064b4923d284b5", + "scenario": "model", + "subscribers": 64, + "per_group": 9736.783127 + }, + { + "label": "base-84d9b018b83d", + "revision": "84d9b018b83d880d901cfd84a9064b4923d284b5", + "scenario": "model", + "subscribers": 512, + "per_group": 9540.758499192361 + }, + { + "label": "base-bdfa5b1d0639", + "revision": "bdfa5b1d06394e05e30f167e297b45cc913af1f0", + "scenario": "model", + "subscribers": 64, + "per_group": 10101.811168 + }, + { + "label": "base-bdfa5b1d0639", + "revision": "bdfa5b1d06394e05e30f167e297b45cc913af1f0", + "scenario": "model", + "subscribers": 512, + "per_group": 9905.499159376599 + }, + { + "label": "base-db92d18a24c2", + "revision": "db92d18a24c2dc4e72274bc04b4a49ed296dcad1", + "scenario": "model", + "subscribers": 64, + "per_group": 12171.450912 + }, + { + "label": "base-db92d18a24c2", + "revision": "db92d18a24c2dc4e72274bc04b4a49ed296dcad1", + "scenario": "model", + "subscribers": 512, + "per_group": 11965.120981800153 + }, + { + "label": "base-3292489a4608", + "revision": "3292489a46083d9ef75ac1ec83258fb040f59fba", + "scenario": "model", + "subscribers": 64, + "per_group": 12171.445025 + }, + { + "label": "base-3292489a4608", + "revision": "3292489a46083d9ef75ac1ec83258fb040f59fba", + "scenario": "model", + "subscribers": 512, + "per_group": 11965.122121289662 + }, + { + "label": "base-097dfc42257f", + "revision": "097dfc42257fc893dee5116006a265660fd71b01", + "scenario": "model", + "subscribers": 64, + "per_group": 12997.10948 + }, + { + "label": "base-097dfc42257f", + "revision": "097dfc42257fc893dee5116006a265660fd71b01", + "scenario": "model", + "subscribers": 512, + "per_group": 12790.695173562244 + }, + { + "label": "base-cc3c1961d849", + "revision": "cc3c1961d849dd536dc203724bf6b6cca46f0d88", + "scenario": "model", + "subscribers": 64, + "per_group": 12997.109092 + }, + { + "label": "base-cc3c1961d849", + "revision": "cc3c1961d849dd536dc203724bf6b6cca46f0d88", + "scenario": "model", + "subscribers": 512, + "per_group": 12790.690385707203 + }, + { + "label": "base-d44ee77b18b0", + "revision": "d44ee77b18b039fd8b72d410b82404ab8353db0e", + "scenario": "model", + "subscribers": 64, + "per_group": 10020.59734 + }, + { + "label": "base-d44ee77b18b0", + "revision": "d44ee77b18b039fd8b72d410b82404ab8353db0e", + "scenario": "model", + "subscribers": 512, + "per_group": 9818.073104249297 + }, + { + "label": "base-e3a1cdf2328f", + "revision": "e3a1cdf2328f853254c467b2b3c9c7327753d31a", + "scenario": "model", + "subscribers": 64, + "per_group": 10020.599767 + }, + { + "label": "base-e3a1cdf2328f", + "revision": "e3a1cdf2328f853254c467b2b3c9c7327753d31a", + "scenario": "model", + "subscribers": 512, + "per_group": 9818.07263745842 + }, + { + "label": "base-cdff3c8f952b", + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "scenario": "model", + "subscribers": 64, + "per_group": 4409.825208 + }, + { + "label": "base-cdff3c8f952b", + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "scenario": "model", + "subscribers": 512, + "per_group": 4285.009976530514 + }, + { + "label": "base-e84ffb3de313", + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "scenario": "model", + "subscribers": 64, + "per_group": 6004.348586 + }, + { + "label": "base-e84ffb3de313", + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "scenario": "model", + "subscribers": 512, + "per_group": 5840.058401835978 + }, + { + "label": "base-21d6da9fc108", + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "scenario": "model", + "subscribers": 64, + "per_group": 12288.032389 + }, + { + "label": "base-21d6da9fc108", + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "scenario": "model", + "subscribers": 512, + "per_group": 12079.328686748337 + }, + { + "label": "base-a6f3d2821693", + "revision": "a6f3d28216931c216d76b0ffd8e9ea2455741341", + "scenario": "model", + "subscribers": 64, + "per_group": 13508.303174 + }, + { + "label": "base-a6f3d2821693", + "revision": "a6f3d28216931c216d76b0ffd8e9ea2455741341", + "scenario": "model", + "subscribers": 512, + "per_group": 13299.745055215264 + }, + { + "label": "current", + "revision": "b243c4046f472d0a547e401deb71036f1a5c94c3", + "scenario": "model", + "subscribers": 64, + "per_group": 10399.519503 + }, + { + "label": "current", + "revision": "b243c4046f472d0a547e401deb71036f1a5c94c3", + "scenario": "model", + "subscribers": 512, + "per_group": 10189.137410440122 + }, + { + "label": "base-55f6ed229250", + "revision": "55f6ed229250d17fc99389237659906d011d8a94", + "scenario": "session", + "subscribers": 64, + "per_group": null + }, + { + "label": "base-55f6ed229250", + "revision": "55f6ed229250d17fc99389237659906d011d8a94", + "scenario": "session", + "subscribers": 512, + "per_group": null + }, + { + "label": "base-caff81eb7f64", + "revision": "caff81eb7f64fff28c554cf218ab393da182f55a", + "scenario": "session", + "subscribers": 64, + "per_group": null + }, + { + "label": "base-caff81eb7f64", + "revision": "caff81eb7f64fff28c554cf218ab393da182f55a", + "scenario": "session", + "subscribers": 512, + "per_group": null + }, + { + "label": "base-30f23c3f8af0", + "revision": "30f23c3f8af0b46eeebff687bfa45397ed70cf0a", + "scenario": "session", + "subscribers": 64, + "per_group": null + }, + { + "label": "base-30f23c3f8af0", + "revision": "30f23c3f8af0b46eeebff687bfa45397ed70cf0a", + "scenario": "session", + "subscribers": 512, + "per_group": null + }, + { + "label": "base-deee150f8f64", + "revision": "deee150f8f6400c14632627601960a89fd8e2b26", + "scenario": "session", + "subscribers": 64, + "per_group": null + }, + { + "label": "base-deee150f8f64", + "revision": "deee150f8f6400c14632627601960a89fd8e2b26", + "scenario": "session", + "subscribers": 512, + "per_group": null + }, + { + "label": "base-947c91d05b4b", + "revision": "947c91d05b4b12594e64b5f0ab84a28c1831a491", + "scenario": "session", + "subscribers": 64, + "per_group": 143788.77370241523 + }, + { + "label": "base-947c91d05b4b", + "revision": "947c91d05b4b12594e64b5f0ab84a28c1831a491", + "scenario": "session", + "subscribers": 512, + "per_group": 141133.06857860333 + }, + { + "label": "base-68fb455bac47", + "revision": "68fb455bac47023b19020266ac1e6d5618dd08bc", + "scenario": "session", + "subscribers": 64, + "per_group": 143324.79522552784 + }, + { + "label": "base-68fb455bac47", + "revision": "68fb455bac47023b19020266ac1e6d5618dd08bc", + "scenario": "session", + "subscribers": 512, + "per_group": 140678.44754464287 + }, + { + "label": "base-3f5939eb2ee3", + "revision": "3f5939eb2ee39f4ff9d52f6c061ff366110ced6a", + "scenario": "session", + "subscribers": 64, + "per_group": 142781.21071257198 + }, + { + "label": "base-3f5939eb2ee3", + "revision": "3f5939eb2ee39f4ff9d52f6c061ff366110ced6a", + "scenario": "session", + "subscribers": 512, + "per_group": 140313.2066127232 + }, + { + "label": "base-81fe8a6631cf", + "revision": "81fe8a6631cf4c3402ab7e04a1dbf3ccb61ba264", + "scenario": "session", + "subscribers": 64, + "per_group": 146287.1747840691 + }, + { + "label": "base-81fe8a6631cf", + "revision": "81fe8a6631cf4c3402ab7e04a1dbf3ccb61ba264", + "scenario": "session", + "subscribers": 512, + "per_group": 143603.11443718112 + }, + { + "label": "base-3d095a319cb6", + "revision": "3d095a319cb695e6812350cb09aef4ee712c247a", + "scenario": "session", + "subscribers": 64, + "per_group": 146079.13113803582 + }, + { + "label": "base-3d095a319cb6", + "revision": "3d095a319cb695e6812350cb09aef4ee712c247a", + "scenario": "session", + "subscribers": 512, + "per_group": 143397.84674944196 + }, + { + "label": "base-3c0dd8a2af6b", + "revision": "3c0dd8a2af6ba98c13965fa13c31c4e3185830ac", + "scenario": "session", + "subscribers": 64, + "per_group": 142925.80393274152 + }, + { + "label": "base-3c0dd8a2af6b", + "revision": "3c0dd8a2af6ba98c13965fa13c31c4e3185830ac", + "scenario": "session", + "subscribers": 512, + "per_group": 140256.0416135204 + }, + { + "label": "base-4eb9c23db5de", + "revision": "4eb9c23db5de2baa48ef838c357392fa6424fa2a", + "scenario": "session", + "subscribers": 64, + "per_group": 118998.90310100767 + }, + { + "label": "base-4eb9c23db5de", + "revision": "4eb9c23db5de2baa48ef838c357392fa6424fa2a", + "scenario": "session", + "subscribers": 512, + "per_group": 116906.18026546556 + }, + { + "label": "base-f1034b7cc973", + "revision": "f1034b7cc973009bb33abfd06404be1a1dea0266", + "scenario": "session", + "subscribers": 64, + "per_group": 122072.00230926104 + }, + { + "label": "base-f1034b7cc973", + "revision": "f1034b7cc973009bb33abfd06404be1a1dea0266", + "scenario": "session", + "subscribers": 512, + "per_group": 119959.71555125956 + }, + { + "label": "base-b7187a4559ed", + "revision": "b7187a4559ed2093b59c6a69b6058e10466cd37c", + "scenario": "session", + "subscribers": 64, + "per_group": 122503.40778950736 + }, + { + "label": "base-b7187a4559ed", + "revision": "b7187a4559ed2093b59c6a69b6058e10466cd37c", + "scenario": "session", + "subscribers": 512, + "per_group": 120387.56217115752 + }, + { + "label": "base-c68c93102534", + "revision": "c68c93102534b0f12382dcad20704fb57385bef1", + "scenario": "session", + "subscribers": 64, + "per_group": 122962.28769793666 + }, + { + "label": "base-c68c93102534", + "revision": "c68c93102534b0f12382dcad20704fb57385bef1", + "scenario": "session", + "subscribers": 512, + "per_group": 120837.39202008929 + }, + { + "label": "base-bd71f72b40d9", + "revision": "bd71f72b40d9a85001acc1b36ab40b6a5e1a6760", + "scenario": "session", + "subscribers": 64, + "per_group": 122839.86978166987 + }, + { + "label": "base-bd71f72b40d9", + "revision": "bd71f72b40d9a85001acc1b36ab40b6a5e1a6760", + "scenario": "session", + "subscribers": 512, + "per_group": 120719.14727160396 + }, + { + "label": "base-25ff6e97546f", + "revision": "25ff6e97546f59dcebde66f43b67ba55f590867c", + "scenario": "session", + "subscribers": 64, + "per_group": 122894.58472288867 + }, + { + "label": "base-25ff6e97546f", + "revision": "25ff6e97546f59dcebde66f43b67ba55f590867c", + "scenario": "session", + "subscribers": 512, + "per_group": 120761.91606544962 + }, + { + "label": "base-84d9b018b83d", + "revision": "84d9b018b83d880d901cfd84a9064b4923d284b5", + "scenario": "session", + "subscribers": 64, + "per_group": null + }, + { + "label": "base-84d9b018b83d", + "revision": "84d9b018b83d880d901cfd84a9064b4923d284b5", + "scenario": "session", + "subscribers": 512, + "per_group": null + }, + { + "label": "base-bdfa5b1d0639", + "revision": "bdfa5b1d06394e05e30f167e297b45cc913af1f0", + "scenario": "session", + "subscribers": 64, + "per_group": null + }, + { + "label": "base-bdfa5b1d0639", + "revision": "bdfa5b1d06394e05e30f167e297b45cc913af1f0", + "scenario": "session", + "subscribers": 512, + "per_group": null + }, + { + "label": "base-db92d18a24c2", + "revision": "db92d18a24c2dc4e72274bc04b4a49ed296dcad1", + "scenario": "session", + "subscribers": 64, + "per_group": 142925.57175703775 + }, + { + "label": "base-db92d18a24c2", + "revision": "db92d18a24c2dc4e72274bc04b4a49ed296dcad1", + "scenario": "session", + "subscribers": 512, + "per_group": 140256.04686503508 + }, + { + "label": "base-3292489a4608", + "revision": "3292489a46083d9ef75ac1ec83258fb040f59fba", + "scenario": "session", + "subscribers": 64, + "per_group": 142986.527331254 + }, + { + "label": "base-3292489a4608", + "revision": "3292489a46083d9ef75ac1ec83258fb040f59fba", + "scenario": "session", + "subscribers": 512, + "per_group": 140316.09682916134 + }, + { + "label": "base-097dfc42257f", + "revision": "097dfc42257fc893dee5116006a265660fd71b01", + "scenario": "session", + "subscribers": 64, + "per_group": 122237.25825735765 + }, + { + "label": "base-097dfc42257f", + "revision": "097dfc42257fc893dee5116006a265660fd71b01", + "scenario": "session", + "subscribers": 512, + "per_group": 120122.85361527423 + }, + { + "label": "base-cc3c1961d849", + "revision": "cc3c1961d849dd536dc203724bf6b6cca46f0d88", + "scenario": "session", + "subscribers": 64, + "per_group": 122259.8310240723 + }, + { + "label": "base-cc3c1961d849", + "revision": "cc3c1961d849dd536dc203724bf6b6cca46f0d88", + "scenario": "session", + "subscribers": 512, + "per_group": 120144.74912308673 + }, + { + "label": "base-d44ee77b18b0", + "revision": "d44ee77b18b039fd8b72d410b82404ab8353db0e", + "scenario": "session", + "subscribers": 64, + "per_group": null + }, + { + "label": "base-d44ee77b18b0", + "revision": "d44ee77b18b039fd8b72d410b82404ab8353db0e", + "scenario": "session", + "subscribers": 512, + "per_group": null + }, + { + "label": "base-e3a1cdf2328f", + "revision": "e3a1cdf2328f853254c467b2b3c9c7327753d31a", + "scenario": "session", + "subscribers": 64, + "per_group": null + }, + { + "label": "base-e3a1cdf2328f", + "revision": "e3a1cdf2328f853254c467b2b3c9c7327753d31a", + "scenario": "session", + "subscribers": 512, + "per_group": null + }, + { + "label": "base-cdff3c8f952b", + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "scenario": "session", + "subscribers": 64, + "per_group": null + }, + { + "label": "base-cdff3c8f952b", + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "scenario": "session", + "subscribers": 512, + "per_group": null + }, + { + "label": "base-e84ffb3de313", + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "scenario": "session", + "subscribers": 64, + "per_group": null + }, + { + "label": "base-e84ffb3de313", + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "scenario": "session", + "subscribers": 512, + "per_group": null + }, + { + "label": "base-21d6da9fc108", + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "scenario": "session", + "subscribers": 64, + "per_group": 135423.53441898592 + }, + { + "label": "base-21d6da9fc108", + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "scenario": "session", + "subscribers": 512, + "per_group": 132932.98566047513 + }, + { + "label": "base-a6f3d2821693", + "revision": "a6f3d28216931c216d76b0ffd8e9ea2455741341", + "scenario": "session", + "subscribers": 64, + "per_group": 123075.42752319257 + }, + { + "label": "base-a6f3d2821693", + "revision": "a6f3d28216931c216d76b0ffd8e9ea2455741341", + "scenario": "session", + "subscribers": 512, + "per_group": 120948.21283083546 + }, + { + "label": "current", + "revision": "b243c4046f472d0a547e401deb71036f1a5c94c3", + "scenario": "session", + "subscribers": 64, + "per_group": 110473.08423304543 + }, + { + "label": "current", + "revision": "b243c4046f472d0a547e401deb71036f1a5c94c3", + "scenario": "session", + "subscribers": 512, + "per_group": 108705.21113679848 + } + ] + }, + { + "artifact": "run.s6IjbN", + "note": "this branch's fixes, each on top of the last", + "harness_revision": "df74d8dbe07f760b2ff5eb6da9dda57d9ee8006d", + "frame_pointers": false, + "counts": [ + { + "label": "base-a6f3d2821693", + "revision": "a6f3d28216931c216d76b0ffd8e9ea2455741341", + "scenario": "model", + "subscribers": 64, + "per_group": 13508.284341 + }, + { + "label": "base-a6f3d2821693", + "revision": "a6f3d28216931c216d76b0ffd8e9ea2455741341", + "scenario": "model", + "subscribers": 512, + "per_group": 13299.745813875385 + }, + { + "label": "base-0db3f6c91b54", + "revision": "0db3f6c91b5482429b3bf3b6d40313c3c923ae98", + "scenario": "model", + "subscribers": 64, + "per_group": 10687.580642 + }, + { + "label": "base-0db3f6c91b54", + "revision": "0db3f6c91b5482429b3bf3b6d40313c3c923ae98", + "scenario": "model", + "subscribers": 512, + "per_group": 10476.74617571328 + }, + { + "label": "base-5132fc4a455b", + "revision": "5132fc4a455bf6a063112d1d5185c6e3b8a2a4f5", + "scenario": "model", + "subscribers": 64, + "per_group": 10336.580476 + }, + { + "label": "base-5132fc4a455b", + "revision": "5132fc4a455bf6a063112d1d5185c6e3b8a2a4f5", + "scenario": "model", + "subscribers": 512, + "per_group": 10125.742104537167 + }, + { + "label": "base-101ed5a8066d", + "revision": "101ed5a8066d4c17ec85ac3cc2d5ec40686c4962", + "scenario": "model", + "subscribers": 64, + "per_group": 10399.485894 + }, + { + "label": "base-101ed5a8066d", + "revision": "101ed5a8066d4c17ec85ac3cc2d5ec40686c4962", + "scenario": "model", + "subscribers": 512, + "per_group": 10188.911252758764 + }, + { + "label": "current", + "revision": "df74d8dbe07f760b2ff5eb6da9dda57d9ee8006d", + "scenario": "model", + "subscribers": 64, + "per_group": 10399.518113 + }, + { + "label": "current", + "revision": "df74d8dbe07f760b2ff5eb6da9dda57d9ee8006d", + "scenario": "model", + "subscribers": 512, + "per_group": 10189.138902771558 + }, + { + "label": "base-a6f3d2821693", + "revision": "a6f3d28216931c216d76b0ffd8e9ea2455741341", + "scenario": "session", + "subscribers": 64, + "per_group": 123077.5850227927 + }, + { + "label": "base-a6f3d2821693", + "revision": "a6f3d28216931c216d76b0ffd8e9ea2455741341", + "scenario": "session", + "subscribers": 512, + "per_group": 120947.86723732462 + }, + { + "label": "base-0db3f6c91b54", + "revision": "0db3f6c91b5482429b3bf3b6d40313c3c923ae98", + "scenario": "session", + "subscribers": 64, + "per_group": 116020.33326335573 + }, + { + "label": "base-0db3f6c91b54", + "revision": "0db3f6c91b5482429b3bf3b6d40313c3c923ae98", + "scenario": "session", + "subscribers": 512, + "per_group": 114139.33255939094 + }, + { + "label": "base-5132fc4a455b", + "revision": "5132fc4a455bf6a063112d1d5185c6e3b8a2a4f5", + "scenario": "session", + "subscribers": 64, + "per_group": 114999.65691978567 + }, + { + "label": "base-5132fc4a455b", + "revision": "5132fc4a455bf6a063112d1d5185c6e3b8a2a4f5", + "scenario": "session", + "subscribers": 512, + "per_group": 113132.77601841518 + }, + { + "label": "base-101ed5a8066d", + "revision": "101ed5a8066d4c17ec85ac3cc2d5ec40686c4962", + "scenario": "session", + "subscribers": 64, + "per_group": 110473.41742642355 + }, + { + "label": "base-101ed5a8066d", + "revision": "101ed5a8066d4c17ec85ac3cc2d5ec40686c4962", + "scenario": "session", + "subscribers": 512, + "per_group": 108709.31261957908 + }, + { + "label": "current", + "revision": "df74d8dbe07f760b2ff5eb6da9dda57d9ee8006d", + "scenario": "session", + "subscribers": 64, + "per_group": 110473.45486444338 + }, + { + "label": "current", + "revision": "df74d8dbe07f760b2ff5eb6da9dda57d9ee8006d", + "scenario": "session", + "subscribers": 512, + "per_group": 108709.42349928252 + } + ] + }, + { + "artifact": "run.evL1wx", + "note": "this branch against main at 619df6e1b3", + "harness_revision": "af83ff87fcb3cb0e2ff3969ed1f96fcebee62043", + "frame_pointers": false, + "counts": [ + { + "label": "base", + "revision": "619df6e1b3a27457b31be76d14d58ae7b5dc7afc", + "scenario": "model", + "subscribers": 64, + "per_group": 13576.320641 + }, + { + "label": "base", + "revision": "619df6e1b3a27457b31be76d14d58ae7b5dc7afc", + "scenario": "model", + "subscribers": 512, + "per_group": 13367.74955919748 + }, + { + "label": "current", + "revision": "af83ff87fcb3cb0e2ff3969ed1f96fcebee62043", + "scenario": "model", + "subscribers": 64, + "per_group": 10329.611195 + }, + { + "label": "current", + "revision": "af83ff87fcb3cb0e2ff3969ed1f96fcebee62043", + "scenario": "model", + "subscribers": 512, + "per_group": 10118.748023885299 + }, + { + "label": "base", + "revision": "619df6e1b3a27457b31be76d14d58ae7b5dc7afc", + "scenario": "session", + "subscribers": 64, + "per_group": 123241.99125279911 + }, + { + "label": "base", + "revision": "619df6e1b3a27457b31be76d14d58ae7b5dc7afc", + "scenario": "session", + "subscribers": 512, + "per_group": 121114.091796875 + }, + { + "label": "current", + "revision": "af83ff87fcb3cb0e2ff3969ed1f96fcebee62043", + "scenario": "session", + "subscribers": 64, + "per_group": 115007.12947856686 + }, + { + "label": "current", + "revision": "af83ff87fcb3cb0e2ff3969ed1f96fcebee62043", + "scenario": "session", + "subscribers": 512, + "per_group": 113142.93781887754 + } + ] + } + ], + "quiet_host_runs": [ + { + "artifact": "nixos2:run.we4umM", + "host": "nixos2 (i7-8700, idle; quest gate not installed there, so its load and other agents were checked first)", + "relay_cpus": "0-3", + "generator_cpus": "4,5,10,11", + "generator_revision": "21d6da9fc1", + "workload": "chat", + "runtime": "io-uring-workers", + "revisions": { "base": "a6f3d28216 (main merge)", "current": "44c0dc494f (the probe and window changes)" }, + "summary": { + "base": { + "4": { + "cells": [ + { + "cell": "relay/base/round-1/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 154075.30345533811, + "cycles_per_frame": 301337.768161681, + "recv_fps": 20002.865948772975, + "groups_lost": 0, + "host_load_1m": 4.68 + }, + { + "cell": "relay/base/round-2/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 155046.662319841, + "cycles_per_frame": 314087.22154518875, + "recv_fps": 19998.415463227855, + "groups_lost": 0, + "host_load_1m": 1.67 + }, + { + "cell": "relay/base/round-3/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 154380.03417891636, + "cycles_per_frame": 351534.3433222535, + "recv_fps": 20001.617599275352, + "groups_lost": 0, + "host_load_1m": 1.45 + }, + { + "cell": "relay/base/round-4/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 154590.6275877289, + "cycles_per_frame": 304639.855884305, + "recv_fps": 20001.623163712316, + "groups_lost": 0, + "host_load_1m": 2.64 + } + ], + "instructions_per_frame": { + "median": 154485.33088332263, + "min": 154075.30345533811, + "max": 155046.662319841 + }, + "cycles_per_frame": { + "median": 309363.5387147469, + "min": 301337.768161681, + "max": 351534.3433222535 + }, + "recv_fps": { + "median": 20001.620381493834, + "min": 19998.415463227855, + "max": 20002.865948772975 + }, + "groups_lost": { "median": 0.0, "min": 0, "max": 0 }, + "host_load_1m": { "median": 2.1550000000000002, "min": 1.45, "max": 4.68 } + } + }, + "current": { + "4": { + "cells": [ + { + "cell": "relay/current/round-1/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 148274.66412811857, + "cycles_per_frame": 337324.0407741121, + "recv_fps": 20000.0, + "groups_lost": 0, + "host_load_1m": 2.41 + }, + { + "cell": "relay/current/round-2/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 148357.681494429, + "cycles_per_frame": 274414.4307833131, + "recv_fps": 20003.999606288755, + "groups_lost": 0, + "host_load_1m": 1.54 + }, + { + "cell": "relay/current/round-3/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 150798.68041245252, + "cycles_per_frame": 322361.04395030695, + "recv_fps": 19997.632603729515, + "groups_lost": 0, + "host_load_1m": 2.75 + }, + { + "cell": "relay/current/round-4/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 149414.94788195015, + "cycles_per_frame": 315090.17196768953, + "recv_fps": 19999.995299019218, + "groups_lost": 0, + "host_load_1m": 1.32 + } + ], + "instructions_per_frame": { + "median": 148886.3146881896, + "min": 148274.66412811857, + "max": 150798.68041245252 + }, + "cycles_per_frame": { + "median": 318725.60795899824, + "min": 274414.4307833131, + "max": 337324.0407741121 + }, + "recv_fps": { + "median": 19999.99764950961, + "min": 19997.632603729515, + "max": 20003.999606288755 + }, + "groups_lost": { "median": 0.0, "min": 0, "max": 0 }, + "host_load_1m": { "median": 1.975, "min": 1.32, "max": 2.75 } + } + } + } + } + ], + "notes": [ + "nixos2:run.cAVCAV (in-tree qmux and receive leases against their parents) was stopped after one round when other agents started fuzzing and compiling there; its round-1 cells (qmux 116k to 122k, leases 130k to 132k) are not conclusive and are not used." + ] +} diff --git a/.github/relay-cpu-bisect.md b/.github/relay-cpu-bisect.md new file mode 100644 index 00000000000..12a1a385969 --- /dev/null +++ b/.github/relay-cpu-bisect.md @@ -0,0 +1,118 @@ +# Relay CPU bisect + +Relay instructions per delivered frame between [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95) ([moq-dev/moq#5926](https://github.com/moq-dev/moq/pull/5926) as merged) and `main` at [c68c931025](https://github.com/moq-dev/moq/commit/c68c931025). Per-cell counts, delivery, loss, host load, and source markers are in [the evidence](relay-cpu-bisect.json). The [first checkpoint](https://github.com/moq-dev/moq/blob/8c62afd3195a3307054120226b54e3309a5dbc1f/.github/relay-cpu-bisect.md) and its [per-cell evidence](https://github.com/moq-dev/moq/blob/8c62afd3195a3307054120226b54e3309a5dbc1f/.github/relay-cpu-bisect.json) are superseded. + +## Method + +`just bench-profile instructions` counts the relay's user and kernel instructions over a five-second window after seven seconds of warmup, divided by the frames the generator's 500 ms reports interpolate over the same window. Builds use the profiling profile with frame pointers and `moq-relay/io-uring,moq-bench/uring`; the relay runs four workers on cores 4-7 and the generator on cores 0-3. Chat is one publisher, 2,000 viewers, 10 fps, one 300-byte frame per group; video is the `video` workload at 960 delivered frames/s. + +One run measures several revisions: every round counts each base and `main` back to back, rotating which goes first. Each revision builds once into its own target and checkout, so later runs reuse it. Every run passed through `quest gate`, and each cell records the host's highest one-minute load during its window. + +Three findings shaped the method during this attempt: + +- **Host load batches instructions down.** A relay contending with other work handles more frames per wakeup, so each costs fewer instructions: in [run.2DucdO](relay-cpu-bisect.json)'s third round, four revisions read 6-12k lower at load 9-19 than in the two rounds at load 2-5, three read the same, and the two at load 31-56 lost groups. Absolute counts from different sessions are not comparable: the first checkpoint read [55f6ed2292](https://github.com/moq-dev/moq/commit/55f6ed229250d17fc99389237659906d011d8a94) at 142.7k against 169-175k here. Only cells from one run, in a quiet stretch, are compared below. The 24-CPU host's load rose to 20-250 during several runs, from other work. +- **The generator is pinned.** From run.1KRu4S on, `BENCH_GENERATOR=21d6da9fc1` drives every relay, so later client changes cannot move what an old relay spends. With main's generator (run.BIL6LP), five of the seven relays it shares with run.qYPSii's quiet round read within 3k of the pinned figures; [0c2a798de5](https://github.com/moq-dev/moq/commit/0c2a798de5c2e9abb29815b86f945b0a980e5345) and [55f6ed2292](https://github.com/moq-dev/moq/commit/55f6ed229250d17fc99389237659906d011d8a94) read 7k and 11k lower. +- **Counting runs with the sampler off.** From run.qYPSii on, fixed-period stacks (period 2,000,003) come from the five seconds after the counter window, so sampling interrupts stay out of the counts. No measurable bias was found: [55f6ed2292](https://github.com/moq-dev/moq/commit/55f6ed229250d17fc99389237659906d011d8a94) read 167-171k with concurrent sampling and 175k without. + +These are sampled-delivery estimates on a shared host. No confidence interval is claimed. + +## Chat on io\_uring + +The quiet round of run.qYPSii (one-minute load 2-4, every cell at 20,000 frames/s with no loss): + +| Revision | kinstr/frame | `learn_end` | `Cursor::probe` | forward drop | syscall leaf | +| --- | ---: | ---: | ---: | ---: | ---: | +| [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95), old endpoint | 117.7 | 0% | 0% | 7.1% | 24.5% | +| [e84ffb3de3](https://github.com/moq-dev/moq/commit/e84ffb3de3139f7245bed5697754fd6add9a022e), before [moq-dev/moq#6224](https://github.com/moq-dev/moq/pull/6224) | 135.7 | 0% | 0% | 7.6% | 21.6% | +| [0c2a798de5](https://github.com/moq-dev/moq/commit/0c2a798de5c2e9abb29815b86f945b0a980e5345) | 169.8 | 15.4% | 0% | 11.7% | 27.5% | +| [55f6ed2292](https://github.com/moq-dev/moq/commit/55f6ed229250d17fc99389237659906d011d8a94), before [moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171) | 174.6 | 17.5% | 0% | 11.7% | 31.6% | +| [991572d636](https://github.com/moq-dev/moq/commit/991572d636d264c6189a3477849d2969e689f55c) | 204.7 | 15.5% | 7.7% | 20.1% | 35.0% | +| [2fff7ea7ae](https://github.com/moq-dev/moq/commit/2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3) | 214.0 | 17.0% | 7.6% | 24.0% | 32.7% | +| [21d6da9fc1](https://github.com/moq-dev/moq/commit/21d6da9fc108b48373a961f34e43c3fcd38b0146), first checkpoint's control | 176.2 | 1.3% | 7.3% | 21.2% | 22.3% | +| `main` | 148.8 | 0.5% | 6.6% | 14.4% | 17.7% | + +Markers are inclusive shares of the fixed-period samples; they overlap and are not additive. The two larger rises account for most of the peak, and `main` has already removed both: + +- **[moq-dev/moq#6224](https://github.com/moq-dev/moq/pull/6224), 26k of a +34k step, accidental, fixed.** Its `Reader::learn_end` polled each subscriber's finish through the track lock on every forward-drop pass (`poll_forward_drop` → `Subscriber::poll_drop` → `Reader::poll_serving` → `sync` → `learn_end`). It takes 10-18% of all relay instructions from [fd59eb18a5](https://github.com/moq-dev/moq/commit/fd59eb18a50f6e0ac819e73d941a3832cbdac91f), the PR's merge, through [2fff7ea7ae](https://github.com/moq-dev/moq/commit/2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3), and none at its parent ([run.16AJUb](relay-cpu-bisect.json) has every point in the interval). [moq-dev/moq#7532](https://github.com/moq-dev/moq/pull/7532) brought it to 0.5% on `main`. +- **[moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171) and the changes after it, +39k to [2fff7ea7ae](https://github.com/moq-dev/moq/commit/2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3), mostly fixed.** Dynamic tracks added `Cursor::probe`, 15.8k at [991572d636](https://github.com/moq-dev/moq/commit/991572d636d264c6189a3477849d2969e689f55c) and 9.8k on `main` (below). The rest is track lock contention: futex syscalls and the forward-drop path. [moq-dev/moq#7493](https://github.com/moq-dev/moq/pull/7493) took the fanout subscribers off the shared lock, and `main`'s syscall leaf is now below the old endpoint's (26.3k against 28.8k). +- **[cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95) to [e84ffb3de3](https://github.com/moq-dev/moq/commit/e84ffb3de3139f7245bed5697754fd6add9a022e), +18k, a staircase of feature steps.** The first checkpoint measured [moq-dev/moq#5929](https://github.com/moq-dev/moq/pull/5929)'s per-reader ranges and time anchors at +6k. Runs eVjz2w and 2DucdO (main's generator, load 2-5 in the rounds used) put about +6k across [moq-dev/moq#5928](https://github.com/moq-dev/moq/pull/5928) (compressed tracks) and [moq-dev/moq#6129](https://github.com/moq-dev/moq/pull/6129) (a subscriber reads only its ranges' frames), and +12k in steps of 2-4k from [2d73eea67d](https://github.com/moq-dev/moq/commit/2d73eea67d083ecf18938f4fc7c4842118eb87cf): qmux in tree ([moq-dev/moq#6231](https://github.com/moq-dev/moq/pull/6231)), signed priorities ([moq-dev/moq#6142](https://github.com/moq-dev/moq/pull/6142)), the m0 simplification audit and public ingest ([moq-dev/moq#6108](https://github.com/moq-dev/moq/pull/6108), [moq-dev/moq#6111](https://github.com/moq-dev/moq/pull/6111)), and receive leases ([moq-dev/moq#5878](https://github.com/moq-dev/moq/pull/5878)). Each step is within twice the spread between rounds, so these name the interval, not a single commit's cost. +- **[moq-dev/moq#7530](https://github.com/moq-dev/moq/pull/7530) moved io\_uring submission, not cost.** `udp::send` doubled after [21d6da9fc1](https://github.com/moq-dev/moq/commit/21d6da9fc108b48373a961f34e43c3fcd38b0146) because each turn now submits its queued sends inside `Shared::push_inner`, while total syscall instructions fell from 39.2k to 26.3k per frame. + +## Main against the old endpoint + +Every runtime and workload, with the pinned generator: + +| Workload, runtime | Run | [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95) | [21d6da9fc1](https://github.com/moq-dev/moq/commit/21d6da9fc108b48373a961f34e43c3fcd38b0146) | `main` | `main` / old | Highest load | +| --- | --- | ---: | ---: | ---: | ---: | ---: | +| chat, io\_uring | qYPSii, quiet round | 117.7 | 176.2 | 148.8 | 1.26 | 3 | +| chat, tokio workers | IV47ua, 4 rounds | 131.1 \[123.6-131.5] | 186.1 \[184.3-189.6] | 171.6 \[167.8-172.8] | 1.31 | 23 | +| video, io\_uring | SkAD0k, 4 rounds | 109.7 \[105.6-119.9] | 140.3 \[138.3-146.8] | 124.9 \[120.5-138.3] | 1.14 | 40 | +| video, tokio workers | Z04ENl, 4 rounds | 152.8 \[148.2-163.9] | 167.3 \[163.2-175.3] | 146.8 \[142.9-147.3] | 0.96 | 40 | + +Medians of four rounds, min-max in brackets; every cell delivered its workload's full rate with no loss. The video rows ran at load up to 40, so treat their ratios as approximate. Chat rose on both runtimes, so the remaining cost is shared code. By the nearest named function, chat's growth on `main` falls in the same places on io\_uring (run.qYPSii) and the tokio workers (run.IV47ua), in instructions per frame: + +| Growth on `main` | io\_uring | tokio | Introduced by | +| --- | ---: | ---: | --- | +| `Cursor::probe` and `Misses::judge` for every peek | 8.6k | 8.7k | [moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171), fills for misses | +| `Cursor` `sync`, `poll_drift`, `poll_peek`, `TrackState::live_edge` and `live` | 11.0k | 9.0k | [moq-dev/moq#5929](https://github.com/moq-dev/moq/pull/5929) and its successors, per-reader ranges and time anchors | +| kio `Tasks::poll_each`, `WaiterList::register` growth, `Waiters::release` | 10.7k | 11.5k | kio's task and waiter rework, including [moq-dev/moq#7340](https://github.com/moq-dev/moq/pull/7340) and [moq-dev/moq#7532](https://github.com/moq-dev/moq/pull/7532) | +| `Subscriber` `poll_drop`, `take`, `poll_peek_group` growth | 5.0k | 4.7k | forward drops and resume | +| `SubscribeServe::poll`, `Writer::poll_close`, transport `poll_closed` | 5.6k | 6.6k | lite publisher serving | + +The rows are self cost under the named function, including the std and libc leaves below it; they add to more than the net growth because other paths shrank. `Cursor::probe` reads the subscription, takes the track's shared state lock to judge misses, clones the track's fills, and runs the ask and resolve loop on every peek, even for a live reader that took every group as it arrived and never misses one. That work pays for relay backfill of missed groups, but the chat workload never needs it. + +## Exact counts by commit + +Relay counts move 5-20% with host load, so a step of a few thousand instructions can't be pinned to one commit from them. `just bench-fanout` (see `bench/README.md`) counts the chat fan-out on one pinned thread instead, against a run of zero cycles, so repeated counts agree within a few instructions per group: + +- `model`: one finished group a cycle through the track model, which every subscriber takes. It builds against every revision back to [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95). +- `session`: a publisher session into a relay origin and a viewer session out of it per subscriber, over the in-memory transport. It counts publisher, relay, and viewers together, without QUIC, and builds back to [moq-dev/moq#7027](https://github.com/moq-dev/moq/pull/7027)'s `kio::clock` (2026-11-01). + +At 512 subscribers, in instructions per delivered group, the model path cost 4,285 at the old endpoint and 13,300 on `main`, 3.1x; its growth is about 9k of the relay's per-frame growth on chat. The features the maintainer asked about, each against its parent (model, built with frame pointers): + +| Feature | Parent | Merge | Change | +| --- | ---: | ---: | ---: | +| Per-reader ranges and time anchors, [moq-dev/moq#5929](https://github.com/moq-dev/moq/pull/5929) | 4,411 | 5,313 | +902 | +| Compressed tracks, [moq-dev/moq#5928](https://github.com/moq-dev/moq/pull/5928) | 5,579 | 5,612 | +33 | +| A subscriber reads only its ranges' frames, [moq-dev/moq#6129](https://github.com/moq-dev/moq/pull/6129) | 5,604 | 5,787 | +183 | +| Signed priorities, [moq-dev/moq#6142](https://github.com/moq-dev/moq/pull/6142) | 5,888 | 5,816 | -72 | +| Public ingest, [moq-dev/moq#6111](https://github.com/moq-dev/moq/pull/6111) | 5,816 | 6,016 | +200 | +| The m0 simplification audit, [moq-dev/moq#6108](https://github.com/moq-dev/moq/pull/6108) | 6,017 | 6,017 | 0 | + +In-tree qmux ([moq-dev/moq#6231](https://github.com/moq-dev/moq/pull/6231)) and receive leases ([moq-dev/moq#5878](https://github.com/moq-dev/moq/pull/5878)) are QUIC-level, which neither scenario reaches. A relay run pairing each with its parent on the quiet test host got one round (qmux 116k to 122k, leases 130k to 132k per frame) before other agents started fuzzing and compiling there, so neither is attributed yet. The model path barely moved before [e84ffb3de3](https://github.com/moq-dev/moq/commit/e84ffb3de3139f7245bed5697754fd6add9a022e) (5,840) and then grew in a few large steps, each against its parent (run.TmGdFy, one build profile throughout): + +| Commit | Model | Session | +| --- | ---: | ---: | +| Dynamic tracks serve fills for misses, [moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171) | +4,264 | | +| Refill groups evicted before a reader took them, [moq-dev/moq#7339](https://github.com/moq-dev/moq/pull/7339) | +1,154 | | +| Lite-07 frame bounds, timed datagram admission, layered windows, [moq-dev/moq#7542](https://github.com/moq-dev/moq/pull/7542) | +876 | +3,054 | +| Fill linearly in range count and span, [moq-dev/moq#6995](https://github.com/moq-dev/moq/pull/6995) | +558 | +3,290 | +| Close the relay demand gaps, [moq-dev/moq#7052](https://github.com/moq-dev/moq/pull/7052) | +365 | | +| Bounded reads end through routes, [c68c931025](https://github.com/moq-dev/moq/commit/c68c931025) | +275 | +450 | +| An idle track read parks without walking the read path, [3c0dd8a2af](https://github.com/moq-dev/moq/commit/3c0dd8a2af) | +213 | -3,142 | +| Take a reserved group without asking its cursor again, [68fb455bac](https://github.com/moq-dev/moq/commit/68fb455bac) | +204 | -455 | + +The session column starts at the 2026-11-01 harness; blank cells predate it. [moq-dev/moq#6995](https://github.com/moq-dev/moq/pull/6995)'s session cost is the probe's walk of the live groups (`TrackState::live`, +4.0k inclusive) on both the reader's cursor and its forward-drop cursor. + +## Fixes in this PR + +Each is its own commit, counted in one run (run.s6IjbN in the evidence) at 512 subscribers per delivered group, each row on top of the one above: + +| Change | Model | Session | +| --- | ---: | ---: | +| `main` | 13,300 | 120,948 | +| A live reader's probe skips the fills while it misses nothing | 10,477 (-2,823) | 114,139 (-6,809) | +| A window with one set of runs reads without growing across two | 10,126 (-351) | 113,133 (-1,007) | +| This branch | 10,126 (-23.9%) | 113,133 (-6.5%) | + +Against `main` at [619df6e1b3](https://github.com/moq-dev/moq/commit/619df6e1b3) (run.evL1wx), after merging it: model 13,368 to 10,119 (-24.3%), session 121,114 to 113,143 (-6.6%). + +`Cursor::probe` is the relay's check for groups a reader has missed. Each time a subscriber asks for its next group, the probe looks at what arrived on the track since its last look and decides whether any group the reader still wants has left the cache without the live feed delivering it (evicted, skipped, or from before the feed started), and asks upstream to fill those. A live reader that takes each group as it arrives never has such a gap, yet it paid for the whole check on every group: locking the subscription and the track's shared state, cloning the track's fills, walking the arrival ledger, checking each new group for holes, and running the ask-and-answer loop. Each time the cache aged out an old group, the eviction also landed with every reader, all of which had already taken it, and forced each one to re-run the full check. The fast path judges a group the arrival walk serves without the hole, backfill, and ledger passes, skips the subscription, fills, and answers when nothing is asked or owed, and starts each reader's eviction inbox above what it handed out. It recovers most of [moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171)'s and [moq-dev/moq#7339](https://github.com/moq-dev/moq/pull/7339)'s cost. + +kio's `WaiterList::register` is the session path's second-largest self cost: 7.2k per group. A waiter records 8 lists, and a relay session's control and serving tasks park on more, so every list past the eighth takes a duplicate entry on every poll. Remembering 16 cut the session by 4,424 (-3.9%) and was reverted: with it, two moq-net integration tests (`errored_end::an_errored_end_keeps_its_history_across_a_hop`, `quiet_catalog::a_quiet_catalog_reaches_an_unfloored_relay_reader_when_a_peer_resumes_past_it`) fail, and they pass with 8. A read whose waiter parks on more lists than it records is retired and walks again; with more records those reads stay parked, and something they depend on doesn't wake them. That is now an m0 correctness quest, [a waiter's record limit changes no behavior](../quest/m0/kio-waiter-records.md), and the per-group costs this report attributes and nothing here cut are accepted in the [release-perf epic](../quest/m1/release-perf/README.md). + +On the quiet test host (run.we4umM, one-minute load at most 4.7), chat over io\_uring went from 154.5k \[154.1-155.0] instructions per frame on `main` to 148.9k \[148.3-150.8] with the probe and window changes (-3.6%), four rounds each, every cell at 20,000 frames/s with no loss. The relay's QUIC, crypto, and syscalls are most of its cost, so the model's -23% shows as a few percent there. + +## Validation + +Every count, delivery estimate, loss figure, and host load cited here is in [the evidence](relay-cpu-bisect.json), keyed by run artifact. Raw captures (`perf.data`, folded stacks, reports) stay in the measuring checkout's ignored `.tmp/bench/`. diff --git a/bench/README.md b/bench/README.md index fc8897240b9..c314333f0e2 100644 --- a/bench/README.md +++ b/bench/README.md @@ -283,6 +283,118 @@ keeps `kernel.perf_event_paranoid=-1`. Profile there only while it is quiet, with no other agents running and load near zero, since other work on the host shows up as lock waits and CPU the relay did not cause. +### Instructions + +Count instructions and cycles (user and kernel) per delivered frame, with +fixed-period instruction stacks for source attribution: + +```bash +BENCH_RELAY_CPUS=4-7 BENCH_GENERATOR_CPUS=0-3 BENCH_RUNTIME_ROUNDS=5 \ + BENCH_INSTRUCTION_PERIOD=10000019 \ + nix develop --command just bench-profile instructions chat 4 io-uring-workers cdff3c8f95 +BENCH_RELAY_CPUS=4-7 BENCH_GENERATOR_CPUS=0-3 \ + nix develop --command just bench-profile instructions video 4 tokio-workers cdff3c8f95 +# A bisect step: every listed base and the current tree, rotated in each round. +BENCH_RELAY_CPUS=4-7 BENCH_GENERATOR_CPUS=0-3 BENCH_GENERATOR=21d6da9fc1 \ + nix develop --command just bench-profile instructions chat 4 io-uring-workers "fe722cec9f 2d73eea67d e84ffb3de3" +``` + +Choose disjoint allowed CPU sets, with at least one relay CPU per worker. +Hybrid CPUs require performance cores, and count their `cpu_core` events. +The relay and generator keep these affinities for every revision. Instructions +mode runs alone. + +Each base builds into its own target under `target/bench/revisions/`, +from a detached worktree in `.tmp/` that the run removes when it exits, so a +later run reuses that build. The revisions build side by side on every allowed +CPU, since fat LTO links on one thread, `BENCH_BUILD_JOBS` (default 4) at a +time: each link takes gigabytes, so lower it on a host with little memory. +Every round measures each base and the +current tree back to back, rotating which goes first, for +`BENCH_RUNTIME_ROUNDS` rounds (default 3). The revisions are each other's +control, so there is no unprofiled run per revision. One generator drives them +all: the current tree's, unless `BENCH_GENERATOR` names a revision. Hold it +fixed across a bisect's runs, so client changes (flow control, subscription +options) can't move what an old relay spends per frame. + +The run ends with each revision's median and range at each worker count in +`instructions-summary.json`, including the groups the generator confirmed lost +between the reports bracketing each counter window and the host's highest +one-minute load during it. Admission only waits for room when the run starts, +so judge cells by the load they actually ran under. + +After seven seconds of warmup, acknowledged perf enable/disable commands bound +five seconds of relay counters. `instructions/instructions.json` records both +acknowledgement brackets, their uncertainty, and the delivered frames linearly +interpolated between the generator's surrounding 500 ms reports at those same +boundaries. This is a sampled delivery estimate, not exact frame accounting. +Missing, uncounted, or multiplexed counters and unbracketed delivery fail the run. +Lost or throttled instruction samples also fail. Inspect the host's sample-rate +limit and buffers before choosing a larger fixed period for the whole comparison; +some sample-rate limits throttle on every interrupt, regardless of the period. +On the profiling host, a 1,000 Hz limit throttled every sample even at a period +of 10,000,019. Temporarily raising `kernel.perf_event_max_sample_rate` to 100,000 +allowed complete captures. Save and restore the original setting around the +whole gated run; the driver records the observed limit and changes no sysctls. +The host sampler lasts nineteen seconds, so delivery reports bracket both edges +of the counter window and cover the sampling window after it. The normal +`summary.json` still describes its own final throughput window; use +`instructions.json` for instruction comparisons. + +`BENCH_INSTRUCTION_PERIOD` defaults to 1,000,003 retired instructions per +sample. The fixed-period recording covers the five seconds after the counter +window, not the window itself, so sampling interrupts, which run as the relay's +kernel instructions, stay out of the counts. `instructions.folded` weights each +stack by that period; scale stacks to a frame by their share of samples. +`perf.data`, raw `stat.json`, `perf.log`, thread names, CPU sets, and delivery +reports remain with the evidence. Both counters and samples cover only the +relay. + +Compare equivalent delivery and loss. Instructions reduce frequency noise, but +not batching: a relay contending with other host work handles more frames per +wakeup, so each frame costs fewer instructions. On the profiling host, cells +measured under a one-minute load above about 9 read 5-20% low, so compare +cells from quiet stretches within one run. Pinning does not remove interference +from other host work. + +### Fan-out + +Relay instruction counts move with host load (see Instructions above), which +hides steps of a few percent. `just bench-fanout` counts the chat fan-out +without a relay process or sockets, so the counts are the code's alone: + +```bash +nix develop --command just bench-fanout "cdff3c8f95 21d6da9fc1" +nix develop --command just bench-fanout "" "model" "1 512" +``` + +Two scenarios run every subscriber count on one thread: + +- `model`: one finished group a cycle through the track model, which every + subscriber takes before parking again, as moq-net's `track_fanout_group` + bench does. It builds against every revision since + [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95). +- `session`: one publisher session into a relay origin and one viewer session + out of it per subscriber, over `transport::memory`, on one current-thread + runtime. It counts the publisher, relay, and viewer sessions together, but no + QUIC: in-tree qmux, receive leases, and crypto only show in relay counts. A + revision whose session API predates this harness reports it unsupported. + +Each `bench/fanout/` crate builds against each base's and the current +tree's own crates and lock, `BENCH_BUILD_JOBS` at a time, into +`target/bench/fanout/`, or +`BENCH_FANOUT_TARGET` where a build cache might evict that during a long sweep. Each +count pins the harness to one CPU (`BENCH_FANOUT_CPU`, by default the last +allowed one) and takes its user instructions from `perf stat` for a run of +zero cycles and one of many; the difference over the groups delivered is +`fanout-summary.json`. Each process draws its own hash seeds, which move a +count by up to about 0.5%, so each is the median of three. `BENCH_FANOUT_CALLGRIND=1`, with `valgrind` on `PATH`, also keeps +callgrind outputs of both runs under `fanout///callgrind/`; +`python3 bench/fanout.py functions ZERO FULL UNITS` ranks functions by +instructions per group, and `python3 bench/fanout.py compare BASE CURRENT +SUBSCRIBERS` ranks what moved between two revisions' `callgrind/` directories. `BENCH_FANOUT_VERSION` picks the session's protocol +version (default `moq-lite-06`). + ### Heap The heap mode builds the relay with `moq-relay/jemalloc` (the other modes keep diff --git a/bench/fanout.py b/bench/fanout.py new file mode 100644 index 00000000000..5e7c04840f2 --- /dev/null +++ b/bench/fanout.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +"""Count chat fan-out instructions per delivered group, independent of host load. + +`bench/run.sh --fanout` builds each `bench/fanout/` harness against every +revision. Each harness runs its whole fan-out on one thread, so the user instructions it +retires depend only on the code: this pins it to one CPU, counts a run of zero cycles and +one of many, and divides the difference by the groups delivered. +""" + +import argparse +import json +import math +import os +import re +import subprocess +from pathlib import Path + +# Group deliveries per measurement: subscribers times cycles, so every subscriber count +# runs about as long. The session harness costs about ten times the model's per group. +UNITS = {"model": 1_000_000, "session": 100_000} + + +def cycles(scenario, subscribers): + return max(1, math.ceil(UNITS[scenario] / subscribers)) + + +def parse_stat(text): + """Sum the user instructions perf stat's CSV reports, one line per PMU on a hybrid CPU.""" + total, counted = 0, False + for line in text.splitlines(): + fields = line.split(",") + if len(fields) < 5 or "instructions" not in fields[2]: + continue + if fields[0] in ("", ""): + continue + if float(fields[4] or 0) != 100: + raise ValueError(f"instructions were multiplexed and undercounted: {line}") + total += int(fields[0]) + counted = True + if not counted: + raise ValueError(f"perf counted no instructions:\n{text}") + return total + + +def count(binary, args, cpu): + """User instructions `binary args` retires pinned to `cpu`.""" + run = subprocess.run( + ["taskset", "-c", str(cpu), "perf", "stat", "-x,", "-e", "instructions:u", binary, *map(str, args)], + capture_output=True, + text=True, + env=dict(os.environ, LC_ALL="C"), + ) + if run.returncode != 0: + raise ValueError(f"{binary} {args} failed:\n{run.stderr}") + return parse_stat(run.stderr) + + +# Each process draws its own hash seeds, which move a count by up to about 0.5%, so each +# count is the median of a few. +REPEATS = 3 + + +def per_group(scenario, binary, subscribers, cpu, extra=()): + from statistics import median + + n = cycles(scenario, subscribers) + counts = [] + for _ in range(REPEATS): + base = count(binary, [subscribers, 0, *extra], cpu) + full = count(binary, [subscribers, n, *extra], cpu) + counts.append((full - base) / (n * subscribers)) + return median(counts) + + +def callgrind(scenario, binary, subscribers, out, extra=()): + """Exact per-function attribution: both runs under callgrind, kept for callgrind_annotate.""" + n = max(1, cycles(scenario, subscribers) // 10) + out.mkdir(parents=True, exist_ok=True) + for label, run in (("zero", 0), ("full", n)): + subprocess.run( + [ + "valgrind", + "--tool=callgrind", + f"--callgrind-out-file={out / f'{subscribers}.{label}.out'}", + binary, + str(subscribers), + str(run), + *extra, + ], + check=True, + capture_output=True, + ) + (out / f"{subscribers}.units").write_text(f"{n * subscribers}\n") + + +def functions(zero, full, units, inclusive=False): + """Instructions per group by function, from two callgrind outputs of the same binary.""" + + def load(path): + text = subprocess.run( + [ + "callgrind_annotate", + f"--inclusive={'yes' if inclusive else 'no'}", + "--threshold=100", + "--auto=no", + "--show-percs=no", + str(path), + ], + capture_output=True, + text=True, + check=True, + ).stdout + totals = {} + for line in text.splitlines(): + match = re.match(r"\s*([\d,]+)\s+(\S.*)$", line) + if not match: + continue + name = re.sub(r"\s+\[[^\]]*\]$", "", match.group(2)) + name = re.sub(r"^([^ <]*\.rs|\?\?\?):", "", name) + totals[name] = totals.get(name, 0) + int(match.group(1).replace(",", "")) + return totals + + before, after = load(zero), load(full) + names = set(before) | set(after) + return {name: (after.get(name, 0) - before.get(name, 0)) / units for name in names} + + +def differences(base, current, subscribers, inclusive, top): + """What each function costs per group in `current` against `base`: the largest moves + either way, from the callgrind outputs `--callgrind` kept for each.""" + + def per_group(directory): + units = int((directory / f"{subscribers}.units").read_text()) + zero, full = directory / f"{subscribers}.zero.out", directory / f"{subscribers}.full.out" + return functions(zero, full, units, inclusive) + + before, after = per_group(base), per_group(current) + moves = sorted( + ( + (after.get(name, 0) - before.get(name, 0), before.get(name, 0), after.get(name, 0), name) + for name in set(before) | set(after) + ), + reverse=True, + ) + shown = moves[:top] + [None] + moves[-top // 3 :] + return [ + " ..." if row is None else f"{row[0]:+10.1f} {row[1]:10.1f} {row[2]:10.1f} {row[3][:160]}" for row in shown + ] + + +def summarize(run): + """Print and keep each scenario's instructions per group, by subscribers and revision.""" + rows = [json.loads(line) for line in (run / "fanout.jsonl").read_text().splitlines()] + (run / "fanout-summary.json").write_text(json.dumps(rows, indent=2) + "\n") + labels = list(dict.fromkeys(row["label"] for row in rows)) + for scenario in dict.fromkeys(row["scenario"] for row in rows): + print(f"\n{scenario}: instructions per delivered group") + print(f"{'subscribers':>11}" + "".join(f"{label[:16]:>18}" for label in labels)) + for subscribers in sorted({row["subscribers"] for row in rows if row["scenario"] == scenario}): + cells = { + row["label"]: row.get("per_group") + for row in rows + if row["scenario"] == scenario and row["subscribers"] == subscribers + } + print( + f"{subscribers:>11}" + + "".join( + f"{cells[label]:>18.1f}" if cells.get(label) is not None else f"{'unsupported':>18}" + for label in labels + ) + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + measure = sub.add_parser("measure") + measure.add_argument("run", type=Path) + measure.add_argument("label") + measure.add_argument("revision") + measure.add_argument("scenario", choices=sorted(UNITS)) + measure.add_argument("binary") + measure.add_argument("cpu") + measure.add_argument("subscribers", type=int, nargs="+") + measure.add_argument("--version", default=None) + measure.add_argument("--callgrind", action="store_true") + summary = sub.add_parser("summarize") + summary.add_argument("run", type=Path) + compare = sub.add_parser("compare") + compare.add_argument("base", type=Path, help="callgrind/ directory of the base revision") + compare.add_argument("current", type=Path, help="callgrind/ directory of the revision compared") + compare.add_argument("subscribers", type=int) + compare.add_argument("--inclusive", action="store_true") + compare.add_argument("--top", type=int, default=30) + top = sub.add_parser("functions") + top.add_argument("zero", type=Path) + top.add_argument("full", type=Path) + top.add_argument("units", type=int) + top.add_argument("--inclusive", action="store_true") + top.add_argument("--top", type=int, default=40) + args = parser.parse_args() + try: + if args.command == "measure": + extra = [args.version] if args.version and args.scenario == "session" else [] + with (args.run / "fanout.jsonl").open("a") as out: + for subscribers in args.subscribers: + row = dict( + label=args.label, + revision=args.revision, + scenario=args.scenario, + subscribers=subscribers, + cycles=cycles(args.scenario, subscribers), + per_group=None, + ) + if args.binary != "-": + row["per_group"] = per_group(args.scenario, args.binary, subscribers, args.cpu, extra) + if args.callgrind: + target = args.run / "fanout" / args.label / args.scenario / "callgrind" + callgrind(args.scenario, args.binary, subscribers, target, extra) + out.write(json.dumps(row) + "\n") + elif args.command == "summarize": + summarize(args.run) + elif args.command == "compare": + for line in differences(args.base, args.current, args.subscribers, args.inclusive, args.top): + print(line) + else: + ranked = functions(args.zero, args.full, args.units, args.inclusive) + for name, value in sorted(ranked.items(), key=lambda item: -item[1])[: args.top]: + print(f"{value:10.1f} {name[:180]}") + except (ValueError, OSError, subprocess.CalledProcessError) as error: + parser.exit(1, f"fanout: {error}\n") + + +if __name__ == "__main__": + main() diff --git a/bench/fanout.test.py b/bench/fanout.test.py new file mode 100644 index 00000000000..33f2d738a1b --- /dev/null +++ b/bench/fanout.test.py @@ -0,0 +1,55 @@ +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location("fanout", Path(__file__).with_name("fanout.py")) +fanout = importlib.util.module_from_spec(spec) +spec.loader.exec_module(fanout) + + +class Fanout(unittest.TestCase): + def test_hybrid_counts_sum_each_pmu_that_ran(self): + stat = "\n".join( + [ + "1200,,cpu_atom/instructions/u,100,100.00,,", + ",,cpu_core/instructions/u,0,0.00,,", + "34,,cpu_atom/instructions/u,100,100.00,,", + ] + ) + self.assertEqual(fanout.parse_stat(stat), 1234) + + def test_multiplexed_or_absent_counts_are_refused(self): + with self.assertRaisesRegex(ValueError, "multiplexed"): + fanout.parse_stat("1200,,instructions/u,100,50.00,,") + with self.assertRaisesRegex(ValueError, "no instructions"): + fanout.parse_stat(",,cpu_core/instructions/u,0,0.00,,") + + def test_each_count_divides_the_cycles_it_added_by_the_groups_delivered(self): + counts = {0: 1_000, fanout.cycles("model", 4): 1_000 + 7 * fanout.cycles("model", 4) * 4} + with patch.object(fanout, "count", side_effect=lambda _, args, __: counts[args[1]]) as counted: + self.assertEqual(fanout.per_group("model", "bin", 4, "0"), 7) + self.assertEqual(counted.call_count, 2 * fanout.REPEATS) + self.assertEqual(fanout.cycles("session", 512), 196) + self.assertEqual(fanout.cycles("model", 2_000_000), 1) + + def test_summary_names_unsupported_cells(self): + with tempfile.TemporaryDirectory() as directory: + run = Path(directory) + rows = [ + dict(label="base", scenario="session", subscribers=8, per_group=None), + dict(label="current", scenario="session", subscribers=8, per_group=12.5), + ] + (run / "fanout.jsonl").write_text("".join(json.dumps(row) + "\n" for row in rows)) + with patch("builtins.print") as printed: + fanout.summarize(run) + table = "\n".join(str(call.args[0]) for call in printed.call_args_list) + self.assertIn("unsupported", table) + self.assertIn("12.5", table) + self.assertEqual(json.loads((run / "fanout-summary.json").read_text()), rows) + + +if __name__ == "__main__": + unittest.main() diff --git a/bench/fanout/model/Cargo.toml b/bench/fanout/model/Cargo.toml new file mode 100644 index 00000000000..dc7272be082 --- /dev/null +++ b/bench/fanout/model/Cargo.toml @@ -0,0 +1,19 @@ +# Built by `bench/run.sh --fanout` against each revision's moq-net, outside the workspace, +# so a historical checkout measures the same scenario. Its own lock is that revision's. +[package] +name = "fanout-model" +version = "0.0.0" +edition = "2024" +publish = false + +[workspace] + +[dependencies] +bytes = "1" +kio = { path = "../../../rs/kio" } +moq-net = { path = "../../../rs/moq-net", features = ["async"] } + +[profile.release] +codegen-units = 1 +debug = "line-tables-only" +lto = "fat" diff --git a/bench/fanout/model/src/main.rs b/bench/fanout/model/src/main.rs new file mode 100644 index 00000000000..93ef1b310ae --- /dev/null +++ b/bench/fanout/model/src/main.rs @@ -0,0 +1,43 @@ +//! Chat fan-out through the model alone: one finished group a cycle, which every +//! subscriber takes before parking again, as moq-net's `track_fanout_group` bench does. +//! Usage: `fanout-model SUBSCRIBERS CYCLES`. The work is deterministic, so its retired +//! instructions, less a run of zero cycles, are the per-group cost whatever the host does. +use std::{hint::black_box, task::Poll}; + +use bytes::Bytes; +use moq_net::{Timestamp, broadcast, cache, track}; + +fn main() { + let mut args = std::env::args().skip(1).map(|a| a.parse::().unwrap()); + let (subscribers, cycles) = (args.next().unwrap(), args.next().unwrap()); + let mut info = broadcast::Info::default(); + info.pool = cache::Pool::new(cache::Config::default().with_capacity(64 * 1024).with_expiry(cache::DEFAULT_EXPIRY)); + let broadcast = broadcast::Producer::new(info); + let track = broadcast + .create_track("bench", track::Info::default().with_timescale(moq_net::Timescale::MILLI)) + .unwrap(); + let mut subs: Vec<_> = (0..subscribers).map(|_| track.subscribe(None)).collect(); + let waiters: Vec<_> = (0..subscribers).map(|_| kio::Waiter::noop()).collect(); + for (s, w) in subs.iter_mut().zip(&waiters) { + assert!(matches!(s.poll_recv_group(w), Poll::Pending)); + } + let payload = Bytes::from(vec![0; 64]); + // Warm up past the first groups' one-time work, so both runs share it. + for _ in 0..(16 + cycles) { + let mut group = track.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); + group.finish().unwrap(); + for (s, w) in subs.iter_mut().zip(&waiters) { + match s.poll_recv_group(w) { + Poll::Ready(Ok(Some(g))) => { + black_box(g); + } + _ => unreachable!(), + } + assert!(matches!(s.poll_recv_group(w), Poll::Pending)); + } + } + // Left standing, so teardown stays out of the count. + std::mem::forget((broadcast, track, subs)); + std::process::exit(0); +} diff --git a/bench/fanout/session/Cargo.toml b/bench/fanout/session/Cargo.toml new file mode 100644 index 00000000000..76eaa646abc --- /dev/null +++ b/bench/fanout/session/Cargo.toml @@ -0,0 +1,21 @@ +# Built by `bench/run.sh --fanout` against each revision's moq-net, outside the workspace, +# so a historical checkout measures the same scenario. Its own lock is that revision's. +[package] +name = "fanout-session" +version = "0.0.0" +edition = "2024" +publish = false + +[workspace] + +[dependencies] +bytes = "1" +futures = "0.3" +tokio = { version = "1", features = ["rt", "time"] } +kio = { path = "../../../rs/kio" } +moq-net = { path = "../../../rs/moq-net", features = ["async"] } + +[profile.release] +codegen-units = 1 +debug = "line-tables-only" +lto = "fat" diff --git a/bench/fanout/session/src/main.rs b/bench/fanout/session/src/main.rs new file mode 100644 index 00000000000..d692315eae2 --- /dev/null +++ b/bench/fanout/session/src/main.rs @@ -0,0 +1,95 @@ +//! Chat fan-out through a relay over the in-memory transport: one publisher session into +//! a relay origin and one viewer session out of it per subscriber, all on one thread. Each +//! cycle writes one 64-byte group that every viewer receives and reads. +//! +//! Usage: `fanout-session VIEWERS CYCLES [VERSION]`. Every session runs on this thread's +//! runtime and the memory transport has no sockets or clock waits, so the retired +//! instructions, less a run of zero cycles, are the per-group cost whatever the host does. +use std::hint::black_box; + +use bytes::Bytes; +use moq_net::{Client, Hop, Server, Timestamp, Version, cache, origin, transport::memory}; + +/// An origin whose cache holds what the model scenario's does, so neither the relay's nor a +/// viewer's grows with the groups a run delivers. +fn origin(hop: u64) -> origin::Producer { + let mut config = origin::Config::new(Hop::new(hop).unwrap()); + config.pool = cache::Pool::new(cache::Config::default().with_capacity(64 * 1024)); + let (producer, driver) = origin::Producer::new(config); + tokio::spawn(moq_net::time::run(driver)); + producer +} + +/// Connect `client` to `server` over one in-memory pair, returning both sessions. +async fn link(version: Version, client: Client, server: Server) -> (moq_net::Session, moq_net::Session) { + let mut config = memory::Config::default(); + config.protocol = Some(version.alpn().to_string()); + let (near, far) = memory::pair(config); + // Each driver runs as soon as its handshake resolves: a newer draft's accept waits on + // the client's SETUP, which only its driver sends. + futures::join!( + async { + let (session, driver) = client.connect(kio::clock::now(), near).await.expect("client handshake"); + tokio::spawn(moq_net::time::run(driver)); + session + }, + async { + let (session, driver) = server.accept(kio::clock::now(), far).await.expect("server handshake"); + tokio::spawn(moq_net::time::run(driver)); + session + } + ) +} + +fn main() { + let mut args = std::env::args().skip(1); + let viewers: u64 = args.next().unwrap().parse().unwrap(); + let cycles: usize = args.next().unwrap().parse().unwrap(); + let version: Version = args.next().as_deref().unwrap_or("moq-lite-06").parse().unwrap(); + let runtime = tokio::runtime::Builder::new_current_thread().enable_time().build().unwrap(); + runtime.block_on(async move { + let relay = origin(1); + let publisher = origin(2); + let ingest = link( + version, + Client::new().with_versions(version.into()).with_publisher(publisher.consume()), + Server::new().with_versions(version.into()).with_subscriber(relay.clone()), + ) + .await; + let broadcast = publisher.create_broadcast("room").unwrap(); + let info = moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI); + let mut track = broadcast.create_track("chat", info).unwrap(); + broadcast.announce(Default::default()).unwrap(); + + let mut audience = Vec::new(); + for viewer in 0..viewers { + let local = origin(3 + viewer); + let sessions = link( + version, + Client::new().with_versions(version.into()).with_subscriber(local.clone()), + Server::new().with_versions(version.into()).with_publisher(relay.consume()), + ) + .await; + let remote = local.consume().follow("room", None).unwrap().available().await.expect("broadcast resolves"); + let subscriber = remote.track("chat").unwrap().subscribe(None).await.expect("subscribe"); + audience.push((sessions, local, subscriber)); + } + while track.subscription_changed().await.expect("track open").is_none() {} + + let payload = Bytes::from(vec![0; 64]); + // Past the first groups' one-time work, so the zero-cycle run shares it. + for _ in 0..(16 + cycles) { + let mut group = track.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); + group.finish().unwrap(); + for (_, _, subscriber) in &mut audience { + let mut group = subscriber.recv_group().await.unwrap().expect("a group"); + black_box(group.read_frame().await.unwrap().expect("a frame")); + } + } + // Left standing: their teardown grows with what they delivered, which the zero-cycle + // run would not subtract. + std::mem::forget((ingest, audience, broadcast, relay, publisher)); + }); + std::process::exit(0); +} diff --git a/bench/instructions.py b/bench/instructions.py new file mode 100644 index 00000000000..82ce8eaaaed --- /dev/null +++ b/bench/instructions.py @@ -0,0 +1,393 @@ +#!/usr/bin/env python3 +"""Count relay instructions and sample their callers on disjoint pinned cores.""" + +import argparse +import bisect +import json +import math +import os +import profile as profiles +import signal +import struct +import subprocess +import time +from pathlib import Path + + +def cpus(value): + if not value: + raise ValueError("instructions mode requires BENCH_RELAY_CPUS and BENCH_GENERATOR_CPUS") + result = set() + for part in value.split(","): + first, _, last = part.partition("-") + first, last = int(first), int(last or first) + if first < 0 or last < first: + raise ValueError(f"invalid CPU range: {part}") + result.update(range(first, last + 1)) + return result + + +def settings(workers): + relay = cpus(os.environ.get("BENCH_RELAY_CPUS", "")) + generator = cpus(os.environ.get("BENCH_GENERATOR_CPUS", "")) + allowed = os.sched_getaffinity(0) + if relay & generator or not (relay | generator) <= allowed: + raise ValueError("relay and generator CPU sets must be disjoint subsets of the allowed CPUs") + if len(relay) < max(map(int, workers.split())): + raise ValueError("relay CPU set has fewer CPUs than workers") + # A hybrid event silently counts nothing when its task runs on the other core type. + core = Path("/sys/devices/cpu_core/cpus") + if core.exists() and not (relay | generator) <= cpus(core.read_text().strip()): + raise ValueError("instructions mode requires performance cores on a hybrid CPU") + period = int(os.environ.get("BENCH_INSTRUCTION_PERIOD", "1000003")) + if period <= 0: + raise ValueError("BENCH_INSTRUCTION_PERIOD must be positive") + return dict(relay_cpus=sorted(relay), generator_cpus=sorted(generator), period=period) + + +def events(): + prefix = "cpu_core/" if Path("/sys/devices/cpu_core").exists() else "" + return [f"{prefix}{name}{'/' if prefix else ''}" for name in ("instructions", "cycles")] + + +class Controlled: + """An acknowledged perf control channel bounds the actual counter window.""" + + def __init__(self, command, log): + self.record = command[1] == "record" + read, self.write = os.pipe() + self.read, write = os.pipe() + try: + self.process = subprocess.Popen( + [*command, "--delay", "-1", "--control", f"fd:{read},{write}"], + pass_fds=(read, write), + stdout=log, + stderr=log, + env=dict(os.environ, LC_ALL="C", DEBUGINFOD_URLS=""), + ) + finally: + os.close(read) + os.close(write) + + def command(self, command): + before = time.time_ns() + os.write(self.write, command.encode() + b"\n") + ack = b"" + while not ack.endswith(b"\0"): + if not (part := os.read(self.read, 1)): + raise ValueError(f"perf exited before acknowledging {command}; see perf.log") + ack += part + after = time.time_ns() + if ack != b"ack\n\0": + raise ValueError(f"unexpected perf acknowledgement: {ack!r}") + return [before / 1e6, after / 1e6] + + def close(self): + try: + if self.process.poll() is None: + if self.record: + # Record must finish its header before exiting; stat exits on SIGINT. + self.command("stop") + else: + self.process.send_signal(signal.SIGINT) + except (OSError, ValueError): + self.process.kill() + raise + finally: + os.close(self.write) + os.close(self.read) + self.process.wait() + status = self.process.wait() + if status != 0 and (self.record or status != -signal.SIGINT): + raise ValueError(f"perf failed ({status}); see perf.log") + + +def capture(pid, directory, delay, duration): + if not math.isfinite(delay) or delay < 0 or not math.isfinite(duration) or duration <= 0: + raise ValueError("capture needs a nonnegative delay and a positive duration") + config = settings("1") + for task in (Path("/proc") / str(pid) / "task").iterdir(): + if os.sched_getaffinity(int(task.name)) != set(config["relay_cpus"]): + raise ValueError(f"relay thread {task.name} has unexpected affinity") + directory.mkdir(parents=True, exist_ok=True) + counters = events() + names = profiles.threads(pid) + running = [] + try: + with (directory / "perf.log").open("w") as log: + stat = Controlled( + [ + "perf", + "stat", + "-j", + "-o", + str(directory / "stat.json"), + "-p", + str(pid), + "-e", + ",".join([*counters, "task-clock"]), + ], + log, + ) + running.append(stat) + record = Controlled( + [ + "perf", + "record", + "--quiet", + "--no-buildid-cache", + "-p", + str(pid), + "-e", + counters[0], + "-c", + str(config["period"]), + "--call-graph", + "fp", + "-k", + "monotonic", + "-o", + str(directory / "perf.data"), + ], + log, + ) + running.append(record) + # These also prove both processes have attached before the warmup starts. + stat.command("disable") + record.command("disable") + time.sleep(delay) + # Sampling interrupts run as the relay's kernel instructions, and their cost + # follows its stack depth, so the stacks come from the next window instead. + start = stat.command("enable") + load = [host_load()] + time.sleep(duration) + end = stat.command("disable") + load.append(host_load()) + record.command("enable") + time.sleep(duration) + record.command("disable") + finally: + errors = [] + for process in reversed(running): + try: + process.close() + except (OSError, ValueError) as error: + errors.append(str(error)) + if errors: + raise ValueError("; ".join(errors)) + names.update(profiles.threads(pid)) + (directory / "window.json").write_text( + json.dumps( + dict( + **config, + start_ms=start, + end_ms=end, + threads=names, + events=counters, + sample_rate_limit=int(Path("/proc/sys/kernel/perf_event_max_sample_rate").read_text()), + host_load_1m=load, + ), + indent=2, + ) + + "\n" + ) + + +def host_load(): + """Other host work can change after admission, so each window keeps the load it ran under.""" + return float(Path("/proc/loadavg").read_text().split()[0]) + + +def delivered(rows, at): + """Interpolate delivery, never extrapolate past the generator's reports.""" + index = bisect.bisect_left([row["timestamp_ms"] for row in rows], at) + if index == len(rows) or at < rows[0]["timestamp_ms"]: + raise ValueError("generator reports do not bracket the counter window") + if rows[index]["timestamp_ms"] == at: + return rows[index]["frames_recv"] + first, last = rows[index - 1], rows[index] + weight = (at - first["timestamp_ms"]) / (last["timestamp_ms"] - first["timestamp_ms"]) + return first["frames_recv"] + weight * (last["frames_recv"] - first["frames_recv"]) + + +def reduce(window, stats, rows): + if len(rows) < 2: + raise ValueError("fewer than two generator reports") + for row in rows: + for key in ("timestamp_ms", "frames_recv"): + if not isinstance(row[key], (int, float)) or not math.isfinite(row[key]) or row[key] < 0: + raise ValueError(f"invalid generator counter: {key}") + for first, last in zip(rows, rows[1:]): + if last["timestamp_ms"] < first["timestamp_ms"] or last["frames_recv"] < first["frames_recv"]: + raise ValueError("generator counters moved backwards") + # Catch-up ticks can observe the same cumulative snapshot at millisecond + # resolution. Different counters at that timestamp cannot be interpolated. + if last["timestamp_ms"] == first["timestamp_ms"] and last["frames_recv"] != first["frames_recv"]: + raise ValueError("conflicting generator counters at the same timestamp") + start, end = [sum(window[key]) / 2 for key in ("start_ms", "end_ms")] + if end <= start: + raise ValueError("counter window did not advance") + frames = delivered(rows, end) - delivered(rows, start) + if frames <= 0: + raise ValueError("no frames delivered in the counter window") + counts = {} + for event in [*window["events"], "task-clock"]: + matches = [row for row in stats if row["event"] == event] + if len(matches) != 1: + raise ValueError(f"missing or duplicate perf counter: {event}") + row = matches[0] + value = float(row["counter-value"]) + if not math.isfinite(value) or value <= 0 or float(row["pcnt-running"]) != 100: + raise ValueError(f"invalid or multiplexed perf counter: {event}") + counts[event] = value + return dict( + version=1, + **window, + window_start_ms=start, + window_end_ms=end, + boundary_uncertainty_ms=[b - a for a, b in (window["start_ms"], window["end_ms"])], + delivered_frames=frames, + recv_fps=frames * 1000 / (end - start), + counters=counts, + instructions_per_frame=counts[window["events"][0]] / frames, + cycles_per_frame=counts[window["events"][1]] / frames, + scope="all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + ) + + +def report(directory, load): + validate_data(directory / "perf.data") + window = json.loads((directory / "window.json").read_text()) + stats = [ + json.loads(line.rstrip(",")) + for line in (directory / "stat.json").read_text().splitlines() + if line.strip().startswith("{") + ] + result = reduce(window, stats, [json.loads(line) for line in load.read_text().splitlines()]) + records = profiles.parse(profiles.perf_script(directory / "perf.data"), kernel=True) + records = [row for row in records if row[2].startswith(window["events"][0])] + if not records or not any(profiles.RESOLVED.search(frame) for _, _, _, frames in records for frame in frames): + raise ValueError("no instruction samples have resolved MoQ frames") + from collections import Counter + + folded = Counter() + for tid, _, _, frames in records: + folded[";".join([window["threads"].get(tid, f"tid-{tid}"), *reversed(frames)])] += window["period"] + result["instruction_samples"] = len(records) + (directory / "instructions.folded").write_text( + "".join(f"{stack} {count}\n" for stack, count in folded.most_common()) + ) + (directory / "instructions.json").write_text(json.dumps(result, indent=2) + "\n") + + +def lost(rows, start, end): + """Groups the generator confirmed lost between the reports bracketing the window.""" + before = [row for row in rows if row["timestamp_ms"] <= start] + after = [row for row in rows if row["timestamp_ms"] >= end] + if not before or not after: + raise ValueError("generator reports do not bracket the counter window") + return after[0]["groups_lost"] - before[-1]["groups_lost"] + + +def summarize(run, labels): + """Print each revision's median and range across rounds at each worker count, and keep them.""" + from statistics import median + + result = {} + for label in labels: + configurations = {} + for path in sorted((run / "relay" / label).glob("round-*/instructions-*/*/instructions/instructions.json")): + # Worker counts are separate configurations, never one revision-wide range. + workers = path.relative_to(run / "relay" / label).parts[1].removeprefix("instructions-") + cell = json.loads(path.read_text()) + rows = [json.loads(line) for line in (path.parent.parent / "load.jsonl").read_text().splitlines()] + configurations.setdefault(workers, dict(cells=[]))["cells"].append( + dict( + cell=str(path.relative_to(run)), + instructions_per_frame=cell["instructions_per_frame"], + cycles_per_frame=cell["cycles_per_frame"], + recv_fps=cell["recv_fps"], + groups_lost=lost(rows, cell["window_start_ms"], cell["window_end_ms"]), + host_load_1m=max(cell.get("host_load_1m", [math.nan])), + ) + ) + if not configurations: + raise ValueError(f"no instruction cells for {label}") + for configuration in configurations.values(): + cells = configuration["cells"] + for key in ("instructions_per_frame", "cycles_per_frame", "recv_fps", "groups_lost", "host_load_1m"): + values = [cell[key] for cell in cells] + configuration[key] = dict(median=median(values), min=min(values), max=max(values)) + result[label] = dict(sorted(configurations.items(), key=lambda item: int(item[0]))) + (run / "instructions-summary.json").write_text(json.dumps(result, indent=2) + "\n") + print( + f"\n{'revision':<20} {'workers':>7} {'kinstr/frame [min-max]':>26} {'kcycles/frame':>14} " + f"{'fps [min-max]':>24} {'lost':>6} {'load':>5}" + ) + for label, configurations in result.items(): + for workers, row in configurations.items(): + i, c, f = row["instructions_per_frame"], row["cycles_per_frame"], row["recv_fps"] + print( + f"{label:<20} {workers:>7} {i['median'] / 1e3:>9.1f} [{i['min'] / 1e3:.1f}-{i['max'] / 1e3:.1f}] " + f"{c['median'] / 1e3:>14.1f} {f['median']:>9.0f} [{f['min']:.0f}-{f['max']:.0f}] " + f"{row['groups_lost']['max']:>6} {row['host_load_1m']['max']:>5.1f}" + ) + + +def validate_data(path): + """Reject lost or throttled samples before expensive symbolization.""" + data = path.read_bytes() + if data[:8] != b"PERFILE2" or len(data) < 56: + raise ValueError("unsupported or incomplete perf.data header") + offset, size = struct.unpack_from(" len(data): + raise ValueError("incomplete perf.data recording") + while offset < end: + if end - offset < 8: + raise ValueError("incomplete perf.data record header") + kind, _, size = struct.unpack_from(" end: + raise ValueError("invalid perf.data record") + # perf_event_header types: LOST, THROTTLE, LOST_SAMPLES. perf script's + # --show-lost-events omits THROTTLE, which would bias fixed-period attribution. + if kind in (2, 5, 13): + raise ValueError( + f"perf lost or throttled samples (record type {kind}); inspect the host's " + "kernel.perf_event_max_sample_rate and capture buffers before choosing a fixed period" + ) + offset += size + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + check = sub.add_parser("check") + check.add_argument("workers") + capture_args = sub.add_parser("capture") + capture_args.add_argument("pid", type=int) + capture_args.add_argument("directory", type=Path) + capture_args.add_argument("--delay", type=float, default=7) + capture_args.add_argument("--duration", type=float, default=5) + report_args = sub.add_parser("report") + report_args.add_argument("directory", type=Path) + report_args.add_argument("load", type=Path) + summarize_args = sub.add_parser("summarize") + summarize_args.add_argument("run", type=Path) + summarize_args.add_argument("labels", nargs="+") + args = parser.parse_args() + try: + if args.command == "check": + settings(args.workers) + elif args.command == "capture": + capture(args.pid, args.directory, args.delay, args.duration) + elif args.command == "summarize": + summarize(args.run, args.labels) + else: + report(args.directory, args.load) + except (ValueError, OSError, subprocess.CalledProcessError) as error: + parser.exit(1, f"instructions: {error}\n") + + +if __name__ == "__main__": + main() diff --git a/bench/metadata.py b/bench/metadata.py index 7bf6e731f25..d4e24cd2005 100644 --- a/bench/metadata.py +++ b/bench/metadata.py @@ -69,11 +69,27 @@ def command(*args): "profilers": {tool: command(tool, "--version").splitlines()[0] for tool in ("perf", "jeprof") if shutil.which(tool)} if mode == "--profile" else None, - "workload": None if mode == "--criterion" else {"startup": "2s", "duration": "10s", "report": "500ms"}, + "workload": None + if mode == "--criterion" + else {"startup": "2s", "duration": "10s", "report": "500ms", "instructions_duration": "19s"}, } (out / "metadata.json").write_text(json.dumps(metadata, indent=2) + "\n") subprocess.run( - ["cargo", "tree", "--locked", "-p", "moq-relay", "-e", "features"], + [ + "cargo", + "--config", + str(Path.cwd() / ".cargo/config.toml"), + "tree", + "--manifest-path", + str(Path.cwd() / "Cargo.toml"), + "--locked", + "-p", + "moq-relay", + "-e", + "features", + ], + # Nested revision checkouts must not inherit the current tree's Cargo config. + cwd="/", stdout=(out / "features.txt").open("w"), check=True, ) diff --git a/bench/profile.py b/bench/profile.py index d9c5381ef43..9e7b86c6931 100644 --- a/bench/profile.py +++ b/bench/profile.py @@ -432,7 +432,7 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) commands = parser.add_subparsers(dest="command", required=True) check = commands.add_parser("check", help="refuse a capture this host cannot take") - check.add_argument("mode", choices=["lock", "cpu", "heap"]) + check.add_argument("mode", choices=["lock", "cpu", "heap", "instructions"]) for name, help in [ ("lock", "record worker off-CPU stacks of PID into DIRECTORY"), ("cpu", "sample CPU stacks of every PID thread into DIRECTORY"), diff --git a/bench/profile.test.py b/bench/profile.test.py index 3896329606b..59723561df3 100644 --- a/bench/profile.test.py +++ b/bench/profile.test.py @@ -2,6 +2,7 @@ import json import os import signal +import struct import subprocess import tempfile import unittest @@ -16,6 +17,117 @@ runtime = importlib.util.module_from_spec(spec) spec.loader.exec_module(runtime) +spec = importlib.util.spec_from_file_location("instructions", Path(__file__).with_name("instructions.py")) +instructions = importlib.util.module_from_spec(spec) +spec.loader.exec_module(instructions) + + +class Instructions(unittest.TestCase): + def setUp(self): + self.window = dict(start_ms=[1250, 1252], end_ms=[2250, 2252], events=["instructions", "cycles"]) + self.stats = [ + {"event": event, "counter-value": count, "pcnt-running": 100} + for event, count in [("instructions", 20000), ("cycles", 10000), ("task-clock", 10)] + ] + self.load = [dict(timestamp_ms=t, frames_recv=t * 2) for t in [1000, 1500, 2000, 2500]] + + def test_frames_match_the_acknowledged_counter_window(self): + result = instructions.reduce(self.window, self.stats, self.load) + self.assertEqual(result["delivered_frames"], 2000) + self.assertEqual(result["instructions_per_frame"], 10) + self.assertEqual(result["window_start_ms"], 1251) + self.assertEqual(result["boundary_uncertainty_ms"], [2, 2]) + with self.assertRaisesRegex(ValueError, "bracket"): + instructions.reduce(self.window, self.stats, self.load[:-1]) + self.load[2]["frames_recv"] = 0 + with self.assertRaisesRegex(ValueError, "backwards"): + instructions.reduce(self.window, self.stats, self.load) + + def test_partial_or_multiplexed_counts_are_refused(self): + for value in ["", "nan", "0"]: + with self.subTest(value=value), self.assertRaises(ValueError): + instructions.reduce( + self.window, [dict(self.stats[0], **{"counter-value": value}), *self.stats[1:]], self.load + ) + with self.assertRaisesRegex(ValueError, "multiplexed"): + instructions.reduce(self.window, [dict(self.stats[0], **{"pcnt-running": 99}), *self.stats[1:]], self.load) + with self.assertRaisesRegex(ValueError, "missing"): + instructions.reduce(self.window, self.stats[1:], self.load) + + def test_identical_cumulative_snapshots_remain_interpolatable(self): + self.load.insert(2, dict(self.load[1])) + result = instructions.reduce(self.window, self.stats, self.load) + self.assertEqual(result["delivered_frames"], 2000) + self.load[2]["frames_recv"] += 1 + with self.assertRaisesRegex(ValueError, "same timestamp"): + instructions.reduce(self.window, self.stats, self.load) + + def test_cpu_sets_cannot_overlap_or_escape_the_allowed_set(self): + with patch.object(os, "sched_getaffinity", return_value={0, 1, 2, 3}, create=True): + for relay, generator in [("0-1", "1-2"), ("4", "0"), ("0", ""), ("0-1", "3-2")]: + with ( + self.subTest(relay=relay, generator=generator), + patch.dict(os.environ, BENCH_RELAY_CPUS=relay, BENCH_GENERATOR_CPUS=generator), + self.assertRaises(ValueError), + ): + instructions.settings("1") + + def test_summary_reports_each_revision_across_rounds(self): + with tempfile.TemporaryDirectory() as directory: + run = Path(directory) + for label, workers, values in [ + ("base-0123", 4, [12, 10, 11]), + ("base-0123", 16, [40]), + ("current", 4, [20, 21]), + ]: + for round, value in enumerate(values, 1): + cell = run / "relay" / label / f"round-{round}" / f"instructions-{workers}" / "chat" + (cell / "instructions").mkdir(parents=True) + (cell / "instructions" / "instructions.json").write_text( + json.dumps( + dict( + instructions_per_frame=value, + cycles_per_frame=value * 2, + recv_fps=1000, + window_start_ms=1250, + window_end_ms=2250, + host_load_1m=[round, 0.5], + ) + ) + ) + rows = [dict(timestamp_ms=t, groups_lost=lost) for t, lost in [(1000, 1), (1500, 2), (2500, round)]] + (cell / "load.jsonl").write_text("".join(json.dumps(row) + "\n" for row in rows)) + with patch("builtins.print"): + instructions.summarize(run, ["base-0123", "current"]) + summary = json.loads((run / "instructions-summary.json").read_text()) + # Each worker count is its own configuration, not part of one revision-wide range. + self.assertEqual(list(summary["base-0123"]), ["4", "16"]) + self.assertEqual(summary["base-0123"]["4"]["instructions_per_frame"], dict(median=11, min=10, max=12)) + self.assertEqual(summary["base-0123"]["16"]["instructions_per_frame"], dict(median=40, min=40, max=40)) + self.assertEqual(summary["current"]["4"]["instructions_per_frame"]["median"], 20.5) + # Loss counts between the reports bracketing the window, not over the lifetime. + self.assertEqual([cell["groups_lost"] for cell in summary["base-0123"]["4"]["cells"]], [0, 1, 2]) + self.assertEqual(summary["base-0123"]["4"]["host_load_1m"], dict(median=2, min=1, max=3)) + with self.assertRaisesRegex(ValueError, "no instruction cells"): + instructions.summarize(run, ["missing"]) + + def test_lost_or_throttled_samples_refuse_attribution(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "perf.data" + for kind in [2, 5, 13, 9]: + header = bytearray(56) + header[:8] = b"PERFILE2" + struct.pack_into(""$log" 2>&1 & + local -a affinity=() + if [[ ${INSTRUCTION_AFFINITY:-false} == true ]]; then + affinity=(taskset -c "$BENCH_RELAY_CPUS") + fi + env "${environment[@]}" "${affinity[@]}" "$relay" "$config" >"$log" 2>&1 & RELAY_PID=$! for _ in {1..100}; do @@ -296,16 +300,27 @@ run_workload() { if [[ $(uname -s) == Linux ]]; then # The sampler owns the measurement lifetime. The generator runs until # interrupted after sampling, so its normal exit cannot race /proc reads. - "${load[@]}" >"$directory/load.log" 2>&1 & + local -a affinity=() + if [[ ${INSTRUCTION_AFFINITY:-false} == true ]]; then + affinity=(taskset -c "$BENCH_GENERATOR_CPUS") + fi + "${affinity[@]}" "${load[@]}" >"$directory/load.log" 2>&1 & LOAD_PID=$! generator_pid=$LOAD_PID + local duration=10s + if [[ $PROFILE == instructions ]]; then duration=19s; fi "$HOST_BIN" --pid "$relay_pid" --pid "$generator_pid" --threads \ - --interval 500ms --duration 10s --output "$host" \ + --interval 500ms --duration "$duration" --output "$host" \ >"$directory/host.log" 2>&1 & HOST_PID=$! # The second half of the host window, matching the summary's throughput window. # Its own process group, so cleanup also stops perf and its workload. case $PROFILE in + instructions) + setsid python3 "$ROOT/bench/instructions.py" capture "$relay_pid" "$profile" \ + --delay 7 --duration 5 >"$directory/profile.log" 2>&1 & + PROFILE_PID=$! + ;; lock | cpu) setsid python3 "$ROOT/bench/profile.py" "$PROFILE" "$relay_pid" "$profile" --delay 5 --duration 5 \ >"$directory/profile.log" 2>&1 & @@ -348,6 +363,9 @@ run_workload() { --settled "http://127.0.0.1:$((RELAY_PORT + 1))/sessions" fi stop_relay + if [[ $PROFILE == instructions ]]; then + python3 "$ROOT/bench/instructions.py" report "$profile" "$stats" + fi if [[ $PROFILE == heap ]]; then python3 "$ROOT/bench/profile.py" heap "$relay" "$profile" fi diff --git a/bench/run.sh b/bench/run.sh index 79fafa2345a..356c7282b8a 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -10,6 +10,8 @@ set -euo pipefail # `--profile` captures relay profiles: std mutex waits by stack (lock), CPU stacks # (cpu), or jemalloc heap snapshots and allocation churn (heap). # `--shard` sweeps the QUIC shard count against the connection count. +# `--fanout` counts the chat fan-out's instructions per delivered group on one thread, +# for each BENCH_BASE revision and the current tree, independent of host load. MODE=${1:-} ROUNDS=${BENCH_RUNTIME_ROUNDS:-3} @@ -24,6 +26,12 @@ ROUNDS=$((10#$ROUNDS)) # the performance cores it may already use. The kernel only lists them on Intel # hybrid parts. PINNED= +# Builds don't affect what a benchmark measures, so instructions mode compiles on +# every CPU this shell may use before it pins itself. Linux only, like that mode. +ALLOWED_CPUS= +if [[ $(uname -s) == Linux ]]; then + ALLOWED_CPUS=$(python3 -c 'import os; print(",".join(map(str, sorted(os.sched_getaffinity(0)))))') +fi if [[ -r /sys/devices/cpu_core/cpus ]]; then PINNED=$( python3 - "$$" <<'EOF' @@ -70,9 +78,17 @@ case $MODE in BASE= ;; --profile) - BASE=${BENCH_BASE:-} - if [[ -n $BASE ]]; then - BASE=$(git rev-parse --verify "$BASE^{commit}") + # Instructions mode accepts several space-separated bases, rotated with current in each round. + BASES=() + for value in ${BENCH_BASE:-}; do + BASES+=("$(git rev-parse --verify "$value^{commit}")") + done + BASE=${BASES[*]} + # A client revision held fixed across bisect runs, so a newer client's behavior + # doesn't move the historical relays it drives. + GENERATOR= + if [[ -n ${BENCH_GENERATOR:-} ]]; then + GENERATOR=$(git rev-parse --verify "$BENCH_GENERATOR^{commit}") fi PROFILE_RUNTIME=${BENCH_PROFILE_RUNTIME:-tokio-workers} if [[ ! $PROFILE_RUNTIME =~ ^(tokio-workers|io-uring-workers)$ ]]; then @@ -88,15 +104,24 @@ case $MODE in fi PROFILE_WORKERS=${BENCH_PROFILE_WORKERS:-1 4 16} if [[ -z $PROFILE_MODES ]]; then - printf 'BENCH_PROFILE_MODES must name one or more of: lock cpu heap\n' >&2 + printf 'BENCH_PROFILE_MODES must name one or more of: lock cpu heap instructions\n' >&2 exit 1 fi for value in $PROFILE_MODES; do - if [[ ! $value =~ ^(lock|cpu|heap)$ ]]; then - printf 'unknown profile mode %q; expected lock, cpu, or heap\n' "$value" >&2 + if [[ ! $value =~ ^(lock|cpu|heap|instructions)$ ]]; then + printf 'unknown profile mode %q; expected lock, cpu, heap, or instructions\n' "$value" >&2 exit 1 fi done + # Instruction counts pair revisions in rotated rounds, so they run on their own. + if [[ " $PROFILE_MODES " == *' instructions '* && $PROFILE_MODES != instructions ]]; then + printf 'BENCH_PROFILE_MODES=instructions runs alone, not with %q\n' "$PROFILE_MODES" >&2 + exit 1 + fi + if { ((${#BASES[@]} > 1)) || [[ -n $GENERATOR ]]; } && [[ $PROFILE_MODES != instructions ]]; then + printf 'only BENCH_PROFILE_MODES=instructions accepts several BENCH_BASE revisions or BENCH_GENERATOR\n' >&2 + exit 1 + fi for value in $PROFILE_WORKERS; do if [[ ! $value =~ ^[1-9][0-9]{0,2}$ ]] || ((value > 256)); then printf 'profile workers must be integers from 1 to 256, got %q\n' "$value" >&2 @@ -107,6 +132,32 @@ case $MODE in # and CPU stacks unwind by frame pointer alone. export RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }-C force-frame-pointers=yes" ;; + --fanout) + # Several bases, each counted beside the current tree; see bench/fanout.py. + BASES=() + for value in ${BENCH_BASE:-}; do + BASES+=("$(git rev-parse --verify "$value^{commit}")") + done + BASE=${BASES[*]} + FANOUT_SCENARIOS=${BENCH_FANOUT_SCENARIOS:-model session} + FANOUT_SUBSCRIBERS=${BENCH_FANOUT_SUBSCRIBERS:-1 64 512} + for value in $FANOUT_SCENARIOS; do + if [[ ! $value =~ ^(model|session)$ ]]; then + printf 'unknown fan-out scenario %q; expected model or session\n' "$value" >&2 + exit 1 + fi + done + for value in $FANOUT_SUBSCRIBERS; do + if [[ ! $value =~ ^[1-9][0-9]{0,4}$ ]]; then + printf 'fan-out subscribers must be integers from 1 to 99999, got %q\n' "$value" >&2 + exit 1 + fi + done + if [[ -n ${BENCH_FANOUT_CALLGRIND:-} ]] && ! command -v valgrind >/dev/null; then + printf 'BENCH_FANOUT_CALLGRIND needs valgrind on PATH\n' >&2 + exit 1 + fi + ;; --shard) BASE= SHARD_COUNTS=${BENCH_SHARD_COUNTS:-0 1 4 16} @@ -151,6 +202,9 @@ TARGET=${CARGO_TARGET_DIR:-$ROOT/target/bench} if [[ $TARGET != /* ]]; then TARGET=$ROOT/$TARGET fi +# A build cache may evict the directory a `target` symlink points to: recreate it rather +# than fail every build through a dangling link. +mkdir -p "$(readlink -m "$TARGET")" BASE_TARGET=$TARGET/base CURRENT_TARGET=$TARGET/current @@ -167,6 +221,8 @@ fi RUN=$(mktemp -d "$ARTIFACTS/run.XXXXXX") printf 'Benchmark evidence: %s\n' "$RUN" WORKTREE= +WORKTREES=() +BUILD_PIDS=() RELAY_PID= HOST_PID= LOAD_PID= @@ -192,10 +248,16 @@ cleanup() { kill "$RELAY_PID" 2>/dev/null || true wait "$RELAY_PID" 2>/dev/null || true fi - if [[ -n $WORKTREE ]]; then - git -C "$ROOT" worktree remove --force "$WORKTREE" >/dev/null 2>&1 || true - rmdir "$WORKTREE" 2>/dev/null || true - fi + # Builds read the worktrees below, so stop them first. Each leads its own group. + local build worktree + for build in ${BUILD_PIDS[@]+"${BUILD_PIDS[@]}"}; do + kill -- -"$build" 2>/dev/null || true + wait "$build" 2>/dev/null || true + done + for worktree in ${WORKTREE:+"$WORKTREE"} ${WORKTREES[@]+"${WORKTREES[@]}"}; do + git -C "$ROOT" worktree remove --force "$worktree" >/dev/null 2>&1 || true + rmdir "$worktree" 2>/dev/null || true + done # Close both pipe writers before waiting for the owned log copiers. if [[ -n $STDOUT_PID ]]; then exec 1>&3 2>&4 @@ -733,8 +795,10 @@ build_workers() { fi printf 'Building worker runtimes: %s (%s)\n' "$checkout" "$profile" ( - cd "$checkout" - CARGO_TARGET_DIR=$target cargo build --locked --profile "$profile" \ + # Read just this revision's configuration, even in a nested scratch worktree. + cd / + CARGO_TARGET_DIR=$target cargo --config "$checkout/.cargo/config.toml" build \ + --manifest-path "$checkout/Cargo.toml" --locked --profile "$profile" \ -p moq-relay -p moq-bench --features "$features" ) } @@ -846,9 +910,17 @@ run_profiles() { local mode for mode in $PROFILE_MODES; do python3 "$ROOT/bench/profile.py" check "$mode" + if [[ $mode == instructions ]]; then + python3 "$ROOT/bench/instructions.py" check "$PROFILE_WORKERS" + fi done generate_certificate + if [[ $PROFILE_MODES == instructions ]]; then + run_instructions + return + fi + if [[ -n $BASE ]]; then checkout_base "$BASE" mkdir -p "$RUN/base-context" @@ -877,11 +949,193 @@ run_profiles() { done } +# Run each QUEUED_BUILDS shell command with its output in the matching QUEUED_LOGS file, on +# every allowed CPU and BENCH_BUILD_JOBS at a time: fat LTO links on one thread, but each +# link takes gigabytes. Each runs as its own process group, so cleanup stops its rustc too. +# Sets BUILD_STATUS to each command's exit status. +run_queued_builds() { + local jobs=${BENCH_BUILD_JOBS:-4} first index launched + BUILD_STATUS=() + for ((first = 0; first < ${#QUEUED_BUILDS[@]}; first += jobs)); do + BUILD_PIDS=() + for ((index = first; index < first + jobs && index < ${#QUEUED_BUILDS[@]}; index++)); do + # setsid makes each build a group leader without forking. + setsid taskset -c "$ALLOWED_CPUS" bash -c "${QUEUED_BUILDS[$index]}" >"${QUEUED_LOGS[$index]}" 2>&1 & + BUILD_PIDS+=($!) + done + launched=${#BUILD_PIDS[@]} + for ((index = 0; index < launched; index++)); do + BUILD_STATUS[first + index]=0 + wait "${BUILD_PIDS[$index]}" || BUILD_STATUS[first + index]=$? + # Reaped, so cleanup must not signal a group that ID may now name. + unset "BUILD_PIDS[$index]" + done + done +} + +# Check out one revision for this run, in the checkout's ignored scratch space, +# where it loads only its own Cargo configuration. Its target persists, keyed by +# revision; the compiler cache keys on remapped paths, so a fresh checkout reuses it. +revision_checkout() { + local sha=$1 label=$2 worktree + mkdir -p "$ROOT/.tmp" + worktree=$(mktemp -d "$ROOT/.tmp/moq-bench-base.XXXXXX") + WORKTREES+=("$worktree") + git worktree add --detach "$worktree" "$sha" >/dev/null + mkdir -p "$RUN/base-context/$label" + (cd "$worktree" && python3 "$ROOT/bench/metadata.py" "$RUN/base-context/$label" "$sha" --profile) + REVISION_CHECKOUT=$worktree +} + +# Count every base and the current tree in rotated order, so each round measures +# all revisions back to back and no revision always runs first. One generator and +# sampler drive every relay, including old bases whose own harness cannot select +# the io_uring runtime: BENCH_GENERATOR's, or else the current tree's. +run_instructions() { + local label sha round index next generator built + # Measured revisions, then what gets built: those plus a generator measured nowhere. + local -a labels=() targets=() checkouts=() + for sha in ${BASES[@]+"${BASES[@]}"}; do + label=base + if ((${#BASES[@]} > 1)); then label=base-${sha:0:12}; fi + revision_checkout "$sha" "$label" + labels+=("$label") + checkouts+=("$REVISION_CHECKOUT") + targets+=("$TARGET/revisions/$sha/profile") + done + labels+=(current) + checkouts+=("$ROOT") + # The working tree changes under one target; Cargo's fingerprints track it. + targets+=("$TARGET/revisions/current/profile") + local -a build_labels=("${labels[@]}") build_checkouts=("${checkouts[@]}") build_targets=("${targets[@]}") + generator=${targets[-1]} + if [[ -n $GENERATOR ]]; then + generator=$TARGET/revisions/$GENERATOR/profile + if [[ " ${BASES[*]} " != *" $GENERATOR "* ]]; then + revision_checkout "$GENERATOR" generator + build_labels+=(generator) + build_checkouts+=("$REVISION_CHECKOUT") + build_targets+=("$generator") + fi + fi + LOAD_BIN=$generator/profiling/moq-bench + HOST_BIN=$generator/profiling/moq-bench-host + + mkdir -p "$RUN/build" + QUEUED_BUILDS=() QUEUED_LOGS=() + for ((index = 0; index < ${#build_labels[@]}; index++)); do + QUEUED_BUILDS+=("$(declare -f build_workers); build_workers $(printf '%q ' \ + "${build_checkouts[$index]}" "${build_targets[$index]}" profiling instructions)") + QUEUED_LOGS+=("$RUN/build/${build_labels[$index]}.log") + done + run_queued_builds + for ((index = 0; index < ${#build_labels[@]}; index++)); do + if ((BUILD_STATUS[index] != 0)); then + printf 'build failed for %s:\n' "${build_labels[$index]}" >&2 + tail -n 40 "$RUN/build/${build_labels[$index]}.log" >&2 + return 1 + fi + tail -n 1 "$RUN/build/${build_labels[$index]}.log" + done + if [[ $PROFILE_RUNTIME == io-uring-workers ]]; then + for built in "${build_targets[@]}"; do + check_uring "$built/profiling" + done + fi + for ((round = 1; round <= ROUNDS; round++)); do + for ((index = 0; index < ${#labels[@]}; index++)); do + next=$(((index + round - 1) % ${#labels[@]})) + run_profile instructions "${labels[$next]}/round-$round" "${targets[$next]}/profiling/moq-relay" + done + done + python3 "$ROOT/bench/instructions.py" summarize "$RUN" "${labels[@]}" +} + +# Count the chat fan-out per delivered group on one thread, for every base and the current +# tree: each `bench/fanout/` harness builds against that revision's own crates and +# lock, so a count is the code's alone, whatever else the host runs. A scenario whose API a +# revision predates is reported unsupported rather than failing the run. +run_fanout() { + if [[ $(uname -s) != Linux ]] || ! command -v perf >/dev/null; then + printf 'fan-out counts need Linux perf\n' >&2 + return 1 + fi + local cpu=${BENCH_FANOUT_CPU:-${ALLOWED_CPUS##*,}} + local label sha index scenario dir key binary status + local -a labels=() revisions=() checkouts=() keys=() build_dirs=() extra=() + for sha in ${BASES[@]+"${BASES[@]}"}; do + label=base + if ((${#BASES[@]} > 1)); then label=base-${sha:0:12}; fi + revision_checkout "$sha" "$label" + labels+=("$label") + revisions+=("$sha") + checkouts+=("$REVISION_CHECKOUT") + keys+=("$sha") + done + labels+=(current) + revisions+=("$(git rev-parse HEAD)") + checkouts+=("$ROOT") + # The working tree changes under one target; Cargo's fingerprints track it. + keys+=(current) + + # A build cache that evicts the `target` directory under a long sweep fails every later + # build; BENCH_FANOUT_TARGET names a directory it doesn't manage. + local harnesses=${BENCH_FANOUT_TARGET:-$TARGET/fanout} target + QUEUED_BUILDS=() QUEUED_LOGS=() + for ((index = 0; index < ${#labels[@]}; index++)); do + for scenario in $FANOUT_SCENARIOS; do + dir=$RUN/fanout/${labels[$index]}/$scenario + mkdir -p "$dir" + cp -R "$ROOT/bench/fanout/$scenario/." "$dir/" + # This revision's crates, resolved by its own lock. + sed -i "s|path = \"../../../rs/|path = \"${checkouts[$index]}/rs/|" "$dir/Cargo.toml" + cp "${checkouts[$index]}/Cargo.lock" "$dir/Cargo.lock" + target=$harnesses/${keys[$index]}/$scenario + # shellcheck disable=SC2016 # expanded by the build's own shell + QUEUED_BUILDS+=("$(printf 'mkdir -p "$(readlink -m %q)" && cd %q && CARGO_TARGET_DIR=%q cargo build --release' \ + "$target" "$dir" "$target")") + QUEUED_LOGS+=("$dir/build.log") + build_dirs+=("$dir") + done + done + run_queued_builds + for ((index = 0; index < ${#build_dirs[@]}; index++)); do + dir=${build_dirs[$index]} + if ((BUILD_STATUS[index] == 0)); then + continue + fi + if [[ $dir == "$RUN/fanout/current/"* ]]; then + printf 'the current tree'"'"'s fan-out harness failed to build:\n' >&2 + tail -n 40 "$dir/build.log" >&2 + return 1 + fi + printf 'fan-out harness %s does not build; reported unsupported\n' "${dir#"$RUN"/fanout/}" + touch "$dir/unsupported" + done + + if [[ -n ${BENCH_FANOUT_VERSION:-} ]]; then extra+=(--version "$BENCH_FANOUT_VERSION"); fi + if [[ -n ${BENCH_FANOUT_CALLGRIND:-} ]]; then extra+=(--callgrind); fi + printf 'Counting user instructions on CPU %s\n' "$cpu" + for scenario in $FANOUT_SCENARIOS; do + for ((index = 0; index < ${#labels[@]}; index++)); do + dir=$RUN/fanout/${labels[$index]}/$scenario + binary=$harnesses/${keys[$index]}/$scenario/release/fanout-$scenario + if [[ -f $dir/unsupported ]]; then binary=-; fi + # shellcheck disable=SC2086 # one argument per subscriber count + python3 "$ROOT/bench/fanout.py" measure "$RUN" "${labels[$index]}" "${revisions[$index]}" \ + "$scenario" "$binary" "$cpu" $FANOUT_SUBSCRIBERS ${extra[@]+"${extra[@]}"} + done + done + python3 "$ROOT/bench/fanout.py" summarize "$RUN" +} + # Each profiled run follows an unprofiled control on the same binary, so the # throughput and CPU difference shows the profiler's overhead. run_profile() { local mode=$1 label=$2 relay=$3 local workers control profiled + local INSTRUCTION_AFFINITY=false + if [[ $mode == instructions ]]; then INSTRUCTION_AFFINITY=true; fi TRANSPORT=$PROFILE_TRANSPORT case $mode in lock) @@ -900,15 +1154,26 @@ run_profile() { workers control-fps profiled-fps control-CPU profiled-CPU control-RSS profiled-RSS \ before steady teardown churn-MiB/s allocs/s ;; + instructions) + printf '\nInstructions: %s over %s, %s, %s\n' "$PROFILE_WORKLOAD" "$TRANSPORT" "$PROFILE_RUNTIME" "$label" + ;; esac for workers in $PROFILE_WORKERS; do - run_workload "$label/control-$mode-$workers" "$relay" "$PROFILE_WORKLOAD" "$PROFILE_RUNTIME" "$workers" + # Instruction counts compare revisions under the same profiler, so the paired + # revisions are the control instead of an unprofiled run of each. + if [[ $mode != instructions ]]; then + run_workload "$label/control-$mode-$workers" "$relay" "$PROFILE_WORKLOAD" "$PROFILE_RUNTIME" "$workers" + fi PROFILE=$mode run_workload "$label/$mode-$workers" "$relay" "$PROFILE_WORKLOAD" "$PROFILE_RUNTIME" "$workers" PROFILE= control=$RUN/relay/$label/control-$mode-$workers/$PROFILE_WORKLOAD/summary.json profiled=$RUN/relay/$label/$mode-$workers/$PROFILE_WORKLOAD case $mode in + instructions) + jq -r '"\(.instructions_per_frame / 1000 | round) kinstr/frame at \(.recv_fps | round) fps"' \ + "$profiled/instructions/instructions.json" + ;; lock) jq -r -s '[ .[0].recv_fps, .[1].recv_fps, .[0].cpu_cores, .[1].cpu_cores, (.[2].idle | length), @@ -977,6 +1242,9 @@ elif [[ $MODE == --profile ]]; then elif [[ $MODE == --shard ]]; then run_shard_sweep exit 0 +elif [[ $MODE == --fanout ]]; then + run_fanout + exit 0 elif [[ -n $BASE ]]; then BASE_COMMIT=$(git rev-parse --verify "$BASE^{commit}") checkout_base "$BASE_COMMIT" diff --git a/justfile b/justfile index 7f442f26520..f3a9d59a0be 100644 --- a/justfile +++ b/justfile @@ -70,11 +70,18 @@ bench-shard $SHARDS="0 1 4 16" $CONNECTIONS="16 64 256": BENCH_SHARD_COUNTS="$SHARDS" BENCH_SHARD_CONNECTIONS="$CONNECTIONS" bench/run.sh --shard # Profile the relay at each worker count: std mutex waits by stack (lock), CPU -# stacks (cpu), or jemalloc heap and churn (heap), over one transport (quic, ws, wss, tcp, or tls). +# stacks (cpu), hardware instructions and fixed-period stacks (instructions), or +# jemalloc heap and churn (heap), over one transport (quic, ws, wss, tcp, or tls). # Example: `just bench-profile "cpu heap" fanout-heavy 4`, or `just bench-profile cpu chat 4 tokio-workers "" ws`. bench-profile $MODES $WORKLOAD="fanout" $WORKERS="1 4 16" $RUNTIME="tokio-workers" $BASE="" $TRANSPORT="quic": BENCH_PROFILE_MODES="$MODES" BENCH_PROFILE_WORKLOAD="$WORKLOAD" BENCH_PROFILE_WORKERS="$WORKERS" BENCH_PROFILE_RUNTIME="$RUNTIME" BENCH_BASE="$BASE" BENCH_PROFILE_TRANSPORT="$TRANSPORT" bench/run.sh --profile +# Count the chat fan-out's instructions per delivered group on one thread, through the model +# alone and through a relay over the in-memory transport, for each revision in BASE and the +# current tree. The counts don't move with host load: see the Fan-out section of bench/README.md. +bench-fanout $BASE="" $SCENARIOS="model session" $SUBSCRIBERS="1 64 512": + BENCH_BASE="$BASE" BENCH_FANOUT_SCENARIOS="$SCENARIOS" BENCH_FANOUT_SUBSCRIBERS="$SUBSCRIBERS" bench/run.sh --fanout + # Real-browser transport, CMAF, and publish/watch costs, optionally against a base: # `just bench-browser origin/main`. `--smoke` is the bounded correctness lane. [positional-arguments] diff --git a/quest/m0/README.md b/quest/m0/README.md index 25a135c7f0d..b0090933bb2 100644 --- a/quest/m0/README.md +++ b/quest/m0/README.md @@ -35,6 +35,7 @@ since cutting segments from first-frame pts waits on nothing in m1. - [Backport the joined-head FETCH race fix](/quest/m0/backport-fill-head-race.md) - a `release` relay stops FETCHing a group head its own SUBSCRIBE is bringing, which left live readers hanging - [CI cost and latency](/quest/m0/ci/README.md) - cloud CI runs only in a batched merge gate as one `just ci` pass per batch, expensive checks run nightly and gate `release`, caches move to R2, and runners are chosen last - [More tests under load](/quest/m0/test-flakes-2/README.md) - load-only and queue failures, one quest per flake, fixed at the cause, since each one fails a whole batch +- [A waiter's record limit changes no behavior](/quest/m0/kio-waiter-records.md) - two moq-net tests stop failing when kio's waiters remember 16 lists, and the limit becomes a tuning knob worth 3.9% per delivered group - [Backport the relay and auth hardening to release](/quest/m0/backport-release-hardening.md) - `release`'s relay refuses dot-segment paths, serves its HTTP routes with main's response hardening, and withdraws advertisements a peer starves of send credit; its auth client refuses redirects - [Release's MKV importer bounds what it buffers](/quest/m0/backport-mkv-buffer.md) - on `release`, an MKV input that declares a huge element and trickles its bytes no longer grows the importer's memory without limit - [HLS export on a declared grid](/quest/m0/hls-media-grid.md) - the catalog's `hls` field and record tracks go away, and the export numbers segments on `media.join.interval` from media tracks diff --git a/quest/m0/kio-waiter-records.md b/quest/m0/kio-waiter-records.md new file mode 100644 index 00000000000..f4225dae312 --- /dev/null +++ b/quest/m0/kio-waiter-records.md @@ -0,0 +1,35 @@ +# [M] A waiter's record limit changes no behavior + +## Goal + +Raising kio's `PARKED`, the number of lists a `kio::Waiter` remembers parking on, from 8 to 16 changes only what it costs: no moq-net test or relay behavior depends on the limit. Today two moq-net integration tests fail at 16 and pass at 8, so a missed wake, or a test leaning on one by accident, hides behind it. Find and fix that, then set the limit to 16 and take the 3.9% per delivered group the chat fan-out measured. + +## Plan + +Decided 2026-11-03 (maintainer, on [moq-dev/moq#7605](https://github.com/moq-dev/moq/pull/7605)): m0, as a correctness bug rather than a performance tweak, since "the fact that tests fail if we change a constant is a HUGE red flag". + +Reproduce: set `const PARKED: usize = 16;` in `rs/kio/src/waiter.rs`, then + +```bash +nix develop --command cargo nextest run -p moq-net --all-features \ + -E 'test(an_errored_end_keeps_its_history_across_a_hop) | test(a_quiet_catalog_reaches_an_unfloored_relay_reader_when_a_peer_resumes_past_it)' +``` + +- `errored_end::an_errored_end_keeps_its_history_across_a_hop` fails with `subscribe: Stream(App(7))` at `rs/moq-net/tests/integration/errored_end.rs:35`. +- `quiet_catalog::a_quiet_catalog_reaches_an_unfloored_relay_reader_when_a_peer_resumes_past_it` fails with `resume subscribe: Unroutable` at `rs/moq-net/tests/integration/quiet_catalog.rs:178`. +- Both pass at 8, on [0e9b7ae6bb](https://github.com/moq-dev/moq/commit/0e9b7ae6bb4d22b50da244a3c1146acd9bb91f5a) and on `main` at the time. + +The lead: `rs/moq-net/src/util/idle.rs` keeps a read's waiter, and reports the read parked without walking it again, only while every list it parked on is recorded. Past the limit the waiter is marked lost and retired, and the read walks again. At 8 the reads in these tests overflowed and walked on every poll; at 16 they stay parked, so a change they depend on doesn't wake them, through a list not registered with this waiter or a wake that only a walk noticed. Find the read and the change, and fix the wake or the test's hidden dependency, not the limit. + +Then set the limit to 16: + +- `util::idle::test::a_waiter_parked_past_its_records_is_retired` parks on 9 lists, the old limit plus one; make it the limit plus one. +- The larger waiter makes clippy's `large_enum_variant` fire on `Sub` in `rs/moq-net/src/model/resume.rs` and `Reader` in `rs/moq-stats/src/aggregate.rs`; box their pending variants, as [moq-dev/moq#7605](https://github.com/moq-dev/moq/pull/7605)'s reverted commits [101ed5a806](https://github.com/moq-dev/moq/commit/101ed5a806) and [1a5ceeca84](https://github.com/moq-dev/moq/commit/1a5ceeca84) did. +- Measure with `just bench-fanout "" session "64 512"`. On [moq-dev/moq#7605](https://github.com/moq-dev/moq/pull/7605), 16 took the session scenario from 115,000 to 110,524 instructions per group at 64 viewers and from 113,200 to 108,747 at 512 (-3.9%); 12 and 32 each cost about 1-2% more than 16, since every registration searches the records. + +Public API: none expected. Wire: none. + +## Related + +- [Relay CPU per frame since #5926](/quest/m1/release-perf/relay-cpu-bisect.md) - found this, measuring the waiter's registration cost +- [Task sets release memory after a burst](/quest/m2/performance-survey/task-capacity.md) - a task slot's `Park` holds a whole `Waiter`, which 16 records grow diff --git a/quest/m1/release-perf/README.md b/quest/m1/release-perf/README.md index b0c68e29927..725f2cab901 100644 --- a/quest/m1/release-perf/README.md +++ b/quest/m1/release-perf/README.md @@ -285,9 +285,37 @@ bench-only backport to `release`, merged in [moq-dev/moq#7698](https://github.com/moq-dev/moq/pull/7698). Bench-only backports are allowed from now on. +Decided 2026-11-03 (maintainer), in +[moq-dev/moq#7605](https://github.com/moq-dev/moq/pull/7605): these flat +per-group costs of the relay's chat fan-out are accepted as the features they +pay for. Each is instructions per delivered group at 512 subscribers through +the track model, counted by `just bench-fanout` against the merge's parent: + +- Per-reader ranges and time anchors + ([moq-dev/moq#5929](https://github.com/moq-dev/moq/pull/5929)): +902 when + it landed. moq-dev/moq#7605 took back 351 of the window reads it and + [moq-dev/moq#7542](https://github.com/moq-dev/moq/pull/7542) left on every + peek; what remains reads each reader's window and drift edge. +- A subscriber reads only its ranges' frames + ([moq-dev/moq#6129](https://github.com/moq-dev/moq/pull/6129)): +183. +- Public ingest ([moq-dev/moq#6111](https://github.com/moq-dev/moq/pull/6111)): + +200. +- Compressed tracks ([moq-dev/moq#5928](https://github.com/moq-dev/moq/pull/5928)): + +33. +- Closing the relay demand gaps + ([moq-dev/moq#7052](https://github.com/moq-dev/moq/pull/7052)): +365. +- Bounded reads end through routes + ([moq-dev/moq#7625](https://github.com/moq-dev/moq/pull/7625)): +275, and + +450 per group through the in-process session. + +The first four were counted with frame pointers forced and the last two +without, so compare each only with its own parent. None grows with +subscribers. + ## Required - [Relay CPU per frame since #5926](/quest/m1/release-perf/relay-cpu-bisect.md) - the 18-56% rise in relay instructions per delivered frame is bisected, attributed, and its avoidable cost fixed +- [A read moves a small group consumer](/quest/m1/release-perf/group-consumer-size.md) - `group::Consumer` shrinks from 280 bytes, so a read stops copying it through every layer ## Related diff --git a/quest/m1/release-perf/group-consumer-size.md b/quest/m1/release-perf/group-consumer-size.md new file mode 100644 index 00000000000..057debc66c2 --- /dev/null +++ b/quest/m1/release-perf/group-consumer-size.md @@ -0,0 +1,21 @@ +# [M] A read moves a small group consumer + +## Goal + +`moq_net::group::Consumer` shrinks from 280 bytes by boxing what a read rarely touches, so handing a group to each reader stops copying 280 bytes through every layer it passes. `just bench-fanout` shows the drop, and the common read path gains no allocation per group. + +## Plan + +Decided 2026-11-03 (maintainer, on [moq-dev/moq#7605](https://github.com/moq-dev/moq/pull/7605)): a follow-up in this epic. + +Found by [moq-dev/moq#7605](https://github.com/moq-dev/moq/pull/7605)'s session scenario (a publisher, a relay origin, and viewers over the in-memory transport): `memcpy` is its largest self cost, about 9.4k of 113k instructions per delivered group at 64 viewers, mostly moves of `group::Consumer` and the `Peeked` that carries one, by value through `Poll>>` layers. The largest callers per group: `track::Subscriber::poll_peek_group` (982), the viewer's `lite::subscriber::UniServe::poll_serve` (715), `track::Subscriber::take` (566, 442, and 428 across its inlined copies), `track::Cursor::poll_peek` (507), and `kio::Tasks::insert` (454). + +- Find which fields a read touches per group and box the rest, or the rarely-set state, behind one pointer. Check `size_of` before and after. +- Measure with `just bench-fanout "" "model session" "64 512"`, and `BENCH_FANOUT_CALLGRIND=1` with `bench/fanout.py compare` to see the moves shrink. Keep the session scenario's allocations per group flat: the [relay allocations](/quest/m2/perf/relay-allocs.md) quest tracks them. + +Public API: `group::Consumer`'s layout only; no signature change expected. Wire: none. + +## Related + +- [Relay CPU per frame since #5926](/quest/m1/release-perf/relay-cpu-bisect.md) - found this +- [The relay allocates less per delivered frame](/quest/m2/perf/relay-allocs.md) - boxing must not add an allocation per delivered group diff --git a/quest/m1/release-perf/relay-cpu-bisect.md b/quest/m1/release-perf/relay-cpu-bisect.md index 96ce6aa7863..c64a984f82d 100644 --- a/quest/m1/release-perf/relay-cpu-bisect.md +++ b/quest/m1/release-perf/relay-cpu-bisect.md @@ -1,33 +1,61 @@ -# [M] Find and fix the relay's per-frame CPU growth since #5926 +# [M] Find and fix the relay's per-frame CPU growth ## Goal -The commits that raised the relay's instructions per delivered frame between [`cdff3c8f95`](https://github.com/moq-dev/moq/commit/cdff3c8f95) ([moq-dev/moq#5926](https://github.com/moq-dev/moq/pull/5926) as merged) and `main` are found, each rise is attributed to a cause, and the avoidable costs are fixed. A rise that pays for a deliberate feature is accepted with its reason in the [release-perf epic](/quest/m1/release-perf/README.md). +The commits that raised the relay's instructions per delivered frame between [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95) ([moq-dev/moq#5926](https://github.com/moq-dev/moq/pull/5926) as merged) and `main` are found, each rise is attributed to a cause, and the avoidable costs are fixed. A rise that pays for a deliberate feature is accepted with its reason in the [release-perf epic](/quest/m1/release-perf/README.md). ## Plan -Found by [moq-dev/moq#7505](https://github.com/moq-dev/moq/pull/7505), which profiled the io_uring clock read. At `main` [`30f23c3f8a`](https://github.com/moq-dev/moq/commit/30f23c3f8a), relay instructions per delivered frame had risen 25-56% on io_uring since `cdff3c8f95` and 18-38% on the tokio workers since `1dd9b83d1`, with the same delivery: +Found by [moq-dev/moq#7505](https://github.com/moq-dev/moq/pull/7505). Decided 2026-11-01 (maintainer): m1, in this epic, since `main` must not reach the cut carrying accidental per-frame cost. -| workload | runtime | before, kinstr/frame | `30f23c3f8a`, kinstr/frame | -|---|---|---|---| -| chat | io_uring | 86.3 [84.5-87.4] at `cdff3c8f95` | 134.8 [128.5-151.3] | -| video | io_uring | 108.7 [106.6-116.2] at `cdff3c8f95` | 136.1 [132.6-152.8] | -| chat | tokio workers | 104.0 [101.4-105.0] at `1dd9b83d1` | 143.1 [127.5-205.3] | -| video | tokio workers | 141.6 [139.3-148.3] at `1dd9b83d1` | 167.1 [164.9-173.6] | +Bisected in [moq-dev/moq#7605](https://github.com/moq-dev/moq/pull/7605), which committed the driver (`just bench-profile instructions`, several rotated bases per run, a pinned `BENCH_GENERATOR`, per-cell host load) and [the report](/.github/relay-cpu-bisect.md). On chat over io_uring, the relay peaked at 1.82x the old endpoint and `main` is at 1.26x: -Medians of 5 interleaved reps, min-max in brackets. The tokio rows start at [`1dd9b83d1`](https://github.com/moq-dev/moq/commit/1dd9b83d19), #5926's base, since #7505 measured the tokio workers only there and at `main`. Both runtimes moved, so at least one cause is shared code (moq-net's model, kio, or the session layer). That does not rule out a further io_uring-only cost, such as the clock read #7505 profiled. The tokio range also spans #5926 itself, so the two rises can't be subtracted to find one. +- [moq-dev/moq#6224](https://github.com/moq-dev/moq/pull/6224)'s `Reader::learn_end` polling (+26k instructions per frame) and the track lock contention after [moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171) were accidental. [moq-dev/moq#7532](https://github.com/moq-dev/moq/pull/7532) and [moq-dev/moq#7493](https://github.com/moq-dev/moq/pull/7493) already removed both from `main`. +- `main` against the old endpoint: chat 1.26x on io_uring and 1.31x on the tokio workers, video 1.14x on io_uring and 0.96x on the tokio workers. Chat's growth is shared code, in the same functions on both runtimes. -Decided 2026-11-01 (maintainer, follow-up planning after the 2026-11-01 /quest-afk run): m1, in this epic, since `main` must not reach the cut carrying accidental per-frame cost. +Decided 2026-11-03 (maintainer, on [moq-dev/moq#7605](https://github.com/moq-dev/moq/pull/7605)): -- Measure as #7505 did: `--profile profiling` builds with frame pointers and `moq-relay/io-uring,moq-bench/uring`, one target directory per revision, `perf stat` `cpu_core` instructions (user and kernel) over the steady window, divided by the frames the generator's 500 ms reports count over the same window. 4 io_uring workers, not 1, so chat does not saturate a worker. Pin the relay and generator cores so a loaded host still reads instruction deltas. -- Bisect `cdff3c8f95..main` on chat over io_uring first (the largest rise and the tightest ranges), with a step size that keeps each build's cost bounded, then confirm the culprits on video and on the tokio workers. A rise that shows on io_uring but not on the tokio workers is attributed on its own, not folded into a shared culprit. -- Attribute each step with `perf record` at a fixed period, as #7505 did, before deciding fix or accept. Record accepted costs in the epic's Plan with the feature they pay for. -- Commit the driver this needs as a deterministic mode of `just bench-profile` or `just bench-runtime` (pinned cores, instructions per delivered frame, fixed-period samples) instead of another throwaway script; #7505's driver on `bench/relay.sh` was not committed. -- [moq-dev/moq#7493](https://github.com/moq-dev/moq/pull/7493) fixed the bimodal futex cost on the fanout workload; don't fix it twice. A bisect step that lands on the same lock starts from that PR's lock counts. +1. `Cursor::probe` on every peek ([moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171), about 8.6k instructions per frame): + - ✅ Add a fast path in this quest that skips the work while nothing is missable. + - Accept it as the cost of relay backfill. +2. The cursor's window, drift, and live-edge reads ([moq-dev/moq#5929](https://github.com/moq-dev/moq/pull/5929), about 9-11k) and the 2-4k steps from compressed tracks, ranged reads, in-tree qmux, signed priorities, public ingest, and receive leases: + - Accept them as feature costs. + - ✅ Investigate each: profile each step's added instructions, fix what can be cut without changing behavior, and attribute what can't with the evidence. +3. kio's task and waiter growth (about 11k) and the lite publisher's serving and close polling (about 6k): + - ✅ Profile them in this quest, then fix or attribute them before closing. + - Hand them to the [performance audit](/quest/m2/audit-perf.md). + +`Cursor::probe` is the relay's check for groups a reader has missed. Each time a subscriber asks for its next group, the probe looks at what arrived on the track since its last look and decides whether any group the reader still wants has left the cache without the live feed delivering it (evicted, skipped, or from before the feed started), and asks upstream to fill those. A live reader that takes each group as it arrives never has such a gap, yet it paid for the whole check on every group: locking the subscription and the track's shared state, cloning the track's fills, walking the arrival ledger, checking each new group for holes, and running the ask-and-answer loop. On top of that, each time the cache aged out an old group, the eviction landed with every reader, all of which had already taken it, and forced each one to re-run the full check. + +Follow-up planning on 2026-11-03 (maintainer) added a load-independent mode to the driver: `just bench-fanout` counts the chat fan-out per delivered group on one pinned thread, through the model alone and through a relay over the in-memory transport, for any list of revisions. [The report](/.github/relay-cpu-bisect.md) attributes each step with it, and the fixes so far take `main`'s model path from 13,300 to 10,126 instructions per group (-23.9%) and the in-process session from 120,948 to 113,133 (-6.5%), at 512 subscribers. + +Decided 2026-11-03 (maintainer, on [moq-dev/moq#7605](https://github.com/moq-dev/moq/pull/7605)), after the attribution: + +1. Costs measured and not cut: + - ✅ Accept the flat ones (per-reader ranges' remainder, ranged reads, public ingest, compressed tracks, [moq-dev/moq#7052](https://github.com/moq-dev/moq/pull/7052), and bounded reads), recorded with their numbers in the [epic](/quest/m1/release-perf/README.md). Keep [moq-dev/moq#6995](https://github.com/moq-dev/moq/pull/6995)'s and [moq-dev/moq#7542](https://github.com/moq-dev/moq/pull/7542)'s session cost here for a re-measure against this branch. + - Accept all of them now. + - Keep investigating all of them. +2. The waiter's list records: + - ✅ A follow-up at m0, as a correctness bug: [a waiter's record limit changes no behavior](/quest/m0/kio-waiter-records.md). + - Leave the limit at 8. +3. `group::Consumer` moves: + - ✅ A follow-up in this epic: [a read moves a small group consumer](/quest/m1/release-perf/group-consumer-size.md). + - Leave it. + +Remaining: + +- Re-measure [moq-dev/moq#6995](https://github.com/moq-dev/moq/pull/6995)'s and [moq-dev/moq#7542](https://github.com/moq-dev/moq/pull/7542)'s session cost against `main` with the probe fast path (about +3.3k and +3.1k per group when each landed; much of it was the probe's walk of the live groups, which the fast path now skips), then fix what is accidental or record what remains in the epic. +- Attribute in-tree qmux ([moq-dev/moq#6231](https://github.com/moq-dev/moq/pull/6231)) and receive leases ([moq-dev/moq#5878](https://github.com/moq-dev/moq/pull/5878)), each against its parent, with relay runs on a quiet host: they are QUIC-level, which `just bench-fanout` doesn't reach, and the one attempt on `nixos2` lost the host to other agents after a round. +- Re-pair `main` against [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95) on chat and video over both runtimes on a quiet host, then delete this quest. + +Measure as the report does. For a model or session change, `just bench-fanout` against the base, which host load doesn't move. For the relay, `BENCH_GENERATOR=21d6da9fc1`, `BENCH_INSTRUCTION_PERIOD=2000003`, `kernel.perf_event_max_sample_rate` raised to 100,000 and restored after, on a quiet host such as `nixos2` (6 cores: relay on 0-3, generator on 4,5,10,11, `BENCH_BUILD_JOBS=2` for its 15 GiB), comparing only cells from one run; instructions per frame fall by up to 20% when other host work makes the relay batch more frames per wakeup. Public API: none expected. Wire: none. ## Related +- [A waiter's record limit changes no behavior](/quest/m0/kio-waiter-records.md) - found here; the session's waiter registration cost waits on it +- [A read moves a small group consumer](/quest/m1/release-perf/group-consumer-size.md) - found here; the session's largest remaining self cost + - [Calibrate the paired nightly benchmarks](/quest/m1/bench-ci.md) - the nightly series that should have caught this growth - [Performance audit](/quest/m2/audit-perf.md) - counts instructions when bisecting, for the same reason diff --git a/rs/moq-net/src/model/subscription/runs.rs b/rs/moq-net/src/model/subscription/runs.rs index cbce7e8f218..e60e6f43bb3 100644 --- a/rs/moq-net/src/model/subscription/runs.rs +++ b/rs/moq-net/src/model/subscription/runs.rs @@ -133,8 +133,22 @@ impl Layered { self.fixed.contains(sequence) || self.layer.contains(sequence) } + /// The one set holding runs, when the other holds none: its runs are the union's, each + /// already whole, so a read needs no growing across the two. + fn only(&self) -> Option<&Runs> { + match (self.fixed.0.is_empty(), self.layer.0.is_empty()) { + (_, true) => Some(&self.fixed), + (true, false) => Some(&self.layer), + (false, false) => None, + } + } + /// The lowest whole run of the union reaching into `lo..=hi`, its start clipped to `lo`. pub fn first(&self, lo: u64, hi: u64) -> Option> { + // Most windows have no layer: their fixed runs are already the union's. + if let Some(only) = self.only() { + return only.first(lo, hi).map(|run| (*run.start()).max(lo)..=*run.end()); + } let starts = [self.fixed.first(lo, hi), self.layer.first(lo, hi)]; let start = starts.into_iter().flatten().map(|run| (*run.start()).max(lo)).min()?; // Grow the end through whatever either set holds next to it: each step crosses into @@ -159,6 +173,9 @@ impl Layered { /// The highest whole run of the union reaching into `lo..=hi`, its end clipped to `hi`. pub fn last(&self, lo: u64, hi: u64) -> Option> { + if let Some(only) = self.only() { + return only.last(lo, hi).map(|run| *run.start()..=(*run.end()).min(hi)); + } let ends = [self.fixed.last(lo, hi), self.layer.last(lo, hi)]; let end = ends.into_iter().flatten().map(|run| (*run.end()).min(hi)).max()?; let mut start = end; diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 951c6e65c2f..36ca52ffa7b 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -6299,6 +6299,17 @@ impl Admitted { } /// `runs`, lowest first, below the exclusive `cap`. +/// What a reader asks its fills with: read only once it asks, since every probe that asks +/// nothing would otherwise take the subscription's lock for it. +fn asking(subscription: &Preferences) -> miss::Asking { + let subscription = subscription.read(); + miss::Asking { + priority: subscription.priority, + max_delay: subscription.delay.max, + order: subscription.order, + } +} + fn below(runs: impl Iterator>, cap: Option) -> RangeInclusiveSet { let mut below = RangeInclusiveSet::new(); let last = match cap { @@ -7531,11 +7542,12 @@ impl Cursor { /// Look up each time start the window can't resolve from the cache, once, and take in /// the answers, parking `waiter` on the lookups still out. - fn look_up(&mut self, asking: &miss::Asking, waiter: &kio::Waiter) { + fn look_up(&mut self, waiter: &kio::Waiter) { let lookups: Vec<(Timestamp, u64)> = self.filter.window.lookups().collect(); if lookups.is_empty() && self.filter.lookups.iter().all(|lookup| lookup.join.is_none()) { return; } + let asking = &asking(&self.subscription); let fills = self.state.read().fills.clone(); for (pts, reach) in lookups { if self.filter.lookups.iter().any(|lookup| lookup.pts == pts) { @@ -7599,16 +7611,12 @@ impl Cursor { /// Ask fills for what this cursor misses from `from`, and take in what they answered, /// parking `waiter` on the fills still out. Whatever it gives up goes to the drop cursor. /// `taken` is what its subscriber handed out, which an eviction never makes a miss. + /// + /// A live reader that takes each group as it arrives never misses one, so most probes + /// find nothing to ask and hold no fill: those skip the fills, the subscription, and + /// the answers entirely, paying only for the judgement that the new groups arrived. fn probe(&mut self, from: u64, drift: &Drift, taken: &Delivered, waiter: &kio::Waiter) { - let asking = { - let subscription = self.subscription.read(); - miss::Asking { - priority: subscription.priority, - max_delay: subscription.delay.max, - order: subscription.order, - } - }; - self.look_up(&asking, waiter); + self.look_up(waiter); let (window, cap, revision) = (&self.filter.window, self.end_sequence, self.filter.revision); let misses = self.misses.get_or_insert_with(Box::default); let mut given_up = Vec::new(); @@ -7641,15 +7649,26 @@ impl Cursor { &mut given_up, waiter, ); - (asks, state.fills.clone()) + // Only an ask needs the fills. + let fills = (!asks.is_empty()).then(|| state.fills.clone()); + (asks, fills) }; - misses.ask(&self.state, &fills, &asks, &asking, &mut given_up); - loop { - let retry = misses.resolve(&self.state, asking.max_delay, &mut given_up, waiter); - if retry.is_empty() { - break; + // Nothing to ask, no fill to hear from, and no gap to time: no answer can arrive. + if fills.is_some() || !misses.is_idle() { + let asking = asking(&self.subscription); + match &fills { + Some(fills) => misses.ask(&self.state, fills, &asks, &asking, &mut given_up), + // What the reader asks with still reaches the fills it joined. + None => misses.update(&asking), + } + loop { + let retry = misses.resolve(&self.state, asking.max_delay, &mut given_up, waiter); + if retry.is_empty() { + break; + } + let fills = self.state.read().fills.clone(); + misses.ask(&self.state, &fills, &retry, &asking, &mut given_up); } - misses.ask(&self.state, &fills, &retry, &asking, &mut given_up); } // Only what a fill still owes can expire. if misses.is_waiting() { @@ -21469,9 +21488,57 @@ mod test { assert!(misses.refilled().is_empty(), "a refill handed out is forgotten"); } + /// A live arrival judged on its own keeps the history a resumed copy's live floor sits + /// above: a cached group the reader hasn't taken yet, evicted after, is still refilled. + #[test] + #[cfg(feature = "async")] + fn history_below_a_resumed_live_floor_is_refilled_after_a_live_arrival() { + let mut producer = track_producer("test", Info::default()); + let dynamic = producer.requests(); + producer.received(); + for sequence in 0..5 { + framed(&producer, sequence, 1).finish().unwrap(); + } + producer.set_idle(); + // The feed resumes at 8, above the cached 0..=4: its live floor is 5. + producer.set_live(Some(Position::group(8)), None); + framed(&producer, 8, 1).finish().unwrap(); + let mut reader = producer.subscribe( + Subscription::default() + .with_range(0..) + .with_order(Order::Ascending) + .with_delay(Delay::default().with_max(None)), + ); + let waiter = kio::Waiter::noop(); + let next = |reader: &mut Subscriber| match reader.poll_next_group(&waiter) { + Poll::Ready(Ok(Some(group))) => Some(group.sequence), + _ => None, + }; + assert_eq!(next(&mut reader), Some(0)); + // The gap the feed may still deliver arrives late, so nothing is deferred after. + for sequence in 5..8 { + framed(&producer, sequence, 1).finish().unwrap(); + } + assert_eq!(next(&mut reader), Some(1)); + // Only the newest group moves before this read: judged on its own. + framed(&producer, 9, 1).finish().unwrap(); + assert_eq!(next(&mut reader), Some(2)); + evict(&producer, 3); + let _ = next(&mut reader); + let mut asked = Vec::new(); + while let Poll::Ready(Ok(fill)) = dynamic.poll_requested_fill(&waiter) { + asked.push((fill.start().group, fill.end())); + } + assert!( + asked.iter().any(|&(start, end)| start <= 3 && 3 < end), + "group 3 was evicted before the reader took it: asked {asked:?}" + ); + } + /// An eviction waits only for a reader that judged the evicted group cached: a reader /// that judged once and stalled holds no more than what was cached then, however many - /// groups the feed caches and evicts after, late lower ones included. + /// groups the feed caches and evicts after, late lower ones included. A reader that + /// took every group up to it holds none, since a group it took is never a miss again. #[test] #[cfg(feature = "async")] fn an_eviction_waits_only_for_readers_that_judged_it_cached() { @@ -21498,9 +21565,17 @@ mod test { assert_eq!(held(), [vec![], vec![(198, 199)]], "the stalled reader judged none"); drain_groups(&mut active); evict(&producer, 1); - assert_eq!(held(), [vec![(1, 2)], vec![(1, 2)]], "both judged group 1 cached"); + assert_eq!( + held(), + [vec![(1, 2)], vec![]], + "the active reader judged after taking every group up to 2; the stalled one judged before it took group 1" + ); drain_groups(&mut active); - assert_eq!(held(), [vec![(1, 2)], vec![]], "the active reader took it in"); + assert_eq!( + held(), + [vec![(1, 2)], vec![]], + "the active reader has nothing to take in" + ); drop(stalled); assert_eq!(held(), [vec![]], "a dropped reader holds nothing"); write_group(&producer, 3); diff --git a/rs/moq-net/src/model/track/miss.rs b/rs/moq-net/src/model/track/miss.rs index 8e941a3b681..f9272e7dee7 100644 --- a/rs/moq-net/src/model/track/miss.rs +++ b/rs/moq-net/src/model/track/miss.rs @@ -263,6 +263,12 @@ impl Misses { self.release(lost); } + /// Whether no fill can answer this reader and no gap is timed: nothing joined, nothing + /// deferred. Its probe then has nothing to resolve. + pub fn is_idle(&self) -> bool { + self.joins.is_empty() && self.deferred.runs.is_empty() + } + /// Whether a fill still owes this reader a group. pub fn is_waiting(&self) -> bool { !self.wanted.is_empty() @@ -630,6 +636,10 @@ impl Misses { return asks; } let grace = state.miss_grace(); + if self.judge_arrivals(state, key, at, view, &mut target, taken, drift, grace, ended) { + self.ledger_revision = ledger.revision(); + return asks; + } let unchanged = unchanged && self.grace == Some(grace); let deadline = self.deferred.deadline(); if unchanged && deadline.is_none() { @@ -782,7 +792,7 @@ impl Misses { .register(inbox.clone()); inbox }); - let evicted = inbox.judge(state.cached, low, top); + let evicted = inbox.judge(state.cached, inbox_floor(low, taken), top); self.evictions = state.evictions; let mut refills = RangeSet::new(); let mut again = RangeSet::new(); @@ -937,6 +947,99 @@ impl Misses { asks } + /// Judge what changed when only the feed's newest group moved up, and the reader's floor + /// with it, returning whether that was all. Each group arriving live changes nothing else, + /// so a reader that takes them as they come judges every one here. + /// + /// With nothing deferred or refused, the arrival ledger's changes touch nothing this + /// reader holds, and with no eviction counted, its inbox holds none. A new group the + /// arrival walk serves is then neither a hole nor backfill: judged, with nothing to ask. + /// Anything else, a group not walked included, is left to [`Self::judge`] whole. + #[expect( + clippy::too_many_arguments, + reason = "the judgement's inputs, as Self::judge takes them" + )] + fn judge_arrivals( + &mut self, + state: &TrackState, + key: (u64, u64, Option), + at: JudgedAt, + view: Option<&Layered>, + target: &mut impl FnMut(RangeInclusive) -> RangeInclusiveSet, + taken: &Delivered, + drift: &Drift, + grace: Duration, + ended: bool, + ) -> bool { + let Some(previous) = self.at else { + return false; + }; + let (Some(judged_to), Some(newest)) = (previous.3, at.3) else { + return false; + }; + let only_arrivals = previous.0 <= at.0 + && judged_to <= newest + && (previous.1, previous.2, previous.4, previous.5, previous.6) == (at.1, at.2, at.4, at.5, at.6); + if !only_arrivals + || !self.deferred.runs.is_empty() + || !self.refused.is_empty() + || self.budget != Some(drift.budget) + || self.grace != Some(grace) + || self.ended != ended + || self.drops_revision != state.drops_changes.revision() + || self.evictions != state.evictions + || self.view.as_ref() != view + { + return false; + } + let Some(inbox) = &self.inbox else { + return false; + }; + // The reader's floor, as the whole judgement takes it: a resumed copy's live floor + // can sit above history the reader still reads. + let low = key.0; + let top = newest.min(u64::MAX - 1); + let first = judged_to.saturating_add(1).max(low); + let mut fresh = RangeSet::new(); + if first <= top { + for run in target(first..=top).iter() { + for gap in self.judged.gaps(&(*run.start()..*run.end() + 1)) { + fresh.insert(gap); + } + } + } + let walked = fresh.iter().all(|run| { + let mut next = run.start; + for (&sequence, slot) in state.lookup.range(run.clone()) { + if sequence != next || !slot.is_walked() { + return false; + } + next = sequence + 1; + } + next == run.end + }); + if !walked { + return false; + } + if low > 0 { + for set in [ + &mut self.judged, + &mut self.refused, + &mut self.given_up, + &mut self.refilled, + ] { + set.remove(0..low); + } + } + let evicted = inbox.judge(state.cached, inbox_floor(low, taken), top); + debug_assert!(evicted.is_empty(), "an eviction landed without counting"); + for run in fresh.iter() { + self.judged.insert(run.clone()); + } + self.at = Some(at); + true + } + /// Give up what a fill still owes this reader once it is stale under the reader's budget, /// as the reader skips a stale cached group, parking `waiter` on each run going stale. /// A fill answers on its handler's time, so an ordered reader would otherwise wait out @@ -1285,6 +1388,14 @@ impl TrackState { } /// The runs of `run` the cache holds as backfill: servable, but out of the arrival walk. +/// Where a reader's inbox starts taking evictions: no lower than what it handed out every +/// group below, since an evicted group it took is never a miss again. A live reader that +/// reads by arrival never raises its floor, so without this every old group the cache +/// evicts would land with it and count as an eviction to judge again. +fn inbox_floor(low: u64, taken: &Delivered) -> u64 { + low.max(taken.floor().unwrap_or(u64::MAX)) +} + fn backfill(state: &TrackState, run: ops::Range) -> RangeSet { state .lookup