From 3eeefa2d2a88b3151ec8f8edc417a238fe2ddea1 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 9 Oct 2026 20:00:44 +0200 Subject: [PATCH] perf(sha256): compress blocks on ARMv8 SHA2 / x86 SHA-NI (#2441) Every process start fingerprints its own executable for the daemon build identity, and the hook path pays for that before it can answer. With the portable transform that was ~1.2 s of CPU for the ~300 MB binary, so on a slower machine hook-augment hit its 2000 ms deadline and printed nothing (#2441). Two changes in src/foundation/sha256.c: - cbm_sha256_update compresses whole blocks straight from the input instead of copying every byte through the context buffer. - Block compression runs on the CPU's SHA-256 instructions when it has them: ARMv8 SHA2 (sysctl on macOS, HWCAP on Linux, IsProcessorFeaturePresent on Windows) or x86 SHA-NI (CPUID). The backend is probed once; the portable transform stays the fallback. Measured on an Apple arm64 Mac (macOS 26.6), release builds of main 72a2c0bc and of this change, hook-augment SessionStart in fresh isolated dirs, 9 interleaved runs, medians: - hook-augment: wall 1.448 s -> 0.376 s, user CPU 1.162 s -> 0.097 s - SHA-256 of the 303 MB binary: 1.18 s -> 0.095 s CPU per pass (the block-direct update alone, portable rounds: 0.74 s) - with CBM_HOOK_DEADLINE_MS=1000 (a slower machine): before, exit 0 with empty stdout after 1.01 s; after, the full 188-byte answer in 0.37 s Digests are unchanged: the build id the new binary logs equals the file's SHA-256, and the hook output is byte-identical. A new test compares the hardware and portable backends at every length 0..299 and at block-boundary sizes up to 1 MiB under eight update chunkings, plus the NIST million-'a' vector; it fails if the ARM schedule is broken or if Apple arm64 falls back to portable. The x86 path was checked under qemu-x86_64 -cpu max (2,436 comparisons equal, million-'a' correct) and falls back to portable on a CPU without SHA-NI. On a Windows arm64 VM the probe picks the hardware path: 0.094 s against 0.82 s portable for the 303 MB binary, digest equal to sha256sum's. Signed-off-by: Martin Vogel --- src/foundation/sha256.c | 254 ++++++++++++++++++++++++++++++++++++++-- src/foundation/sha256.h | 9 ++ tests/test_cli.c | 66 +++++++++++ 3 files changed, 320 insertions(+), 9 deletions(-) diff --git a/src/foundation/sha256.c b/src/foundation/sha256.c index 84d37453e1..b5f6bcd6dd 100644 --- a/src/foundation/sha256.c +++ b/src/foundation/sha256.c @@ -1,11 +1,49 @@ /* SHA-256 per FIPS 180-4. Straightforward reference implementation; validated - * against the NIST test vectors in tests/test_cli.c. */ + * against the NIST test vectors in tests/test_cli.c. + * + * Block compression runs on the CPU's SHA-256 instructions when it has them + * (ARMv8 SHA2, x86 SHA-NI), chosen once at runtime; the portable transform + * below stays the fallback and the reference the tests compare against. + * #2441: every process start fingerprints its own ~300 MB executable, and the + * portable transform made that ~1.2 s of CPU — past the hook deadline. */ #include "foundation/sha256.h" #include "foundation/secure_random.h" +#include +#include #include +/* The ARM path needs a way to ask the OS whether the CPU has SHA2; on any + * other aarch64 OS the portable transform is used. */ +#if defined(__aarch64__) && (defined(__clang__) || defined(__GNUC__)) && \ + (defined(__APPLE__) || defined(__linux__) || defined(_WIN32)) +#define SHA256_HW_ARM 1 +#include +#if defined(__APPLE__) +#include +#elif defined(__linux__) +#include +#ifndef HWCAP_SHA2 +#define HWCAP_SHA2 (1UL << 6) +#endif +#else +#include +#endif +#if defined(__clang__) +#define SHA256_ARM_TARGET __attribute__((target("sha2"))) +#else +#define SHA256_ARM_TARGET __attribute__((target("+sha2"))) +#endif +#elif defined(__x86_64__) && (defined(__clang__) || defined(__GNUC__)) +#define SHA256_HW_X86 1 +#include +#include +#define SHA256_X86_TARGET __attribute__((target("sha,sse4.1,ssse3"))) +#endif + +enum { SHA256_BLOCK_BYTES = 64 }; + static const uint32_t K[64] = { 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, @@ -64,6 +102,181 @@ static void sha256_transform(cbm_sha256_ctx *c, const uint8_t *data) { c->state[7] += h; } +static void sha256_blocks_portable(cbm_sha256_ctx *c, const uint8_t *data, size_t blocks) { + for (; blocks > 0; blocks--, data += SHA256_BLOCK_BYTES) { + sha256_transform(c, data); + } +} + +#if defined(SHA256_HW_ARM) +/* ARMv8 SHA2: the state stays {a,b,c,d} / {e,f,g,h}; each SHA256H/H2 pair runs + * four rounds, and SU0/SU1 extend the schedule four words at a time. */ +SHA256_ARM_TARGET static void sha256_blocks_arm(cbm_sha256_ctx *c, const uint8_t *data, + size_t blocks) { + uint32x4_t abcd = vld1q_u32(&c->state[0]); + uint32x4_t efgh = vld1q_u32(&c->state[4]); + for (; blocks > 0; blocks--, data += SHA256_BLOCK_BYTES) { + uint32x4_t abcd_in = abcd; + uint32x4_t efgh_in = efgh; + uint32x4_t w[4]; + for (int i = 0; i < 4; i++) { + w[i] = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(data + i * 16))); + } + for (int i = 0; i < 16; i++) { + uint32x4_t wk = vaddq_u32(w[i & 3], vld1q_u32(&K[i * 4])); + uint32x4_t abcd_prev = abcd; + abcd = vsha256hq_u32(abcd, efgh, wk); + efgh = vsha256h2q_u32(efgh, abcd_prev, wk); + if (i < 12) { + w[i & 3] = vsha256su1q_u32(vsha256su0q_u32(w[i & 3], w[(i + 1) & 3]), + w[(i + 2) & 3], w[(i + 3) & 3]); + } + } + abcd = vaddq_u32(abcd, abcd_in); + efgh = vaddq_u32(efgh, efgh_in); + } + vst1q_u32(&c->state[0], abcd); + vst1q_u32(&c->state[4], efgh); +} + +static bool sha256_cpu_has_arm(void) { +#if defined(__APPLE__) + /* Present on every Apple arm64 CPU; the key itself exists since macOS 12. */ + int has = 0; + size_t size = sizeof(has); + return sysctlbyname("hw.optional.arm.FEAT_SHA256", &has, &size, NULL, 0) == 0 && has != 0; +#elif defined(__linux__) + return (getauxval(AT_HWCAP) & HWCAP_SHA2) != 0; +#else + return IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE) != 0; +#endif +} +#endif + +#if defined(SHA256_HW_X86) +/* x86 SHA-NI: SHA256RNDS2 wants the state as {a,b,e,f} / {c,d,g,h}, so it is + * shuffled in once per call and back out at the end. */ +SHA256_X86_TARGET static void sha256_blocks_x86(cbm_sha256_ctx *c, const uint8_t *data, + size_t blocks) { + const __m128i byteswap = _mm_set_epi64x(0x0c0d0e0f08090a0bULL, 0x0405060700010203ULL); + __m128i dcba = _mm_shuffle_epi32(_mm_loadu_si128((const __m128i *)&c->state[0]), 0xB1); + __m128i efgh = _mm_shuffle_epi32(_mm_loadu_si128((const __m128i *)&c->state[4]), 0x1B); + __m128i abef = _mm_alignr_epi8(dcba, efgh, 8); + __m128i cdgh = _mm_blend_epi16(efgh, dcba, 0xF0); + for (; blocks > 0; blocks--, data += SHA256_BLOCK_BYTES) { + __m128i abef_in = abef; + __m128i cdgh_in = cdgh; + __m128i w[4]; + for (int i = 0; i < 4; i++) { + w[i] = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)(data + i * 16)), byteswap); + } + for (int i = 0; i < 16; i++) { + __m128i wk = _mm_add_epi32(w[i & 3], _mm_loadu_si128((const __m128i *)&K[i * 4])); + cdgh = _mm_sha256rnds2_epu32(cdgh, abef, wk); + abef = _mm_sha256rnds2_epu32(abef, cdgh, _mm_shuffle_epi32(wk, 0x0E)); + if (i < 12) { + __m128i next = _mm_add_epi32(_mm_sha256msg1_epu32(w[i & 3], w[(i + 1) & 3]), + _mm_alignr_epi8(w[(i + 3) & 3], w[(i + 2) & 3], 4)); + w[i & 3] = _mm_sha256msg2_epu32(next, w[(i + 3) & 3]); + } + } + abef = _mm_add_epi32(abef, abef_in); + cdgh = _mm_add_epi32(cdgh, cdgh_in); + } + __m128i feba = _mm_shuffle_epi32(abef, 0x1B); + __m128i dchg = _mm_shuffle_epi32(cdgh, 0xB1); + _mm_storeu_si128((__m128i *)&c->state[0], _mm_blend_epi16(feba, dchg, 0xF0)); + _mm_storeu_si128((__m128i *)&c->state[4], _mm_alignr_epi8(dchg, feba, 8)); +} + +static bool sha256_cpu_has_x86(void) { + unsigned int eax = 0; + unsigned int ebx = 0; + unsigned int ecx = 0; + unsigned int edx = 0; + if (!__get_cpuid(1, &eax, &ebx, &ecx, &edx)) { + return false; + } + bool ssse3 = (ecx & (1U << 9)) != 0; + bool sse41 = (ecx & (1U << 19)) != 0; + if (!ssse3 || !sse41 || !__get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx)) { + return false; + } + return (ebx & (1U << 29)) != 0; /* CPUID.(7,0):EBX.SHA */ +} +#endif + +typedef enum { + SHA256_BACKEND_UNPROBED = 0, + SHA256_BACKEND_PORTABLE, + SHA256_BACKEND_ARM, + SHA256_BACKEND_X86, +} sha256_backend_t; + +/* Probed once; every thread computes the same answer, so relaxed is enough. */ +static _Atomic int g_sha256_backend = SHA256_BACKEND_UNPROBED; +static _Atomic bool g_sha256_force_portable = false; + +static sha256_backend_t sha256_probe_backend(void) { +#if defined(SHA256_HW_ARM) + if (sha256_cpu_has_arm()) { + return SHA256_BACKEND_ARM; + } +#elif defined(SHA256_HW_X86) + if (sha256_cpu_has_x86()) { + return SHA256_BACKEND_X86; + } +#endif + return SHA256_BACKEND_PORTABLE; +} + +static sha256_backend_t sha256_backend(void) { + if (atomic_load_explicit(&g_sha256_force_portable, memory_order_relaxed)) { + return SHA256_BACKEND_PORTABLE; + } + int backend = atomic_load_explicit(&g_sha256_backend, memory_order_relaxed); + if (backend == SHA256_BACKEND_UNPROBED) { + backend = (int)sha256_probe_backend(); + atomic_store_explicit(&g_sha256_backend, backend, memory_order_relaxed); + } + return (sha256_backend_t)backend; +} + +static void sha256_blocks(cbm_sha256_ctx *c, const uint8_t *data, size_t blocks) { + switch (sha256_backend()) { +#if defined(SHA256_HW_ARM) + case SHA256_BACKEND_ARM: + sha256_blocks_arm(c, data, blocks); + return; +#endif +#if defined(SHA256_HW_X86) + case SHA256_BACKEND_X86: + sha256_blocks_x86(c, data, blocks); + return; +#endif + default: + sha256_blocks_portable(c, data, blocks); + return; + } +} + +#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS +const char *cbm_sha256_backend_name_for_testing(void) { + switch (sha256_backend()) { + case SHA256_BACKEND_ARM: + return "arm-sha2"; + case SHA256_BACKEND_X86: + return "x86-sha-ni"; + default: + return "portable"; + } +} + +void cbm_sha256_force_portable_for_testing(bool force) { + atomic_store_explicit(&g_sha256_force_portable, force, memory_order_relaxed); +} +#endif + void cbm_sha256_init(cbm_sha256_ctx *c) { c->bitlen = 0; c->buflen = 0; @@ -78,14 +291,37 @@ void cbm_sha256_init(cbm_sha256_ctx *c) { } void cbm_sha256_update(cbm_sha256_ctx *c, const void *data, size_t len) { + if (len == 0) { + return; /* data may be NULL when len is 0 */ + } const uint8_t *p = (const uint8_t *)data; - for (size_t i = 0; i < len; i++) { - c->buf[c->buflen++] = p[i]; - if (c->buflen == 64) { - sha256_transform(c, c->buf); - c->bitlen += 512; - c->buflen = 0; + if (c->buflen > 0) { + size_t take = SHA256_BLOCK_BYTES - c->buflen; + if (take > len) { + take = len; + } + memcpy(c->buf + c->buflen, p, take); + c->buflen += take; + p += take; + len -= take; + if (c->buflen < SHA256_BLOCK_BYTES) { + return; } + sha256_blocks(c, c->buf, 1); + c->bitlen += 512; + c->buflen = 0; + } + /* Whole blocks straight from the input, without staging them in buf. */ + size_t blocks = len / SHA256_BLOCK_BYTES; + if (blocks > 0) { + sha256_blocks(c, p, blocks); + c->bitlen += (uint64_t)blocks * 512; + p += blocks * SHA256_BLOCK_BYTES; + len -= blocks * SHA256_BLOCK_BYTES; + } + if (len > 0) { + memcpy(c->buf, p, len); + c->buflen = len; } } @@ -98,7 +334,7 @@ void cbm_sha256_final(cbm_sha256_ctx *c, uint8_t out[CBM_SHA256_DIGEST_LEN]) { while (i < 64) { c->buf[i++] = 0; } - sha256_transform(c, c->buf); + sha256_blocks(c, c->buf, 1); i = 0; } while (i < 56) { @@ -108,7 +344,7 @@ void cbm_sha256_final(cbm_sha256_ctx *c, uint8_t out[CBM_SHA256_DIGEST_LEN]) { for (int j = 0; j < 8; j++) { c->buf[56 + j] = (uint8_t)(c->bitlen >> (56 - 8 * j)); } - sha256_transform(c, c->buf); + sha256_blocks(c, c->buf, 1); for (int j = 0; j < 8; j++) { out[j * 4] = (uint8_t)(c->state[j] >> 24); diff --git a/src/foundation/sha256.h b/src/foundation/sha256.h index f3c7f9c55e..23fb3b8c94 100644 --- a/src/foundation/sha256.h +++ b/src/foundation/sha256.h @@ -41,4 +41,13 @@ void cbm_sha256_hex(const void *data, size_t len, char out[CBM_SHA256_HEX_LEN + void cbm_hmac_sha256(const void *key, size_t key_len, const void *data, size_t data_len, uint8_t out[CBM_SHA256_DIGEST_LEN]); +#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS +#include +/* The block backend in use: "arm-sha2", "x86-sha-ni" or "portable". */ +const char *cbm_sha256_backend_name_for_testing(void); +/* Force the portable transform (true) or return to the probed backend (false), + * so a test can hash the same input both ways. */ +void cbm_sha256_force_portable_for_testing(bool force); +#endif + #endif /* CBM_SHA256_H */ diff --git a/tests/test_cli.c b/tests/test_cli.c index a77abc0a31..a3fda1d5f8 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -17174,6 +17174,71 @@ TEST(cli_sha256_file_matches_known_vector) { PASS(); } +/* Hash `len` bytes fed in `chunk`-sized pieces (0 = one update) to hex. */ +static void sha256_chunked_hex(const uint8_t *data, size_t len, size_t chunk, + char out[CBM_SHA256_HEX_LEN + 1]) { + static const char hex[] = "0123456789abcdef"; + cbm_sha256_ctx ctx; + cbm_sha256_init(&ctx); + size_t step = chunk == 0 ? len : chunk; + for (size_t off = 0; off < len; off += step) { + cbm_sha256_update(&ctx, data + off, len - off < step ? len - off : step); + } + uint8_t digest[CBM_SHA256_DIGEST_LEN]; + cbm_sha256_final(&ctx, digest); + for (int i = 0; i < CBM_SHA256_DIGEST_LEN; i++) { + out[i * 2] = hex[digest[i] >> 4]; + out[i * 2 + 1] = hex[digest[i] & 0x0f]; + } + out[CBM_SHA256_HEX_LEN] = '\0'; +} + +/* #2441: block compression runs on ARMv8 SHA2 / x86 SHA-NI when the CPU has + * them. The hardware path must agree with the portable transform bit for bit + * at every length around the block and padding boundaries, for every way the + * input is split across update calls, and on the NIST million-'a' vector that + * runs 15,625 blocks through it. */ +TEST(cli_sha256_hardware_matches_portable_issue2441) { + enum { MAX_LEN = (1 << 20) + 13 }; + uint8_t *data = malloc(MAX_LEN); + ASSERT_NOT_NULL(data); + uint32_t x = 0x2441u; + for (size_t i = 0; i < MAX_LEN; i++) { + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + data[i] = (uint8_t)x; + } + static const size_t big[] = {4096, 65535, 65536, 65537, MAX_LEN}; + static const size_t chunks[] = {0, 1, 3, 55, 63, 64, 65, 1000}; + char hw[CBM_SHA256_HEX_LEN + 1]; + char portable[CBM_SHA256_HEX_LEN + 1]; + for (size_t n = 0; n < 300 + sizeof(big) / sizeof(big[0]); n++) { + size_t len = n < 300 ? n : big[n - 300]; + for (size_t c = 0; c < sizeof(chunks) / sizeof(chunks[0]); c++) { + if (len > 70000 && chunks[c] != 0 && chunks[c] < 64) { + continue; /* byte-sized updates over 1 MiB add time, not coverage */ + } + cbm_sha256_force_portable_for_testing(false); + sha256_chunked_hex(data, len, chunks[c], hw); + cbm_sha256_force_portable_for_testing(true); + sha256_chunked_hex(data, len, chunks[c], portable); + cbm_sha256_force_portable_for_testing(false); + ASSERT_STR_EQ(hw, portable); + } + } + memset(data, 'a', 1000000); + sha256_chunked_hex(data, 1000000, 0, hw); + ASSERT_STR_EQ(hw, "cdc76e5c9914fb9281a1c7e284d73e67f1809a48a497200e046d39ccc7112cd0"); + free(data); +#if defined(__APPLE__) && defined(__aarch64__) + /* Every Apple arm64 CPU has the SHA2 extension: a portable answer here + * means the probe or dispatch broke, and startup is slow again. */ + ASSERT_STR_EQ(cbm_sha256_backend_name_for_testing(), "arm-sha2"); +#endif + PASS(); +} + /* #1544: v0.10.2 deleted the ui/standard chooser AND the flags that drove it, * so `update --ui` — a command people had in scripts and aliases — started * failing with "unknown update option". Retiring a choice is fine; breaking the @@ -17669,6 +17734,7 @@ SUITE(cli) { RUN_TEST(cli_progress_sink_preserves_explicit_verbose_diagnostics); RUN_TEST(cli_progress_sink_serializes_concurrent_callbacks); RUN_TEST(cli_sha256_file_matches_known_vector); + RUN_TEST(cli_sha256_hardware_matches_portable_issue2441); RUN_TEST(cli_checksum_manifest_requires_exact_filename_and_accepts_star); RUN_TEST(cli_checksum_manifest_rejects_invalid_missing_and_conflicting_digest); RUN_TEST(cli_checksum_manifest_rejects_oversized_input);