From f3ecae9889dd0aeb33859a30b85021477ebc7495 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Wed, 14 Oct 2026 14:27:49 -0400 Subject: [PATCH 1/2] Support big-endian platforms in the scalar codecs (fixes #83) The scalar encode/decode paths copied integers to and from the byte stream with memcpy, which silently assumes a little-endian host. On big-endian systems such as s390x this produced corrupted streams. Add small svb_load_le/svb_store_le helpers that keep using memcpy on little-endian hosts (no codegen change) and fall back to explicit byte shifts on big-endian hosts. Add an s390x CI job running under QEMU. --- .github/workflows/s390x.yml | 30 +++++++++++++++++++++++++++ src/streamvbyte_0124_decode.c | 6 +++--- src/streamvbyte_0124_encode.c | 5 +++-- src/streamvbyte_decode.c | 9 ++++----- src/streamvbyte_encode.c | 7 ++++--- src/streamvbyte_endian.h | 38 +++++++++++++++++++++++++++++++++++ src/streamvbytedelta_decode.c | 9 ++++----- src/streamvbytedelta_encode.c | 7 ++++--- 8 files changed, 90 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/s390x.yml create mode 100644 src/streamvbyte_endian.h diff --git a/.github/workflows/s390x.yml b/.github/workflows/s390x.yml new file mode 100644 index 0000000..108e308 --- /dev/null +++ b/.github/workflows/s390x.yml @@ -0,0 +1,30 @@ +name: s390x (big endian) CI + +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + s390x-build: + if: >- + ! contains(toJSON(github.event.commits.*.message), '[skip ci]') && + ! contains(toJSON(github.event.commits.*.message), '[skip github]') + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: docker/setup-qemu-action@v3 + with: + platforms: s390x + - name: Build and test + run: | + docker run --rm --platform linux/s390x -v "$PWD":/src -w /src debian:stable-slim sh -c ' + apt-get update -qq && + apt-get install -y -qq gcc g++ cmake make git ca-certificates > /dev/null && + cmake -B build -DSTREAMVBYTE_ENABLE_EXAMPLES=ON -DSTREAMVBYTE_ENABLE_TESTS=ON -DSTREAMVBYTE_WALL=ON -DSTREAMVBYTE_WERROR=ON && + cmake --build build -j4 && + ctest --test-dir build --output-on-failure && + ./build/example' diff --git a/src/streamvbyte_0124_decode.c b/src/streamvbyte_0124_decode.c index a6bb6f3..ea42b72 100644 --- a/src/streamvbyte_0124_decode.c +++ b/src/streamvbyte_0124_decode.c @@ -5,6 +5,7 @@ #include "streamvbyte_shuffle_tables_0124_decode.h" #endif +#include "streamvbyte_endian.h" #include // for memcpy #ifdef __clang__ @@ -45,11 +46,10 @@ static inline uint32_t svb_decode_data(const uint8_t **dataPtrPtr, uint8_t code) val = (uint32_t)*dataPtr; dataPtr += 1; } else if (code == 2) { // 2 bytes - val = 0; - memcpy(&val, dataPtr, 2); // assumes little endian + val = svb_load_le(dataPtr, 2); dataPtr += 2; } else { // code == 3, 4 bytes - memcpy(&val, dataPtr, 4); + val = svb_load_le(dataPtr, 4); dataPtr += 4; } diff --git a/src/streamvbyte_0124_encode.c b/src/streamvbyte_0124_encode.c index 376d673..76435b4 100644 --- a/src/streamvbyte_0124_encode.c +++ b/src/streamvbyte_0124_encode.c @@ -2,6 +2,7 @@ #include "streamvbyte_isadetection.h" #include "streamvbyte_shuffle_tables_0124_encode.h" +#include "streamvbyte_endian.h" #include // for memcpy #ifdef __clang__ @@ -20,11 +21,11 @@ static uint8_t svb_encode_data(uint32_t val, uint8_t *__restrict__ *dataPtrPtr) *dataPtrPtr += 1; code = 1; } else if (val < (1 << 16)) { // 2 bytes - memcpy(dataPtr, &val, 2); // assumes little endian + svb_store_le(dataPtr, val, 2); *dataPtrPtr += 2; code = 2; } else { // 4 bytes - memcpy(dataPtr, &val, sizeof(uint32_t)); + svb_store_le(dataPtr, val, sizeof(uint32_t)); *dataPtrPtr += sizeof(uint32_t); code = 3; } diff --git a/src/streamvbyte_decode.c b/src/streamvbyte_decode.c index 1f8c1aa..f861236 100644 --- a/src/streamvbyte_decode.c +++ b/src/streamvbyte_decode.c @@ -1,6 +1,7 @@ #include "streamvbyte.h" #include "streamvbyte_isadetection.h" +#include "streamvbyte_endian.h" #include // for memcpy #ifdef __clang__ @@ -23,15 +24,13 @@ static inline uint32_t svb_decode_data(const uint8_t **dataPtrPtr, uint8_t code) val = (uint32_t)*dataPtr; dataPtr += 1; } else if (code == 1) { // 2 bytes - val = 0; - memcpy(&val, dataPtr, 2); // assumes little endian + val = svb_load_le(dataPtr, 2); dataPtr += 2; } else if (code == 2) { // 3 bytes - val = 0; - memcpy(&val, dataPtr, 3); // assumes little endian + val = svb_load_le(dataPtr, 3); dataPtr += 3; } else { // code == 3 - memcpy(&val, dataPtr, 4); + val = svb_load_le(dataPtr, 4); dataPtr += 4; } diff --git a/src/streamvbyte_encode.c b/src/streamvbyte_encode.c index 6e4f8c7..058c076 100644 --- a/src/streamvbyte_encode.c +++ b/src/streamvbyte_encode.c @@ -1,6 +1,7 @@ #include "streamvbyte.h" #include "streamvbyte_isadetection.h" +#include "streamvbyte_endian.h" #include // for memcpy #ifdef __clang__ @@ -20,15 +21,15 @@ static uint8_t svb_encode_data(uint32_t val, uint8_t *__restrict__ *dataPtrPtr) *dataPtrPtr += 1; code = 0; } else if (val < (1 << 16)) { // 2 bytes - memcpy(dataPtr, &val, 2); // assumes little endian + svb_store_le(dataPtr, val, 2); *dataPtrPtr += 2; code = 1; } else if (val < (1 << 24)) { // 3 bytes - memcpy(dataPtr, &val, 3); // assumes little endian + svb_store_le(dataPtr, val, 3); *dataPtrPtr += 3; code = 2; } else { // 4 bytes - memcpy(dataPtr, &val, sizeof(uint32_t)); + svb_store_le(dataPtr, val, sizeof(uint32_t)); *dataPtrPtr += sizeof(uint32_t); code = 3; } diff --git a/src/streamvbyte_endian.h b/src/streamvbyte_endian.h new file mode 100644 index 0000000..9c900da --- /dev/null +++ b/src/streamvbyte_endian.h @@ -0,0 +1,38 @@ +#ifndef STREAMVBYTE_ENDIAN_H +#define STREAMVBYTE_ENDIAN_H + +#include +#include +#include // for memcpy + +// The stream vbyte format stores values in little-endian byte order. +#if defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && \ + (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +#define STREAMVBYTE_BIG_ENDIAN 1 +#endif + +// Reads the 'len' (1 to 4) low-order bytes of a little-endian value. +static inline uint32_t svb_load_le(const uint8_t *p, size_t len) { + uint32_t val = 0; +#ifdef STREAMVBYTE_BIG_ENDIAN + for (size_t i = 0; i < len; i++) { + val |= (uint32_t)p[i] << (8 * i); + } +#else + memcpy(&val, p, len); +#endif + return val; +} + +// Writes the 'len' (1 to 4) low-order bytes of 'val' in little-endian order. +static inline void svb_store_le(uint8_t *p, uint32_t val, size_t len) { +#ifdef STREAMVBYTE_BIG_ENDIAN + for (size_t i = 0; i < len; i++) { + p[i] = (uint8_t)(val >> (8 * i)); + } +#else + memcpy(p, &val, len); +#endif +} + +#endif // STREAMVBYTE_ENDIAN_H diff --git a/src/streamvbytedelta_decode.c b/src/streamvbytedelta_decode.c index b081904..b55bb2d 100644 --- a/src/streamvbytedelta_decode.c +++ b/src/streamvbytedelta_decode.c @@ -1,6 +1,7 @@ #include "streamvbytedelta.h" #include "streamvbyte_isadetection.h" +#include "streamvbyte_endian.h" #include // for memcpy #ifdef __clang__ @@ -15,15 +16,13 @@ static inline uint32_t svb_decode_data(const uint8_t **dataPtrPtr, uint8_t code) val = (uint32_t)*dataPtr; dataPtr += 1; } else if (code == 1) { // 2 bytes - val = 0; - memcpy(&val, dataPtr, 2); // assumes little endian + val = svb_load_le(dataPtr, 2); dataPtr += 2; } else if (code == 2) { // 3 bytes - val = 0; - memcpy(&val, dataPtr, 3); // assumes little endian + val = svb_load_le(dataPtr, 3); dataPtr += 3; } else { // code == 3 - memcpy(&val, dataPtr, 4); + val = svb_load_le(dataPtr, 4); dataPtr += 4; } diff --git a/src/streamvbytedelta_encode.c b/src/streamvbytedelta_encode.c index b745fa1..462b821 100644 --- a/src/streamvbytedelta_encode.c +++ b/src/streamvbytedelta_encode.c @@ -1,6 +1,7 @@ #include "streamvbytedelta.h" #include "streamvbyte_isadetection.h" +#include "streamvbyte_endian.h" #include // for memcpy #ifdef STREAMVBYTE_X64 @@ -20,15 +21,15 @@ static uint8_t svb_encode_data(uint32_t val, uint8_t *__restrict__ *dataPtrPtr) *dataPtrPtr += 1; code = 0; } else if (val < (1 << 16)) { // 2 bytes - memcpy(dataPtr, &val, 2); // assumes little endian + svb_store_le(dataPtr, val, 2); *dataPtrPtr += 2; code = 1; } else if (val < (1 << 24)) { // 3 bytes - memcpy(dataPtr, &val, 3); // assumes little endian + svb_store_le(dataPtr, val, 3); *dataPtrPtr += 3; code = 2; } else { // 4 bytes - memcpy(dataPtr, &val, sizeof(uint32_t)); + svb_store_le(dataPtr, val, sizeof(uint32_t)); *dataPtrPtr += sizeof(uint32_t); code = 3; } From 43c3bc837ad4e7b901b0b4c1a7e26e8248d29d5e Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Wed, 14 Oct 2026 15:02:27 -0400 Subject: [PATCH 2/2] CI: fix windows-2025 builds and refresh workflows - windows-2025 no longer ships Visual Studio 2022; let CMake pick the installed Visual Studio instead of hardcoding the generator. - Replace leftover SIMDJSON_* options (ignored by this project) with a BUILD_SHARED_LIBS axis so the static/shared matrix tests something. - Bump actions/checkout to v7 and docker/setup-qemu-action to v4 (Node 20 deprecation). --- .github/workflows/macos.yml | 2 +- .github/workflows/pages.yml | 2 +- .github/workflows/s390x.yml | 4 ++-- .github/workflows/ubuntu22.yml | 2 +- .github/workflows/ubuntu24.yml | 2 +- .github/workflows/vs.yml | 20 ++++++++++---------- .github/workflows/vs16.yml | 22 +++++++++++----------- 7 files changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index ab8a62b..6687081 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -21,7 +21,7 @@ jobs: - {compiler: gcc} - {compiler: clang} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v7 - name: Use cmake (debug) run: | mkdir builddebug && diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 4d39519..ac857a1 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Publish website/ to gh-pages uses: peaceiris/actions-gh-pages@v4 diff --git a/.github/workflows/s390x.yml b/.github/workflows/s390x.yml index 108e308..282282b 100644 --- a/.github/workflows/s390x.yml +++ b/.github/workflows/s390x.yml @@ -15,8 +15,8 @@ jobs: ! contains(toJSON(github.event.commits.*.message), '[skip github]') runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4 - - uses: docker/setup-qemu-action@v3 + - uses: actions/checkout@v7 + - uses: docker/setup-qemu-action@v4 with: platforms: s390x - name: Build and test diff --git a/.github/workflows/ubuntu22.yml b/.github/workflows/ubuntu22.yml index 4cd3b1b..2704841 100644 --- a/.github/workflows/ubuntu22.yml +++ b/.github/workflows/ubuntu22.yml @@ -25,7 +25,7 @@ jobs: - {compiler: clang-13} - {compiler: clang} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v7 - name: Use cmake (debug) run: | mkdir builddebug && diff --git a/.github/workflows/ubuntu24.yml b/.github/workflows/ubuntu24.yml index 39a0639..f4321d2 100644 --- a/.github/workflows/ubuntu24.yml +++ b/.github/workflows/ubuntu24.yml @@ -25,7 +25,7 @@ jobs: - {compiler: clang-17} - {compiler: clang-18} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Use cmake (debug) run: | mkdir builddebug && diff --git a/.github/workflows/vs.yml b/.github/workflows/vs.yml index 548b878..86e39e8 100644 --- a/.github/workflows/vs.yml +++ b/.github/workflows/vs.yml @@ -19,20 +19,20 @@ jobs: fail-fast: false matrix: include: - - {gen: Visual Studio 17 2022, mode: , arch: Win32, static: ON} - - {gen: Visual Studio 17 2022, mode: , arch: Win32, static: OFF} - - {gen: Visual Studio 17 2022, mode: , arch: x64, static: ON} - - {gen: Visual Studio 17 2022, mode: , arch: x64, static: OFF} - - {gen: Visual Studio 17 2022, mode: -T ClangCL, arch: Win32, static: ON} - - {gen: Visual Studio 17 2022, mode: -T ClangCL, arch: Win32, static: OFF} - - {gen: Visual Studio 17 2022, mode: -T ClangCL, arch: x64, static: ON} - - {gen: Visual Studio 17 2022, mode: -T ClangCL, arch: x64, static: OFF} + - {toolset: '', arch: Win32, shared: OFF} + - {toolset: '', arch: Win32, shared: ON} + - {toolset: '', arch: x64, shared: OFF} + - {toolset: '', arch: x64, shared: ON} + - {toolset: '-T ClangCL', arch: Win32, shared: OFF} + - {toolset: '-T ClangCL', arch: Win32, shared: ON} + - {toolset: '-T ClangCL', arch: x64, shared: OFF} + - {toolset: '-T ClangCL', arch: x64, shared: ON} steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v7 - name: Configure run: | - cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} "${{matrix.mode}}" -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_BUILD_STATIC=${{matrix.static}} -DSTREAMVBYTE_ENABLE_EXAMPLES=ON -DSTREAMVBYTE_ENABLE_TESTS=ON -DSTREAMVBYTE_WALL=ON -DSTREAMVBYTE_WERROR=ON -B build + cmake -S . -G "Visual Studio 17 2022" -A ${{matrix.arch}} ${{matrix.toolset}} -DBUILD_SHARED_LIBS=${{matrix.shared}} -DSTREAMVBYTE_ENABLE_EXAMPLES=ON -DSTREAMVBYTE_ENABLE_TESTS=ON -DSTREAMVBYTE_WALL=ON -DSTREAMVBYTE_WERROR=ON -B build - name: Build Debug run: cmake --build build --config Debug --verbose - name: Build Release diff --git a/.github/workflows/vs16.yml b/.github/workflows/vs16.yml index 2021e35..80cb234 100644 --- a/.github/workflows/vs16.yml +++ b/.github/workflows/vs16.yml @@ -13,26 +13,26 @@ jobs: if: >- ! contains(toJSON(github.event.commits.*.message), '[skip ci]') && ! contains(toJSON(github.event.commits.*.message), '[skip github]') - name: windows-2025 + name: windows-2025 (latest Visual Studio) runs-on: windows-2025 strategy: fail-fast: false matrix: include: - - {gen: Visual Studio 17 2022, mode: , arch: Win32, static: ON} - - {gen: Visual Studio 17 2022, mode: , arch: Win32, static: OFF} - - {gen: Visual Studio 17 2022, mode: , arch: x64, static: ON} - - {gen: Visual Studio 17 2022, mode: , arch: x64, static: OFF} - - {gen: Visual Studio 17 2022, mode: -T ClangCL, arch: Win32, static: ON} - - {gen: Visual Studio 17 2022, mode: -T ClangCL, arch: Win32, static: OFF} - - {gen: Visual Studio 17 2022, mode: -T ClangCL, arch: x64, static: ON} - - {gen: Visual Studio 17 2022, mode: -T ClangCL, arch: x64, static: OFF} + - {toolset: '', arch: Win32, shared: OFF} + - {toolset: '', arch: Win32, shared: ON} + - {toolset: '', arch: x64, shared: OFF} + - {toolset: '', arch: x64, shared: ON} + - {toolset: '-T ClangCL', arch: Win32, shared: OFF} + - {toolset: '-T ClangCL', arch: Win32, shared: ON} + - {toolset: '-T ClangCL', arch: x64, shared: OFF} + - {toolset: '-T ClangCL', arch: x64, shared: ON} steps: - name: checkout - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Configure run: | - cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} "${{matrix.mode}}" -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_BUILD_STATIC=${{matrix.static}} -DSTREAMVBYTE_ENABLE_EXAMPLES=ON -DSTREAMVBYTE_ENABLE_TESTS=ON -DSTREAMVBYTE_WALL=ON -DSTREAMVBYTE_WERROR=ON -B build + cmake -S . -A ${{matrix.arch}} ${{matrix.toolset}} -DBUILD_SHARED_LIBS=${{matrix.shared}} -DSTREAMVBYTE_ENABLE_EXAMPLES=ON -DSTREAMVBYTE_ENABLE_TESTS=ON -DSTREAMVBYTE_WALL=ON -DSTREAMVBYTE_WERROR=ON -B build - name: Build Debug run: cmake --build build --config Debug --verbose - name: Build Release