Support big-endian platforms (s390x) in the scalar codecs

.github/workflows/macos.yml

@@ -21,7 +21,7 @@ jobs:
           - {compiler: gcc}
           - {compiler: clang}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v7
       - name: Use cmake (debug)
         run: |
           mkdir builddebug &&

.github/workflows/pages.yml

@@ -23,7 +23,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
 
       - name: Publish website/ to gh-pages
         uses: peaceiris/actions-gh-pages@v4

.github/workflows/s390x.yml

@@ -0,0 +1,30 @@
+name: s390x (big endian) CI
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  s390x-build:
+    if: >-
+      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
+      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v7
+      - uses: docker/setup-qemu-action@v4
+        with:
+          platforms: s390x
+      - name: Build and test
+        run: |
+          docker run --rm --platform linux/s390x -v "$PWD":/src -w /src debian:stable-slim sh -c '
+            apt-get update -qq &&
+            apt-get install -y -qq gcc g++ cmake make git ca-certificates > /dev/null &&
+            cmake -B build -DSTREAMVBYTE_ENABLE_EXAMPLES=ON -DSTREAMVBYTE_ENABLE_TESTS=ON -DSTREAMVBYTE_WALL=ON -DSTREAMVBYTE_WERROR=ON &&
+            cmake --build build -j4 &&
+            ctest --test-dir build --output-on-failure &&
+            ./build/example'

.github/workflows/ubuntu22.yml

@@ -25,7 +25,7 @@ jobs:
           - {compiler: clang-13}
           - {compiler: clang}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v7
       - name: Use cmake (debug)
         run: |
           mkdir builddebug &&

.github/workflows/ubuntu24.yml

@@ -25,7 +25,7 @@ jobs:
           - {compiler: clang-17}
           - {compiler: clang-18}
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v7
       - name: Use cmake (debug)
         run: |
           mkdir builddebug &&

.github/workflows/vs.yml

@@ -19,20 +19,20 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - {gen: Visual Studio 17 2022, mode: , arch: Win32, static: ON}
-          - {gen: Visual Studio 17 2022, mode: , arch: Win32, static: OFF}
-          - {gen: Visual Studio 17 2022, mode: , arch: x64, static: ON}
-          - {gen: Visual Studio 17 2022, mode: , arch: x64, static: OFF}
-          - {gen: Visual Studio 17 2022, mode: -T ClangCL, arch: Win32, static: ON}
-          - {gen: Visual Studio 17 2022, mode: -T ClangCL, arch: Win32, static: OFF}
-          - {gen: Visual Studio 17 2022, mode: -T ClangCL, arch: x64, static: ON}
-          - {gen: Visual Studio 17 2022, mode: -T ClangCL, arch: x64, static: OFF}
+          - {toolset: '', arch: Win32, shared: OFF}
+          - {toolset: '', arch: Win32, shared: ON}
+          - {toolset: '', arch: x64, shared: OFF}
+          - {toolset: '', arch: x64, shared: ON}
+          - {toolset: '-T ClangCL', arch: Win32, shared: OFF}
+          - {toolset: '-T ClangCL', arch: Win32, shared: ON}
+          - {toolset: '-T ClangCL', arch: x64, shared: OFF}
+          - {toolset: '-T ClangCL', arch: x64, shared: ON}
     steps:
     - name: checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v7
     - name: Configure
       run: |
-        cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} "${{matrix.mode}}" -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_BUILD_STATIC=${{matrix.static}} -DSTREAMVBYTE_ENABLE_EXAMPLES=ON -DSTREAMVBYTE_ENABLE_TESTS=ON -DSTREAMVBYTE_WALL=ON -DSTREAMVBYTE_WERROR=ON -B build
+        cmake -S . -G "Visual Studio 17 2022" -A ${{matrix.arch}} ${{matrix.toolset}} -DBUILD_SHARED_LIBS=${{matrix.shared}} -DSTREAMVBYTE_ENABLE_EXAMPLES=ON -DSTREAMVBYTE_ENABLE_TESTS=ON -DSTREAMVBYTE_WALL=ON -DSTREAMVBYTE_WERROR=ON -B build
     - name: Build Debug
       run: cmake --build build --config Debug --verbose
     - name: Build Release

.github/workflows/vs16.yml

@@ -13,26 +13,26 @@ jobs:
     if: >-
       ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
-    name: windows-2025
+    name: windows-2025 (latest Visual Studio)
     runs-on: windows-2025
     strategy:
       fail-fast: false
       matrix:
         include:
-          - {gen: Visual Studio 17 2022, mode: , arch: Win32, static: ON}
-          - {gen: Visual Studio 17 2022, mode: , arch: Win32, static: OFF}
-          - {gen: Visual Studio 17 2022, mode: , arch: x64, static: ON}
-          - {gen: Visual Studio 17 2022, mode: , arch: x64, static: OFF}
-          - {gen: Visual Studio 17 2022, mode: -T ClangCL, arch: Win32, static: ON}
-          - {gen: Visual Studio 17 2022, mode: -T ClangCL, arch: Win32, static: OFF}
-          - {gen: Visual Studio 17 2022, mode: -T ClangCL, arch: x64, static: ON}
-          - {gen: Visual Studio 17 2022, mode: -T ClangCL, arch: x64, static: OFF}
+          - {toolset: '', arch: Win32, shared: OFF}
+          - {toolset: '', arch: Win32, shared: ON}
+          - {toolset: '', arch: x64, shared: OFF}
+          - {toolset: '', arch: x64, shared: ON}
+          - {toolset: '-T ClangCL', arch: Win32, shared: OFF}
+          - {toolset: '-T ClangCL', arch: Win32, shared: ON}
+          - {toolset: '-T ClangCL', arch: x64, shared: OFF}
+          - {toolset: '-T ClangCL', arch: x64, shared: ON}
     steps:
     - name: checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@v7
     - name: Configure
       run: |
-        cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} "${{matrix.mode}}" -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_BUILD_STATIC=${{matrix.static}} -DSTREAMVBYTE_ENABLE_EXAMPLES=ON -DSTREAMVBYTE_ENABLE_TESTS=ON -DSTREAMVBYTE_WALL=ON -DSTREAMVBYTE_WERROR=ON -B build
+        cmake -S . -A ${{matrix.arch}} ${{matrix.toolset}} -DBUILD_SHARED_LIBS=${{matrix.shared}} -DSTREAMVBYTE_ENABLE_EXAMPLES=ON -DSTREAMVBYTE_ENABLE_TESTS=ON -DSTREAMVBYTE_WALL=ON -DSTREAMVBYTE_WERROR=ON -B build
     - name: Build Debug
       run: cmake --build build --config Debug --verbose
     - name: Build Release

src/streamvbyte_0124_decode.c

@@ -5,6 +5,7 @@
 #include "streamvbyte_shuffle_tables_0124_decode.h"
 #endif
 
+#include "streamvbyte_endian.h"
 #include <string.h> // for memcpy
 
 #ifdef __clang__
@@ -45,11 +46,10 @@ static inline uint32_t svb_decode_data(const uint8_t **dataPtrPtr, uint8_t code)
     val = (uint32_t)*dataPtr;
     dataPtr += 1;
   } else if (code == 2) { // 2 bytes
-    val = 0;
-    memcpy(&val, dataPtr, 2); // assumes little endian
+    val = svb_load_le(dataPtr, 2);
     dataPtr += 2;
   } else { // code == 3, 4 bytes
-    memcpy(&val, dataPtr, 4);
+    val = svb_load_le(dataPtr, 4);
     dataPtr += 4;
   }
 

src/streamvbyte_0124_encode.c

@@ -2,6 +2,7 @@
 #include "streamvbyte_isadetection.h"
 #include "streamvbyte_shuffle_tables_0124_encode.h"
 
+#include "streamvbyte_endian.h"
 #include <string.h> // for memcpy
 
 #ifdef __clang__
@@ -20,11 +21,11 @@ static uint8_t svb_encode_data(uint32_t val, uint8_t *__restrict__ *dataPtrPtr)
     *dataPtrPtr += 1;
     code = 1;
   } else if (val < (1 << 16)) { // 2 bytes
-    memcpy(dataPtr, &val, 2);   // assumes little endian
+    svb_store_le(dataPtr, val, 2);
     *dataPtrPtr += 2;
     code = 2;
   } else { // 4 bytes
-    memcpy(dataPtr, &val, sizeof(uint32_t));
+    svb_store_le(dataPtr, val, sizeof(uint32_t));
     *dataPtrPtr += sizeof(uint32_t);
     code = 3;
   }

src/streamvbyte_decode.c

@@ -1,6 +1,7 @@
 #include "streamvbyte.h"
 #include "streamvbyte_isadetection.h"
 
+#include "streamvbyte_endian.h"
 #include <string.h> // for memcpy
 
 #ifdef __clang__
@@ -23,15 +24,13 @@ static inline uint32_t svb_decode_data(const uint8_t **dataPtrPtr, uint8_t code)
     val = (uint32_t)*dataPtr;
     dataPtr += 1;
   } else if (code == 1) { // 2 bytes
-    val = 0;
-    memcpy(&val, dataPtr, 2); // assumes little endian
+    val = svb_load_le(dataPtr, 2);
     dataPtr += 2;
   } else if (code == 2) { // 3 bytes
-    val = 0;
-    memcpy(&val, dataPtr, 3); // assumes little endian
+    val = svb_load_le(dataPtr, 3);
     dataPtr += 3;
   } else { // code == 3
-    memcpy(&val, dataPtr, 4);
+    val = svb_load_le(dataPtr, 4);
     dataPtr += 4;
   }
 

src/streamvbyte_encode.c

@@ -1,6 +1,7 @@
 #include "streamvbyte.h"
 #include "streamvbyte_isadetection.h"
 
+#include "streamvbyte_endian.h"
 #include <string.h> // for memcpy
 
 #ifdef __clang__
@@ -20,15 +21,15 @@ static uint8_t svb_encode_data(uint32_t val, uint8_t *__restrict__ *dataPtrPtr)
     *dataPtrPtr += 1;
     code = 0;
   } else if (val < (1 << 16)) { // 2 bytes
-    memcpy(dataPtr, &val, 2);   // assumes little endian
+    svb_store_le(dataPtr, val, 2);
     *dataPtrPtr += 2;
     code = 1;
   } else if (val < (1 << 24)) { // 3 bytes
-    memcpy(dataPtr, &val, 3);   // assumes little endian
+    svb_store_le(dataPtr, val, 3);
     *dataPtrPtr += 3;
     code = 2;
   } else { // 4 bytes
-    memcpy(dataPtr, &val, sizeof(uint32_t));
+    svb_store_le(dataPtr, val, sizeof(uint32_t));
     *dataPtrPtr += sizeof(uint32_t);
     code = 3;
   }

src/streamvbyte_endian.h

@@ -0,0 +1,38 @@
+#ifndef STREAMVBYTE_ENDIAN_H
+#define STREAMVBYTE_ENDIAN_H
+
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h> // for memcpy
+
+// The stream vbyte format stores values in little-endian byte order.
+#if defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) &&               \
+    (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
+#define STREAMVBYTE_BIG_ENDIAN 1
+#endif
+
+// Reads the 'len' (1 to 4) low-order bytes of a little-endian value.
+static inline uint32_t svb_load_le(const uint8_t *p, size_t len) {
+  uint32_t val = 0;
+#ifdef STREAMVBYTE_BIG_ENDIAN
+  for (size_t i = 0; i < len; i++) {
+    val |= (uint32_t)p[i] << (8 * i);
+  }
+#else
+  memcpy(&val, p, len);
+#endif
+  return val;
+}
+
+// Writes the 'len' (1 to 4) low-order bytes of 'val' in little-endian order.
+static inline void svb_store_le(uint8_t *p, uint32_t val, size_t len) {
+#ifdef STREAMVBYTE_BIG_ENDIAN
+  for (size_t i = 0; i < len; i++) {
+    p[i] = (uint8_t)(val >> (8 * i));
+  }
+#else
+  memcpy(p, &val, len);
+#endif
+}
+
+#endif // STREAMVBYTE_ENDIAN_H

src/streamvbytedelta_decode.c

@@ -1,6 +1,7 @@
 #include "streamvbytedelta.h"
 #include "streamvbyte_isadetection.h"
 
+#include "streamvbyte_endian.h"
 #include <string.h> // for memcpy
 
 #ifdef __clang__
@@ -15,15 +16,13 @@ static inline uint32_t svb_decode_data(const uint8_t **dataPtrPtr, uint8_t code)
     val = (uint32_t)*dataPtr;
     dataPtr += 1;
   } else if (code == 1) { // 2 bytes
-    val = 0;
-    memcpy(&val, dataPtr, 2); // assumes little endian
+    val = svb_load_le(dataPtr, 2);
     dataPtr += 2;
   } else if (code == 2) { // 3 bytes
-    val = 0;
-    memcpy(&val, dataPtr, 3); // assumes little endian
+    val = svb_load_le(dataPtr, 3);
     dataPtr += 3;
   } else { // code == 3
-    memcpy(&val, dataPtr, 4);
+    val = svb_load_le(dataPtr, 4);
     dataPtr += 4;
   }
 

src/streamvbytedelta_encode.c

@@ -1,6 +1,7 @@
 #include "streamvbytedelta.h"
 #include "streamvbyte_isadetection.h"
 
+#include "streamvbyte_endian.h"
 #include <string.h> // for memcpy
 
 #ifdef STREAMVBYTE_X64
@@ -20,15 +21,15 @@ static uint8_t svb_encode_data(uint32_t val, uint8_t *__restrict__ *dataPtrPtr)
     *dataPtrPtr += 1;
     code = 0;
   } else if (val < (1 << 16)) { // 2 bytes
-    memcpy(dataPtr, &val, 2);   // assumes little endian
+    svb_store_le(dataPtr, val, 2);
     *dataPtrPtr += 2;
     code = 1;
   } else if (val < (1 << 24)) { // 3 bytes
-    memcpy(dataPtr, &val, 3);   // assumes little endian
+    svb_store_le(dataPtr, val, 3);
     *dataPtrPtr += 3;
     code = 2;
   } else { // 4 bytes
-    memcpy(dataPtr, &val, sizeof(uint32_t));
+    svb_store_le(dataPtr, val, sizeof(uint32_t));
     *dataPtrPtr += sizeof(uint32_t);
     code = 3;
   }