Skip to content

Commit 25417e3

Browse files
lemirelemire
authored
Support big-endian platforms (s390x) in the scalar codecs (#85)
* Support big-endian platforms in the scalar codecs (fixes #83) The scalar encode/decode paths copied integers to and from the byte stream with memcpy, which silently assumes a little-endian host. On big-endian systems such as s390x this produced corrupted streams. Add small svb_load_le/svb_store_le helpers that keep using memcpy on little-endian hosts (no codegen change) and fall back to explicit byte shifts on big-endian hosts. Add an s390x CI job running under QEMU. * CI: fix windows-2025 builds and refresh workflows - windows-2025 no longer ships Visual Studio 2022; let CMake pick the installed Visual Studio instead of hardcoding the generator. - Replace leftover SIMDJSON_* options (ignored by this project) with a BUILD_SHARED_LIBS axis so the static/shared matrix tests something. - Bump actions/checkout to v7 and docker/setup-qemu-action to v4 (Node 20 deprecation).
1 parent 7c472d7 commit 25417e3

14 files changed

Lines changed: 115 additions & 46 deletions

‎.github/workflows/macos.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ jobs:
2121
- {compiler: gcc}
2222
- {compiler: clang}
2323
steps:
24-
- uses: actions/checkout@v3
24+
- uses: actions/checkout@v7
2525
- name: Use cmake (debug)
2626
run: |
2727
mkdir builddebug &&

‎.github/workflows/pages.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ jobs:
2323
runs-on: ubuntu-latest
2424
steps:
2525
- name: Checkout
26-
uses: actions/checkout@v4
26+
uses: actions/checkout@v7
2727

2828
- name: Publish website/ to gh-pages
2929
uses: peaceiris/actions-gh-pages@v4

‎.github/workflows/s390x.yml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
name: s390x (big endian) CI
2+
3+
on:
4+
push:
5+
branches:
6+
- master
7+
pull_request:
8+
branches:
9+
- master
10+
11+
jobs:
12+
s390x-build:
13+
if: >-
14+
! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
15+
! contains(toJSON(github.event.commits.*.message), '[skip github]')
16+
runs-on: ubuntu-24.04
17+
steps:
18+
- uses: actions/checkout@v7
19+
- uses: docker/setup-qemu-action@v4
20+
with:
21+
platforms: s390x
22+
- name: Build and test
23+
run: |
24+
docker run --rm --platform linux/s390x -v "$PWD":/src -w /src debian:stable-slim sh -c '
25+
apt-get update -qq &&
26+
apt-get install -y -qq gcc g++ cmake make git ca-certificates > /dev/null &&
27+
cmake -B build -DSTREAMVBYTE_ENABLE_EXAMPLES=ON -DSTREAMVBYTE_ENABLE_TESTS=ON -DSTREAMVBYTE_WALL=ON -DSTREAMVBYTE_WERROR=ON &&
28+
cmake --build build -j4 &&
29+
ctest --test-dir build --output-on-failure &&
30+
./build/example'

‎.github/workflows/ubuntu22.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ jobs:
2525
- {compiler: clang-13}
2626
- {compiler: clang}
2727
steps:
28-
- uses: actions/checkout@v3
28+
- uses: actions/checkout@v7
2929
- name: Use cmake (debug)
3030
run: |
3131
mkdir builddebug &&

‎.github/workflows/ubuntu24.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ jobs:
2525
- {compiler: clang-17}
2626
- {compiler: clang-18}
2727
steps:
28-
- uses: actions/checkout@v4
28+
- uses: actions/checkout@v7
2929
- name: Use cmake (debug)
3030
run: |
3131
mkdir builddebug &&

‎.github/workflows/vs.yml‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -19,20 +19,20 @@ jobs:
1919
fail-fast: false
2020
matrix:
2121
include:
22-
- {gen: Visual Studio 17 2022, mode: , arch: Win32, static: ON}
23-
- {gen: Visual Studio 17 2022, mode: , arch: Win32, static: OFF}
24-
- {gen: Visual Studio 17 2022, mode: , arch: x64, static: ON}
25-
- {gen: Visual Studio 17 2022, mode: , arch: x64, static: OFF}
26-
- {gen: Visual Studio 17 2022, mode: -T ClangCL, arch: Win32, static: ON}
27-
- {gen: Visual Studio 17 2022, mode: -T ClangCL, arch: Win32, static: OFF}
28-
- {gen: Visual Studio 17 2022, mode: -T ClangCL, arch: x64, static: ON}
29-
- {gen: Visual Studio 17 2022, mode: -T ClangCL, arch: x64, static: OFF}
22+
- {toolset: '', arch: Win32, shared: OFF}
23+
- {toolset: '', arch: Win32, shared: ON}
24+
- {toolset: '', arch: x64, shared: OFF}
25+
- {toolset: '', arch: x64, shared: ON}
26+
- {toolset: '-T ClangCL', arch: Win32, shared: OFF}
27+
- {toolset: '-T ClangCL', arch: Win32, shared: ON}
28+
- {toolset: '-T ClangCL', arch: x64, shared: OFF}
29+
- {toolset: '-T ClangCL', arch: x64, shared: ON}
3030
steps:
3131
- name: checkout
32-
uses: actions/checkout@v3
32+
uses: actions/checkout@v7
3333
- name: Configure
3434
run: |
35-
cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} "${{matrix.mode}}" -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_BUILD_STATIC=${{matrix.static}} -DSTREAMVBYTE_ENABLE_EXAMPLES=ON -DSTREAMVBYTE_ENABLE_TESTS=ON -DSTREAMVBYTE_WALL=ON -DSTREAMVBYTE_WERROR=ON -B build
35+
cmake -S . -G "Visual Studio 17 2022" -A ${{matrix.arch}} ${{matrix.toolset}} -DBUILD_SHARED_LIBS=${{matrix.shared}} -DSTREAMVBYTE_ENABLE_EXAMPLES=ON -DSTREAMVBYTE_ENABLE_TESTS=ON -DSTREAMVBYTE_WALL=ON -DSTREAMVBYTE_WERROR=ON -B build
3636
- name: Build Debug
3737
run: cmake --build build --config Debug --verbose
3838
- name: Build Release

‎.github/workflows/vs16.yml‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -13,26 +13,26 @@ jobs:
1313
if: >-
1414
! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
1515
! contains(toJSON(github.event.commits.*.message), '[skip github]')
16-
name: windows-2025
16+
name: windows-2025 (latest Visual Studio)
1717
runs-on: windows-2025
1818
strategy:
1919
fail-fast: false
2020
matrix:
2121
include:
22-
- {gen: Visual Studio 17 2022, mode: , arch: Win32, static: ON}
23-
- {gen: Visual Studio 17 2022, mode: , arch: Win32, static: OFF}
24-
- {gen: Visual Studio 17 2022, mode: , arch: x64, static: ON}
25-
- {gen: Visual Studio 17 2022, mode: , arch: x64, static: OFF}
26-
- {gen: Visual Studio 17 2022, mode: -T ClangCL, arch: Win32, static: ON}
27-
- {gen: Visual Studio 17 2022, mode: -T ClangCL, arch: Win32, static: OFF}
28-
- {gen: Visual Studio 17 2022, mode: -T ClangCL, arch: x64, static: ON}
29-
- {gen: Visual Studio 17 2022, mode: -T ClangCL, arch: x64, static: OFF}
22+
- {toolset: '', arch: Win32, shared: OFF}
23+
- {toolset: '', arch: Win32, shared: ON}
24+
- {toolset: '', arch: x64, shared: OFF}
25+
- {toolset: '', arch: x64, shared: ON}
26+
- {toolset: '-T ClangCL', arch: Win32, shared: OFF}
27+
- {toolset: '-T ClangCL', arch: Win32, shared: ON}
28+
- {toolset: '-T ClangCL', arch: x64, shared: OFF}
29+
- {toolset: '-T ClangCL', arch: x64, shared: ON}
3030
steps:
3131
- name: checkout
32-
uses: actions/checkout@v4
32+
uses: actions/checkout@v7
3333
- name: Configure
3434
run: |
35-
cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} "${{matrix.mode}}" -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_BUILD_STATIC=${{matrix.static}} -DSTREAMVBYTE_ENABLE_EXAMPLES=ON -DSTREAMVBYTE_ENABLE_TESTS=ON -DSTREAMVBYTE_WALL=ON -DSTREAMVBYTE_WERROR=ON -B build
35+
cmake -S . -A ${{matrix.arch}} ${{matrix.toolset}} -DBUILD_SHARED_LIBS=${{matrix.shared}} -DSTREAMVBYTE_ENABLE_EXAMPLES=ON -DSTREAMVBYTE_ENABLE_TESTS=ON -DSTREAMVBYTE_WALL=ON -DSTREAMVBYTE_WERROR=ON -B build
3636
- name: Build Debug
3737
run: cmake --build build --config Debug --verbose
3838
- name: Build Release

‎src/streamvbyte_0124_decode.c‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
#include "streamvbyte_shuffle_tables_0124_decode.h"
66
#endif
77

8+
#include "streamvbyte_endian.h"
89
#include <string.h> // for memcpy
910

1011
#ifdef __clang__
@@ -45,11 +46,10 @@ static inline uint32_t svb_decode_data(const uint8_t **dataPtrPtr, uint8_t code)
4546
val = (uint32_t)*dataPtr;
4647
dataPtr += 1;
4748
} else if (code == 2) { // 2 bytes
48-
val = 0;
49-
memcpy(&val, dataPtr, 2); // assumes little endian
49+
val = svb_load_le(dataPtr, 2);
5050
dataPtr += 2;
5151
} else { // code == 3, 4 bytes
52-
memcpy(&val, dataPtr, 4);
52+
val = svb_load_le(dataPtr, 4);
5353
dataPtr += 4;
5454
}
5555

‎src/streamvbyte_0124_encode.c‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
#include "streamvbyte_isadetection.h"
33
#include "streamvbyte_shuffle_tables_0124_encode.h"
44

5+
#include "streamvbyte_endian.h"
56
#include <string.h> // for memcpy
67

78
#ifdef __clang__
@@ -20,11 +21,11 @@ static uint8_t svb_encode_data(uint32_t val, uint8_t *__restrict__ *dataPtrPtr)
2021
*dataPtrPtr += 1;
2122
code = 1;
2223
} else if (val < (1 << 16)) { // 2 bytes
23-
memcpy(dataPtr, &val, 2); // assumes little endian
24+
svb_store_le(dataPtr, val, 2);
2425
*dataPtrPtr += 2;
2526
code = 2;
2627
} else { // 4 bytes
27-
memcpy(dataPtr, &val, sizeof(uint32_t));
28+
svb_store_le(dataPtr, val, sizeof(uint32_t));
2829
*dataPtrPtr += sizeof(uint32_t);
2930
code = 3;
3031
}

‎src/streamvbyte_decode.c‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#include "streamvbyte.h"
22
#include "streamvbyte_isadetection.h"
33

4+
#include "streamvbyte_endian.h"
45
#include <string.h> // for memcpy
56

67
#ifdef __clang__
@@ -23,15 +24,13 @@ static inline uint32_t svb_decode_data(const uint8_t **dataPtrPtr, uint8_t code)
2324
val = (uint32_t)*dataPtr;
2425
dataPtr += 1;
2526
} else if (code == 1) { // 2 bytes
26-
val = 0;
27-
memcpy(&val, dataPtr, 2); // assumes little endian
27+
val = svb_load_le(dataPtr, 2);
2828
dataPtr += 2;
2929
} else if (code == 2) { // 3 bytes
30-
val = 0;
31-
memcpy(&val, dataPtr, 3); // assumes little endian
30+
val = svb_load_le(dataPtr, 3);
3231
dataPtr += 3;
3332
} else { // code == 3
34-
memcpy(&val, dataPtr, 4);
33+
val = svb_load_le(dataPtr, 4);
3534
dataPtr += 4;
3635
}
3736

0 commit comments

Comments
 (0)