From 34ea2c8592a0d011a4b10babfad2921835ebf85b Mon Sep 17 00:00:00 2001 From: Yagiz Nizipli Date: Mon, 21 Sep 2026 16:15:17 -0400 Subject: [PATCH 1/7] src: compress embedded icu, builtins, and snapshot Store the full ICU data file, one-byte builtin sources, and the V8 startup snapshot plus code caches as zstd frames instead of raw bytes. Decompress each once at startup and keep it for the process lifetime. On macOS Release, link the executable with -dead_strip, -x, and -S. A Release arm64 macOS binary dropped from 140 MB to 59 MB. Intl, builtins, the snapshot, and code cache behavior are unchanged. Startup RSS rose by about 40 MB because those blobs are private heap instead of file-backed pages. Signed-off-by: Yagiz Nizipli --- deps/zstd/zstd.gyp | 13 ++- node.gyp | 22 +++++ node.gypi | 6 +- src/node_i18n.cc | 46 +++++++++- src/node_snapshotable.cc | 179 +++++++++++++++++++++----------------- src/node_union_bytes.h | 22 ++++- src/zstd_blob.cc | 47 ++++++++++ src/zstd_blob.h | 18 ++++ tools/icu/icu-generic.gyp | 116 ++++++++++++++++++++---- tools/js2c.cc | 131 ++++++++++++++++++++++++++++ tools/zstd_compress.cc | 95 ++++++++++++++++++++ 11 files changed, 595 insertions(+), 100 deletions(-) create mode 100644 src/zstd_blob.cc create mode 100644 src/zstd_blob.h create mode 100644 tools/zstd_compress.cc diff --git a/deps/zstd/zstd.gyp b/deps/zstd/zstd.gyp index 6018f12831b..9e82086ca3c 100644 --- a/deps/zstd/zstd.gyp +++ b/deps/zstd/zstd.gyp @@ -100,7 +100,16 @@ ], 'sources': [ '<@(zstd_sources)', - ] - } + ], + 'toolsets': ['host', 'target'], + }, + { + 'target_name': 'zstd_compress', + 'type': 'executable', + 'toolsets': ['host'], + 'dependencies': ['zstd#host'], + 'include_dirs': ['lib'], + 'sources': ['../../tools/zstd_compress.cc'], + }, ] } diff --git a/node.gyp b/node.gyp index 8692f42c897..0e095fb4d07 100644 --- a/node.gyp +++ b/node.gyp @@ -185,6 +185,8 @@ 'src/node_watchdog.cc', 'src/node_worker.cc', 'src/node_zlib.cc', + 'src/zstd_blob.cc', + 'src/zstd_blob.h', 'src/path.cc', 'src/permission/fs_permission.cc', 'src/permission/permission.cc', @@ -659,6 +661,23 @@ 'WARNING_CFLAGS': [ '-Werror' ], }, }], + # The Release executable force-loads several static libraries, so + # unreferenced objects and the symbol table dominate the file. + # -dead_strip drops unreferenced .o files, -x drops local symbols, + # and -S drops STABS. Debug keeps symbols. + ['OS=="mac" or OS=="ios"', { + 'configurations': { + 'Release': { + 'xcode_settings': { + 'OTHER_LDFLAGS': [ + '-Wl,-dead_strip', + '-Wl,-x', + '-Wl,-S', + ], + }, + }, + }, + }], ['node_shared=="true" and OS=="win"', { 'dependencies': ['generate_node_def'], 'msvs_settings': { @@ -1671,6 +1690,9 @@ [ 'node_shared_libuv=="false"', { 'dependencies': [ 'deps/uv/uv.gyp:libuv#host' ], }], + [ 'node_shared_zstd=="false"', { + 'dependencies': [ 'deps/zstd/zstd.gyp:zstd#host' ], + }], [ 'OS in "linux mac openharmony"', { 'defines': ['NODE_JS2C_USE_STRING_LITERALS'], }], diff --git a/node.gypi b/node.gypi index b382784e610..6119a869661 100644 --- a/node.gypi +++ b/node.gypi @@ -135,6 +135,9 @@ '<(icu_gyp_path):icuuc', ], 'conditions': [ + [ 'icu_system!="true"', { + 'defines': [ 'NODE_HAVE_EMBEDDED_ICU_ZSTD=1' ], + }], [ 'icu_small=="true"', { 'defines': [ 'NODE_HAVE_SMALL_ICU=1' ], 'conditions': [ @@ -144,7 +147,8 @@ ], }], ], - }]], + }], + ], }], [ 'node_use_bundled_v8=="true" and \ node_enable_v8_vtunejit=="true" and (target_arch=="x64" or \ diff --git a/src/node_i18n.cc b/src/node_i18n.cc index e743941734a..b110a6883c9 100644 --- a/src/node_i18n.cc +++ b/src/node_i18n.cc @@ -43,6 +43,10 @@ #include "node_i18n.h" #include "node_external_reference.h" #include "simdutf.h" +#include "zstd_blob.h" + +#include +#include #if defined(NODE_HAVE_I18N_SUPPORT) @@ -68,7 +72,7 @@ #include #include "nbytes.h" -#ifdef NODE_HAVE_SMALL_ICU +#if defined(NODE_HAVE_SMALL_ICU) || defined(NODE_HAVE_EMBEDDED_ICU_ZSTD) #include /* if this is defined, we have a 'secondary' entry point. @@ -85,6 +89,38 @@ extern "C" const char U_DATA_API SMALL_ICUDATA_ENTRY_POINT[]; #endif +#ifdef NODE_HAVE_EMBEDDED_ICU_ZSTD +extern "C" const uint8_t node_icu_zstd_dat[]; + +static uint64_t ReadU64LE(const uint8_t* bytes) { + uint64_t value = 0; + for (int i = 0; i < 8; i++) { + value |= static_cast(bytes[i]) << (8 * i); + } + return value; +} + +// Decompress the embedded ICU data file. The returned pointer is aligned and +// lives for the process lifetime. nullptr on failure, with `error` set. +static uint8_t* DecompressEmbeddedICU(std::string* error) { + const uint8_t* bytes = node_icu_zstd_dat; + if (memcmp(bytes, "ICUZ", 4) != 0) { + *error = "embedded ICU data header is invalid"; + return nullptr; + } + uint64_t raw_size = ReadU64LE(bytes + 4); + uint64_t compressed_size = ReadU64LE(bytes + 12); + size_t got = 0; + uint8_t* data = node::ZstdDecompressAligned( + bytes + 20, static_cast(compressed_size), &got); + if (data == nullptr || got != raw_size) { + *error = "failed to decompress embedded ICU data"; + return nullptr; + } + return data; +} +#endif // NODE_HAVE_EMBEDDED_ICU_ZSTD + namespace node { using v8::Context; @@ -555,7 +591,13 @@ ConverterObject::ConverterObject( bool InitializeICUDirectory(const std::string& path, std::string* error) { UErrorCode status = U_ZERO_ERROR; if (path.empty()) { -#ifdef NODE_HAVE_SMALL_ICU +#ifdef NODE_HAVE_EMBEDDED_ICU_ZSTD + static uint8_t* icu_data = DecompressEmbeddedICU(error); + if (icu_data == nullptr) { + return false; + } + udata_setCommonData(icu_data, &status); +#elif defined(NODE_HAVE_SMALL_ICU) // install the 'small' data. udata_setCommonData(&SMALL_ICUDATA_ENTRY_POINT, &status); #else // !NODE_HAVE_SMALL_ICU diff --git a/src/node_snapshotable.cc b/src/node_snapshotable.cc index faf95133bf4..da175a83e75 100644 --- a/src/node_snapshotable.cc +++ b/src/node_snapshotable.cc @@ -8,7 +8,6 @@ #include "base_object-inl.h" #include "blob_serializer_deserializer-inl.h" #include "debug_utils-inl.h" -#include "embedded_data.h" #include "encoding_binding.h" #include "env-inl.h" #include "glob/node_glob.h" @@ -30,6 +29,9 @@ #include "node_v8_platform-inl.h" #include "simdjson.h" #include "timers.h" +#include "zstd.h" + +#include #if HAVE_INSPECTOR #include "inspector/worker_inspector.h" // ParentInspectorHandle @@ -737,17 +739,6 @@ SnapshotData::~SnapshotData() { } } -static std::string GetCodeCacheDefName(const std::string& id) { - char buf[64] = {0}; - size_t size = id.size(); - CHECK_LT(size, sizeof(buf)); - for (size_t i = 0; i < size; ++i) { - char ch = id[i]; - buf[i] = (ch == '-' || ch == '/') ? '_' : ch; - } - return std::string(buf) + std::string("_cache_data"); -} - static std::string FormatSize(size_t size) { char buf[64] = {0}; if (size < 1024) { @@ -761,88 +752,81 @@ static std::string FormatSize(size_t size) { return buf; } -template - requires(std::same_as || std::same_as) -void WriteByteVectorLiteral(std::ostream* ss, - const T* vec, - size_t size, - const char* var_name, - bool use_array_literals) { - constexpr bool is_uint8_t = std::is_same_v; - constexpr const char* type_name = is_uint8_t ? "uint8_t" : "char"; - if (!use_array_literals) { - const uint8_t* data = reinterpret_cast(vec); - *ss << "static const " << type_name << " *" << var_name << " = "; - *ss << (is_uint8_t ? R"(reinterpret_cast(")" : "\""); - for (size_t i = 0; i < size; i++) { - const uint8_t ch = data[i]; - *ss << GetOctalCode(ch); - if (i % 64 == 63) { - // Go to a newline every 64 bytes since many text editors have - // problems with very long lines. - *ss << "\"\n\""; - } +void WriteCompressedByteArray(std::ostream* ss, + const uint8_t* bytes, + size_t size, + const char* name) { + *ss << "static const uint8_t " << name << "[] = {\n"; + for (size_t i = 0; i < size; i++) { + char buf[8]; + snprintf(buf, sizeof(buf), "0x%02x,", bytes[i]); + *ss << buf; + if ((i % 16) == 15) { + *ss << '\n'; } - *ss << (is_uint8_t ? "\");\n" : "\";\n"); - } else { - *ss << "static const " << type_name << " " << var_name << "[] = {"; - for (size_t i = 0; i < size; i++) { - *ss << std::to_string(vec[i]) << (i == size - 1 ? '\n' : ','); - if (i % 64 == 63) { - // Print a newline every 64 units and a offset to improve - // readability. - *ss << " // " << (i / 64) << "\n"; - } - } - *ss << "};\n"; } -} - -static void WriteCodeCacheInitializer(std::ostream* ss, - const std::string& id, - size_t size) { - std::string def_name = GetCodeCacheDefName(id); - *ss << " { \"" << id << "\",\n"; - *ss << " {" << def_name << ",\n"; - *ss << " " << size << ",\n"; - *ss << " }\n"; - *ss << " },\n"; + *ss << "\n};\n"; } void FormatBlob(std::ostream& ss, const SnapshotData* data, bool use_array_literals) { + // The snapshot blob and every builtin code cache are one zstd frame. + // use_array_literals only affected the previous per-byte literal style. + (void)use_array_literals; + std::vector raw; + auto append32 = [&](uint32_t value) { + uint8_t bytes[4]; + memcpy(bytes, &value, sizeof(bytes)); + raw.insert(raw.end(), bytes, bytes + sizeof(bytes)); + }; + auto append_bytes = [&](const uint8_t* bytes, size_t size) { + raw.insert(raw.end(), bytes, bytes + size); + }; + const size_t snapshot_size = data->v8_snapshot_blob_data.raw_size; + append32(static_cast(snapshot_size)); + append32(static_cast(data->code_cache.size())); + for (const auto& item : data->code_cache) { + append32(static_cast(item.data.length)); + } + while (raw.size() % 16 != 0) { + raw.push_back(0); + } + append_bytes(reinterpret_cast(data->v8_snapshot_blob_data.data), + snapshot_size); + for (const auto& item : data->code_cache) { + append_bytes(item.data.data, item.data.length); + } + + std::vector compressed(ZSTD_compressBound(raw.size())); + size_t compressed_size = ZSTD_compress( + compressed.data(), compressed.size(), raw.data(), raw.size(), 19); + CHECK(!ZSTD_isError(compressed_size)); + fprintf(stderr, + "snapshot: embedded blob %zu -> %zu bytes\n", + raw.size(), + compressed_size); + ss << R"(#include +#include +#include +#include +#include + #include "env.h" #include "node_snapshot_builder.h" #include "v8.h" +#include "zstd_blob.h" // This file is generated by tools/snapshot. Do not edit. namespace node { )"; - WriteByteVectorLiteral(&ss, - data->v8_snapshot_blob_data.data, - data->v8_snapshot_blob_data.raw_size, - "v8_snapshot_blob_data", - use_array_literals); - - ss << R"(static const int v8_snapshot_blob_size = )" - << data->v8_snapshot_blob_data.raw_size << ";\n"; + WriteCompressedByteArray( + &ss, compressed.data(), compressed_size, "embedded_snapshot_zstd"); - // Windows can't deal with too many large vector initializers. - // Store the data into static arrays first. - for (const auto& item : data->code_cache) { - std::string var_name = GetCodeCacheDefName(item.id); - WriteByteVectorLiteral(&ss, - item.data.data, - item.data.length, - var_name.c_str(), - use_array_literals); - } - - ss << R"(const SnapshotData snapshot_data { + ss << R"(SnapshotData snapshot_data { // -- data_ownership begins -- SnapshotData::DataOwnership::kNotOwned, // -- data_ownership ends -- @@ -851,7 +835,7 @@ namespace node { << R"(, // -- metadata ends -- // -- v8_snapshot_blob_data begins -- - { v8_snapshot_blob_data, v8_snapshot_blob_size }, + { nullptr, 0 }, // -- v8_snapshot_blob_data ends -- // -- v8_snapshot_blob_data_ownership begins -- SnapshotData::DataOwnership::kNotOwned, @@ -869,7 +853,9 @@ namespace node { // -- code_cache begins -- {)"; for (const auto& item : data->code_cache) { - WriteCodeCacheInitializer(&ss, item.id, item.data.length); + ss << " { \"" << item.id << "\",\n"; + ss << " { nullptr, 0 }\n"; + ss << " },\n"; } ss << R"( } @@ -877,6 +863,43 @@ namespace node { }; const SnapshotData* SnapshotBuilder::GetEmbeddedSnapshotData() { + static std::once_flag once; + std::call_once(once, [] { + size_t raw_size = 0; + static uint8_t* storage = ZstdDecompressAligned( + embedded_snapshot_zstd, sizeof(embedded_snapshot_zstd), &raw_size); + CHECK_NE(storage, nullptr); + const uint8_t* cursor = storage; + auto read32 = [&]() -> uint32_t { + CHECK_LE(static_cast(cursor - storage) + 4, raw_size); + uint32_t value = 0; + memcpy(&value, cursor, 4); + cursor += 4; + return value; + }; + uint32_t snapshot_size = read32(); + uint32_t cache_count = read32(); + CHECK_EQ(static_cast(cache_count), snapshot_data.code_cache.size()); + std::vector lengths(cache_count); + for (uint32_t i = 0; i < cache_count; i++) { + lengths[i] = read32(); + } + while (static_cast(cursor - storage) % 16 != 0) { + cursor++; + } + CHECK_LE(static_cast(cursor - storage) + snapshot_size, raw_size); + snapshot_data.v8_snapshot_blob_data.data = + reinterpret_cast(cursor); + snapshot_data.v8_snapshot_blob_data.raw_size = + static_cast(snapshot_size); + cursor += snapshot_size; + for (uint32_t i = 0; i < cache_count; i++) { + CHECK_LE(static_cast(cursor - storage) + lengths[i], raw_size); + snapshot_data.code_cache[i].data.data = cursor; + snapshot_data.code_cache[i].data.length = lengths[i]; + cursor += lengths[i]; + } + }); return &snapshot_data; } } // namespace node diff --git a/src/node_union_bytes.h b/src/node_union_bytes.h index 4a3f67980fd..d6774b5e92c 100644 --- a/src/node_union_bytes.h +++ b/src/node_union_bytes.h @@ -8,6 +8,11 @@ namespace node { +// Set by the embedded-builtin blob (node_javascript.cc) so external-string +// resources can decompress their source the first time V8 reads it. +using BuiltinSourceEnsure = void (*)(); +inline BuiltinSourceEnsure builtin_source_ensure = nullptr; + // An external resource intended to be used with static lifetime. template class StaticExternalByteResource : public Base { @@ -21,9 +26,22 @@ class StaticExternalByteResource : public Base { : data_(data), length_(length), owning_ptr_(owning_ptr) {} const IChar* data() const override { + if (data_ == nullptr && builtin_source_ensure != nullptr) { + builtin_source_ensure(); + } return reinterpret_cast(data_); } - size_t length() const override { return length_; } + size_t length() const override { + if (data_ == nullptr && builtin_source_ensure != nullptr) { + builtin_source_ensure(); + } + return length_; + } + + void set_data(const Char* data, size_t length) { + data_ = data; + length_ = length; + } void Dispose() override { // We ignore Dispose calls from V8, even if we "own" a resource via @@ -37,7 +55,7 @@ class StaticExternalByteResource : public Base { private: const Char* data_; - const size_t length_; + size_t length_; std::shared_ptr owning_ptr_; }; diff --git a/src/zstd_blob.cc b/src/zstd_blob.cc new file mode 100644 index 00000000000..29ad46f23d0 --- /dev/null +++ b/src/zstd_blob.cc @@ -0,0 +1,47 @@ +#include "zstd_blob.h" + +#include + +#include "zstd.h" + +#if defined(_WIN32) +#include +#endif + +namespace node { + +uint8_t* ZstdDecompressAligned(const uint8_t* src, + size_t src_size, + size_t* raw_size) { + unsigned long long content = ZSTD_getFrameContentSize(src, src_size); + if (content == ZSTD_CONTENTSIZE_ERROR || + content == ZSTD_CONTENTSIZE_UNKNOWN || content == 0) { + return nullptr; + } + size_t size = static_cast(content); + size_t alloc = (size + 15u) & ~size_t{15}; + void* buf = nullptr; +#if defined(_WIN32) + buf = _aligned_malloc(alloc, 16); +#else + if (posix_memalign(&buf, 16, alloc) != 0) { + buf = nullptr; + } +#endif + if (buf == nullptr) { + return nullptr; + } + size_t got = ZSTD_decompress(buf, size, src, src_size); + if (ZSTD_isError(got) || got != size) { +#if defined(_WIN32) + _aligned_free(buf); +#else + free(buf); +#endif + return nullptr; + } + *raw_size = size; + return static_cast(buf); +} + +} // namespace node diff --git a/src/zstd_blob.h b/src/zstd_blob.h new file mode 100644 index 00000000000..9ac27bb1d73 --- /dev/null +++ b/src/zstd_blob.h @@ -0,0 +1,18 @@ +#ifndef SRC_ZSTD_BLOB_H_ +#define SRC_ZSTD_BLOB_H_ + +#include +#include + +namespace node { + +// Decompress one zstd frame into a 16-byte-aligned buffer. The returned +// pointer is process-lifetime storage; callers must not free it. `*raw_size` +// is the decompressed size. Returns nullptr on failure. +uint8_t* ZstdDecompressAligned(const uint8_t* src, + size_t src_size, + size_t* raw_size); + +} // namespace node + +#endif // SRC_ZSTD_BLOB_H_ diff --git a/tools/icu/icu-generic.gyp b/tools/icu/icu-generic.gyp index c4e8c6fbb9f..521e0f3a499 100644 --- a/tools/icu/icu-generic.gyp +++ b/tools/icu/icu-generic.gyp @@ -138,14 +138,32 @@ [ 'icu_small == "false"', { # and OS=win # full data - just build the full data file, then we are done. 'sources': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], - 'dependencies': [ 'genccode#host' ], + 'dependencies': [ + 'genccode#host', + 'icustubdata', + '../../deps/zstd/zstd.gyp:zstd_compress#host', + ], + 'export_dependent_settings': [ 'icustubdata' ], 'conditions': [ [ 'clang==1', { 'actions': [ + { + 'action_name': 'icu_zstd', + 'msvs_quote_cmd': 0, + 'inputs': [ + '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '<(icu_data_in)', + ], + 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], + 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '--icu', + '<(icu_data_in)', + '<@(_outputs)' ], + }, { 'action_name': 'icudata', 'msvs_quote_cmd': 0, - 'inputs': [ '<(icu_data_in)' ], + 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], # on Windows, we can go directly to .obj file (-o) option. # for Clang use "-c <(target_arch)" option @@ -154,16 +172,30 @@ '-c', '<(target_arch)', '-d', '<(SHARED_INTERMEDIATE_DIR)', '-n', 'icudata', - '-e', 'icudt<(icu_ver_major)', + '-e', 'node_icu_zstd', + '-f', 'icudt<(icu_ver_major)<(icu_endianness)_dat', '<@(_inputs)' ], }, ], }, { 'actions': [ + { + 'action_name': 'icu_zstd', + 'msvs_quote_cmd': 0, + 'inputs': [ + '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '<(icu_data_in)', + ], + 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], + 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '--icu', + '<(icu_data_in)', + '<@(_outputs)' ], + }, { 'action_name': 'icudata', 'msvs_quote_cmd': 0, - 'inputs': [ '<(icu_data_in)' ], + 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], # on Windows, we can go directly to .obj file (-o) option. # for MSVC do not use "-c <(target_arch)" option @@ -171,7 +203,8 @@ '<@(icu_asm_opts)', # -o '-d', '<(SHARED_INTERMEDIATE_DIR)', '-n', 'icudata', - '-e', 'icudt<(icu_ver_major)', + '-e', 'node_icu_zstd', + '-f', 'icudt<(icu_ver_major)<(icu_endianness)_dat', '<@(_inputs)' ], }, ], @@ -180,7 +213,8 @@ }, { # icu_small == TRUE and OS == win # link against stub data primarily # then, use icupkg and genccode to rebuild data - 'dependencies': [ 'icustubdata', 'genccode#host', 'icupkg#host', 'genrb#host', 'iculslocs#host' ], + 'dependencies': [ 'icustubdata', 'genccode#host', 'icupkg#host', 'genrb#host', 'iculslocs#host', + '../../deps/zstd/zstd.gyp:zstd_compress#host' ], 'export_dependent_settings': [ 'icustubdata' ], 'actions': [ { @@ -200,18 +234,32 @@ '-v', '-L', '<(icu_locales)'], }, + { + 'action_name': 'icu_zstd', + 'msvs_quote_cmd': 0, + 'inputs': [ + '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat', + ], + 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], + 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '--icu', + '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat', + '<@(_outputs)' ], + }, { # build final .dat -> .obj 'action_name': 'genccode', 'msvs_quote_cmd': 0, - 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat' ], + 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], 'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)', '<@(icu_asm_opts)', # -o '-c', '<(target_arch)', '-d', '<(SHARED_INTERMEDIATE_DIR)/', '-n', 'icudata', - '-e', 'icusmdt<(icu_ver_major)', + '-e', 'node_icu_zstd', + '-f', 'icudt<(icu_ver_major)<(icu_endianness)_dat', '<@(_inputs)' ], }, ], @@ -221,9 +269,19 @@ }, { # OS != win 'conditions': [ [ 'icu_small == "false"', { - # full data - no trim needed + # full data - no trim needed. The bytes embedded below are + # zstd-compressed; node decompresses them before udata_setCommonData. + # icustubdata satisfies ICU's icudtXX_dat link reference. 'sources': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)_dat.<(icu_asm_ext)' ], - 'dependencies': [ 'genccode#host', 'icupkg#host', 'icu_implementation#host', 'icu_uconfig' ], + 'dependencies': [ + 'genccode#host', + 'icupkg#host', + 'icu_implementation#host', + 'icu_uconfig', + 'icustubdata', + '../../deps/zstd/zstd.gyp:zstd_compress#host', + ], + 'export_dependent_settings': [ 'icustubdata' ], 'include_dirs': [ '<(icu_path)/source/common', ], @@ -250,12 +308,26 @@ ], }, { - # convert full ICU data file to .c, or .S, etc. + 'action_name': 'icu_zstd', + 'inputs': [ + '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major).dat', + ], + 'outputs':[ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], + 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '--icu', + '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major).dat', + '<@(_outputs)' ], + }, + { + # convert compressed ICU data to .c, or .S, etc. + # -e names the symbol node_icu_zstd_dat so ICU does not + # treat the compressed bytes as its data entry point. 'action_name': 'icudata', - 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major).dat' ], + 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], 'outputs':[ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)_dat.<(icu_asm_ext)' ], 'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)', - '-e', 'icudt<(icu_ver_major)', + '-e', 'node_icu_zstd', '-d', '<(SHARED_INTERMEDIATE_DIR)', '<@(icu_asm_opts)', '-f', 'icudt<(icu_ver_major)_dat', @@ -266,7 +338,8 @@ # link against stub data (as primary data) # then, use icupkg and genccode to rebuild small data 'dependencies': [ 'icustubdata', 'genccode#host', 'icupkg#host', 'genrb#host', 'iculslocs#host', - 'icu_implementation', 'icu_uconfig' ], + 'icu_implementation', 'icu_uconfig', + '../../deps/zstd/zstd.gyp:zstd_compress#host' ], 'export_dependent_settings': [ 'icustubdata' ], 'actions': [ { @@ -294,13 +367,26 @@ '<@(_inputs)', '<@(_outputs)', ], + }, { + 'action_name': 'icu_zstd', + 'inputs': [ + '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '<(SHARED_INTERMEDIATE_DIR)/icutmp/icusmdt<(icu_ver_major).dat', + ], + 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], + 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '--icu', + '<(SHARED_INTERMEDIATE_DIR)/icutmp/icusmdt<(icu_ver_major).dat', + '<@(_outputs)' ], }, { # For icu-small, always use .c, don't try to use .S, etc. 'action_name': 'genccode', - 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icutmp/icusmdt<(icu_ver_major).dat' ], + 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icusmdt<(icu_ver_major)_dat.<(icu_asm_ext)' ], 'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)', '<@(icu_asm_opts)', + '-e', 'node_icu_zstd', + '-f', 'icusmdt<(icu_ver_major)_dat', '-d', '<(SHARED_INTERMEDIATE_DIR)', '<@(_inputs)' ], }, diff --git a/tools/js2c.cc b/tools/js2c.cc index 2cb09f8e1d7..e5bc0f50f9c 100644 --- a/tools/js2c.cc +++ b/tools/js2c.cc @@ -3,7 +3,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -14,6 +16,7 @@ #include "executable_wrapper.h" #include "simdutf.h" #include "uv.h" +#include "zstd.h" #if defined(_WIN32) #include // _S_IREAD _S_IWRITE @@ -172,10 +175,15 @@ std::vector Join(const Fragments& fragments, } const char* kTemplate = R"( +#include +#include +#include + #include "env-inl.h" #include "node_builtins.h" #include "node_external_reference.h" #include "node_internals.h" +#include "zstd_blob.h" namespace node { @@ -191,6 +199,7 @@ const ThreadsafeCopyOnWrite global_source_map { } // anonymous namespace void BuiltinLoader::LoadJavaScriptSource() { + EnsureEmbeddedBuiltinSourcesImpl(); source_ = global_source_map; } @@ -447,6 +456,12 @@ enum class CodeType { kLatin1, // Code points are all within 0-255 kTwoByte, }; +struct OneBytePiece { + std::string var; + std::vector bytes; +}; +std::vector one_byte_pieces; + template Fragment GetDefinitionImpl(const std::vector& code, const std::string& var, @@ -454,6 +469,21 @@ Fragment GetDefinitionImpl(const std::vector& code, constexpr bool is_two_byte = std::is_same_v; static_assert(is_two_byte || std::is_same_v); + // One-byte builtins are stored in a single zstd blob and inflated the + // first time V8 reads the external string. Two-byte sources stay literal + // so the target compiler picks the endianness. + if (type != CodeType::kTwoByte) { + one_byte_pieces.push_back(OneBytePiece{ + var, + std::vector( + reinterpret_cast(code.data()), + reinterpret_cast(code.data()) + code.size()), + }); + std::string decl = "static StaticExternalOneByteResource " + var + + "_resource(nullptr, 0, nullptr);\n"; + return Fragment(decl.begin(), decl.end()); + } + size_t count = is_two_byte ? simdutf::utf16_length_from_utf8(code.data(), code.size()) : code.size(); @@ -824,10 +854,110 @@ int AddGypi(const std::string& var, return 0; } +Fragment EmitCompressedBuiltins() { + std::vector raw; + auto append32 = [&](uint32_t value) { + uint8_t bytes[4]; + memcpy(bytes, &value, sizeof(bytes)); + raw.insert(raw.end(), bytes, bytes + sizeof(bytes)); + }; + append32(static_cast(one_byte_pieces.size())); + for (const OneBytePiece& piece : one_byte_pieces) { + append32(static_cast(piece.bytes.size())); + raw.insert(raw.end(), piece.bytes.begin(), piece.bytes.end()); + while (raw.size() % 4 != 0) { + raw.push_back(0); + } + } + + size_t bound = ZSTD_compressBound(raw.size()); + std::vector compressed(bound); + size_t compressed_size = + ZSTD_compress(compressed.data(), bound, raw.data(), raw.size(), 19); + if (ZSTD_isError(compressed_size)) { + fprintf(stderr, + "js2c: zstd compress failed: %s\n", + ZSTD_getErrorName(compressed_size)); + exit(1); + } + compressed.resize(compressed_size); + fprintf(stderr, + "js2c: builtin sources %zu -> %zu bytes\n", + raw.size(), + compressed_size); + + std::string out; + out.reserve(compressed.size() * 5 + one_byte_pieces.size() * 64 + 2048); + out += "static const uint8_t node_builtin_sources_zstd[] = {\n"; + for (size_t i = 0; i < compressed.size(); i++) { + char buf[8]; + snprintf(buf, sizeof(buf), "0x%02x,", compressed[i]); + out += buf; + if ((i % 16) == 15) { + out += '\n'; + } + } + out += "\n};\n\n"; + out += "static void EnsureEmbeddedBuiltinSourcesImpl() {\n"; + out += " static std::once_flag once;\n"; + out += " std::call_once(once, [] {\n"; + out += " size_t raw_size = 0;\n"; + out += " static uint8_t* storage = node::ZstdDecompressAligned(\n"; + out += " node_builtin_sources_zstd,\n"; + out += " sizeof(node_builtin_sources_zstd),\n"; + out += " &raw_size);\n"; + out += " CHECK_NE(storage, nullptr);\n"; + out += " const uint8_t* cursor = storage;\n"; + out += " const uint8_t* end = storage + raw_size;\n"; + out += " auto read32 = [&](uint32_t* value) {\n"; + out += " CHECK_LE(cursor + 4, end);\n"; + out += " memcpy(value, cursor, 4);\n"; + out += " cursor += 4;\n"; + out += " };\n"; + out += " uint32_t count = 0;\n"; + out += " read32(&count);\n"; + out += " CHECK_EQ(count, "; + out += std::to_string(one_byte_pieces.size()); + out += "u);\n"; + if (!one_byte_pieces.empty()) { + out += " struct Slot { StaticExternalOneByteResource* resource; };\n"; + out += " static const Slot slots[] = {\n"; + for (const OneBytePiece& piece : one_byte_pieces) { + out += " { &"; + out += piece.var; + out += "_resource },\n"; + } + out += " };\n"; + out += " static uint8_t empty = 0;\n"; + out += " for (uint32_t i = 0; i < count; i++) {\n"; + out += " uint32_t byte_len = 0;\n"; + out += " read32(&byte_len);\n"; + out += " CHECK_LE(cursor + byte_len, end);\n"; + out += " const uint8_t* data = byte_len == 0 ? &empty : cursor;\n"; + out += " slots[i].resource->set_data(data, byte_len);\n"; + out += " cursor += byte_len;\n"; + out += " while (static_cast(cursor - storage) % 4 != 0) {\n"; + out += " cursor++;\n"; + out += " }\n"; + out += " }\n"; + out += " CHECK_EQ(static_cast(sizeof(slots) / sizeof(slots[0])),\n"; + out += " static_cast(count));\n"; + } + out += " });\n"; + out += "}\n\n"; + out += "static struct RegisterBuiltinSourceEnsure {\n"; + out += " RegisterBuiltinSourceEnsure() {\n"; + out += " builtin_source_ensure = EnsureEmbeddedBuiltinSourcesImpl;\n"; + out += " }\n"; + out += "} register_builtin_source_ensure;\n"; + return Fragment(out.begin(), out.end()); +} + int JS2C(const FileList& js_files, const FileList& mjs_files, const std::string& config, const std::string& dest) { + one_byte_pieces.clear(); Fragments definitions; definitions.reserve(js_files.size() + mjs_files.size() + 1); Fragments initializers; @@ -854,6 +984,7 @@ int JS2C(const FileList& js_files, if (r != 0) { return r; } + definitions.push_back(EmitCompressedBuiltins()); Fragment out = Format(definitions, initializers, registrations); return WriteIfChanged(out, dest); } diff --git a/tools/zstd_compress.cc b/tools/zstd_compress.cc new file mode 100644 index 00000000000..13a9f23568d --- /dev/null +++ b/tools/zstd_compress.cc @@ -0,0 +1,95 @@ +// Host tool: compress a file with zstd for embedding in the node binary. +// +// zstd_compress --icu +// +// `--icu` prefixes a little-endian header: +// magic "ICUZ" | uint64 raw_size | uint64 compressed_size | zstd frame + +#include +#include +#include + +#include + +#include "zstd.h" + +namespace { + +void WriteU64LE(FILE* out, uint64_t value) { + uint8_t bytes[8]; + for (int i = 0; i < 8; i++) { + bytes[i] = static_cast((value >> (8 * i)) & 0xff); + } + fwrite(bytes, 1, 8, out); +} + +int Fail(const char* message) { + fprintf(stderr, "zstd_compress: %s\n", message); + return 1; +} + +} // namespace + +int main(int argc, char** argv) { + if (argc != 4 || strcmp(argv[1], "--icu") != 0) { + fprintf(stderr, "usage: zstd_compress --icu \n"); + return 1; + } + + FILE* in = fopen(argv[2], "rb"); + if (in == nullptr) { + return Fail("open input"); + } + if (fseek(in, 0, SEEK_END) != 0) { + return Fail("seek"); + } + long raw_long = ftell(in); + if (raw_long < 0) { + return Fail("ftell"); + } + if (fseek(in, 0, SEEK_SET) != 0) { + return Fail("rewind"); + } + size_t raw_size = static_cast(raw_long); + uint8_t* raw = static_cast(malloc(raw_size)); + if (raw == nullptr || fread(raw, 1, raw_size, in) != raw_size) { + return Fail("read"); + } + fclose(in); + + size_t bound = ZSTD_compressBound(raw_size); + uint8_t* compressed = static_cast(malloc(bound)); + if (compressed == nullptr) { + return Fail("alloc"); + } + size_t compressed_size = + ZSTD_compress(compressed, bound, raw, raw_size, 19); + if (ZSTD_isError(compressed_size)) { + return Fail(ZSTD_getErrorName(compressed_size)); + } + + FILE* out = fopen(argv[3], "wb"); + if (out == nullptr) { + return Fail("open output"); + } + fwrite("ICUZ", 1, 4, out); + WriteU64LE(out, raw_size); + WriteU64LE(out, compressed_size); + if (fwrite(compressed, 1, compressed_size, out) != compressed_size) { + return Fail("write"); + } + // genccode emits the file as 32-bit words and drops a short tail. + size_t total = 20 + compressed_size; + while (total % 16 != 0) { + fputc(0, out); + total++; + } + fclose(out); + fprintf(stderr, + "zstd_compress: %zu -> %zu bytes\n", + raw_size, + compressed_size); + free(raw); + free(compressed); + return 0; +} From b094ea9e2d30ab4e037e8237fd2deb639504cd81 Mon Sep 17 00:00:00 2001 From: Yagiz Nizipli Date: Mon, 21 Sep 2026 16:20:23 -0400 Subject: [PATCH 2/7] src: fix lint on compressed startup blobs Wrap lines past 80 columns and store the zstd frame size in uint64_t. Signed-off-by: Yagiz Nizipli --- src/node_snapshotable.cc | 5 +++-- src/zstd_blob.cc | 2 +- tools/js2c.cc | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/node_snapshotable.cc b/src/node_snapshotable.cc index da175a83e75..9f095f8720b 100644 --- a/src/node_snapshotable.cc +++ b/src/node_snapshotable.cc @@ -792,8 +792,9 @@ void FormatBlob(std::ostream& ss, while (raw.size() % 16 != 0) { raw.push_back(0); } - append_bytes(reinterpret_cast(data->v8_snapshot_blob_data.data), - snapshot_size); + const uint8_t* snapshot_bytes = + reinterpret_cast(data->v8_snapshot_blob_data.data); + append_bytes(snapshot_bytes, snapshot_size); for (const auto& item : data->code_cache) { append_bytes(item.data.data, item.data.length); } diff --git a/src/zstd_blob.cc b/src/zstd_blob.cc index 29ad46f23d0..627fee644c6 100644 --- a/src/zstd_blob.cc +++ b/src/zstd_blob.cc @@ -13,7 +13,7 @@ namespace node { uint8_t* ZstdDecompressAligned(const uint8_t* src, size_t src_size, size_t* raw_size) { - unsigned long long content = ZSTD_getFrameContentSize(src, src_size); + uint64_t content = ZSTD_getFrameContentSize(src, src_size); if (content == ZSTD_CONTENTSIZE_ERROR || content == ZSTD_CONTENTSIZE_UNKNOWN || content == 0) { return nullptr; diff --git a/tools/js2c.cc b/tools/js2c.cc index e5bc0f50f9c..692a2babe2f 100644 --- a/tools/js2c.cc +++ b/tools/js2c.cc @@ -940,7 +940,8 @@ Fragment EmitCompressedBuiltins() { out += " cursor++;\n"; out += " }\n"; out += " }\n"; - out += " CHECK_EQ(static_cast(sizeof(slots) / sizeof(slots[0])),\n"; + out += " CHECK_EQ(static_cast(sizeof(slots) /\n"; + out += " sizeof(slots[0])),\n"; out += " static_cast(count));\n"; } out += " });\n"; From 8afd2ef63523c45c760dc22aec07086b3cb3e866 Mon Sep 17 00:00:00 2001 From: Yagiz Nizipli Date: Tue, 22 Sep 2026 14:56:12 -0400 Subject: [PATCH 3/7] build: keep addon exports in the macOS binary -Wl,-dead_strip on the Release executable removed napi_* and other globals that nothing in the executable calls. Addons resolve those symbols from the host, so doc-kit's lightningcss addon called a null pointer inside napi_register_module_v1 and macOS CI died with SIGSEGV while generating docs. -x and -S stay; they only drop local and debug symbols. Signed-off-by: Yagiz Nizipli Assisted-by: Grok --- node.gyp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/node.gyp b/node.gyp index 0e095fb4d07..2540cc3c2f4 100644 --- a/node.gyp +++ b/node.gyp @@ -661,16 +661,16 @@ 'WARNING_CFLAGS': [ '-Werror' ], }, }], - # The Release executable force-loads several static libraries, so - # unreferenced objects and the symbol table dominate the file. - # -dead_strip drops unreferenced .o files, -x drops local symbols, - # and -S drops STABS. Debug keeps symbols. + # The Release executable's local symbol table and STABS dominate + # LINKEDIT. -x drops local symbols and -S drops STABS. Debug keeps + # them. Do not pass -dead_strip: N-API and libuv symbols are reached + # only from addons loaded at runtime, and dead-stripping removes + # those exports from the executable. ['OS=="mac" or OS=="ios"', { 'configurations': { 'Release': { 'xcode_settings': { 'OTHER_LDFLAGS': [ - '-Wl,-dead_strip', '-Wl,-x', '-Wl,-S', ], From 465e71b1dcd499e1978878f97ebb06dd97e2930d Mon Sep 17 00:00:00 2001 From: Yagiz Nizipli Date: Tue, 22 Sep 2026 14:56:13 -0400 Subject: [PATCH 4/7] src: keep embedded ICU data file-backed udata_setCommonData() stores the package address and later lookups return pointers into that storage, so the decompressed buffer cannot be freed. Inflating the full ICU data file at startup turned every page into private dirty memory. Map the original data from the executable again so unused pages stay demand-paged, clean, and shareable across processes. Builtin sources and the startup snapshot remain compressed. Signed-off-by: Yagiz Nizipli Assisted-by: Grok --- deps/zstd/zstd.gyp | 8 --- node.gypi | 6 +- src/node_i18n.cc | 46 +-------------- tools/icu/icu-generic.gyp | 116 +++++--------------------------------- tools/zstd_compress.cc | 95 ------------------------------- 5 files changed, 18 insertions(+), 253 deletions(-) delete mode 100644 tools/zstd_compress.cc diff --git a/deps/zstd/zstd.gyp b/deps/zstd/zstd.gyp index 9e82086ca3c..d06e903442e 100644 --- a/deps/zstd/zstd.gyp +++ b/deps/zstd/zstd.gyp @@ -103,13 +103,5 @@ ], 'toolsets': ['host', 'target'], }, - { - 'target_name': 'zstd_compress', - 'type': 'executable', - 'toolsets': ['host'], - 'dependencies': ['zstd#host'], - 'include_dirs': ['lib'], - 'sources': ['../../tools/zstd_compress.cc'], - }, ] } diff --git a/node.gypi b/node.gypi index 6119a869661..b382784e610 100644 --- a/node.gypi +++ b/node.gypi @@ -135,9 +135,6 @@ '<(icu_gyp_path):icuuc', ], 'conditions': [ - [ 'icu_system!="true"', { - 'defines': [ 'NODE_HAVE_EMBEDDED_ICU_ZSTD=1' ], - }], [ 'icu_small=="true"', { 'defines': [ 'NODE_HAVE_SMALL_ICU=1' ], 'conditions': [ @@ -147,8 +144,7 @@ ], }], ], - }], - ], + }]], }], [ 'node_use_bundled_v8=="true" and \ node_enable_v8_vtunejit=="true" and (target_arch=="x64" or \ diff --git a/src/node_i18n.cc b/src/node_i18n.cc index b110a6883c9..e743941734a 100644 --- a/src/node_i18n.cc +++ b/src/node_i18n.cc @@ -43,10 +43,6 @@ #include "node_i18n.h" #include "node_external_reference.h" #include "simdutf.h" -#include "zstd_blob.h" - -#include -#include #if defined(NODE_HAVE_I18N_SUPPORT) @@ -72,7 +68,7 @@ #include #include "nbytes.h" -#if defined(NODE_HAVE_SMALL_ICU) || defined(NODE_HAVE_EMBEDDED_ICU_ZSTD) +#ifdef NODE_HAVE_SMALL_ICU #include /* if this is defined, we have a 'secondary' entry point. @@ -89,38 +85,6 @@ extern "C" const char U_DATA_API SMALL_ICUDATA_ENTRY_POINT[]; #endif -#ifdef NODE_HAVE_EMBEDDED_ICU_ZSTD -extern "C" const uint8_t node_icu_zstd_dat[]; - -static uint64_t ReadU64LE(const uint8_t* bytes) { - uint64_t value = 0; - for (int i = 0; i < 8; i++) { - value |= static_cast(bytes[i]) << (8 * i); - } - return value; -} - -// Decompress the embedded ICU data file. The returned pointer is aligned and -// lives for the process lifetime. nullptr on failure, with `error` set. -static uint8_t* DecompressEmbeddedICU(std::string* error) { - const uint8_t* bytes = node_icu_zstd_dat; - if (memcmp(bytes, "ICUZ", 4) != 0) { - *error = "embedded ICU data header is invalid"; - return nullptr; - } - uint64_t raw_size = ReadU64LE(bytes + 4); - uint64_t compressed_size = ReadU64LE(bytes + 12); - size_t got = 0; - uint8_t* data = node::ZstdDecompressAligned( - bytes + 20, static_cast(compressed_size), &got); - if (data == nullptr || got != raw_size) { - *error = "failed to decompress embedded ICU data"; - return nullptr; - } - return data; -} -#endif // NODE_HAVE_EMBEDDED_ICU_ZSTD - namespace node { using v8::Context; @@ -591,13 +555,7 @@ ConverterObject::ConverterObject( bool InitializeICUDirectory(const std::string& path, std::string* error) { UErrorCode status = U_ZERO_ERROR; if (path.empty()) { -#ifdef NODE_HAVE_EMBEDDED_ICU_ZSTD - static uint8_t* icu_data = DecompressEmbeddedICU(error); - if (icu_data == nullptr) { - return false; - } - udata_setCommonData(icu_data, &status); -#elif defined(NODE_HAVE_SMALL_ICU) +#ifdef NODE_HAVE_SMALL_ICU // install the 'small' data. udata_setCommonData(&SMALL_ICUDATA_ENTRY_POINT, &status); #else // !NODE_HAVE_SMALL_ICU diff --git a/tools/icu/icu-generic.gyp b/tools/icu/icu-generic.gyp index 521e0f3a499..c4e8c6fbb9f 100644 --- a/tools/icu/icu-generic.gyp +++ b/tools/icu/icu-generic.gyp @@ -138,32 +138,14 @@ [ 'icu_small == "false"', { # and OS=win # full data - just build the full data file, then we are done. 'sources': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], - 'dependencies': [ - 'genccode#host', - 'icustubdata', - '../../deps/zstd/zstd.gyp:zstd_compress#host', - ], - 'export_dependent_settings': [ 'icustubdata' ], + 'dependencies': [ 'genccode#host' ], 'conditions': [ [ 'clang==1', { 'actions': [ - { - 'action_name': 'icu_zstd', - 'msvs_quote_cmd': 0, - 'inputs': [ - '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', - '<(icu_data_in)', - ], - 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], - 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', - '--icu', - '<(icu_data_in)', - '<@(_outputs)' ], - }, { 'action_name': 'icudata', 'msvs_quote_cmd': 0, - 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], + 'inputs': [ '<(icu_data_in)' ], 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], # on Windows, we can go directly to .obj file (-o) option. # for Clang use "-c <(target_arch)" option @@ -172,30 +154,16 @@ '-c', '<(target_arch)', '-d', '<(SHARED_INTERMEDIATE_DIR)', '-n', 'icudata', - '-e', 'node_icu_zstd', - '-f', 'icudt<(icu_ver_major)<(icu_endianness)_dat', + '-e', 'icudt<(icu_ver_major)', '<@(_inputs)' ], }, ], }, { 'actions': [ - { - 'action_name': 'icu_zstd', - 'msvs_quote_cmd': 0, - 'inputs': [ - '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', - '<(icu_data_in)', - ], - 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], - 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', - '--icu', - '<(icu_data_in)', - '<@(_outputs)' ], - }, { 'action_name': 'icudata', 'msvs_quote_cmd': 0, - 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], + 'inputs': [ '<(icu_data_in)' ], 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], # on Windows, we can go directly to .obj file (-o) option. # for MSVC do not use "-c <(target_arch)" option @@ -203,8 +171,7 @@ '<@(icu_asm_opts)', # -o '-d', '<(SHARED_INTERMEDIATE_DIR)', '-n', 'icudata', - '-e', 'node_icu_zstd', - '-f', 'icudt<(icu_ver_major)<(icu_endianness)_dat', + '-e', 'icudt<(icu_ver_major)', '<@(_inputs)' ], }, ], @@ -213,8 +180,7 @@ }, { # icu_small == TRUE and OS == win # link against stub data primarily # then, use icupkg and genccode to rebuild data - 'dependencies': [ 'icustubdata', 'genccode#host', 'icupkg#host', 'genrb#host', 'iculslocs#host', - '../../deps/zstd/zstd.gyp:zstd_compress#host' ], + 'dependencies': [ 'icustubdata', 'genccode#host', 'icupkg#host', 'genrb#host', 'iculslocs#host' ], 'export_dependent_settings': [ 'icustubdata' ], 'actions': [ { @@ -234,32 +200,18 @@ '-v', '-L', '<(icu_locales)'], }, - { - 'action_name': 'icu_zstd', - 'msvs_quote_cmd': 0, - 'inputs': [ - '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', - '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat', - ], - 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], - 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', - '--icu', - '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat', - '<@(_outputs)' ], - }, { # build final .dat -> .obj 'action_name': 'genccode', 'msvs_quote_cmd': 0, - 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], + 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat' ], 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], 'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)', '<@(icu_asm_opts)', # -o '-c', '<(target_arch)', '-d', '<(SHARED_INTERMEDIATE_DIR)/', '-n', 'icudata', - '-e', 'node_icu_zstd', - '-f', 'icudt<(icu_ver_major)<(icu_endianness)_dat', + '-e', 'icusmdt<(icu_ver_major)', '<@(_inputs)' ], }, ], @@ -269,19 +221,9 @@ }, { # OS != win 'conditions': [ [ 'icu_small == "false"', { - # full data - no trim needed. The bytes embedded below are - # zstd-compressed; node decompresses them before udata_setCommonData. - # icustubdata satisfies ICU's icudtXX_dat link reference. + # full data - no trim needed 'sources': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)_dat.<(icu_asm_ext)' ], - 'dependencies': [ - 'genccode#host', - 'icupkg#host', - 'icu_implementation#host', - 'icu_uconfig', - 'icustubdata', - '../../deps/zstd/zstd.gyp:zstd_compress#host', - ], - 'export_dependent_settings': [ 'icustubdata' ], + 'dependencies': [ 'genccode#host', 'icupkg#host', 'icu_implementation#host', 'icu_uconfig' ], 'include_dirs': [ '<(icu_path)/source/common', ], @@ -308,26 +250,12 @@ ], }, { - 'action_name': 'icu_zstd', - 'inputs': [ - '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', - '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major).dat', - ], - 'outputs':[ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], - 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', - '--icu', - '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major).dat', - '<@(_outputs)' ], - }, - { - # convert compressed ICU data to .c, or .S, etc. - # -e names the symbol node_icu_zstd_dat so ICU does not - # treat the compressed bytes as its data entry point. + # convert full ICU data file to .c, or .S, etc. 'action_name': 'icudata', - 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], + 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major).dat' ], 'outputs':[ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)_dat.<(icu_asm_ext)' ], 'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)', - '-e', 'node_icu_zstd', + '-e', 'icudt<(icu_ver_major)', '-d', '<(SHARED_INTERMEDIATE_DIR)', '<@(icu_asm_opts)', '-f', 'icudt<(icu_ver_major)_dat', @@ -338,8 +266,7 @@ # link against stub data (as primary data) # then, use icupkg and genccode to rebuild small data 'dependencies': [ 'icustubdata', 'genccode#host', 'icupkg#host', 'genrb#host', 'iculslocs#host', - 'icu_implementation', 'icu_uconfig', - '../../deps/zstd/zstd.gyp:zstd_compress#host' ], + 'icu_implementation', 'icu_uconfig' ], 'export_dependent_settings': [ 'icustubdata' ], 'actions': [ { @@ -367,26 +294,13 @@ '<@(_inputs)', '<@(_outputs)', ], - }, { - 'action_name': 'icu_zstd', - 'inputs': [ - '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', - '<(SHARED_INTERMEDIATE_DIR)/icutmp/icusmdt<(icu_ver_major).dat', - ], - 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], - 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', - '--icu', - '<(SHARED_INTERMEDIATE_DIR)/icutmp/icusmdt<(icu_ver_major).dat', - '<@(_outputs)' ], }, { # For icu-small, always use .c, don't try to use .S, etc. 'action_name': 'genccode', - 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], + 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icutmp/icusmdt<(icu_ver_major).dat' ], 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icusmdt<(icu_ver_major)_dat.<(icu_asm_ext)' ], 'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)', '<@(icu_asm_opts)', - '-e', 'node_icu_zstd', - '-f', 'icusmdt<(icu_ver_major)_dat', '-d', '<(SHARED_INTERMEDIATE_DIR)', '<@(_inputs)' ], }, diff --git a/tools/zstd_compress.cc b/tools/zstd_compress.cc deleted file mode 100644 index 13a9f23568d..00000000000 --- a/tools/zstd_compress.cc +++ /dev/null @@ -1,95 +0,0 @@ -// Host tool: compress a file with zstd for embedding in the node binary. -// -// zstd_compress --icu -// -// `--icu` prefixes a little-endian header: -// magic "ICUZ" | uint64 raw_size | uint64 compressed_size | zstd frame - -#include -#include -#include - -#include - -#include "zstd.h" - -namespace { - -void WriteU64LE(FILE* out, uint64_t value) { - uint8_t bytes[8]; - for (int i = 0; i < 8; i++) { - bytes[i] = static_cast((value >> (8 * i)) & 0xff); - } - fwrite(bytes, 1, 8, out); -} - -int Fail(const char* message) { - fprintf(stderr, "zstd_compress: %s\n", message); - return 1; -} - -} // namespace - -int main(int argc, char** argv) { - if (argc != 4 || strcmp(argv[1], "--icu") != 0) { - fprintf(stderr, "usage: zstd_compress --icu \n"); - return 1; - } - - FILE* in = fopen(argv[2], "rb"); - if (in == nullptr) { - return Fail("open input"); - } - if (fseek(in, 0, SEEK_END) != 0) { - return Fail("seek"); - } - long raw_long = ftell(in); - if (raw_long < 0) { - return Fail("ftell"); - } - if (fseek(in, 0, SEEK_SET) != 0) { - return Fail("rewind"); - } - size_t raw_size = static_cast(raw_long); - uint8_t* raw = static_cast(malloc(raw_size)); - if (raw == nullptr || fread(raw, 1, raw_size, in) != raw_size) { - return Fail("read"); - } - fclose(in); - - size_t bound = ZSTD_compressBound(raw_size); - uint8_t* compressed = static_cast(malloc(bound)); - if (compressed == nullptr) { - return Fail("alloc"); - } - size_t compressed_size = - ZSTD_compress(compressed, bound, raw, raw_size, 19); - if (ZSTD_isError(compressed_size)) { - return Fail(ZSTD_getErrorName(compressed_size)); - } - - FILE* out = fopen(argv[3], "wb"); - if (out == nullptr) { - return Fail("open output"); - } - fwrite("ICUZ", 1, 4, out); - WriteU64LE(out, raw_size); - WriteU64LE(out, compressed_size); - if (fwrite(compressed, 1, compressed_size, out) != compressed_size) { - return Fail("write"); - } - // genccode emits the file as 32-bit words and drops a short tail. - size_t total = 20 + compressed_size; - while (total % 16 != 0) { - fputc(0, out); - total++; - } - fclose(out); - fprintf(stderr, - "zstd_compress: %zu -> %zu bytes\n", - raw_size, - compressed_size); - free(raw); - free(compressed); - return 0; -} From 9ce7fe175b6cb2a7d7e15fa5ddcfc667d896313a Mon Sep 17 00:00:00 2001 From: Yagiz Nizipli Date: Tue, 22 Sep 2026 15:29:29 -0400 Subject: [PATCH 5/7] src: compress ICU data into a shared cache file The ICU data file is a zstd frame in the binary. The first process inflates it into a file under the temp directory and maps that file read-only. Later processes map the same file, so the pages stay clean, demand-paged, and shared instead of a private dirty copy. If the temp directory cannot be written, startup keeps the private buffer so ICU still works. Signed-off-by: Yagiz Nizipli Assisted-by: Grok --- deps/zstd/zstd.gyp | 8 + node.gyp | 3 + src/node_i18n.cc | 302 +++++++++++++++++++++++++++++++++++++- tools/embed_sha256.h | 109 ++++++++++++++ tools/icu/icu-generic.gyp | 117 +++++++++++++-- tools/zstd_compress.cc | 100 +++++++++++++ 6 files changed, 619 insertions(+), 20 deletions(-) create mode 100644 tools/embed_sha256.h create mode 100644 tools/zstd_compress.cc diff --git a/deps/zstd/zstd.gyp b/deps/zstd/zstd.gyp index d06e903442e..9e82086ca3c 100644 --- a/deps/zstd/zstd.gyp +++ b/deps/zstd/zstd.gyp @@ -103,5 +103,13 @@ ], 'toolsets': ['host', 'target'], }, + { + 'target_name': 'zstd_compress', + 'type': 'executable', + 'toolsets': ['host'], + 'dependencies': ['zstd#host'], + 'include_dirs': ['lib'], + 'sources': ['../../tools/zstd_compress.cc'], + }, ] } diff --git a/node.gyp b/node.gyp index 2540cc3c2f4..8fbe82a0969 100644 --- a/node.gyp +++ b/node.gyp @@ -920,6 +920,9 @@ 'msvs_disabled_warnings!': [4244], 'conditions': [ + [ 'icu_system!="true" and v8_enable_i18n_support==1', { + 'defines': [ 'NODE_HAVE_EMBEDDED_ICU_ZSTD=1' ], + }], [ 'openssl_default_cipher_list!=""', { 'defines': [ 'NODE_OPENSSL_DEFAULT_CIPHER_LIST="<(openssl_default_cipher_list)"' diff --git a/src/node_i18n.cc b/src/node_i18n.cc index e743941734a..ecea577b0a1 100644 --- a/src/node_i18n.cc +++ b/src/node_i18n.cc @@ -33,9 +33,11 @@ * udata_setCommonData(SMALL_ICUDATA_ENTRY_POINT,...) * to load up the english+root data. * - * - when NOT in NODE_HAVE_SMALL_ICU mode, ICU is linked directly with its full - * data. All of the variables and command line options for changing data at - * runtime are disabled, as they wouldn't fully override the internal data. + * - Full and small ICU data are stored as a zstd frame in the binary. + * The first process inflates that into a file under the temp directory + * and maps it read-only. Later processes map the same file, so the + * pages stay clean, demand-paged, and shared. --icu-data-dir still wins + * when it is set. * See: http://bugs.icu-project.org/trac/ticket/10924 */ @@ -68,8 +70,28 @@ #include #include "nbytes.h" -#ifdef NODE_HAVE_SMALL_ICU +#if defined(NODE_HAVE_SMALL_ICU) || defined(NODE_HAVE_EMBEDDED_ICU_ZSTD) #include +#endif + +#ifdef NODE_HAVE_EMBEDDED_ICU_ZSTD +#include "uv.h" +#include "zstd.h" + +#include "../tools/embed_sha256.h" + +#include +#include + +#ifdef _WIN32 +#include +#include +#else +#include +#endif +#endif + +#ifdef NODE_HAVE_SMALL_ICU /* if this is defined, we have a 'secondary' entry point. compare following to utypes.h defs for U_ICUDATA_ENTRY_POINT */ @@ -85,6 +107,270 @@ extern "C" const char U_DATA_API SMALL_ICUDATA_ENTRY_POINT[]; #endif +#ifdef NODE_HAVE_EMBEDDED_ICU_ZSTD +extern "C" const uint8_t node_icu_zstd_dat[]; + +namespace { + +constexpr size_t kIcuHeaderSize = 52; +constexpr uint64_t kMaxIcuBytes = 256 * 1024 * 1024; + +size_t Align16(size_t size) { + return (size + 15u) & ~size_t{15}; +} + +// Anonymous mapping, not the malloc heap. munmap actually drops the +// decompress buffer once the cache file is mapped. +uint8_t* AllocRaw(size_t size) { +#ifdef _WIN32 + return static_cast( + VirtualAlloc(nullptr, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE)); +#else +#if !defined(MAP_ANON) && defined(MAP_ANONYMOUS) +#define MAP_ANON MAP_ANONYMOUS +#endif + void* ptr = mmap(nullptr, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON, -1, 0); + if (ptr == MAP_FAILED) { + return nullptr; + } + return static_cast(ptr); +#endif +} + +void FreeRaw(uint8_t* ptr, size_t size) { + if (ptr == nullptr) { + return; + } +#ifdef _WIN32 + VirtualFree(ptr, 0, MEM_RELEASE); +#else + munmap(ptr, size); +#endif +} + +uint64_t ReadU64LE(const uint8_t* bytes) { + uint64_t value = 0; + for (int i = 0; i < 8; i++) { + value |= static_cast(bytes[i]) << (8 * i); + } + return value; +} + +void AppendHex(std::string* out, const uint8_t hash[32]) { + static const char kHex[] = "0123456789abcdef"; + for (int i = 0; i < 32; i++) { + out->push_back(kHex[hash[i] >> 4]); + out->push_back(kHex[hash[i] & 0xf]); + } +} + +void CleanupFs(uv_fs_t* req) { + uv_fs_req_cleanup(req); +} + +void CloseFd(uv_file fd) { + uv_fs_t req; + uv_fs_close(nullptr, &req, fd, nullptr); + CleanupFs(&req); +} + +bool CachePath(const uint8_t hash[32], std::string* out) { + char tmp[4096]; + size_t len = sizeof(tmp); + if (uv_os_tmpdir(tmp, &len) != 0) { + return false; + } + std::string path(tmp); + if (path.empty()) { + return false; + } + char tail = path.back(); + if (tail != '/' && tail != '\\') { +#ifdef _WIN32 + path.push_back('\\'); +#else + path.push_back('/'); +#endif + } + path += "node-icu-"; + AppendHex(&path, hash); + path += ".dat"; + *out = path; + return true; +} + +// Map the cache file read-only. Clean file pages stay shared across +// processes of this user and are faulted only when ICU touches them. +uint8_t* MapIfSize(const std::string& path, size_t size) { + uv_fs_t req; + int fd = uv_fs_open(nullptr, &req, path.c_str(), UV_FS_O_RDONLY, 0, nullptr); + CleanupFs(&req); + if (fd < 0) { + return nullptr; + } + int st = uv_fs_fstat(nullptr, &req, fd, nullptr); + uint64_t file_size = st == 0 ? req.statbuf.st_size : 0; + CleanupFs(&req); + if (st != 0 || file_size != size) { + CloseFd(fd); + return nullptr; + } +#ifdef _WIN32 + intptr_t osf = _get_osfhandle(fd); + if (osf == -1) { + CloseFd(fd); + return nullptr; + } + HANDLE mapping = CreateFileMappingW(reinterpret_cast(osf), + nullptr, + PAGE_READONLY, + 0, + 0, + nullptr); + if (mapping == nullptr) { + CloseFd(fd); + return nullptr; + } + void* view = MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, size); + CloseFd(fd); + if (view == nullptr) { + CloseHandle(mapping); + return nullptr; + } + // ICU holds pointers into this view for the process lifetime. + static HANDLE keep_mapping = nullptr; + keep_mapping = mapping; + if (keep_mapping == nullptr) { + return nullptr; + } + return static_cast(view); +#else + void* view = mmap(nullptr, size, PROT_READ, MAP_PRIVATE, fd, 0); + CloseFd(fd); + if (view == MAP_FAILED) { + return nullptr; + } + return static_cast(view); +#endif +} + +bool WriteAll(uv_file fd, const uint8_t* data, size_t size) { + size_t off = 0; + while (off < size) { + size_t remain = size - off; + unsigned int chunk = remain > 0x40000000u + ? 0x40000000u + : static_cast(remain); + uv_buf_t buf = uv_buf_init( + const_cast(reinterpret_cast(data + off)), chunk); + uv_fs_t req; + int n = uv_fs_write(nullptr, &req, fd, &buf, 1, + static_cast(off), nullptr); + CleanupFs(&req); + if (n <= 0) { + return false; + } + off += static_cast(n); + } + return true; +} + +uint8_t* PublishCache(const uint8_t* data, + size_t size, + const uint8_t hash[32]) { + std::string path; + if (!CachePath(hash, &path)) { + return nullptr; + } + uint8_t* existing = MapIfSize(path, size); + if (existing != nullptr) { + return existing; + } + std::string tmp = path + ".tmp." + std::to_string(uv_os_getpid()); + uv_fs_t req; + int fd = uv_fs_open(nullptr, &req, tmp.c_str(), + UV_FS_O_CREAT | UV_FS_O_EXCL | UV_FS_O_WRONLY, 0600, + nullptr); + CleanupFs(&req); + if (fd < 0) { + return MapIfSize(path, size); + } + bool wrote = WriteAll(fd, data, size); + if (wrote) { + int sync = uv_fs_fsync(nullptr, &req, fd, nullptr); + CleanupFs(&req); + wrote = sync == 0; + } + CloseFd(fd); + if (!wrote) { + uv_fs_unlink(nullptr, &req, tmp.c_str(), nullptr); + CleanupFs(&req); + return nullptr; + } + int renamed = uv_fs_rename(nullptr, &req, tmp.c_str(), path.c_str(), nullptr); + CleanupFs(&req); + if (renamed != 0) { + uv_fs_unlink(nullptr, &req, tmp.c_str(), nullptr); + CleanupFs(&req); + } + return MapIfSize(path, size); +} + +uint8_t* LoadEmbeddedICU(std::string* error) { + const uint8_t* bytes = node_icu_zstd_dat; + if (memcmp(bytes, "ICUZ", 4) != 0) { + *error = "embedded ICU data header is invalid"; + return nullptr; + } + uint64_t raw_size = ReadU64LE(bytes + 4); + uint64_t compressed_size = ReadU64LE(bytes + 12); + const uint8_t* expect_hash = bytes + 20; + if (raw_size == 0 || raw_size > kMaxIcuBytes || compressed_size == 0 || + compressed_size > raw_size) { + *error = "embedded ICU data header is invalid"; + return nullptr; + } + std::string path; + if (CachePath(expect_hash, &path)) { + uint8_t* cached = MapIfSize(path, static_cast(raw_size)); + if (cached != nullptr) { + return cached; + } + } + size_t raw = static_cast(raw_size); + size_t alloc = Align16(raw); + uint8_t* data = AllocRaw(alloc); + if (data == nullptr) { + *error = "failed to decompress embedded ICU data"; + return nullptr; + } + size_t got = ZSTD_decompress( + data, raw, bytes + kIcuHeaderSize, static_cast(compressed_size)); + if (ZSTD_isError(got) || got != raw) { + FreeRaw(data, alloc); + *error = "failed to decompress embedded ICU data"; + return nullptr; + } + uint8_t hash[32]; + EmbedSha256(data, got, hash); + if (memcmp(hash, expect_hash, 32) != 0) { + FreeRaw(data, alloc); + *error = "embedded ICU data hash mismatch"; + return nullptr; + } + uint8_t* mapped = PublishCache(data, got, hash); + if (mapped != nullptr) { + FreeRaw(data, alloc); + return mapped; + } + // No writable temp directory. Keep the private buffer so ICU still works. + return data; +} + +} // namespace +#endif // NODE_HAVE_EMBEDDED_ICU_ZSTD + namespace node { using v8::Context; @@ -555,7 +841,13 @@ ConverterObject::ConverterObject( bool InitializeICUDirectory(const std::string& path, std::string* error) { UErrorCode status = U_ZERO_ERROR; if (path.empty()) { -#ifdef NODE_HAVE_SMALL_ICU +#ifdef NODE_HAVE_EMBEDDED_ICU_ZSTD + static uint8_t* icu_data = LoadEmbeddedICU(error); + if (icu_data == nullptr) { + return false; + } + udata_setCommonData(icu_data, &status); +#elif defined(NODE_HAVE_SMALL_ICU) // install the 'small' data. udata_setCommonData(&SMALL_ICUDATA_ENTRY_POINT, &status); #else // !NODE_HAVE_SMALL_ICU diff --git a/tools/embed_sha256.h b/tools/embed_sha256.h new file mode 100644 index 00000000000..ac58d7dde2b --- /dev/null +++ b/tools/embed_sha256.h @@ -0,0 +1,109 @@ +#ifndef TOOLS_EMBED_SHA256_H_ +#define TOOLS_EMBED_SHA256_H_ + +// SHA-256 of an embedded blob. Shared by the host compressor and node so the +// cache filename and the header digest match. + +#include +#include +#include + +inline uint32_t EmbedSha256Rotr(uint32_t x, uint32_t n) { + return (x >> n) | (x << (32 - n)); +} + +inline void EmbedSha256Transform(uint32_t state[8], const uint8_t block[64]) { + static const uint32_t k[64] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, + 0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, + 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, + 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, + 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, + 0x5b9cca4f, 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2}; + uint32_t w[64]; + for (int i = 0; i < 16; i++) { + w[i] = (static_cast(block[i * 4]) << 24) | + (static_cast(block[i * 4 + 1]) << 16) | + (static_cast(block[i * 4 + 2]) << 8) | + static_cast(block[i * 4 + 3]); + } + for (int i = 16; i < 64; i++) { + uint32_t s0 = EmbedSha256Rotr(w[i - 15], 7) ^ + EmbedSha256Rotr(w[i - 15], 18) ^ (w[i - 15] >> 3); + uint32_t s1 = EmbedSha256Rotr(w[i - 2], 17) ^ + EmbedSha256Rotr(w[i - 2], 19) ^ (w[i - 2] >> 10); + w[i] = w[i - 16] + s0 + w[i - 7] + s1; + } + uint32_t a = state[0]; + uint32_t b = state[1]; + uint32_t c = state[2]; + uint32_t d = state[3]; + uint32_t e = state[4]; + uint32_t f = state[5]; + uint32_t g = state[6]; + uint32_t h = state[7]; + for (int i = 0; i < 64; i++) { + uint32_t s1 = EmbedSha256Rotr(e, 6) ^ EmbedSha256Rotr(e, 11) ^ + EmbedSha256Rotr(e, 25); + uint32_t ch = (e & f) ^ (~e & g); + uint32_t t1 = h + s1 + ch + k[i] + w[i]; + uint32_t s0 = EmbedSha256Rotr(a, 2) ^ EmbedSha256Rotr(a, 13) ^ + EmbedSha256Rotr(a, 22); + uint32_t maj = (a & b) ^ (a & c) ^ (b & c); + uint32_t t2 = s0 + maj; + h = g; + g = f; + f = e; + e = d + t1; + d = c; + c = b; + b = a; + a = t1 + t2; + } + state[0] += a; + state[1] += b; + state[2] += c; + state[3] += d; + state[4] += e; + state[5] += f; + state[6] += g; + state[7] += h; +} + +inline void EmbedSha256(const uint8_t* data, size_t len, uint8_t out[32]) { + uint32_t state[8] = {0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, + 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19}; + uint8_t block[64]; + size_t off = 0; + while (len - off >= 64) { + EmbedSha256Transform(state, data + off); + off += 64; + } + size_t rem = len - off; + memcpy(block, data + off, rem); + block[rem++] = 0x80; + if (rem > 56) { + memset(block + rem, 0, 64 - rem); + EmbedSha256Transform(state, block); + rem = 0; + } + memset(block + rem, 0, 56 - rem); + uint64_t bits = static_cast(len) * 8; + for (int i = 0; i < 8; i++) { + block[63 - i] = static_cast(bits >> (8 * i)); + } + EmbedSha256Transform(state, block); + for (int i = 0; i < 8; i++) { + out[i * 4] = static_cast(state[i] >> 24); + out[i * 4 + 1] = static_cast(state[i] >> 16); + out[i * 4 + 2] = static_cast(state[i] >> 8); + out[i * 4 + 3] = static_cast(state[i]); + } +} + +#endif // TOOLS_EMBED_SHA256_H_ diff --git a/tools/icu/icu-generic.gyp b/tools/icu/icu-generic.gyp index c4e8c6fbb9f..ecb2396ee2b 100644 --- a/tools/icu/icu-generic.gyp +++ b/tools/icu/icu-generic.gyp @@ -138,14 +138,32 @@ [ 'icu_small == "false"', { # and OS=win # full data - just build the full data file, then we are done. 'sources': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], - 'dependencies': [ 'genccode#host' ], + 'dependencies': [ + 'genccode#host', + 'icustubdata', + '../../deps/zstd/zstd.gyp:zstd_compress#host', + ], + 'export_dependent_settings': [ 'icustubdata' ], 'conditions': [ [ 'clang==1', { 'actions': [ + { + 'action_name': 'icu_zstd', + 'msvs_quote_cmd': 0, + 'inputs': [ + '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '<(icu_data_in)', + ], + 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], + 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '--icu', + '<(icu_data_in)', + '<@(_outputs)' ], + }, { 'action_name': 'icudata', 'msvs_quote_cmd': 0, - 'inputs': [ '<(icu_data_in)' ], + 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], # on Windows, we can go directly to .obj file (-o) option. # for Clang use "-c <(target_arch)" option @@ -154,16 +172,30 @@ '-c', '<(target_arch)', '-d', '<(SHARED_INTERMEDIATE_DIR)', '-n', 'icudata', - '-e', 'icudt<(icu_ver_major)', + '-e', 'node_icu_zstd', + '-f', 'icudt<(icu_ver_major)<(icu_endianness)_dat', '<@(_inputs)' ], }, ], }, { 'actions': [ + { + 'action_name': 'icu_zstd', + 'msvs_quote_cmd': 0, + 'inputs': [ + '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '<(icu_data_in)', + ], + 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], + 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '--icu', + '<(icu_data_in)', + '<@(_outputs)' ], + }, { 'action_name': 'icudata', 'msvs_quote_cmd': 0, - 'inputs': [ '<(icu_data_in)' ], + 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], # on Windows, we can go directly to .obj file (-o) option. # for MSVC do not use "-c <(target_arch)" option @@ -171,7 +203,8 @@ '<@(icu_asm_opts)', # -o '-d', '<(SHARED_INTERMEDIATE_DIR)', '-n', 'icudata', - '-e', 'icudt<(icu_ver_major)', + '-e', 'node_icu_zstd', + '-f', 'icudt<(icu_ver_major)<(icu_endianness)_dat', '<@(_inputs)' ], }, ], @@ -180,7 +213,8 @@ }, { # icu_small == TRUE and OS == win # link against stub data primarily # then, use icupkg and genccode to rebuild data - 'dependencies': [ 'icustubdata', 'genccode#host', 'icupkg#host', 'genrb#host', 'iculslocs#host' ], + 'dependencies': [ 'icustubdata', 'genccode#host', 'icupkg#host', 'genrb#host', 'iculslocs#host', + '../../deps/zstd/zstd.gyp:zstd_compress#host' ], 'export_dependent_settings': [ 'icustubdata' ], 'actions': [ { @@ -200,18 +234,32 @@ '-v', '-L', '<(icu_locales)'], }, + { + 'action_name': 'icu_zstd', + 'msvs_quote_cmd': 0, + 'inputs': [ + '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat', + ], + 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], + 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '--icu', + '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat', + '<@(_outputs)' ], + }, { # build final .dat -> .obj 'action_name': 'genccode', 'msvs_quote_cmd': 0, - 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat' ], + 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], 'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)', '<@(icu_asm_opts)', # -o '-c', '<(target_arch)', '-d', '<(SHARED_INTERMEDIATE_DIR)/', '-n', 'icudata', - '-e', 'icusmdt<(icu_ver_major)', + '-e', 'node_icu_zstd', + '-f', 'icudt<(icu_ver_major)<(icu_endianness)_dat', '<@(_inputs)' ], }, ], @@ -221,9 +269,20 @@ }, { # OS != win 'conditions': [ [ 'icu_small == "false"', { - # full data - no trim needed + # full data - no trim needed. The bytes embedded below are + # zstd-compressed. Node maps a per-user cache file of the + # decompressed bytes so the pages stay shareable. + # icustubdata satisfies ICU's icudtXX_dat link reference. 'sources': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)_dat.<(icu_asm_ext)' ], - 'dependencies': [ 'genccode#host', 'icupkg#host', 'icu_implementation#host', 'icu_uconfig' ], + 'dependencies': [ + 'genccode#host', + 'icupkg#host', + 'icu_implementation#host', + 'icu_uconfig', + 'icustubdata', + '../../deps/zstd/zstd.gyp:zstd_compress#host', + ], + 'export_dependent_settings': [ 'icustubdata' ], 'include_dirs': [ '<(icu_path)/source/common', ], @@ -250,12 +309,26 @@ ], }, { - # convert full ICU data file to .c, or .S, etc. + 'action_name': 'icu_zstd', + 'inputs': [ + '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major).dat', + ], + 'outputs':[ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], + 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '--icu', + '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major).dat', + '<@(_outputs)' ], + }, + { + # convert compressed ICU data to .c, or .S, etc. + # -e names the symbol node_icu_zstd_dat so ICU does not + # treat the compressed bytes as its data entry point. 'action_name': 'icudata', - 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major).dat' ], + 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], 'outputs':[ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)_dat.<(icu_asm_ext)' ], 'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)', - '-e', 'icudt<(icu_ver_major)', + '-e', 'node_icu_zstd', '-d', '<(SHARED_INTERMEDIATE_DIR)', '<@(icu_asm_opts)', '-f', 'icudt<(icu_ver_major)_dat', @@ -266,7 +339,8 @@ # link against stub data (as primary data) # then, use icupkg and genccode to rebuild small data 'dependencies': [ 'icustubdata', 'genccode#host', 'icupkg#host', 'genrb#host', 'iculslocs#host', - 'icu_implementation', 'icu_uconfig' ], + 'icu_implementation', 'icu_uconfig', + '../../deps/zstd/zstd.gyp:zstd_compress#host' ], 'export_dependent_settings': [ 'icustubdata' ], 'actions': [ { @@ -294,13 +368,26 @@ '<@(_inputs)', '<@(_outputs)', ], + }, { + 'action_name': 'icu_zstd', + 'inputs': [ + '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '<(SHARED_INTERMEDIATE_DIR)/icutmp/icusmdt<(icu_ver_major).dat', + ], + 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], + 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '--icu', + '<(SHARED_INTERMEDIATE_DIR)/icutmp/icusmdt<(icu_ver_major).dat', + '<@(_outputs)' ], }, { # For icu-small, always use .c, don't try to use .S, etc. 'action_name': 'genccode', - 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icutmp/icusmdt<(icu_ver_major).dat' ], + 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icusmdt<(icu_ver_major)_dat.<(icu_asm_ext)' ], 'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)', '<@(icu_asm_opts)', + '-e', 'node_icu_zstd', + '-f', 'icusmdt<(icu_ver_major)_dat', '-d', '<(SHARED_INTERMEDIATE_DIR)', '<@(_inputs)' ], }, diff --git a/tools/zstd_compress.cc b/tools/zstd_compress.cc new file mode 100644 index 00000000000..ab48134e039 --- /dev/null +++ b/tools/zstd_compress.cc @@ -0,0 +1,100 @@ +// Host tool: compress a file with zstd for embedding in the node binary. +// +// zstd_compress --icu +// +// `--icu` prefixes a little-endian header: +// magic "ICUZ" | uint64 raw_size | uint64 compressed_size +// | sha256(raw) | zstd frame + +#include +#include +#include + +#include + +#include "embed_sha256.h" +#include "zstd.h" + +namespace { + +void WriteU64LE(FILE* out, uint64_t value) { + uint8_t bytes[8]; + for (int i = 0; i < 8; i++) { + bytes[i] = static_cast((value >> (8 * i)) & 0xff); + } + fwrite(bytes, 1, 8, out); +} + +int Fail(const char* message) { + fprintf(stderr, "zstd_compress: %s\n", message); + return 1; +} + +} // namespace + +int main(int argc, char** argv) { + if (argc != 4 || strcmp(argv[1], "--icu") != 0) { + fprintf(stderr, "usage: zstd_compress --icu \n"); + return 1; + } + + FILE* in = fopen(argv[2], "rb"); + if (in == nullptr) { + return Fail("open input"); + } + if (fseek(in, 0, SEEK_END) != 0) { + return Fail("seek"); + } + long raw_long = ftell(in); + if (raw_long < 0) { + return Fail("ftell"); + } + if (fseek(in, 0, SEEK_SET) != 0) { + return Fail("rewind"); + } + size_t raw_size = static_cast(raw_long); + uint8_t* raw = static_cast(malloc(raw_size)); + if (raw == nullptr || fread(raw, 1, raw_size, in) != raw_size) { + return Fail("read"); + } + fclose(in); + + size_t bound = ZSTD_compressBound(raw_size); + uint8_t* compressed = static_cast(malloc(bound)); + if (compressed == nullptr) { + return Fail("alloc"); + } + size_t compressed_size = + ZSTD_compress(compressed, bound, raw, raw_size, 19); + if (ZSTD_isError(compressed_size)) { + return Fail(ZSTD_getErrorName(compressed_size)); + } + + FILE* out = fopen(argv[3], "wb"); + if (out == nullptr) { + return Fail("open output"); + } + uint8_t hash[32]; + EmbedSha256(raw, raw_size, hash); + fwrite("ICUZ", 1, 4, out); + WriteU64LE(out, raw_size); + WriteU64LE(out, compressed_size); + if (fwrite(hash, 1, sizeof(hash), out) != sizeof(hash) || + fwrite(compressed, 1, compressed_size, out) != compressed_size) { + return Fail("write"); + } + // genccode emits the file as 32-bit words and drops a short tail. + size_t total = 52 + compressed_size; + while (total % 16 != 0) { + fputc(0, out); + total++; + } + fclose(out); + fprintf(stderr, + "zstd_compress: %zu -> %zu bytes\n", + raw_size, + compressed_size); + free(raw); + free(compressed); + return 0; +} From 8e075c9809d210b8880aba0ca0a087ac8a2d7006 Mon Sep 17 00:00:00 2001 From: Yagiz Nizipli Date: Tue, 22 Sep 2026 16:03:43 -0400 Subject: [PATCH 6/7] src: fix clang-format in ICU cache helpers Signed-off-by: Yagiz Nizipli Assisted-by: Grok --- src/node_i18n.cc | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/src/node_i18n.cc b/src/node_i18n.cc index ecea577b0a1..e111dee2afa 100644 --- a/src/node_i18n.cc +++ b/src/node_i18n.cc @@ -41,7 +41,6 @@ * See: http://bugs.icu-project.org/trac/ticket/10924 */ - #include "node_i18n.h" #include "node_external_reference.h" #include "simdutf.h" @@ -129,8 +128,8 @@ uint8_t* AllocRaw(size_t size) { #if !defined(MAP_ANON) && defined(MAP_ANONYMOUS) #define MAP_ANON MAP_ANONYMOUS #endif - void* ptr = mmap(nullptr, size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANON, -1, 0); + void* ptr = mmap( + nullptr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); if (ptr == MAP_FAILED) { return nullptr; } @@ -222,12 +221,8 @@ uint8_t* MapIfSize(const std::string& path, size_t size) { CloseFd(fd); return nullptr; } - HANDLE mapping = CreateFileMappingW(reinterpret_cast(osf), - nullptr, - PAGE_READONLY, - 0, - 0, - nullptr); + HANDLE mapping = CreateFileMappingW( + reinterpret_cast(osf), nullptr, PAGE_READONLY, 0, 0, nullptr); if (mapping == nullptr) { CloseFd(fd); return nullptr; @@ -259,14 +254,13 @@ bool WriteAll(uv_file fd, const uint8_t* data, size_t size) { size_t off = 0; while (off < size) { size_t remain = size - off; - unsigned int chunk = remain > 0x40000000u - ? 0x40000000u - : static_cast(remain); + unsigned int chunk = + remain > 0x40000000u ? 0x40000000u : static_cast(remain); uv_buf_t buf = uv_buf_init( const_cast(reinterpret_cast(data + off)), chunk); uv_fs_t req; - int n = uv_fs_write(nullptr, &req, fd, &buf, 1, - static_cast(off), nullptr); + int n = uv_fs_write( + nullptr, &req, fd, &buf, 1, static_cast(off), nullptr); CleanupFs(&req); if (n <= 0) { return false; @@ -289,8 +283,11 @@ uint8_t* PublishCache(const uint8_t* data, } std::string tmp = path + ".tmp." + std::to_string(uv_os_getpid()); uv_fs_t req; - int fd = uv_fs_open(nullptr, &req, tmp.c_str(), - UV_FS_O_CREAT | UV_FS_O_EXCL | UV_FS_O_WRONLY, 0600, + int fd = uv_fs_open(nullptr, + &req, + tmp.c_str(), + UV_FS_O_CREAT | UV_FS_O_EXCL | UV_FS_O_WRONLY, + 0600, nullptr); CleanupFs(&req); if (fd < 0) { From 248ac4cd6ce3da99817d0567d9abe419b6faca9a Mon Sep 17 00:00:00 2001 From: Yagiz Nizipli Date: Tue, 22 Sep 2026 17:49:16 -0400 Subject: [PATCH 7/7] src: make compressed ICU opt-in and check the cache Packagers pass --with-icu-compress. The default build keeps the data file mapped from the binary. The compressed blob is named icudt.dat.z and its symbol stays an ICU data name. The cache file is checked against the hash stored in the binary before ICU is pointed at it. A rewritten file is discarded and built again. Signed-off-by: Yagiz Nizipli Assisted-by: Grok --- BUILDING.md | 4 + configure.py | 16 + node.gyp | 2 +- src/node_i18n.cc | 72 +++- tools/icu/icu-generic.gyp | 698 +++++++++++++++++++++++--------------- 5 files changed, 514 insertions(+), 278 deletions(-) diff --git a/BUILDING.md b/BUILDING.md index c32d1889c96..a2240fd5898 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -923,6 +923,10 @@ This is the default option. ./configure --with-intl=full-icu ``` +`--with-icu-compress` stores that data file compressed. The first run +unpacks it into the temp directory and maps the result, so later +processes share those pages. It is off unless a packager opts in. + #### Windows ```powershell diff --git a/configure.py b/configure.py index 8b3332a461f..ef2e255a317 100755 --- a/configure.py +++ b/configure.py @@ -1002,6 +1002,15 @@ 'the icu4c source archive. ' f"v{icu_versions['minimum_icu']}.x or later recommended.") +intl_optgroup.add_argument('--with-icu-compress', + action='store_true', + dest='with_icu_compress', + default=False, + help='Compress bundled ICU data and map a per-user cache file at runtime. ' + 'Off by default. Packagers opt in when a smaller on-disk binary is ' + 'worth the first-launch unpack. Requires --with-intl=full-icu or ' + 'small-icu.') + intl_optgroup.add_argument('--with-icu-default-data-dir', action='store', dest='with_icu_default_data_dir', @@ -2544,11 +2553,15 @@ def icu_download(path): # always set icu_small, node.gyp depends on it being defined. o['variables']['icu_small'] = b(False) o['variables']['icu_system'] = b(False) + # Off unless a packager passes --with-icu-compress. + o['variables']['icu_compress_data'] = b(False) # prevent data override o['defines'] += ['ICU_NO_USER_DATA_OVERRIDE'] with_intl = options.with_intl + if options.with_icu_compress and with_intl not in ('small-icu', 'full-icu'): + error('--with-icu-compress requires --with-intl=full-icu or small-icu') with_icu_source = options.with_icu_source have_icu_path = bool(options.with_icu_path) if have_icu_path and with_intl != 'none': @@ -2603,6 +2616,9 @@ def icu_download(path): o['variables']['icu_gyp_path'] = 'tools/icu/icu-system.gyp' return + if options.with_icu_compress: + o['variables']['icu_compress_data'] = b(True) + # this is just the 'deps' dir. Used for unpacking. icu_parent_path = 'deps' diff --git a/node.gyp b/node.gyp index 8fbe82a0969..50b2e8c48b2 100644 --- a/node.gyp +++ b/node.gyp @@ -920,7 +920,7 @@ 'msvs_disabled_warnings!': [4244], 'conditions': [ - [ 'icu_system!="true" and v8_enable_i18n_support==1', { + [ 'icu_compress_data=="true"', { 'defines': [ 'NODE_HAVE_EMBEDDED_ICU_ZSTD=1' ], }], [ 'openssl_default_cipher_list!=""', { diff --git a/src/node_i18n.cc b/src/node_i18n.cc index e111dee2afa..a79c6798b5d 100644 --- a/src/node_i18n.cc +++ b/src/node_i18n.cc @@ -107,10 +107,20 @@ extern "C" const char U_DATA_API SMALL_ICUDATA_ENTRY_POINT[]; #endif #ifdef NODE_HAVE_EMBEDDED_ICU_ZSTD -extern "C" const uint8_t node_icu_zstd_dat[]; +// genccode appends _dat to -e icudt_dat_zstd. +// The extra macro level expands the version and endianness before paste. +#define NODE_ICU_ZSTD_ENTRY \ + NODE_ICU_ZSTD_CALL(U_ICU_VERSION_MAJOR_NUM, U_ICUDATA_TYPE_LITLETTER) +#define NODE_ICU_ZSTD_CALL(major, letter) NODE_ICU_ZSTD_PASTE(major, letter) +#define NODE_ICU_ZSTD_PASTE(major, letter) icudt##major##letter##_dat_zstd_dat +extern "C" const uint8_t NODE_ICU_ZSTD_ENTRY[]; namespace { +#ifdef _WIN32 +HANDLE icu_file_mapping = nullptr; +#endif + constexpr size_t kIcuHeaderSize = 52; constexpr uint64_t kMaxIcuBytes = 256 * 1024 * 1024; @@ -192,7 +202,11 @@ bool CachePath(const uint8_t hash[32], std::string* out) { path.push_back('/'); #endif } - path += "node-icu-"; + // icudt78l-.dat so the version and endianness stay visible. + path += "icudt"; + path += U_ICU_VERSION_SHORT; + path += U_ICUDATA_TYPE_LETTER; + path += "-"; AppendHex(&path, hash); path += ".dat"; *out = path; @@ -234,11 +248,7 @@ uint8_t* MapIfSize(const std::string& path, size_t size) { return nullptr; } // ICU holds pointers into this view for the process lifetime. - static HANDLE keep_mapping = nullptr; - keep_mapping = mapping; - if (keep_mapping == nullptr) { - return nullptr; - } + icu_file_mapping = mapping; return static_cast(view); #else void* view = mmap(nullptr, size, PROT_READ, MAP_PRIVATE, fd, 0); @@ -270,6 +280,43 @@ bool WriteAll(uv_file fd, const uint8_t* data, size_t size) { return true; } +void UnmapView(uint8_t* view, size_t size) { +#ifdef _WIN32 + UnmapViewOfFile(view); + if (icu_file_mapping != nullptr) { + CloseHandle(icu_file_mapping); + icu_file_mapping = nullptr; + } +#else + munmap(view, size); +#endif +} + +bool DigestMatches(const uint8_t* data, size_t size, const uint8_t expect[32]) { + uint8_t hash[32]; + EmbedSha256(data, size, hash); + return memcmp(hash, expect, 32) == 0; +} + +// Reject a cache file whose bytes do not match the hash in the binary. +// A same-user process can rewrite the file; ICU would not notice. +uint8_t* MapTrusted(const std::string& path, + size_t size, + const uint8_t expect[32]) { + uint8_t* view = MapIfSize(path, size); + if (view == nullptr) { + return nullptr; + } + if (!DigestMatches(view, size, expect)) { + UnmapView(view, size); + uv_fs_t req; + uv_fs_unlink(nullptr, &req, path.c_str(), nullptr); + CleanupFs(&req); + return nullptr; + } + return view; +} + uint8_t* PublishCache(const uint8_t* data, size_t size, const uint8_t hash[32]) { @@ -277,7 +324,7 @@ uint8_t* PublishCache(const uint8_t* data, if (!CachePath(hash, &path)) { return nullptr; } - uint8_t* existing = MapIfSize(path, size); + uint8_t* existing = MapTrusted(path, size, hash); if (existing != nullptr) { return existing; } @@ -291,7 +338,7 @@ uint8_t* PublishCache(const uint8_t* data, nullptr); CleanupFs(&req); if (fd < 0) { - return MapIfSize(path, size); + return MapTrusted(path, size, hash); } bool wrote = WriteAll(fd, data, size); if (wrote) { @@ -311,11 +358,11 @@ uint8_t* PublishCache(const uint8_t* data, uv_fs_unlink(nullptr, &req, tmp.c_str(), nullptr); CleanupFs(&req); } - return MapIfSize(path, size); + return MapTrusted(path, size, hash); } uint8_t* LoadEmbeddedICU(std::string* error) { - const uint8_t* bytes = node_icu_zstd_dat; + const uint8_t* bytes = NODE_ICU_ZSTD_ENTRY; if (memcmp(bytes, "ICUZ", 4) != 0) { *error = "embedded ICU data header is invalid"; return nullptr; @@ -330,7 +377,8 @@ uint8_t* LoadEmbeddedICU(std::string* error) { } std::string path; if (CachePath(expect_hash, &path)) { - uint8_t* cached = MapIfSize(path, static_cast(raw_size)); + uint8_t* cached = + MapTrusted(path, static_cast(raw_size), expect_hash); if (cached != nullptr) { return cached; } diff --git a/tools/icu/icu-generic.gyp b/tools/icu/icu-generic.gyp index ecb2396ee2b..91ff5df3357 100644 --- a/tools/icu/icu-generic.gyp +++ b/tools/icu/icu-generic.gyp @@ -133,273 +133,441 @@ 'type': '<(library)', 'toolsets': [ 'target' ], 'conditions': [ - [ 'OS == "win"', { + [ 'icu_compress_data=="true"', { 'conditions': [ - [ 'icu_small == "false"', { # and OS=win - # full data - just build the full data file, then we are done. - 'sources': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], - 'dependencies': [ - 'genccode#host', - 'icustubdata', - '../../deps/zstd/zstd.gyp:zstd_compress#host', - ], - 'export_dependent_settings': [ 'icustubdata' ], - 'conditions': [ - [ 'clang==1', { - 'actions': [ - { - 'action_name': 'icu_zstd', - 'msvs_quote_cmd': 0, - 'inputs': [ - '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', - '<(icu_data_in)', - ], - 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], - 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', - '--icu', - '<(icu_data_in)', - '<@(_outputs)' ], - }, - { - 'action_name': 'icudata', - 'msvs_quote_cmd': 0, - 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], - 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], - # on Windows, we can go directly to .obj file (-o) option. - # for Clang use "-c <(target_arch)" option - 'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)', - '<@(icu_asm_opts)', # -o - '-c', '<(target_arch)', - '-d', '<(SHARED_INTERMEDIATE_DIR)', - '-n', 'icudata', - '-e', 'node_icu_zstd', - '-f', 'icudt<(icu_ver_major)<(icu_endianness)_dat', - '<@(_inputs)' ], - }, - ], - }, { - 'actions': [ - { - 'action_name': 'icu_zstd', - 'msvs_quote_cmd': 0, - 'inputs': [ - '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', - '<(icu_data_in)', - ], - 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], - 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', - '--icu', - '<(icu_data_in)', - '<@(_outputs)' ], - }, - { - 'action_name': 'icudata', - 'msvs_quote_cmd': 0, - 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], - 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], - # on Windows, we can go directly to .obj file (-o) option. - # for MSVC do not use "-c <(target_arch)" option - 'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)', - '<@(icu_asm_opts)', # -o - '-d', '<(SHARED_INTERMEDIATE_DIR)', - '-n', 'icudata', - '-e', 'node_icu_zstd', - '-f', 'icudt<(icu_ver_major)<(icu_endianness)_dat', - '<@(_inputs)' ], - }, - ], - }] - ], - }, { # icu_small == TRUE and OS == win - # link against stub data primarily - # then, use icupkg and genccode to rebuild data - 'dependencies': [ 'icustubdata', 'genccode#host', 'icupkg#host', 'genrb#host', 'iculslocs#host', - '../../deps/zstd/zstd.gyp:zstd_compress#host' ], - 'export_dependent_settings': [ 'icustubdata' ], - 'actions': [ - { - # trim down ICU - 'action_name': 'icutrim', - 'msvs_quote_cmd': 0, - 'inputs': [ '<(icu_data_in)', 'icu_small.json' ], - 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat' ], - 'action': [ '<(python)', - 'icutrim.py', - '-P', '<(PRODUCT_DIR)/.', # '.' suffix is a workaround against GYP assumptions :( - '-D', '<(icu_data_in)', - '--delete-tmp', - '-T', '<(SHARED_INTERMEDIATE_DIR)/icutmp', - '-F', 'icu_small.json', - '-O', 'icudt<(icu_ver_major)<(icu_endianness).dat', - '-v', - '-L', '<(icu_locales)'], - }, - { - 'action_name': 'icu_zstd', - 'msvs_quote_cmd': 0, - 'inputs': [ - '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', - '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat', - ], - 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], - 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', - '--icu', - '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat', - '<@(_outputs)' ], - }, - { - # build final .dat -> .obj - 'action_name': 'genccode', - 'msvs_quote_cmd': 0, - 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], - 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], - 'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)', - '<@(icu_asm_opts)', # -o - '-c', '<(target_arch)', - '-d', '<(SHARED_INTERMEDIATE_DIR)/', - '-n', 'icudata', - '-e', 'node_icu_zstd', - '-f', 'icudt<(icu_ver_major)<(icu_endianness)_dat', - '<@(_inputs)' ], - }, - ], - # This file contains the small ICU data. - 'sources': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], - } ] ], #end of OS==win and icu_small == true - }, { # OS != win + [ 'OS == "win"', { + 'conditions': [ + [ 'icu_small == "false"', { # and OS=win + # full data - just build the full data file, then we are done. + 'sources': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], + 'dependencies': [ + 'genccode#host', + 'icustubdata', + '../../deps/zstd/zstd.gyp:zstd_compress#host', + ], + 'export_dependent_settings': [ 'icustubdata' ], + 'conditions': [ + [ 'clang==1', { + 'actions': [ + { + 'action_name': 'icu_zstd', + 'msvs_quote_cmd': 0, + 'inputs': [ + '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '<(icu_data_in)', + ], + 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness).dat.z' ], + 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '--icu', + '<(icu_data_in)', + '<@(_outputs)' ], + }, + { + 'action_name': 'icudata', + 'msvs_quote_cmd': 0, + 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness).dat.z' ], + 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], + # on Windows, we can go directly to .obj file (-o) option. + # for Clang use "-c <(target_arch)" option + 'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)', + '<@(icu_asm_opts)', # -o + '-c', '<(target_arch)', + '-d', '<(SHARED_INTERMEDIATE_DIR)', + '-n', 'icudata', + '-e', 'icudt<(icu_ver_major)<(icu_endianness)_dat_zstd', + '-f', 'icudt<(icu_ver_major)<(icu_endianness)_dat', + '<@(_inputs)' ], + }, + ], + }, { + 'actions': [ + { + 'action_name': 'icu_zstd', + 'msvs_quote_cmd': 0, + 'inputs': [ + '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '<(icu_data_in)', + ], + 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness).dat.z' ], + 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '--icu', + '<(icu_data_in)', + '<@(_outputs)' ], + }, + { + 'action_name': 'icudata', + 'msvs_quote_cmd': 0, + 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness).dat.z' ], + 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], + # on Windows, we can go directly to .obj file (-o) option. + # for MSVC do not use "-c <(target_arch)" option + 'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)', + '<@(icu_asm_opts)', # -o + '-d', '<(SHARED_INTERMEDIATE_DIR)', + '-n', 'icudata', + '-e', 'icudt<(icu_ver_major)<(icu_endianness)_dat_zstd', + '-f', 'icudt<(icu_ver_major)<(icu_endianness)_dat', + '<@(_inputs)' ], + }, + ], + }] + ], + }, { # icu_small == TRUE and OS == win + # link against stub data primarily + # then, use icupkg and genccode to rebuild data + 'dependencies': [ 'icustubdata', 'genccode#host', 'icupkg#host', 'genrb#host', 'iculslocs#host', + '../../deps/zstd/zstd.gyp:zstd_compress#host' ], + 'export_dependent_settings': [ 'icustubdata' ], + 'actions': [ + { + # trim down ICU + 'action_name': 'icutrim', + 'msvs_quote_cmd': 0, + 'inputs': [ '<(icu_data_in)', 'icu_small.json' ], + 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat' ], + 'action': [ '<(python)', + 'icutrim.py', + '-P', '<(PRODUCT_DIR)/.', # '.' suffix is a workaround against GYP assumptions :( + '-D', '<(icu_data_in)', + '--delete-tmp', + '-T', '<(SHARED_INTERMEDIATE_DIR)/icutmp', + '-F', 'icu_small.json', + '-O', 'icudt<(icu_ver_major)<(icu_endianness).dat', + '-v', + '-L', '<(icu_locales)'], + }, + { + 'action_name': 'icu_zstd', + 'msvs_quote_cmd': 0, + 'inputs': [ + '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat', + ], + 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness).dat.z' ], + 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '--icu', + '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat', + '<@(_outputs)' ], + }, + { + # build final .dat -> .obj + 'action_name': 'genccode', + 'msvs_quote_cmd': 0, + 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness).dat.z' ], + 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], + 'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)', + '<@(icu_asm_opts)', # -o + '-c', '<(target_arch)', + '-d', '<(SHARED_INTERMEDIATE_DIR)/', + '-n', 'icudata', + '-e', 'icudt<(icu_ver_major)<(icu_endianness)_dat_zstd', + '-f', 'icudt<(icu_ver_major)<(icu_endianness)_dat', + '<@(_inputs)' ], + }, + ], + # This file contains the small ICU data. + 'sources': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], + } ] ], #end of OS==win and icu_small == true + }, { # OS != win + 'conditions': [ + [ 'icu_small == "false"', { + # full data - no trim needed. The bytes embedded below are + # zstd-compressed. Node maps a per-user cache file of the + # decompressed bytes so the pages stay shareable. + # icustubdata satisfies ICU's icudtXX_dat link reference. + 'sources': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)_dat.<(icu_asm_ext)' ], + 'dependencies': [ + 'genccode#host', + 'icupkg#host', + 'icu_implementation#host', + 'icu_uconfig', + 'icustubdata', + '../../deps/zstd/zstd.gyp:zstd_compress#host', + ], + 'export_dependent_settings': [ 'icustubdata' ], + 'include_dirs': [ + '<(icu_path)/source/common', + ], + 'actions': [ + { + # Copy the .dat file, swapping endianness if needed. + 'action_name': 'icupkg', + 'inputs': [ '<(icu_data_in)' ], + 'outputs':[ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness).dat' ], + 'action': [ '<(PRODUCT_DIR)/icupkg<(EXECUTABLE_SUFFIX)', + '-t<(icu_endianness)', + '<@(_inputs)', + '<@(_outputs)', + ], + }, + { + # Keep the version and endianness in the name (icudt78l.dat.z). + 'action_name': 'icu_zstd', + 'inputs': [ + '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness).dat', + ], + 'outputs':[ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness).dat.z' ], + 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '--icu', + '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness).dat', + '<@(_outputs)' ], + }, + { + # genccode appends _dat to -e. The symbol is + # icudt_dat_zstd_dat, not the real ICU entry. + 'action_name': 'icudata', + 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness).dat.z' ], + 'outputs':[ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)_dat.<(icu_asm_ext)' ], + 'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)', + '-e', 'icudt<(icu_ver_major)<(icu_endianness)_dat_zstd', + '-d', '<(SHARED_INTERMEDIATE_DIR)', + '<@(icu_asm_opts)', + '-f', 'icudt<(icu_ver_major)_dat', + '<@(_inputs)' ], + }, + ], # end actions + }, { # icu_small == true ( and OS != win ) + # link against stub data (as primary data) + # then, use icupkg and genccode to rebuild small data + 'dependencies': [ 'icustubdata', 'genccode#host', 'icupkg#host', 'genrb#host', 'iculslocs#host', + 'icu_implementation', 'icu_uconfig', + '../../deps/zstd/zstd.gyp:zstd_compress#host' ], + 'export_dependent_settings': [ 'icustubdata' ], + 'actions': [ + { + # Trim down ICU. + # Note that icupkg is invoked automatically, swapping endianness if needed. + 'action_name': 'icutrim', + 'inputs': [ '<(icu_data_in)', 'icu_small.json' ], + 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat' ], + 'action': [ '<(python)', + 'icutrim.py', + '-P', '<(PRODUCT_DIR)', + '-D', '<(icu_data_in)', + '--delete-tmp', + '-T', '<(SHARED_INTERMEDIATE_DIR)/icutmp', + '-F', 'icu_small.json', + '-O', 'icudt<(icu_ver_major)<(icu_endianness).dat', + '-v', + '-L', '<(icu_locales)'], + }, { + 'action_name': 'icu_zstd', + 'inputs': [ + '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat', + ], + 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness).dat.z' ], + 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '--icu', + '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat', + '<@(_outputs)' ], + }, { + # For icu-small, always use .c, don't try to use .S, etc. + 'action_name': 'genccode', + 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness).dat.z' ], + 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icusmdt<(icu_ver_major)_dat.<(icu_asm_ext)' ], + 'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)', + '<@(icu_asm_opts)', + '-e', 'icudt<(icu_ver_major)<(icu_endianness)_dat_zstd', + '-f', 'icusmdt<(icu_ver_major)_dat', + '-d', '<(SHARED_INTERMEDIATE_DIR)', + '<@(_inputs)' ], + }, + ], + # This file contains the small ICU data + 'sources': [ '<(SHARED_INTERMEDIATE_DIR)/icusmdt<(icu_ver_major)_dat.<(icu_asm_ext)' ], + # for umachine.h + 'include_dirs': [ + '<(icu_path)/source/common', + ], + }]], # end icu_small == true + }]], # end OS != win + }, { 'conditions': [ - [ 'icu_small == "false"', { - # full data - no trim needed. The bytes embedded below are - # zstd-compressed. Node maps a per-user cache file of the - # decompressed bytes so the pages stay shareable. - # icustubdata satisfies ICU's icudtXX_dat link reference. - 'sources': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)_dat.<(icu_asm_ext)' ], - 'dependencies': [ - 'genccode#host', - 'icupkg#host', - 'icu_implementation#host', - 'icu_uconfig', - 'icustubdata', - '../../deps/zstd/zstd.gyp:zstd_compress#host', - ], - 'export_dependent_settings': [ 'icustubdata' ], - 'include_dirs': [ - '<(icu_path)/source/common', - ], - 'actions': [ - { - # Copy the .dat file, swapping endianness if needed. - 'action_name': 'icupkg', - 'inputs': [ '<(icu_data_in)' ], - 'outputs':[ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness).dat' ], - 'action': [ '<(PRODUCT_DIR)/icupkg<(EXECUTABLE_SUFFIX)', - '-t<(icu_endianness)', - '<@(_inputs)', - '<@(_outputs)', - ], - }, - { - # Rename without the endianness marker (icudt64l.dat -> icudt64.dat) - 'action_name': 'copy', - 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness).dat' ], - 'outputs':[ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major).dat' ], - 'action': [ 'cp', - '<@(_inputs)', - '<@(_outputs)', - ], - }, - { - 'action_name': 'icu_zstd', - 'inputs': [ - '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', - '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major).dat', - ], - 'outputs':[ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], - 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', - '--icu', - '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major).dat', - '<@(_outputs)' ], - }, - { - # convert compressed ICU data to .c, or .S, etc. - # -e names the symbol node_icu_zstd_dat so ICU does not - # treat the compressed bytes as its data entry point. - 'action_name': 'icudata', - 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], - 'outputs':[ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)_dat.<(icu_asm_ext)' ], - 'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)', - '-e', 'node_icu_zstd', - '-d', '<(SHARED_INTERMEDIATE_DIR)', - '<@(icu_asm_opts)', - '-f', 'icudt<(icu_ver_major)_dat', - '<@(_inputs)' ], - }, - ], # end actions - }, { # icu_small == true ( and OS != win ) - # link against stub data (as primary data) - # then, use icupkg and genccode to rebuild small data - 'dependencies': [ 'icustubdata', 'genccode#host', 'icupkg#host', 'genrb#host', 'iculslocs#host', - 'icu_implementation', 'icu_uconfig', - '../../deps/zstd/zstd.gyp:zstd_compress#host' ], - 'export_dependent_settings': [ 'icustubdata' ], - 'actions': [ - { - # Trim down ICU. - # Note that icupkg is invoked automatically, swapping endianness if needed. - 'action_name': 'icutrim', - 'inputs': [ '<(icu_data_in)', 'icu_small.json' ], - 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat' ], - 'action': [ '<(python)', - 'icutrim.py', - '-P', '<(PRODUCT_DIR)', - '-D', '<(icu_data_in)', - '--delete-tmp', - '-T', '<(SHARED_INTERMEDIATE_DIR)/icutmp', - '-F', 'icu_small.json', - '-O', 'icudt<(icu_ver_major)<(icu_endianness).dat', - '-v', - '-L', '<(icu_locales)'], - }, { - # rename to get the final entrypoint name right (icudt64l.dat -> icusmdt64.dat) - 'action_name': 'rename', - 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat' ], - 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icutmp/icusmdt<(icu_ver_major).dat' ], - 'action': [ 'cp', - '<@(_inputs)', - '<@(_outputs)', - ], - }, { - 'action_name': 'icu_zstd', - 'inputs': [ - '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', - '<(SHARED_INTERMEDIATE_DIR)/icutmp/icusmdt<(icu_ver_major).dat', - ], - 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], - 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', - '--icu', - '<(SHARED_INTERMEDIATE_DIR)/icutmp/icusmdt<(icu_ver_major).dat', - '<@(_outputs)' ], - }, { - # For icu-small, always use .c, don't try to use .S, etc. - 'action_name': 'genccode', - 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], - 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icusmdt<(icu_ver_major)_dat.<(icu_asm_ext)' ], - 'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)', - '<@(icu_asm_opts)', - '-e', 'node_icu_zstd', - '-f', 'icusmdt<(icu_ver_major)_dat', - '-d', '<(SHARED_INTERMEDIATE_DIR)', - '<@(_inputs)' ], - }, - ], - # This file contains the small ICU data - 'sources': [ '<(SHARED_INTERMEDIATE_DIR)/icusmdt<(icu_ver_major)_dat.<(icu_asm_ext)' ], - # for umachine.h - 'include_dirs': [ - '<(icu_path)/source/common', - ], - }]], # end icu_small == true - }]], # end OS != win + [ 'OS == "win"', { + 'conditions': [ + [ 'icu_small == "false"', { # and OS=win + # full data - just build the full data file, then we are done. + 'sources': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], + 'dependencies': [ 'genccode#host' ], + 'conditions': [ + [ 'clang==1', { + 'actions': [ + { + 'action_name': 'icudata', + 'msvs_quote_cmd': 0, + 'inputs': [ '<(icu_data_in)' ], + 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], + # on Windows, we can go directly to .obj file (-o) option. + # for Clang use "-c <(target_arch)" option + 'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)', + '<@(icu_asm_opts)', # -o + '-c', '<(target_arch)', + '-d', '<(SHARED_INTERMEDIATE_DIR)', + '-n', 'icudata', + '-e', 'icudt<(icu_ver_major)', + '<@(_inputs)' ], + }, + ], + }, { + 'actions': [ + { + 'action_name': 'icudata', + 'msvs_quote_cmd': 0, + 'inputs': [ '<(icu_data_in)' ], + 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], + # on Windows, we can go directly to .obj file (-o) option. + # for MSVC do not use "-c <(target_arch)" option + 'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)', + '<@(icu_asm_opts)', # -o + '-d', '<(SHARED_INTERMEDIATE_DIR)', + '-n', 'icudata', + '-e', 'icudt<(icu_ver_major)', + '<@(_inputs)' ], + }, + ], + }] + ], + }, { # icu_small == TRUE and OS == win + # link against stub data primarily + # then, use icupkg and genccode to rebuild data + 'dependencies': [ 'icustubdata', 'genccode#host', 'icupkg#host', 'genrb#host', 'iculslocs#host' ], + 'export_dependent_settings': [ 'icustubdata' ], + 'actions': [ + { + # trim down ICU + 'action_name': 'icutrim', + 'msvs_quote_cmd': 0, + 'inputs': [ '<(icu_data_in)', 'icu_small.json' ], + 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat' ], + 'action': [ '<(python)', + 'icutrim.py', + '-P', '<(PRODUCT_DIR)/.', # '.' suffix is a workaround against GYP assumptions :( + '-D', '<(icu_data_in)', + '--delete-tmp', + '-T', '<(SHARED_INTERMEDIATE_DIR)/icutmp', + '-F', 'icu_small.json', + '-O', 'icudt<(icu_ver_major)<(icu_endianness).dat', + '-v', + '-L', '<(icu_locales)'], + }, + { + # build final .dat -> .obj + 'action_name': 'genccode', + 'msvs_quote_cmd': 0, + 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat' ], + 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], + 'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)', + '<@(icu_asm_opts)', # -o + '-c', '<(target_arch)', + '-d', '<(SHARED_INTERMEDIATE_DIR)/', + '-n', 'icudata', + '-e', 'icusmdt<(icu_ver_major)', + '<@(_inputs)' ], + }, + ], + # This file contains the small ICU data. + 'sources': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], + } ] ], #end of OS==win and icu_small == true + }, { # OS != win + 'conditions': [ + [ 'icu_small == "false"', { + # full data - no trim needed + 'sources': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)_dat.<(icu_asm_ext)' ], + 'dependencies': [ 'genccode#host', 'icupkg#host', 'icu_implementation#host', 'icu_uconfig' ], + 'include_dirs': [ + '<(icu_path)/source/common', + ], + 'actions': [ + { + # Copy the .dat file, swapping endianness if needed. + 'action_name': 'icupkg', + 'inputs': [ '<(icu_data_in)' ], + 'outputs':[ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness).dat' ], + 'action': [ '<(PRODUCT_DIR)/icupkg<(EXECUTABLE_SUFFIX)', + '-t<(icu_endianness)', + '<@(_inputs)', + '<@(_outputs)', + ], + }, + { + # Rename without the endianness marker (icudt64l.dat -> icudt64.dat) + 'action_name': 'copy', + 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness).dat' ], + 'outputs':[ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major).dat' ], + 'action': [ 'cp', + '<@(_inputs)', + '<@(_outputs)', + ], + }, + { + # convert full ICU data file to .c, or .S, etc. + 'action_name': 'icudata', + 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major).dat' ], + 'outputs':[ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)_dat.<(icu_asm_ext)' ], + 'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)', + '-e', 'icudt<(icu_ver_major)', + '-d', '<(SHARED_INTERMEDIATE_DIR)', + '<@(icu_asm_opts)', + '-f', 'icudt<(icu_ver_major)_dat', + '<@(_inputs)' ], + }, + ], # end actions + }, { # icu_small == true ( and OS != win ) + # link against stub data (as primary data) + # then, use icupkg and genccode to rebuild small data + 'dependencies': [ 'icustubdata', 'genccode#host', 'icupkg#host', 'genrb#host', 'iculslocs#host', + 'icu_implementation', 'icu_uconfig' ], + 'export_dependent_settings': [ 'icustubdata' ], + 'actions': [ + { + # Trim down ICU. + # Note that icupkg is invoked automatically, swapping endianness if needed. + 'action_name': 'icutrim', + 'inputs': [ '<(icu_data_in)', 'icu_small.json' ], + 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat' ], + 'action': [ '<(python)', + 'icutrim.py', + '-P', '<(PRODUCT_DIR)', + '-D', '<(icu_data_in)', + '--delete-tmp', + '-T', '<(SHARED_INTERMEDIATE_DIR)/icutmp', + '-F', 'icu_small.json', + '-O', 'icudt<(icu_ver_major)<(icu_endianness).dat', + '-v', + '-L', '<(icu_locales)'], + }, { + # rename to get the final entrypoint name right (icudt64l.dat -> icusmdt64.dat) + 'action_name': 'rename', + 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat' ], + 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icutmp/icusmdt<(icu_ver_major).dat' ], + 'action': [ 'cp', + '<@(_inputs)', + '<@(_outputs)', + ], + }, { + # For icu-small, always use .c, don't try to use .S, etc. + 'action_name': 'genccode', + 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icutmp/icusmdt<(icu_ver_major).dat' ], + 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icusmdt<(icu_ver_major)_dat.<(icu_asm_ext)' ], + 'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)', + '<@(icu_asm_opts)', + '-d', '<(SHARED_INTERMEDIATE_DIR)', + '<@(_inputs)' ], + }, + ], + # This file contains the small ICU data + 'sources': [ '<(SHARED_INTERMEDIATE_DIR)/icusmdt<(icu_ver_major)_dat.<(icu_asm_ext)' ], + # for umachine.h + 'include_dirs': [ + '<(icu_path)/source/common', + ], + }]], # end icu_small == true + }]], # end OS != win + }], + ], + }, # end icudata # icustubdata is a tiny (~1k) symbol with no ICU data in it. # tools must link against it as they are generating the full data.