From 34ea2c8592a0d011a4b10babfad2921835ebf85b Mon Sep 17 00:00:00 2001 From: Yagiz Nizipli Date: Mon, 21 Sep 2026 16:15:17 -0400 Subject: [PATCH 1/4] src: compress embedded icu, builtins, and snapshot Store the full ICU data file, one-byte builtin sources, and the V8 startup snapshot plus code caches as zstd frames instead of raw bytes. Decompress each once at startup and keep it for the process lifetime. On macOS Release, link the executable with -dead_strip, -x, and -S. A Release arm64 macOS binary dropped from 140 MB to 59 MB. Intl, builtins, the snapshot, and code cache behavior are unchanged. Startup RSS rose by about 40 MB because those blobs are private heap instead of file-backed pages. Signed-off-by: Yagiz Nizipli --- deps/zstd/zstd.gyp | 13 ++- node.gyp | 22 +++++ node.gypi | 6 +- src/node_i18n.cc | 46 +++++++++- src/node_snapshotable.cc | 179 +++++++++++++++++++++----------------- src/node_union_bytes.h | 22 ++++- src/zstd_blob.cc | 47 ++++++++++ src/zstd_blob.h | 18 ++++ tools/icu/icu-generic.gyp | 116 ++++++++++++++++++++---- tools/js2c.cc | 131 ++++++++++++++++++++++++++++ tools/zstd_compress.cc | 95 ++++++++++++++++++++ 11 files changed, 595 insertions(+), 100 deletions(-) create mode 100644 src/zstd_blob.cc create mode 100644 src/zstd_blob.h create mode 100644 tools/zstd_compress.cc diff --git a/deps/zstd/zstd.gyp b/deps/zstd/zstd.gyp index 6018f12831b..9e82086ca3c 100644 --- a/deps/zstd/zstd.gyp +++ b/deps/zstd/zstd.gyp @@ -100,7 +100,16 @@ ], 'sources': [ '<@(zstd_sources)', - ] - } + ], + 'toolsets': ['host', 'target'], + }, + { + 'target_name': 'zstd_compress', + 'type': 'executable', + 'toolsets': ['host'], + 'dependencies': ['zstd#host'], + 'include_dirs': ['lib'], + 'sources': ['../../tools/zstd_compress.cc'], + }, ] } diff --git a/node.gyp b/node.gyp index 8692f42c897..0e095fb4d07 100644 --- a/node.gyp +++ b/node.gyp @@ -185,6 +185,8 @@ 'src/node_watchdog.cc', 'src/node_worker.cc', 'src/node_zlib.cc', + 'src/zstd_blob.cc', + 'src/zstd_blob.h', 'src/path.cc', 'src/permission/fs_permission.cc', 'src/permission/permission.cc', @@ -659,6 +661,23 @@ 'WARNING_CFLAGS': [ '-Werror' ], }, }], + # The Release executable force-loads several static libraries, so + # unreferenced objects and the symbol table dominate the file. + # -dead_strip drops unreferenced .o files, -x drops local symbols, + # and -S drops STABS. Debug keeps symbols. + ['OS=="mac" or OS=="ios"', { + 'configurations': { + 'Release': { + 'xcode_settings': { + 'OTHER_LDFLAGS': [ + '-Wl,-dead_strip', + '-Wl,-x', + '-Wl,-S', + ], + }, + }, + }, + }], ['node_shared=="true" and OS=="win"', { 'dependencies': ['generate_node_def'], 'msvs_settings': { @@ -1671,6 +1690,9 @@ [ 'node_shared_libuv=="false"', { 'dependencies': [ 'deps/uv/uv.gyp:libuv#host' ], }], + [ 'node_shared_zstd=="false"', { + 'dependencies': [ 'deps/zstd/zstd.gyp:zstd#host' ], + }], [ 'OS in "linux mac openharmony"', { 'defines': ['NODE_JS2C_USE_STRING_LITERALS'], }], diff --git a/node.gypi b/node.gypi index b382784e610..6119a869661 100644 --- a/node.gypi +++ b/node.gypi @@ -135,6 +135,9 @@ '<(icu_gyp_path):icuuc', ], 'conditions': [ + [ 'icu_system!="true"', { + 'defines': [ 'NODE_HAVE_EMBEDDED_ICU_ZSTD=1' ], + }], [ 'icu_small=="true"', { 'defines': [ 'NODE_HAVE_SMALL_ICU=1' ], 'conditions': [ @@ -144,7 +147,8 @@ ], }], ], - }]], + }], + ], }], [ 'node_use_bundled_v8=="true" and \ node_enable_v8_vtunejit=="true" and (target_arch=="x64" or \ diff --git a/src/node_i18n.cc b/src/node_i18n.cc index e743941734a..b110a6883c9 100644 --- a/src/node_i18n.cc +++ b/src/node_i18n.cc @@ -43,6 +43,10 @@ #include "node_i18n.h" #include "node_external_reference.h" #include "simdutf.h" +#include "zstd_blob.h" + +#include +#include #if defined(NODE_HAVE_I18N_SUPPORT) @@ -68,7 +72,7 @@ #include #include "nbytes.h" -#ifdef NODE_HAVE_SMALL_ICU +#if defined(NODE_HAVE_SMALL_ICU) || defined(NODE_HAVE_EMBEDDED_ICU_ZSTD) #include /* if this is defined, we have a 'secondary' entry point. @@ -85,6 +89,38 @@ extern "C" const char U_DATA_API SMALL_ICUDATA_ENTRY_POINT[]; #endif +#ifdef NODE_HAVE_EMBEDDED_ICU_ZSTD +extern "C" const uint8_t node_icu_zstd_dat[]; + +static uint64_t ReadU64LE(const uint8_t* bytes) { + uint64_t value = 0; + for (int i = 0; i < 8; i++) { + value |= static_cast(bytes[i]) << (8 * i); + } + return value; +} + +// Decompress the embedded ICU data file. The returned pointer is aligned and +// lives for the process lifetime. nullptr on failure, with `error` set. +static uint8_t* DecompressEmbeddedICU(std::string* error) { + const uint8_t* bytes = node_icu_zstd_dat; + if (memcmp(bytes, "ICUZ", 4) != 0) { + *error = "embedded ICU data header is invalid"; + return nullptr; + } + uint64_t raw_size = ReadU64LE(bytes + 4); + uint64_t compressed_size = ReadU64LE(bytes + 12); + size_t got = 0; + uint8_t* data = node::ZstdDecompressAligned( + bytes + 20, static_cast(compressed_size), &got); + if (data == nullptr || got != raw_size) { + *error = "failed to decompress embedded ICU data"; + return nullptr; + } + return data; +} +#endif // NODE_HAVE_EMBEDDED_ICU_ZSTD + namespace node { using v8::Context; @@ -555,7 +591,13 @@ ConverterObject::ConverterObject( bool InitializeICUDirectory(const std::string& path, std::string* error) { UErrorCode status = U_ZERO_ERROR; if (path.empty()) { -#ifdef NODE_HAVE_SMALL_ICU +#ifdef NODE_HAVE_EMBEDDED_ICU_ZSTD + static uint8_t* icu_data = DecompressEmbeddedICU(error); + if (icu_data == nullptr) { + return false; + } + udata_setCommonData(icu_data, &status); +#elif defined(NODE_HAVE_SMALL_ICU) // install the 'small' data. udata_setCommonData(&SMALL_ICUDATA_ENTRY_POINT, &status); #else // !NODE_HAVE_SMALL_ICU diff --git a/src/node_snapshotable.cc b/src/node_snapshotable.cc index faf95133bf4..da175a83e75 100644 --- a/src/node_snapshotable.cc +++ b/src/node_snapshotable.cc @@ -8,7 +8,6 @@ #include "base_object-inl.h" #include "blob_serializer_deserializer-inl.h" #include "debug_utils-inl.h" -#include "embedded_data.h" #include "encoding_binding.h" #include "env-inl.h" #include "glob/node_glob.h" @@ -30,6 +29,9 @@ #include "node_v8_platform-inl.h" #include "simdjson.h" #include "timers.h" +#include "zstd.h" + +#include #if HAVE_INSPECTOR #include "inspector/worker_inspector.h" // ParentInspectorHandle @@ -737,17 +739,6 @@ SnapshotData::~SnapshotData() { } } -static std::string GetCodeCacheDefName(const std::string& id) { - char buf[64] = {0}; - size_t size = id.size(); - CHECK_LT(size, sizeof(buf)); - for (size_t i = 0; i < size; ++i) { - char ch = id[i]; - buf[i] = (ch == '-' || ch == '/') ? '_' : ch; - } - return std::string(buf) + std::string("_cache_data"); -} - static std::string FormatSize(size_t size) { char buf[64] = {0}; if (size < 1024) { @@ -761,88 +752,81 @@ static std::string FormatSize(size_t size) { return buf; } -template - requires(std::same_as || std::same_as) -void WriteByteVectorLiteral(std::ostream* ss, - const T* vec, - size_t size, - const char* var_name, - bool use_array_literals) { - constexpr bool is_uint8_t = std::is_same_v; - constexpr const char* type_name = is_uint8_t ? "uint8_t" : "char"; - if (!use_array_literals) { - const uint8_t* data = reinterpret_cast(vec); - *ss << "static const " << type_name << " *" << var_name << " = "; - *ss << (is_uint8_t ? R"(reinterpret_cast(")" : "\""); - for (size_t i = 0; i < size; i++) { - const uint8_t ch = data[i]; - *ss << GetOctalCode(ch); - if (i % 64 == 63) { - // Go to a newline every 64 bytes since many text editors have - // problems with very long lines. - *ss << "\"\n\""; - } +void WriteCompressedByteArray(std::ostream* ss, + const uint8_t* bytes, + size_t size, + const char* name) { + *ss << "static const uint8_t " << name << "[] = {\n"; + for (size_t i = 0; i < size; i++) { + char buf[8]; + snprintf(buf, sizeof(buf), "0x%02x,", bytes[i]); + *ss << buf; + if ((i % 16) == 15) { + *ss << '\n'; } - *ss << (is_uint8_t ? "\");\n" : "\";\n"); - } else { - *ss << "static const " << type_name << " " << var_name << "[] = {"; - for (size_t i = 0; i < size; i++) { - *ss << std::to_string(vec[i]) << (i == size - 1 ? '\n' : ','); - if (i % 64 == 63) { - // Print a newline every 64 units and a offset to improve - // readability. - *ss << " // " << (i / 64) << "\n"; - } - } - *ss << "};\n"; } -} - -static void WriteCodeCacheInitializer(std::ostream* ss, - const std::string& id, - size_t size) { - std::string def_name = GetCodeCacheDefName(id); - *ss << " { \"" << id << "\",\n"; - *ss << " {" << def_name << ",\n"; - *ss << " " << size << ",\n"; - *ss << " }\n"; - *ss << " },\n"; + *ss << "\n};\n"; } void FormatBlob(std::ostream& ss, const SnapshotData* data, bool use_array_literals) { + // The snapshot blob and every builtin code cache are one zstd frame. + // use_array_literals only affected the previous per-byte literal style. + (void)use_array_literals; + std::vector raw; + auto append32 = [&](uint32_t value) { + uint8_t bytes[4]; + memcpy(bytes, &value, sizeof(bytes)); + raw.insert(raw.end(), bytes, bytes + sizeof(bytes)); + }; + auto append_bytes = [&](const uint8_t* bytes, size_t size) { + raw.insert(raw.end(), bytes, bytes + size); + }; + const size_t snapshot_size = data->v8_snapshot_blob_data.raw_size; + append32(static_cast(snapshot_size)); + append32(static_cast(data->code_cache.size())); + for (const auto& item : data->code_cache) { + append32(static_cast(item.data.length)); + } + while (raw.size() % 16 != 0) { + raw.push_back(0); + } + append_bytes(reinterpret_cast(data->v8_snapshot_blob_data.data), + snapshot_size); + for (const auto& item : data->code_cache) { + append_bytes(item.data.data, item.data.length); + } + + std::vector compressed(ZSTD_compressBound(raw.size())); + size_t compressed_size = ZSTD_compress( + compressed.data(), compressed.size(), raw.data(), raw.size(), 19); + CHECK(!ZSTD_isError(compressed_size)); + fprintf(stderr, + "snapshot: embedded blob %zu -> %zu bytes\n", + raw.size(), + compressed_size); + ss << R"(#include +#include +#include +#include +#include + #include "env.h" #include "node_snapshot_builder.h" #include "v8.h" +#include "zstd_blob.h" // This file is generated by tools/snapshot. Do not edit. namespace node { )"; - WriteByteVectorLiteral(&ss, - data->v8_snapshot_blob_data.data, - data->v8_snapshot_blob_data.raw_size, - "v8_snapshot_blob_data", - use_array_literals); - - ss << R"(static const int v8_snapshot_blob_size = )" - << data->v8_snapshot_blob_data.raw_size << ";\n"; + WriteCompressedByteArray( + &ss, compressed.data(), compressed_size, "embedded_snapshot_zstd"); - // Windows can't deal with too many large vector initializers. - // Store the data into static arrays first. - for (const auto& item : data->code_cache) { - std::string var_name = GetCodeCacheDefName(item.id); - WriteByteVectorLiteral(&ss, - item.data.data, - item.data.length, - var_name.c_str(), - use_array_literals); - } - - ss << R"(const SnapshotData snapshot_data { + ss << R"(SnapshotData snapshot_data { // -- data_ownership begins -- SnapshotData::DataOwnership::kNotOwned, // -- data_ownership ends -- @@ -851,7 +835,7 @@ namespace node { << R"(, // -- metadata ends -- // -- v8_snapshot_blob_data begins -- - { v8_snapshot_blob_data, v8_snapshot_blob_size }, + { nullptr, 0 }, // -- v8_snapshot_blob_data ends -- // -- v8_snapshot_blob_data_ownership begins -- SnapshotData::DataOwnership::kNotOwned, @@ -869,7 +853,9 @@ namespace node { // -- code_cache begins -- {)"; for (const auto& item : data->code_cache) { - WriteCodeCacheInitializer(&ss, item.id, item.data.length); + ss << " { \"" << item.id << "\",\n"; + ss << " { nullptr, 0 }\n"; + ss << " },\n"; } ss << R"( } @@ -877,6 +863,43 @@ namespace node { }; const SnapshotData* SnapshotBuilder::GetEmbeddedSnapshotData() { + static std::once_flag once; + std::call_once(once, [] { + size_t raw_size = 0; + static uint8_t* storage = ZstdDecompressAligned( + embedded_snapshot_zstd, sizeof(embedded_snapshot_zstd), &raw_size); + CHECK_NE(storage, nullptr); + const uint8_t* cursor = storage; + auto read32 = [&]() -> uint32_t { + CHECK_LE(static_cast(cursor - storage) + 4, raw_size); + uint32_t value = 0; + memcpy(&value, cursor, 4); + cursor += 4; + return value; + }; + uint32_t snapshot_size = read32(); + uint32_t cache_count = read32(); + CHECK_EQ(static_cast(cache_count), snapshot_data.code_cache.size()); + std::vector lengths(cache_count); + for (uint32_t i = 0; i < cache_count; i++) { + lengths[i] = read32(); + } + while (static_cast(cursor - storage) % 16 != 0) { + cursor++; + } + CHECK_LE(static_cast(cursor - storage) + snapshot_size, raw_size); + snapshot_data.v8_snapshot_blob_data.data = + reinterpret_cast(cursor); + snapshot_data.v8_snapshot_blob_data.raw_size = + static_cast(snapshot_size); + cursor += snapshot_size; + for (uint32_t i = 0; i < cache_count; i++) { + CHECK_LE(static_cast(cursor - storage) + lengths[i], raw_size); + snapshot_data.code_cache[i].data.data = cursor; + snapshot_data.code_cache[i].data.length = lengths[i]; + cursor += lengths[i]; + } + }); return &snapshot_data; } } // namespace node diff --git a/src/node_union_bytes.h b/src/node_union_bytes.h index 4a3f67980fd..d6774b5e92c 100644 --- a/src/node_union_bytes.h +++ b/src/node_union_bytes.h @@ -8,6 +8,11 @@ namespace node { +// Set by the embedded-builtin blob (node_javascript.cc) so external-string +// resources can decompress their source the first time V8 reads it. +using BuiltinSourceEnsure = void (*)(); +inline BuiltinSourceEnsure builtin_source_ensure = nullptr; + // An external resource intended to be used with static lifetime. template class StaticExternalByteResource : public Base { @@ -21,9 +26,22 @@ class StaticExternalByteResource : public Base { : data_(data), length_(length), owning_ptr_(owning_ptr) {} const IChar* data() const override { + if (data_ == nullptr && builtin_source_ensure != nullptr) { + builtin_source_ensure(); + } return reinterpret_cast(data_); } - size_t length() const override { return length_; } + size_t length() const override { + if (data_ == nullptr && builtin_source_ensure != nullptr) { + builtin_source_ensure(); + } + return length_; + } + + void set_data(const Char* data, size_t length) { + data_ = data; + length_ = length; + } void Dispose() override { // We ignore Dispose calls from V8, even if we "own" a resource via @@ -37,7 +55,7 @@ class StaticExternalByteResource : public Base { private: const Char* data_; - const size_t length_; + size_t length_; std::shared_ptr owning_ptr_; }; diff --git a/src/zstd_blob.cc b/src/zstd_blob.cc new file mode 100644 index 00000000000..29ad46f23d0 --- /dev/null +++ b/src/zstd_blob.cc @@ -0,0 +1,47 @@ +#include "zstd_blob.h" + +#include + +#include "zstd.h" + +#if defined(_WIN32) +#include +#endif + +namespace node { + +uint8_t* ZstdDecompressAligned(const uint8_t* src, + size_t src_size, + size_t* raw_size) { + unsigned long long content = ZSTD_getFrameContentSize(src, src_size); + if (content == ZSTD_CONTENTSIZE_ERROR || + content == ZSTD_CONTENTSIZE_UNKNOWN || content == 0) { + return nullptr; + } + size_t size = static_cast(content); + size_t alloc = (size + 15u) & ~size_t{15}; + void* buf = nullptr; +#if defined(_WIN32) + buf = _aligned_malloc(alloc, 16); +#else + if (posix_memalign(&buf, 16, alloc) != 0) { + buf = nullptr; + } +#endif + if (buf == nullptr) { + return nullptr; + } + size_t got = ZSTD_decompress(buf, size, src, src_size); + if (ZSTD_isError(got) || got != size) { +#if defined(_WIN32) + _aligned_free(buf); +#else + free(buf); +#endif + return nullptr; + } + *raw_size = size; + return static_cast(buf); +} + +} // namespace node diff --git a/src/zstd_blob.h b/src/zstd_blob.h new file mode 100644 index 00000000000..9ac27bb1d73 --- /dev/null +++ b/src/zstd_blob.h @@ -0,0 +1,18 @@ +#ifndef SRC_ZSTD_BLOB_H_ +#define SRC_ZSTD_BLOB_H_ + +#include +#include + +namespace node { + +// Decompress one zstd frame into a 16-byte-aligned buffer. The returned +// pointer is process-lifetime storage; callers must not free it. `*raw_size` +// is the decompressed size. Returns nullptr on failure. +uint8_t* ZstdDecompressAligned(const uint8_t* src, + size_t src_size, + size_t* raw_size); + +} // namespace node + +#endif // SRC_ZSTD_BLOB_H_ diff --git a/tools/icu/icu-generic.gyp b/tools/icu/icu-generic.gyp index c4e8c6fbb9f..521e0f3a499 100644 --- a/tools/icu/icu-generic.gyp +++ b/tools/icu/icu-generic.gyp @@ -138,14 +138,32 @@ [ 'icu_small == "false"', { # and OS=win # full data - just build the full data file, then we are done. 'sources': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], - 'dependencies': [ 'genccode#host' ], + 'dependencies': [ + 'genccode#host', + 'icustubdata', + '../../deps/zstd/zstd.gyp:zstd_compress#host', + ], + 'export_dependent_settings': [ 'icustubdata' ], 'conditions': [ [ 'clang==1', { 'actions': [ + { + 'action_name': 'icu_zstd', + 'msvs_quote_cmd': 0, + 'inputs': [ + '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '<(icu_data_in)', + ], + 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], + 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '--icu', + '<(icu_data_in)', + '<@(_outputs)' ], + }, { 'action_name': 'icudata', 'msvs_quote_cmd': 0, - 'inputs': [ '<(icu_data_in)' ], + 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], # on Windows, we can go directly to .obj file (-o) option. # for Clang use "-c <(target_arch)" option @@ -154,16 +172,30 @@ '-c', '<(target_arch)', '-d', '<(SHARED_INTERMEDIATE_DIR)', '-n', 'icudata', - '-e', 'icudt<(icu_ver_major)', + '-e', 'node_icu_zstd', + '-f', 'icudt<(icu_ver_major)<(icu_endianness)_dat', '<@(_inputs)' ], }, ], }, { 'actions': [ + { + 'action_name': 'icu_zstd', + 'msvs_quote_cmd': 0, + 'inputs': [ + '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '<(icu_data_in)', + ], + 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], + 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '--icu', + '<(icu_data_in)', + '<@(_outputs)' ], + }, { 'action_name': 'icudata', 'msvs_quote_cmd': 0, - 'inputs': [ '<(icu_data_in)' ], + 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], # on Windows, we can go directly to .obj file (-o) option. # for MSVC do not use "-c <(target_arch)" option @@ -171,7 +203,8 @@ '<@(icu_asm_opts)', # -o '-d', '<(SHARED_INTERMEDIATE_DIR)', '-n', 'icudata', - '-e', 'icudt<(icu_ver_major)', + '-e', 'node_icu_zstd', + '-f', 'icudt<(icu_ver_major)<(icu_endianness)_dat', '<@(_inputs)' ], }, ], @@ -180,7 +213,8 @@ }, { # icu_small == TRUE and OS == win # link against stub data primarily # then, use icupkg and genccode to rebuild data - 'dependencies': [ 'icustubdata', 'genccode#host', 'icupkg#host', 'genrb#host', 'iculslocs#host' ], + 'dependencies': [ 'icustubdata', 'genccode#host', 'icupkg#host', 'genrb#host', 'iculslocs#host', + '../../deps/zstd/zstd.gyp:zstd_compress#host' ], 'export_dependent_settings': [ 'icustubdata' ], 'actions': [ { @@ -200,18 +234,32 @@ '-v', '-L', '<(icu_locales)'], }, + { + 'action_name': 'icu_zstd', + 'msvs_quote_cmd': 0, + 'inputs': [ + '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat', + ], + 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], + 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '--icu', + '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat', + '<@(_outputs)' ], + }, { # build final .dat -> .obj 'action_name': 'genccode', 'msvs_quote_cmd': 0, - 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat' ], + 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], 'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)', '<@(icu_asm_opts)', # -o '-c', '<(target_arch)', '-d', '<(SHARED_INTERMEDIATE_DIR)/', '-n', 'icudata', - '-e', 'icusmdt<(icu_ver_major)', + '-e', 'node_icu_zstd', + '-f', 'icudt<(icu_ver_major)<(icu_endianness)_dat', '<@(_inputs)' ], }, ], @@ -221,9 +269,19 @@ }, { # OS != win 'conditions': [ [ 'icu_small == "false"', { - # full data - no trim needed + # full data - no trim needed. The bytes embedded below are + # zstd-compressed; node decompresses them before udata_setCommonData. + # icustubdata satisfies ICU's icudtXX_dat link reference. 'sources': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)_dat.<(icu_asm_ext)' ], - 'dependencies': [ 'genccode#host', 'icupkg#host', 'icu_implementation#host', 'icu_uconfig' ], + 'dependencies': [ + 'genccode#host', + 'icupkg#host', + 'icu_implementation#host', + 'icu_uconfig', + 'icustubdata', + '../../deps/zstd/zstd.gyp:zstd_compress#host', + ], + 'export_dependent_settings': [ 'icustubdata' ], 'include_dirs': [ '<(icu_path)/source/common', ], @@ -250,12 +308,26 @@ ], }, { - # convert full ICU data file to .c, or .S, etc. + 'action_name': 'icu_zstd', + 'inputs': [ + '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major).dat', + ], + 'outputs':[ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], + 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '--icu', + '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major).dat', + '<@(_outputs)' ], + }, + { + # convert compressed ICU data to .c, or .S, etc. + # -e names the symbol node_icu_zstd_dat so ICU does not + # treat the compressed bytes as its data entry point. 'action_name': 'icudata', - 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major).dat' ], + 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], 'outputs':[ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)_dat.<(icu_asm_ext)' ], 'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)', - '-e', 'icudt<(icu_ver_major)', + '-e', 'node_icu_zstd', '-d', '<(SHARED_INTERMEDIATE_DIR)', '<@(icu_asm_opts)', '-f', 'icudt<(icu_ver_major)_dat', @@ -266,7 +338,8 @@ # link against stub data (as primary data) # then, use icupkg and genccode to rebuild small data 'dependencies': [ 'icustubdata', 'genccode#host', 'icupkg#host', 'genrb#host', 'iculslocs#host', - 'icu_implementation', 'icu_uconfig' ], + 'icu_implementation', 'icu_uconfig', + '../../deps/zstd/zstd.gyp:zstd_compress#host' ], 'export_dependent_settings': [ 'icustubdata' ], 'actions': [ { @@ -294,13 +367,26 @@ '<@(_inputs)', '<@(_outputs)', ], + }, { + 'action_name': 'icu_zstd', + 'inputs': [ + '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '<(SHARED_INTERMEDIATE_DIR)/icutmp/icusmdt<(icu_ver_major).dat', + ], + 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], + 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', + '--icu', + '<(SHARED_INTERMEDIATE_DIR)/icutmp/icusmdt<(icu_ver_major).dat', + '<@(_outputs)' ], }, { # For icu-small, always use .c, don't try to use .S, etc. 'action_name': 'genccode', - 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icutmp/icusmdt<(icu_ver_major).dat' ], + 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icusmdt<(icu_ver_major)_dat.<(icu_asm_ext)' ], 'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)', '<@(icu_asm_opts)', + '-e', 'node_icu_zstd', + '-f', 'icusmdt<(icu_ver_major)_dat', '-d', '<(SHARED_INTERMEDIATE_DIR)', '<@(_inputs)' ], }, diff --git a/tools/js2c.cc b/tools/js2c.cc index 2cb09f8e1d7..e5bc0f50f9c 100644 --- a/tools/js2c.cc +++ b/tools/js2c.cc @@ -3,7 +3,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -14,6 +16,7 @@ #include "executable_wrapper.h" #include "simdutf.h" #include "uv.h" +#include "zstd.h" #if defined(_WIN32) #include // _S_IREAD _S_IWRITE @@ -172,10 +175,15 @@ std::vector Join(const Fragments& fragments, } const char* kTemplate = R"( +#include +#include +#include + #include "env-inl.h" #include "node_builtins.h" #include "node_external_reference.h" #include "node_internals.h" +#include "zstd_blob.h" namespace node { @@ -191,6 +199,7 @@ const ThreadsafeCopyOnWrite global_source_map { } // anonymous namespace void BuiltinLoader::LoadJavaScriptSource() { + EnsureEmbeddedBuiltinSourcesImpl(); source_ = global_source_map; } @@ -447,6 +456,12 @@ enum class CodeType { kLatin1, // Code points are all within 0-255 kTwoByte, }; +struct OneBytePiece { + std::string var; + std::vector bytes; +}; +std::vector one_byte_pieces; + template Fragment GetDefinitionImpl(const std::vector& code, const std::string& var, @@ -454,6 +469,21 @@ Fragment GetDefinitionImpl(const std::vector& code, constexpr bool is_two_byte = std::is_same_v; static_assert(is_two_byte || std::is_same_v); + // One-byte builtins are stored in a single zstd blob and inflated the + // first time V8 reads the external string. Two-byte sources stay literal + // so the target compiler picks the endianness. + if (type != CodeType::kTwoByte) { + one_byte_pieces.push_back(OneBytePiece{ + var, + std::vector( + reinterpret_cast(code.data()), + reinterpret_cast(code.data()) + code.size()), + }); + std::string decl = "static StaticExternalOneByteResource " + var + + "_resource(nullptr, 0, nullptr);\n"; + return Fragment(decl.begin(), decl.end()); + } + size_t count = is_two_byte ? simdutf::utf16_length_from_utf8(code.data(), code.size()) : code.size(); @@ -824,10 +854,110 @@ int AddGypi(const std::string& var, return 0; } +Fragment EmitCompressedBuiltins() { + std::vector raw; + auto append32 = [&](uint32_t value) { + uint8_t bytes[4]; + memcpy(bytes, &value, sizeof(bytes)); + raw.insert(raw.end(), bytes, bytes + sizeof(bytes)); + }; + append32(static_cast(one_byte_pieces.size())); + for (const OneBytePiece& piece : one_byte_pieces) { + append32(static_cast(piece.bytes.size())); + raw.insert(raw.end(), piece.bytes.begin(), piece.bytes.end()); + while (raw.size() % 4 != 0) { + raw.push_back(0); + } + } + + size_t bound = ZSTD_compressBound(raw.size()); + std::vector compressed(bound); + size_t compressed_size = + ZSTD_compress(compressed.data(), bound, raw.data(), raw.size(), 19); + if (ZSTD_isError(compressed_size)) { + fprintf(stderr, + "js2c: zstd compress failed: %s\n", + ZSTD_getErrorName(compressed_size)); + exit(1); + } + compressed.resize(compressed_size); + fprintf(stderr, + "js2c: builtin sources %zu -> %zu bytes\n", + raw.size(), + compressed_size); + + std::string out; + out.reserve(compressed.size() * 5 + one_byte_pieces.size() * 64 + 2048); + out += "static const uint8_t node_builtin_sources_zstd[] = {\n"; + for (size_t i = 0; i < compressed.size(); i++) { + char buf[8]; + snprintf(buf, sizeof(buf), "0x%02x,", compressed[i]); + out += buf; + if ((i % 16) == 15) { + out += '\n'; + } + } + out += "\n};\n\n"; + out += "static void EnsureEmbeddedBuiltinSourcesImpl() {\n"; + out += " static std::once_flag once;\n"; + out += " std::call_once(once, [] {\n"; + out += " size_t raw_size = 0;\n"; + out += " static uint8_t* storage = node::ZstdDecompressAligned(\n"; + out += " node_builtin_sources_zstd,\n"; + out += " sizeof(node_builtin_sources_zstd),\n"; + out += " &raw_size);\n"; + out += " CHECK_NE(storage, nullptr);\n"; + out += " const uint8_t* cursor = storage;\n"; + out += " const uint8_t* end = storage + raw_size;\n"; + out += " auto read32 = [&](uint32_t* value) {\n"; + out += " CHECK_LE(cursor + 4, end);\n"; + out += " memcpy(value, cursor, 4);\n"; + out += " cursor += 4;\n"; + out += " };\n"; + out += " uint32_t count = 0;\n"; + out += " read32(&count);\n"; + out += " CHECK_EQ(count, "; + out += std::to_string(one_byte_pieces.size()); + out += "u);\n"; + if (!one_byte_pieces.empty()) { + out += " struct Slot { StaticExternalOneByteResource* resource; };\n"; + out += " static const Slot slots[] = {\n"; + for (const OneBytePiece& piece : one_byte_pieces) { + out += " { &"; + out += piece.var; + out += "_resource },\n"; + } + out += " };\n"; + out += " static uint8_t empty = 0;\n"; + out += " for (uint32_t i = 0; i < count; i++) {\n"; + out += " uint32_t byte_len = 0;\n"; + out += " read32(&byte_len);\n"; + out += " CHECK_LE(cursor + byte_len, end);\n"; + out += " const uint8_t* data = byte_len == 0 ? &empty : cursor;\n"; + out += " slots[i].resource->set_data(data, byte_len);\n"; + out += " cursor += byte_len;\n"; + out += " while (static_cast(cursor - storage) % 4 != 0) {\n"; + out += " cursor++;\n"; + out += " }\n"; + out += " }\n"; + out += " CHECK_EQ(static_cast(sizeof(slots) / sizeof(slots[0])),\n"; + out += " static_cast(count));\n"; + } + out += " });\n"; + out += "}\n\n"; + out += "static struct RegisterBuiltinSourceEnsure {\n"; + out += " RegisterBuiltinSourceEnsure() {\n"; + out += " builtin_source_ensure = EnsureEmbeddedBuiltinSourcesImpl;\n"; + out += " }\n"; + out += "} register_builtin_source_ensure;\n"; + return Fragment(out.begin(), out.end()); +} + int JS2C(const FileList& js_files, const FileList& mjs_files, const std::string& config, const std::string& dest) { + one_byte_pieces.clear(); Fragments definitions; definitions.reserve(js_files.size() + mjs_files.size() + 1); Fragments initializers; @@ -854,6 +984,7 @@ int JS2C(const FileList& js_files, if (r != 0) { return r; } + definitions.push_back(EmitCompressedBuiltins()); Fragment out = Format(definitions, initializers, registrations); return WriteIfChanged(out, dest); } diff --git a/tools/zstd_compress.cc b/tools/zstd_compress.cc new file mode 100644 index 00000000000..13a9f23568d --- /dev/null +++ b/tools/zstd_compress.cc @@ -0,0 +1,95 @@ +// Host tool: compress a file with zstd for embedding in the node binary. +// +// zstd_compress --icu +// +// `--icu` prefixes a little-endian header: +// magic "ICUZ" | uint64 raw_size | uint64 compressed_size | zstd frame + +#include +#include +#include + +#include + +#include "zstd.h" + +namespace { + +void WriteU64LE(FILE* out, uint64_t value) { + uint8_t bytes[8]; + for (int i = 0; i < 8; i++) { + bytes[i] = static_cast((value >> (8 * i)) & 0xff); + } + fwrite(bytes, 1, 8, out); +} + +int Fail(const char* message) { + fprintf(stderr, "zstd_compress: %s\n", message); + return 1; +} + +} // namespace + +int main(int argc, char** argv) { + if (argc != 4 || strcmp(argv[1], "--icu") != 0) { + fprintf(stderr, "usage: zstd_compress --icu \n"); + return 1; + } + + FILE* in = fopen(argv[2], "rb"); + if (in == nullptr) { + return Fail("open input"); + } + if (fseek(in, 0, SEEK_END) != 0) { + return Fail("seek"); + } + long raw_long = ftell(in); + if (raw_long < 0) { + return Fail("ftell"); + } + if (fseek(in, 0, SEEK_SET) != 0) { + return Fail("rewind"); + } + size_t raw_size = static_cast(raw_long); + uint8_t* raw = static_cast(malloc(raw_size)); + if (raw == nullptr || fread(raw, 1, raw_size, in) != raw_size) { + return Fail("read"); + } + fclose(in); + + size_t bound = ZSTD_compressBound(raw_size); + uint8_t* compressed = static_cast(malloc(bound)); + if (compressed == nullptr) { + return Fail("alloc"); + } + size_t compressed_size = + ZSTD_compress(compressed, bound, raw, raw_size, 19); + if (ZSTD_isError(compressed_size)) { + return Fail(ZSTD_getErrorName(compressed_size)); + } + + FILE* out = fopen(argv[3], "wb"); + if (out == nullptr) { + return Fail("open output"); + } + fwrite("ICUZ", 1, 4, out); + WriteU64LE(out, raw_size); + WriteU64LE(out, compressed_size); + if (fwrite(compressed, 1, compressed_size, out) != compressed_size) { + return Fail("write"); + } + // genccode emits the file as 32-bit words and drops a short tail. + size_t total = 20 + compressed_size; + while (total % 16 != 0) { + fputc(0, out); + total++; + } + fclose(out); + fprintf(stderr, + "zstd_compress: %zu -> %zu bytes\n", + raw_size, + compressed_size); + free(raw); + free(compressed); + return 0; +} From b094ea9e2d30ab4e037e8237fd2deb639504cd81 Mon Sep 17 00:00:00 2001 From: Yagiz Nizipli Date: Mon, 21 Sep 2026 16:20:23 -0400 Subject: [PATCH 2/4] src: fix lint on compressed startup blobs Wrap lines past 80 columns and store the zstd frame size in uint64_t. Signed-off-by: Yagiz Nizipli --- src/node_snapshotable.cc | 5 +++-- src/zstd_blob.cc | 2 +- tools/js2c.cc | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/node_snapshotable.cc b/src/node_snapshotable.cc index da175a83e75..9f095f8720b 100644 --- a/src/node_snapshotable.cc +++ b/src/node_snapshotable.cc @@ -792,8 +792,9 @@ void FormatBlob(std::ostream& ss, while (raw.size() % 16 != 0) { raw.push_back(0); } - append_bytes(reinterpret_cast(data->v8_snapshot_blob_data.data), - snapshot_size); + const uint8_t* snapshot_bytes = + reinterpret_cast(data->v8_snapshot_blob_data.data); + append_bytes(snapshot_bytes, snapshot_size); for (const auto& item : data->code_cache) { append_bytes(item.data.data, item.data.length); } diff --git a/src/zstd_blob.cc b/src/zstd_blob.cc index 29ad46f23d0..627fee644c6 100644 --- a/src/zstd_blob.cc +++ b/src/zstd_blob.cc @@ -13,7 +13,7 @@ namespace node { uint8_t* ZstdDecompressAligned(const uint8_t* src, size_t src_size, size_t* raw_size) { - unsigned long long content = ZSTD_getFrameContentSize(src, src_size); + uint64_t content = ZSTD_getFrameContentSize(src, src_size); if (content == ZSTD_CONTENTSIZE_ERROR || content == ZSTD_CONTENTSIZE_UNKNOWN || content == 0) { return nullptr; diff --git a/tools/js2c.cc b/tools/js2c.cc index e5bc0f50f9c..692a2babe2f 100644 --- a/tools/js2c.cc +++ b/tools/js2c.cc @@ -940,7 +940,8 @@ Fragment EmitCompressedBuiltins() { out += " cursor++;\n"; out += " }\n"; out += " }\n"; - out += " CHECK_EQ(static_cast(sizeof(slots) / sizeof(slots[0])),\n"; + out += " CHECK_EQ(static_cast(sizeof(slots) /\n"; + out += " sizeof(slots[0])),\n"; out += " static_cast(count));\n"; } out += " });\n"; From 8afd2ef63523c45c760dc22aec07086b3cb3e866 Mon Sep 17 00:00:00 2001 From: Yagiz Nizipli Date: Tue, 22 Sep 2026 14:56:12 -0400 Subject: [PATCH 3/4] build: keep addon exports in the macOS binary -Wl,-dead_strip on the Release executable removed napi_* and other globals that nothing in the executable calls. Addons resolve those symbols from the host, so doc-kit's lightningcss addon called a null pointer inside napi_register_module_v1 and macOS CI died with SIGSEGV while generating docs. -x and -S stay; they only drop local and debug symbols. Signed-off-by: Yagiz Nizipli Assisted-by: Grok --- node.gyp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/node.gyp b/node.gyp index 0e095fb4d07..2540cc3c2f4 100644 --- a/node.gyp +++ b/node.gyp @@ -661,16 +661,16 @@ 'WARNING_CFLAGS': [ '-Werror' ], }, }], - # The Release executable force-loads several static libraries, so - # unreferenced objects and the symbol table dominate the file. - # -dead_strip drops unreferenced .o files, -x drops local symbols, - # and -S drops STABS. Debug keeps symbols. + # The Release executable's local symbol table and STABS dominate + # LINKEDIT. -x drops local symbols and -S drops STABS. Debug keeps + # them. Do not pass -dead_strip: N-API and libuv symbols are reached + # only from addons loaded at runtime, and dead-stripping removes + # those exports from the executable. ['OS=="mac" or OS=="ios"', { 'configurations': { 'Release': { 'xcode_settings': { 'OTHER_LDFLAGS': [ - '-Wl,-dead_strip', '-Wl,-x', '-Wl,-S', ], From 465e71b1dcd499e1978878f97ebb06dd97e2930d Mon Sep 17 00:00:00 2001 From: Yagiz Nizipli Date: Tue, 22 Sep 2026 14:56:13 -0400 Subject: [PATCH 4/4] src: keep embedded ICU data file-backed udata_setCommonData() stores the package address and later lookups return pointers into that storage, so the decompressed buffer cannot be freed. Inflating the full ICU data file at startup turned every page into private dirty memory. Map the original data from the executable again so unused pages stay demand-paged, clean, and shareable across processes. Builtin sources and the startup snapshot remain compressed. Signed-off-by: Yagiz Nizipli Assisted-by: Grok --- deps/zstd/zstd.gyp | 8 --- node.gypi | 6 +- src/node_i18n.cc | 46 +-------------- tools/icu/icu-generic.gyp | 116 +++++--------------------------------- tools/zstd_compress.cc | 95 ------------------------------- 5 files changed, 18 insertions(+), 253 deletions(-) delete mode 100644 tools/zstd_compress.cc diff --git a/deps/zstd/zstd.gyp b/deps/zstd/zstd.gyp index 9e82086ca3c..d06e903442e 100644 --- a/deps/zstd/zstd.gyp +++ b/deps/zstd/zstd.gyp @@ -103,13 +103,5 @@ ], 'toolsets': ['host', 'target'], }, - { - 'target_name': 'zstd_compress', - 'type': 'executable', - 'toolsets': ['host'], - 'dependencies': ['zstd#host'], - 'include_dirs': ['lib'], - 'sources': ['../../tools/zstd_compress.cc'], - }, ] } diff --git a/node.gypi b/node.gypi index 6119a869661..b382784e610 100644 --- a/node.gypi +++ b/node.gypi @@ -135,9 +135,6 @@ '<(icu_gyp_path):icuuc', ], 'conditions': [ - [ 'icu_system!="true"', { - 'defines': [ 'NODE_HAVE_EMBEDDED_ICU_ZSTD=1' ], - }], [ 'icu_small=="true"', { 'defines': [ 'NODE_HAVE_SMALL_ICU=1' ], 'conditions': [ @@ -147,8 +144,7 @@ ], }], ], - }], - ], + }]], }], [ 'node_use_bundled_v8=="true" and \ node_enable_v8_vtunejit=="true" and (target_arch=="x64" or \ diff --git a/src/node_i18n.cc b/src/node_i18n.cc index b110a6883c9..e743941734a 100644 --- a/src/node_i18n.cc +++ b/src/node_i18n.cc @@ -43,10 +43,6 @@ #include "node_i18n.h" #include "node_external_reference.h" #include "simdutf.h" -#include "zstd_blob.h" - -#include -#include #if defined(NODE_HAVE_I18N_SUPPORT) @@ -72,7 +68,7 @@ #include #include "nbytes.h" -#if defined(NODE_HAVE_SMALL_ICU) || defined(NODE_HAVE_EMBEDDED_ICU_ZSTD) +#ifdef NODE_HAVE_SMALL_ICU #include /* if this is defined, we have a 'secondary' entry point. @@ -89,38 +85,6 @@ extern "C" const char U_DATA_API SMALL_ICUDATA_ENTRY_POINT[]; #endif -#ifdef NODE_HAVE_EMBEDDED_ICU_ZSTD -extern "C" const uint8_t node_icu_zstd_dat[]; - -static uint64_t ReadU64LE(const uint8_t* bytes) { - uint64_t value = 0; - for (int i = 0; i < 8; i++) { - value |= static_cast(bytes[i]) << (8 * i); - } - return value; -} - -// Decompress the embedded ICU data file. The returned pointer is aligned and -// lives for the process lifetime. nullptr on failure, with `error` set. -static uint8_t* DecompressEmbeddedICU(std::string* error) { - const uint8_t* bytes = node_icu_zstd_dat; - if (memcmp(bytes, "ICUZ", 4) != 0) { - *error = "embedded ICU data header is invalid"; - return nullptr; - } - uint64_t raw_size = ReadU64LE(bytes + 4); - uint64_t compressed_size = ReadU64LE(bytes + 12); - size_t got = 0; - uint8_t* data = node::ZstdDecompressAligned( - bytes + 20, static_cast(compressed_size), &got); - if (data == nullptr || got != raw_size) { - *error = "failed to decompress embedded ICU data"; - return nullptr; - } - return data; -} -#endif // NODE_HAVE_EMBEDDED_ICU_ZSTD - namespace node { using v8::Context; @@ -591,13 +555,7 @@ ConverterObject::ConverterObject( bool InitializeICUDirectory(const std::string& path, std::string* error) { UErrorCode status = U_ZERO_ERROR; if (path.empty()) { -#ifdef NODE_HAVE_EMBEDDED_ICU_ZSTD - static uint8_t* icu_data = DecompressEmbeddedICU(error); - if (icu_data == nullptr) { - return false; - } - udata_setCommonData(icu_data, &status); -#elif defined(NODE_HAVE_SMALL_ICU) +#ifdef NODE_HAVE_SMALL_ICU // install the 'small' data. udata_setCommonData(&SMALL_ICUDATA_ENTRY_POINT, &status); #else // !NODE_HAVE_SMALL_ICU diff --git a/tools/icu/icu-generic.gyp b/tools/icu/icu-generic.gyp index 521e0f3a499..c4e8c6fbb9f 100644 --- a/tools/icu/icu-generic.gyp +++ b/tools/icu/icu-generic.gyp @@ -138,32 +138,14 @@ [ 'icu_small == "false"', { # and OS=win # full data - just build the full data file, then we are done. 'sources': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], - 'dependencies': [ - 'genccode#host', - 'icustubdata', - '../../deps/zstd/zstd.gyp:zstd_compress#host', - ], - 'export_dependent_settings': [ 'icustubdata' ], + 'dependencies': [ 'genccode#host' ], 'conditions': [ [ 'clang==1', { 'actions': [ - { - 'action_name': 'icu_zstd', - 'msvs_quote_cmd': 0, - 'inputs': [ - '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', - '<(icu_data_in)', - ], - 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], - 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', - '--icu', - '<(icu_data_in)', - '<@(_outputs)' ], - }, { 'action_name': 'icudata', 'msvs_quote_cmd': 0, - 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], + 'inputs': [ '<(icu_data_in)' ], 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], # on Windows, we can go directly to .obj file (-o) option. # for Clang use "-c <(target_arch)" option @@ -172,30 +154,16 @@ '-c', '<(target_arch)', '-d', '<(SHARED_INTERMEDIATE_DIR)', '-n', 'icudata', - '-e', 'node_icu_zstd', - '-f', 'icudt<(icu_ver_major)<(icu_endianness)_dat', + '-e', 'icudt<(icu_ver_major)', '<@(_inputs)' ], }, ], }, { 'actions': [ - { - 'action_name': 'icu_zstd', - 'msvs_quote_cmd': 0, - 'inputs': [ - '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', - '<(icu_data_in)', - ], - 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], - 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', - '--icu', - '<(icu_data_in)', - '<@(_outputs)' ], - }, { 'action_name': 'icudata', 'msvs_quote_cmd': 0, - 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], + 'inputs': [ '<(icu_data_in)' ], 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], # on Windows, we can go directly to .obj file (-o) option. # for MSVC do not use "-c <(target_arch)" option @@ -203,8 +171,7 @@ '<@(icu_asm_opts)', # -o '-d', '<(SHARED_INTERMEDIATE_DIR)', '-n', 'icudata', - '-e', 'node_icu_zstd', - '-f', 'icudt<(icu_ver_major)<(icu_endianness)_dat', + '-e', 'icudt<(icu_ver_major)', '<@(_inputs)' ], }, ], @@ -213,8 +180,7 @@ }, { # icu_small == TRUE and OS == win # link against stub data primarily # then, use icupkg and genccode to rebuild data - 'dependencies': [ 'icustubdata', 'genccode#host', 'icupkg#host', 'genrb#host', 'iculslocs#host', - '../../deps/zstd/zstd.gyp:zstd_compress#host' ], + 'dependencies': [ 'icustubdata', 'genccode#host', 'icupkg#host', 'genrb#host', 'iculslocs#host' ], 'export_dependent_settings': [ 'icustubdata' ], 'actions': [ { @@ -234,32 +200,18 @@ '-v', '-L', '<(icu_locales)'], }, - { - 'action_name': 'icu_zstd', - 'msvs_quote_cmd': 0, - 'inputs': [ - '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', - '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat', - ], - 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], - 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', - '--icu', - '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat', - '<@(_outputs)' ], - }, { # build final .dat -> .obj 'action_name': 'genccode', 'msvs_quote_cmd': 0, - 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], + 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat' ], 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], 'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)', '<@(icu_asm_opts)', # -o '-c', '<(target_arch)', '-d', '<(SHARED_INTERMEDIATE_DIR)/', '-n', 'icudata', - '-e', 'node_icu_zstd', - '-f', 'icudt<(icu_ver_major)<(icu_endianness)_dat', + '-e', 'icusmdt<(icu_ver_major)', '<@(_inputs)' ], }, ], @@ -269,19 +221,9 @@ }, { # OS != win 'conditions': [ [ 'icu_small == "false"', { - # full data - no trim needed. The bytes embedded below are - # zstd-compressed; node decompresses them before udata_setCommonData. - # icustubdata satisfies ICU's icudtXX_dat link reference. + # full data - no trim needed 'sources': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)_dat.<(icu_asm_ext)' ], - 'dependencies': [ - 'genccode#host', - 'icupkg#host', - 'icu_implementation#host', - 'icu_uconfig', - 'icustubdata', - '../../deps/zstd/zstd.gyp:zstd_compress#host', - ], - 'export_dependent_settings': [ 'icustubdata' ], + 'dependencies': [ 'genccode#host', 'icupkg#host', 'icu_implementation#host', 'icu_uconfig' ], 'include_dirs': [ '<(icu_path)/source/common', ], @@ -308,26 +250,12 @@ ], }, { - 'action_name': 'icu_zstd', - 'inputs': [ - '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', - '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major).dat', - ], - 'outputs':[ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], - 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', - '--icu', - '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major).dat', - '<@(_outputs)' ], - }, - { - # convert compressed ICU data to .c, or .S, etc. - # -e names the symbol node_icu_zstd_dat so ICU does not - # treat the compressed bytes as its data entry point. + # convert full ICU data file to .c, or .S, etc. 'action_name': 'icudata', - 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], + 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major).dat' ], 'outputs':[ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)_dat.<(icu_asm_ext)' ], 'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)', - '-e', 'node_icu_zstd', + '-e', 'icudt<(icu_ver_major)', '-d', '<(SHARED_INTERMEDIATE_DIR)', '<@(icu_asm_opts)', '-f', 'icudt<(icu_ver_major)_dat', @@ -338,8 +266,7 @@ # link against stub data (as primary data) # then, use icupkg and genccode to rebuild small data 'dependencies': [ 'icustubdata', 'genccode#host', 'icupkg#host', 'genrb#host', 'iculslocs#host', - 'icu_implementation', 'icu_uconfig', - '../../deps/zstd/zstd.gyp:zstd_compress#host' ], + 'icu_implementation', 'icu_uconfig' ], 'export_dependent_settings': [ 'icustubdata' ], 'actions': [ { @@ -367,26 +294,13 @@ '<@(_inputs)', '<@(_outputs)', ], - }, { - 'action_name': 'icu_zstd', - 'inputs': [ - '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', - '<(SHARED_INTERMEDIATE_DIR)/icutmp/icusmdt<(icu_ver_major).dat', - ], - 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], - 'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)', - '--icu', - '<(SHARED_INTERMEDIATE_DIR)/icutmp/icusmdt<(icu_ver_major).dat', - '<@(_outputs)' ], }, { # For icu-small, always use .c, don't try to use .S, etc. 'action_name': 'genccode', - 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ], + 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icutmp/icusmdt<(icu_ver_major).dat' ], 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icusmdt<(icu_ver_major)_dat.<(icu_asm_ext)' ], 'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)', '<@(icu_asm_opts)', - '-e', 'node_icu_zstd', - '-f', 'icusmdt<(icu_ver_major)_dat', '-d', '<(SHARED_INTERMEDIATE_DIR)', '<@(_inputs)' ], }, diff --git a/tools/zstd_compress.cc b/tools/zstd_compress.cc deleted file mode 100644 index 13a9f23568d..00000000000 --- a/tools/zstd_compress.cc +++ /dev/null @@ -1,95 +0,0 @@ -// Host tool: compress a file with zstd for embedding in the node binary. -// -// zstd_compress --icu -// -// `--icu` prefixes a little-endian header: -// magic "ICUZ" | uint64 raw_size | uint64 compressed_size | zstd frame - -#include -#include -#include - -#include - -#include "zstd.h" - -namespace { - -void WriteU64LE(FILE* out, uint64_t value) { - uint8_t bytes[8]; - for (int i = 0; i < 8; i++) { - bytes[i] = static_cast((value >> (8 * i)) & 0xff); - } - fwrite(bytes, 1, 8, out); -} - -int Fail(const char* message) { - fprintf(stderr, "zstd_compress: %s\n", message); - return 1; -} - -} // namespace - -int main(int argc, char** argv) { - if (argc != 4 || strcmp(argv[1], "--icu") != 0) { - fprintf(stderr, "usage: zstd_compress --icu \n"); - return 1; - } - - FILE* in = fopen(argv[2], "rb"); - if (in == nullptr) { - return Fail("open input"); - } - if (fseek(in, 0, SEEK_END) != 0) { - return Fail("seek"); - } - long raw_long = ftell(in); - if (raw_long < 0) { - return Fail("ftell"); - } - if (fseek(in, 0, SEEK_SET) != 0) { - return Fail("rewind"); - } - size_t raw_size = static_cast(raw_long); - uint8_t* raw = static_cast(malloc(raw_size)); - if (raw == nullptr || fread(raw, 1, raw_size, in) != raw_size) { - return Fail("read"); - } - fclose(in); - - size_t bound = ZSTD_compressBound(raw_size); - uint8_t* compressed = static_cast(malloc(bound)); - if (compressed == nullptr) { - return Fail("alloc"); - } - size_t compressed_size = - ZSTD_compress(compressed, bound, raw, raw_size, 19); - if (ZSTD_isError(compressed_size)) { - return Fail(ZSTD_getErrorName(compressed_size)); - } - - FILE* out = fopen(argv[3], "wb"); - if (out == nullptr) { - return Fail("open output"); - } - fwrite("ICUZ", 1, 4, out); - WriteU64LE(out, raw_size); - WriteU64LE(out, compressed_size); - if (fwrite(compressed, 1, compressed_size, out) != compressed_size) { - return Fail("write"); - } - // genccode emits the file as 32-bit words and drops a short tail. - size_t total = 20 + compressed_size; - while (total % 16 != 0) { - fputc(0, out); - total++; - } - fclose(out); - fprintf(stderr, - "zstd_compress: %zu -> %zu bytes\n", - raw_size, - compressed_size); - free(raw); - free(compressed); - return 0; -}