From 83a870d4c6ea2c02f6a2ffe7e4b51f74d2307477 Mon Sep 17 00:00:00 2001 From: Jenny Shen Date: Fri, 3 Jul 2026 12:55:29 -0400 Subject: [PATCH 1/4] Add client support for content-addressable gems Content-addressable ("skinny") gems replace the platform slot in a gem's filename with a content address (a hex prefix of the gem's SHA-256), letting multiple per-Ruby-ABI precompiled binaries coexist for one version. - Gem::BasicSpecification: content_address primitive, derived content_addressable?, content-addressed full_name, and the CONTENT_ADDRESS format predicate. - Gem::Specification: settable content_address; to_ruby writes the SHA in the stub slot, a '# stub-target: platform=' line, and s.content_address in the body. - Gem::StubSpecification: parse the stub slot + stub-target; content-addressed full_name on the fast path. - Gem::Installer: derive the content address from the .gem filename, verify it against the file's SHA-256, and raise Gem::InstallError on mismatch. - Resolver (APISet/APISpecification/Resolver): parse the compact-index platform:= requirement into a content address + real platform, and prefer a content-addressable variant among equally-specific candidates. - SpecificationPolicy: warn if a gemspec sets content_address (assigned at install). Assisted-By: devx/4ddf66ed-6c08-4b3b-b805-28b76b59ebd4 --- lib/rubygems/basic_specification.rb | 34 ++++++++++-- lib/rubygems/installer.rb | 24 +++++++++ lib/rubygems/resolver.rb | 5 +- lib/rubygems/resolver/api_set.rb | 6 +-- lib/rubygems/resolver/api_specification.rb | 31 +++++++++-- lib/rubygems/resolver/specification.rb | 10 ++++ lib/rubygems/specification.rb | 13 ++++- lib/rubygems/specification_policy.rb | 2 + lib/rubygems/stub_specification.rb | 53 ++++++++++++++----- test/rubygems/test_gem_installer.rb | 41 ++++++++++++++ test/rubygems/test_gem_resolver.rb | 38 +++++++++++++ test/rubygems/test_gem_resolver_api_set.rb | 8 +-- .../test_gem_resolver_api_specification.rb | 43 +++++++++++---- test/rubygems/test_gem_specification.rb | 43 +++++++++++++++ test/rubygems/test_gem_stub_specification.rb | 48 +++++++++++++++++ 15 files changed, 356 insertions(+), 43 deletions(-) diff --git a/lib/rubygems/basic_specification.rb b/lib/rubygems/basic_specification.rb index 0ed7fc60bbe4..677c45f11231 100644 --- a/lib/rubygems/basic_specification.rb +++ b/lib/rubygems/basic_specification.rb @@ -140,18 +140,44 @@ def full_gem_path end ## - # Returns the full name (name-version) of this Gem. Platform information - # is included (name-version-platform) if it is specified and not the - # default Ruby platform. + # Returns the full name (name-version) of this Gem. For a content-addressable + # gem (one targeting a single Ruby ABI on a non-default platform), the content + # address takes the platform's place (name-version-). + # Otherwise platform information is included (name-version-platform) unless it + # is the default Ruby platform. def full_name - if platform == Gem::Platform::RUBY || platform.nil? + if content_addressable? + "#{name}-#{version}-#{content_address}" + elsif platform == Gem::Platform::RUBY || platform.nil? "#{name}-#{version}" else "#{name}-#{version}-#{platform}" end end + ## + # The content address (a hex prefix of the gem file's SHA256) used in place of + # the platform in #full_name for a Gem that targets one CRuby ABI + + def content_address + raise NotImplementedError + end + + ## + # True if this Gem is content-addressable, i.e. it carries a + # content address. + + def content_addressable? + !content_address.nil? + end + + CONTENT_ADDRESS = /\A[0-9a-f]{8,64}\z/ # :nodoc: + + def self.content_address?(string) # :nodoc: + string.is_a?(String) && CONTENT_ADDRESS.match?(string) + end + ## # Returns the full name of this Gem (see `Gem::BasicSpecification#full_name`). # Information about where the gem is installed is also included if not diff --git a/lib/rubygems/installer.rb b/lib/rubygems/installer.rb index a6e1dc4730a6..242911712a9b 100644 --- a/lib/rubygems/installer.rb +++ b/lib/rubygems/installer.rb @@ -114,6 +114,9 @@ def extract_files(destination_dir, pattern = "*") def copy_to(path) end + + def gem + end end ## @@ -266,6 +269,8 @@ def spec # specifications/.gemspec #=> the Gem::Specification def install + assign_content_address + pre_install_checks run_pre_install_hooks @@ -966,6 +971,25 @@ def ensure_writable_dir(dir) # :nodoc: private + def assign_content_address + address = content_address_from_filename(@package.gem&.path) + spec.content_address = address if address + end + + def content_address_from_filename(path) + return unless path + + token = File.basename(path, ".gem").split("-").last + return unless Gem::BasicSpecification.content_address?(token) + + require "digest" + unless Digest::SHA256.file(path).hexdigest.start_with?(token) + raise Gem::InstallError, "content address mismatch for #{File.basename(path)}" + end + + token + end + def user_install_dir # never install to user home in --build-root mode return unless @build_root.nil? diff --git a/lib/rubygems/resolver.rb b/lib/rubygems/resolver.rb index 788206c0566f..0140e640c58e 100644 --- a/lib/rubygems/resolver.rb +++ b/lib/rubygems/resolver.rb @@ -396,10 +396,11 @@ def build_spec_for_cache(name) next installed.first if installed.length == 1 candidates = installed if installed.any? - # Among remaining candidates, prefer the most specific platform, then the - # earlier-supplied source. + # Prefer the most specific platform; among equally specific candidates + # prefer a content-addressable (skinny) variant, then the earlier source. candidates.min_by do |s| [Gem::Platform.platform_specificity_match(s.platform, Gem::Platform.local), + s.content_addressable? ? 0 : 1, source_rank[s.source]] end end diff --git a/lib/rubygems/resolver/api_set.rb b/lib/rubygems/resolver/api_set.rb index c3dc4c0bcabc..9574f9885c1d 100644 --- a/lib/rubygems/resolver/api_set.rb +++ b/lib/rubygems/resolver/api_set.rb @@ -108,12 +108,12 @@ def versions(name) # :nodoc: [] end - infos.each do |_, number, platform, dependencies, requirements| - platform ||= "ruby" + infos.each do |_, number, suffix, dependencies, requirements| + suffix ||= "ruby" dependencies = dependencies.map {|dep_name, reqs| [dep_name, reqs.join(", ")] } requirements = requirements.map {|req_name, reqs| [req_name.to_sym, reqs] }.to_h - @data[name] << { name: name, number: number, platform: platform, dependencies: dependencies, requirements: requirements } + @data[name] << { name: name, number: number, suffix: suffix, dependencies: dependencies, requirements: requirements } end @data[name] diff --git a/lib/rubygems/resolver/api_specification.rb b/lib/rubygems/resolver/api_specification.rb index c45d83832aba..1cb62bfe1245 100644 --- a/lib/rubygems/resolver/api_specification.rb +++ b/lib/rubygems/resolver/api_specification.rb @@ -31,8 +31,7 @@ def initialize(set, api_data) @set = set @name = api_data[:name] @version = Gem::Version.new(api_data[:number]).freeze - @platform = Gem::Platform.new(api_data[:platform]).freeze - @original_platform = api_data[:platform].freeze + assign_platform api_data @dependencies = api_data[:dependencies].map do |name, ver| Gem::Dependency.new(name, ver.split(/\s*,\s*/)).freeze end.freeze @@ -46,11 +45,12 @@ def ==(other) # :nodoc: @set == other.set && @name == other.name && @version == other.version && - @platform == other.platform + @platform == other.platform && + @content_address == other.content_address end def hash - @set.hash ^ @name.hash ^ @version.hash ^ @platform.hash + @set.hash ^ @name.hash ^ @version.hash ^ @platform.hash ^ @content_address.hash end def fetch_development_dependencies # :nodoc: @@ -97,6 +97,7 @@ def spec # :nodoc: s.version = @version s.platform = @platform s.original_platform = @original_platform + s.content_address = @content_address s.required_ruby_version = @required_ruby_version s.required_rubygems_version = @required_rubygems_version @@ -112,6 +113,28 @@ def source # :nodoc: private + def required_platform_from(requirement) + operator, platform = Array(requirement).last.to_s.strip.split(/\s+/, 2) + return unless operator == "=" && platform + + Gem::Platform.new(platform) + end + + def assign_platform(api_data) + suffix = api_data[:suffix].freeze + required_platform = required_platform_from(api_data.dig(:requirements, :platform)) + + if required_platform + @content_address = suffix + @platform = required_platform.freeze + @original_platform = required_platform.to_s.freeze + else + @content_address = nil + @platform = Gem::Platform.new(suffix).freeze + @original_platform = suffix + end + end + def parse_created_at(value) value = value.first if value.is_a?(Array) return unless value.is_a?(String) diff --git a/lib/rubygems/resolver/specification.rb b/lib/rubygems/resolver/specification.rb index 986fa7c9ae82..7cefeaa9bb7b 100644 --- a/lib/rubygems/resolver/specification.rb +++ b/lib/rubygems/resolver/specification.rb @@ -64,6 +64,7 @@ class Gem::Resolver::Specification # Sets default instance variables for the specification. def initialize + @content_address = nil @created_at = nil @dependencies = nil @name = nil @@ -91,6 +92,15 @@ def full_name "#{@name}-#{@version}" end + ## + # The content address of a content-addressable gem, or nil for ordinary gems. + + attr_reader :content_address + + def content_addressable? + !content_address.nil? + end + ## # Installs this specification using the Gem::Installer +options+. The # install method yields a Gem::Installer instance, which indicates the diff --git a/lib/rubygems/specification.rb b/lib/rubygems/specification.rb index 50eecedb7727..6dd6a984865f 100644 --- a/lib/rubygems/specification.rb +++ b/lib/rubygems/specification.rb @@ -407,6 +407,13 @@ def licenses=(licenses) attr_accessor :metadata + ## + # The content address of a platformed gem that targets one Ruby ABI: a hex + # prefix of the gem's SHA256 that stands in for the platform in the .gem file + # name. Assigned at install time, not declared in the gemspec. + + attr_accessor :content_address # :nodoc: + ###################################################################### # :section: Optional gemspec attributes @@ -1949,6 +1956,7 @@ def initialize(name = nil, version = nil) @loaded_from = nil @original_platform = nil @installed_by_version = nil + @content_address = nil set_nil_attributes_to_nil set_not_nil_attributes_to_default_values @@ -2370,9 +2378,11 @@ def test_files # :nodoc: def to_ruby result = [] result << "# -*- encoding: utf-8 -*-" - result << "#{Gem::StubSpecification::PREFIX}#{name} #{version} #{platform} #{raw_require_paths.join("\0")}" + slot = content_addressable? ? content_address : platform + result << "#{Gem::StubSpecification::PREFIX}#{name} #{version} #{slot} #{raw_require_paths.join("\0")}" result << "#{Gem::StubSpecification::PREFIX}#{extensions.join "\0"}" unless extensions.empty? + result << "#{Gem::StubSpecification::TARGET_PREFIX}platform=#{platform}" if content_addressable? result << nil result << "Gem::Specification.new do |s|" @@ -2381,6 +2391,7 @@ def to_ruby unless platform.nil? || platform == Gem::Platform::RUBY result << " s.platform = #{ruby_code original_platform}" end + result << " s.content_address = #{ruby_code content_address}" if content_addressable? result << "" result << " s.required_rubygems_version = #{ruby_code required_rubygems_version} if s.respond_to? :required_rubygems_version=" diff --git a/lib/rubygems/specification_policy.rb b/lib/rubygems/specification_policy.rb index 478e294e0997..f8aede058bef 100644 --- a/lib/rubygems/specification_policy.rb +++ b/lib/rubygems/specification_policy.rb @@ -434,6 +434,8 @@ def validate_values # TODO: raise at some given date warning "deprecated autorequire specified" if @specification.autorequire + warning "content_address is assigned automatically at install time; remove it from your gemspec" if @specification.content_address + @specification.executables.each do |executable| validate_executable(executable) validate_shebang_line_in(executable) diff --git a/lib/rubygems/stub_specification.rb b/lib/rubygems/stub_specification.rb index 53b337ed8554..1f77ba80800d 100644 --- a/lib/rubygems/stub_specification.rb +++ b/lib/rubygems/stub_specification.rb @@ -9,12 +9,15 @@ class Gem::StubSpecification < Gem::BasicSpecification # :nodoc: PREFIX = "# stub: " + # :nodoc: + TARGET_PREFIX = "# stub-target: " + # :nodoc: OPEN_MODE = "r:UTF-8:-" class StubLine # :nodoc: all attr_reader :name, :version, :platform, :require_paths, :extensions, - :full_name + :full_name, :content_address NO_EXTENSIONS = [].freeze @@ -33,7 +36,7 @@ class StubLine # :nodoc: all "lib" => ["lib"].freeze, }.freeze - def initialize(data, extensions) + def initialize(data, extensions, target = nil) parts = data[PREFIX.length..-1].split(" ", 4) @name = -parts[0] @version = if Gem::Version.correct?(parts[1]) @@ -42,12 +45,15 @@ def initialize(data, extensions) Gem::Version.new(0) end - @platform = Gem::Platform.new parts[2] - @extensions = extensions - @full_name = if platform == Gem::Platform::RUBY + suffix = parts[2] + @extensions = extensions + target_platform = target && target["platform"] + @platform = Gem::Platform.new(target_platform || suffix) + @content_address = Gem::BasicSpecification.content_address?(suffix) ? suffix : nil + @full_name = if @platform == Gem::Platform::RUBY "#{name}-#{version}" else - "#{name}-#{version}-#{platform}" + "#{name}-#{version}-#{suffix}" end path_list = parts.last @@ -110,18 +116,24 @@ def data file.readline # discard encoding line stubline = file.readline if stubline.start_with?(PREFIX) - extline = file.readline - - extensions = - if extline.delete_prefix!(PREFIX) - extline.chomp! - extline.split "\0" + extensions = StubLine::NO_EXTENSIONS + target = nil + + loop do + line = file.readline + if line.delete_prefix!(TARGET_PREFIX) + line.chomp! + target = parse_target(line) + elsif line.delete_prefix!(PREFIX) + line.chomp! + extensions = line.split "\0" else - StubLine::NO_EXTENSIONS + break end + end stubline.chomp! # readline(chomp: true) allocates 3x as much as .readline.chomp! - @data = StubLine.new stubline, extensions + @data = StubLine.new stubline, extensions, target end rescue EOFError end @@ -135,6 +147,15 @@ def data private :data + def parse_target(line) + line.split(" ").each_with_object({}) do |pair, target| + key, value = pair.split("=", 2) + target[key] = value if value + end + end + + private :parse_target + def raw_require_paths # :nodoc: data.require_paths end @@ -180,6 +201,10 @@ def full_name data.full_name end + def content_address + data.content_address + end + ## # The full Gem::Specification for this gem, loaded from evalling its gemspec diff --git a/test/rubygems/test_gem_installer.rb b/test/rubygems/test_gem_installer.rb index 8947694f53f3..94ecf52ff1aa 100644 --- a/test/rubygems/test_gem_installer.rb +++ b/test/rubygems/test_gem_installer.rb @@ -291,6 +291,47 @@ def test_ensure_dependency assert_equal "a requires b (> 2)", e.message end + def test_install_assigns_content_address_from_filename + require "digest" + _, a_gem = util_gem "a", 2 + address = Digest::SHA256.file(a_gem).hexdigest[0, 8] + skinny = File.join(File.dirname(a_gem), "a-2-#{address}.gem") + FileUtils.cp a_gem, skinny + + installer = Gem::Installer.at skinny, install_dir: @gemhome, force: true + spec = installer.install + + assert_equal address, spec.content_address + assert spec.content_addressable? + assert_equal "a-2-#{address}", spec.full_name + assert_path_exist File.join(@gemhome, "gems", "a-2-#{address}") + assert_path_exist File.join(@gemhome, "specifications", "a-2-#{address}.gemspec") + end + + def test_install_raises_when_content_address_token_does_not_match_sha + _, a_gem = util_gem "a", 2 + bogus = File.join(File.dirname(a_gem), "a-2-deadbeef.gem") + FileUtils.cp a_gem, bogus + + installer = Gem::Installer.at bogus, install_dir: @gemhome, force: true + + e = assert_raise Gem::InstallError do + installer.install + end + assert_match "content address mismatch", e.message + end + + def test_install_does_not_content_address_ordinary_gem + _, a_gem = util_gem "a", 2 + + installer = Gem::Installer.at a_gem, install_dir: @gemhome, force: true + spec = installer.install + + refute spec.content_addressable? + assert_nil spec.content_address + assert_equal "a-2", spec.full_name + end + def test_ensure_loadable_spec a, a_gem = util_gem "a", 2 do |s| s.add_dependency "garbage ~> 5" diff --git a/test/rubygems/test_gem_resolver.rb b/test/rubygems/test_gem_resolver.rb index 84ede36b6c85..4363af82e08a 100644 --- a/test/rubygems/test_gem_resolver.rb +++ b/test/rubygems/test_gem_resolver.rb @@ -285,6 +285,44 @@ def test_picks_highest_version assert_resolves_to [a2], res end + def test_prefers_content_addressable_variant + set = Gem::Resolver::APISet.new + fat = Gem::Resolver::APISpecification.new(set, + { name: "a", number: "1", suffix: "arm64-darwin", dependencies: [], requirements: {} }) + skinny = Gem::Resolver::APISpecification.new(set, + { name: "a", number: "1", suffix: "abc12345", dependencies: [], + requirements: { ruby: [">= 0"], platform: ["= arm64-darwin"] } }) + + resolver = Gem::Resolver.new [], set + all = Hash.new {|h, k| h[k] = [] } + all["a"] = [fat, skinny] + resolver.instance_variable_set(:@all_specs, all) + + chosen = resolver.send(:build_spec_for_cache, "a") + + assert_equal skinny, chosen[v(1)] + end + + def test_prefers_more_specific_platform_over_content_addressable + util_set_arch "arm64-darwin-23" do + set = Gem::Resolver::APISet.new + fat = Gem::Resolver::APISpecification.new(set, + { name: "a", number: "1", suffix: "arm64-darwin-23", dependencies: [], requirements: {} }) + skinny = Gem::Resolver::APISpecification.new(set, + { name: "a", number: "1", suffix: "abc12345", dependencies: [], + requirements: { ruby: [">= 0"], platform: ["= arm64-darwin"] } }) + + resolver = Gem::Resolver.new [], set + all = Hash.new {|h, k| h[k] = [] } + all["a"] = [skinny, fat] + resolver.instance_variable_set(:@all_specs, all) + + chosen = resolver.send(:build_spec_for_cache, "a") + + assert_equal fat, chosen[v(1)] + end + end + def test_picks_best_platform is = Gem::Resolver::IndexSpecification unknown = Gem::Platform.new "unknown" diff --git a/test/rubygems/test_gem_resolver_api_set.rb b/test/rubygems/test_gem_resolver_api_set.rb index af855f1692ad..1be5bff800ba 100644 --- a/test/rubygems/test_gem_resolver_api_set.rb +++ b/test/rubygems/test_gem_resolver_api_set.rb @@ -38,7 +38,7 @@ def test_find_all data = [ { name: "a", number: "1", - platform: "ruby", + suffix: "ruby", dependencies: [] }, ] @@ -61,11 +61,11 @@ def test_find_all_prereleases data = [ { name: "a", number: "1", - platform: "ruby", + suffix: "ruby", dependencies: [] }, { name: "a", number: "2.a", - platform: "ruby", + suffix: "ruby", dependencies: [] }, ] @@ -90,7 +90,7 @@ def test_find_all_cache data = [ { name: "a", number: "1", - platform: "ruby", + suffix: "ruby", dependencies: [] }, ] diff --git a/test/rubygems/test_gem_resolver_api_specification.rb b/test/rubygems/test_gem_resolver_api_specification.rb index a58dd75de05f..ff477ce2d499 100644 --- a/test/rubygems/test_gem_resolver_api_specification.rb +++ b/test/rubygems/test_gem_resolver_api_specification.rb @@ -8,7 +8,7 @@ def test_initialize data = { name: "rails", number: "3.0.3", - platform: Gem::Platform.local.to_s, + suffix: Gem::Platform.local.to_s, dependencies: [ ["bundler", "~> 1.0"], ["railties", "= 3.0.3"], @@ -20,6 +20,8 @@ def test_initialize assert_equal "rails", spec.name assert_equal Gem::Version.new("3.0.3"), spec.version assert_equal Gem::Platform.local, spec.platform + refute spec.content_addressable? + assert_nil spec.content_address expected = [ Gem::Dependency.new("bundler", "~> 1.0"), @@ -30,12 +32,31 @@ def test_initialize assert_nil spec.created_at end + def test_initialize_content_addressable + set = Gem::Resolver::APISet.new + data = { + name: "darwin-demo", + number: "1.0.0", + suffix: "bd0ec167", + dependencies: [], + requirements: { ruby: ["~> 3.4.0"], platform: ["= arm64-darwin"] }, + } + + spec = Gem::Resolver::APISpecification.new set, data + + assert spec.content_addressable? + assert_equal "bd0ec167", spec.content_address + assert_equal Gem::Platform.new("arm64-darwin"), spec.platform + assert_equal "darwin-demo-1.0.0-bd0ec167", spec.spec.full_name + assert_equal "bd0ec167", spec.spec.content_address + end + def test_initialize_created_at set = Gem::Resolver::APISet.new data = { name: "rails", number: "3.0.3", - platform: "ruby", + suffix: "ruby", dependencies: [], requirements: { created_at: ["2026-06-05T10:30:45Z"] }, } @@ -50,7 +71,7 @@ def test_initialize_created_at_invalid data = { name: "rails", number: "3.0.3", - platform: "ruby", + suffix: "ruby", dependencies: [], requirements: { created_at: ["not a timestamp"] }, } @@ -65,7 +86,7 @@ def test_initialize_created_at_non_iso8601 data = { name: "rails", number: "3.0.3", - platform: "ruby", + suffix: "ruby", dependencies: [], requirements: { created_at: ["2026"] }, } @@ -93,7 +114,7 @@ def test_fetch_development_dependencies data = { name: "rails", number: "3.0.3", - platform: "ruby", + suffix: "ruby", dependencies: [ ["bundler", "~> 1.0"], ["railties", "= 3.0.3"], @@ -120,7 +141,7 @@ def test_installable_platform_eh data = { name: "a", number: "1", - platform: "ruby", + suffix: "ruby", dependencies: [], } @@ -131,7 +152,7 @@ def test_installable_platform_eh data = { name: "b", number: "1", - platform: "cpu-other_platform-1", + suffix: "cpu-other_platform-1", dependencies: [], } @@ -142,7 +163,7 @@ def test_installable_platform_eh data = { name: "c", number: "1", - platform: Gem::Platform.local.to_s, + suffix: Gem::Platform.local.to_s, dependencies: [], } @@ -156,7 +177,7 @@ def test_source data = { name: "a", number: "1", - platform: "ruby", + suffix: "ruby", dependencies: [], } @@ -175,7 +196,7 @@ def test_spec data = { name: "a", number: "1", - platform: "ruby", + suffix: "ruby", dependencies: [], } @@ -199,7 +220,7 @@ def test_spec_jruby_platform data = { name: "j", number: "1", - platform: "jruby", + suffix: "jruby", dependencies: [], } diff --git a/test/rubygems/test_gem_specification.rb b/test/rubygems/test_gem_specification.rb index 79be0c996d23..13423b58fced 100644 --- a/test/rubygems/test_gem_specification.rb +++ b/test/rubygems/test_gem_specification.rb @@ -1940,6 +1940,18 @@ def test_full_name_windows end end + def test_content_addressable_full_name + refute @a2.content_addressable? + assert_nil @a2.content_address + + @a2.platform = "arm64-darwin" + @a2.content_address = "bd0ec167" + + assert @a2.content_addressable? + assert_equal "a-2-bd0ec167", @a2.full_name + assert_equal Gem::Platform.new("arm64-darwin"), @a2.platform + end + def test_gem_build_complete_path expected = File.join @a1.extension_dir, "gem.build_complete" assert_equal expected, @a1.gem_build_complete_path @@ -2513,6 +2525,22 @@ def test_to_ruby_platform assert_equal "old_platform", same_spec.original_platform end + def test_to_ruby_content_addressable + @a2.platform = "arm64-darwin" + @a2.content_address = "bd0ec167" + + ruby_code = @a2.to_ruby + lines = ruby_code.lines + + assert_equal "# stub: a 2 bd0ec167 lib\n", lines[1] + assert_includes ruby_code, "# stub-target: platform=arm64-darwin" + assert_includes ruby_code, %(s.content_address = "bd0ec167") + + reloaded = eval ruby_code + assert_equal "bd0ec167", reloaded.content_address + assert_equal "a-2-bd0ec167", reloaded.full_name + end + def test_to_yaml yaml_str = @a1.to_yaml @@ -2681,6 +2709,21 @@ def test_validate_autorequire end end + def test_validate_content_address_warns + util_setup_validate + + Dir.chdir @tempdir do + @a1.content_address = "bd0ec167" + + use_ui @ui do + @a1.validate + end + + assert_match "content_address is assigned automatically at install time", + @ui.error, "error" + end + end + def test_validate_dependencies util_setup_validate diff --git a/test/rubygems/test_gem_stub_specification.rb b/test/rubygems/test_gem_stub_specification.rb index 6c07480c7fd4..63590aa0364f 100644 --- a/test/rubygems/test_gem_stub_specification.rb +++ b/test/rubygems/test_gem_stub_specification.rb @@ -47,6 +47,19 @@ def test_initialize_with_empty_version assert_equal v(0), stub.version end + def test_content_addressable_stub + refute @foo.content_addressable? + assert_nil @foo.content_address + + stub = stub_content_addressed + + assert stub.content_addressable? + assert_equal "bd0ec167", stub.content_address + assert_equal Gem::Platform.new("arm64-darwin"), stub.platform + assert_equal "skinny-1.0.0-bd0ec167", stub.full_name + assert_equal ["lib"], stub.require_paths + end + def test_initialize_missing_stubline stub = Gem::StubSpecification.gemspec_stub(BAR, @base_dir, @gems_dir) assert_equal "bar", stub.name @@ -212,6 +225,15 @@ def test_to_spec_default refute_same real_bar, bar_default.to_spec end + def test_content_addressable_to_spec_carries_address + spec = stub_content_addressed.to_spec + + assert_equal "bd0ec167", spec.content_address + assert spec.content_addressable? + assert_equal "skinny-1.0.0-bd0ec167", spec.full_name + assert_equal Gem::Platform.new("arm64-darwin"), spec.platform + end + def test_to_spec_with_other_specs_loaded_does_not_warn real_foo = util_spec @foo.name, @foo.version real_foo.activate @@ -243,6 +265,32 @@ def stub_with_version end end + def stub_content_addressed + spec = File.join @gemhome, "specifications", "skinny-1.0.0-bd0ec167.gemspec" + File.open spec, "w" do |io| + io.write <<~STUB + # -*- encoding: utf-8 -*- + # stub: skinny 1.0.0 bd0ec167 lib + # stub-target: platform=arm64-darwin + + Gem::Specification.new do |s| + s.name = 'skinny' + s.version = Gem::Version.new '1.0.0' + s.platform = 'arm64-darwin' + s.content_address = 'bd0ec167' + end + STUB + + io.flush + + stub = Gem::StubSpecification.gemspec_stub io.path, @gemhome, File.join(@gemhome, "gems") + + yield stub if block_given? + + return stub + end + end + def stub_without_version spec = File.join @gemhome, "specifications", "stub_v-without-version.gemspec" File.open spec, "w" do |io| From 39e302b745d61bf4267869b8c2ab2801f3fadbe5 Mon Sep 17 00:00:00 2001 From: Jenny Shen Date: Fri, 3 Jul 2026 19:15:52 -0400 Subject: [PATCH 2/4] Show content-addressable gems under their content-addressed name gem install --explain (and the resolver's handling of an already-installed skinny) printed the fat name-version-platform instead of the content-addressed name-version-, because Gem::NameTuple -- which ActivationRequest#full_name builds from -- had no field for the content address, and SpecSpecification never forwarded it. Give Gem::NameTuple a content_address field (with matching ==/eql?/hash so two skinny variants of the same name-version-platform don't collapse under uniq) and have Gem::Resolver::SpecSpecification delegate content_address to its backing spec. ActivationRequest#full_name now reports the content-addressed name for both freshly-resolved and already-installed skinnies. The SafeMarshal allowlist permits the new marshaled ivar; older cached tuples simply lack it. Assisted-By: devx/4ddf66ed-6c08-4b3b-b805-28b76b59ebd4 --- lib/rubygems/name_tuple.rb | 33 +++++++++++++------ lib/rubygems/resolver/activation_request.rb | 2 +- lib/rubygems/resolver/spec_specification.rb | 7 ++++ lib/rubygems/safe_marshal.rb | 2 +- test/rubygems/test_gem_name_tuple.rb | 18 ++++++++++ .../test_gem_resolver_activation_request.rb | 13 ++++++++ ...st_gem_resolver_installed_specification.rb | 11 +++++++ 7 files changed, 74 insertions(+), 12 deletions(-) diff --git a/lib/rubygems/name_tuple.rb b/lib/rubygems/name_tuple.rb index cbdf4d7ac5f2..b59f4b96e72f 100644 --- a/lib/rubygems/name_tuple.rb +++ b/lib/rubygems/name_tuple.rb @@ -6,16 +6,17 @@ # wrap the data returned from the indexes. class Gem::NameTuple - def initialize(name, version, platform = Gem::Platform::RUBY) + def initialize(name, version, platform = Gem::Platform::RUBY, content_address = nil) @name = name @version = version platform &&= platform.to_s platform = Gem::Platform::RUBY if !platform || platform.empty? @platform = platform + @content_address = content_address end - attr_reader :name, :version, :platform + attr_reader :name, :version, :platform, :content_address ## # Turn an array of [name, version, platform] into an array of @@ -46,11 +47,15 @@ def self.null # of Gem::Specification#full_name. def full_name - case @platform - when nil, "", Gem::Platform::RUBY - "#{@name}-#{@version}" + if @content_address + "#{@name}-#{@version}-#{@content_address}" else - "#{@name}-#{@version}-#{@platform}" + case @platform + when nil, "", Gem::Platform::RUBY + "#{@name}-#{@version}" + else + "#{@name}-#{@version}-#{@platform}" + end end end @@ -61,6 +66,13 @@ def match_platform? Gem::Platform.match_gem? @platform, @name end + ## + # Indicate if this NameTuple is content-addressable (carries a content address). + + def content_addressable? + !@content_address.nil? + end + ## # Indicate if this NameTuple is for a prerelease version. def prerelease? @@ -94,8 +106,8 @@ def inspect # :nodoc: alias_method :to_s, :inspect # :nodoc: def <=>(other) - [@name, @version, Gem::Platform.sort_priority(@platform)] <=> - [other.name, other.version, Gem::Platform.sort_priority(other.platform)] + [@name, @version, Gem::Platform.sort_priority(@platform), @content_address.to_s] <=> + [other.name, other.version, Gem::Platform.sort_priority(other.platform), other.content_address.to_s] end include Comparable @@ -109,7 +121,8 @@ def ==(other) when self.class @name == other.name && @version == other.version && - @platform == other.platform + @platform == other.platform && + @content_address == other.content_address when Array to_a == other else @@ -120,6 +133,6 @@ def ==(other) alias_method :eql?, :== def hash - to_a.hash + [@name, @version, @platform, @content_address].hash end end diff --git a/lib/rubygems/resolver/activation_request.rb b/lib/rubygems/resolver/activation_request.rb index 5c722001b1ae..9b84f8efc2f6 100644 --- a/lib/rubygems/resolver/activation_request.rb +++ b/lib/rubygems/resolver/activation_request.rb @@ -154,6 +154,6 @@ def platform private def name_tuple - @name_tuple ||= Gem::NameTuple.new(name, version, platform) + @name_tuple ||= Gem::NameTuple.new(name, version, platform, @spec.content_address) end end diff --git a/lib/rubygems/resolver/spec_specification.rb b/lib/rubygems/resolver/spec_specification.rb index 00ef9fdba05b..09d967d2a943 100644 --- a/lib/rubygems/resolver/spec_specification.rb +++ b/lib/rubygems/resolver/spec_specification.rb @@ -23,6 +23,13 @@ def dependencies spec.dependencies end + ## + # The content address of the backing gem + + def content_address + spec.content_address + end + ## # The required_ruby_version constraint for this specification diff --git a/lib/rubygems/safe_marshal.rb b/lib/rubygems/safe_marshal.rb index 871f24727dcb..1ba0e9d0b8dd 100644 --- a/lib/rubygems/safe_marshal.rb +++ b/lib/rubygems/safe_marshal.rb @@ -51,7 +51,7 @@ module SafeMarshal @name @requirement @prerelease @version_requirement @version_requirements @type @force_ruby_platform ], - "Gem::NameTuple" => %w[@name @version @platform], + "Gem::NameTuple" => %w[@name @version @platform @content_address], "Gem::Platform" => %w[@os @cpu @version], "Psych::PrivateType" => %w[@value @type_id], "YAML::PrivateType" => %w[@value @type_id], diff --git a/test/rubygems/test_gem_name_tuple.rb b/test/rubygems/test_gem_name_tuple.rb index 4876737c83db..6c4bede22c37 100644 --- a/test/rubygems/test_gem_name_tuple.rb +++ b/test/rubygems/test_gem_name_tuple.rb @@ -16,6 +16,24 @@ def test_full_name n = Gem::NameTuple.new "a", Gem::Version.new(0), "other" assert_equal "a-0-other", n.full_name + + n = Gem::NameTuple.new "a", Gem::Version.new(0), "arm64-darwin", "bd0ec167" + assert_equal "a-0-bd0ec167", n.full_name + end + + def test_content_address_variants_are_distinct + a = Gem::NameTuple.new "a", Gem::Version.new(0), "arm64-darwin", "bd0ec167" + b = Gem::NameTuple.new "a", Gem::Version.new(0), "arm64-darwin", "3a41cc2c08" + + assert a.content_addressable? + refute Gem::NameTuple.new("a", Gem::Version.new(0), "ruby").content_addressable? + refute_equal a, b + refute_equal a.hash, b.hash + assert_equal 2, [a, b].uniq.size + + same = Gem::NameTuple.new "a", Gem::Version.new(0), "arm64-darwin", "bd0ec167" + assert_equal a, same + assert_equal a.hash, same.hash end def test_platform_normalization diff --git a/test/rubygems/test_gem_resolver_activation_request.rb b/test/rubygems/test_gem_resolver_activation_request.rb index 53875b72cb55..b611d0f2b230 100644 --- a/test/rubygems/test_gem_resolver_activation_request.rb +++ b/test/rubygems/test_gem_resolver_activation_request.rb @@ -26,6 +26,19 @@ def test_development_eh assert act_req.development? end + def test_full_name + assert_equal "a-3", @req.full_name + + set = Gem::Resolver::APISet.new + data = { name: "a", number: "1", suffix: "bd0ec167", dependencies: [], + requirements: { platform: ["= arm64-darwin"] } } + ca = Gem::Resolver::ActivationRequest.new( + Gem::Resolver::APISpecification.new(set, data), @dep + ) + + assert_equal "a-1-bd0ec167", ca.full_name + end + def test_inspect assert_match "a-3", @req.inspect assert_match "from a (>= 0)", @req.inspect diff --git a/test/rubygems/test_gem_resolver_installed_specification.rb b/test/rubygems/test_gem_resolver_installed_specification.rb index 5b10273b1368..102dc79aa2d7 100644 --- a/test/rubygems/test_gem_resolver_installed_specification.rb +++ b/test/rubygems/test_gem_resolver_installed_specification.rb @@ -19,6 +19,17 @@ def test_initialize assert_equal Gem::Platform::RUBY, spec.platform end + def test_content_address_delegates_to_spec + plain = Gem::Resolver::InstalledSpecification.new @set, util_spec("a") + refute plain.content_addressable? + assert_nil plain.content_address + + skinny_spec = util_spec("a") {|s| s.content_address = "bd0ec167" } + skinny = Gem::Resolver::InstalledSpecification.new @set, skinny_spec + assert skinny.content_addressable? + assert_equal "bd0ec167", skinny.content_address + end + def test_install a = util_spec "a" From 1636f46b6c8a001f9ed44a2fb12060a8904cd848 Mon Sep 17 00:00:00 2001 From: Jenny Shen Date: Fri, 3 Jul 2026 19:15:52 -0400 Subject: [PATCH 3/4] Activate the content-addressable build matching the running Ruby When several content-addressable variants of the same name-version-platform are installed in one GEM_HOME (e.g. a GEM_HOME shared across Ruby versions), activation/require chose one arbitrarily and could load a native extension built for a different Ruby ABI -- or a fat gem over a matching skinny. _resort! now breaks that tie in favour of the build whose required_ruby_version matches the running Ruby, then the content-addressed one. The tiebreak is only reached when name, version, and platform all tie, and only loads a full spec when a content-addressable variant is actually involved, so ordinary duplicates (a default gem plus an installed copy) stay lazy. Assisted-By: devx/4ddf66ed-6c08-4b3b-b805-28b76b59ebd4 --- lib/rubygems/specification.rb | 14 +++++++ test/rubygems/test_gem_specification.rb | 51 +++++++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/lib/rubygems/specification.rb b/lib/rubygems/specification.rb index 6dd6a984865f..097ca38b2660 100644 --- a/lib/rubygems/specification.rb +++ b/lib/rubygems/specification.rb @@ -836,12 +836,26 @@ def self._resort!(specs) # :nodoc: next versions if versions.nonzero? platforms = Gem::Platform.sort_priority(b.platform) <=> Gem::Platform.sort_priority(a.platform) next platforms if platforms.nonzero? + content = _compare_content_address(a, b) + next content if content.nonzero? default_gem = a.default_gem_priority <=> b.default_gem_priority next default_gem if default_gem.nonzero? a.base_dir_priority(gem_path) <=> b.base_dir_priority(gem_path) end end + def self._compare_content_address(a, b) # :nodoc: + return 0 unless a.content_addressable? || b.content_addressable? + + _content_address_priority(a) <=> _content_address_priority(b) + end + + def self._content_address_priority(spec) # :nodoc: + full = spec.to_spec + built_for_running_ruby = full && full.required_ruby_version.satisfied_by?(Gem.ruby_version) + [built_for_running_ruby ? 0 : 1, spec.content_addressable? ? 0 : 1] + end + ## # Loads the default specifications. It should be called only once. diff --git a/test/rubygems/test_gem_specification.rb b/test/rubygems/test_gem_specification.rb index 13423b58fced..b1d14c16470f 100644 --- a/test/rubygems/test_gem_specification.rb +++ b/test/rubygems/test_gem_specification.rb @@ -1952,6 +1952,57 @@ def test_content_addressable_full_name assert_equal Gem::Platform.new("arm64-darwin"), @a2.platform end + def test_resort_orders_content_addressed_variants_by_ruby_abi + fat = util_spec("a", "1") {|s| s.platform = "arm64-darwin" } + old = util_spec("a", "1") do |s| + s.platform = "arm64-darwin" + s.content_address = "3a41cc2c08" + s.required_ruby_version = "~> 3.3.0" + end + new = util_spec("a", "1") do |s| + s.platform = "arm64-darwin" + s.content_address = "bd0ec167" + s.required_ruby_version = "~> 3.4.0" + end + + Gem.stub :ruby_version, Gem::Version.new("3.4.5") do + specs = [old, fat, new] + Gem::Specification._resort! specs + assert_equal "bd0ec167", specs.first.content_address, "running-Ruby skinny first" + assert specs[1].content_address.nil?, "then the compatible fat" + end + + Gem.stub :ruby_version, Gem::Version.new("3.3.9") do + specs = [new, fat, old] + Gem::Specification._resort! specs + assert_equal "3a41cc2c08", specs.first.content_address + end + end + + def test_resort_keeps_ordinary_duplicates_lazy + # A default gem plus an installed copy tie on name-version-platform but are + # not content-addressable; _resort! must not load their full specs. + a = util_spec("a", "1") + b = util_spec("a", "1") + [a, b].each do |s| + def s.to_spec + raise "to_spec must not be called for ordinary gems" + end + end + + sorted = Gem::Specification._resort!([a, b]) # must not call to_spec + assert_equal 2, sorted.size + end + + def test_content_address_priority_tolerates_missing_spec + spec = util_spec("a", "1") {|s| s.content_address = "bd0ec167" } + def spec.to_spec + nil + end + + assert_equal [1, 0], Gem::Specification._content_address_priority(spec) + end + def test_gem_build_complete_path expected = File.join @a1.extension_dir, "gem.build_complete" assert_equal expected, @a1.gem_build_complete_path From 8574341547ea5b25bf0f578494b555a3e552af1f Mon Sep 17 00:00:00 2001 From: Jenny Shen Date: Fri, 3 Jul 2026 19:15:53 -0400 Subject: [PATCH 4/4] Support content-addressable gems in gem fetch and SpecFetcher gem fetch downloaded the fat gem and gem list --remote showed the SHA as a platform, because the SpecFetcher/NameTuple index path treated the content address (carried in the /versions platform slot) as a bogus platform and rejected the skinny as a platform mismatch. Teach that path about content addresses so gem fetch mirrors gem install: load_compact_index_specs records a content-address token as the tuple's content_address; search_for_dependency skips the platform-mismatch rejection for such tuples (their slot holds a SHA, not a platform -- the fetched spec supplies the real platform); fetch_spec fetches the content-addressed gemspec and carries the address onto it; and FetchCommand prefers the running-Ruby content-addressed build. Assisted-By: devx/4ddf66ed-6c08-4b3b-b805-28b76b59ebd4 --- lib/rubygems/commands/fetch_command.rb | 14 +++++++++++ lib/rubygems/source.rb | 18 +++++++++++--- lib/rubygems/spec_fetcher.rb | 2 +- .../test_gem_commands_fetch_command.rb | 24 +++++++++++++++++++ 4 files changed, 54 insertions(+), 4 deletions(-) diff --git a/lib/rubygems/commands/fetch_command.rb b/lib/rubygems/commands/fetch_command.rb index 8e64a18cee9a..be3e8ecdb758 100644 --- a/lib/rubygems/commands/fetch_command.rb +++ b/lib/rubygems/commands/fetch_command.rb @@ -93,6 +93,8 @@ def fetch_gems specs_and_sources = filtered unless filtered.empty? end + specs_and_sources = prefer_content_addressed(specs_and_sources) + spec, source = specs_and_sources.max_by {|s,| s } if spec.nil? @@ -106,4 +108,16 @@ def fetch_gems exit_code end + + def prefer_content_addressed(specs_and_sources) + return specs_and_sources unless specs_and_sources.any? {|s,| s.content_addressable? } + + compatible = specs_and_sources.select do |spec,| + spec.required_ruby_version.satisfied_by?(Gem.ruby_version) + end + specs_and_sources = compatible unless compatible.empty? + + skinny = specs_and_sources.select {|s,| s.content_addressable? } + skinny.empty? ? specs_and_sources : skinny + end end diff --git a/lib/rubygems/source.rb b/lib/rubygems/source.rb index 348bf6a24bc0..49ed6020f0ff 100644 --- a/lib/rubygems/source.rb +++ b/lib/rubygems/source.rb @@ -123,7 +123,10 @@ def fetch_spec(name_tuple) rescue StandardError nil end - return spec if spec + if spec + spec.content_address = name_tuple.content_address if name_tuple.content_address + return spec + end end source_uri.path << ".rz" @@ -142,7 +145,9 @@ def fetch_spec(name_tuple) Gem.load_safe_marshal # TODO: Investigate setting Gem::Specification#loaded_from to a URI - Gem::SafeMarshal.safe_load spec + spec = Gem::SafeMarshal.safe_load spec + spec.content_address = name_tuple.content_address if name_tuple.content_address + spec end ## @@ -258,7 +263,14 @@ def load_compact_index_specs(type) version = Gem::Version.new(version_string) next if version.prerelease? != (type == :prerelease) - Gem::NameTuple.new(name, version, platform || "ruby") + if Gem::BasicSpecification.content_address?(platform) + # The /versions index carries the content address in the platform slot, + # not the real platform. Record it as the content address (which keeps + # variants distinct) and let the fetched spec supply the real platform. + Gem::NameTuple.new(name, version, platform, platform) + else + Gem::NameTuple.new(name, version, platform || "ruby") + end end gem_tuples = max_versions_by_platform(gem_tuples) if type == :latest diff --git a/lib/rubygems/spec_fetcher.rb b/lib/rubygems/spec_fetcher.rb index 835dedf9489a..254bfb6d86f3 100644 --- a/lib/rubygems/spec_fetcher.rb +++ b/lib/rubygems/spec_fetcher.rb @@ -99,7 +99,7 @@ def search_for_dependency(dependency, matching_platform = true) found[source] = specs.select do |tup| if dependency.match?(tup) - if matching_platform && !Gem::Platform.match_gem?(tup.platform, tup.name) + if matching_platform && !tup.content_addressable? && !Gem::Platform.match_gem?(tup.platform, tup.name) pm = ( rejected_specs[dependency] ||= \ Gem::PlatformMismatch.new(tup.name, tup.version)) diff --git a/test/rubygems/test_gem_commands_fetch_command.rb b/test/rubygems/test_gem_commands_fetch_command.rb index e673e391fe45..b850e4b35e0c 100644 --- a/test/rubygems/test_gem_commands_fetch_command.rb +++ b/test/rubygems/test_gem_commands_fetch_command.rb @@ -12,6 +12,30 @@ def setup @cmd = Gem::Commands::FetchCommand.new end + def test_prefer_content_addressed + fat = util_spec("a", "1") {|s| s.platform = "arm64-darwin" } + s33 = util_spec("a", "1") do |s| + s.platform = "arm64-darwin" + s.content_address = "3a41cc2c08" + s.required_ruby_version = "~> 3.3.0" + end + s34 = util_spec("a", "1") do |s| + s.platform = "arm64-darwin" + s.content_address = "bd0ec167" + s.required_ruby_version = "~> 3.4.0" + end + pairs = [[fat, nil], [s33, nil], [s34, nil]] + + Gem.stub :ruby_version, Gem::Version.new("3.4.5") do + chosen = @cmd.send(:prefer_content_addressed, pairs).map {|s,| s.content_address } + assert_equal ["bd0ec167"], chosen, "the running-Ruby skinny" + end + + # No content-addressable candidates: leave the set untouched. + plain = [[fat, nil]] + assert_same plain, @cmd.send(:prefer_content_addressed, plain) + end + def test_execute specs = spec_fetcher do |fetcher| fetcher.gem "a", 2