Content addressable gems support

lib/rubygems/basic_specification.rb

@@ -140,18 +140,44 @@ def full_gem_path
   end
 
   ##
-  # Returns the full name (name-version) of this Gem.  Platform information
-  # is included (name-version-platform) if it is specified and not the
-  # default Ruby platform.
+  # Returns the full name (name-version) of this Gem.  For a content-addressable
+  # gem (one targeting a single Ruby ABI on a non-default platform), the content
+  # address takes the platform's place (name-version-<content-address>).
+  # Otherwise platform information is included (name-version-platform) unless it
+  # is the default Ruby platform.
 
   def full_name
-    if platform == Gem::Platform::RUBY || platform.nil?
+    if content_addressable?
+      "#{name}-#{version}-#{content_address}"
+    elsif platform == Gem::Platform::RUBY || platform.nil?
       "#{name}-#{version}"
     else
       "#{name}-#{version}-#{platform}"
     end
   end
 
+  ##
+  # The content address (a hex prefix of the gem file's SHA256) used in place of
+  # the platform in #full_name for a Gem that targets one CRuby ABI
+
+  def content_address
+    raise NotImplementedError
+  end
+
+  ##
+  # True if this Gem is content-addressable, i.e. it carries a
+  # content address.
+
+  def content_addressable?
+    !content_address.nil?
+  end
+
+  CONTENT_ADDRESS = /\A[0-9a-f]{8,64}\z/ # :nodoc:
+
+  def self.content_address?(string) # :nodoc:
+    string.is_a?(String) && CONTENT_ADDRESS.match?(string)
+  end
+
   ##
   # Returns the full name of this Gem (see `Gem::BasicSpecification#full_name`).
   # Information about where the gem is installed is also included if not

lib/rubygems/commands/fetch_command.rb

@@ -93,6 +93,8 @@ def fetch_gems
         specs_and_sources = filtered unless filtered.empty?
       end
 
+      specs_and_sources = prefer_content_addressed(specs_and_sources)
+
       spec, source = specs_and_sources.max_by {|s,| s }
 
       if spec.nil?
@@ -106,4 +108,16 @@ def fetch_gems
 
     exit_code
   end
+
+  def prefer_content_addressed(specs_and_sources)
+    return specs_and_sources unless specs_and_sources.any? {|s,| s.content_addressable? }
+
+    compatible = specs_and_sources.select do |spec,|
+      spec.required_ruby_version.satisfied_by?(Gem.ruby_version)
+    end
+    specs_and_sources = compatible unless compatible.empty?
+
+    skinny = specs_and_sources.select {|s,| s.content_addressable? }
+    skinny.empty? ? specs_and_sources : skinny
+  end
 end

lib/rubygems/installer.rb

@@ -114,6 +114,9 @@ def extract_files(destination_dir, pattern = "*")
 
     def copy_to(path)
     end
+
+    def gem
+    end
   end
 
   ##
@@ -266,6 +269,8 @@ def spec
   #     specifications/<gem-version>.gemspec #=> the Gem::Specification
 
   def install
+    assign_content_address
+
     pre_install_checks
 
     run_pre_install_hooks
@@ -966,6 +971,25 @@ def ensure_writable_dir(dir) # :nodoc:
 
   private
 
+  def assign_content_address
+    address = content_address_from_filename(@package.gem&.path)
+    spec.content_address = address if address
+  end
+
+  def content_address_from_filename(path)
+    return unless path
+
+    token = File.basename(path, ".gem").split("-").last
+    return unless Gem::BasicSpecification.content_address?(token)
+
+    require "digest"
+    unless Digest::SHA256.file(path).hexdigest.start_with?(token)
+      raise Gem::InstallError, "content address mismatch for #{File.basename(path)}"
+    end
+
+    token
+  end
+
   def user_install_dir
     # never install to user home in --build-root mode
     return unless @build_root.nil?

lib/rubygems/name_tuple.rb

@@ -6,16 +6,17 @@
 # wrap the data returned from the indexes.
 
 class Gem::NameTuple
-  def initialize(name, version, platform = Gem::Platform::RUBY)
+  def initialize(name, version, platform = Gem::Platform::RUBY, content_address = nil)
     @name = name
     @version = version
 
     platform &&= platform.to_s
     platform = Gem::Platform::RUBY if !platform || platform.empty?
     @platform = platform
+    @content_address = content_address
   end
 
-  attr_reader :name, :version, :platform
+  attr_reader :name, :version, :platform, :content_address
 
   ##
   # Turn an array of [name, version, platform] into an array of
@@ -46,11 +47,15 @@ def self.null
   # of Gem::Specification#full_name.
 
   def full_name
-    case @platform
-    when nil, "", Gem::Platform::RUBY
-      "#{@name}-#{@version}"
+    if @content_address
+      "#{@name}-#{@version}-#{@content_address}"
     else
-      "#{@name}-#{@version}-#{@platform}"
+      case @platform
+      when nil, "", Gem::Platform::RUBY
+        "#{@name}-#{@version}"
+      else
+        "#{@name}-#{@version}-#{@platform}"
+      end
     end
   end
 
@@ -61,6 +66,13 @@ def match_platform?
     Gem::Platform.match_gem? @platform, @name
   end
 
+  ##
+  # Indicate if this NameTuple is content-addressable (carries a content address).
+
+  def content_addressable?
+    !@content_address.nil?
+  end
+
   ##
   # Indicate if this NameTuple is for a prerelease version.
   def prerelease?
@@ -94,8 +106,8 @@ def inspect # :nodoc:
   alias_method :to_s, :inspect # :nodoc:
 
   def <=>(other)
-    [@name, @version, Gem::Platform.sort_priority(@platform)] <=>
-      [other.name, other.version, Gem::Platform.sort_priority(other.platform)]
+    [@name, @version, Gem::Platform.sort_priority(@platform), @content_address.to_s] <=>
+      [other.name, other.version, Gem::Platform.sort_priority(other.platform), other.content_address.to_s]
   end
 
   include Comparable
@@ -109,7 +121,8 @@ def ==(other)
     when self.class
       @name == other.name &&
         @version == other.version &&
-        @platform == other.platform
+        @platform == other.platform &&
+        @content_address == other.content_address
     when Array
       to_a == other
     else
@@ -120,6 +133,6 @@ def ==(other)
   alias_method :eql?, :==
 
   def hash
-    to_a.hash
+    [@name, @version, @platform, @content_address].hash
   end
 end

lib/rubygems/resolver.rb

@@ -396,10 +396,11 @@ def build_spec_for_cache(name)
       next installed.first if installed.length == 1
       candidates = installed if installed.any?
 
-      # Among remaining candidates, prefer the most specific platform, then the
-      # earlier-supplied source.
+      # Prefer the most specific platform; among equally specific candidates
+      # prefer a content-addressable (skinny) variant, then the earlier source.
       candidates.min_by do |s|
         [Gem::Platform.platform_specificity_match(s.platform, Gem::Platform.local),
+         s.content_addressable? ? 0 : 1,
          source_rank[s.source]]
       end
     end

lib/rubygems/resolver/activation_request.rb

@@ -154,6 +154,6 @@ def platform
   private
 
   def name_tuple
-    @name_tuple ||= Gem::NameTuple.new(name, version, platform)
+    @name_tuple ||= Gem::NameTuple.new(name, version, platform, @spec.content_address)
   end
 end

lib/rubygems/resolver/api_set.rb

@@ -108,12 +108,12 @@ def versions(name) # :nodoc:
       []
     end
 
-    infos.each do |_, number, platform, dependencies, requirements|
-      platform ||= "ruby"
+    infos.each do |_, number, suffix, dependencies, requirements|
+      suffix ||= "ruby"
       dependencies = dependencies.map {|dep_name, reqs| [dep_name, reqs.join(", ")] }
       requirements = requirements.map {|req_name, reqs| [req_name.to_sym, reqs] }.to_h
 
-      @data[name] << { name: name, number: number, platform: platform, dependencies: dependencies, requirements: requirements }
+      @data[name] << { name: name, number: number, suffix: suffix, dependencies: dependencies, requirements: requirements }
     end
 
     @data[name]

lib/rubygems/resolver/api_specification.rb

@@ -31,8 +31,7 @@ def initialize(set, api_data)
     @set = set
     @name = api_data[:name]
     @version = Gem::Version.new(api_data[:number]).freeze
-    @platform = Gem::Platform.new(api_data[:platform]).freeze
-    @original_platform = api_data[:platform].freeze
+    assign_platform api_data
     @dependencies = api_data[:dependencies].map do |name, ver|
       Gem::Dependency.new(name, ver.split(/\s*,\s*/)).freeze
     end.freeze
@@ -46,11 +45,12 @@ def ==(other) # :nodoc:
       @set          == other.set &&
       @name         == other.name &&
       @version      == other.version &&
-      @platform     == other.platform
+      @platform     == other.platform &&
+      @content_address == other.content_address
   end
 
   def hash
-    @set.hash ^ @name.hash ^ @version.hash ^ @platform.hash
+    @set.hash ^ @name.hash ^ @version.hash ^ @platform.hash ^ @content_address.hash
   end
 
   def fetch_development_dependencies # :nodoc:
@@ -97,6 +97,7 @@ def spec # :nodoc:
       s.version  = @version
       s.platform = @platform
       s.original_platform = @original_platform
+      s.content_address = @content_address
       s.required_ruby_version = @required_ruby_version
       s.required_rubygems_version = @required_rubygems_version
 
@@ -112,6 +113,28 @@ def source # :nodoc:
 
   private
 
+  def required_platform_from(requirement)
+    operator, platform = Array(requirement).last.to_s.strip.split(/\s+/, 2)
+    return unless operator == "=" && platform
+
+    Gem::Platform.new(platform)
+  end
+
+  def assign_platform(api_data)
+    suffix = api_data[:suffix].freeze
+    required_platform = required_platform_from(api_data.dig(:requirements, :platform))
+
+    if required_platform
+      @content_address = suffix
+      @platform = required_platform.freeze
+      @original_platform = required_platform.to_s.freeze
+    else
+      @content_address = nil
+      @platform = Gem::Platform.new(suffix).freeze
+      @original_platform = suffix
+    end
+  end
+
   def parse_created_at(value)
     value = value.first if value.is_a?(Array)
     return unless value.is_a?(String)

lib/rubygems/resolver/spec_specification.rb

@@ -23,6 +23,13 @@ def dependencies
     spec.dependencies
   end
 
+  ##
+  # The content address of the backing gem
+
+  def content_address
+    spec.content_address
+  end
+
   ##
   # The required_ruby_version constraint for this specification
 

lib/rubygems/resolver/specification.rb

@@ -64,6 +64,7 @@ class Gem::Resolver::Specification
   # Sets default instance variables for the specification.
 
   def initialize
+    @content_address = nil
     @created_at   = nil
     @dependencies = nil
     @name         = nil
@@ -91,6 +92,15 @@ def full_name
     "#{@name}-#{@version}"
   end
 
+  ##
+  # The content address of a content-addressable gem, or nil for ordinary gems.
+
+  attr_reader :content_address
+
+  def content_addressable?
+    !content_address.nil?
+  end
+
   ##
   # Installs this specification using the Gem::Installer +options+.  The
   # install method yields a Gem::Installer instance, which indicates the

lib/rubygems/safe_marshal.rb

@@ -51,7 +51,7 @@ module SafeMarshal
         @name @requirement @prerelease @version_requirement @version_requirements @type
         @force_ruby_platform
       ],
-      "Gem::NameTuple" => %w[@name @version @platform],
+      "Gem::NameTuple" => %w[@name @version @platform @content_address],
       "Gem::Platform" => %w[@os @cpu @version],
       "Psych::PrivateType" => %w[@value @type_id],
       "YAML::PrivateType" => %w[@value @type_id],

lib/rubygems/source.rb

@@ -123,7 +123,10 @@ def fetch_spec(name_tuple)
              rescue StandardError
                nil
              end
-      return spec if spec
+      if spec
+        spec.content_address = name_tuple.content_address if name_tuple.content_address
+        return spec
+      end
     end
 
     source_uri.path << ".rz"
@@ -142,7 +145,9 @@ def fetch_spec(name_tuple)
 
     Gem.load_safe_marshal
     # TODO: Investigate setting Gem::Specification#loaded_from to a URI
-    Gem::SafeMarshal.safe_load spec
+    spec = Gem::SafeMarshal.safe_load spec
+    spec.content_address = name_tuple.content_address if name_tuple.content_address
+    spec
   end
 
   ##
@@ -258,7 +263,14 @@ def load_compact_index_specs(type)
         version = Gem::Version.new(version_string)
         next if version.prerelease? != (type == :prerelease)
 
-        Gem::NameTuple.new(name, version, platform || "ruby")
+        if Gem::BasicSpecification.content_address?(platform)
+          # The /versions index carries the content address in the platform slot,
+          # not the real platform. Record it as the content address (which keeps
+          # variants distinct) and let the fetched spec supply the real platform.
+          Gem::NameTuple.new(name, version, platform, platform)
+        else
+          Gem::NameTuple.new(name, version, platform || "ruby")
+        end
       end
 
       gem_tuples = max_versions_by_platform(gem_tuples) if type == :latest

lib/rubygems/spec_fetcher.rb

@@ -99,7 +99,7 @@ def search_for_dependency(dependency, matching_platform = true)
 
       found[source] = specs.select do |tup|
         if dependency.match?(tup)
-          if matching_platform && !Gem::Platform.match_gem?(tup.platform, tup.name)
+          if matching_platform && !tup.content_addressable? && !Gem::Platform.match_gem?(tup.platform, tup.name)
             pm = (
               rejected_specs[dependency] ||= \
                 Gem::PlatformMismatch.new(tup.name, tup.version))