Speed up default gem loading by adding files stub line to gemspec

lib/rubygems/specification.rb

@@ -839,10 +839,18 @@ def self._resort!(specs) # :nodoc:
   # Loads the default specifications. It should be called only once.
 
   def self.load_defaults
-    each_spec([Gem.default_specifications_dir]) do |spec|
-      # #load returns nil if the spec is bad, so we just ignore
-      # it at this stage
-      Gem.register_default_spec(spec)
+    default_dir = Gem.default_specifications_dir
+    base_dir = Gem.default_dir
+    gems_dir = File.join(base_dir, "gems")
+
+    each_gemspec([default_dir]) do |path|
+      stub = Gem::StubSpecification.default_gemspec_stub(path, base_dir, gems_dir)
+      if stub.stubbed? && !stub.stubbed_files.equal?(Gem::StubSpecification::StubLine::NO_FILES)
+        Gem.register_default_spec(stub)
+      else
+        spec = load(path)
+        Gem.register_default_spec(spec) if spec
+      end
     end
   end
 
@@ -2373,6 +2381,12 @@ def to_ruby
     result << "#{Gem::StubSpecification::PREFIX}#{name} #{version} #{platform} #{raw_require_paths.join("\0")}"
     result << "#{Gem::StubSpecification::PREFIX}#{extensions.join "\0"}" unless
       extensions.empty?
+    unless files.empty?
+      files_line = "#{Gem::StubSpecification::FILES_PREFIX}#{files.join "\0"}"
+      # A newline in a file path would split the stub line and corrupt parsing,
+      # so fall back to the eval path for such gems by omitting the line.
+      result << files_line unless files_line.include?("\n")
+    end
     result << nil
     result << "Gem::Specification.new do |s|"
 

lib/rubygems/stub_specification.rb

@@ -9,14 +9,19 @@ class Gem::StubSpecification < Gem::BasicSpecification
   # :nodoc:
   PREFIX = "# stub: "
 
+  # :nodoc:
+  FILES_PREFIX = "# files: "
+
   # :nodoc:
   OPEN_MODE = "r:UTF-8:-"
 
   class StubLine # :nodoc: all
     attr_reader :name, :version, :platform, :require_paths, :extensions,
                 :full_name
+    attr_accessor :files
 
     NO_EXTENSIONS = [].freeze
+    NO_FILES = [].freeze
 
     # These are common require paths.
     REQUIRE_PATHS = { # :nodoc:
@@ -44,6 +49,7 @@ def initialize(data, extensions)
 
       @platform      = Gem::Platform.new parts[2]
       @extensions    = extensions
+      @files         = NO_FILES
       @full_name     = if platform == Gem::Platform::RUBY
         "#{name}-#{version}"
       else
@@ -122,6 +128,13 @@ def data
 
             stubline.chomp! # readline(chomp: true) allocates 3x as much as .readline.chomp!
             @data = StubLine.new stubline, extensions
+
+            # Read files stub line if present
+            filesline = extensions.equal?(StubLine::NO_EXTENSIONS) ? extline : file.readline
+            if filesline.start_with?(FILES_PREFIX)
+              filesline.chomp!
+              @data.files = filesline.byteslice(FILES_PREFIX.bytesize..).split("\0")
+            end
           end
         rescue EOFError
         end
@@ -139,6 +152,31 @@ def raw_require_paths # :nodoc:
     data.require_paths
   end
 
+  ##
+  # Files recorded in the files stub line, without loading the full
+  # specification. Returns StubLine::NO_FILES when the gemspec has no files
+  # stub line.
+
+  def stubbed_files
+    data.files
+  end
+
+  ##
+  # Files in the gem, from the files stub line if available,
+  # otherwise from the full specification.
+
+  def files
+    stubbed = stubbed_files
+    stubbed.equal?(StubLine::NO_FILES) ? to_spec.files : stubbed
+  end
+
+  ##
+  # Activate this spec, loading the full specification if needed.
+
+  def activate
+    to_spec.activate
+  end
+
   def missing_extensions?
     return false if RUBY_ENGINE == "jruby"
     return false if default_gem?

spec/update/git_spec.rb

@@ -299,7 +299,8 @@
 
     it "the --source flag updates version of gems that were originally pulled in by the source" do
       spec_lines = lib_path("bar/foo.gemspec").read.split("\n")
-      spec_lines[5] = "s.version = '2.0'"
+      version_line = spec_lines.index {|l| l.include?("s.version") }
+      spec_lines[version_line] = "s.version = '2.0'"
 
       update_git "foo", "2.0", path: @git.path do |s|
         s.write "foo.gemspec", spec_lines.join("\n")

test/rubygems/test_gem_installer.rb

@@ -2545,6 +2545,27 @@ def test_write_default_spec
     assert_equal @spec.version, loaded.version
   end
 
+  def test_write_default_spec_includes_files_stub
+    @spec = setup_base_spec
+    @spec.files = %w[lib/a.rb lib/b.rb]
+
+    installer = Gem::Installer.for_spec @spec
+    installer.gem_home = @gemhome
+
+    installer.write_default_spec
+
+    stub = Gem::StubSpecification.default_gemspec_stub(
+      installer.default_spec_file,
+      @gemhome,
+      File.join(@gemhome, "gems")
+    )
+
+    assert stub.stubbed?
+    assert_includes stub.files, "lib/a.rb"
+    assert_includes stub.files, "lib/b.rb"
+    assert_equal @spec.name, stub.name
+  end
+
   def test_dir
     installer = setup_base_installer
 

test/rubygems/test_gem_specification.rb

@@ -2315,6 +2315,7 @@ def test_to_ruby
     expected = <<-SPEC
 # -*- encoding: utf-8 -*-
 # stub: a 2 ruby lib\0other
+# files: lib/code.rb
 
 Gem::Specification.new do |s|
   s.name = "a".freeze
@@ -2344,6 +2345,18 @@ def test_to_ruby
     assert_equal @a2, same_spec
   end
 
+  def test_to_ruby_omits_files_stub_when_path_contains_newline
+    @a2.files = ["lib/code.rb", "lib/with\nnewline.rb"]
+
+    ruby_code = @a2.to_ruby
+
+    refute_includes ruby_code, "# files:"
+
+    same_spec = eval ruby_code
+
+    assert_equal @a2.files.sort, same_spec.files.sort
+  end
+
   def test_to_ruby_with_rsa_key
     require "rubygems/openssl"
     pend "openssl is missing" unless defined?(OpenSSL::PKey::RSA)
@@ -2355,6 +2368,7 @@ def test_to_ruby_with_rsa_key
     expected = <<-SPEC
 # -*- encoding: utf-8 -*-
 # stub: a 2 ruby lib
+# files: lib/code.rb
 
 Gem::Specification.new do |s|
   s.name = "a".freeze
@@ -2432,10 +2446,13 @@ def test_to_ruby_fancy
       @c1.instance_variable_get(:@require_paths).join "\u0000"
     extensions = @c1.extensions.join "\u0000"
 
+    files_stub = @c1.files.join("\0")
+
     expected = <<-SPEC
 # -*- encoding: utf-8 -*-
 # stub: a 1 #{Gem.win_platform? ? "x86-mswin32-60" : "x86-darwin-8"} #{stub_require_paths}
 # stub: #{extensions}
+# files: #{files_stub}
 
 Gem::Specification.new do |s|
   s.name = "a".freeze
@@ -3990,6 +4007,7 @@ def test_metadata_specs
     valid_ruby_spec = <<-EOF
 # -*- encoding: utf-8 -*-
 # stub: m 1 ruby lib
+# files: lib/code.rb
 
 Gem::Specification.new do |s|
   s.name = "m".freeze

test/rubygems/test_gem_stub_specification.rb

@@ -220,6 +220,42 @@ def test_to_spec_with_other_specs_loaded_does_not_warn
     assert bar.to_spec
   end
 
+  def test_initialize_with_files_stub
+    stub = stub_with_files
+
+    assert_equal "stub_f",                    stub.name
+    assert_equal v(2),                        stub.version
+    assert_equal Gem::Platform::RUBY,         stub.platform
+    assert_equal ["lib"],                     stub.require_paths
+    assert_equal %w[lib/stub_f.rb lib/stub_f/util.rb], stub.files
+    assert stub.stubbed?
+  end
+
+  def test_initialize_with_extension_and_files_stub
+    stub = stub_with_extension_and_files
+
+    assert_equal "stub_ef",                   stub.name
+    assert_equal v(2),                        stub.version
+    assert_equal %w[ext/stub_ef/extconf.rb],  stub.extensions
+    assert_equal %w[lib/stub_ef.rb ext/stub_ef/extconf.rb], stub.files
+    assert stub.stubbed?
+  end
+
+  def test_files_stub_missing
+    stub = stub_without_extension
+
+    assert_equal Gem::StubSpecification::StubLine::NO_FILES, stub.stubbed_files
+    assert stub.stubbed?
+  end
+
+  def test_files_falls_back_to_full_spec_without_files_stub
+    stub = stub_with_files_in_body
+
+    assert_equal Gem::StubSpecification::StubLine::NO_FILES, stub.stubbed_files
+    assert_equal %w[lib/stub_b.rb lib/stub_b/util.rb], stub.files
+    assert stub.stubbed?
+  end
+
   def stub_with_version
     spec = File.join @gemhome, "specifications", "stub_v-with-version.gemspec"
     File.open spec, "w" do |io|
@@ -292,6 +328,76 @@ def stub_with_extension
     end
   end
 
+  def stub_with_files
+    spec = File.join @gemhome, "specifications", "stub_f-2.gemspec"
+    File.open spec, "w" do |io|
+      io.write "# -*- encoding: utf-8 -*-\n"
+      io.write "# stub: stub_f 2 ruby lib\n"
+      io.write "# files: lib/stub_f.rb\0lib/stub_f/util.rb\n"
+      io.write "\n"
+      io.write "Gem::Specification.new do |s|\n"
+      io.write "  s.name = 'stub_f'\n"
+      io.write "  s.version = Gem::Version.new '2'\n"
+      io.write "  s.files = ['lib/stub_f.rb', 'lib/stub_f/util.rb']\n"
+      io.write "end\n"
+
+      io.flush
+
+      stub = Gem::StubSpecification.gemspec_stub io.path, @gemhome, File.join(@gemhome, "gems")
+
+      yield stub if block_given?
+
+      return stub
+    end
+  end
+
+  def stub_with_files_in_body
+    spec = File.join @gemhome, "specifications", "stub_b-2.gemspec"
+    File.open spec, "w" do |io|
+      io.write "# -*- encoding: utf-8 -*-\n"
+      io.write "# stub: stub_b 2 ruby lib\n"
+      io.write "\n"
+      io.write "Gem::Specification.new do |s|\n"
+      io.write "  s.name = 'stub_b'\n"
+      io.write "  s.version = Gem::Version.new '2'\n"
+      io.write "  s.files = ['lib/stub_b.rb', 'lib/stub_b/util.rb']\n"
+      io.write "end\n"
+
+      io.flush
+
+      stub = Gem::StubSpecification.gemspec_stub io.path, @gemhome, File.join(@gemhome, "gems")
+
+      yield stub if block_given?
+
+      return stub
+    end
+  end
+
+  def stub_with_extension_and_files
+    spec = File.join @gemhome, "specifications", "stub_ef-2.gemspec"
+    File.open spec, "w" do |io|
+      io.write "# -*- encoding: utf-8 -*-\n"
+      io.write "# stub: stub_ef 2 ruby lib\n"
+      io.write "# stub: ext/stub_ef/extconf.rb\n"
+      io.write "# files: lib/stub_ef.rb\0ext/stub_ef/extconf.rb\n"
+      io.write "\n"
+      io.write "Gem::Specification.new do |s|\n"
+      io.write "  s.name = 'stub_ef'\n"
+      io.write "  s.version = Gem::Version.new '2'\n"
+      io.write "  s.extensions = ['ext/stub_ef/extconf.rb']\n"
+      io.write "  s.files = ['lib/stub_ef.rb', 'ext/stub_ef/extconf.rb']\n"
+      io.write "end\n"
+
+      io.flush
+
+      stub = Gem::StubSpecification.gemspec_stub io.path, @gemhome, File.join(@gemhome, "gems")
+
+      yield stub if block_given?
+
+      return stub
+    end
+  end
+
   def stub_without_extension
     spec = File.join @gemhome, "specifications", "stub-2.gemspec"
     File.open spec, "w" do |io|