From 2ffc1da029edc0cb5c8ee91dda960e0fd726ca75 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 11 Mar 2026 11:35:56 +0900 Subject: [PATCH 1/5] Speed up default gem loading by adding files stub line to gemspec During Ruby boot, RubyGems loads all default gem specifications by reading and eval'ing each gemspec file to build the path-to-spec mapping used by Kernel#require. This eval overhead is the largest contributor to RubyGems initialization time (~43% of samples in profiling). This change adds a "# files:" comment line to gemspec headers, alongside the existing "# stub:" lines for name/version/extensions. When present, StubSpecification can read the file list from the first few lines of the gemspec without eval'ing the full Ruby code. For default gems with many files (>200 bytes in the files stub line), the line is omitted and the traditional eval path is used as fallback. On a typical Ruby installation, ~73% of default gems (33 of 45) fit within this limit, reducing the eval cost of load_defaults by ~64%. Co-Authored-By: Claude Opus 4.6 --- lib/rubygems/specification.rb | 20 ++++-- lib/rubygems/stub_specification.rb | 28 ++++++++ test/rubygems/test_gem_installer.rb | 21 ++++++ test/rubygems/test_gem_stub_specification.rb | 76 ++++++++++++++++++++ 4 files changed, 141 insertions(+), 4 deletions(-) diff --git a/lib/rubygems/specification.rb b/lib/rubygems/specification.rb index 50eecedb7727..0addb35e407b 100644 --- a/lib/rubygems/specification.rb +++ b/lib/rubygems/specification.rb @@ -839,10 +839,18 @@ def self._resort!(specs) # :nodoc: # Loads the default specifications. It should be called only once. def self.load_defaults - each_spec([Gem.default_specifications_dir]) do |spec| - # #load returns nil if the spec is bad, so we just ignore - # it at this stage - Gem.register_default_spec(spec) + default_dir = Gem.default_specifications_dir + base_dir = Gem.default_dir + gems_dir = File.join(base_dir, "gems") + + each_gemspec([default_dir]) do |path| + stub = Gem::StubSpecification.default_gemspec_stub(path, base_dir, gems_dir) + if stub.stubbed? && !stub.files.equal?(Gem::StubSpecification::StubLine::NO_FILES) + Gem.register_default_spec(stub) + else + spec = load(path) + Gem.register_default_spec(spec) if spec + end end end @@ -2373,6 +2381,10 @@ def to_ruby result << "#{Gem::StubSpecification::PREFIX}#{name} #{version} #{platform} #{raw_require_paths.join("\0")}" result << "#{Gem::StubSpecification::PREFIX}#{extensions.join "\0"}" unless extensions.empty? + unless files.empty? + files_line = "#{Gem::StubSpecification::FILES_PREFIX}#{files.join "\0"}" + result << files_line if files_line.bytesize <= 200 + end result << nil result << "Gem::Specification.new do |s|" diff --git a/lib/rubygems/stub_specification.rb b/lib/rubygems/stub_specification.rb index 53b337ed8554..889170a5ddce 100644 --- a/lib/rubygems/stub_specification.rb +++ b/lib/rubygems/stub_specification.rb @@ -9,14 +9,19 @@ class Gem::StubSpecification < Gem::BasicSpecification # :nodoc: PREFIX = "# stub: " + # :nodoc: + FILES_PREFIX = "# files: " + # :nodoc: OPEN_MODE = "r:UTF-8:-" class StubLine # :nodoc: all attr_reader :name, :version, :platform, :require_paths, :extensions, :full_name + attr_accessor :files NO_EXTENSIONS = [].freeze + NO_FILES = [].freeze # These are common require paths. REQUIRE_PATHS = { # :nodoc: @@ -44,6 +49,7 @@ def initialize(data, extensions) @platform = Gem::Platform.new parts[2] @extensions = extensions + @files = NO_FILES @full_name = if platform == Gem::Platform::RUBY "#{name}-#{version}" else @@ -122,6 +128,13 @@ def data stubline.chomp! # readline(chomp: true) allocates 3x as much as .readline.chomp! @data = StubLine.new stubline, extensions + + # Read files stub line if present + filesline = extensions == StubLine::NO_EXTENSIONS ? extline : file.readline + if filesline.start_with?(FILES_PREFIX) + filesline.chomp! + @data.files = filesline.byteslice(FILES_PREFIX.bytesize..).split("\0") + end end rescue EOFError end @@ -139,6 +152,21 @@ def raw_require_paths # :nodoc: data.require_paths end + ## + # Files in the gem, from the files stub line if available, + # otherwise from the full specification. + + def files + data.files + end + + ## + # Activate this spec, loading the full specification if needed. + + def activate + to_spec.activate + end + def missing_extensions? return false if RUBY_ENGINE == "jruby" return false if default_gem? diff --git a/test/rubygems/test_gem_installer.rb b/test/rubygems/test_gem_installer.rb index 8947694f53f3..244a6ad148ad 100644 --- a/test/rubygems/test_gem_installer.rb +++ b/test/rubygems/test_gem_installer.rb @@ -2545,6 +2545,27 @@ def test_write_default_spec assert_equal @spec.version, loaded.version end + def test_write_default_spec_includes_files_stub + @spec = setup_base_spec + @spec.files = %w[lib/a.rb lib/b.rb] + + installer = Gem::Installer.for_spec @spec + installer.gem_home = @gemhome + + installer.write_default_spec + + stub = Gem::StubSpecification.default_gemspec_stub( + installer.default_spec_file, + @gemhome, + File.join(@gemhome, "gems") + ) + + assert stub.stubbed? + assert_includes stub.files, "lib/a.rb" + assert_includes stub.files, "lib/b.rb" + assert_equal @spec.name, stub.name + end + def test_dir installer = setup_base_installer diff --git a/test/rubygems/test_gem_stub_specification.rb b/test/rubygems/test_gem_stub_specification.rb index 6c07480c7fd4..94ca8d8de166 100644 --- a/test/rubygems/test_gem_stub_specification.rb +++ b/test/rubygems/test_gem_stub_specification.rb @@ -220,6 +220,34 @@ def test_to_spec_with_other_specs_loaded_does_not_warn assert bar.to_spec end + def test_initialize_with_files_stub + stub = stub_with_files + + assert_equal "stub_f", stub.name + assert_equal v(2), stub.version + assert_equal Gem::Platform::RUBY, stub.platform + assert_equal ["lib"], stub.require_paths + assert_equal %w[lib/stub_f.rb lib/stub_f/util.rb], stub.files + assert stub.stubbed? + end + + def test_initialize_with_extension_and_files_stub + stub = stub_with_extension_and_files + + assert_equal "stub_ef", stub.name + assert_equal v(2), stub.version + assert_equal %w[ext/stub_ef/extconf.rb], stub.extensions + assert_equal %w[lib/stub_ef.rb ext/stub_ef/extconf.rb], stub.files + assert stub.stubbed? + end + + def test_files_stub_missing + stub = stub_without_extension + + assert_equal Gem::StubSpecification::StubLine::NO_FILES, stub.files + assert stub.stubbed? + end + def stub_with_version spec = File.join @gemhome, "specifications", "stub_v-with-version.gemspec" File.open spec, "w" do |io| @@ -292,6 +320,54 @@ def stub_with_extension end end + def stub_with_files + spec = File.join @gemhome, "specifications", "stub_f-2.gemspec" + File.open spec, "w" do |io| + io.write "# -*- encoding: utf-8 -*-\n" + io.write "# stub: stub_f 2 ruby lib\n" + io.write "# files: lib/stub_f.rb\0lib/stub_f/util.rb\n" + io.write "\n" + io.write "Gem::Specification.new do |s|\n" + io.write " s.name = 'stub_f'\n" + io.write " s.version = Gem::Version.new '2'\n" + io.write " s.files = ['lib/stub_f.rb', 'lib/stub_f/util.rb']\n" + io.write "end\n" + + io.flush + + stub = Gem::StubSpecification.gemspec_stub io.path, @gemhome, File.join(@gemhome, "gems") + + yield stub if block_given? + + return stub + end + end + + def stub_with_extension_and_files + spec = File.join @gemhome, "specifications", "stub_ef-2.gemspec" + File.open spec, "w" do |io| + io.write "# -*- encoding: utf-8 -*-\n" + io.write "# stub: stub_ef 2 ruby lib\n" + io.write "# stub: ext/stub_ef/extconf.rb\n" + io.write "# files: lib/stub_ef.rb\0ext/stub_ef/extconf.rb\n" + io.write "\n" + io.write "Gem::Specification.new do |s|\n" + io.write " s.name = 'stub_ef'\n" + io.write " s.version = Gem::Version.new '2'\n" + io.write " s.extensions = ['ext/stub_ef/extconf.rb']\n" + io.write " s.files = ['lib/stub_ef.rb', 'ext/stub_ef/extconf.rb']\n" + io.write "end\n" + + io.flush + + stub = Gem::StubSpecification.gemspec_stub io.path, @gemhome, File.join(@gemhome, "gems") + + yield stub if block_given? + + return stub + end + end + def stub_without_extension spec = File.join @gemhome, "specifications", "stub-2.gemspec" File.open spec, "w" do |io| From b45831b7f0314879855c67bc2d55b758b8a5ce94 Mon Sep 17 00:00:00 2001 From: lolwut Date: Wed, 11 Mar 2026 21:31:34 +0900 Subject: [PATCH 2/5] Fix test expectations to include files stub line in to_ruby output The files stub line feature added in the previous commit causes to_ruby to output a `# files:` comment line, but four tests were not updated to expect this new line in their expected output strings. Co-Authored-By: Claude Opus 4.6 --- test/rubygems/test_gem_specification.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/rubygems/test_gem_specification.rb b/test/rubygems/test_gem_specification.rb index 79be0c996d23..94db1a5c601c 100644 --- a/test/rubygems/test_gem_specification.rb +++ b/test/rubygems/test_gem_specification.rb @@ -2315,6 +2315,7 @@ def test_to_ruby expected = <<-SPEC # -*- encoding: utf-8 -*- # stub: a 2 ruby lib\0other +# files: lib/code.rb Gem::Specification.new do |s| s.name = "a".freeze @@ -2355,6 +2356,7 @@ def test_to_ruby_with_rsa_key expected = <<-SPEC # -*- encoding: utf-8 -*- # stub: a 2 ruby lib +# files: lib/code.rb Gem::Specification.new do |s| s.name = "a".freeze @@ -2432,10 +2434,13 @@ def test_to_ruby_fancy @c1.instance_variable_get(:@require_paths).join "\u0000" extensions = @c1.extensions.join "\u0000" + files_stub = @c1.files.join("\0") + expected = <<-SPEC # -*- encoding: utf-8 -*- # stub: a 1 #{Gem.win_platform? ? "x86-mswin32-60" : "x86-darwin-8"} #{stub_require_paths} # stub: #{extensions} +# files: #{files_stub} Gem::Specification.new do |s| s.name = "a".freeze @@ -3990,6 +3995,7 @@ def test_metadata_specs valid_ruby_spec = <<-EOF # -*- encoding: utf-8 -*- # stub: m 1 ruby lib +# files: lib/code.rb Gem::Specification.new do |s| s.name = "m".freeze From 59e97a6c05928a097e45a72586866d9db9bc40c4 Mon Sep 17 00:00:00 2001 From: lolwut Date: Thu, 12 Mar 2026 06:46:10 +0900 Subject: [PATCH 3/5] Fix bundler git_spec to find version line dynamically The files stub line added to to_ruby output shifted line numbers in generated gemspecs, causing the hardcoded spec_lines[5] to overwrite the name attribute instead of the version. Use dynamic index lookup to find the version line regardless of gemspec header format. Co-Authored-By: Claude Opus 4.6 --- spec/update/git_spec.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spec/update/git_spec.rb b/spec/update/git_spec.rb index 526e988ab7c7..2d510ac0864e 100644 --- a/spec/update/git_spec.rb +++ b/spec/update/git_spec.rb @@ -299,7 +299,8 @@ it "the --source flag updates version of gems that were originally pulled in by the source" do spec_lines = lib_path("bar/foo.gemspec").read.split("\n") - spec_lines[5] = "s.version = '2.0'" + version_line = spec_lines.index {|l| l.include?("s.version") } + spec_lines[version_line] = "s.version = '2.0'" update_git "foo", "2.0", path: @git.path do |s| s.write "foo.gemspec", spec_lines.join("\n") From 67e83fa810178cdb7b4b457608953b0e0d259c97 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Thu, 25 Jun 2026 11:23:25 +0900 Subject: [PATCH 4/5] Drop the size cap on the default gem files stub line Putting every default gem's file list on the "# files:" stub line lets load_defaults skip the gemspec eval for all of them, not just the 33 that fit under 200 bytes. Cold load_defaults drops from ~23ms to ~7.6ms (3.1x), and the cost stays bounded to ~20KB in the default specifications dir because to_ruby_for_cache still strips files from regular installed gems. Co-Authored-By: Claude Opus 4.8 --- lib/rubygems/specification.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/rubygems/specification.rb b/lib/rubygems/specification.rb index 0addb35e407b..315dca166cfb 100644 --- a/lib/rubygems/specification.rb +++ b/lib/rubygems/specification.rb @@ -2382,8 +2382,7 @@ def to_ruby result << "#{Gem::StubSpecification::PREFIX}#{extensions.join "\0"}" unless extensions.empty? unless files.empty? - files_line = "#{Gem::StubSpecification::FILES_PREFIX}#{files.join "\0"}" - result << files_line if files_line.bytesize <= 200 + result << "#{Gem::StubSpecification::FILES_PREFIX}#{files.join "\0"}" end result << nil result << "Gem::Specification.new do |s|" From 6885dbd32d0497f02414d5f70a3158be5c4504ee Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Thu, 25 Jun 2026 11:44:16 +0900 Subject: [PATCH 5/5] Harden the default gem files stub line per review StubSpecification#files now falls back to the full specification when no files stub line is present, matching its documented behavior. load_defaults uses the raw stubbed_files accessor for the presence check so it keeps skipping the eval without triggering that fallback. Compare the extensions sentinel with equal? so an explicitly empty stub extension line is not mistaken for "no extensions" and the files line is still read. Omit the files stub line when a path contains a newline, which would otherwise split the single-line stub and corrupt parsing. Co-Authored-By: Claude Opus 4.8 --- lib/rubygems/specification.rb | 7 +++-- lib/rubygems/stub_specification.rb | 14 +++++++-- test/rubygems/test_gem_specification.rb | 12 ++++++++ test/rubygems/test_gem_stub_specification.rb | 32 +++++++++++++++++++- 4 files changed, 60 insertions(+), 5 deletions(-) diff --git a/lib/rubygems/specification.rb b/lib/rubygems/specification.rb index 315dca166cfb..0c6d12667b4e 100644 --- a/lib/rubygems/specification.rb +++ b/lib/rubygems/specification.rb @@ -845,7 +845,7 @@ def self.load_defaults each_gemspec([default_dir]) do |path| stub = Gem::StubSpecification.default_gemspec_stub(path, base_dir, gems_dir) - if stub.stubbed? && !stub.files.equal?(Gem::StubSpecification::StubLine::NO_FILES) + if stub.stubbed? && !stub.stubbed_files.equal?(Gem::StubSpecification::StubLine::NO_FILES) Gem.register_default_spec(stub) else spec = load(path) @@ -2382,7 +2382,10 @@ def to_ruby result << "#{Gem::StubSpecification::PREFIX}#{extensions.join "\0"}" unless extensions.empty? unless files.empty? - result << "#{Gem::StubSpecification::FILES_PREFIX}#{files.join "\0"}" + files_line = "#{Gem::StubSpecification::FILES_PREFIX}#{files.join "\0"}" + # A newline in a file path would split the stub line and corrupt parsing, + # so fall back to the eval path for such gems by omitting the line. + result << files_line unless files_line.include?("\n") end result << nil result << "Gem::Specification.new do |s|" diff --git a/lib/rubygems/stub_specification.rb b/lib/rubygems/stub_specification.rb index 889170a5ddce..912c549d58e7 100644 --- a/lib/rubygems/stub_specification.rb +++ b/lib/rubygems/stub_specification.rb @@ -130,7 +130,7 @@ def data @data = StubLine.new stubline, extensions # Read files stub line if present - filesline = extensions == StubLine::NO_EXTENSIONS ? extline : file.readline + filesline = extensions.equal?(StubLine::NO_EXTENSIONS) ? extline : file.readline if filesline.start_with?(FILES_PREFIX) filesline.chomp! @data.files = filesline.byteslice(FILES_PREFIX.bytesize..).split("\0") @@ -152,12 +152,22 @@ def raw_require_paths # :nodoc: data.require_paths end + ## + # Files recorded in the files stub line, without loading the full + # specification. Returns StubLine::NO_FILES when the gemspec has no files + # stub line. + + def stubbed_files + data.files + end + ## # Files in the gem, from the files stub line if available, # otherwise from the full specification. def files - data.files + stubbed = stubbed_files + stubbed.equal?(StubLine::NO_FILES) ? to_spec.files : stubbed end ## diff --git a/test/rubygems/test_gem_specification.rb b/test/rubygems/test_gem_specification.rb index 94db1a5c601c..abcea11c77a7 100644 --- a/test/rubygems/test_gem_specification.rb +++ b/test/rubygems/test_gem_specification.rb @@ -2345,6 +2345,18 @@ def test_to_ruby assert_equal @a2, same_spec end + def test_to_ruby_omits_files_stub_when_path_contains_newline + @a2.files = ["lib/code.rb", "lib/with\nnewline.rb"] + + ruby_code = @a2.to_ruby + + refute_includes ruby_code, "# files:" + + same_spec = eval ruby_code + + assert_equal @a2.files.sort, same_spec.files.sort + end + def test_to_ruby_with_rsa_key require "rubygems/openssl" pend "openssl is missing" unless defined?(OpenSSL::PKey::RSA) diff --git a/test/rubygems/test_gem_stub_specification.rb b/test/rubygems/test_gem_stub_specification.rb index 94ca8d8de166..9ccc1b15a81c 100644 --- a/test/rubygems/test_gem_stub_specification.rb +++ b/test/rubygems/test_gem_stub_specification.rb @@ -244,7 +244,15 @@ def test_initialize_with_extension_and_files_stub def test_files_stub_missing stub = stub_without_extension - assert_equal Gem::StubSpecification::StubLine::NO_FILES, stub.files + assert_equal Gem::StubSpecification::StubLine::NO_FILES, stub.stubbed_files + assert stub.stubbed? + end + + def test_files_falls_back_to_full_spec_without_files_stub + stub = stub_with_files_in_body + + assert_equal Gem::StubSpecification::StubLine::NO_FILES, stub.stubbed_files + assert_equal %w[lib/stub_b.rb lib/stub_b/util.rb], stub.files assert stub.stubbed? end @@ -343,6 +351,28 @@ def stub_with_files end end + def stub_with_files_in_body + spec = File.join @gemhome, "specifications", "stub_b-2.gemspec" + File.open spec, "w" do |io| + io.write "# -*- encoding: utf-8 -*-\n" + io.write "# stub: stub_b 2 ruby lib\n" + io.write "\n" + io.write "Gem::Specification.new do |s|\n" + io.write " s.name = 'stub_b'\n" + io.write " s.version = Gem::Version.new '2'\n" + io.write " s.files = ['lib/stub_b.rb', 'lib/stub_b/util.rb']\n" + io.write "end\n" + + io.flush + + stub = Gem::StubSpecification.gemspec_stub io.path, @gemhome, File.join(@gemhome, "gems") + + yield stub if block_given? + + return stub + end + end + def stub_with_extension_and_files spec = File.join @gemhome, "specifications", "stub_ef-2.gemspec" File.open spec, "w" do |io|