From de25d6182eb43d3408d3efe7e444b937b2a3a1d9 Mon Sep 17 00:00:00 2001 From: Erik Berlin Date: Sun, 25 Oct 2026 07:50:23 -0700 Subject: [PATCH] Add Duel performance regression guards Benchmark parse and generate through the default adapter in the shapes and sizes of benchmark.rb, plus blank, scalar, IO, binary and invalid input, options, the Oj and fast_jsonparser adapters, adapter resolution, the options cache and the deprecated MultiJson names. The perf workflow compares each pull request with main, each push with the commit before it, and the last release on a schedule. Duel needs Ruby 3.4 or later, so the Gemfile and Rakefile skip it on older Rubies. --- .duel.rb | 13 ++++++ .github/workflows/perf.yml | 64 +++++++++++++++++++++++++ .rubocop.yml | 8 ++++ Gemfile | 5 ++ README.md | 1 + Rakefile | 7 +++ duel/adapter_duel.rb | 44 ++++++++++++++++++ duel/duel_helper.rb | 95 ++++++++++++++++++++++++++++++++++++++ duel/generate_duel.rb | 49 ++++++++++++++++++++ duel/parse_duel.rb | 76 ++++++++++++++++++++++++++++++ 10 files changed, 362 insertions(+) create mode 100644 .duel.rb create mode 100644 .github/workflows/perf.yml create mode 100644 duel/adapter_duel.rb create mode 100644 duel/duel_helper.rb create mode 100644 duel/generate_duel.rb create mode 100644 duel/parse_duel.rb diff --git a/.duel.rb b/.duel.rb new file mode 100644 index 00000000..5c938d58 --- /dev/null +++ b/.duel.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +# Duel's configuration: https://github.com/sferik/duel-ruby/blob/main/docs/configuration.md +# +# Duel follows conventions, so set only what differs: benchmarks are in +# duel/**/*_duel.rb, each version's lib/ is first on the load path, a +# change is compared when it touches lib/ or ext/, and releases are +# tagged v*. +Duel.configure do |c| + # The library each version's workers require (from the gemspec, unless + # this says otherwise). + # c.library = "multi_json" +end diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml new file mode 100644 index 00000000..22ffbe04 --- /dev/null +++ b/.github/workflows/perf.yml @@ -0,0 +1,64 @@ +name: Perf + +on: + pull_request: + push: + branches: [main] + schedule: + - cron: "0 4 * * *" + workflow_dispatch: + +permissions: + contents: read + pull-requests: write # to comment on a pull request with the report + +jobs: + duel: + name: Duel + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 # the baseline comes from history, tags included + # (And whatever else your tests need set up: packages, services.) + - uses: ruby/setup-ruby@v1 + with: + ruby-version: "3.4" + bundler-cache: true + # Against the main branch for a pull request, what was there before + # for a push, and the last release on a schedule (slow creep). Duel + # adds its report to the job's summary, and writes it to duel.md for + # the comment. + - name: Compare + env: + DUEL_BASE: ${{ github.event_name == 'schedule' && 'last-release' || github.event_name == 'push' && github.event.before || '' }} + # (Run by hand, it compares anyway: with nothing changed, the code + # with itself, which checks the setup.) + DUEL_FORCE: ${{ github.event_name == 'workflow_dispatch' && '1' || '' }} + run: | + # (A branch's first push, or one that rewrote history, has no + # commit before it here to compare against.) + if [ "$GITHUB_EVENT_NAME" = push ] && ! git cat-file -e "$DUEL_BASE^{commit}" 2>/dev/null; then + echo "::notice::No commit before this push to compare against" + exit 0 + fi + bundle exec duel --markdown "$RUNNER_TEMP/duel.md" + # The report, as a comment on the pull request: one comment, updated + # on every push. (A pull request from a fork gets no comment, as its + # token can't write one: the job's summary has the report.) + - name: Comment on the pull request + if: ${{ !cancelled() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} + env: + GH_TOKEN: ${{ github.token }} + PR: ${{ github.event.pull_request.number }} + run: | + [ -f "$RUNNER_TEMP/duel.md" ] || exit 0 + marker="" + { echo "$marker"; cat "$RUNNER_TEMP/duel.md"; } > "$RUNNER_TEMP/comment.md" + ids=$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR/comments" \ + --jq ".[] | select(.user.login == \"github-actions[bot]\" and (.body | startswith(\"$marker\"))) | .id") + if [ -n "$ids" ]; then + gh api --method PATCH "repos/$GITHUB_REPOSITORY/issues/comments/${ids%%$'\n'*}" -F body=@"$RUNNER_TEMP/comment.md" > /dev/null + else + gh api --method POST "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F body=@"$RUNNER_TEMP/comment.md" > /dev/null + fi diff --git a/.rubocop.yml b/.rubocop.yml index 991328b8..f1b424b3 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -7,7 +7,15 @@ plugins: - rubocop-rake - standard-performance +# Merge AllCops/Include with RuboCop's defaults, so the list below adds +# to them. Dotfiles are left out unless they're named there. +inherit_mode: + merge: + - Include + AllCops: + Include: + - ".duel.rb" Exclude: - ".git/**/*" - "tmp/**/*" diff --git a/Gemfile b/Gemfile index a7233a3a..ae18bc4a 100644 --- a/Gemfile +++ b/Gemfile @@ -6,6 +6,8 @@ gem "activesupport", ">= 8", require: false gem "concurrent-ruby", ">= 1.2", require: false gem "json", ">= 2", require: false +# Duel requires Ruby 3.4 or later +gem "duel", ">= 0.1" if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("3.4") gem "minitest", ">= 5.20" gem "mutant-minitest", ">= 0.12" gem "rake", ">= 13.1" @@ -30,3 +32,6 @@ gem "jrjackson", ">= 0.4.18", platforms: [:jruby], require: false gem "oj", ">= 3", platforms: %i[ruby windows], require: false gemspec + +# For profiles of regressions (StackProf runs on CRuby only) +gem "stackprof", platforms: :mri diff --git a/README.md b/README.md index 596b3a2c..711863b8 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ [![Mutant](https://github.com/sferik/multi_json/actions/workflows/mutant.yml/badge.svg)][mutant] [![Typecheck](https://github.com/sferik/multi_json/actions/workflows/typecheck.yml/badge.svg)][typecheck] [![Docs](https://github.com/sferik/multi_json/actions/workflows/docs.yml/badge.svg)][docs] +[![Duel](https://raw.githubusercontent.com/sferik/duel-ruby/main/assets/badge.svg)](https://github.com/sferik/duel-ruby) [![Maintainability](https://qlty.sh/badges/fde3f4a8-c331-44be-b1e6-45842137def9/maintainability.svg)][qlty] [![Gem Version](https://badge.fury.io/rb/multi_json.svg)][gem] diff --git a/Rakefile b/Rakefile index 60168184..ea8c2878 100644 --- a/Rakefile +++ b/Rakefile @@ -55,3 +55,10 @@ desc "Run the default task" default_tasks = %i[test lint mutant yardstick] default_tasks << :steep if Rake::Task.task_defined?(:steep) task default: default_tasks + +begin + require "duel/rake_task" + Duel::RakeTask.new +rescue LoadError + # Duel requires Ruby 3.4 or later +end diff --git a/duel/adapter_duel.rb b/duel/adapter_duel.rb new file mode 100644 index 00000000..c48f5429 --- /dev/null +++ b/duel/adapter_duel.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +# What runs around every parse and generate: resolving an adapter by +# name, a block with a temporary adapter, the options cache when it's +# full, and the deprecated names that older callers still use. + +require_relative "duel_helper" + +duel "adapter: resolve one by name" do + MultiJSON.current_adapter(adapter: :oj) +end + +duel "adapter: parse in with_adapter" do + MultiJSON.with_adapter(:oj) { MultiJSON.parse('{"a":1}') } +end + +# A store of its own, so that its evictions leave the shared caches +# alone. It holds the most entries it can, and every fetch is of the key +# it evicted longest ago, so every fetch misses and evicts one. +duel "options: a cache miss in a full cache" do + once do + @store = MultiJSON::OptionsCache::Store.new + @keys = Array.new(MultiJSON::OptionsCache.max_cache_size + 1) { |index| {max_nesting: index + 1}.freeze } + @value = {}.freeze + @next = 0 + @keys.each { |key| @store.fetch(key) { @value } } + end + key = @keys[@next] + @next = (@next + 1) % @keys.size + @store.fetch(key) { @value } +end + +duel "legacy: MultiJson.load" do + once { MultiJson.load("{}") } # warns once, before it's timed + MultiJson.load('{"a":1}') +end + +duel "legacy: MultiJSON.dump" do + once do + MultiJSON.dump({}) # warns once, before it's timed + @object = DuelPayloads.flat_scalars(40) + end + MultiJSON.dump(@object) +end diff --git a/duel/duel_helper.rb b/duel/duel_helper.rb new file mode 100644 index 00000000..f186d986 --- /dev/null +++ b/duel/duel_helper.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +require "json" + +# Payloads for the benchmarks, in the shapes and sizes of benchmark.rb's +# matrix: flat objects of scalars, nested records, and arrays of numbers +# and strings. Every one is built from its index alone, so both versions' +# workers parse and generate exactly the same documents. +module DuelPayloads + COUNTRIES = %w[US DE BR JP IN AU GB CA].freeze + EVENT_TYPES = %w[create update sync archive].freeze + RECORD_TYPES = %w[user team account project].freeze + + module_function + + # A Hash of count fields, cycling through integers, floats, strings, + # booleans, nulls and short arrays of strings + def flat_scalars(count) + Array.new(count) { |index| ["field_#{index}", scalar_value(index)] }.to_h + end + + # A Hash of count user records, each with nested objects and arrays + def nested_records(count) + { + "meta" => {"count" => count, "generated_at" => "2026-01-01T00:00:00Z", "source" => "benchmark"}, + "data" => Array.new(count) { |index| record(index) } + } + end + + # A Hash of long arrays of integers, floats, booleans and strings, and + # a matrix of integers + def array_heavy(count) + { + "ints" => Array.new(count) { |index| index }, + "floats" => Array.new(count) { |index| ((index * 7) % 100_000) / 31.0 }, + "booleans" => Array.new(count, &:even?), + "strings" => Array.new(count / 2) { |index| token("str", index, 18) }, + "matrix" => Array.new(count / 25) { |row| Array.new(20) { |column| (row * 20) + column } } + } + end + + # The JSON text of an object, frozen, as an application receives it + def json(object) = JSON.generate(object).freeze + + def scalar_value(index) + case index % 6 + when 0 then index + when 1 then ((index * 13) % 10_000) / 10.0 + when 2 then token("value", index, 20) + when 3 then index.even? + when 4 then nil + else [token("tag", index, 6), token("tag", index + 1, 6), token("tag", index + 2, 6)] + end + end + + def record(index) + { + "id" => index, + "external_id" => token("user", index, 10), + "type" => RECORD_TYPES[index % RECORD_TYPES.length], + "active" => (index % 3).zero?, + "profile" => profile(index), + "metrics" => metrics(index), + "events" => Array.new(3) { |offset| event(index + offset) } + } + end + + def profile(index) + { + "name" => token("name", index, 14), + "email" => "user#{index}@example.test", + "country" => COUNTRIES[index % COUNTRIES.length], + "segments" => Array.new(4) { |offset| token("segment", index + offset, 6) } + } + end + + def metrics(index) + used = index % 500 + { + "logins" => index * 3, + "score" => ((index * 17) % 10_000) / 100.0, + "quota" => {"used" => used, "limit" => 500, "overage" => used > 450} + } + end + + def event(index) + { + "at" => format("2026-02-%02dT12:%02d:00Z", day: (index % 28) + 1, minute: index % 60), + "type" => EVENT_TYPES[index % EVENT_TYPES.length], + "delta" => (index % 17) - 8 + } + end + + def token(prefix, index, width) = "#{prefix}_#{index.to_s(36).rjust(width, "0")}" +end diff --git a/duel/generate_duel.rb b/duel/generate_duel.rb new file mode 100644 index 00000000..fbb1b2ec --- /dev/null +++ b/duel/generate_duel.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +# MultiJSON.generate, through the adapter MultiJSON picks by default (the +# JSON gem), across benchmark.rb's shapes and sizes, and through each +# adapter's handling of options such as pretty. + +require_relative "duel_helper" + +duel "generate: a tiny hash" do + MultiJSON.generate({"a" => 1}) +end + +duel "generate: a small flat object" do + once { @object = DuelPayloads.flat_scalars(40) } + MultiJSON.generate(@object) +end + +duel "generate: a small set of records" do + once { @object = DuelPayloads.nested_records(8) } + MultiJSON.generate(@object) +end + +duel "generate: a large set of records" do + once { @object = DuelPayloads.nested_records(384) } + MultiJSON.generate(@object) +end + +duel "generate: a large array-heavy document" do + once { @object = DuelPayloads.array_heavy(25_000) } + MultiJSON.generate(@object) +end + +duel "generate: pretty" do + once { @object = DuelPayloads.nested_records(8) } + MultiJSON.generate(@object, pretty: true) +end + +duel "generate: with Oj" do + MultiJSON.generate({"a" => 1}, adapter: :oj) +end + +duel "generate: pretty, with Oj" do + MultiJSON.generate({"a" => 1}, adapter: :oj, pretty: true) +end + +duel "generate: through fast_jsonparser" do + once { @object = DuelPayloads.nested_records(8) } + MultiJSON.generate(@object, adapter: :fast_jsonparser) +end diff --git a/duel/parse_duel.rb b/duel/parse_duel.rb new file mode 100644 index 00000000..efd4a45c --- /dev/null +++ b/duel/parse_duel.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +# MultiJSON.parse, through the adapter MultiJSON picks by default (the +# JSON gem), across benchmark.rb's shapes and sizes, and through each +# path that parse takes before and after the adapter: blank and scalar +# input, options, IO input, binary input, and parse errors. + +require "stringio" +require_relative "duel_helper" + +duel "parse: a tiny object" do + MultiJSON.parse('{"a":1}') +end + +duel "parse: a scalar" do + MultiJSON.parse("42") +end + +duel "parse: blank input" do + MultiJSON.parse(" \n") +end + +duel "parse: a small flat object" do + once { @json = DuelPayloads.json(DuelPayloads.flat_scalars(40)) } + MultiJSON.parse(@json) +end + +duel "parse: a small set of records" do + once { @json = DuelPayloads.json(DuelPayloads.nested_records(8)) } + MultiJSON.parse(@json) +end + +duel "parse: a medium set of records" do + once { @json = DuelPayloads.json(DuelPayloads.nested_records(64)) } + MultiJSON.parse(@json) +end + +duel "parse: a large set of records" do + once { @json = DuelPayloads.json(DuelPayloads.nested_records(384)) } + MultiJSON.parse(@json) +end + +duel "parse: a large array-heavy document" do + once { @json = DuelPayloads.json(DuelPayloads.array_heavy(25_000)) } + MultiJSON.parse(@json) +end + +duel "parse: with symbolize_names" do + once { @json = DuelPayloads.json(DuelPayloads.nested_records(8)) } + MultiJSON.parse(@json, symbolize_names: true) +end + +duel "parse: from an IO" do |n| + once { @json = DuelPayloads.json(DuelPayloads.nested_records(8)) } + ios = untimed { Array.new(n) { StringIO.new(@json) } } + ios.each { |io| MultiJSON.parse(io) } +end + +duel "parse: binary-encoded input" do + once { @json = DuelPayloads.json(DuelPayloads.nested_records(64)).b.freeze } + MultiJSON.parse(@json) +end + +duel "parse: invalid JSON" do + MultiJSON.parse('{"a":[1,2,') +rescue MultiJSON::ParseError + nil +end + +duel "parse: with Oj" do + MultiJSON.parse('{"a":1}', adapter: :oj) +end + +duel "parse: with fast_jsonparser" do + MultiJSON.parse('{"a":1}', adapter: :fast_jsonparser) +end