diff --git a/CHANGELOG.md b/CHANGELOG.md index af66b1d5..a8ae8457 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # sqlite3-ruby Changelog +## next / unreleased + +### Added + +- Experimental Ractor support. On Ruby 3.4 and later, when SQLite is compiled thread-safe, the extension declares itself Ractor-safe, and each Ractor can open and use its own connections. `SQLite3.ractor_safe?` reports whether the extension made the declaration. See [RACTOR.md](RACTOR.md) for the limits on sharing connections between Ractors and on fork safety. We welcome bug reports, but we recommend caution before using Ractor support in production applications. #749 @rubys @sandlerr @tenderlove @flavorjones + +### Changed + +- `SQLite3::VERSION`, `SQLite3::VERSION_INFO`, `SQLite3::SQLITE_VERSION` and `SQLite3::SQLITE_LOADED_VERSION` are frozen, so that any Ractor can read them. Code that mutates them now raises `FrozenError`. #749 @rubys @tenderlove + ## 2.10.2 / 2026-10-26 ### Security / Stability diff --git a/RACTOR.md b/RACTOR.md new file mode 100644 index 00000000..40885458 --- /dev/null +++ b/RACTOR.md @@ -0,0 +1,98 @@ +# Ractor support in SQLite3-Ruby + +Ractor support is experimental at this time. We welcome bug reports, but we +recommend caution before exercising this support in production applications. + +SQLite3 has [3 different modes of threading support](https://www.sqlite.org/threadsafe.html). + +1. Single-thread +2. Multi-thread +3. Serialized + +"Single thread" mode means there are no mutexes, so the library itself is not +thread safe. In other words if two threads do `SQLite3::Database.new` on the +same file, it will have thread safety problems. + +"Multi thread" mode means that SQLite3 puts mutexes in place, but it does not +mean that the SQLite3 API itself is thread safe. In other words, in this mode +it is SAFE for two threads to do `SQLite3::Database.new` on the same file, but +it is NOT SAFE to share that database object between two threads. + +"Serialized" mode is like "Multi thread" mode except that there are mutexes in +place such that it IS SAFE to share the same database object between threads. + +## Ractor Safety + +When a C extension claims to be Ractor safe by calling `rb_ext_ractor_safe`, +it's merely claiming that its C API is thread safe. This _does not_ mean that +objects allocated from said C extension are allowed to cross between Ractor +boundaries. + +In other words, `rb_ext_ractor_safe` matches the expectations of the +"multi-thread" mode of SQLite3. We can detect the multithread mode via the +`sqlite3_threadsafe` function. In other words, it's fine to declare this +extension is Ractor safe, but only if `sqlite3_threadsafe` returns true. +`SQLite3.ractor_safe?` reports whether the extension made that declaration. + +The supported pattern is that each Ractor opens and uses its own connections. +Everything works inside a Ractor except `Database#create_aggregate`, which +raises `Ractor::IsolationError` outside the main Ractor; use +`Database#define_aggregator` or `Database#create_aggregate_handler` instead. +User-defined functions, collations, the authorizer, `trace`, and +`busy_handler` all work. Several +Ractors may open connections to the same database file; SQLite's own locking +coordinates them, just as it does for separate processes. + +```ruby +require "sqlite3" + +ractors = 4.times.map do |i| + Ractor.new(i) do |i| + db = SQLite3::Database.new("app.db") + db.busy_timeout = 5000 + db.execute("INSERT INTO log (worker) VALUES (?)", [i]) + db.close + end +end +ractors.each(&:value) # use Ractor#take on Ruby 3.4 +``` + +## Supported Ruby versions + +Ractor support requires Ruby 3.4 or later. SQLite3-Ruby needs +`rb_thread_lock_native_thread()` to run callbacks safely in non-main Ractors, +and only Ruby 3.4 and later have it. On earlier versions, +`SQLite3.ractor_safe?` returns false, and opening a database in a non-main +Ractor raises `Ractor::UnsafeError` ("ractor unsafe method called from not +main ractor"). + +## Database objects can't be shared between Ractors + +A `SQLite3::Database` can't be shared between Ractors. Sending one to another +Ractor with `Ractor#send` raises a `Ractor::Error`. + +```ruby +require "sqlite3" + +r = Ractor.new { Ractor.receive } + +db = SQLite3::Database.new ":memory:" + +begin + r.send db + puts "unreachable" +rescue Ractor::Error +end +``` + +This is true even for a connection opened in "Serialized" mode +(`SQLite3::Constants::Open::FULLMUTEX`). Serialized mode makes the SQLite +calls safe, but a `Database` also holds Ruby state: the procs and objects +registered as functions, aggregators, collations, and handlers. Those can't +be shared between Ractors. + +## Fork Safety + +⚠️ Forking a process that has opened writable connections in non-main Ractors is +not supported and may corrupt those databases. Only connections opened in the +main Ractor are closed in the forked child. diff --git a/ext/sqlite3/exception.c b/ext/sqlite3/exception.c index 40abbbb7..ec8c279a 100644 --- a/ext/sqlite3/exception.c +++ b/ext/sqlite3/exception.c @@ -1,4 +1,5 @@ #include +#include static VALUE status2klass(int status) @@ -158,6 +159,43 @@ current_callback_frames(void) return NIL_P(holder) ? NULL : (callback_frames_t *)RTYPEDDATA_DATA(holder); } +#ifdef HAVE_RB_THREAD_LOCK_NATIVE_THREAD +static ID id_native_thread_locked; + +static VALUE +mark_native_thread_locked(VALUE thread) +{ + return rb_thread_local_aset(thread, id_native_thread_locked, thread); +} + +/* Pin the current Ruby thread to its native thread before a callback runs any Ruby code. SQLite + * holds the connection mutex while it runs a callback, and only the native thread that locked the + * mutex can unlock it. In a non-main Ractor, the M:N scheduler can move a thread that blocks in a + * callback to another native thread, and the connection deadlocks. + * + * Each call to rb_thread_lock_native_thread increments a counter with no matching decrement, so + * call it once per fiber, and record the call in a fiber-local. Reading a fiber-local runs no Ruby + * code, and Ruby code here could block before the pin. The fiber-local holds the Thread itself, + * so a copy of it in another thread doesn't count. */ +static void +lock_native_thread(void) +{ + VALUE thread = rb_thread_current(); + int state; + + if (rb_thread_local_aref(thread, id_native_thread_locked) == thread) { return; } + + rb_thread_lock_native_thread(); + rb_protect(mark_native_thread_locked, thread, &state); + if (state) { rb_set_errinfo(Qnil); } +} +#else +static void +lock_native_thread(void) +{ +} +#endif + void rb_sqlite3_push_callback_frame(rb_sqlite3_callback_frame_t *frame) { @@ -191,6 +229,8 @@ rb_sqlite3_protect_callback(VALUE (*body)(VALUE), VALUE data, int *failed) VALUE result; int state; + lock_native_thread(); + if (frame && frame->state) { *failed = 1; return Qnil; @@ -214,4 +254,7 @@ void init_sqlite3_exception(void) { id_callback_frames = rb_intern("__sqlite3_callback_frames__"); +#ifdef HAVE_RB_THREAD_LOCK_NATIVE_THREAD + id_native_thread_locked = rb_intern("__sqlite3_native_thread_locked__"); +#endif } diff --git a/ext/sqlite3/extconf.rb b/ext/sqlite3/extconf.rb index 1fbefd73..96d8ad06 100644 --- a/ext/sqlite3/extconf.rb +++ b/ext/sqlite3/extconf.rb @@ -129,6 +129,9 @@ def configure_extension # Functions defined in 2.1 but not 2.0 have_func("rb_integer_pack") + # Functions defined in 3.4 but not 3.3 + have_func("rb_thread_lock_native_thread", "ruby/thread.h") + # These functions may not be defined have_func("sqlite3_initialize") have_func("sqlite3_backup_init") diff --git a/ext/sqlite3/sqlite3.c b/ext/sqlite3/sqlite3.c index 3e506d19..d15c53b4 100644 --- a/ext/sqlite3/sqlite3.c +++ b/ext/sqlite3/sqlite3.c @@ -98,6 +98,20 @@ threadsafe_p(VALUE UNUSED(klass)) return INT2NUM(sqlite3_threadsafe()); } +/* Returns true if the extension declared itself Ractor-safe. That needs SQLite compiled + * thread-safe and Ruby 3.4 or later, which has rb_thread_lock_native_thread (see + * lock_native_thread in exception.c). + */ +static VALUE +ractor_safe_p(VALUE UNUSED(klass)) +{ +#if defined(HAVE_RB_EXT_RACTOR_SAFE) && defined(HAVE_RB_THREAD_LOCK_NATIVE_THREAD) + return sqlite3_threadsafe() ? Qtrue : Qfalse; +#else + return Qfalse; +#endif +} + /* * call-seq: * status(parameter) → Hash @@ -140,6 +154,12 @@ init_sqlite3_constants(void) VALUE mSqlite3Constants; VALUE mSqlite3Open; +#ifdef HAVE_RB_EXT_RACTOR_SAFE + if (RTEST(ractor_safe_p(mSqlite3))) { + rb_ext_ractor_safe(true); + } +#endif + mSqlite3Constants = rb_define_module_under(mSqlite3, "Constants"); /* sqlite3_open_v2 flags for Database::new */ @@ -214,16 +234,17 @@ Init_sqlite3_native(void) rb_define_singleton_method(mSqlite3, "sqlcipher?", using_sqlcipher, 0); rb_define_singleton_method(mSqlite3, "libversion", libversion, 0); rb_define_singleton_method(mSqlite3, "threadsafe", threadsafe_p, 0); + rb_define_singleton_method(mSqlite3, "ractor_safe?", ractor_safe_p, 0); rb_define_singleton_method(mSqlite3, "status", rb_sqlite3_status, -1); /* (String) The version of the sqlite3 library compiled with (e.g., "3.46.1") */ - rb_define_const(mSqlite3, "SQLITE_VERSION", rb_str_new2(SQLITE_VERSION)); + rb_define_const(mSqlite3, "SQLITE_VERSION", rb_obj_freeze(rb_str_new2(SQLITE_VERSION))); /* (Integer) The version of the sqlite3 library compiled with (e.g., 346001) */ rb_define_const(mSqlite3, "SQLITE_VERSION_NUMBER", INT2FIX(SQLITE_VERSION_NUMBER)); /* (String) The version of the sqlite3 library loaded at runtime (e.g., "3.46.1") */ - rb_define_const(mSqlite3, "SQLITE_LOADED_VERSION", rb_str_new2(sqlite3_libversion())); + rb_define_const(mSqlite3, "SQLITE_LOADED_VERSION", rb_obj_freeze(rb_str_new2(sqlite3_libversion()))); #ifdef USING_PACKAGED_LIBRARIES rb_define_const(mSqlite3, "SQLITE_PACKAGED_LIBRARIES", Qtrue); diff --git a/ext/sqlite3/statement.c b/ext/sqlite3/statement.c index 258a78b4..67ede656 100644 --- a/ext/sqlite3/statement.c +++ b/ext/sqlite3/statement.c @@ -596,26 +596,24 @@ static VALUE stmt_stat_symbols[stmt_stat_sym_last]; static void setup_stmt_stat_symbols(void) { - if (stmt_stat_symbols[0] == 0) { #define S(s) stmt_stat_symbols[stmt_stat_sym_##s] = ID2SYM(rb_intern_const(#s)) - S(fullscan_steps); - S(sorts); - S(autoindexes); - S(vm_steps); + S(fullscan_steps); + S(sorts); + S(autoindexes); + S(vm_steps); #ifdef SQLITE_STMTSTATUS_REPREPARE - S(reprepares); + S(reprepares); #endif #ifdef SQLITE_STMTSTATUS_RUN - S(runs); + S(runs); #endif #ifdef SQLITE_STMTSTATUS_FILTER_MISS - S(filter_misses); + S(filter_misses); #endif #ifdef SQLITE_STMTSTATUS_FILTER_HIT - S(filter_hits); + S(filter_hits); #endif #undef S - } } static size_t @@ -623,8 +621,6 @@ stmt_stat_internal(VALUE hash_or_sym, sqlite3_stmt *stmt) { VALUE hash = Qnil, key = Qnil; - setup_stmt_stat_symbols(); - if (RB_TYPE_P(hash_or_sym, T_HASH)) { hash = hash_or_sym; } else if (SYMBOL_P(hash_or_sym)) { @@ -788,6 +784,9 @@ init_sqlite3_statement(void) { cSqlite3Statement = rb_define_class_under(mSqlite3, "Statement", rb_cObject); + /* populate eagerly: a lazy fill would race when the extension is used from multiple Ractors */ + setup_stmt_stat_symbols(); + rb_define_alloc_func(cSqlite3Statement, allocate); rb_define_method(cSqlite3Statement, "close", sqlite3_rb_close, 0); rb_define_method(cSqlite3Statement, "closed?", closed_p, 0); diff --git a/lib/sqlite3/database.rb b/lib/sqlite3/database.rb index 8939dd8e..1b5be1bc 100644 --- a/lib/sqlite3/database.rb +++ b/lib/sqlite3/database.rb @@ -180,7 +180,9 @@ def initialize file, options = {}, zvfs = nil initialize_extensions(options[:extensions]) - ForkSafety.track(self) + if !defined?(Ractor) || Ractor.main == Ractor.current + ForkSafety.track(self) + end if block_given? begin diff --git a/lib/sqlite3/version_info.rb b/lib/sqlite3/version_info.rb index cf869528..d2d5185e 100644 --- a/lib/sqlite3/version_info.rb +++ b/lib/sqlite3/version_info.rb @@ -1,10 +1,11 @@ module SQLite3 - # a hash of descriptive metadata about the current version of the sqlite3 gem + # a hash of descriptive metadata about the current version of the sqlite3 gem, + # deeply frozen so that it can be read from any Ractor VERSION_INFO = { ruby: RUBY_DESCRIPTION, gem: { - version: SQLite3::VERSION - }, + version: SQLite3::VERSION.freeze + }.freeze, sqlite: { compiled: SQLite3::SQLITE_VERSION, loaded: SQLite3::SQLITE_LOADED_VERSION, @@ -12,6 +13,6 @@ module SQLite3 precompiled: SQLite3::SQLITE_PRECOMPILED_LIBRARIES, sqlcipher: SQLite3.sqlcipher?, threadsafe: SQLite3.threadsafe? - } - } + }.freeze + }.freeze end diff --git a/test/test_integration_ractor.rb b/test/test_integration_ractor.rb new file mode 100644 index 00000000..47ad315b --- /dev/null +++ b/test/test_integration_ractor.rb @@ -0,0 +1,230 @@ +# frozen_string_literal: true + +require "helper" +require "open3" +require "rbconfig" + +# Each Ractor test runs in a child process. Once a process has started a +# Ractor, some process-wide behavior changes for good (for example, +# ObjectSpace.each_object stops seeing objects), which would leak into +# unrelated tests. A child process also lets us fail on a hang instead of +# stalling the suite. +class IntegrationRactorTestCase < SQLite3::TestCase + TIMEOUT = 60 + + PRELUDE = <<~RUBY + require "sqlite3" + Warning[:experimental] = false + def ractor_value(r) = r.respond_to?(:value) ? r.value : r.take + RUBY + + def setup + super + skip("Requires a Ractor-safe build") unless SQLite3.ractor_safe? + + # valgrind follows the child processes, and a Ruby process that has created a Ractor can't + # exit cleanly under it: with RUBY_FREE_AT_EXIT, Ruby 4.0.7 reads freed memory in + # ractor_free (RUBY_FREE_AT_EXIT=1 valgrind ruby -e 'Ractor.new { 1 }.value'); without it, + # everything Ruby allocated is reported as leaked. + skip("Ractor processes can't exit cleanly under valgrind") if i_am_running_in_valgrind + end + + def test_ractor_safe + assert_predicate SQLite3, :ractor_safe? + end + + def test_open_and_query_in_ractors + assert_ractor_script(<<~RUBY, "4950,4950,4950,4950") + ractors = 4.times.map do + Ractor.new do + db = SQLite3::Database.new(":memory:") + begin + db.execute "CREATE TABLE t (a INTEGER)" + db.transaction do + stmt = db.prepare("INSERT INTO t VALUES (?)") + 100.times { |n| stmt.execute(n) } + stmt.close + end + db.get_first_value("SELECT sum(a) FROM t") + ensure + db.close + end + end + end + print ractors.map { ractor_value(_1) }.join(",") + RUBY + end + + def test_callbacks_in_ractor + assert_ractor_script(<<~RUBY, "20,b,true,true") + class Summer + def initialize = @sum = 0 + def step(v) = @sum += v + def finalize = @sum + end + + class Reverse + def compare(a, b) = b <=> a + end + + r = Ractor.new do + db = SQLite3::Database.new(":memory:") + begin + traced = false + db.trace { traced = true } + db.authorizer = ->(*) { true } + db.create_function("double", 1) { |fn, x| fn.result = x * 2 } + db.define_aggregator("summer", Summer.new) + db.collation("rev", Reverse.new) + + db.execute "CREATE TABLE t (a INTEGER, s TEXT)" + db.execute "INSERT INTO t VALUES (1, 'a'), (2, 'b'), (3, 'a'), (4, 'b')" + stmt = db.prepare("SELECT s FROM t ORDER BY s COLLATE rev LIMIT 1") + first = stmt.execute.to_a.flatten.first + stats = stmt.stat.key?(:vm_steps) + stmt.close + + [db.get_first_value("SELECT summer(double(a)) FROM t"), first, stats, traced].join(",") + ensure + db.close + end + end + print ractor_value(r) + RUBY + end + + def test_callback_that_blocks_in_ractors + assert_ractor_script(<<~RUBY, "20,20") + ractors = 2.times.map do + Ractor.new do + db = SQLite3::Database.new(":memory:") + begin + db.create_function("nap", 0) { |fn| sleep 0.01; fn.result = 1 } + 20.times.sum { db.get_first_value("SELECT nap()") } + ensure + db.close + end + end + end + print ractors.map { ractor_value(_1) }.join(",") + RUBY + end + + def test_callback_that_blocks_outside_a_statement_in_ractors + skip("Requires sqlite3_serialize") unless SQLite3::Database.method_defined?(:serialize) + + assert_ractor_script(<<~RUBY, "20,20") + ractors = 2.times.map do + Ractor.new do + db = SQLite3::Database.new(":memory:") + begin + db.authorizer = ->(*) { sleep 0.01; true } + 20.times.count { db.serialize } + ensure + db.close + end + end + end + print ractors.map { ractor_value(_1) }.join(",") + RUBY + end + + def test_trace_point_that_blocks_in_ractor_callbacks + assert_ractor_script(<<~RUBY, "8") + ractors = 8.times.map do + Ractor.new do + db = SQLite3::Database.new(":memory:") + begin + db.create_function("one", 0) { |fn| fn.result = 1 } + TracePoint.new(:c_return) { sleep 0.001 }.enable do + db.get_first_value("SELECT one()") + end + ensure + db.close + end + end + end + print ractors.sum { ractor_value(_1) } + RUBY + end + + def test_callback_that_blocks_in_a_thread_with_copied_fiber_locals_in_ractors + assert_ractor_script(<<~RUBY, "20,20") + ractors = 2.times.map do + Ractor.new do + db = SQLite3::Database.new(":memory:") + begin + db.create_function("nap", 0) { |fn| sleep 0.01; fn.result = 1 } + db.get_first_value("SELECT nap()") + locals = Thread.current.keys.to_h { [_1, Thread.current[_1]] } + Thread.new do + locals.each { |key, value| Thread.current[key] = value } + 20.times.sum { db.get_first_value("SELECT nap()") } + end.value + ensure + db.close + end + end + end + print ractors.map { ractor_value(_1) }.join(",") + RUBY + end + + def test_exit_while_a_ractor_callback_blocks + assert_ractor_script(<<~RUBY, "exiting") + 2.times do + Ractor.new do + db = SQLite3::Database.new(":memory:") + db.trace do + Ractor.main.send(:started) + sleep + end + db.execute("SELECT 1") + end + end + 2.times { Ractor.receive } + print "exiting" + RUBY + end + + def test_version_info_is_shareable + assert_ractor_script(<<~RUBY, SQLite3::VERSION) + print ractor_value(Ractor.new { SQLite3::VERSION_INFO[:gem][:version] }) + RUBY + end + + def test_database_cannot_be_sent_to_another_ractor + assert_ractor_script(<<~RUBY, "Ractor::Error") + db = SQLite3::Database.new(":memory:") + r = Ractor.new { Ractor.receive } + begin + r.send(db) + print "sent" + rescue Ractor::Error => e + print e.class + end + RUBY + end + + private + + def assert_ractor_script(script, expected) + load_path = $LOAD_PATH.select { |dir| File.directory?(dir) }.flat_map { |dir| ["-I", dir] } + cmd = [RbConfig.ruby, *load_path, "-e", PRELUDE + script] + + Open3.popen3(*cmd) do |stdin, out, err, wait_thr| + stdin.close + out_reader = Thread.new { out.read } + err_reader = Thread.new { err.read } + + unless wait_thr.join(TIMEOUT) + Process.kill(:KILL, wait_thr.pid) + flunk "Ractor script did not finish within #{TIMEOUT}s:\n#{script}" + end + + output = out_reader.value + assert_predicate wait_thr.value, :success?, "Ractor script failed:\n#{output}#{err_reader.value}" + assert_equal expected, output + end + end +end diff --git a/test/test_sqlite3.rb b/test/test_sqlite3.rb index 364734b2..aabb3591 100644 --- a/test/test_sqlite3.rb +++ b/test/test_sqlite3.rb @@ -1,4 +1,6 @@ require "helper" +require "open3" +require "rbconfig" module SQLite3 class TestSQLite3 < SQLite3::TestCase @@ -18,6 +20,27 @@ def test_threadsafe? end end + def test_non_main_ractor_cannot_open_a_database_unless_ractor_safe + skip("Requires Ractor") unless defined?(Ractor) + skip("Requires a build that is not Ractor-safe") if SQLite3.ractor_safe? + skip("Ractor processes can't exit cleanly under valgrind") if i_am_running_in_valgrind + + load_path = $LOAD_PATH.flat_map { |dir| ["-I", dir] } + output, error, status = Open3.capture3(RbConfig.ruby, *load_path, "-e", <<~RUBY) + require "sqlite3" + Warning[:experimental] = false + r = Ractor.new { SQLite3::Database.new(":memory:") } + begin + r.respond_to?(:value) ? r.value : r.take + rescue Ractor::RemoteError => e + print e.cause.class + end + RUBY + + assert_predicate status, :success?, error + assert_equal "Ractor::UnsafeError", output + end + def test_compiled_version_and_loaded_version assert_equal(SQLite3::SQLITE_VERSION, SQLite3::SQLITE_LOADED_VERSION) end