From 958dbf41f3b8f00c18e2e01ae1c6c818e0738c1b Mon Sep 17 00:00:00 2001 From: Roman Sandler <5535625+sandlerr@users.noreply.github.com> Date: Mon, 26 Dec 2022 23:19:01 +1100 Subject: [PATCH 1/9] Initial Ractor support --- ext/sqlite3/extconf.rb | 3 ++ ext/sqlite3/sqlite3.c | 6 +++ lib/sqlite3.rb | 5 ++ sqlite3.gemspec | 1 + test/helper.rb | 1 + test/test_integration_ractor.rb | 87 +++++++++++++++++++++++++++++++++ 6 files changed, 103 insertions(+) create mode 100644 test/test_integration_ractor.rb diff --git a/ext/sqlite3/extconf.rb b/ext/sqlite3/extconf.rb index 733e22d5..9152f471 100644 --- a/ext/sqlite3/extconf.rb +++ b/ext/sqlite3/extconf.rb @@ -115,6 +115,9 @@ def configure_extension # Functions defined in 2.1 but not 2.0 have_func("rb_integer_pack") + # Functions defined in 3.0 but not 2.7 + have_func("rb_ext_ractor_safe") + # These functions may not be defined have_func("sqlite3_initialize") have_func("sqlite3_backup_init") diff --git a/ext/sqlite3/sqlite3.c b/ext/sqlite3/sqlite3.c index c0672a06..b7bdd729 100644 --- a/ext/sqlite3/sqlite3.c +++ b/ext/sqlite3/sqlite3.c @@ -91,6 +91,12 @@ init_sqlite3_constants(void) VALUE mSqlite3Constants; VALUE mSqlite3Open; +#ifdef HAVE_RB_EXT_RACTOR_SAFE + if (sqlite3_threadsafe()) { + rb_ext_ractor_safe(true); + } +#endif + mSqlite3Constants = rb_define_module_under(mSqlite3, "Constants"); /* sqlite3_open_v2 flags for Database::new */ diff --git a/lib/sqlite3.rb b/lib/sqlite3.rb index 790fd754..9fab21e7 100644 --- a/lib/sqlite3.rb +++ b/lib/sqlite3.rb @@ -14,4 +14,9 @@ module SQLite3 def self.threadsafe? threadsafe > 0 end + + # Is the gem's C extension marked as Ractor-safe? + def self.ractor_safe? + threadsafe? && !defined?(Ractor).nil? + end end diff --git a/sqlite3.gemspec b/sqlite3.gemspec index d0746454..183dcc2a 100644 --- a/sqlite3.gemspec +++ b/sqlite3.gemspec @@ -83,6 +83,7 @@ Gem::Specification.new do |s| "test/test_integration_aggregate.rb", "test/test_integration_open_close.rb", "test/test_integration_pending.rb", + "test/test_integration_ractor.rb", "test/test_integration_resultset.rb", "test/test_integration_statement.rb", "test/test_pragmas.rb", diff --git a/test/helper.rb b/test/helper.rb index 5ea6da11..930858b6 100644 --- a/test/helper.rb +++ b/test/helper.rb @@ -9,6 +9,7 @@ puts "info: sqlite version: #{SQLite3::SQLITE_VERSION}/#{SQLite3::SQLITE_LOADED_VERSION}" puts "info: sqlcipher?: #{SQLite3.sqlcipher?}" puts "info: threadsafe?: #{SQLite3.threadsafe?}" +puts "info: ractor_safe?: #{SQLite3.ractor_safe?}" module SQLite3 class TestCase < Minitest::Test diff --git a/test/test_integration_ractor.rb b/test/test_integration_ractor.rb new file mode 100644 index 00000000..e058029d --- /dev/null +++ b/test/test_integration_ractor.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +require "helper" +require "fileutils" + +class IntegrationRactorTestCase < SQLite3::TestCase + STRESS_DB_NAME = "stress.db" + + def setup + teardown + end + + def teardown + FileUtils.rm_rf(Dir.glob("#{STRESS_DB_NAME}*")) + end + + def test_ractor_safe + skip unless RUBY_VERSION >= "3.0" && SQLite3.threadsafe? + assert_predicate SQLite3, :ractor_safe? + end + + def test_ractor_share_database + skip("Requires Ruby with Ractors") unless SQLite3.ractor_safe? + + db_receiver = Ractor.new do + db = Ractor.receive + Ractor.yield db.object_id + begin + db.execute("create table test_table ( b integer primary key)") + raise "Should have raised an exception in db.execute()" + rescue => e + Ractor.yield e + end + end + db_creator = Ractor.new(db_receiver) do |db_receiver| + db = SQLite3::Database.open(":memory:") + Ractor.yield db.object_id + db_receiver.send(db) + sleep 0.1 + db.execute("create table test_table ( a integer primary key)") + end + first_oid = db_creator.take + second_oid = db_receiver.take + assert_not_equal first_oid, second_oid + ex = db_receiver.take + # For now, let's assert that you can't pass database connections around + # between different Ractors. Letting a live DB connection exist in two + # threads that are running concurrently might expose us to footguns and + # lead to data corruption, so we should avoid this possibility and wait + # until connections can be given away using `yield` or `send`. + assert_equal "prepare called on a closed database", ex.message + end + + def test_ractor_stress + skip("Requires Ruby with Ractors") unless SQLite3.ractor_safe? + + # Testing with a file instead of :memory: since it can be more realistic + # compared with real production use, and so discover problems that in- + # memory testing won't find. Trivial example: STRESS_DB_NAME needs to be + # frozen to pass into the Ractor, but :memory: might avoid that problem by + # using a literal string. + db = SQLite3::Database.open(STRESS_DB_NAME) + db.execute("PRAGMA journal_mode=WAL") # A little slow without this + db.execute("create table stress_test (a integer primary_key, b text)") + random = Random.new.freeze + ractors = (0..9).map do |ractor_number| + Ractor.new(random, ractor_number) do |random, ractor_number| + db_in_ractor = SQLite3::Database.open(STRESS_DB_NAME) + db_in_ractor.busy_handler do + sleep random.rand / 100 # Lots of busy errors happen with multiple concurrent writers + true + end + 100.times do |i| + db_in_ractor.execute("insert into stress_test(a, b) values (#{ractor_number * 100 + i}, '#{random.rand}')") + end + end + end + ractors.each { |r| r.take } + final_check = Ractor.new do + db_in_ractor = SQLite3::Database.open(STRESS_DB_NAME) + res = db_in_ractor.execute("select count(*) from stress_test") + Ractor.yield res + end + res = final_check.take + assert_equal 1000, res[0][0] + end +end From 6ccb27b405f50787e7a1a0c487549fec589c2d05 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Wed, 10 Jan 2024 16:21:39 -0500 Subject: [PATCH 2/9] test: handle Ruby 3.3 behavior around passing unshareable objects --- test/test_integration_ractor.rb | 43 ++++++++++++++------------------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/test/test_integration_ractor.rb b/test/test_integration_ractor.rb index e058029d..f4f4d5d0 100644 --- a/test/test_integration_ractor.rb +++ b/test/test_integration_ractor.rb @@ -22,33 +22,26 @@ def test_ractor_safe def test_ractor_share_database skip("Requires Ruby with Ractors") unless SQLite3.ractor_safe? - db_receiver = Ractor.new do - db = Ractor.receive - Ractor.yield db.object_id - begin - db.execute("create table test_table ( b integer primary key)") - raise "Should have raised an exception in db.execute()" - rescue => e - Ractor.yield e + db = SQLite3::Database.open(":memory:") + + if RUBY_VERSION >= "3.3" + # after ruby/ruby@ce47ee00 + ractor = Ractor.new do + Ractor.receive end + + assert_raises(Ractor::Error) { ractor.send(db) } + else + # before ruby/ruby@ce47ee00 T_DATA objects could be copied + ractor = Ractor.new do + local_db = Ractor.receive + Ractor.yield local_db.object_id + end + ractor.send(db) + copy_id = ractor.take + + assert_not_equal db.object_id, copy_id end - db_creator = Ractor.new(db_receiver) do |db_receiver| - db = SQLite3::Database.open(":memory:") - Ractor.yield db.object_id - db_receiver.send(db) - sleep 0.1 - db.execute("create table test_table ( a integer primary key)") - end - first_oid = db_creator.take - second_oid = db_receiver.take - assert_not_equal first_oid, second_oid - ex = db_receiver.take - # For now, let's assert that you can't pass database connections around - # between different Ractors. Letting a live DB connection exist in two - # threads that are running concurrently might expose us to footguns and - # lead to data corruption, so we should avoid this possibility and wait - # until connections can be given away using `yield` or `send`. - assert_equal "prepare called on a closed database", ex.message end def test_ractor_stress From 2b0fff51eccb34dd75d07e495b33042468590e47 Mon Sep 17 00:00:00 2001 From: Aaron Patterson Date: Wed, 26 Mar 2025 15:11:30 -0700 Subject: [PATCH 3/9] Allow databases to be shareable but only in full-mutex mode https://www.sqlite.org/threadsafe.html "Full mutex" mode is the only time when a SQLite database can be shared between multiple threads. Multi-thread mode only means that multiple threads can have their own instances of a database object that point at the same file, but multiple threads cannot access the object concurrently. --- ext/sqlite3/database.c | 4 +++ lib/sqlite3/database.rb | 2 +- test/test_integration_ractor.rb | 43 +++++++++------------------------ 3 files changed, 16 insertions(+), 33 deletions(-) diff --git a/ext/sqlite3/database.c b/ext/sqlite3/database.c index a35ff3a1..885aa3c7 100644 --- a/ext/sqlite3/database.c +++ b/ext/sqlite3/database.c @@ -158,6 +158,10 @@ rb_sqlite3_open_v2(VALUE self, VALUE file, VALUE mode, VALUE zvfs) ctx->flags |= SQLITE3_RB_DATABASE_READONLY; } + if (flags & SQLITE_OPEN_FULLMUTEX) { + FL_SET_RAW(self, RUBY_FL_SHAREABLE); + } + return self; } diff --git a/lib/sqlite3/database.rb b/lib/sqlite3/database.rb index efa91a95..e12e81c1 100644 --- a/lib/sqlite3/database.rb +++ b/lib/sqlite3/database.rb @@ -181,7 +181,7 @@ def initialize file, options = {}, zvfs = nil initialize_extensions(options[:extensions]) - ForkSafety.track(self) + ForkSafety.track(self) if Ractor.main? if block_given? begin diff --git a/test/test_integration_ractor.rb b/test/test_integration_ractor.rb index f4f4d5d0..cebd112b 100644 --- a/test/test_integration_ractor.rb +++ b/test/test_integration_ractor.rb @@ -44,37 +44,16 @@ def test_ractor_share_database end end - def test_ractor_stress - skip("Requires Ruby with Ractors") unless SQLite3.ractor_safe? - - # Testing with a file instead of :memory: since it can be more realistic - # compared with real production use, and so discover problems that in- - # memory testing won't find. Trivial example: STRESS_DB_NAME needs to be - # frozen to pass into the Ractor, but :memory: might avoid that problem by - # using a literal string. - db = SQLite3::Database.open(STRESS_DB_NAME) - db.execute("PRAGMA journal_mode=WAL") # A little slow without this - db.execute("create table stress_test (a integer primary_key, b text)") - random = Random.new.freeze - ractors = (0..9).map do |ractor_number| - Ractor.new(random, ractor_number) do |random, ractor_number| - db_in_ractor = SQLite3::Database.open(STRESS_DB_NAME) - db_in_ractor.busy_handler do - sleep random.rand / 100 # Lots of busy errors happen with multiple concurrent writers - true - end - 100.times do |i| - db_in_ractor.execute("insert into stress_test(a, b) values (#{ractor_number * 100 + i}, '#{random.rand}')") - end - end - end - ractors.each { |r| r.take } - final_check = Ractor.new do - db_in_ractor = SQLite3::Database.open(STRESS_DB_NAME) - res = db_in_ractor.execute("select count(*) from stress_test") - Ractor.yield res - end - res = final_check.take - assert_equal 1000, res[0][0] + def test_shareable_db + # databases are shareable between ractors, but only if they're opened + # in "full mutex" mode + db = SQLite3::Database.new ":memory:", + flags: SQLite3::Constants::Open::FULLMUTEX | + SQLite3::Constants::Open::READWRITE | + SQLite3::Constants::Open::CREATE + assert Ractor.shareable?(db) + + db = SQLite3::Database.new ":memory:" + refute Ractor.shareable?(db) end end From 46fcc4206972855abea4b16050749ee4d4ead5a4 Mon Sep 17 00:00:00 2001 From: Aaron Patterson Date: Wed, 26 Mar 2025 16:09:28 -0700 Subject: [PATCH 4/9] Fix cop? --- test/test_integration_ractor.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_integration_ractor.rb b/test/test_integration_ractor.rb index cebd112b..fdc5f23a 100644 --- a/test/test_integration_ractor.rb +++ b/test/test_integration_ractor.rb @@ -49,8 +49,8 @@ def test_shareable_db # in "full mutex" mode db = SQLite3::Database.new ":memory:", flags: SQLite3::Constants::Open::FULLMUTEX | - SQLite3::Constants::Open::READWRITE | - SQLite3::Constants::Open::CREATE + SQLite3::Constants::Open::READWRITE | + SQLite3::Constants::Open::CREATE assert Ractor.shareable?(db) db = SQLite3::Database.new ":memory:" From 9e64a7ad43ad2cd0118ff05919c33c27fa10778f Mon Sep 17 00:00:00 2001 From: Aaron Patterson Date: Wed, 26 Mar 2025 16:16:14 -0700 Subject: [PATCH 5/9] remove dead code --- test/test_integration_ractor.rb | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/test/test_integration_ractor.rb b/test/test_integration_ractor.rb index fdc5f23a..66d707de 100644 --- a/test/test_integration_ractor.rb +++ b/test/test_integration_ractor.rb @@ -4,16 +4,6 @@ require "fileutils" class IntegrationRactorTestCase < SQLite3::TestCase - STRESS_DB_NAME = "stress.db" - - def setup - teardown - end - - def teardown - FileUtils.rm_rf(Dir.glob("#{STRESS_DB_NAME}*")) - end - def test_ractor_safe skip unless RUBY_VERSION >= "3.0" && SQLite3.threadsafe? assert_predicate SQLite3, :ractor_safe? From c7a24ba7372775cb183d6519bb1c9e94060e6ef4 Mon Sep 17 00:00:00 2001 From: Aaron Patterson Date: Wed, 26 Mar 2025 16:49:06 -0700 Subject: [PATCH 6/9] not all versions of Ruby have Ractor.main? --- lib/sqlite3/database.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sqlite3/database.rb b/lib/sqlite3/database.rb index c9dc491b..a04338b5 100644 --- a/lib/sqlite3/database.rb +++ b/lib/sqlite3/database.rb @@ -183,7 +183,7 @@ def initialize file, options = {}, zvfs = nil initialize_extensions(options[:extensions]) - ForkSafety.track(self) if Ractor.main? + ForkSafety.track(self) if Ractor.main == Ractor.current if block_given? begin From 90dec0aeeaf44f7b62009a8286ad38e93eb889f6 Mon Sep 17 00:00:00 2001 From: Aaron Patterson Date: Wed, 26 Mar 2025 16:53:22 -0700 Subject: [PATCH 7/9] Check for Ractor constant TruffleRuby support --- lib/sqlite3/database.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sqlite3/database.rb b/lib/sqlite3/database.rb index a04338b5..4d3279e8 100644 --- a/lib/sqlite3/database.rb +++ b/lib/sqlite3/database.rb @@ -183,7 +183,7 @@ def initialize file, options = {}, zvfs = nil initialize_extensions(options[:extensions]) - ForkSafety.track(self) if Ractor.main == Ractor.current + ForkSafety.track(self) if !defined?(Ractor) || (Ractor.main == Ractor.current) if block_given? begin From 0d2ee9682c89da22fb9c02831b3d7a94d0df5cab Mon Sep 17 00:00:00 2001 From: Aaron Patterson Date: Wed, 26 Mar 2025 17:20:45 -0700 Subject: [PATCH 8/9] more TruffleRuby --- test/test_integration_ractor.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/test_integration_ractor.rb b/test/test_integration_ractor.rb index 66d707de..49dee789 100644 --- a/test/test_integration_ractor.rb +++ b/test/test_integration_ractor.rb @@ -35,6 +35,8 @@ def test_ractor_share_database end def test_shareable_db + skip unless defined?(Ractor) + # databases are shareable between ractors, but only if they're opened # in "full mutex" mode db = SQLite3::Database.new ":memory:", From 433c2d3174b9b9c57255e3f8975c71bdcc567890 Mon Sep 17 00:00:00 2001 From: Aaron Patterson Date: Thu, 27 Mar 2025 09:36:24 -0700 Subject: [PATCH 9/9] more truffle ruby --- test/test_integration_ractor.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_integration_ractor.rb b/test/test_integration_ractor.rb index 49dee789..009d386e 100644 --- a/test/test_integration_ractor.rb +++ b/test/test_integration_ractor.rb @@ -6,6 +6,7 @@ class IntegrationRactorTestCase < SQLite3::TestCase def test_ractor_safe skip unless RUBY_VERSION >= "3.0" && SQLite3.threadsafe? + skip unless defined?(Ractor) assert_predicate SQLite3, :ractor_safe? end