From 4f1ba5c39e9ce4da777cf695c97eb22e2cf2d077 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 19:49:42 +0000 Subject: [PATCH 01/27] Bump rdoc from 6.6.2 to 6.6.3.1 Bumps [rdoc](https://github.com/ruby/rdoc) from 6.6.2 to 6.6.3.1. - [Release notes](https://github.com/ruby/rdoc/releases) - [Changelog](https://github.com/ruby/rdoc/blob/master/History.rdoc) - [Commits](https://github.com/ruby/rdoc/compare/v6.6.2...v6.6.3.1) --- updated-dependencies: - dependency-name: rdoc dependency-type: indirect ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 0f0bf73..c1abb38 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -118,7 +118,7 @@ GEM zeitwerk (~> 2.6) rainbow (3.1.1) rake (13.1.0) - rdoc (6.6.2) + rdoc (6.6.3.1) psych (>= 4.0.0) regexp_parser (2.8.3) reline (0.4.1) From 33d1629f3bc12fdfff4d2de31141c6fa23098aaf Mon Sep 17 00:00:00 2001 From: Marek de Heus Date: Mon, 8 Apr 2024 11:19:57 +0200 Subject: [PATCH 02/27] Fix typo "and" should be "an" --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ca58640..24b3d3e 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,7 @@ Solid Cache supports these options in addition to the standard `ActiveSupport::C - `max_entries` - the maximum number of entries allowed in the cache (default: `nil`, meaning no limit) - `max_size` - the maximum size of the cache entries (default `nil`, meaning no limit) - `cluster` - a Hash of options for the cache database cluster, e.g `{ shards: [:database1, :database2, :database3] }` -- `clusters` - and Array of Hashes for multiple cache clusters (ignored if `:cluster` is set) +- `clusters` - an Array of Hashes for multiple cache clusters (ignored if `:cluster` is set) - `active_record_instrumentation` - whether to instrument the cache's queries (default: `true`) - `clear_with` - clear the cache with `:truncate` or `:delete` (default `truncate`, except for when `Rails.env.test?` then `delete`) - `max_key_bytesize` - the maximum size of a normalized key in bytes (default `1024`) From 03a7da4248432f514698f37b0285bf8c3ae61295 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Tue, 23 Apr 2024 14:37:29 +0100 Subject: [PATCH 03/27] Remove clusters The are unnecessarily complicated - let's just allow you to list shards. Multi-cluster setups will now raise an error. Setups with `cluster` or `clusters` (single cluster) will still work, but give a deprecation warning. --- README.md | 33 +---- Rakefile | 2 +- lib/solid_cache/cluster.rb | 18 --- lib/solid_cache/cluster/connections.rb | 55 ------- lib/solid_cache/store.rb | 6 +- lib/solid_cache/store/clusters.rb | 83 ----------- lib/solid_cache/store/connections.rb | 108 ++++++++++++++ lib/solid_cache/store/entries.rb | 8 +- .../{cluster => store}/execution.rb | 8 +- lib/solid_cache/{cluster => store}/expiry.rb | 2 +- lib/solid_cache/{cluster => store}/stats.rb | 4 +- test/dummy/config/solid_cache_cluster.yml | 14 -- .../config/solid_cache_cluster_inferred.yml | 12 -- .../config/solid_cache_clusters_named.yml | 19 --- test/dummy/config/solid_cache_connects_to.yml | 3 - ...he_clusters.yml => solid_cache_shards.yml} | 8 +- test/models/solid_cache/record_test.rb | 8 +- test/test_helper.rb | 22 +-- test/unit/clear_test.rb | 1 - test/unit/cluster_test.rb | 136 ------------------ test/unit/execution_test.rb | 6 +- test/unit/expiry_test.rb | 37 +++-- test/unit/query_cache_test.rb | 4 +- test/unit/rails_cache_test.rb | 2 +- test/unit/solid_cache_test.rb | 4 +- test/unit/stats_test.rb | 4 +- 26 files changed, 162 insertions(+), 445 deletions(-) delete mode 100644 lib/solid_cache/cluster.rb delete mode 100644 lib/solid_cache/cluster/connections.rb delete mode 100644 lib/solid_cache/store/clusters.rb create mode 100644 lib/solid_cache/store/connections.rb rename lib/solid_cache/{cluster => store}/execution.rb (90%) rename lib/solid_cache/{cluster => store}/expiry.rb (99%) rename lib/solid_cache/{cluster => store}/stats.rb (95%) delete mode 100644 test/dummy/config/solid_cache_cluster.yml delete mode 100644 test/dummy/config/solid_cache_cluster_inferred.yml delete mode 100644 test/dummy/config/solid_cache_clusters_named.yml rename test/dummy/config/{solid_cache_clusters.yml => solid_cache_shards.yml} (56%) delete mode 100644 test/unit/cluster_test.rb diff --git a/README.md b/README.md index 24b3d3e..a48728b 100644 --- a/README.md +++ b/README.md @@ -125,8 +125,9 @@ Solid Cache supports these options in addition to the standard `ActiveSupport::C - `max_age` - the maximum age of entries in the cache (default: `2.weeks.to_i`). Can be set to `nil`, but this is not recommended unless using `max_entries` to limit the size of the cache. - `max_entries` - the maximum number of entries allowed in the cache (default: `nil`, meaning no limit) - `max_size` - the maximum size of the cache entries (default `nil`, meaning no limit) -- `cluster` - a Hash of options for the cache database cluster, e.g `{ shards: [:database1, :database2, :database3] }` -- `clusters` - an Array of Hashes for multiple cache clusters (ignored if `:cluster` is set) +- `cluster` - (deprecated) a Hash of options for the cache database cluster, e.g `{ shards: [:database1, :database2, :database3] }` +- `clusters` - (deprecated) an Array of Hashes for multiple cache clusters (ignored if `:cluster` is set) +- `shards` - an Array or Hash of databases. When a Hash is supplied, the keys are the database names, and the values are the sharding key. - `active_record_instrumentation` - whether to instrument the cache's queries (default: `true`) - `clear_with` - clear the cache with `:truncate` or `:delete` (default `truncate`, except for when `Rails.env.test?` then `delete`) - `max_key_bytesize` - the maximum size of a normalized key in bytes (default `1024`) @@ -220,24 +221,6 @@ production: databases: [cache_shard1, cache_shard2, cache_shard3] ``` -### Secondary cache clusters - -You can add secondary cache clusters. Reads will only be sent to the primary cluster (i.e. the first one listed). - -Writes will go to all clusters. The writes to the primary cluster are synchronous, but asynchronous to the secondary clusters. - -To specific multiple clusters you can do: - -```yaml -# config/solid_cache.yml -production: - databases: [cache_primary_shard1, cache_primary_shard2, cache_secondary_shard1, cache_secondary_shard2] - store_options: - clusters: - - shards: [cache_primary_shard1, cache_primary_shard2] - - shards: [cache_secondary_shard1, cache_secondary_shard2] -``` - ### Named shard destinations By default, the node key used for sharding is the name of the database in `database.yml`. @@ -248,13 +231,9 @@ It is possible to add names for the shards in the cluster config. This will allo production: databases: [cache_primary_shard1, cache_primary_shard2, cache_secondary_shard1, cache_secondary_shard2] store_options: - clusters: - - shards: - cache_primary_shard1: node1 - cache_primary_shard2: node2 - - shards: - cache_secondary_shard1: node3 - cache_secondary_shard2: node4 + shards: + cache_primary_shard1: node1 + cache_primary_shard2: node2 ``` ### Enabling encryption diff --git a/Rakefile b/Rakefile index da46064..167e23c 100644 --- a/Rakefile +++ b/Rakefile @@ -23,7 +23,7 @@ def run_without_aborting(*tasks) end def configs - [ :default, :cluster, :cluster_inferred, :clusters, :clusters_named, :database, :no_database ] + [ :default, :connects_to, :database, :no_database, :shards ] end task :test do diff --git a/lib/solid_cache/cluster.rb b/lib/solid_cache/cluster.rb deleted file mode 100644 index 30b8eec..0000000 --- a/lib/solid_cache/cluster.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -module SolidCache - class Cluster - include Connections, Execution, Expiry, Stats - - attr_reader :error_handler - - def initialize(options = {}) - @error_handler = options[:error_handler] - super(options) - end - - def setup! - super - end - end -end diff --git a/lib/solid_cache/cluster/connections.rb b/lib/solid_cache/cluster/connections.rb deleted file mode 100644 index 03bcfa1..0000000 --- a/lib/solid_cache/cluster/connections.rb +++ /dev/null @@ -1,55 +0,0 @@ -# frozen_string_literal: true - -module SolidCache - class Cluster - module Connections - attr_reader :shard_options - - def initialize(options = {}) - super(options) - @shard_options = options.fetch(:shards, nil) - - if [ Hash, Array, NilClass ].none? { |klass| @shard_options.is_a? klass } - raise ArgumentError, "`shards` is a `#{@shard_options.class.name}`, it should be one of Array, Hash or nil" - end - end - - def with_each_connection(async: false, &block) - return enum_for(:with_each_connection) unless block_given? - - connections.with_each do - execute(async, &block) - end - end - - def with_connection_for(key, async: false, &block) - connections.with_connection_for(key) do - execute(async, &block) - end - end - - def with_connection(name, async: false, &block) - connections.with(name) do - execute(async, &block) - end - end - - def group_by_connection(keys) - connections.assign(keys) - end - - def connection_names - connections.names - end - - def connections - @connections ||= SolidCache::Connections.from_config(@shard_options) - end - - private - def setup! - connections - end - end - end -end diff --git a/lib/solid_cache/store.rb b/lib/solid_cache/store.rb index 2c5d457..ebb865e 100644 --- a/lib/solid_cache/store.rb +++ b/lib/solid_cache/store.rb @@ -2,7 +2,7 @@ module SolidCache class Store < ActiveSupport::Cache::Store - include Api, Clusters, Entries, Failsafe + include Api, Connections, Entries, Execution, Expiry, Failsafe, Stats prepend ActiveSupport::Cache::Strategy::LocalCache def initialize(options = {}) @@ -16,9 +16,5 @@ def self.supports_cache_versioning? def setup! super end - - def stats - primary_cluster.stats - end end end diff --git a/lib/solid_cache/store/clusters.rb b/lib/solid_cache/store/clusters.rb deleted file mode 100644 index 38b2339..0000000 --- a/lib/solid_cache/store/clusters.rb +++ /dev/null @@ -1,83 +0,0 @@ -# frozen_string_literal: true - -module SolidCache - class Store - module Clusters - attr_reader :primary_cluster, :clusters - - def initialize(options = {}) - super(options) - - clusters_options = options.fetch(:clusters) { [ options.fetch(:cluster, {}) ] } - - @clusters = clusters_options.map.with_index do |cluster_options, index| - Cluster.new(options.merge(cluster_options).merge(async_writes: index != 0, error_handler: error_handler)) - end - - @primary_cluster = clusters.first - end - - def setup! - clusters.each(&:setup!) - end - - private - def reading_key(key, failsafe:, failsafe_returning: nil, &block) - failsafe(failsafe, returning: failsafe_returning) do - primary_cluster.with_connection_for(key, &block) - end - end - - def reading_keys(keys, failsafe:, failsafe_returning: nil) - connection_keys = primary_cluster.group_by_connection(keys) - - connection_keys.map do |connection, keys| - failsafe(failsafe, returning: failsafe_returning) do - primary_cluster.with_connection(connection) do - yield keys - end - end - end - end - - - def writing_key(key, failsafe:, failsafe_returning: nil) - first_cluster_sync_rest_async do |cluster, async| - failsafe(failsafe, returning: failsafe_returning) do - cluster.with_connection_for(key, async: async) do - yield cluster - end - end - end - end - - def writing_keys(entries, failsafe:, failsafe_returning: nil) - first_cluster_sync_rest_async do |cluster, async| - connection_entries = cluster.group_by_connection(entries) - - connection_entries.map do |connection, entries| - failsafe(failsafe, returning: failsafe_returning) do - cluster.with_connection(connection, async: async) do - yield cluster, entries - end - end - end - end - end - - def writing_all(failsafe:, failsafe_returning: nil, &block) - first_cluster_sync_rest_async do |cluster, async| - cluster.connection_names.map do |connection| - failsafe(failsafe, returning: failsafe_returning) do - cluster.with_connection(connection, async: async, &block) - end - end - end.first - end - - def first_cluster_sync_rest_async - clusters.map.with_index { |cluster, index| yield cluster, index != 0 }.first - end - end - end -end diff --git a/lib/solid_cache/store/connections.rb b/lib/solid_cache/store/connections.rb new file mode 100644 index 0000000..5f8adb3 --- /dev/null +++ b/lib/solid_cache/store/connections.rb @@ -0,0 +1,108 @@ +# frozen_string_literal: true + +module SolidCache + class Store + module Connections + attr_reader :shard_options + + def initialize(options = {}) + super(options) + if options[:clusters].present? + if options[:clusters].size > 1 + raise ArgumentError, "Multiple clusters are no longer supported" + else + ActiveSupport.deprecator.warn(":clusters is deprecated, use :shards instead.") + end + @shard_options = options.fetch(:clusters).first[:shards] + elsif options[:cluster].present? + ActiveSupport.deprecator.warn(":cluster is deprecated, use :shards instead.") + @shard_options = options.fetch(:cluster, {})[:shards] + else + @shard_options = options.fetch(:shards, nil) + end + + if [ Hash, Array, NilClass ].none? { |klass| @shard_options.is_a? klass } + raise ArgumentError, "`shards` is a `#{@shard_options.class.name}`, it should be one of Array, Hash or nil" + end + end + + def with_each_connection(async: false, &block) + return enum_for(:with_each_connection) unless block_given? + + connections.with_each do + execute(async, &block) + end + end + + def with_connection_for(key, async: false, &block) + connections.with_connection_for(key) do + execute(async, &block) + end + end + + def with_connection(name, async: false, &block) + connections.with(name) do + execute(async, &block) + end + end + + def group_by_connection(keys) + connections.assign(keys) + end + + def connection_names + connections.names + end + + def connections + @connections ||= SolidCache::Connections.from_config(@shard_options) + end + + private + def setup! + connections + end + + def reading_key(key, failsafe:, failsafe_returning: nil, &block) + failsafe(failsafe, returning: failsafe_returning) do + with_connection_for(key, &block) + end + end + + def reading_keys(keys, failsafe:, failsafe_returning: nil) + group_by_connection(keys).map do |connection, keys| + failsafe(failsafe, returning: failsafe_returning) do + with_connection(connection) do + yield keys + end + end + end + end + + + def writing_key(key, failsafe:, failsafe_returning: nil, &block) + failsafe(failsafe, returning: failsafe_returning) do + with_connection_for(key, &block) + end + end + + def writing_keys(entries, failsafe:, failsafe_returning: nil) + group_by_connection(entries).map do |connection, entries| + failsafe(failsafe, returning: failsafe_returning) do + with_connection(connection) do + yield entries + end + end + end + end + + def writing_all(failsafe:, failsafe_returning: nil, &block) + connection_names.map do |connection| + failsafe(failsafe, returning: failsafe_returning) do + with_connection(connection, &block) + end + end.first + end + end + end +end diff --git a/lib/solid_cache/store/entries.rb b/lib/solid_cache/store/entries.rb index f91dca1..d3edf63 100644 --- a/lib/solid_cache/store/entries.rb +++ b/lib/solid_cache/store/entries.rb @@ -46,17 +46,17 @@ def entry_read_multi(keys) end def entry_write(key, payload) - writing_key(key, failsafe: :write_entry, failsafe_returning: nil) do |cluster| + writing_key(key, failsafe: :write_entry, failsafe_returning: nil) do Entry.write(key, payload) - cluster.track_writes(1) + track_writes(1) true end end def entry_write_multi(entries) - writing_keys(entries, failsafe: :write_multi_entries, failsafe_returning: false) do |cluster, entries| + writing_keys(entries, failsafe: :write_multi_entries, failsafe_returning: false) do |entries| Entry.write_multi(entries) - cluster.track_writes(entries.count) + track_writes(entries.count) true end end diff --git a/lib/solid_cache/cluster/execution.rb b/lib/solid_cache/store/execution.rb similarity index 90% rename from lib/solid_cache/cluster/execution.rb rename to lib/solid_cache/store/execution.rb index 1b3a3c7..01457ce 100644 --- a/lib/solid_cache/cluster/execution.rb +++ b/lib/solid_cache/store/execution.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module SolidCache - class Cluster + class Store module Execution def initialize(options = {}) super(options) @@ -16,7 +16,7 @@ def async(&block) @background << ->() do wrap_in_rails_executor do connections.with(current_shard) do - instrument(&block) + setup_instrumentation(&block) end end rescue Exception => exception @@ -28,7 +28,7 @@ def execute(async, &block) if async async(&block) else - instrument(&block) + setup_instrumentation(&block) end end @@ -44,7 +44,7 @@ def active_record_instrumentation? @active_record_instrumentation end - def instrument(&block) + def setup_instrumentation(&block) if active_record_instrumentation? block.call else diff --git a/lib/solid_cache/cluster/expiry.rb b/lib/solid_cache/store/expiry.rb similarity index 99% rename from lib/solid_cache/cluster/expiry.rb rename to lib/solid_cache/store/expiry.rb index 6a00855..ae40912 100644 --- a/lib/solid_cache/cluster/expiry.rb +++ b/lib/solid_cache/store/expiry.rb @@ -3,7 +3,7 @@ require "concurrent/atomic/atomic_fixnum" module SolidCache - class Cluster + class Store module Expiry # For every write that we do, we attempt to delete EXPIRY_MULTIPLIER times as many records. # This ensures there is downward pressure on the cache size while there is valid data to delete diff --git a/lib/solid_cache/cluster/stats.rb b/lib/solid_cache/store/stats.rb similarity index 95% rename from lib/solid_cache/cluster/stats.rb rename to lib/solid_cache/store/stats.rb index 16f0059..7c64fad 100644 --- a/lib/solid_cache/cluster/stats.rb +++ b/lib/solid_cache/store/stats.rb @@ -1,10 +1,10 @@ # frozen_string_literal: true module SolidCache - class Cluster + class Store module Stats def initialize(options = {}) - super() + super(options) end def stats diff --git a/test/dummy/config/solid_cache_cluster.yml b/test/dummy/config/solid_cache_cluster.yml deleted file mode 100644 index bb6607d..0000000 --- a/test/dummy/config/solid_cache_cluster.yml +++ /dev/null @@ -1,14 +0,0 @@ -default: &default - databases: [primary_shard_one, primary_shard_two] - - store_options: - max_age: 3600 - max_size: - cluster: - shards: [primary_shard_one, primary_shard_two] - -development: - <<: *default - -test: - <<: *default diff --git a/test/dummy/config/solid_cache_cluster_inferred.yml b/test/dummy/config/solid_cache_cluster_inferred.yml deleted file mode 100644 index bca21ba..0000000 --- a/test/dummy/config/solid_cache_cluster_inferred.yml +++ /dev/null @@ -1,12 +0,0 @@ -default: &default - databases: [primary_shard_one, primary_shard_two] - - store_options: - max_age: 3600 - max_size: - -development: - <<: *default - -test: - <<: *default diff --git a/test/dummy/config/solid_cache_clusters_named.yml b/test/dummy/config/solid_cache_clusters_named.yml deleted file mode 100644 index 33a8dc5..0000000 --- a/test/dummy/config/solid_cache_clusters_named.yml +++ /dev/null @@ -1,19 +0,0 @@ -default: &default - databases: [primary_shard_one, primary_shard_two, secondary_shard_one, secondary_shard_two] - - store_options: - max_age: 3600 - max_size: - clusters: - - shards: - primary_shard_one: node1 - primary_shard_two: node2 - - shards: - secondary_shard_one: node3 - secondary_shard_two: node4 - -development: - <<: *default - -test: - <<: *default diff --git a/test/dummy/config/solid_cache_connects_to.yml b/test/dummy/config/solid_cache_connects_to.yml index 3fc9e29..7b5f279 100644 --- a/test/dummy/config/solid_cache_connects_to.yml +++ b/test/dummy/config/solid_cache_connects_to.yml @@ -2,9 +2,6 @@ default: &default connects_to: shards: - default: - writing: :primary - reading: :primary_replica primary_shard_one: writing: :primary_shard_one primary_shard_two: diff --git a/test/dummy/config/solid_cache_clusters.yml b/test/dummy/config/solid_cache_shards.yml similarity index 56% rename from test/dummy/config/solid_cache_clusters.yml rename to test/dummy/config/solid_cache_shards.yml index e149870..6820ed4 100644 --- a/test/dummy/config/solid_cache_clusters.yml +++ b/test/dummy/config/solid_cache_shards.yml @@ -4,13 +4,7 @@ default: &default store_options: max_age: 3600 max_size: - clusters: - - shards: - - primary_shard_one - - primary_shard_two - - shards: - - secondary_shard_one - - secondary_shard_two + shards: [primary_shard_one, primary_shard_two, secondary_shard_one, secondary_shard_two] development: <<: *default diff --git a/test/models/solid_cache/record_test.rb b/test/models/solid_cache/record_test.rb index c89e3e3..d426374 100644 --- a/test/models/solid_cache/record_test.rb +++ b/test/models/solid_cache/record_test.rb @@ -9,10 +9,8 @@ class RecordTest < ActiveSupport::TestCase case ENV["SOLID_CACHE_CONFIG"] when "config/solid_cache_no_database.yml", "config/solid_cache_database.yml" assert_equal [ :default ], shards - when "config/solid_cache_clusters.yml", "config/solid_cache_clusters_named.yml", nil + when "config/solid_cache_connects_to.yml", "config/solid_cache_shards.yml", nil assert_equal [ :primary_shard_one, :primary_shard_two, :secondary_shard_one, :secondary_shard_two ], shards - when "config/solid_cache_cluster.yml", "config/solid_cache_cluster_inferred.yml" - assert_equal [ :primary_shard_one, :primary_shard_two ], shards else raise "Unknown SOLID_CACHE_CONFIG: #{ENV["SOLID_CACHE_CONFIG"]}" end @@ -23,10 +21,8 @@ class RecordTest < ActiveSupport::TestCase case ENV["SOLID_CACHE_CONFIG"] when "config/solid_cache_no_database.yml", "config/solid_cache_database.yml" assert_equal [ :reading ], role - when "config/solid_cache_clusters.yml", "config/solid_cache_clusters_named.yml", nil + when "config/solid_cache_connects_to.yml", "config/solid_cache_shards.yml", nil assert_equal [ :writing, :writing, :writing, :writing ], role - when "config/solid_cache_cluster.yml", "config/solid_cache_cluster_inferred.yml" - assert_equal [ :writing, :writing ], role else raise "Unknown SOLID_CACHE_CONFIG: #{ENV["SOLID_CACHE_CONFIG"]}" end diff --git a/test/test_helper.rb b/test/test_helper.rb index 2028a21..faa71be 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -49,7 +49,7 @@ def cleanup_stores end def send_entries_back_in_time(distance) - @cache.primary_cluster.with_each_connection do + @cache.with_each_connection do SolidCache::Entry.uncached do SolidCache::Entry.all.each do |entry| entry.update_columns(created_at: entry.created_at - distance) @@ -60,14 +60,12 @@ def send_entries_back_in_time(distance) def wait_for_background_tasks(cache, timeout: 2) timeout_at = Time.now + timeout - threadpools = cache.clusters.map { |cluster| cluster.instance_variable_get("@background") } + threadpool = cache.instance_variable_get("@background") - threadpools.each do |threadpool| - loop do - break if threadpool.completed_task_count == threadpool.scheduled_task_count - raise "Timeout waiting for cache background tasks" if Time.now > timeout_at - sleep 0.001 - end + loop do + break if threadpool.completed_task_count == threadpool.scheduled_task_count + raise "Timeout waiting for cache background tasks" if Time.now > timeout_at + sleep 0.001 end end @@ -86,12 +84,4 @@ def second_shard_key def single_database? [ "config/solid_cache_database.yml", "config/solid_cache_no_database.yml" ].include?(ENV["SOLID_CACHE_CONFIG"]) end - - def multi_cluster? - self.class.multi_cluster? - end - - def self.multi_cluster? - [ "config/solid_cache_clusters.yml", "config/solid_cache_clusters_named.yml" ].include?(ENV["SOLID_CACHE_CONFIG"]) - end end diff --git a/test/unit/clear_test.rb b/test/unit/clear_test.rb index 2a52822..c14675e 100644 --- a/test/unit/clear_test.rb +++ b/test/unit/clear_test.rb @@ -5,7 +5,6 @@ class ClearTest < ActiveSupport::TestCase setup do @namespace = "test-#{SecureRandom.hex}" - skip if multi_cluster? end test "clear by truncation" do diff --git a/test/unit/cluster_test.rb b/test/unit/cluster_test.rb deleted file mode 100644 index c528f01..0000000 --- a/test/unit/cluster_test.rb +++ /dev/null @@ -1,136 +0,0 @@ -# frozen_string_literal: true - -require "test_helper" - -class ClusterTest < ActiveSupport::TestCase - setup do - skip unless multi_cluster? - - @cache = nil - @namespace = "test-#{SecureRandom.hex}" - - @cache = lookup_store(expires_in: 60) - p - @primary_cache = lookup_store(expires_in: 60, clusters: [{ shards: @cache.clusters.first.shard_options }]) - @secondary_cache = lookup_store(expires_in: 60, clusters: [{ shards: @cache.clusters.second.shard_options }]) - end - - teardown do - wait_for_background_tasks(@cache) if @cache - end - - test "writes to both clusters" do - @cache.write("foo", 1) - wait_for_background_tasks(@cache) - assert_equal 1, @cache.read("foo") - assert_equal 1, @primary_cache.read("foo") - assert_equal 1, @secondary_cache.read("foo") - end - - test "reads from primary cluster" do - @cache.write("foo", 1) - wait_for_background_tasks(@cache) - assert_equal 1, @cache.read("foo") - - @secondary_cache.delete("foo") - assert_equal 1, @cache.read("foo") - - @primary_cache.delete("foo") - assert_nil @cache.read("foo") - end - - test "fetch writes to both clusters" do - @cache.fetch("foo") { 1 } - wait_for_background_tasks(@cache) - - assert_equal 1, @cache.read("foo") - assert_equal 1, @primary_cache.read("foo") - assert_equal 1, @secondary_cache.read("foo") - end - - test "fetch reads from primary clusters" do - @cache.fetch("foo") { 1 } - wait_for_background_tasks(@cache) - assert_equal 1, @cache.read("foo") - - @primary_cache.delete("foo") - @cache.fetch("foo") { 2 } - wait_for_background_tasks(@cache) - - assert_equal 2, @cache.read("foo") - assert_equal 2, @primary_cache.read("foo") - assert_equal 2, @secondary_cache.read("foo") - - @secondary_cache.delete("foo") - assert_equal 2, @cache.fetch("foo") { 3 } - - assert_equal 2, @primary_cache.read("foo") - assert_nil @secondary_cache.read("foo") - end - - test "deletes from both cluster" do - @cache.write("foo", 1) - wait_for_background_tasks(@cache) - assert_equal 1, @cache.read("foo") - - @cache.delete("foo") - wait_for_background_tasks(@cache) - - assert_nil @cache.read("foo") - assert_nil @primary_cache.read("foo") - assert_nil @secondary_cache.read("foo") - end - - test "multi_writes to both clusters" do - values = { "foo" => "bar", "egg" => "spam" } - @cache.write_multi(values) - wait_for_background_tasks(@cache) - assert_equal values, @cache.read_multi("foo", "egg") - assert_equal values, @primary_cache.read_multi("foo", "egg") - assert_equal values, @secondary_cache.read_multi("foo", "egg") - end - - test "increment and decrement hit both clusters" do - @cache.write("foo", 1, raw: true) - wait_for_background_tasks(@cache) - - assert_equal 1, @cache.read("foo", raw: true).to_i - assert_equal 1, @primary_cache.read("foo", raw: true).to_i - assert_equal 1, @secondary_cache.read("foo", raw: true).to_i - - @cache.increment("foo") - wait_for_background_tasks(@cache) - - assert_equal 2, @cache.read("foo", raw: true).to_i - assert_equal 2, @primary_cache.read("foo", raw: true).to_i - assert_equal 2, @secondary_cache.read("foo", raw: true).to_i - - @secondary_cache.write("foo", 4, raw: true) - - @cache.decrement("foo") - wait_for_background_tasks(@cache) - - assert_equal 1, @cache.read("foo", raw: true).to_i - assert_equal 1, @primary_cache.read("foo", raw: true).to_i - assert_equal 3, @secondary_cache.read("foo", raw: true).to_i - end - - test "cache with node names" do - @namespace = "test-#{SecureRandom.hex}" - primary_cluster = { shards: { primary_shard_one: :node1, primary_shard_two: :node2 } } - secondary_cluster = { shards: { secondary_shard_one: :node3, secondary_shard_two: :node4 } } - - @cache = lookup_store(expires_in: 60, clusters: [ primary_cluster, secondary_cluster ]) - @primary_cache = lookup_store(expires_in: 60, clusters: [ primary_cluster ]) - @secondary_cache = lookup_store(expires_in: 60, clusters: [ secondary_cluster ]) - - @cache.write("foo", 1) - wait_for_background_tasks(@cache) - assert_equal 1, @cache.read("foo") - assert_equal 1, @primary_cache.read("foo") - assert_equal 1, @secondary_cache.read("foo") - - assert_equal [ :node1, :node2 ], @cache.primary_cluster.connections.consistent_hash.nodes - assert_equal [ :node3, :node4 ], @cache.clusters[1].connections.consistent_hash.nodes - end -end diff --git a/test/unit/execution_test.rb b/test/unit/execution_test.rb index 380d65f..590f2aa 100644 --- a/test/unit/execution_test.rb +++ b/test/unit/execution_test.rb @@ -17,7 +17,7 @@ def test_async_errors_are_reported error_subscriber = ErrorSubscriber.new Rails.error.subscribe(error_subscriber) - @cache.primary_cluster.send(:async) do + @cache.send(:async) do raise "Boom!" end @@ -106,8 +106,6 @@ def test_active_record_instrumention_expiry end def test_no_connections_uninstrumented - skip if multi_cluster? - ActiveRecord::ConnectionAdapters::ConnectionPool.any_instance.stubs(connection).raises(ActiveRecord::StatementInvalid) cache = lookup_store(expires_in: 60, active_record_instrumentation: false) @@ -122,8 +120,6 @@ def test_no_connections_uninstrumented end def test_no_connections_instrumented - skip if multi_cluster? - ActiveRecord::ConnectionAdapters::ConnectionPool.any_instance.stubs(connection).raises(ActiveRecord::StatementInvalid) cache = lookup_store(expires_in: 60) diff --git a/test/unit/expiry_test.rb b/test/unit/expiry_test.rb index 6313132..d713a05 100644 --- a/test/unit/expiry_test.rb +++ b/test/unit/expiry_test.rb @@ -10,7 +10,6 @@ class SolidCache::ExpiryTest < ActiveSupport::TestCase setup do @namespace = "test-#{SecureRandom.hex}" @single_shard_cluster = single_database? ? {} : { shards: [ first_shard_key ] } - skip if multi_cluster? end teardown do @@ -19,7 +18,7 @@ class SolidCache::ExpiryTest < ActiveSupport::TestCase [ :thread, :job ].each do |expiry_method| test "expires old records (#{expiry_method})" do - SolidCache::Cluster.any_instance.stubs(:rand).returns(0) + SolidCache::Store.any_instance.stubs(:rand).returns(0) @cache = lookup_store(expiry_batch_size: 3, max_age: 2.weeks, expiry_method: expiry_method) default_shard_keys = shard_keys(@cache, first_shard_key) @@ -43,7 +42,7 @@ class SolidCache::ExpiryTest < ActiveSupport::TestCase end test "expires records when the cache is full (#{expiry_method})" do - SolidCache::Cluster.any_instance.stubs(:rand).returns(0) + SolidCache::Store.any_instance.stubs(:rand).returns(0) @cache = lookup_store(expiry_batch_size: 3, max_age: nil, max_entries: 2, expiry_method: expiry_method) default_shard_keys = shard_keys(@cache, first_shard_key) @@ -63,7 +62,7 @@ class SolidCache::ExpiryTest < ActiveSupport::TestCase end test "expires records when the cache is full via max_size (#{expiry_method})" do - SolidCache::Cluster.any_instance.stubs(:rand).returns(0) + SolidCache::Store.any_instance.stubs(:rand).returns(0) @cache = lookup_store(expiry_batch_size: 3, max_age: nil, max_size: 1000, expiry_method: expiry_method) default_shard_keys = shard_keys(@cache, first_shard_key) @@ -83,9 +82,9 @@ class SolidCache::ExpiryTest < ActiveSupport::TestCase end test "expires records no shards (#{expiry_method})" do - SolidCache::Cluster.any_instance.stubs(:rand).returns(0) + SolidCache::Store.any_instance.stubs(:rand).returns(0) - @cache = ActiveSupport::Cache.lookup_store(:solid_cache_store, expiry_batch_size: 3, namespace: @namespace, max_entries: 2, expiry_method: expiry_method, clusters: [ @single_shard_cluster ]) + @cache = ActiveSupport::Cache.lookup_store(:solid_cache_store, expiry_batch_size: 3, namespace: @namespace, max_entries: 2, expiry_method: expiry_method, **@single_shard_cluster) default_shard_keys = shard_keys(@cache, first_shard_key) @cache.write(default_shard_keys[0], 1) @@ -104,7 +103,7 @@ class SolidCache::ExpiryTest < ActiveSupport::TestCase end test "expires when random number is below threshold (#{expiry_method})" do - SolidCache::Cluster.any_instance.stubs(:rand).returns(0.416) + SolidCache::Store.any_instance.stubs(:rand).returns(0.416) @cache = ActiveSupport::Cache.lookup_store(:solid_cache_store, expiry_batch_size: 3, namespace: @namespace, max_entries: 1, expiry_method: expiry_method) default_shard_keys = shard_keys(@cache, first_shard_key) @@ -119,7 +118,7 @@ class SolidCache::ExpiryTest < ActiveSupport::TestCase end test "doesn't expire when random number is above threshold (#{expiry_method})" do - SolidCache::Cluster.any_instance.stubs(:rand).returns(0.417) + SolidCache::Store.any_instance.stubs(:rand).returns(0.417) @cache = ActiveSupport::Cache.lookup_store(:solid_cache_store, expiry_batch_size: 6, namespace: @namespace, max_entries: 1, expiry_method: expiry_method) default_shard_keys = shard_keys(@cache, first_shard_key) @@ -136,8 +135,8 @@ class SolidCache::ExpiryTest < ActiveSupport::TestCase test "expires old records multiple shards (#{expiry_method})" do skip if single_database? - SolidCache::Cluster.any_instance.stubs(:rand).returns(0, 1, 0, 1, 0, 1, 0, 1) - @cache = lookup_store(expiry_batch_size: 2, clusters: [ { shards: [ first_shard_key, second_shard_key ] } ], expiry_method: expiry_method) + SolidCache::Store.any_instance.stubs(:rand).returns(0, 1, 0, 1, 0, 1, 0, 1) + @cache = lookup_store(expiry_batch_size: 2, shards: [ first_shard_key, second_shard_key ], expiry_method: expiry_method) default_shard_keys = shard_keys(@cache, first_shard_key) shard_one_keys = shard_keys(@cache, second_shard_key) @@ -180,7 +179,7 @@ class SolidCache::ExpiryTest < ActiveSupport::TestCase end test "expires old records with a custom queue" do - SolidCache::Cluster.any_instance.stubs(:rand).returns(0, 1, 0, 1) + SolidCache::Store.any_instance.stubs(:rand).returns(0, 1, 0, 1) @cache = lookup_store(expiry_batch_size: 3, max_entries: 2, expiry_method: :job, expiry_queue: :cache_expiry) @@ -198,10 +197,10 @@ class SolidCache::ExpiryTest < ActiveSupport::TestCase end test "triggers multiple expiry tasks when there are many writes" do - @cache = lookup_store(expiry_batch_size: 20, max_entries: 2, expiry_queue: :cache_expiry, clusters: [ @single_shard_cluster ]) - background = @cache.primary_cluster.instance_variable_get("@background") + @cache = lookup_store(expiry_batch_size: 20, max_entries: 2, expiry_queue: :cache_expiry, **@single_shard_cluster) + background = @cache.instance_variable_get("@background") - SolidCache::Cluster.any_instance.stubs(:rand).returns(0.25, 0.24) + SolidCache::Store.any_instance.stubs(:rand).returns(0.25, 0.24) # We expect 2 expiry job for 8 writes assert_difference -> { background.scheduled_task_count }, +1 do @cache.write_multi(8.times.index_by { |i| "key#{i}" }) @@ -214,7 +213,7 @@ class SolidCache::ExpiryTest < ActiveSupport::TestCase end # Whether we overflow an extra job depends on rand - SolidCache::Cluster.any_instance.stubs(:rand).returns(0.25, 0.24) + SolidCache::Store.any_instance.stubs(:rand).returns(0.25, 0.24) assert_difference -> { background.scheduled_task_count }, +1 do @cache.write_multi(10.times.index_by { |i| "key#{i}" }) wait_for_background_tasks(@cache) @@ -227,9 +226,9 @@ class SolidCache::ExpiryTest < ActiveSupport::TestCase end test "triggers multiple expiry jobs when there are many writes" do - @cache = lookup_store(expiry_batch_size: 10, max_entries: 4, expiry_queue: :cache_expiry, expiry_method: :job, clusters: [ @single_shard_cluster ]) + @cache = lookup_store(expiry_batch_size: 10, max_entries: 4, expiry_queue: :cache_expiry, expiry_method: :job, **@single_shard_cluster) - SolidCache::Cluster.any_instance.stubs(:rand).returns(0.25, 0.24) + SolidCache::Store.any_instance.stubs(:rand).returns(0.25, 0.24) # We expect 1 expiry job for 8 writes assert_enqueued_jobs(2, only: SolidCache::ExpiryJob) do @cache.write_multi(8.times.index_by { |i| "key#{i}" }) @@ -240,7 +239,7 @@ class SolidCache::ExpiryTest < ActiveSupport::TestCase end # Whether we overflow an extra job depends on rand - SolidCache::Cluster.any_instance.stubs(:rand).returns(0.125, 0.124) + SolidCache::Store.any_instance.stubs(:rand).returns(0.125, 0.124) assert_enqueued_jobs(2, only: SolidCache::ExpiryJob) do @cache.write_multi(10.times.index_by { |i| "key#{i}" }) end @@ -253,7 +252,7 @@ class SolidCache::ExpiryTest < ActiveSupport::TestCase private def shard_keys(cache, shard) namespaced_keys = 100.times.map { |i| @cache.send(:normalize_key, "key#{i}", {}) } - shard_keys = cache.primary_cluster.send(:connections).assign(namespaced_keys)[shard] + shard_keys = cache.send(:connections).assign(namespaced_keys)[shard] shard_keys.map { |key| key.delete_prefix("#{@namespace}:") } end end diff --git a/test/unit/query_cache_test.rb b/test/unit/query_cache_test.rb index c17de7d..ff5df0f 100644 --- a/test/unit/query_cache_test.rb +++ b/test/unit/query_cache_test.rb @@ -12,8 +12,8 @@ class QueryCacheTest < ActiveSupport::TestCase @cache = lookup_store(expires_in: 60) @peek = lookup_store(expires_in: 60) else - @cache = lookup_store(expires_in: 60, clusters: [ { shards: [ first_shard_key ] } ]) - @peek = lookup_store(expires_in: 60, clusters: [ { shards: [ first_shard_key ] } ]) + @cache = lookup_store(expires_in: 60, shards: [ first_shard_key ]) + @peek = lookup_store(expires_in: 60, shards: [ first_shard_key ]) end end diff --git a/test/unit/rails_cache_test.rb b/test/unit/rails_cache_test.rb index b10677b..a49b113 100644 --- a/test/unit/rails_cache_test.rb +++ b/test/unit/rails_cache_test.rb @@ -4,6 +4,6 @@ class RailsCacheTest < ActiveSupport::TestCase test "reads cache yml config" do - assert_equal 3600, Rails.cache.primary_cluster.max_age + assert_equal 3600, Rails.cache.max_age end end diff --git a/test/unit/solid_cache_test.rb b/test/unit/solid_cache_test.rb index 472c979..f4d782f 100644 --- a/test/unit/solid_cache_test.rb +++ b/test/unit/solid_cache_test.rb @@ -47,12 +47,12 @@ class SolidCacheTest < ActiveSupport::TestCase test "loads defaults from config/solid_cache.yml" do cache = lookup_store - assert_equal 3600, cache.primary_cluster.max_age + assert_equal 3600, cache.max_age end test "cache options override defaults" do cache = lookup_store(max_age: 7200) - assert_equal 7200, cache.primary_cluster.max_age + assert_equal 7200, cache.max_age end end diff --git a/test/unit/stats_test.rb b/test/unit/stats_test.rb index f25952f..dcf3856 100644 --- a/test/unit/stats_test.rb +++ b/test/unit/stats_test.rb @@ -62,7 +62,7 @@ def test_stats_with_entries def test_stats_one_shard skip if ENV["SOLID_CACHE_CONFIG"] - @cache = lookup_store(expiry_batch_size: 2, max_age: 2.weeks.to_i, max_entries: 1000, clusters: [{ shards: [ first_shard_key ] }]) + @cache = lookup_store(expiry_batch_size: 2, max_age: 2.weeks.to_i, max_entries: 1000, shards: [ first_shard_key ]) expected = { connections: 1, @@ -77,7 +77,7 @@ def test_stats_one_shard def test_stats_multiple_shards skip if ENV["SOLID_CACHE_CONFIG"] - @cache = lookup_store(expiry_batch_size: 2, max_age: 2.weeks.to_i, max_entries: 1000, clusters: [{ shards: [ first_shard_key, second_shard_key ] }]) + @cache = lookup_store(expiry_batch_size: 2, max_age: 2.weeks.to_i, max_entries: 1000, shards: [ first_shard_key, second_shard_key ]) expected = { connections: 2, From 529baf37fd1cead70688870fc112aa033c02a023 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Tue, 23 Apr 2024 15:14:43 +0100 Subject: [PATCH 04/27] Remove node names --- README.md | 17 +---------------- lib/solid_cache/connections.rb | 7 +------ lib/solid_cache/connections/sharded.rb | 7 +++---- lib/solid_cache/store/connections.rb | 4 ++-- 4 files changed, 7 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index a48728b..9100955 100644 --- a/README.md +++ b/README.md @@ -127,7 +127,7 @@ Solid Cache supports these options in addition to the standard `ActiveSupport::C - `max_size` - the maximum size of the cache entries (default `nil`, meaning no limit) - `cluster` - (deprecated) a Hash of options for the cache database cluster, e.g `{ shards: [:database1, :database2, :database3] }` - `clusters` - (deprecated) an Array of Hashes for multiple cache clusters (ignored if `:cluster` is set) -- `shards` - an Array or Hash of databases. When a Hash is supplied, the keys are the database names, and the values are the sharding key. +- `shards` - an Array of databases. - `active_record_instrumentation` - whether to instrument the cache's queries (default: `true`) - `clear_with` - clear the cache with `:truncate` or `:delete` (default `truncate`, except for when `Rails.env.test?` then `delete`) - `max_key_bytesize` - the maximum size of a normalized key in bytes (default `1024`) @@ -221,21 +221,6 @@ production: databases: [cache_shard1, cache_shard2, cache_shard3] ``` -### Named shard destinations - -By default, the node key used for sharding is the name of the database in `database.yml`. - -It is possible to add names for the shards in the cluster config. This will allow you to shuffle or remove shards without breaking consistent hashing. - -```yaml -production: - databases: [cache_primary_shard1, cache_primary_shard2, cache_secondary_shard1, cache_secondary_shard2] - store_options: - shards: - cache_primary_shard1: node1 - cache_primary_shard2: node2 -``` - ### Enabling encryption Add this to an initializer: diff --git a/lib/solid_cache/connections.rb b/lib/solid_cache/connections.rb index e8c6f0c..2531921 100644 --- a/lib/solid_cache/connections.rb +++ b/lib/solid_cache/connections.rb @@ -7,13 +7,8 @@ def self.from_config(options) case options when NilClass names = SolidCache.configuration.shard_keys - nodes = names.to_h { |name| [ name, name ] } when Array names = options.map(&:to_sym) - nodes = names.to_h { |name| [ name, name ] } - when Hash - names = options.keys.map(&:to_sym) - nodes = options.to_h { |names, nodes| [ nodes.to_sym, names.to_sym ] } end if (unknown_shards = names - SolidCache.configuration.shard_keys).any? @@ -23,7 +18,7 @@ def self.from_config(options) if names.size == 1 Single.new(names.first) else - Sharded.new(names, nodes) + Sharded.new(names) end else Unmanaged.new diff --git a/lib/solid_cache/connections/sharded.rb b/lib/solid_cache/connections/sharded.rb index c6d750c..b84a028 100644 --- a/lib/solid_cache/connections/sharded.rb +++ b/lib/solid_cache/connections/sharded.rb @@ -5,10 +5,9 @@ module Connections class Sharded attr_reader :names, :nodes, :consistent_hash - def initialize(names, nodes) + def initialize(names) @names = names - @nodes = nodes - @consistent_hash = MaglevHash.new(@nodes.keys) + @consistent_hash = MaglevHash.new(names) end def with_each(&block) @@ -35,7 +34,7 @@ def count private def shard_for(key) - nodes[consistent_hash.node(key)] + consistent_hash.node(key) end end end diff --git a/lib/solid_cache/store/connections.rb b/lib/solid_cache/store/connections.rb index 5f8adb3..ce669ee 100644 --- a/lib/solid_cache/store/connections.rb +++ b/lib/solid_cache/store/connections.rb @@ -21,8 +21,8 @@ def initialize(options = {}) @shard_options = options.fetch(:shards, nil) end - if [ Hash, Array, NilClass ].none? { |klass| @shard_options.is_a? klass } - raise ArgumentError, "`shards` is a `#{@shard_options.class.name}`, it should be one of Array, Hash or nil" + if [ Array, NilClass ].none? { |klass| @shard_options.is_a? klass } + raise ArgumentError, "`shards` is a `#{@shard_options.class.name}`, it should be Array or nil" end end From a71afec6f0c777baeda25233e28ed672f88570bc Mon Sep 17 00:00:00 2001 From: Rafael Montas Date: Wed, 24 Apr 2024 16:55:32 -0400 Subject: [PATCH 05/27] Fix punctuation and typo in README --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 24b3d3e..f74e7ef 100644 --- a/README.md +++ b/README.md @@ -137,15 +137,15 @@ For more information on cache clusters see [Sharding the cache](#sharding-the-ca Solid Cache tracks writes to the cache. For every write it increments a counter by 1. Once the counter reaches 50% of the `expiry_batch_size` it adds a task to run on a background thread. That task will: -1. Check if we have exceeded the `max_entries` or `max_size` values (if set) +1. Check if we have exceeded the `max_entries` or `max_size` values (if set). The current entries are estimated by subtracting the max and min IDs from the `SolidCache::Entry` table. The current size is estimated by sampling the entry `byte_size` columns. -2. If we have it will delete `expiry_batch_size` entries -3. If not it will delete up to `expiry_batch_size` entries, provided they are all older than `max_age`. +2. If we have, it will delete `expiry_batch_size` entries. +3. If not, it will delete up to `expiry_batch_size` entries, provided they are all older than `max_age`. Expiring when we reach 50% of the batch size allows us to expire records from the cache faster than we write to it when we need to reduce the cache size. -Only triggering expiry when we write means that the if the cache is idle, the background thread is also idle. +Only triggering expiry when we write means that if the cache is idle, the background thread is also idle. If you want the cache expiry to be run in a background job instead of a thread, you can set `expiry_method` to `:job`. This will enqueue a `SolidCache::ExpiryJob`. From 62e92047b8be4dd201b221b887e73d088214ac5b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 May 2024 23:40:54 +0000 Subject: [PATCH 06/27] Bump nokogiri from 1.16.2 to 1.16.5 Bumps [nokogiri](https://github.com/sparklemotion/nokogiri) from 1.16.2 to 1.16.5. - [Release notes](https://github.com/sparklemotion/nokogiri/releases) - [Changelog](https://github.com/sparklemotion/nokogiri/blob/main/CHANGELOG.md) - [Commits](https://github.com/sparklemotion/nokogiri/compare/v1.16.2...v1.16.5) --- updated-dependencies: - dependency-name: nokogiri dependency-type: indirect ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index c1abb38..ef8f5ab 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -79,11 +79,11 @@ GEM msgpack (1.7.2) mutex_m (0.2.0) mysql2 (0.5.5) - nokogiri (1.16.2-arm64-darwin) + nokogiri (1.16.5-arm64-darwin) racc (~> 1.4) - nokogiri (1.16.2-x86_64-darwin) + nokogiri (1.16.5-x86_64-darwin) racc (~> 1.4) - nokogiri (1.16.2-x86_64-linux) + nokogiri (1.16.5-x86_64-linux) racc (~> 1.4) parallel (1.24.0) parser (3.2.2.4) From 7acbee5f86d49f7b6dce5dc5bdf200d92284045d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 May 2024 21:46:02 +0000 Subject: [PATCH 07/27] Bump rexml from 3.2.6 to 3.2.8 Bumps [rexml](https://github.com/ruby/rexml) from 3.2.6 to 3.2.8. - [Release notes](https://github.com/ruby/rexml/releases) - [Changelog](https://github.com/ruby/rexml/blob/master/NEWS.md) - [Commits](https://github.com/ruby/rexml/compare/v3.2.6...v3.2.8) --- updated-dependencies: - dependency-name: rexml dependency-type: indirect ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index c1abb38..4fd53a5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -123,7 +123,8 @@ GEM regexp_parser (2.8.3) reline (0.4.1) io-console (~> 0.5) - rexml (3.2.6) + rexml (3.2.8) + strscan (>= 3.0.9) rubocop (1.59.0) json (~> 2.3) language_server-protocol (>= 3.17.0) @@ -165,6 +166,7 @@ GEM sqlite3 (1.7.0-x86_64-darwin) sqlite3 (1.7.0-x86_64-linux) stringio (3.1.0) + strscan (3.1.0) thor (1.3.0) timeout (0.4.1) tzinfo (2.0.6) From a02959d64718603b1c51bc153aaf8550177315df Mon Sep 17 00:00:00 2001 From: Devin Starks Date: Sat, 25 May 2024 10:37:17 +0300 Subject: [PATCH 08/27] Fix documentation typos and grammar --- README.md | 34 +++++++++---------- app/models/solid_cache/entry.rb | 2 +- app/models/solid_cache/entry/size/estimate.rb | 2 +- .../entry/size/moving_average_estimate.rb | 4 +-- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 9477ce4..2896a90 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Solid Cache is a database-backed Active Support cache store implementation. -Using SQL databases backed by SSDs we can have caches that are much larger and cheaper than traditional memory only Redis or Memcached backed caches. +Using SQL databases backed by SSDs we can have caches that are much larger and cheaper than traditional memory-only Redis or Memcached backed caches. ## Usage @@ -14,15 +14,15 @@ To set Solid Cache as your Rails cache, you should add this to your environment config.cache_store = :solid_cache_store ``` -Solid Cache is a FIFO (first in, first out) cache. While this is not as efficient as an LRU cache, this is mitigated by the longer cache lifespan. +Solid Cache is a FIFO (first in, first out) cache. While this is not as efficient as an LRU cache, it is mitigated by the longer cache lifespan. A FIFO cache is much easier to manage: -1. We don't need to track when items are read +1. We don't need to track when items are read. 2. We can estimate and control the cache size by comparing the maximum and minimum IDs. 3. By deleting from one end of the table and adding at the other end we can avoid fragmentation (on MySQL at least). ### Installation -Add this line to your application's Gemfile: +Add this line to your application's `Gemfile`: ```ruby gem "solid_cache" @@ -93,9 +93,9 @@ Setting `databases` to `[cache_db, cache_db2]` is the equivalent of: SolidCache::Record.connects_to shards: { cache_db1: { writing: :cache_db1 }, cache_db2: { writing: :cache_db2 } } ``` -If `connects_to` is set it will be passed directly. +If `connects_to` is set, it will be passed directly. -If none of these are set, then Solid Cache will use the `ActiveRecord::Base` connection pool. This means that cache reads and writes will be part of any wrapping +If none of these are set, Solid Cache will use the `ActiveRecord::Base` connection pool. This means that cache reads and writes will be part of any wrapping database transaction. #### Engine configuration @@ -104,7 +104,7 @@ There are three options that can be set on the engine: - `executor` - the [Rails executor](https://guides.rubyonrails.org/threading_and_code_execution.html#executor) used to wrap asynchronous operations, defaults to the app executor - `connects_to` - a custom connects to value for the abstract `SolidCache::Record` active record model. Required for sharding and/or using a separate cache database to the main app. This will overwrite any value set in `config/solid_cache.yml` -- `size_estimate_samples` - if `max_size` is set on the cache, the number of the samples used to estimates the size. +- `size_estimate_samples` - if `max_size` is set on the cache, the number of the samples used to estimates the size These can be set in your Rails configuration: @@ -116,7 +116,7 @@ end #### Cache configuration -Solid Cache supports these options in addition to the standard `ActiveSupport::Cache::Store` options. +Solid Cache supports these options in addition to the standard `ActiveSupport::Cache::Store` options: - `error_handler` - a Proc to call to handle any `ActiveRecord::ActiveRecordError`s that are raises (default: log errors as warnings) - `expiry_batch_size` - the batch size to use when deleting old records (default: `100`) @@ -127,12 +127,12 @@ Solid Cache supports these options in addition to the standard `ActiveSupport::C - `max_size` - the maximum size of the cache entries (default `nil`, meaning no limit) - `cluster` - (deprecated) a Hash of options for the cache database cluster, e.g `{ shards: [:database1, :database2, :database3] }` - `clusters` - (deprecated) an Array of Hashes for multiple cache clusters (ignored if `:cluster` is set) -- `shards` - an Array of databases. +- `shards` - an Array of databases - `active_record_instrumentation` - whether to instrument the cache's queries (default: `true`) - `clear_with` - clear the cache with `:truncate` or `:delete` (default `truncate`, except for when `Rails.env.test?` then `delete`) - `max_key_bytesize` - the maximum size of a normalized key in bytes (default `1024`) -For more information on cache clusters see [Sharding the cache](#sharding-the-cache) +For more information on cache clusters, see [Sharding the cache](#sharding-the-cache) ### Cache expiry @@ -196,9 +196,9 @@ Solid Cache uses the [Maglev](https://static.googleusercontent.com/media/researc To shard: -1. Add the configuration for the database shards to database.yml -2. Configure the shards via `config.solid_cache.connects_to` -3. Pass the shards for the cache to use via the cluster option +1. Add the configuration for the database shards to database.yml. +2. Configure the shards via `config.solid_cache.connects_to`. +3. Pass the shards for the cache to use via the cluster option. For example: ```yml @@ -234,8 +234,8 @@ end ### Index size limits The Solid Cache migrations try to create an index with 1024 byte entries. If that is too big for your database, you should: -1. Edit the index size in the migration -2. Set `max_key_bytesize` on your cache to the new value +1. Edit the index size in the migration. +2. Set `max_key_bytesize` on your cache to the new value. ## Development @@ -262,10 +262,10 @@ $ TARGET_DB=mysql bin/rake test $ TARGET_DB=postgres bin/rake test ``` -### Testing with multiple Rails version +### Testing with multiple Rails versions Solid Cache relies on [appraisal](https://github.com/thoughtbot/appraisal/tree/main) to test -multiple Rails version. +multiple Rails versions. To run a test for a specific version run: diff --git a/app/models/solid_cache/entry.rb b/app/models/solid_cache/entry.rb index a3d3752..8d4dc76 100644 --- a/app/models/solid_cache/entry.rb +++ b/app/models/solid_cache/entry.rb @@ -5,7 +5,7 @@ class Entry < Record include Expiration, Size # The estimated cost of an extra row in bytes, including fixed size columns, overhead, indexes and free space - # Based on expirimentation on SQLite, MySQL and Postgresql. + # Based on experimentation on SQLite, MySQL and Postgresql. # A bit high for SQLite (more like 90 bytes), but about right for MySQL/Postgresql. ESTIMATED_ROW_OVERHEAD = 140 KEY_HASH_ID_RANGE = -(2**63)..(2**63 - 1) diff --git a/app/models/solid_cache/entry/size/estimate.rb b/app/models/solid_cache/entry/size/estimate.rb index c6b3d7c..ad2c745 100644 --- a/app/models/solid_cache/entry/size/estimate.rb +++ b/app/models/solid_cache/entry/size/estimate.rb @@ -27,7 +27,7 @@ class Entry # We then calculate the fraction of the rows we want to sample by dividing the sample size by the estimated number # of rows. # - # The we grab the byte_size sum of the rows in the range of key_hash values excluding any rows that are larger than + # Then we grab the byte_size sum of the rows in the range of key_hash values excluding any rows that are larger than # our minimum outlier cutoff. We then divide this by the sampling fraction to get an estimate of the size of the # non outlier rows # diff --git a/app/models/solid_cache/entry/size/moving_average_estimate.rb b/app/models/solid_cache/entry/size/moving_average_estimate.rb index a435d18..70a81cd 100644 --- a/app/models/solid_cache/entry/size/moving_average_estimate.rb +++ b/app/models/solid_cache/entry/size/moving_average_estimate.rb @@ -3,9 +3,9 @@ module SolidCache class Entry module Size - # Moving averate cache size estimation + # Moving average cache size estimation # - # To reduce variablitity in the cache size estimate, we'll use a moving average of the previous 20 estimates. + # To reduce variability in the cache size estimate, we'll use a moving average of the previous 20 estimates. # The estimates are stored directly in the cache, under the "__solid_cache_entry_size_moving_average_estimates" key. # # We'll remove the largest and smallest estimates, and then average remaining ones. From e4e583b2aebf4939bb5c2fd8179c8451d92782b2 Mon Sep 17 00:00:00 2001 From: Nick Pezza Date: Thu, 30 May 2024 15:24:13 -0400 Subject: [PATCH 09/27] Support rails 7.2 and update gemfiles --- .github/workflows/main.yml | 4 +- Appraisals | 4 + app/models/solid_cache/entry.rb | 2 +- gemfiles/rails_7_2.gemfile | 21 ++ gemfiles/rails_7_2.gemfile.lock | 232 ++++++++++++++++++ gemfiles/rails_main.gemfile.lock | 112 ++++----- .../cache_store_compression_behavior.rb | 5 - .../cache_store_format_version_behavior.rb | 12 +- .../cache_store_compression_behavior.rb | 5 - .../cache_store_format_version_behavior.rb | 12 +- 10 files changed, 319 insertions(+), 90 deletions(-) create mode 100644 gemfiles/rails_7_2.gemfile create mode 100644 gemfiles/rails_7_2.gemfile.lock diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 114e778..42dde80 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -27,8 +27,8 @@ jobs: strategy: fail-fast: false matrix: - ruby-version: ["3.0", "3.1", "3.2", "3.3"] - gemfile: [ rails_7, rails_7_1, rails_main ] + ruby-version: ["3.1", "3.2", "3.3"] + gemfile: [ rails_7, rails_7_1, rails_7_2, rails_main ] database: [sqlite, postgres, mysql] exclude: - ruby-version: "3.0" diff --git a/Appraisals b/Appraisals index 857d069..95e6247 100644 --- a/Appraisals +++ b/Appraisals @@ -8,6 +8,10 @@ appraise "rails-7-1" do gem "railties", github: "rails/rails", branch: "7-1-stable" end +appraise "rails-7-2" do + gem "railties", github: "rails/rails", branch: "7-2-stable" +end + appraise "rails-main" do gem "railties", github: "rails/rails", branch: "main" end diff --git a/app/models/solid_cache/entry.rb b/app/models/solid_cache/entry.rb index a3d3752..a21ad85 100644 --- a/app/models/solid_cache/entry.rb +++ b/app/models/solid_cache/entry.rb @@ -66,7 +66,7 @@ def id_range private def upsert_all_no_query_cache(payloads) - args = [ self, + args = [ self.all, connection_for_insert_all, add_key_hash_and_byte_size(payloads) ].compact options = { unique_by: upsert_unique_by, diff --git a/gemfiles/rails_7_2.gemfile b/gemfiles/rails_7_2.gemfile new file mode 100644 index 0000000..6ac62ce --- /dev/null +++ b/gemfiles/rails_7_2.gemfile @@ -0,0 +1,21 @@ +# This file was generated by Appraisal + +source "https://rubygems.org" + +gem "sqlite3" +gem "mysql2" +gem "pg" +gem "sprockets-rails" +gem "appraisal" +gem "railties", branch: "7-2-stable", git: "https://github.com/rails/rails.git" + +group :rubocop do + gem "rubocop", ">= 1.25.1", require: false + gem "rubocop-minitest", require: false + gem "rubocop-packaging", require: false + gem "rubocop-performance", require: false + gem "rubocop-rails", require: false + gem "rubocop-md", require: false +end + +gemspec path: "../" diff --git a/gemfiles/rails_7_2.gemfile.lock b/gemfiles/rails_7_2.gemfile.lock new file mode 100644 index 0000000..9d5e7ea --- /dev/null +++ b/gemfiles/rails_7_2.gemfile.lock @@ -0,0 +1,232 @@ +GIT + remote: https://github.com/rails/rails.git + revision: c3eaf8707dd988ecfeb6bc5a8e6c2f2677a8356d + branch: 7-2-stable + specs: + actionpack (7.2.0.beta1) + actionview (= 7.2.0.beta1) + activesupport (= 7.2.0.beta1) + nokogiri (>= 1.8.5) + racc + rack (>= 2.2.4) + rack-session (>= 1.0.1) + rack-test (>= 0.6.3) + rails-dom-testing (~> 2.2) + rails-html-sanitizer (~> 1.6) + useragent (~> 0.16) + actionview (7.2.0.beta1) + activesupport (= 7.2.0.beta1) + builder (~> 3.1) + erubi (~> 1.11) + rails-dom-testing (~> 2.2) + rails-html-sanitizer (~> 1.6) + activejob (7.2.0.beta1) + activesupport (= 7.2.0.beta1) + globalid (>= 0.3.6) + activemodel (7.2.0.beta1) + activesupport (= 7.2.0.beta1) + activerecord (7.2.0.beta1) + activemodel (= 7.2.0.beta1) + activesupport (= 7.2.0.beta1) + timeout (>= 0.4.0) + activesupport (7.2.0.beta1) + base64 + bigdecimal + concurrent-ruby (~> 1.0, >= 1.3.1) + connection_pool (>= 2.2.5) + drb + i18n (>= 1.6, < 2) + minitest (>= 5.1) + tzinfo (~> 2.0, >= 2.0.5) + railties (7.2.0.beta1) + actionpack (= 7.2.0.beta1) + activesupport (= 7.2.0.beta1) + irb (~> 1.13) + rackup (>= 1.0.0) + rake (>= 12.2) + thor (~> 1.0, >= 1.2.2) + zeitwerk (~> 2.6) + +PATH + remote: .. + specs: + solid_cache (0.6.0) + activejob (>= 7) + activerecord (>= 7) + railties (>= 7) + +GEM + remote: https://rubygems.org/ + specs: + appraisal (2.5.0) + bundler + rake + thor (>= 0.14.0) + ast (2.4.2) + base64 (0.2.0) + bigdecimal (3.1.8) + builder (3.2.4) + concurrent-ruby (1.3.1) + connection_pool (2.4.1) + crass (1.0.6) + debug (1.9.2) + irb (~> 1.10) + reline (>= 0.3.8) + drb (2.2.1) + erubi (1.12.0) + globalid (1.2.1) + activesupport (>= 6.1) + i18n (1.14.5) + concurrent-ruby (~> 1.0) + io-console (0.7.2) + irb (1.13.1) + rdoc (>= 4.0.0) + reline (>= 0.4.2) + json (2.7.2) + language_server-protocol (3.17.0.3) + loofah (2.22.0) + crass (~> 1.0.2) + nokogiri (>= 1.12.0) + minitest (5.23.1) + mocha (2.3.0) + ruby2_keywords (>= 0.0.5) + msgpack (1.7.2) + mysql2 (0.5.6) + nokogiri (1.16.5-aarch64-linux) + racc (~> 1.4) + nokogiri (1.16.5-arm-linux) + racc (~> 1.4) + nokogiri (1.16.5-arm64-darwin) + racc (~> 1.4) + nokogiri (1.16.5-x86-linux) + racc (~> 1.4) + nokogiri (1.16.5-x86_64-darwin) + racc (~> 1.4) + nokogiri (1.16.5-x86_64-linux) + racc (~> 1.4) + parallel (1.24.0) + parser (3.3.2.0) + ast (~> 2.4.1) + racc + pg (1.5.6) + psych (5.1.2) + stringio + racc (1.8.0) + rack (3.0.11) + rack-session (2.0.0) + rack (>= 3.0.0) + rack-test (2.1.0) + rack (>= 1.3) + rackup (2.1.0) + rack (>= 3) + webrick (~> 1.8) + rails-dom-testing (2.2.0) + activesupport (>= 5.0.0) + minitest + nokogiri (>= 1.6) + rails-html-sanitizer (1.6.0) + loofah (~> 2.21) + nokogiri (~> 1.14) + rainbow (3.1.1) + rake (13.2.1) + rdoc (6.7.0) + psych (>= 4.0.0) + regexp_parser (2.9.2) + reline (0.5.8) + io-console (~> 0.5) + rexml (3.2.8) + strscan (>= 3.0.9) + rubocop (1.64.0) + json (~> 2.3) + language_server-protocol (>= 3.17.0) + parallel (~> 1.10) + parser (>= 3.3.0.2) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml (>= 3.2.5, < 4.0) + rubocop-ast (>= 1.31.1, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 3.0) + rubocop-ast (1.31.3) + parser (>= 3.3.1.0) + rubocop-md (1.2.2) + rubocop (>= 1.0) + rubocop-minitest (0.35.0) + rubocop (>= 1.61, < 2.0) + rubocop-ast (>= 1.31.1, < 2.0) + rubocop-packaging (0.5.2) + rubocop (>= 1.33, < 2.0) + rubocop-performance (1.21.0) + rubocop (>= 1.48.1, < 2.0) + rubocop-ast (>= 1.31.1, < 2.0) + rubocop-rails (2.25.0) + activesupport (>= 4.2.0) + rack (>= 1.1) + rubocop (>= 1.33.0, < 2.0) + rubocop-ast (>= 1.31.1, < 2.0) + ruby-progressbar (1.13.0) + ruby2_keywords (0.0.5) + sprockets (4.2.1) + concurrent-ruby (~> 1.0) + rack (>= 2.2.4, < 4) + sprockets-rails (3.4.2) + actionpack (>= 5.2) + activesupport (>= 5.2) + sprockets (>= 3.0.0) + sqlite3 (2.0.2-aarch64-linux-gnu) + sqlite3 (2.0.2-aarch64-linux-musl) + sqlite3 (2.0.2-arm-linux-gnu) + sqlite3 (2.0.2-arm-linux-musl) + sqlite3 (2.0.2-arm64-darwin) + sqlite3 (2.0.2-x86-linux-gnu) + sqlite3 (2.0.2-x86-linux-musl) + sqlite3 (2.0.2-x86_64-darwin) + sqlite3 (2.0.2-x86_64-linux-gnu) + sqlite3 (2.0.2-x86_64-linux-musl) + stringio (3.1.0) + strscan (3.1.0) + thor (1.3.1) + timeout (0.4.1) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + unicode-display_width (2.5.0) + useragent (0.16.10) + webrick (1.8.1) + zeitwerk (2.6.15) + +PLATFORMS + aarch64-linux + aarch64-linux-gnu + aarch64-linux-musl + arm-linux + arm-linux-gnu + arm-linux-musl + arm64-darwin + x86-linux + x86-linux-gnu + x86-linux-musl + x86_64-darwin + x86_64-linux + x86_64-linux-gnu + x86_64-linux-musl + +DEPENDENCIES + appraisal + debug + mocha + msgpack + mysql2 + pg + railties! + rubocop (>= 1.25.1) + rubocop-md + rubocop-minitest + rubocop-packaging + rubocop-performance + rubocop-rails + solid_cache! + sprockets-rails + sqlite3 + +BUNDLED WITH + 2.5.9 diff --git a/gemfiles/rails_main.gemfile.lock b/gemfiles/rails_main.gemfile.lock index 25fdd97..fc684f0 100644 --- a/gemfiles/rails_main.gemfile.lock +++ b/gemfiles/rails_main.gemfile.lock @@ -1,11 +1,11 @@ GIT remote: https://github.com/rails/rails.git - revision: 029d31ca31ab72df7bb79372f4ff057231fd0196 + revision: ea0f0a2c964659aa2f9ead9ba999f53a6cc39366 branch: main specs: - actionpack (7.2.0.alpha) - actionview (= 7.2.0.alpha) - activesupport (= 7.2.0.alpha) + actionpack (8.0.0.alpha) + actionview (= 8.0.0.alpha) + activesupport (= 8.0.0.alpha) nokogiri (>= 1.8.5) racc rack (>= 2.2.4) @@ -14,34 +14,34 @@ GIT rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) useragent (~> 0.16) - actionview (7.2.0.alpha) - activesupport (= 7.2.0.alpha) + actionview (8.0.0.alpha) + activesupport (= 8.0.0.alpha) builder (~> 3.1) erubi (~> 1.11) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - activejob (7.2.0.alpha) - activesupport (= 7.2.0.alpha) + activejob (8.0.0.alpha) + activesupport (= 8.0.0.alpha) globalid (>= 0.3.6) - activemodel (7.2.0.alpha) - activesupport (= 7.2.0.alpha) - activerecord (7.2.0.alpha) - activemodel (= 7.2.0.alpha) - activesupport (= 7.2.0.alpha) + activemodel (8.0.0.alpha) + activesupport (= 8.0.0.alpha) + activerecord (8.0.0.alpha) + activemodel (= 8.0.0.alpha) + activesupport (= 8.0.0.alpha) timeout (>= 0.4.0) - activesupport (7.2.0.alpha) + activesupport (8.0.0.alpha) base64 bigdecimal - concurrent-ruby (~> 1.0, >= 1.0.2) + concurrent-ruby (~> 1.0, >= 1.3.1) connection_pool (>= 2.2.5) drb i18n (>= 1.6, < 2) - minitest (>= 5.1, < 5.22.0) + minitest (>= 5.1) tzinfo (~> 2.0, >= 2.0.5) - railties (7.2.0.alpha) - actionpack (= 7.2.0.alpha) - activesupport (= 7.2.0.alpha) - irb + railties (8.0.0.alpha) + actionpack (= 8.0.0.alpha) + activesupport (= 8.0.0.alpha) + irb (~> 1.13) rackup (>= 1.0.0) rake (>= 12.2) thor (~> 1.0, >= 1.2.2) @@ -64,49 +64,49 @@ GEM thor (>= 0.14.0) ast (2.4.2) base64 (0.2.0) - bigdecimal (3.1.6) + bigdecimal (3.1.8) builder (3.2.4) - concurrent-ruby (1.2.3) + concurrent-ruby (1.3.1) connection_pool (2.4.1) crass (1.0.6) - debug (1.9.1) + debug (1.9.2) irb (~> 1.10) reline (>= 0.3.8) drb (2.2.1) erubi (1.12.0) globalid (1.2.1) activesupport (>= 6.1) - i18n (1.14.4) + i18n (1.14.5) concurrent-ruby (~> 1.0) io-console (0.7.2) - irb (1.12.0) - rdoc + irb (1.13.1) + rdoc (>= 4.0.0) reline (>= 0.4.2) - json (2.7.1) + json (2.7.2) language_server-protocol (3.17.0.3) loofah (2.22.0) crass (~> 1.0.2) nokogiri (>= 1.12.0) - minitest (5.21.2) - mocha (2.1.0) + minitest (5.23.1) + mocha (2.3.0) ruby2_keywords (>= 0.0.5) msgpack (1.7.2) - mysql2 (0.5.5) - nokogiri (1.16.2-arm64-darwin) + mysql2 (0.5.6) + nokogiri (1.16.5-arm64-darwin) racc (~> 1.4) - nokogiri (1.16.2-x86_64-darwin) + nokogiri (1.16.5-x86_64-darwin) racc (~> 1.4) - nokogiri (1.16.2-x86_64-linux) + nokogiri (1.16.5-x86_64-linux) racc (~> 1.4) parallel (1.24.0) - parser (3.3.0.5) + parser (3.3.2.0) ast (~> 2.4.1) racc pg (1.5.6) psych (5.1.2) stringio - racc (1.7.3) - rack (3.0.9.1) + racc (1.8.0) + rack (3.0.11) rack-session (2.0.0) rack (>= 3.0.0) rack-test (2.1.0) @@ -122,14 +122,15 @@ GEM loofah (~> 2.21) nokogiri (~> 1.14) rainbow (3.1.1) - rake (13.1.0) - rdoc (6.6.2) + rake (13.2.1) + rdoc (6.7.0) psych (>= 4.0.0) - regexp_parser (2.9.0) - reline (0.4.3) + regexp_parser (2.9.2) + reline (0.5.8) io-console (~> 0.5) - rexml (3.2.6) - rubocop (1.62.0) + rexml (3.2.8) + strscan (>= 3.0.9) + rubocop (1.64.0) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) @@ -140,19 +141,19 @@ GEM rubocop-ast (>= 1.31.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.31.2) - parser (>= 3.3.0.4) + rubocop-ast (1.31.3) + parser (>= 3.3.1.0) rubocop-md (1.2.2) rubocop (>= 1.0) - rubocop-minitest (0.34.5) - rubocop (>= 1.39, < 2.0) - rubocop-ast (>= 1.30.0, < 2.0) + rubocop-minitest (0.35.0) + rubocop (>= 1.61, < 2.0) + rubocop-ast (>= 1.31.1, < 2.0) rubocop-packaging (0.5.2) rubocop (>= 1.33, < 2.0) - rubocop-performance (1.20.2) + rubocop-performance (1.21.0) rubocop (>= 1.48.1, < 2.0) - rubocop-ast (>= 1.30.0, < 2.0) - rubocop-rails (2.24.0) + rubocop-ast (>= 1.31.1, < 2.0) + rubocop-rails (2.25.0) activesupport (>= 4.2.0) rack (>= 1.1) rubocop (>= 1.33.0, < 2.0) @@ -166,10 +167,11 @@ GEM actionpack (>= 5.2) activesupport (>= 5.2) sprockets (>= 3.0.0) - sqlite3 (1.7.2-arm64-darwin) - sqlite3 (1.7.2-x86_64-darwin) - sqlite3 (1.7.2-x86_64-linux) + sqlite3 (2.0.2-arm64-darwin) + sqlite3 (2.0.2-x86_64-darwin) + sqlite3 (2.0.2-x86_64-linux-gnu) stringio (3.1.0) + strscan (3.1.0) thor (1.3.1) timeout (0.4.1) tzinfo (2.0.6) @@ -177,7 +179,7 @@ GEM unicode-display_width (2.5.0) useragent (0.16.10) webrick (1.8.1) - zeitwerk (2.6.13) + zeitwerk (2.6.15) PLATFORMS arm64-darwin-21 @@ -206,4 +208,4 @@ DEPENDENCIES sqlite3 BUNDLED WITH - 2.3.26 + 2.5.9 diff --git a/test/unit/behaviors/cache_store_compression_behavior.rb b/test/unit/behaviors/cache_store_compression_behavior.rb index 02df1ac..870b3f3 100644 --- a/test/unit/behaviors/cache_store_compression_behavior.rb +++ b/test/unit/behaviors/cache_store_compression_behavior.rb @@ -7,11 +7,6 @@ module CacheStoreCompressionBehavior extend ActiveSupport::Concern included do - test "compression works with cache format version 6.1 (using Marshal61WithFallback)" do - @cache = with_format(6.1) { lookup_store(compress: true) } - assert_compression true - end - test "compression works with cache format version 7.0 (using Marshal70WithFallback)" do @cache = with_format(7.0) { lookup_store(compress: true) } assert_compression true diff --git a/test/unit/behaviors/cache_store_format_version_behavior.rb b/test/unit/behaviors/cache_store_format_version_behavior.rb index bf23d45..9f9d80e 100644 --- a/test/unit/behaviors/cache_store_format_version_behavior.rb +++ b/test/unit/behaviors/cache_store_format_version_behavior.rb @@ -6,10 +6,6 @@ module CacheStoreFormatVersionBehavior extend ActiveSupport::Concern FORMAT_VERSION_SIGNATURES = { - 6.1 => [ - "\x04\x08o".b, # Marshal.dump(entry) - "\x04\x08o".b, # Marshal.dump(entry.compressed(...)) - ], 7.0 => [ "\x00\x04\x08[".b, # "\x00" + Marshal.dump(entry.pack) "\x01\x78".b, # "\x01" + Zlib::Deflate.deflate(...) @@ -121,12 +117,6 @@ module CacheStoreFormatVersionBehavior private def with_format(format_version, &block) - if format_version == 6.1 - assert_deprecated(ActiveSupport.deprecator) do - ActiveSupport::Cache.with(format_version: format_version, &block) - end - else - ActiveSupport::Cache.with(format_version: format_version, &block) - end + ActiveSupport::Cache.with(format_version: format_version, &block) end end diff --git a/test/unit/behaviors_rails_7_1/cache_store_compression_behavior.rb b/test/unit/behaviors_rails_7_1/cache_store_compression_behavior.rb index 02df1ac..870b3f3 100644 --- a/test/unit/behaviors_rails_7_1/cache_store_compression_behavior.rb +++ b/test/unit/behaviors_rails_7_1/cache_store_compression_behavior.rb @@ -7,11 +7,6 @@ module CacheStoreCompressionBehavior extend ActiveSupport::Concern included do - test "compression works with cache format version 6.1 (using Marshal61WithFallback)" do - @cache = with_format(6.1) { lookup_store(compress: true) } - assert_compression true - end - test "compression works with cache format version 7.0 (using Marshal70WithFallback)" do @cache = with_format(7.0) { lookup_store(compress: true) } assert_compression true diff --git a/test/unit/behaviors_rails_7_1/cache_store_format_version_behavior.rb b/test/unit/behaviors_rails_7_1/cache_store_format_version_behavior.rb index ba2a192..04da6f6 100644 --- a/test/unit/behaviors_rails_7_1/cache_store_format_version_behavior.rb +++ b/test/unit/behaviors_rails_7_1/cache_store_format_version_behavior.rb @@ -6,10 +6,6 @@ module CacheStoreFormatVersionBehavior extend ActiveSupport::Concern FORMAT_VERSION_SIGNATURES = { - 6.1 => [ - "\x04\x08o".b, # Marshal.dump(entry) - "\x04\x08o".b, # Marshal.dump(entry.compressed(...)) - ], 7.0 => [ "\x00\x04\x08[".b, # "\x00" + Marshal.dump(entry.pack) "\x01\x78".b, # "\x01" + Zlib::Deflate.deflate(...) @@ -121,12 +117,6 @@ module CacheStoreFormatVersionBehavior private def with_format(format_version, &block) - if format_version == 6.1 - assert_deprecated(ActiveSupport.deprecator) do - ActiveSupport::Cache.with(format_version: format_version, &block) - end - else - ActiveSupport::Cache.with(format_version: format_version, &block) - end + ActiveSupport::Cache.with(format_version: format_version, &block) end end From 9e056bda8b9a6f8ef58c3e706a54d09c958f0791 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 Jun 2024 22:36:22 +0000 Subject: [PATCH 10/27] Bump actionpack from 7.1.3.2 to 7.1.3.4 Bumps [actionpack](https://github.com/rails/rails) from 7.1.3.2 to 7.1.3.4. - [Release notes](https://github.com/rails/rails/releases) - [Changelog](https://github.com/rails/rails/blob/v7.1.3.4/actionpack/CHANGELOG.md) - [Commits](https://github.com/rails/rails/compare/v7.1.3.2...v7.1.3.4) --- updated-dependencies: - dependency-name: actionpack dependency-type: indirect ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index c386248..b471f49 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -9,9 +9,9 @@ PATH GEM remote: https://rubygems.org/ specs: - actionpack (7.1.3.2) - actionview (= 7.1.3.2) - activesupport (= 7.1.3.2) + actionpack (7.1.3.4) + actionview (= 7.1.3.4) + activesupport (= 7.1.3.4) nokogiri (>= 1.8.5) racc rack (>= 2.2.4) @@ -19,22 +19,22 @@ GEM rack-test (>= 0.6.3) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - actionview (7.1.3.2) - activesupport (= 7.1.3.2) + actionview (7.1.3.4) + activesupport (= 7.1.3.4) builder (~> 3.1) erubi (~> 1.11) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - activejob (7.1.3.2) - activesupport (= 7.1.3.2) + activejob (7.1.3.4) + activesupport (= 7.1.3.4) globalid (>= 0.3.6) - activemodel (7.1.3.2) - activesupport (= 7.1.3.2) - activerecord (7.1.3.2) - activemodel (= 7.1.3.2) - activesupport (= 7.1.3.2) + activemodel (7.1.3.4) + activesupport (= 7.1.3.4) + activerecord (7.1.3.4) + activemodel (= 7.1.3.4) + activesupport (= 7.1.3.4) timeout (>= 0.4.0) - activesupport (7.1.3.2) + activesupport (7.1.3.4) base64 bigdecimal concurrent-ruby (~> 1.0, >= 1.0.2) @@ -50,9 +50,9 @@ GEM thor (>= 0.14.0) ast (2.4.2) base64 (0.2.0) - bigdecimal (3.1.6) + bigdecimal (3.1.8) builder (3.2.4) - concurrent-ruby (1.2.3) + concurrent-ruby (1.3.1) connection_pool (2.4.1) crass (1.0.6) debug (1.9.1) @@ -62,7 +62,7 @@ GEM erubi (1.12.0) globalid (1.2.1) activesupport (>= 6.1) - i18n (1.14.1) + i18n (1.14.5) concurrent-ruby (~> 1.0) io-console (0.7.1) irb (1.11.0) @@ -73,7 +73,7 @@ GEM loofah (2.22.0) crass (~> 1.0.2) nokogiri (>= 1.12.0) - minitest (5.22.2) + minitest (5.23.1) mocha (2.1.0) ruby2_keywords (>= 0.0.5) msgpack (1.7.2) @@ -92,8 +92,8 @@ GEM pg (1.5.4) psych (5.1.2) stringio - racc (1.7.3) - rack (3.0.9.1) + racc (1.8.0) + rack (3.0.11) rack-session (2.0.0) rack (>= 3.0.0) rack-test (2.1.0) @@ -108,9 +108,9 @@ GEM rails-html-sanitizer (1.6.0) loofah (~> 2.21) nokogiri (~> 1.14) - railties (7.1.3.2) - actionpack (= 7.1.3.2) - activesupport (= 7.1.3.2) + railties (7.1.3.4) + actionpack (= 7.1.3.4) + activesupport (= 7.1.3.4) irb rackup (>= 1.0.0) rake (>= 12.2) From f06bb0bd6cd03429ec3056ab0411c1135a7d6da9 Mon Sep 17 00:00:00 2001 From: Oliver Morgan Date: Wed, 3 Jul 2024 12:14:46 +0100 Subject: [PATCH 11/27] fix unprepared statements --- app/models/solid_cache/entry.rb | 6 +++++- test/unit/behaviors/cache_store_behavior.rb | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/app/models/solid_cache/entry.rb b/app/models/solid_cache/entry.rb index a3d3752..20ebcd9 100644 --- a/app/models/solid_cache/entry.rb +++ b/app/models/solid_cache/entry.rb @@ -116,7 +116,11 @@ def get_all_sql(key_hashes) @get_all_sql_binds ||= {} @get_all_sql_binds[key_hashes.count] ||= build_sql(where(key_hash: key_hashes).select(:key, :value)) else - @get_all_sql_no_binds ||= build_sql(where(key_hash: [ 1, 2 ]).select(:key, :value)).gsub("?, ?", "?") + if keys.size == 1 + @get_all_sql_no_binds_one_key ||= build_sql(where(key_hash: 1).select(:key, :value)) + else + @get_all_sql_no_binds_many_keys ||= build_sql(where(key_hash: [ 1, 2 ]).select(:key, :value)).gsub("?, ?", "?") + end end end diff --git a/test/unit/behaviors/cache_store_behavior.rb b/test/unit/behaviors/cache_store_behavior.rb index fdf50b4..8666c92 100644 --- a/test/unit/behaviors/cache_store_behavior.rb +++ b/test/unit/behaviors/cache_store_behavior.rb @@ -145,6 +145,7 @@ def test_read_multi @cache.write(other_key, "baz") @cache.write(SecureRandom.alphanumeric, "biz") assert_equal({ key => "bar", other_key => "baz" }, @cache.read_multi(key, other_key)) + assert_equal({ key => "bar" }, @cache.read_multi(key)) end def test_read_multi_empty_list From ba32f8311e506d93511429341387c737fce37437 Mon Sep 17 00:00:00 2001 From: Oliver Morgan Date: Wed, 3 Jul 2024 12:17:59 +0100 Subject: [PATCH 12/27] keys => key_hashes --- app/models/solid_cache/entry.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/solid_cache/entry.rb b/app/models/solid_cache/entry.rb index 20ebcd9..b89570a 100644 --- a/app/models/solid_cache/entry.rb +++ b/app/models/solid_cache/entry.rb @@ -116,7 +116,7 @@ def get_all_sql(key_hashes) @get_all_sql_binds ||= {} @get_all_sql_binds[key_hashes.count] ||= build_sql(where(key_hash: key_hashes).select(:key, :value)) else - if keys.size == 1 + if key_hashes.size == 1 @get_all_sql_no_binds_one_key ||= build_sql(where(key_hash: 1).select(:key, :value)) else @get_all_sql_no_binds_many_keys ||= build_sql(where(key_hash: [ 1, 2 ]).select(:key, :value)).gsub("?, ?", "?") From 13d69c5d996b3dcaa563197380f5ede428c32843 Mon Sep 17 00:00:00 2001 From: Oliver Morgan Date: Wed, 3 Jul 2024 13:00:41 +0100 Subject: [PATCH 13/27] fix tests --- Rakefile | 2 +- test/dummy/config/database.yml | 8 ++++++++ .../config/solid_cache_unprepared_statements.yml | 12 ++++++++++++ test/unit/behaviors/cache_store_behavior.rb | 1 - 4 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 test/dummy/config/solid_cache_unprepared_statements.yml diff --git a/Rakefile b/Rakefile index 167e23c..403d497 100644 --- a/Rakefile +++ b/Rakefile @@ -23,7 +23,7 @@ def run_without_aborting(*tasks) end def configs - [ :default, :connects_to, :database, :no_database, :shards ] + [ :default, :connects_to, :database, :no_database, :shards, :unprepared_statements ] end task :test do diff --git a/test/dummy/config/database.yml b/test/dummy/config/database.yml index 3f404ce..087a56d 100644 --- a/test/dummy/config/database.yml +++ b/test/dummy/config/database.yml @@ -42,6 +42,10 @@ development: primary: <<: *default database: <%= database("database_cache_development") %> + primary_unprepared_statements: + <<: *default + database: <%= database("database_cache_development") %> + prepared_statements: false primary_replica: <<: *default database: <%= database("database_cache_development") %> @@ -67,6 +71,10 @@ test: primary: <<: *default database: <%= database("database_cache_test") %> + primary_unprepared_statements: + <<: *default + database: <%= database("database_cache_development") %> + prepared_statements: false primary_replica: <<: *default database: <%= database("database_cache_test") %> diff --git a/test/dummy/config/solid_cache_unprepared_statements.yml b/test/dummy/config/solid_cache_unprepared_statements.yml new file mode 100644 index 0000000..87b471f --- /dev/null +++ b/test/dummy/config/solid_cache_unprepared_statements.yml @@ -0,0 +1,12 @@ +default: &default + database: primary_unprepared_statements + + store_options: + max_age: 3600 + max_size: + +development: + <<: *default + +test: + <<: *default diff --git a/test/unit/behaviors/cache_store_behavior.rb b/test/unit/behaviors/cache_store_behavior.rb index 8666c92..fdf50b4 100644 --- a/test/unit/behaviors/cache_store_behavior.rb +++ b/test/unit/behaviors/cache_store_behavior.rb @@ -145,7 +145,6 @@ def test_read_multi @cache.write(other_key, "baz") @cache.write(SecureRandom.alphanumeric, "biz") assert_equal({ key => "bar", other_key => "baz" }, @cache.read_multi(key, other_key)) - assert_equal({ key => "bar" }, @cache.read_multi(key)) end def test_read_multi_empty_list From 2b7cebd6f6530a7187abfcb949eb55350b892f51 Mon Sep 17 00:00:00 2001 From: Oliver Morgan Date: Wed, 3 Jul 2024 13:04:45 +0100 Subject: [PATCH 14/27] Create primary_unprepared_statements_schema.rb --- .../primary_unprepared_statements_schema.rb | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 test/dummy/db/primary_unprepared_statements_schema.rb diff --git a/test/dummy/db/primary_unprepared_statements_schema.rb b/test/dummy/db/primary_unprepared_statements_schema.rb new file mode 100644 index 0000000..1adf031 --- /dev/null +++ b/test/dummy/db/primary_unprepared_statements_schema.rb @@ -0,0 +1,25 @@ +# This file is auto-generated from the current state of the database. Instead +# of editing this file, please use the migrations feature of Active Record to +# incrementally modify your database, and then regenerate this schema definition. +# +# This file is the source Rails uses to define your schema when running `bin/rails +# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to +# be faster and is potentially less error prone than running all of your +# migrations from scratch. Old migrations may fail to apply correctly if those +# migrations use external dependencies or application code. +# +# It's strongly recommended that you check this file into your version control system. + +ActiveRecord::Schema[7.0].define(version: 2024_01_10_111702) do + create_table "solid_cache_entries", force: :cascade do |t| + t.binary "key", limit: 1024, null: false + t.binary "value", limit: 536870912, null: false + t.datetime "created_at", null: false + t.integer "key_hash", limit: 8, null: false + t.integer "byte_size", limit: 4, null: false + t.index ["byte_size"], name: "index_solid_cache_entries_on_byte_size" + t.index ["key_hash", "byte_size"], name: "index_solid_cache_entries_on_key_hash_and_byte_size" + t.index ["key_hash"], name: "index_solid_cache_entries_on_key_hash", unique: true + end + +end From 3bd7510614205d9456b5bb8802fb1a4b103b98a7 Mon Sep 17 00:00:00 2001 From: Oliver Morgan Date: Wed, 3 Jul 2024 13:29:26 +0100 Subject: [PATCH 15/27] exclude some tests --- test/models/solid_cache/record_test.rb | 4 ++-- test/test_helper.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/models/solid_cache/record_test.rb b/test/models/solid_cache/record_test.rb index d426374..e7d569b 100644 --- a/test/models/solid_cache/record_test.rb +++ b/test/models/solid_cache/record_test.rb @@ -7,7 +7,7 @@ class RecordTest < ActiveSupport::TestCase test "each_shard" do shards = SolidCache::Record.each_shard.map { SolidCache::Record.current_shard } case ENV["SOLID_CACHE_CONFIG"] - when "config/solid_cache_no_database.yml", "config/solid_cache_database.yml" + when "config/solid_cache_no_database.yml", "config/solid_cache_database.yml", "config/solid_cache_unprepared_statements.yml" assert_equal [ :default ], shards when "config/solid_cache_connects_to.yml", "config/solid_cache_shards.yml", nil assert_equal [ :primary_shard_one, :primary_shard_two, :secondary_shard_one, :secondary_shard_two ], shards @@ -19,7 +19,7 @@ class RecordTest < ActiveSupport::TestCase test "each_shard uses the default role" do role = ActiveRecord::Base.connected_to(role: :reading) { SolidCache::Record.each_shard.map { SolidCache::Record.current_role } } case ENV["SOLID_CACHE_CONFIG"] - when "config/solid_cache_no_database.yml", "config/solid_cache_database.yml" + when "config/solid_cache_no_database.yml", "config/solid_cache_database.yml", "config/solid_cache_unprepared_statements.yml" assert_equal [ :reading ], role when "config/solid_cache_connects_to.yml", "config/solid_cache_shards.yml", nil assert_equal [ :writing, :writing, :writing, :writing ], role diff --git a/test/test_helper.rb b/test/test_helper.rb index faa71be..862f0f4 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -82,6 +82,6 @@ def second_shard_key end def single_database? - [ "config/solid_cache_database.yml", "config/solid_cache_no_database.yml" ].include?(ENV["SOLID_CACHE_CONFIG"]) + [ "config/solid_cache_database.yml", "config/solid_cache_no_database.yml", "config/solid_cache_unprepared_statements.yml" ].include?(ENV["SOLID_CACHE_CONFIG"]) end end From 3e083de0e7a151f204813c2b8041aaa06cd5e1c1 Mon Sep 17 00:00:00 2001 From: Oliver Morgan Date: Wed, 3 Jul 2024 13:32:04 +0100 Subject: [PATCH 16/27] remove unnecessary schema --- .../primary_unprepared_statements_schema.rb | 25 ------------------- 1 file changed, 25 deletions(-) delete mode 100644 test/dummy/db/primary_unprepared_statements_schema.rb diff --git a/test/dummy/db/primary_unprepared_statements_schema.rb b/test/dummy/db/primary_unprepared_statements_schema.rb deleted file mode 100644 index 1adf031..0000000 --- a/test/dummy/db/primary_unprepared_statements_schema.rb +++ /dev/null @@ -1,25 +0,0 @@ -# This file is auto-generated from the current state of the database. Instead -# of editing this file, please use the migrations feature of Active Record to -# incrementally modify your database, and then regenerate this schema definition. -# -# This file is the source Rails uses to define your schema when running `bin/rails -# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to -# be faster and is potentially less error prone than running all of your -# migrations from scratch. Old migrations may fail to apply correctly if those -# migrations use external dependencies or application code. -# -# It's strongly recommended that you check this file into your version control system. - -ActiveRecord::Schema[7.0].define(version: 2024_01_10_111702) do - create_table "solid_cache_entries", force: :cascade do |t| - t.binary "key", limit: 1024, null: false - t.binary "value", limit: 536870912, null: false - t.datetime "created_at", null: false - t.integer "key_hash", limit: 8, null: false - t.integer "byte_size", limit: 4, null: false - t.index ["byte_size"], name: "index_solid_cache_entries_on_byte_size" - t.index ["key_hash", "byte_size"], name: "index_solid_cache_entries_on_key_hash_and_byte_size" - t.index ["key_hash"], name: "index_solid_cache_entries_on_key_hash", unique: true - end - -end From 2a529cbee83ec178e1a5f21a9ba098b4bc7030fe Mon Sep 17 00:00:00 2001 From: Oliver Morgan Date: Wed, 3 Jul 2024 15:44:51 +0100 Subject: [PATCH 17/27] typo --- test/dummy/config/database.yml | 2 +- .../primary_unprepared_statements_schema.rb | 25 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 test/dummy/db/primary_unprepared_statements_schema.rb diff --git a/test/dummy/config/database.yml b/test/dummy/config/database.yml index 087a56d..793bf42 100644 --- a/test/dummy/config/database.yml +++ b/test/dummy/config/database.yml @@ -73,7 +73,7 @@ test: database: <%= database("database_cache_test") %> primary_unprepared_statements: <<: *default - database: <%= database("database_cache_development") %> + database: <%= database("database_cache_test") %> prepared_statements: false primary_replica: <<: *default diff --git a/test/dummy/db/primary_unprepared_statements_schema.rb b/test/dummy/db/primary_unprepared_statements_schema.rb new file mode 100644 index 0000000..1adf031 --- /dev/null +++ b/test/dummy/db/primary_unprepared_statements_schema.rb @@ -0,0 +1,25 @@ +# This file is auto-generated from the current state of the database. Instead +# of editing this file, please use the migrations feature of Active Record to +# incrementally modify your database, and then regenerate this schema definition. +# +# This file is the source Rails uses to define your schema when running `bin/rails +# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to +# be faster and is potentially less error prone than running all of your +# migrations from scratch. Old migrations may fail to apply correctly if those +# migrations use external dependencies or application code. +# +# It's strongly recommended that you check this file into your version control system. + +ActiveRecord::Schema[7.0].define(version: 2024_01_10_111702) do + create_table "solid_cache_entries", force: :cascade do |t| + t.binary "key", limit: 1024, null: false + t.binary "value", limit: 536870912, null: false + t.datetime "created_at", null: false + t.integer "key_hash", limit: 8, null: false + t.integer "byte_size", limit: 4, null: false + t.index ["byte_size"], name: "index_solid_cache_entries_on_byte_size" + t.index ["key_hash", "byte_size"], name: "index_solid_cache_entries_on_key_hash_and_byte_size" + t.index ["key_hash"], name: "index_solid_cache_entries_on_key_hash", unique: true + end + +end From c8221c979a7f178dab5c5027d817159656314ca4 Mon Sep 17 00:00:00 2001 From: Oliver Morgan Date: Wed, 3 Jul 2024 16:48:23 +0100 Subject: [PATCH 18/27] remove prepared_statements switch --- app/models/solid_cache/entry.rb | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/app/models/solid_cache/entry.rb b/app/models/solid_cache/entry.rb index b89570a..4f4b31f 100644 --- a/app/models/solid_cache/entry.rb +++ b/app/models/solid_cache/entry.rb @@ -112,16 +112,8 @@ def get_sql end def get_all_sql(key_hashes) - if connection.prepared_statements? - @get_all_sql_binds ||= {} - @get_all_sql_binds[key_hashes.count] ||= build_sql(where(key_hash: key_hashes).select(:key, :value)) - else - if key_hashes.size == 1 - @get_all_sql_no_binds_one_key ||= build_sql(where(key_hash: 1).select(:key, :value)) - else - @get_all_sql_no_binds_many_keys ||= build_sql(where(key_hash: [ 1, 2 ]).select(:key, :value)).gsub("?, ?", "?") - end - end + @get_all_sql_binds ||= {} + @get_all_sql_binds[key_hashes.count] ||= build_sql(where(key_hash: key_hashes).select(:key, :value)) end def build_sql(relation) @@ -135,12 +127,12 @@ def build_sql(relation) def select_all_no_query_cache(query, values) uncached do - if connection.prepared_statements? - result = connection.select_all(sanitize_sql(query), "#{name} Load", Array(values), preparable: true) - else - result = connection.select_all(sanitize_sql([ query, values ]), "#{name} Load", Array(values), preparable: false) + binds = Array(values).map do |value| + ActiveRecord::Relation::QueryAttribute.new("key_hash", value, SolidCache::Entry.type_for_attribute("key_hash")) end + result = connection.select_all(sanitize_sql(query), "#{name} Load", binds, preparable: true) + result.cast_values(SolidCache::Entry.attribute_types) end end From e957b5b1426fcd932708a387a690a4396af77f90 Mon Sep 17 00:00:00 2001 From: Oliver Morgan Date: Wed, 3 Jul 2024 17:02:36 +0100 Subject: [PATCH 19/27] go back to what what it was before --- app/models/solid_cache/entry.rb | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/app/models/solid_cache/entry.rb b/app/models/solid_cache/entry.rb index 4f4b31f..d25e1cd 100644 --- a/app/models/solid_cache/entry.rb +++ b/app/models/solid_cache/entry.rb @@ -112,8 +112,12 @@ def get_sql end def get_all_sql(key_hashes) - @get_all_sql_binds ||= {} - @get_all_sql_binds[key_hashes.count] ||= build_sql(where(key_hash: key_hashes).select(:key, :value)) + if connection.prepared_statements? + @get_all_sql_binds ||= {} + @get_all_sql_binds[key_hashes.count] ||= build_sql(where(key_hash: key_hashes).select(:key, :value)) + else + @get_all_sql_no_binds ||= build_sql(where(key_hash: [ 1, 2 ]).select(:key, :value)).gsub("?, ?", "?") + end end def build_sql(relation) @@ -127,12 +131,12 @@ def build_sql(relation) def select_all_no_query_cache(query, values) uncached do - binds = Array(values).map do |value| - ActiveRecord::Relation::QueryAttribute.new("key_hash", value, SolidCache::Entry.type_for_attribute("key_hash")) + if connection.prepared_statements? + result = connection.select_all(sanitize_sql(query), "#{name} Load", Array(values), preparable: true) + else + result = connection.select_all(sanitize_sql([ query, Array(values) ]), "#{name} Load", Array(values), preparable: false) end - result = connection.select_all(sanitize_sql(query), "#{name} Load", binds, preparable: true) - result.cast_values(SolidCache::Entry.attribute_types) end end From 820f8ddbf6d9183fc25ffab40c8c08776aaaa642 Mon Sep 17 00:00:00 2001 From: Oliver Morgan Date: Wed, 3 Jul 2024 17:15:39 +0100 Subject: [PATCH 20/27] all the way back --- app/models/solid_cache/entry.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/solid_cache/entry.rb b/app/models/solid_cache/entry.rb index d25e1cd..3b76dc3 100644 --- a/app/models/solid_cache/entry.rb +++ b/app/models/solid_cache/entry.rb @@ -134,7 +134,7 @@ def select_all_no_query_cache(query, values) if connection.prepared_statements? result = connection.select_all(sanitize_sql(query), "#{name} Load", Array(values), preparable: true) else - result = connection.select_all(sanitize_sql([ query, Array(values) ]), "#{name} Load", Array(values), preparable: false) + result = connection.select_all(sanitize_sql([ query, values ]), "#{name} Load", binds, preparable: false) end result.cast_values(SolidCache::Entry.attribute_types) From 4f07a826942771d8a7ad5a2ef446924692130049 Mon Sep 17 00:00:00 2001 From: Oliver Morgan Date: Wed, 3 Jul 2024 17:23:33 +0100 Subject: [PATCH 21/27] Update entry.rb --- app/models/solid_cache/entry.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/solid_cache/entry.rb b/app/models/solid_cache/entry.rb index 3b76dc3..9949eb3 100644 --- a/app/models/solid_cache/entry.rb +++ b/app/models/solid_cache/entry.rb @@ -134,7 +134,7 @@ def select_all_no_query_cache(query, values) if connection.prepared_statements? result = connection.select_all(sanitize_sql(query), "#{name} Load", Array(values), preparable: true) else - result = connection.select_all(sanitize_sql([ query, values ]), "#{name} Load", binds, preparable: false) + result = connection.select_all(sanitize_sql([ query, values ]), "#{name} Load", values, preparable: false) end result.cast_values(SolidCache::Entry.attribute_types) From 6c4c234e16af0b48f83196513dc54d4e2ace01ca Mon Sep 17 00:00:00 2001 From: Oliver Morgan Date: Wed, 3 Jul 2024 17:25:24 +0100 Subject: [PATCH 22/27] Update entry.rb --- app/models/solid_cache/entry.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/solid_cache/entry.rb b/app/models/solid_cache/entry.rb index 9949eb3..a3d3752 100644 --- a/app/models/solid_cache/entry.rb +++ b/app/models/solid_cache/entry.rb @@ -134,7 +134,7 @@ def select_all_no_query_cache(query, values) if connection.prepared_statements? result = connection.select_all(sanitize_sql(query), "#{name} Load", Array(values), preparable: true) else - result = connection.select_all(sanitize_sql([ query, values ]), "#{name} Load", values, preparable: false) + result = connection.select_all(sanitize_sql([ query, values ]), "#{name} Load", Array(values), preparable: false) end result.cast_values(SolidCache::Entry.attribute_types) From b8c5c1745291e0e0961ce2b9ed99efc80d1b499b Mon Sep 17 00:00:00 2001 From: Oliver Morgan Date: Wed, 3 Jul 2024 17:49:44 +0100 Subject: [PATCH 23/27] Update entry.rb --- app/models/solid_cache/entry.rb | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/app/models/solid_cache/entry.rb b/app/models/solid_cache/entry.rb index a3d3752..a37fc0b 100644 --- a/app/models/solid_cache/entry.rb +++ b/app/models/solid_cache/entry.rb @@ -112,12 +112,8 @@ def get_sql end def get_all_sql(key_hashes) - if connection.prepared_statements? - @get_all_sql_binds ||= {} - @get_all_sql_binds[key_hashes.count] ||= build_sql(where(key_hash: key_hashes).select(:key, :value)) - else - @get_all_sql_no_binds ||= build_sql(where(key_hash: [ 1, 2 ]).select(:key, :value)).gsub("?, ?", "?") - end + @get_all_sql_binds ||= {} + @get_all_sql_binds[key_hashes.count] ||= build_sql(where(key_hash: key_hashes).select(:key, :value)) end def build_sql(relation) @@ -134,7 +130,7 @@ def select_all_no_query_cache(query, values) if connection.prepared_statements? result = connection.select_all(sanitize_sql(query), "#{name} Load", Array(values), preparable: true) else - result = connection.select_all(sanitize_sql([ query, values ]), "#{name} Load", Array(values), preparable: false) + result = connection.select_all(sanitize_sql([ query, *values ]), "#{name} Load", Array(values), preparable: false) end result.cast_values(SolidCache::Entry.attribute_types) From 35cf90a906c868d9559570cce7901157b5923fc9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Jul 2024 14:37:43 +0000 Subject: [PATCH 24/27] Bump rexml from 3.2.8 to 3.3.2 Bumps [rexml](https://github.com/ruby/rexml) from 3.2.8 to 3.3.2. - [Release notes](https://github.com/ruby/rexml/releases) - [Changelog](https://github.com/ruby/rexml/blob/master/NEWS.md) - [Commits](https://github.com/ruby/rexml/compare/v3.2.8...v3.3.2) --- updated-dependencies: - dependency-name: rexml dependency-type: indirect ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index c386248..92ae241 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -123,8 +123,8 @@ GEM regexp_parser (2.8.3) reline (0.4.1) io-console (~> 0.5) - rexml (3.2.8) - strscan (>= 3.0.9) + rexml (3.3.2) + strscan rubocop (1.59.0) json (~> 2.3) language_server-protocol (>= 3.17.0) From b5570f4e5fed4e3af6a3c9470ae9f35fd0a865a9 Mon Sep 17 00:00:00 2001 From: Oliver Morgan Date: Mon, 22 Jul 2024 20:03:38 +0100 Subject: [PATCH 25/27] rename get_all_sql_binds --- app/models/solid_cache/entry.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/solid_cache/entry.rb b/app/models/solid_cache/entry.rb index a37fc0b..a8aef18 100644 --- a/app/models/solid_cache/entry.rb +++ b/app/models/solid_cache/entry.rb @@ -112,8 +112,8 @@ def get_sql end def get_all_sql(key_hashes) - @get_all_sql_binds ||= {} - @get_all_sql_binds[key_hashes.count] ||= build_sql(where(key_hash: key_hashes).select(:key, :value)) + @get_all_sql ||= {} + @get_all_sql[key_hashes.count] ||= build_sql(where(key_hash: key_hashes).select(:key, :value)) end def build_sql(relation) From 6a0b3d546f823f679d34e066115c622cea08a147 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Tue, 23 Jul 2024 09:56:32 +0100 Subject: [PATCH 26/27] Implment unless_exist support We can't just insert and see if the record exists, we need to lock and check if it has expired. --- app/models/solid_cache/entry.rb | 2 +- lib/solid_cache/store/api.rb | 23 +++++++++++++++++------ lib/solid_cache/store/entries.rb | 4 +++- test/unit/solid_cache_test.rb | 11 +++++++++++ 4 files changed, 32 insertions(+), 8 deletions(-) diff --git a/app/models/solid_cache/entry.rb b/app/models/solid_cache/entry.rb index 5e3137a..b02b81a 100644 --- a/app/models/solid_cache/entry.rb +++ b/app/models/solid_cache/entry.rb @@ -52,7 +52,7 @@ def lock_and_write(key, &block) uncached do result = lock.where(key_hash: key_hash_for(key)).pick(:key, :value) new_value = block.call(result&.first == key ? result[1] : nil) - write(key, new_value) + write(key, new_value) if new_value new_value end end diff --git a/lib/solid_cache/store/api.rb b/lib/solid_cache/store/api.rb index 042fb28..8a24056 100644 --- a/lib/solid_cache/store/api.rb +++ b/lib/solid_cache/store/api.rb @@ -39,16 +39,27 @@ def read_serialized_entry(key, **options) entry_read(key) end - def write_entry(key, entry, raw: false, **options) + def write_entry(key, entry, raw: false, unless_exist: false, **options) payload = serialize_entry(entry, raw: raw, **options) - # No-op for us, but this writes it to the local cache - write_serialized_entry(key, payload, raw: raw, **options) - entry_write(key, payload) + if unless_exist + written = false + entry_lock_and_write(key) do |value| + if value.nil? || deserialize_entry(value, **options).expired? + written = true + payload + end + end + else + written = entry_write(key, payload) + end + + write_serialized_entry(key, payload, raw: raw, returning: written, **options) + written end - def write_serialized_entry(key, payload, raw: false, unless_exist: false, expires_in: nil, race_condition_ttl: nil, **options) - true + def write_serialized_entry(key, payload, raw: false, unless_exist: false, expires_in: nil, race_condition_ttl: nil, returning: true, **options) + returning end def read_serialized_entries(keys) diff --git a/lib/solid_cache/store/entries.rb b/lib/solid_cache/store/entries.rb index d3edf63..5c18fe3 100644 --- a/lib/solid_cache/store/entries.rb +++ b/lib/solid_cache/store/entries.rb @@ -29,7 +29,9 @@ def entry_clear def entry_lock_and_write(key, &block) writing_key(key, failsafe: :increment) do - Entry.lock_and_write(key, &block) + Entry.lock_and_write(key) do |value| + block.call(value).tap { |result| track_writes(1) if result } + end end end diff --git a/test/unit/solid_cache_test.rb b/test/unit/solid_cache_test.rb index f4d782f..d3306f7 100644 --- a/test/unit/solid_cache_test.rb +++ b/test/unit/solid_cache_test.rb @@ -54,6 +54,17 @@ class SolidCacheTest < ActiveSupport::TestCase cache = lookup_store(max_age: 7200) assert_equal 7200, cache.max_age end + + def test_write_with_unless_exist + assert_equal true, @cache.write("foo", 1) + assert_equal false, @cache.write("foo", 1, unless_exist: true) + end + + def test_write_expired_value_with_unless_exist + assert_equal true, @cache.write(1, "aaaa", expires_in: 1.second) + travel 2.seconds + assert_equal true, @cache.write(1, "bbbb", expires_in: 1.second, unless_exist: true) + end end class SolidCacheFailsafeTest < ActiveSupport::TestCase From d5304884fa25d5b47ac70ff27b92776d02a2cbb5 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 26 Jul 2024 09:38:30 +0100 Subject: [PATCH 27/27] Bump version for 0.7.0 --- Gemfile.lock | 2 +- gemfiles/rails_7.gemfile.lock | 2 +- gemfiles/rails_7_1.gemfile.lock | 2 +- gemfiles/rails_7_2.gemfile.lock | 2 +- gemfiles/rails_main.gemfile.lock | 2 +- lib/solid_cache/version.rb | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 5396525..f4b20c7 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - solid_cache (0.6.0) + solid_cache (0.7.0) activejob (>= 7) activerecord (>= 7) railties (>= 7) diff --git a/gemfiles/rails_7.gemfile.lock b/gemfiles/rails_7.gemfile.lock index 41c29ac..e098582 100644 --- a/gemfiles/rails_7.gemfile.lock +++ b/gemfiles/rails_7.gemfile.lock @@ -45,7 +45,7 @@ GIT PATH remote: .. specs: - solid_cache (0.6.0) + solid_cache (0.7.0) activejob (>= 7) activerecord (>= 7) railties (>= 7) diff --git a/gemfiles/rails_7_1.gemfile.lock b/gemfiles/rails_7_1.gemfile.lock index 0224f13..f82b570 100644 --- a/gemfiles/rails_7_1.gemfile.lock +++ b/gemfiles/rails_7_1.gemfile.lock @@ -50,7 +50,7 @@ GIT PATH remote: .. specs: - solid_cache (0.6.0) + solid_cache (0.7.0) activejob (>= 7) activerecord (>= 7) railties (>= 7) diff --git a/gemfiles/rails_7_2.gemfile.lock b/gemfiles/rails_7_2.gemfile.lock index 9d5e7ea..d34ac78 100644 --- a/gemfiles/rails_7_2.gemfile.lock +++ b/gemfiles/rails_7_2.gemfile.lock @@ -50,7 +50,7 @@ GIT PATH remote: .. specs: - solid_cache (0.6.0) + solid_cache (0.7.0) activejob (>= 7) activerecord (>= 7) railties (>= 7) diff --git a/gemfiles/rails_main.gemfile.lock b/gemfiles/rails_main.gemfile.lock index fc684f0..d7d3eff 100644 --- a/gemfiles/rails_main.gemfile.lock +++ b/gemfiles/rails_main.gemfile.lock @@ -50,7 +50,7 @@ GIT PATH remote: .. specs: - solid_cache (0.6.0) + solid_cache (0.7.0) activejob (>= 7) activerecord (>= 7) railties (>= 7) diff --git a/lib/solid_cache/version.rb b/lib/solid_cache/version.rb index 8495542..9e6f64d 100644 --- a/lib/solid_cache/version.rb +++ b/lib/solid_cache/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SolidCache - VERSION = "0.6.0" + VERSION = "0.7.0" end