Open master merge

.github/CODEOWNERS

@@ -1 +1 @@
-.github/workflows @toptal/platform-sre
+* @toptal/sre

.github/dependabot.yml

@@ -20,7 +20,7 @@ updates:
       - "ruby"
       - "dependencies"
     reviewers:
-      - "toptal/devx"
+      - "toptal/sre"
     registries:
       - toptal-github
     insecure-external-code-execution: allow
@@ -38,5 +38,5 @@ updates:
       - "dependencies"
       - "gha"
     reviewers:
-      - "toptal/devx"
+      - "toptal/sre"
     open-pull-requests-limit: 3

.github/workflows/ruby.yml

@@ -16,8 +16,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '3.0', '3.1', '3.2' ]
-        gemfile: [rails.6.1.activerecord, rails.7.0.activerecord, rails.7.1.activerecord]
+        ruby: [ '3.2', '3.3', '3.4' ]
+        gemfile: [rails.6.1.activerecord, rails.7.0.activerecord, rails.7.1.activerecord, rails.7.2.activerecord, rails.8.0.activerecord]
     name: ${{ matrix.ruby }}-${{ matrix.gemfile }}
 
     env:
@@ -41,11 +41,11 @@ jobs:
         with:
           ruby-version: ${{ matrix.ruby }}
           bundler-cache: true
-      - name: Run Elasticsearch
-        uses: elastic/elastic-github-actions/elasticsearch@9de0f78f306e4ebc0838f057e6b754364685e759
-        with:
-          stack-version: 7.10.1
-          port: 9250
+      - name: Start containers
+        run: |
+          docker compose up elasticsearch_test -d
+          sleep 15
+
       - name: Tests
         run: bundle exec rspec
 

CHANGELOG.md

@@ -8,6 +8,20 @@
 
 ### Bugs Fixed
 
+* [#964](https://github.com/toptal/chewy/pull/964): Fix `delayed_sidekiq` worker to handle UUID primary keys correctly.
+
+## 8.0.0-beta (2024-08-27)
+
+### New Features
+
+* [#962](https://github.com/toptal/chewy/pull/962): ElasticSearch v.8 support added
+
+* `delete_all_enabled` setting introduced to align Chewy.massacre with wildcard indices deletion disabled in ES 8 by default
+
+### Changes
+
+### Bugs Fixed
+
 ## 7.6.0 (2024-05-03)
 
 ### Changes

README.md

@@ -43,14 +43,15 @@ Or install it yourself as:
 
 ### Ruby
 
-Chewy is compatible with MRI 3.0-3.2¹.
+Chewy is compatible with MRI 3.0-3.3¹.
 
-> ¹ Ruby 3 is only supported with Rails 6.1
+> ¹ Ruby 3 is supported with Rails 6.1, 7.0, 7.1 and 7.2
 
 ### Elasticsearch compatibility matrix
 
 | Chewy version | Elasticsearch version              |
 | ------------- | ---------------------------------- |
+| 8.0.0         | 8.x                                |
 | 7.2.x         | 7.x                                |
 | 7.1.x         | 7.x                                |
 | 7.0.x         | 6.8, 7.x                           |
@@ -66,7 +67,7 @@ various Chewy versions.
 
 ### Active Record
 
-5.2, 6.0, 6.1 Active Record versions are supported by all Chewy versions.
+6.1, 7.0, 7.1, 7.2 Active Record versions are supported by Chewy.
 
 ## Getting Started
 
@@ -97,7 +98,36 @@ development:
 Make sure you have Elasticsearch up and running. You can [install](https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html) it locally, but the easiest way is to use [Docker](https://www.docker.com/get-started):
 
 ```shell
-$ docker run --rm --name elasticsearch -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" elasticsearch:7.11.1
+$ docker run --rm --name elasticsearch -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" -e "xpack.security.enabled=false" elasticsearch:8.15.0
+```
+
+### Security
+
+Please note that starting from version 8 ElasticSearch has security features enabled by default.
+Docker command above has it disabled for local testing convenience.  If you want to enable it, omit
+`"xpack.security.enabled=false"` part from Docker command, and run these command after starting container (container name `es8` assumed):
+
+Reset password for `elastic` user:
+```
+docker container exec es8 '/usr/share/elasticsearch/bin/elasticsearch-reset-password' -u elastic
+```
+
+Extract CA certificate generated by ElasticSearch on first run:
+```
+docker container cp es8:/usr/share/elasticsearch/config/certs/http_ca.crt tmp/
+```
+
+And then add them to settings:
+
+```yaml
+# config/chewy.yml
+development:
+  host: 'localhost:9200'
+  user: 'elastic'
+  password: 'SomeLongPassword'
+  transport_options:
+    ssl:
+      ca_file: './tmp/http_ca.crt'
 ```
 
 ### Index
@@ -941,6 +971,8 @@ Controller actions are wrapped with the configurable value of `Chewy.request_str
 
 It is also a good idea to set up the `:bypass` strategy inside your test suite and import objects manually only when needed, and use `Chewy.massacre` when needed to flush test ES indices before every example. This will allow you to minimize unnecessary ES requests and reduce overhead.
 
+Deprecation note: since version 8 wildcard removing of indices is disabled by default. You can enable it for a cluster with setting `action.destructive_requires_name` to false.
+
 ```ruby
 RSpec.configure do |config|
   config.before(:suite) do

chewy.gemspec

@@ -11,13 +11,14 @@ Gem::Specification.new do |spec|
   spec.description   = 'Chewy provides functionality for Elasticsearch index handling, documents import mappings and chainable query DSL'
   spec.homepage      = 'https://github.com/toptal/chewy'
   spec.license       = 'MIT'
+  spec.required_ruby_version = '~> 3.0'
 
   spec.files         = `git ls-files`.split($RS)
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'activesupport', '>= 5.2' # Remove with major version bump, 8.x
-  spec.add_dependency 'elasticsearch', '>= 7.14.0', '< 8'
+  spec.add_dependency 'activesupport', '>= 6.1'
+  spec.add_dependency 'elasticsearch', '>= 8.14', '< 9.0'
   spec.add_dependency 'elasticsearch-dsl'
   spec.metadata['rubygems_mfa_required'] = 'true'
 end

docker-compose.yml

@@ -0,0 +1,15 @@
+version: "3.4"
+services:
+  elasticsearch_test:
+    image: "elasticsearch:8.15.0"
+    environment:
+      - bootstrap.memory_lock=${ES_MEMORY_LOCK:-false}
+      - "ES_JAVA_OPTS=-Xms${TEST_ES_HEAP_SIZE:-500m} -Xmx${TEST_ES_HEAP_SIZE:-500m}"
+      - discovery.type=single-node
+      - xpack.security.enabled=false
+    ports:
+      - "127.0.0.1:9250:9200"
+    ulimits:
+      nofile:
+        soft: 65536
+        hard: 65536

gemfiles/base.gemfile

@@ -6,7 +6,7 @@ gem 'redis', require: false
 gem 'rspec', '>= 3.7.0'
 gem 'rspec-collection_matchers'
 gem 'rspec-its'
-gem 'rubocop', '1.63.4'
+gem 'rubocop', '1.65.1'
 gem 'sqlite3', '~> 1.4'
 gem 'timecop'
 gem 'unparser'

gemfiles/rails.7.2.activerecord.gemfile

@@ -0,0 +1,14 @@
+source 'https://rubygems.org'
+
+gem 'activejob', '~> 7.2.0'
+gem 'activerecord', '~> 7.2.0'
+gem 'activesupport', '~> 7.2.0'
+gem 'kaminari-core', '~> 1.1.0', require: false
+gem 'parallel', require: false
+gem 'rspec_junit_formatter', '~> 0.4.1'
+gem 'sidekiq', require: false
+
+gem 'rexml'
+
+gemspec path: '../'
+eval_gemfile 'base.gemfile'

gemfiles/rails.8.0.activerecord.gemfile

@@ -0,0 +1,14 @@
+source 'https://rubygems.org'
+
+gem 'activejob', '~> 8.0.0'
+gem 'activerecord', '~> 8.0.0'
+gem 'activesupport', '~> 8.0.0'
+gem 'kaminari-core', '~> 1.1.0', require: false
+gem 'parallel', require: false
+gem 'rspec_junit_formatter', '~> 0.4.1'
+gem 'sidekiq', require: false
+
+gem 'rexml'
+
+gemspec path: '../'
+eval_gemfile 'base.gemfile'

lib/chewy.rb

@@ -116,6 +116,10 @@ def wait_for_status
     # Be careful, if current prefix is blank, this will destroy all the indexes.
     #
     def massacre
+      unless Chewy.settings[:delete_all_enabled]
+        raise FeatureDisabled, 'Feature disabled by default in ES 8. You can enable it in the cluster and set `delete_all_enabled` option in settings'
+      end
+
       Chewy.client.indices.delete(index: [Chewy.configuration[:prefix], '*'].reject(&:blank?).join('_'))
       Chewy.wait_for_status
     end

lib/chewy/config.rb

@@ -70,12 +70,12 @@ def initialize
     end
 
     def transport_logger=(logger)
-      Chewy.client.transport.transport.logger = logger
+      Chewy.client.transport.logger = logger
       @transport_logger = logger
     end
 
     def transport_tracer=(tracer)
-      Chewy.client.transport.transport.tracer = tracer
+      Chewy.client.transport.tracer = tracer
       @transport_tracer = tracer
     end
 

lib/chewy/errors.rb

@@ -39,4 +39,7 @@ def initialize(join_field_type, join_field_name, relations)
 
   class ImportScopeCleanupError < Error
   end
+
+  class FeatureDisabled < Error
+  end
 end

lib/chewy/fields/root.rb

@@ -4,7 +4,7 @@ class Root < Chewy::Fields::Base
       attr_reader :dynamic_templates, :id
 
       def initialize(name, **options)
-        super(name, **options)
+        super
 
         @value ||= -> { self }
         @dynamic_templates = []

lib/chewy/index/actions.rb

@@ -32,7 +32,7 @@ def exists?
         #
         def create(*args, **kwargs)
           create!(*args, **kwargs)
-        rescue Elasticsearch::Transport::Transport::Errors::BadRequest
+        rescue Elastic::Transport::Transport::Errors::BadRequest
           false
         end
 
@@ -83,9 +83,9 @@ def delete(suffix = nil)
           result = client.indices.delete index: index_names.join(',')
           Chewy.wait_for_status if result
           result
-          # es-ruby >= 1.0.10 handles Elasticsearch::Transport::Transport::Errors::NotFound
+          # es-ruby >= 1.0.10 handles Elastic::Transport::Transport::Errors::NotFound
           # by itself, rescue is for previous versions
-        rescue Elasticsearch::Transport::Transport::Errors::NotFound
+        rescue Elastic::Transport::Transport::Errors::NotFound
           false
         end
 
@@ -99,9 +99,9 @@ def delete(suffix = nil)
         #   UsersIndex.delete '01-2014' # deletes `users_01-2014` index
         #
         def delete!(suffix = nil)
-          # es-ruby >= 1.0.10 handles Elasticsearch::Transport::Transport::Errors::NotFound
+          # es-ruby >= 1.0.10 handles Elastic::Transport::Transport::Errors::NotFound
           # by itself, so it is raised here
-          delete(suffix) or raise Elasticsearch::Transport::Transport::Errors::NotFound
+          delete(suffix) or raise Elastic::Transport::Transport::Errors::NotFound
         end
 
         # Deletes and recreates index. Supports suffixes.

lib/chewy/index/aliases.rb

@@ -22,7 +22,7 @@ def aliases
 
         def empty_if_not_found
           yield
-        rescue Elasticsearch::Transport::Transport::Errors::NotFound
+        rescue Elastic::Transport::Transport::Errors::NotFound
           []
         end
       end

lib/chewy/index/syncer.rb

@@ -213,7 +213,7 @@ def outdated_sync_field_type
         @outdated_sync_field_type = mappings
           .fetch('properties', {})
           .fetch(@index.outdated_sync_field.to_s, {})['type']
-      rescue Elasticsearch::Transport::Transport::Errors::NotFound
+      rescue Elastic::Transport::Transport::Errors::NotFound
         nil
       end
     end

lib/chewy/minitest/helpers.rb

@@ -142,7 +142,7 @@ def index_everything!
         teardown do
           # always destroy indexes between tests
           # Prevent croll pollution of test cases due to indexing
-          Chewy.massacre
+          drop_indices
         end
       end
     end

lib/chewy/search/request.rb

@@ -46,7 +46,7 @@ class Request
 
       delegate :hits, :wrappers, :objects, :records, :documents,
                :object_hash, :record_hash, :document_hash,
-               :total, :max_score, :took, :timed_out?, to: :response
+               :total, :max_score, :took, :timed_out?, :terminated_early?, to: :response
       delegate :each, :size, :to_a, :[], to: :wrappers
       alias_method :to_ary, :to_a
       alias_method :total_count, :total
@@ -854,7 +854,7 @@ def count
         else
           Chewy.client.count(only(WHERE_STORAGES).render)['count']
         end
-      rescue Elasticsearch::Transport::Transport::Errors::NotFound
+      rescue Elastic::Transport::Transport::Errors::NotFound
         0
       end
 
@@ -891,7 +891,7 @@ def exists?
       def first(limit = UNDEFINED)
         request_limit = limit == UNDEFINED ? 1 : limit
 
-        if performed? && (request_limit <= size || size == total)
+        if performed? && (terminated_early? || request_limit <= size || size == total)
           limit == UNDEFINED ? wrappers.first : wrappers.first(limit)
         else
           result = except(EXTRA_STORAGES).limit(request_limit).to_a
@@ -1035,7 +1035,7 @@ def perform(additional = {})
         request_body = render.merge(additional)
         ActiveSupport::Notifications.instrument 'search_query.chewy', notification_payload(request: request_body) do
           Chewy.client.search(request_body)
-        rescue Elasticsearch::Transport::Transport::Errors::NotFound
+        rescue Elastic::Transport::Transport::Errors::NotFound
           {}
         end
       end

lib/chewy/search/response.rb

@@ -47,6 +47,13 @@ def timed_out?
         @timed_out ||= @body['timed_out']
       end
 
+      # Has the request been terminated early?
+      #
+      # @return [true, false]
+      def terminated_early?
+        @terminated_early ||= @body['terminated_early']
+      end
+
       # The `suggest` response part. Returns empty hash if suggests
       # were not requested.
       #

lib/chewy/search/scrolling.rb

@@ -39,7 +39,8 @@ def scroll_batches(batch_size: Request::DEFAULT_BATCH_SIZE, scroll: Request::DEF
           hits = hits.first(last_batch_size) if last_batch_size != 0 && fetched >= total
           yield(hits) if hits.present?
           scroll_id = result['_scroll_id']
-          break if fetched >= total
+
+          break if result['terminated_early'] || fetched >= total
 
           result = perform_scroll(scroll: scroll, scroll_id: scroll_id)
         end

lib/chewy/strategy/delayed_sidekiq/worker.rb

@@ -68,7 +68,7 @@ def extract_ids_and_fields(members)
 
           fields = nil if fields.include?(Scheduler::FALLBACK_FIELDS)
 
-          [ids.map(&:to_i), fields]
+          [ids, fields]
         end
       end
     end

lib/chewy/version.rb

@@ -1,3 +1,3 @@
 module Chewy
-  VERSION = '7.6.0'.freeze
+  VERSION = '8.0.0-beta'.freeze
 end