From 3c224377c5e905d3bfd8bf410bd6a05f4ae8b447 Mon Sep 17 00:00:00 2001 From: Samuel Giddins Date: Sun, 21 Jul 2024 12:39:31 -0700 Subject: [PATCH 01/18] Add missing jruby binstub --- spec/support/jruby_binstubs/gem_server_conformance | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 spec/support/jruby_binstubs/gem_server_conformance diff --git a/spec/support/jruby_binstubs/gem_server_conformance b/spec/support/jruby_binstubs/gem_server_conformance new file mode 100644 index 00000000..44bc4900 --- /dev/null +++ b/spec/support/jruby_binstubs/gem_server_conformance @@ -0,0 +1,3 @@ +require "rubygems" + +load Gem.bin_path("gem_server_conformance", "gem_server_conformance") From f5b5800d2d1e3d5e2030ae9aebc79b42a95bbd45 Mon Sep 17 00:00:00 2001 From: Samuel Giddins Date: Sun, 21 Jul 2024 15:32:06 -0700 Subject: [PATCH 02/18] Ensure new enough sqlite3 is used --- gemstash.gemspec | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gemstash.gemspec b/gemstash.gemspec index dc630fd1..d2e79ed1 100644 --- a/gemstash.gemspec +++ b/gemstash.gemspec @@ -54,6 +54,7 @@ you push your own private gems as well." if RUBY_PLATFORM == "java" spec.add_runtime_dependency "jdbc-sqlite3", "~> 3.8" else - spec.add_runtime_dependency "sqlite3", ">= 1.3", "< 3.0" + # SQLite 3.44+ is required for string_agg support + spec.add_runtime_dependency "sqlite3", ">= 1.68", "< 3.0" end end From cedac9cbbc075da7a2e3e00a6e2ce0601e79b8ee Mon Sep 17 00:00:00 2001 From: Samuel Giddins Date: Sun, 21 Jul 2024 15:32:39 -0700 Subject: [PATCH 03/18] Reapply "Add compact index support for private sources" This reverts commit 53d67693acb5072af41f484f272fa591b61773d5. --- .rubocop-bundler.yml | 8 +- Gemfile | 1 + gemstash.gemspec | 1 + lib/gemstash.rb | 1 + lib/gemstash/cache.rb | 5 +- lib/gemstash/cli/info.rb | 3 + lib/gemstash/compact_index_builder.rb | 257 ++++++++++++++++++++ lib/gemstash/db.rb | 1 + lib/gemstash/db/version.rb | 44 +++- lib/gemstash/gem_pusher.rb | 2 +- lib/gemstash/gem_source.rb | 3 +- lib/gemstash/gem_source/private_source.rb | 12 +- lib/gemstash/gem_yanker.rb | 1 + lib/gemstash/migrations/06_compact_index.rb | 14 ++ lib/gemstash/web.rb | 11 +- schema.rb | 94 +++++++ spec/gemstash/compact_index_builder_spec.rb | 233 ++++++++++++++++++ spec/gemstash/web_spec.rb | 4 +- spec/integration_spec.rb | 17 ++ spec/support/db_helpers.rb | 17 +- spec/support/exec_helpers.rb | 5 +- spec/support/test_gemstash_server.ru | 14 ++ 22 files changed, 728 insertions(+), 20 deletions(-) create mode 100644 lib/gemstash/compact_index_builder.rb create mode 100644 lib/gemstash/migrations/06_compact_index.rb create mode 100644 schema.rb create mode 100644 spec/gemstash/compact_index_builder_spec.rb diff --git a/.rubocop-bundler.yml b/.rubocop-bundler.yml index c5a7bd5f..31122b06 100644 --- a/.rubocop-bundler.yml +++ b/.rubocop-bundler.yml @@ -15,6 +15,9 @@ Lint/AssignmentInCondition: Lint/UnusedMethodArgument: Enabled: false +Lint/UriEscapeUnescape: + Enabled: true + # Style Layout/EndAlignment: @@ -88,7 +91,10 @@ Style/SpecialGlobalVars: Enabled: false Naming/VariableNumber: - EnforcedStyle: 'snake_case' + EnforcedStyle: "snake_case" + AllowedIdentifiers: + - sha256 + - capture3 Naming/MemoizedInstanceVariableName: Enabled: false diff --git a/Gemfile b/Gemfile index c2643d27..da9e15c6 100644 --- a/Gemfile +++ b/Gemfile @@ -29,5 +29,6 @@ group :linting do end group :test do + gem "gem_server_conformance", "~> 0.1.4" gem "mock_redis" end diff --git a/gemstash.gemspec b/gemstash.gemspec index d2e79ed1..5522f6ea 100644 --- a/gemstash.gemspec +++ b/gemstash.gemspec @@ -32,6 +32,7 @@ you push your own private gems as well." spec.required_ruby_version = ">= 3.1" spec.add_runtime_dependency "activesupport", ">= 4.2", "< 8" + spec.add_runtime_dependency "compact_index", "~> 0.15.0" spec.add_runtime_dependency "dalli", ">= 3.2.3", "< 4" spec.add_runtime_dependency "faraday", ">= 1", "< 3" spec.add_runtime_dependency "faraday_middleware", "~> 1.0" diff --git a/lib/gemstash.rb b/lib/gemstash.rb index d56b31ab..75939d76 100644 --- a/lib/gemstash.rb +++ b/lib/gemstash.rb @@ -7,6 +7,7 @@ module Gemstash autoload :DB, "gemstash/db" autoload :Cache, "gemstash/cache" autoload :CLI, "gemstash/cli" + autoload :CompactIndexBuilder, "gemstash/compact_index_builder" autoload :Configuration, "gemstash/configuration" autoload :Dependencies, "gemstash/dependencies" autoload :Env, "gemstash/env" diff --git a/lib/gemstash/cache.rb b/lib/gemstash/cache.rb index 404a82e6..3143722a 100644 --- a/lib/gemstash/cache.rb +++ b/lib/gemstash/cache.rb @@ -43,7 +43,10 @@ def set_dependency(scope, gem, value) def invalidate_gem(scope, gem) @client.delete("deps/v1/#{scope}/#{gem}") - Gemstash::SpecsBuilder.invalidate_stored if scope == "private" + if scope == "private" + Gemstash::SpecsBuilder.invalidate_stored + Gemstash::CompactIndexBuilder.invalidate_stored(gem) + end end end diff --git a/lib/gemstash/cli/info.rb b/lib/gemstash/cli/info.rb index 100dcfe2..5ee921d9 100644 --- a/lib/gemstash/cli/info.rb +++ b/lib/gemstash/cli/info.rb @@ -12,6 +12,9 @@ class Info < Gemstash::CLI::Base def run prepare list_config + + # Gemstash::DB + # Gemstash::Env.current.db.dump_schema_migration(same_db: true) end private diff --git a/lib/gemstash/compact_index_builder.rb b/lib/gemstash/compact_index_builder.rb new file mode 100644 index 00000000..cef36ad2 --- /dev/null +++ b/lib/gemstash/compact_index_builder.rb @@ -0,0 +1,257 @@ +# frozen_string_literal: true + +require "active_support/core_ext/string/filters" +require "compact_index" +require "gemstash" +require "stringio" +require "zlib" + +module Gemstash + # Comment + class CompactIndexBuilder + include Gemstash::Env::Helper + attr_reader :result + + def self.serve(app, ...) + app.content_type "text/plain; charset=utf-8" + body = new(app.auth, ...).serve + app.etag Digest::MD5.hexdigest(body) + sha256 = Digest::SHA256.base64digest(body) + app.headers "Accept-Ranges" => "bytes", "Digest" => "sha-256=#{sha256}", "Repr-Digest" => "sha-256=:#{sha256}:", + "Content-Length" => body.bytesize.to_s + body + end + + def self.invalidate_stored(name) + storage = Gemstash::Storage.for("private").for("compact_index") + storage.resource("names").delete(:names) + storage.resource("versions").delete(:versions) + storage.resource("info/#{name}").delete(:info) + end + + def initialize(auth) + @auth = auth + end + + def serve + check_auth if gemstash_env.config[:protected_fetch] + fetch_from_storage + return result if result + + build_result + store_result + result + end + + private + + def storage + @storage ||= Gemstash::Storage.for("private").for("compact_index") + end + + def fetch_from_storage + resource = fetch_resource + return unless resource.exist?(key) + + @result = resource.load(key).content(key) + rescue StandardError + # On the off-chance of a race condition between specs.exist? and specs.load + @result = nil + end + + def store_result + fetch_resource.save(key => @result) + end + + def check_auth + @auth.check("fetch") + end + + # Comment + class Versions < CompactIndexBuilder + def fetch_resource + storage.resource("versions") + end + + def build_result(force_rebuild: false) + resource = fetch_resource + base = !force_rebuild && resource.exist?("versions.list") && resource.content("versions.list") + Tempfile.create("versions.list") do |file| + versions_file = CompactIndex::VersionsFile.new(file.path) + if base + file.write(base) + file.close + @result = versions_file.contents( + compact_index_versions(versions_file.updated_at.to_time) + ) + else + ts = Time.now.iso8601 + versions_file.create( + compact_index_public_versions(ts), ts + ) + @result = file.read + resource.save("versions.list" => @result) + end + end + end + + private + + def compact_index_versions(date) + all_versions = Sequel::Model.db[<<~SQL.squish, date, date].to_a + SELECT r.name as name, v.created_at as date, v.info_checksum as info_checksum, v.number as number, v.platform as platform + FROM rubygems AS r, versions AS v + WHERE v.rubygem_id = r.id AND + v.created_at > ? + + UNION ALL + + SELECT r.name as name, v.yanked_at as date, v.yanked_info_checksum as info_checksum, '-'||v.number as number, v.platform as platform + FROM rubygems AS r, versions AS v + WHERE v.rubygem_id = r.id AND + v.indexed is false AND + v.yanked_at > ? + + ORDER BY date, number, platform, name + SQL + + # not ordered correctly in sqlite for some reason + all_versions.sort_by! {|v| [v[:date], v[:number], v[:platform], v[:name]] } + map_gem_versions(all_versions.map {|v| [v[:name], [v]] }) + end + + def compact_index_public_versions(date) + all_versions = Sequel::Model.db[<<~SQL.squish, date, date].to_a + SELECT r.name, v.indexed, COALESCE(v.yanked_at, v.created_at) as stamp, + COALESCE(v.yanked_info_checksum, v.info_checksum) as info_checksum, v.number, v.platform + FROM rubygems AS r, versions AS v + WHERE v.rubygem_id = r.id AND + (v.created_at <= ? OR v.yanked_at <= ?) + ORDER BY name, COALESCE(v.yanked_at, v.created_at), number, platform + SQL + + versions_by_gem = all_versions.group_by {|row| row[:name] } + versions_by_gem.each_value do |versions| + info_checksum = versions.last[:info_checksum] + versions.select! {|v| v[:indexed] == true } + # Set all versions' info_checksum to work around https://github.com/bundler/compact_index/pull/20 + versions.each {|v| v[:info_checksum] = info_checksum } + end + + map_gem_versions(versions_by_gem) + end + + def map_gem_versions(versions_by_gem) + versions_by_gem.map do |name, versions| + CompactIndex::Gem.new( + name, + versions.map do |row| + CompactIndex::GemVersion.new( + row[:number], + row[:platform], + nil, # sha256 + row[:info_checksum], + nil, # dependencies + nil, # version.required_ruby_version, + nil, # version.required_rubygems_version + ) + end + ) + end + end + + def key + :versions + end + end + + # Comment + class Info < CompactIndexBuilder + def initialize(auth, name) + super(auth) + @name = name + end + + def fetch_resource + storage.resource("info/#{@name}") + end + + def build_result + @result = CompactIndex.info(requirements_and_dependencies) + end + + private + + def requirements_and_dependencies + group_by_columns = "number, platform, sha256, info_checksum, required_ruby_version, required_rubygems_version, versions.created_at" + + dep_req_agg = "string_agg(dependencies.requirements, '@' ORDER BY dependencies.rubygem_name, dependencies.id) as dep_req_agg" + + dep_name_agg = "string_agg(dependencies.rubygem_name, ',' ORDER BY dependencies.rubygem_name) AS dep_name_agg" + + DB::Rubygem.db[<<~SQL.squish, @name]. + SELECT #{group_by_columns}, #{dep_req_agg}, #{dep_name_agg} + FROM rubygems + LEFT JOIN versions ON versions.rubygem_id = rubygems.id + LEFT JOIN dependencies ON dependencies.version_id = versions.id + WHERE rubygems.name = ? AND versions.indexed = true + GROUP BY #{group_by_columns} + ORDER BY versions.created_at, number, platform, dep_name_agg + SQL + map do |row| + reqs = row[:dep_req_agg]&.split("@") + dep_names = row[:dep_name_agg]&.split(",") + + raise "Dependencies and requirements are not the same size:\n reqs: #{reqs.inspect}\n dep_names: #{dep_names.inspect}\n row: #{row.inspect}" if dep_names&.size != reqs&.size + + deps = [] + if reqs + dep_names.zip(reqs).each do |name, req| + deps << CompactIndex::Dependency.new(name, req) + end + end + + CompactIndex::GemVersion.new( + row[:number], + row[:platform], + row[:sha256], + nil, # info_checksum + deps, + row[:required_ruby_version], + row[:required_rubygems_version] + ) + end + end + + def key + :info + end + end + + # Comment + class Names < CompactIndexBuilder + def fetch_resource + storage.resource("names") + end + + def build_result + names = DB::Rubygem.db[<<~SQL.squish].map {|row| row[:name] } + SELECT name + FROM rubygems + LEFT JOIN versions ON versions.rubygem_id = rubygems.id + WHERE versions.indexed = true + GROUP BY name + HAVING COUNT(versions.id) > 0 + ORDER BY name + SQL + @result = CompactIndex.names(names).encode("UTF-8") + end + + private + + def key + :names + end + end + end +end diff --git a/lib/gemstash/db.rb b/lib/gemstash/db.rb index e28743f2..47789d73 100644 --- a/lib/gemstash/db.rb +++ b/lib/gemstash/db.rb @@ -10,6 +10,7 @@ module DB Sequel::Model.db = Gemstash::Env.current.db Sequel::Model.raise_on_save_failure = true Sequel::Model.plugin :timestamps, update_on_create: true + Sequel::Model.db.extension :schema_dumper autoload :Authorization, "gemstash/db/authorization" autoload :CachedRubygem, "gemstash/db/cached_rubygem" autoload :Dependency, "gemstash/db/dependency" diff --git a/lib/gemstash/db/version.rb b/lib/gemstash/db/version.rb index 3f42c3a5..032c03fa 100644 --- a/lib/gemstash/db/version.rb +++ b/lib/gemstash/db/version.rb @@ -9,7 +9,11 @@ class Version < Sequel::Model many_to_one :rubygem def deindex - update(indexed: false) + info = Gemstash::CompactIndexBuilder::Info.new(nil, rubygem.name).tap(&:build_result).result + prefix = number.dup + prefix << "-#{platform}" if platform != "ruby" + info.gsub!(/^#{Regexp.escape(prefix)} .*?\n/, "") + update(indexed: false, yanked_at: Time.now.utc, yanked_info_checksum: Digest::MD5.hexdigest(info)) end def reindex @@ -28,8 +32,10 @@ def self.slug(params) end def self.for_spec_collection(prerelease: false, latest: false) - versions = where(indexed: true, prerelease: prerelease).association_join(:rubygem) - latest ? select_latest(versions) : versions + versions = where(indexed: true, prerelease: prerelease).association_join(:rubygem). + order { [rubygem[:name], platform.desc] } + versions = select_latest(versions) if latest + order_for_spec_collection(versions) end def self.select_latest(versions) @@ -40,6 +46,17 @@ def self.select_latest(versions) map {|gem_versions| gem_versions.max_by {|version| Gem::Version.new(version.number) } } end + def self.order_for_spec_collection(versions) + versions.to_enum.group_by(&:rubygem_id).flat_map do |_, gem_versions| + versions = Hash.new {|h, k| h[k] = Gem::Version.new(k) } + numbers = gem_versions.map {|version| versions[version.number] } + numbers.sort! + gem_versions.sort_by do |version| + [-numbers.index(version.number), version.platform] + end.reverse + end + end + def self.find_by_spec(gem_id, spec) self[rubygem_id: gem_id, number: spec.version.to_s, @@ -54,14 +71,33 @@ def self.find_by_full_name(full_name) self[full_name: "#{full_name}-ruby"] end - def self.insert_by_spec(gem_id, spec) + def self.insert_by_spec(gem_id, spec, sha256:) gem_name = Gemstash::DB::Rubygem[gem_id].name + info = Gemstash::CompactIndexBuilder::Info.new(nil, gem_name).tap(&:build_result).result + info << CompactIndex::GemVersion.new( + spec.version.to_s, + spec.platform.to_s, + sha256, + nil, # info_checksum + spec.runtime_dependencies.map do |dep| + requirements = dep.requirement.requirements + requirements = requirements.map {|r| "#{r.first} #{r.last}" } + requirements = requirements.join(", ") + CompactIndex::Dependency.new(dep.name, requirements) + end, + spec.required_ruby_version&.to_s, + spec.required_rubygems_version&.to_s + ).to_line << "\n" new(rubygem_id: gem_id, number: spec.version.to_s, platform: spec.platform.to_s, full_name: "#{gem_name}-#{spec.version}-#{spec.platform}", storage_id: spec.full_name, indexed: true, + sha256: sha256, + info_checksum: Digest::MD5.hexdigest(info), + required_ruby_version: spec.required_ruby_version&.to_s, + required_rubygems_version: spec.required_rubygems_version&.to_s, prerelease: spec.version.prerelease?).tap(&:save).id end end diff --git a/lib/gemstash/gem_pusher.rb b/lib/gemstash/gem_pusher.rb index 8cc4f8a2..01a1a7aa 100644 --- a/lib/gemstash/gem_pusher.rb +++ b/lib/gemstash/gem_pusher.rb @@ -79,7 +79,7 @@ def save_to_database raise ExistingVersionError, "Cannot push to an existing version!" if existing && existing.indexed raise YankedVersionError, "Cannot push to a yanked version!" if existing && !existing.indexed - version_id = Gemstash::DB::Version.insert_by_spec(gem_id, spec) + version_id = Gemstash::DB::Version.insert_by_spec(gem_id, spec, sha256: Digest::SHA256.hexdigest(@content)) Gemstash::DB::Dependency.insert_by_spec(version_id, spec) end end diff --git a/lib/gemstash/gem_source.rb b/lib/gemstash/gem_source.rb index 56cce779..48c02c07 100644 --- a/lib/gemstash/gem_source.rb +++ b/lib/gemstash/gem_source.rb @@ -29,7 +29,8 @@ class Base include Gemstash::Logging def_delegators :@app, :cache_control, :content_type, :env, :halt, - :headers, :http_client_for, :params, :redirect, :request + :headers, :http_client_for, :params, :redirect, :request, + :etag def initialize(app) @app = app diff --git a/lib/gemstash/gem_source/private_source.rb b/lib/gemstash/gem_source/private_source.rb index 025312d0..606e8cab 100644 --- a/lib/gemstash/gem_source/private_source.rb +++ b/lib/gemstash/gem_source/private_source.rb @@ -43,15 +43,17 @@ def serve_remove_spec_json end def serve_names - halt 403, "Not yet supported" + protected(CompactIndexBuilder::Names) end def serve_versions - halt 404, "Not yet supported" + protected(CompactIndexBuilder::Versions) end def serve_info(name) - halt 403, "Not yet supported" + halt(404, { "Content-Type" => "text/plain; charset=utf-8" }, "This gem could not be found") unless DB::Rubygem.where(name: name).limit(1).count > 0 + + protected(CompactIndexBuilder::Info, name) end def serve_marshal(id) @@ -96,8 +98,8 @@ def serve_prerelease_specs private - def protected(servable) - authorization.protect(self) { servable.serve(self) } + def protected(servable, ...) + authorization.protect(self) { servable.serve(self, ...) } end def authorization diff --git a/lib/gemstash/gem_yanker.rb b/lib/gemstash/gem_yanker.rb index c27ecf5f..dd9824d4 100644 --- a/lib/gemstash/gem_yanker.rb +++ b/lib/gemstash/gem_yanker.rb @@ -66,6 +66,7 @@ def update_database def invalidate_cache gemstash_env.cache.invalidate_gem("private", @gem_name) + Gemstash::CompactIndexBuilder.invalidate_stored(@gem_name) end end end diff --git a/lib/gemstash/migrations/06_compact_index.rb b/lib/gemstash/migrations/06_compact_index.rb new file mode 100644 index 00000000..d55e0d34 --- /dev/null +++ b/lib/gemstash/migrations/06_compact_index.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +Sequel.migration do + change do + alter_table :versions do # TODO: backfill info_checksum, sha256, required_ruby_version, required_rubygems_version + add_column :info_checksum, String, :size => 40 + add_column :yanked_info_checksum, String, :size => 40 + add_column :yanked_at, DateTime, :null => true + add_column :sha256, String, :size => 64 + add_column :required_ruby_version, String, :size => 255 + add_column :required_rubygems_version, String, :size => 255 + end + end +end diff --git a/lib/gemstash/web.rb b/lib/gemstash/web.rb index ae2428c8..bff5e95f 100644 --- a/lib/gemstash/web.rb +++ b/lib/gemstash/web.rb @@ -27,12 +27,19 @@ def http_client_for(server_url) not_found do status 404 + return body response.body if response.body && !response.body.empty? + body JSON.dump("error" => "Not found", "code" => 404) end error GemPusher::ExistingVersionError do - status 422 - body JSON.dump("error" => "Version already exists", "code" => 422) + status 409 + body JSON.dump("error" => "Version already exists", "code" => 409) + end + + error Gemstash::GemYanker::UnknownGemError, Gemstash::GemYanker::UnknownVersionError do |e| + status 404 + body JSON.dump("error" => e.message, "code" => 404) end get "/" do diff --git a/schema.rb b/schema.rb new file mode 100644 index 00000000..76b704fa --- /dev/null +++ b/schema.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +Sequel.migration do + change do + create_table(:authorizations) do + primary_key :id + column :auth_key, "varchar(191)", :null => false + column :permissions, "varchar(191)", :null => false + column :created_at, "timestamp", :null => false + column :updated_at, "timestamp", :null => false + column :name, "varchar(191)" + + index [:auth_key], :unique => true + index [:name], :unique => true + end + + create_table(:cached_rubygems) do + primary_key :id + column :upstream_id, "INTEGER", :null => false + column :name, "varchar(191)", :null => false + column :resource_type, "varchar(191)", :null => false + column :created_at, "timestamp", :null => false + column :updated_at, "timestamp", :null => false + + index [:name] + index %i[upstream_id resource_type name], :unique => true + end + + create_table(:dependencies) do + primary_key :id + column :version_id, "INTEGER", :null => false + column :rubygem_name, "varchar(191)", :null => false + column :requirements, "varchar(191)", :null => false + column :created_at, "timestamp", :null => false + column :updated_at, "timestamp", :null => false + + index [:rubygem_name] + index [:version_id] + end + + create_table(:health_tests) do + primary_key :id + column :string, "varchar(255)" + end + + create_table(:rubygems) do + primary_key :id + column :name, "varchar(191)", :null => false + column :created_at, "timestamp", :null => false + column :updated_at, "timestamp", :null => false + + index [:name], :unique => true + end + + create_table(:schema_info) do + column :version, "INTEGER", :default => 0, :null => false + end + + create_table(:upstreams) do + primary_key :id + column :uri, "varchar(191)", :null => false + column :host_id, "varchar(191)", :null => false + column :created_at, "timestamp", :null => false + column :updated_at, "timestamp", :null => false + + index [:host_id], :unique => true + index [:uri], :unique => true + end + + create_table(:versions) do + primary_key :id + column :rubygem_id, "INTEGER", :null => false + column :storage_id, "varchar(191)", :null => false + column :number, "varchar(191)", :null => false + column :platform, "varchar(191)", :null => false + column :full_name, "varchar(191)", :null => false + column :indexed, "boolean", :default => true, :null => false + column :prerelease, "boolean", :null => false + column :created_at, "timestamp", :null => false + column :updated_at, "timestamp", :null => false + column :info_checksum, "varchar(40)" + column :sha256, "varchar(64)" + column :required_ruby_version, "varchar(191)" + column :required_rubygems_version, "varchar(191)" + + index [:full_name], :unique => true + index [:indexed] + index %i[indexed prerelease] + index [:number] + index %i[rubygem_id number platform], :unique => true + index [:storage_id], :unique => true + end + end +end diff --git a/spec/gemstash/compact_index_builder_spec.rb b/spec/gemstash/compact_index_builder_spec.rb new file mode 100644 index 00000000..729c9687 --- /dev/null +++ b/spec/gemstash/compact_index_builder_spec.rb @@ -0,0 +1,233 @@ +# frozen_string_literal: true + +require "spec_helper" + +RSpec.describe Gemstash::CompactIndexBuilder do + let(:auth) { Gemstash::ApiKeyAuthorization.new(auth_key) } + let(:auth_with_invalid_auth_key) { Gemstash::ApiKeyAuthorization.new(invalid_auth_key) } + let(:auth_without_permission) { Gemstash::ApiKeyAuthorization.new(auth_key_without_permission) } + let(:auth_key) { "auth-key" } + let(:invalid_auth_key) { "invalid-auth-key" } + let(:auth_key_without_permission) { "auth-key-without-permission" } + + before do + Gemstash::Authorization.authorize(auth_key, "all") + Gemstash::Authorization.authorize(auth_key_without_permission, ["push"]) + allow(Time).to receive(:now).and_return(Time.new(1990, in: "UTC")) + end + + context "with no private gems" do + it "returns empty versions" do + result = described_class::Versions.new(auth).serve + expect(result).to eq(<<~VERSIONS) + created_at: 1990-01-01T00:00:00Z + --- + VERSIONS + end + + it "returns empty names" do + result = described_class::Names.new(auth).serve + expect(result).to eq(<<~NAMES) + --- + + NAMES + end + + it "returns 404 for info" do + result = described_class::Info.new(auth, "something").serve + expect(result).to eq(<<~INFO) + --- + INFO + end + end + + context "with some private gems" do + before do + gem_id = insert_rubygem("example") + insert_version(gem_id, "0.0.1") + insert_version(gem_id, "0.0.2") + insert_version(gem_id, "0.0.2", platform: "java") + gem_id = insert_rubygem("other-example") + insert_version(gem_id, "0.1.0") + end + + it "returns versions" do + Gemstash::CompactIndexBuilder::Versions.new(auth).serve + result = Gemstash::CompactIndexBuilder::Versions.new(auth).serve + expect(result).to eq <<~VERSIONS + created_at: 1990-01-01T00:00:00Z + --- + example 0.0.1 1e6fae87f01f5e16ef83205a1a12646c + example 0.0.2-java 02fd7dc9130d37b37fb21e7b3c870ada + example 0.0.2 be6954d4377b5262bee5bf4018e6227f + other-example 0.1.0 ff0722a59d13124677a2edd0da268bd1 + VERSIONS + end + + it "returns info" do + result = Gemstash::CompactIndexBuilder::Info.new(auth, "example").serve + expect(result).to eq <<~INFO + --- + 0.0.1 |checksum:786b0634cdc056d7fbb027802bbd6e13a6056143adc69047db6aded595754554 + 0.0.2-java |checksum:fd67cdfe89ddbd20e499efccffdc828384acf01e4a3068dbf414150ad7515f5f + 0.0.2 |checksum:bfad311d42610c3d1be9d18064f6e688152560e75c716ff63abb5cbb29673f63 + INFO + end + + it "returns names" do + result = Gemstash::CompactIndexBuilder::Names.new(auth).serve + expect(result).to eq <<~NAMES + --- + example + other-example + NAMES + end + end + + context "with some yanked gems" do + let(:expected_specs) do + [["example", Gem::Version.new("0.0.1"), "ruby"], + ["example", Gem::Version.new("0.0.2"), "ruby"], + ["example", Gem::Version.new("0.0.2"), "java"], + ["other-example", Gem::Version.new("0.1.0"), "ruby"]] + end + + let(:expected_latest_specs) do + [["example", Gem::Version.new("0.0.2"), "ruby"], + ["example", Gem::Version.new("0.0.2"), "java"], + ["other-example", Gem::Version.new("0.1.0"), "ruby"]] + end + + let(:expected_prerelease_specs) do + [["example", Gem::Version.new("0.0.2.rc1"), "ruby"], + ["example", Gem::Version.new("0.0.2.rc2"), "ruby"], + ["example", Gem::Version.new("0.0.2.rc2"), "java"], + ["other-example", Gem::Version.new("0.1.1.rc1"), "ruby"]] + end + + before do + Gemstash::CompactIndexBuilder::Versions.new(auth).serve + gem_id = insert_rubygem("example") + insert_version(gem_id, "0.0.1") + insert_version(gem_id, "0.0.2.rc1", prerelease: true) + insert_version(gem_id, "0.0.2.rc2", prerelease: true) + insert_version(gem_id, "0.0.2.rc2", platform: "java", prerelease: true) + insert_version(gem_id, "0.0.2") + insert_version(gem_id, "0.0.2", platform: "java") + insert_version(gem_id, "0.0.3.rc1", indexed: false, prerelease: true) + insert_version(gem_id, "0.0.3", indexed: false) + insert_version(gem_id, "0.0.3.rc1", indexed: false, prerelease: true, platform: "java") + insert_version(gem_id, "0.0.3", indexed: false, platform: "java") + gem_id = insert_rubygem("other-example") + insert_version(gem_id, "0.0.1", indexed: false) + insert_version(gem_id, "0.0.1.rc1", indexed: false, prerelease: true) + insert_version(gem_id, "0.1.0") + insert_version(gem_id, "0.1.1.rc1", prerelease: true) + end + + it "returns versions" do + result = Gemstash::CompactIndexBuilder::Versions.new(auth).serve + expect(result).to eq <<~VERSIONS + created_at: 1990-01-01T00:00:00Z + --- + example 0.0.1 1e6fae87f01f5e16ef83205a1a12646c + other-example 0.0.1 6105347ebb9825ac754615ca55ff3b0c + other-example 0.0.1.rc1 6105347ebb9825ac754615ca55ff3b0c + example 0.0.2-java 30b1ce74f9d06e512e354c697280c5e0 + example 0.0.2 1c60ca76f3375ac0473e16c9920a41c6 + example 0.0.2.rc1 d6f36de1e2fbebb92b6051fc6977ff0a + example 0.0.2.rc2-java 11850dde5a9df04c3fb2aba44704085d + example 0.0.2.rc2 48a1807ddf7e6a29c84d0f261cf4df64 + example 0.0.3-java 30b1ce74f9d06e512e354c697280c5e0 + example 0.0.3 30b1ce74f9d06e512e354c697280c5e0 + example 0.0.3.rc1-java 30b1ce74f9d06e512e354c697280c5e0 + example 0.0.3.rc1 30b1ce74f9d06e512e354c697280c5e0 + other-example 0.1.0 ff0722a59d13124677a2edd0da268bd1 + other-example 0.1.1.rc1 1b239fe769f037ab38a4c89ea6b37320 + VERSIONS + end + end + + context "with a new spec pushed" do + before do + Gemstash::Authorization.authorize(auth_key, "all") + gem_id = insert_rubygem("example") + insert_version(gem_id, "0.0.1") + end + + it "busts the cache" do + # before + Gemstash::GemPusher.new(auth, read_gem("example", "0.1.0")).serve + # after + end + end + + context "with a spec yanked" do + let(:initial_specs) do + [["example", Gem::Version.new("0.0.1"), "ruby"], + ["example", Gem::Version.new("0.1.0"), "ruby"]] + end + + let(:latest_specs) { [["example", Gem::Version.new("0.1.0"), "ruby"]] } + + let(:specs_after_yank) { [["example", Gem::Version.new("0.0.1"), "ruby"]] } + + before do + Gemstash::Authorization.authorize(auth_key, "all") + gem_id = insert_rubygem("example") + insert_version(gem_id, "0.0.1") + Gemstash::GemPusher.new(auth, read_gem("example", "0.1.0")).serve + end + + it "busts the cache" do + result = Gemstash::SpecsBuilder.new(auth).serve + expect(Marshal.load(gunzip(result))).to match_array(initial_specs) + Gemstash::GemYanker.new(auth, "example", "0.1.0").serve + result = Gemstash::SpecsBuilder.new(auth).serve + expect(Marshal.load(gunzip(result))).to match_array(specs_after_yank) + end + end + + context "with protected fetch disabled" do + it "serves versions without authorization" do + result = Gemstash::CompactIndexBuilder::Versions.new(auth).serve + expect(result).to eq(<<~VERSIONS) + created_at: 1990-01-01T00:00:00Z + --- + VERSIONS + end + end + + xcontext "with protected fetch enabled" do + before do + @test_env = test_env + config = Gemstash::Configuration.new(config: { protected_fetch: true }) + Gemstash::Env.current = Gemstash::Env.new(config) + end + + after do + Gemstash::Env.current = @test_env + end + + context "with valid authorization" do + it "serves specs" do + result = Gemstash::SpecsBuilder.new(auth).serve + expect(Marshal.load(gunzip(result))).to eq([]) + end + end + + context "with invalid authorization" do + it "prevents serving specs" do + expect { Gemstash::SpecsBuilder.new(auth_with_invalid_auth_key).serve }. + to raise_error(Gemstash::NotAuthorizedError) + end + end + + context "with invalid permission" do + it "prevents serving specs" do + expect { Gemstash::SpecsBuilder.new(auth_without_permission).serve }. + to raise_error(Gemstash::NotAuthorizedError) + end + end + end +end diff --git a/spec/gemstash/web_spec.rb b/spec/gemstash/web_spec.rb index 57ffe839..f745a868 100644 --- a/spec/gemstash/web_spec.rb +++ b/spec/gemstash/web_spec.rb @@ -431,8 +431,8 @@ def for(server_url, timeout = 20) post "/api/v1/gems", read_gem("example", "0.1.0"), env expect(last_response).to_not be_ok - expect(last_response.status).to eq(422) - expect(JSON.parse(last_response.body)).to eq("error" => "Version already exists", "code" => 422) + expect(last_response.status).to eq(409) + expect(JSON.parse(last_response.body)).to eq("error" => "Version already exists", "code" => 409) end end end diff --git a/spec/integration_spec.rb b/spec/integration_spec.rb index ab0890c1..de81a79f 100644 --- a/spec/integration_spec.rb +++ b/spec/integration_spec.rb @@ -69,6 +69,14 @@ @gemstash_empty_rubygems.start end + before(:each) do + $test_gemstash_server_current_test = self # rubocop:disable Style/GlobalVars + end + + after(:each) do + $test_gemstash_server_current_test = nil # rubocop:disable Style/GlobalVars + end + let(:platform_message) do if RUBY_PLATFORM == "java" "Java" @@ -116,6 +124,15 @@ clean_env env_name end + it "is a conformant gem server", db_transaction: false do + @gemstash.env.cache.flush + expect( + execute("gem_server_conformance", ["--fail-fast", "--format", "progress", "--tag=~content_length_header"], + env: { "UPSTREAM" => host, "GEM_HOST_API_KEY" => auth_key }) + ). + to exit_success + end + context "pushing a gem" do before do expect(deps.fetch(%w[speaker])).to match_dependencies([]) diff --git a/spec/support/db_helpers.rb b/spec/support/db_helpers.rb index d5a478ca..ec0075d9 100644 --- a/spec/support/db_helpers.rb +++ b/spec/support/db_helpers.rb @@ -31,9 +31,12 @@ def insert_version(gem_id, number, platform: "ruby", indexed: true, prerelease: :storage_id => storage_id, :indexed => indexed, :prerelease => prerelease, + :sha256 => Digest::SHA256.hexdigest(storage_id), :created_at => Sequel::SQL::Constants::CURRENT_TIMESTAMP, :updated_at => Sequel::SQL::Constants::CURRENT_TIMESTAMP - ) + ).tap do |version_id| + update_info_checksum(version_id) + end end def insert_dependency(version_id, gem_name, requirements) @@ -43,6 +46,16 @@ def insert_dependency(version_id, gem_name, requirements) :requirements => requirements, :created_at => Sequel::SQL::Constants::CURRENT_TIMESTAMP, :updated_at => Sequel::SQL::Constants::CURRENT_TIMESTAMP - ) + ).tap do + update_info_checksum(version_id) + end + end + + def update_info_checksum(version_id) + gem_id = Gemstash::Env.current.db[:versions][id: version_id][:rubygem_id] + gem_name = Gemstash::Env.current.db[:rubygems][id: gem_id][:name] + info = Gemstash::CompactIndexBuilder::Info.new(nil, gem_name).tap(&:build_result).result + Gemstash::DB::Version.where(id: version_id).update(info_checksum: Digest::MD5.hexdigest(info)) + raise "Failed to update info checksum for version #{version_id}" unless Gemstash::Env.current.db[:versions][id: version_id][:info_checksum] end end diff --git a/spec/support/exec_helpers.rb b/spec/support/exec_helpers.rb index 3820bad8..1838a688 100644 --- a/spec/support/exec_helpers.rb +++ b/spec/support/exec_helpers.rb @@ -76,7 +76,10 @@ def clear_ruby_env "BUNDLER_SETUP" => nil, "GEM_PATH" => original_gem_path, "RUBYLIB" => nil, - "RUBYOPT" => nil + "RUBYOPT" => nil, + "XDG_CONFIG_HOME" => nil, + "XDG_CACHE_HOME" => nil, + "XDG_DATA_HOME" => nil } end diff --git a/spec/support/test_gemstash_server.ru b/spec/support/test_gemstash_server.ru index 358a5878..c7317dcb 100644 --- a/spec/support/test_gemstash_server.ru +++ b/spec/support/test_gemstash_server.ru @@ -5,4 +5,18 @@ use Rack::Deflater use Gemstash::Env::RackMiddleware, $test_gemstash_server_env use Gemstash::GemSource::RackMiddleware use Gemstash::Health::RackMiddleware +map "/set_time" do + run lambda {|env| + now = Time.iso8601(Rack::Request.new(env).body.read) + $test_gemstash_server_current_test.allow(Time). + to $test_gemstash_server_current_test.receive(:now).and_return(now) + [200, {}, ["OK"]] + } +end +map "/rebuild_versions_list" do + run lambda {|_env| + Gemstash::CompactIndexBuilder::Versions.new(nil).build_result(force_rebuild: true) + [200, {}, ["OK"]] + } +end run Gemstash::Web.new(gemstash_env: $test_gemstash_server_env) From 5437e326d0d6f5f470d8d7b5527fc096fb2c5770 Mon Sep 17 00:00:00 2001 From: Samuel Giddins Date: Sun, 21 Jul 2024 15:34:43 -0700 Subject: [PATCH 04/18] Use inner join to simplify query --- lib/gemstash/compact_index_builder.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/gemstash/compact_index_builder.rb b/lib/gemstash/compact_index_builder.rb index cef36ad2..b535d73e 100644 --- a/lib/gemstash/compact_index_builder.rb +++ b/lib/gemstash/compact_index_builder.rb @@ -238,10 +238,9 @@ def build_result names = DB::Rubygem.db[<<~SQL.squish].map {|row| row[:name] } SELECT name FROM rubygems - LEFT JOIN versions ON versions.rubygem_id = rubygems.id + INNER JOIN versions ON versions.rubygem_id = rubygems.id WHERE versions.indexed = true GROUP BY name - HAVING COUNT(versions.id) > 0 ORDER BY name SQL @result = CompactIndex.names(names).encode("UTF-8") From 67a908e6efe2bbdd73d53fd27ac36efabefcafb7 Mon Sep 17 00:00:00 2001 From: Samuel Giddins Date: Sun, 21 Jul 2024 16:12:46 -0700 Subject: [PATCH 05/18] Use Sequel builder methods for complex compact index queries --- lib/gemstash/compact_index_builder.rb | 37 ++++++++++----------------- lib/gemstash/db/rubygem.rb | 2 ++ lib/gemstash/db/version.rb | 1 + 3 files changed, 17 insertions(+), 23 deletions(-) diff --git a/lib/gemstash/compact_index_builder.rb b/lib/gemstash/compact_index_builder.rb index b535d73e..cdcf65c6 100644 --- a/lib/gemstash/compact_index_builder.rb +++ b/lib/gemstash/compact_index_builder.rb @@ -183,21 +183,17 @@ def build_result private def requirements_and_dependencies - group_by_columns = "number, platform, sha256, info_checksum, required_ruby_version, required_rubygems_version, versions.created_at" - - dep_req_agg = "string_agg(dependencies.requirements, '@' ORDER BY dependencies.rubygem_name, dependencies.id) as dep_req_agg" - - dep_name_agg = "string_agg(dependencies.rubygem_name, ',' ORDER BY dependencies.rubygem_name) AS dep_name_agg" - - DB::Rubygem.db[<<~SQL.squish, @name]. - SELECT #{group_by_columns}, #{dep_req_agg}, #{dep_name_agg} - FROM rubygems - LEFT JOIN versions ON versions.rubygem_id = rubygems.id - LEFT JOIN dependencies ON dependencies.version_id = versions.id - WHERE rubygems.name = ? AND versions.indexed = true - GROUP BY #{group_by_columns} - ORDER BY versions.created_at, number, platform, dep_name_agg - SQL + DB::Rubygem.association_left_join(versions: :dependencies). + where(name: @name). + where { versions[:indexed] }. + order { [versions[:created_at], versions[:number], versions[:platform], dep_name_agg] }. + select_group do + [versions[:number], versions[:platform], versions[:sha256], versions[:info_checksum], versions[:required_ruby_version], versions[:required_rubygems_version], versions[:created_at]] + end. # rubocop:disable Style/MultilineBlockChain + select_more do + [string_agg(dependencies[:requirements], "@").order(dependencies[:rubygem_name], dependencies[:id]).as(:dep_req_agg), + string_agg(dependencies[:rubygem_name], ",").order(dependencies[:rubygem_name]).as(:dep_name_agg)] + end. # rubocop:disable Style/MultilineBlockChain map do |row| reqs = row[:dep_req_agg]&.split("@") dep_names = row[:dep_name_agg]&.split(",") @@ -235,14 +231,9 @@ def fetch_resource end def build_result - names = DB::Rubygem.db[<<~SQL.squish].map {|row| row[:name] } - SELECT name - FROM rubygems - INNER JOIN versions ON versions.rubygem_id = rubygems.id - WHERE versions.indexed = true - GROUP BY name - ORDER BY name - SQL + names = DB::Rubygem.association_join(:versions). + where { versions[:indexed] }. + order(:name).group(:name).select_map(:name) @result = CompactIndex.names(names).encode("UTF-8") end diff --git a/lib/gemstash/db/rubygem.rb b/lib/gemstash/db/rubygem.rb index 7083d416..a2112195 100644 --- a/lib/gemstash/db/rubygem.rb +++ b/lib/gemstash/db/rubygem.rb @@ -6,6 +6,8 @@ module Gemstash module DB # Sequel model for rubygems table. class Rubygem < Sequel::Model + one_to_many :versions + def self.find_or_insert(spec) record = self[name: spec.name] return record.id if record diff --git a/lib/gemstash/db/version.rb b/lib/gemstash/db/version.rb index 032c03fa..d8925762 100644 --- a/lib/gemstash/db/version.rb +++ b/lib/gemstash/db/version.rb @@ -7,6 +7,7 @@ module DB # Sequel model for versions table. class Version < Sequel::Model many_to_one :rubygem + one_to_many :dependencies def deindex info = Gemstash::CompactIndexBuilder::Info.new(nil, rubygem.name).tap(&:build_result).result From d7fbc7c516470b8196e12e1bab97186771360bcd Mon Sep 17 00:00:00 2001 From: Samuel Giddins Date: Sun, 21 Jul 2024 16:24:32 -0700 Subject: [PATCH 06/18] [DNM] Debug sql errors --- lib/gemstash/compact_index_builder.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/gemstash/compact_index_builder.rb b/lib/gemstash/compact_index_builder.rb index cdcf65c6..7f8ef754 100644 --- a/lib/gemstash/compact_index_builder.rb +++ b/lib/gemstash/compact_index_builder.rb @@ -178,6 +178,8 @@ def fetch_resource def build_result @result = CompactIndex.info(requirements_and_dependencies) + rescue Sequel::DatabaseError => e + raise "Error building info for #{@name}\n\n```sql\n#{e.sql}\n```\n" end private From 0c34760cddec449aca9c708d067a47605b7d9b97 Mon Sep 17 00:00:00 2001 From: Samuel Giddins Date: Sun, 21 Jul 2024 16:26:51 -0700 Subject: [PATCH 07/18] Enable string_agg extension --- lib/gemstash/db.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/gemstash/db.rb b/lib/gemstash/db.rb index 47789d73..2b6fc11a 100644 --- a/lib/gemstash/db.rb +++ b/lib/gemstash/db.rb @@ -10,6 +10,7 @@ module DB Sequel::Model.db = Gemstash::Env.current.db Sequel::Model.raise_on_save_failure = true Sequel::Model.plugin :timestamps, update_on_create: true + Sequel::Model.db.extension :string_agg Sequel::Model.db.extension :schema_dumper autoload :Authorization, "gemstash/db/authorization" autoload :CachedRubygem, "gemstash/db/cached_rubygem" From 801b4d2568abfda2f70832f4966ba94c8e91b7a4 Mon Sep 17 00:00:00 2001 From: Samuel Giddins Date: Sun, 21 Jul 2024 16:28:41 -0700 Subject: [PATCH 08/18] [DNM] Enable error_sql extension --- lib/gemstash/db.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/gemstash/db.rb b/lib/gemstash/db.rb index 2b6fc11a..4ecd7aa5 100644 --- a/lib/gemstash/db.rb +++ b/lib/gemstash/db.rb @@ -10,6 +10,7 @@ module DB Sequel::Model.db = Gemstash::Env.current.db Sequel::Model.raise_on_save_failure = true Sequel::Model.plugin :timestamps, update_on_create: true + Sequel::Model.db.extension :error_sql Sequel::Model.db.extension :string_agg Sequel::Model.db.extension :schema_dumper autoload :Authorization, "gemstash/db/authorization" From d36a7d289c89090b361e0cdbe6b1a24d05da6963 Mon Sep 17 00:00:00 2001 From: Samuel Giddins Date: Sun, 21 Jul 2024 16:39:36 -0700 Subject: [PATCH 09/18] x-plat string_agg --- lib/gemstash/compact_index_builder.rb | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/gemstash/compact_index_builder.rb b/lib/gemstash/compact_index_builder.rb index 7f8ef754..6dc3489a 100644 --- a/lib/gemstash/compact_index_builder.rb +++ b/lib/gemstash/compact_index_builder.rb @@ -185,6 +185,7 @@ def build_result private def requirements_and_dependencies + db_type = DB::Rubygem.db.database_type DB::Rubygem.association_left_join(versions: :dependencies). where(name: @name). where { versions[:indexed] }. @@ -193,8 +194,12 @@ def requirements_and_dependencies [versions[:number], versions[:platform], versions[:sha256], versions[:info_checksum], versions[:required_ruby_version], versions[:required_rubygems_version], versions[:created_at]] end. # rubocop:disable Style/MultilineBlockChain select_more do - [string_agg(dependencies[:requirements], "@").order(dependencies[:rubygem_name], dependencies[:id]).as(:dep_req_agg), - string_agg(dependencies[:rubygem_name], ",").order(dependencies[:rubygem_name]).as(:dep_name_agg)] + sa = case db_type + when :sqlite then ->(a, b) { string_agg(a, b) } + else ->(a, b) { Sequel.string_agg(a, b) } + end + [sa.call(dependencies[:requirements], "@").order(dependencies[:rubygem_name], dependencies[:id]).as(:dep_req_agg), + sa.call(dependencies[:rubygem_name], ",").order(dependencies[:rubygem_name]).as(:dep_name_agg)] end. # rubocop:disable Style/MultilineBlockChain map do |row| reqs = row[:dep_req_agg]&.split("@") From 1796ac5ebf99d1d988d9f6a7713dd58cf24d57e9 Mon Sep 17 00:00:00 2001 From: Samuel Giddins Date: Sun, 21 Jul 2024 16:59:19 -0700 Subject: [PATCH 10/18] use concat function for mysql compatibility --- lib/gemstash/compact_index_builder.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gemstash/compact_index_builder.rb b/lib/gemstash/compact_index_builder.rb index 6dc3489a..6ca290e3 100644 --- a/lib/gemstash/compact_index_builder.rb +++ b/lib/gemstash/compact_index_builder.rb @@ -106,7 +106,7 @@ def compact_index_versions(date) UNION ALL - SELECT r.name as name, v.yanked_at as date, v.yanked_info_checksum as info_checksum, '-'||v.number as number, v.platform as platform + SELECT r.name as name, v.yanked_at as date, v.yanked_info_checksum as info_checksum, concat('-', v.number) as number, v.platform as platform FROM rubygems AS r, versions AS v WHERE v.rubygem_id = r.id AND v.indexed is false AND From 48bce30e2fcdc495911e0d1d715b6d7e0bf3d59b Mon Sep 17 00:00:00 2001 From: Samuel Giddins Date: Mon, 22 Jul 2024 10:46:20 -0700 Subject: [PATCH 11/18] Attempt to fix integration specs on postgres on ci --- lib/gemstash/compact_index_builder.rb | 2 -- spec/integration_spec.rb | 2 +- spec/support/exec_helpers.rb | 9 ++++++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/gemstash/compact_index_builder.rb b/lib/gemstash/compact_index_builder.rb index 6ca290e3..bec54407 100644 --- a/lib/gemstash/compact_index_builder.rb +++ b/lib/gemstash/compact_index_builder.rb @@ -115,8 +115,6 @@ def compact_index_versions(date) ORDER BY date, number, platform, name SQL - # not ordered correctly in sqlite for some reason - all_versions.sort_by! {|v| [v[:date], v[:number], v[:platform], v[:name]] } map_gem_versions(all_versions.map {|v| [v[:name], [v]] }) end diff --git a/spec/integration_spec.rb b/spec/integration_spec.rb index de81a79f..772085ff 100644 --- a/spec/integration_spec.rb +++ b/spec/integration_spec.rb @@ -245,7 +245,7 @@ shared_examples "a bundleable project" do it "successfully bundles" do env = { "HOME" => dir } - expect(execute("bundle", dir: dir, env: env)).to exit_success + expect(execute("bundle", %w[install --verbose], dir: dir, env: env)).to exit_success expect(execute("bundle", %w[exec speaker hi], dir: dir, env: env)). to exit_success.and_output("Hello world, #{platform_message}\n") end diff --git a/spec/support/exec_helpers.rb b/spec/support/exec_helpers.rb index 1838a688..cf526fb7 100644 --- a/spec/support/exec_helpers.rb +++ b/spec/support/exec_helpers.rb @@ -16,7 +16,7 @@ def execute(command, args = [], dir: nil, env: {}) # Executes and stores the results for an external command. class Result - attr_reader :command, :args, :dir, :output + attr_reader :command, :args, :dir, :output, :err def initialize(env, command, args, dir) @command = command @@ -29,7 +29,7 @@ def initialize(env, command, args, dir) end def exec - @output, @status = Open3.capture2(patched_env, command, *args, chdir: dir) + @output, @err, @status = Open3.capture3(patched_env, command, *args, chdir: dir) end def successful? @@ -148,7 +148,10 @@ def exec_in_process(binstub) else "expected '#{actual.display_command}' in '#{actual.dir}' to exit with a success code, but it didn't. the command output was: -#{actual.output}" +#{actual.output} + +and the error was: +#{actual.err}" end end end From d37f7583b7090c68b56c2285828beb98b1ba2a5d Mon Sep 17 00:00:00 2001 From: Samuel Giddins Date: Tue, 23 Jul 2024 14:52:30 -0700 Subject: [PATCH 12/18] Fail early if running old sqlite --- lib/gemstash/env.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gemstash/env.rb b/lib/gemstash/env.rb index 221aa1c7..bb4d3261 100644 --- a/lib/gemstash/env.rb +++ b/lib/gemstash/env.rb @@ -130,7 +130,7 @@ def db Sequel.connect("jdbc:sqlite:#{db_path}", config.database_connection_config) else Sequel.connect("sqlite://#{db_path}", config.database_connection_config) - end + end.tap {|db| raise "SQLite 3.44+ required, have #{db.sqlite_version}" unless db.sqlite_version >= 34_400 } when "postgres", "mysql", "mysql2" db_url = config[:db_url] raise "Missing DB URL" unless db_url From 203da4bf07e2c834e3a75e415aad7d0612e50c5b Mon Sep 17 00:00:00 2001 From: Samuel Giddins Date: Wed, 2 Oct 2024 18:08:04 -0700 Subject: [PATCH 13/18] jruby fixes Signed-off-by: Samuel Giddins --- gemstash.gemspec | 4 ++-- lib/gemstash/compact_index_builder.rb | 26 ++++++++------------------ lib/gemstash/env.rb | 3 ++- spec/integration_spec.rb | 2 +- 4 files changed, 13 insertions(+), 22 deletions(-) diff --git a/gemstash.gemspec b/gemstash.gemspec index 5522f6ea..ce232a01 100644 --- a/gemstash.gemspec +++ b/gemstash.gemspec @@ -39,7 +39,7 @@ you push your own private gems as well." spec.add_runtime_dependency "lru_redux", "~> 1.1" spec.add_runtime_dependency "psych", ">= 3.2.1" spec.add_runtime_dependency "puma", "~> 6.1" - spec.add_runtime_dependency "sequel", "~> 5.0" + spec.add_runtime_dependency "sequel", "~> 5.85" spec.add_runtime_dependency "server_health_check-rack", "~> 0.1" spec.add_runtime_dependency "sinatra", ">= 1.4", "< 5.0" spec.add_runtime_dependency "terminal-table", "~> 3.0" @@ -53,7 +53,7 @@ you push your own private gems as well." # spec.add_runtime_dependency "mysql2", "~> 0.4" if RUBY_PLATFORM == "java" - spec.add_runtime_dependency "jdbc-sqlite3", "~> 3.8" + spec.add_runtime_dependency "jdbc-sqlite3", "~> 3.46" else # SQLite 3.44+ is required for string_agg support spec.add_runtime_dependency "sqlite3", ">= 1.68", "< 3.0" diff --git a/lib/gemstash/compact_index_builder.rb b/lib/gemstash/compact_index_builder.rb index bec54407..a0b31b55 100644 --- a/lib/gemstash/compact_index_builder.rb +++ b/lib/gemstash/compact_index_builder.rb @@ -87,7 +87,7 @@ def build_result(force_rebuild: false) else ts = Time.now.iso8601 versions_file.create( - compact_index_public_versions(ts), ts + compact_index_public_versions(ts), ts.to_s.sub(/\+00:00\z/, "Z") ) @result = file.read resource.save("versions.list" => @result) @@ -176,14 +176,11 @@ def fetch_resource def build_result @result = CompactIndex.info(requirements_and_dependencies) - rescue Sequel::DatabaseError => e - raise "Error building info for #{@name}\n\n```sql\n#{e.sql}\n```\n" end private def requirements_and_dependencies - db_type = DB::Rubygem.db.database_type DB::Rubygem.association_left_join(versions: :dependencies). where(name: @name). where { versions[:indexed] }. @@ -192,24 +189,17 @@ def requirements_and_dependencies [versions[:number], versions[:platform], versions[:sha256], versions[:info_checksum], versions[:required_ruby_version], versions[:required_rubygems_version], versions[:created_at]] end. # rubocop:disable Style/MultilineBlockChain select_more do - sa = case db_type - when :sqlite then ->(a, b) { string_agg(a, b) } - else ->(a, b) { Sequel.string_agg(a, b) } - end - [sa.call(dependencies[:requirements], "@").order(dependencies[:rubygem_name], dependencies[:id]).as(:dep_req_agg), - sa.call(dependencies[:rubygem_name], ",").order(dependencies[:rubygem_name]).as(:dep_name_agg)] + [coalesce(Sequel.string_agg(dependencies[:requirements], "@").order(dependencies[:rubygem_name], dependencies[:id]), "").as(:dep_req_agg), + coalesce(Sequel.string_agg(dependencies[:rubygem_name], ",").order(dependencies[:rubygem_name]), "").as(:dep_name_agg)] end. # rubocop:disable Style/MultilineBlockChain map do |row| - reqs = row[:dep_req_agg]&.split("@") - dep_names = row[:dep_name_agg]&.split(",") + reqs = row[:dep_req_agg].split("@") + dep_names = row[:dep_name_agg].split(",") - raise "Dependencies and requirements are not the same size:\n reqs: #{reqs.inspect}\n dep_names: #{dep_names.inspect}\n row: #{row.inspect}" if dep_names&.size != reqs&.size + raise "Dependencies and requirements are not the same size:\n reqs: #{reqs.inspect}\n dep_names: #{dep_names.inspect}\n row: #{row.inspect}" if dep_names.size != reqs.size - deps = [] - if reqs - dep_names.zip(reqs).each do |name, req| - deps << CompactIndex::Dependency.new(name, req) - end + deps = dep_names.zip(reqs).map! do |name, req| + CompactIndex::Dependency.new(name, req) end CompactIndex::GemVersion.new( diff --git a/lib/gemstash/env.rb b/lib/gemstash/env.rb index bb4d3261..828ca4a9 100644 --- a/lib/gemstash/env.rb +++ b/lib/gemstash/env.rb @@ -130,7 +130,8 @@ def db Sequel.connect("jdbc:sqlite:#{db_path}", config.database_connection_config) else Sequel.connect("sqlite://#{db_path}", config.database_connection_config) - end.tap {|db| raise "SQLite 3.44+ required, have #{db.sqlite_version}" unless db.sqlite_version >= 34_400 } + end + raise "SQLite 3.44+ required, have #{db.sqlite_version}" unless db.sqlite_version >= 34_400 when "postgres", "mysql", "mysql2" db_url = config[:db_url] raise "Missing DB URL" unless db_url diff --git a/spec/integration_spec.rb b/spec/integration_spec.rb index 772085ff..a8d1879f 100644 --- a/spec/integration_spec.rb +++ b/spec/integration_spec.rb @@ -127,7 +127,7 @@ it "is a conformant gem server", db_transaction: false do @gemstash.env.cache.flush expect( - execute("gem_server_conformance", ["--fail-fast", "--format", "progress", "--tag=~content_length_header"], + execute("gem_server_conformance", ["--format", "progress", "--tag=~content_length_header"], env: { "UPSTREAM" => host, "GEM_HOST_API_KEY" => auth_key }) ). to exit_success From 07925fa43a9c00d61a98c0b34d3c812b587a703e Mon Sep 17 00:00:00 2001 From: Samuel Giddins Date: Tue, 19 Nov 2024 13:38:11 -0800 Subject: [PATCH 14/18] Fix failing spec on ruby 3.3 Signed-off-by: Samuel Giddins --- lib/gemstash/version.rb | 2 +- spec/support/exec_helpers.rb | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/gemstash/version.rb b/lib/gemstash/version.rb index 70ee42a4..b857f637 100644 --- a/lib/gemstash/version.rb +++ b/lib/gemstash/version.rb @@ -2,5 +2,5 @@ # :nodoc: module Gemstash - VERSION = "2.7.1" + VERSION = "2.8.0" end diff --git a/spec/support/exec_helpers.rb b/spec/support/exec_helpers.rb index cf526fb7..2ab98a96 100644 --- a/spec/support/exec_helpers.rb +++ b/spec/support/exec_helpers.rb @@ -144,7 +144,10 @@ def exec_in_process(binstub) #{@expected_output} but instead it output: -#{actual.output}" +#{actual.output} + +and the error was: +#{actual.err}" else "expected '#{actual.display_command}' in '#{actual.dir}' to exit with a success code, but it didn't. the command output was: @@ -154,4 +157,12 @@ def exec_in_process(binstub) #{actual.err}" end end + + failure_message_when_negated do |actual| + "expected '#{actual.display_command}' in '#{actual.dir}' to not exit with a success code, but it did +#{actual.output} + +and the error was: +#{actual.err}" + end end From a46268c947561bbdc13e08bdb5f61572f24e12a5 Mon Sep 17 00:00:00 2001 From: Samuel Giddins Date: Tue, 25 Feb 2025 20:42:09 +0100 Subject: [PATCH 15/18] Bump gem_server_conformance requirement Signed-off-by: Samuel Giddins --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index da9e15c6..b32c7666 100644 --- a/Gemfile +++ b/Gemfile @@ -29,6 +29,6 @@ group :linting do end group :test do - gem "gem_server_conformance", "~> 0.1.4" + gem "gem_server_conformance", "~> 0.1.5" gem "mock_redis" end From 42a33e3d92f35ff0ac848df03922c0f31ac1c4bc Mon Sep 17 00:00:00 2001 From: Samuel Giddins Date: Tue, 25 Feb 2025 20:57:21 +0100 Subject: [PATCH 16/18] Debug jruby gem server conformance failures Signed-off-by: Samuel Giddins --- spec/support/jruby_binstubs/gem_server_conformance | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/spec/support/jruby_binstubs/gem_server_conformance b/spec/support/jruby_binstubs/gem_server_conformance index 44bc4900..5331e474 100644 --- a/spec/support/jruby_binstubs/gem_server_conformance +++ b/spec/support/jruby_binstubs/gem_server_conformance @@ -1,3 +1,9 @@ require "rubygems" -load Gem.bin_path("gem_server_conformance", "gem_server_conformance") +begin + load Gem.bin_path("gem_server_conformance", "gem_server_conformance") +rescue Gem::GemNotFoundException + require "bundler" + Bundler::Env.write($stderr) + raise +end From d0cedb5b482c8eeb5931869689c7e54f9250ee50 Mon Sep 17 00:00:00 2001 From: Samuel Giddins Date: Mon, 10 Mar 2025 14:42:55 -0700 Subject: [PATCH 17/18] Try requiring bundler/setup on jruby binstub Signed-off-by: Samuel Giddins --- lib/gemstash/cli/info.rb | 3 --- spec/support/jruby_binstubs/gem_server_conformance | 1 + 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/gemstash/cli/info.rb b/lib/gemstash/cli/info.rb index 5ee921d9..100dcfe2 100644 --- a/lib/gemstash/cli/info.rb +++ b/lib/gemstash/cli/info.rb @@ -12,9 +12,6 @@ class Info < Gemstash::CLI::Base def run prepare list_config - - # Gemstash::DB - # Gemstash::Env.current.db.dump_schema_migration(same_db: true) end private diff --git a/spec/support/jruby_binstubs/gem_server_conformance b/spec/support/jruby_binstubs/gem_server_conformance index 5331e474..3d1726eb 100644 --- a/spec/support/jruby_binstubs/gem_server_conformance +++ b/spec/support/jruby_binstubs/gem_server_conformance @@ -1,4 +1,5 @@ require "rubygems" +require "bundler/setup" begin load Gem.bin_path("gem_server_conformance", "gem_server_conformance") From 28535179b4864cc48e2c2d625b8e9fbc2ed6f810 Mon Sep 17 00:00:00 2001 From: Samuel Giddins Date: Wed, 9 Jul 2025 14:46:55 -0400 Subject: [PATCH 18/18] Clean up compact index builder and remove schema.rb Remove boilerplate comment placeholders from CompactIndexBuilder classes and delete schema.rb file as requested in review comments. --- lib/gemstash/compact_index_builder.rb | 4 -- schema.rb | 94 --------------------------- 2 files changed, 98 deletions(-) delete mode 100644 schema.rb diff --git a/lib/gemstash/compact_index_builder.rb b/lib/gemstash/compact_index_builder.rb index a0b31b55..20ec86eb 100644 --- a/lib/gemstash/compact_index_builder.rb +++ b/lib/gemstash/compact_index_builder.rb @@ -7,7 +7,6 @@ require "zlib" module Gemstash - # Comment class CompactIndexBuilder include Gemstash::Env::Helper attr_reader :result @@ -67,7 +66,6 @@ def check_auth @auth.check("fetch") end - # Comment class Versions < CompactIndexBuilder def fetch_resource storage.resource("versions") @@ -163,7 +161,6 @@ def key end end - # Comment class Info < CompactIndexBuilder def initialize(auth, name) super(auth) @@ -219,7 +216,6 @@ def key end end - # Comment class Names < CompactIndexBuilder def fetch_resource storage.resource("names") diff --git a/schema.rb b/schema.rb deleted file mode 100644 index 76b704fa..00000000 --- a/schema.rb +++ /dev/null @@ -1,94 +0,0 @@ -# frozen_string_literal: true - -Sequel.migration do - change do - create_table(:authorizations) do - primary_key :id - column :auth_key, "varchar(191)", :null => false - column :permissions, "varchar(191)", :null => false - column :created_at, "timestamp", :null => false - column :updated_at, "timestamp", :null => false - column :name, "varchar(191)" - - index [:auth_key], :unique => true - index [:name], :unique => true - end - - create_table(:cached_rubygems) do - primary_key :id - column :upstream_id, "INTEGER", :null => false - column :name, "varchar(191)", :null => false - column :resource_type, "varchar(191)", :null => false - column :created_at, "timestamp", :null => false - column :updated_at, "timestamp", :null => false - - index [:name] - index %i[upstream_id resource_type name], :unique => true - end - - create_table(:dependencies) do - primary_key :id - column :version_id, "INTEGER", :null => false - column :rubygem_name, "varchar(191)", :null => false - column :requirements, "varchar(191)", :null => false - column :created_at, "timestamp", :null => false - column :updated_at, "timestamp", :null => false - - index [:rubygem_name] - index [:version_id] - end - - create_table(:health_tests) do - primary_key :id - column :string, "varchar(255)" - end - - create_table(:rubygems) do - primary_key :id - column :name, "varchar(191)", :null => false - column :created_at, "timestamp", :null => false - column :updated_at, "timestamp", :null => false - - index [:name], :unique => true - end - - create_table(:schema_info) do - column :version, "INTEGER", :default => 0, :null => false - end - - create_table(:upstreams) do - primary_key :id - column :uri, "varchar(191)", :null => false - column :host_id, "varchar(191)", :null => false - column :created_at, "timestamp", :null => false - column :updated_at, "timestamp", :null => false - - index [:host_id], :unique => true - index [:uri], :unique => true - end - - create_table(:versions) do - primary_key :id - column :rubygem_id, "INTEGER", :null => false - column :storage_id, "varchar(191)", :null => false - column :number, "varchar(191)", :null => false - column :platform, "varchar(191)", :null => false - column :full_name, "varchar(191)", :null => false - column :indexed, "boolean", :default => true, :null => false - column :prerelease, "boolean", :null => false - column :created_at, "timestamp", :null => false - column :updated_at, "timestamp", :null => false - column :info_checksum, "varchar(40)" - column :sha256, "varchar(64)" - column :required_ruby_version, "varchar(191)" - column :required_rubygems_version, "varchar(191)" - - index [:full_name], :unique => true - index [:indexed] - index %i[indexed prerelease] - index [:number] - index %i[rubygem_id number platform], :unique => true - index [:storage_id], :unique => true - end - end -end