Add Shard#merged_with when a canonical repo becomes a mirror

db/migrations/20200506215505_add_shards_merged_with.sql

@@ -0,0 +1,52 @@
+-- migrate:up
+
+ALTER TABLE shards
+ ADD COLUMN merged_with bigint,
+ ADD CONSTRAINT shards_merged_with_fk FOREIGN KEY (merged_with) REFERENCES shards(id),
+ ADD CONSTRAINT shards_merged_with_archived_at CHECK (merged_with IS NULL OR (archived_at IS NOT NULL AND categories = '{}')),
+ DROP CONSTRAINT shards_name_unique,
+ ADD CONSTRAINT shards_name_unique UNIQUE (name, qualifier) DEFERRABLE INITIALLY IMMEDIATE
+;
+
+-- Update all existing archived shards to point to the shard they were merged into
+UPDATE shards
+SET
+ merged_with = log.shard_id
+FROM activity_log log
+WHERE log.event = 'import_catalog:mirror:switched'
+ AND log.metadata->>'old_role' = 'canonical'
+ AND log.metadata->'old_shard_id' <> 'null'
+ AND shards.id = (metadata->'old_shard_id')::bigint
+;
+
+-- Remove dependencies on the merged shard (they should be picked up by the merge target)
+DELETE FROM shard_dependencies
+USING shards
+WHERE shards.id = shard_id
+ AND shards.merged_with IS NOT NULL
+;
+
+-- Switch qualifiers if a merged shard has the empty qualifier
+SET CONSTRAINTS shards_name_unique DEFERRED;
+UPDATE shards
+ SET qualifier = main.qualifier
+FROM shards main
+WHERE shards.merged_with = main.id
+ AND shards.name = main.name
+ AND shards.qualifier = ''
+;
+UPDATE shards
+ SET qualifier = ''
+FROM shards merged
+WHERE shards.id = merged.merged_with
+ AND shards.name = merged.name
+ AND shards.qualifier = merged.qualifier
+;
+
+-- migrate:down
+
+ALTER TABLE shards
+ DROP COLUMN merged_with,
+ DROP CONSTRAINT shards_name_unique,
+ ADD CONSTRAINT shards_name_unique UNIQUE (name, qualifier)
+;

db/schema.sql

@@ -990,6 +990,8 @@ CREATE TABLE public.shards (
  updated_at timestamp with time zone DEFAULT now() NOT NULL,
  categories bigint[] DEFAULT '{}'::bigint[] NOT NULL,
  archived_at timestamp with time zone,
+ merged_with bigint,
+ CONSTRAINT shards_merged_with_archived_at CHECK (((merged_with IS NULL) OR ((archived_at IS NOT NULL) AND (categories = '{}'::bigint[])))),
  CONSTRAINT shards_name_check CHECK ((name OPERATOR(public.~) '^[A-Za-z0-9_\-.]{1,100}$'::text)),
  CONSTRAINT shards_qualifier_check CHECK ((qualifier OPERATOR(public.~) '^[A-Za-z0-9_\-.]{0,100}$'::public.citext))
 );
@@ -1174,7 +1176,7 @@ ALTER TABLE ONLY public.shard_metrics
 --
 
 ALTER TABLE ONLY public.shards
- ADD CONSTRAINT shards_name_unique UNIQUE (name, qualifier);
+ ADD CONSTRAINT shards_name_unique UNIQUE (name, qualifier) DEFERRABLE;
 
 
 --
@@ -1395,6 +1397,14 @@ ALTER TABLE ONLY public.shard_metrics
  ADD CONSTRAINT shard_metrics_shard_id_fkey FOREIGN KEY (shard_id) REFERENCES public.shards(id) ON DELETE CASCADE DEFERRABLE;
 
 
+--
+-- Name: shards shards_merged_with_fk; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.shards
+ ADD CONSTRAINT shards_merged_with_fk FOREIGN KEY (merged_with) REFERENCES public.shards(id);
+
+
 --
 -- PostgreSQL database dump complete
 --
@@ -1411,4 +1421,5 @@ INSERT INTO public.schema_migrations (version) VALUES
  ('20191106093828'),
  ('20191115142944'),
  ('20191122122940'),
- ('20200503132444');
+ ('20200503132444'),
+ ('20200506215505');

spec/service/import_catalog_spec.cr

@@ -520,9 +520,10 @@ describe Service::ImportCatalog do
  {"git", "foo/foo", "canonical", foo_shard_id},
  ]
 
- db.get_shards.map { |shard| {shard.id, shard.name} }.should eq [
- {foo_shard_id, "foo"},
+ db.get_shards.map { |shard| {shard.id, shard.name, shard.qualifier} }.should eq [
+ {foo_shard_id, "foo", ""},
  ]
+ db.get_shard(bar_shard_id).merged_with.should eq foo_shard_id
  shard_categorizations(db).should eq [
  {"foo", "", ["category"]},
  ]
@@ -546,6 +547,28 @@ describe Service::ImportCatalog do
  end
  end
 
+ it "archives unreferenced shard and moves repo to mirror, taking over the main shard qualifier" do
+ transaction do |db|
+ foo_shard_id = Factory.create_shard(db, "baz", "foo")
+ bar_shard_id = Factory.create_shard(db, "baz")
+
+ service = Service::ImportCatalog.new(db, Catalog.empty)
+ service.merge_shard(bar_shard_id, foo_shard_id)
+
+ foo_shard = db.get_shard(foo_shard_id)
+ foo_shard.name.should eq "baz"
+ foo_shard.qualifier.should eq ""
+ bar_shard = db.get_shard(bar_shard_id)
+ bar_shard.name.should eq "baz"
+ bar_shard.qualifier.should eq "foo"
+ bar_shard.merged_with.should eq foo_shard_id
+
+ db.last_activities.map { |a| {a.event, a.repo_id, a.shard_id, a.metadata} }.should eq [
+ {"import_catalog:shard:archived", nil, bar_shard_id, nil},
+ ]
+ end
+ end
+
  it "archives and re-activates shard" do
  with_tempdir("import_catalog-mirrors") do |catalog_path|
  File.write(File.join(catalog_path, "category.yml"), <<-YAML)

src/db.cr

@@ -116,17 +116,17 @@ class ShardsDB
  end
 
  def get_shard(shard_id : Int64)
- result = connection.query_one <<-SQL, shard_id, as: {Int64, String, String, String?, Time?}
+ result = connection.query_one <<-SQL, shard_id, as: {Int64, String, String, String?, Time?, Int64?}
  SELECT
- id, name::text, qualifier::text, description, archived_at
+ id, name::text, qualifier::text, description, archived_at, merged_with
  FROM
  shards
  WHERE
  id = $1
  SQL
 
- id, name, qualifier, description, archived_at = result
- Shard.new(name, qualifier, description, archived_at, id: id)
+ id, name, qualifier, description, archived_at, merged_with = result
+ Shard.new(name, qualifier, description, archived_at, merged_with, id: id)
  end
 
  def get_shards

src/service/import_catalog.cr

@@ -220,6 +220,10 @@ struct Service::ImportCatalog
  "old_shard_id" => repo.shard_id,
  "old_role" => repo.role,
  }
+
+ if repo.role.canonical? && (old_shard_id = repo.shard_id)
+ merge_shard(old_shard_id, shard_id)
+ end
  else
  repo = Repo.new(mirror.repo_ref, shard_id, mirror.role)
  repo.id = @db.create_repo(repo)
@@ -248,6 +252,31 @@ struct Service::ImportCatalog
  end
  end
 
+ def merge_shard(old_shard_id, shard_id)
+ old_shard = @db.get_shard(old_shard_id)
+ shard = @db.get_shard(shard_id)
+
+ if old_shard.name == shard.name && old_shard.qualifier == ""
+ @db.connection.exec "SET CONSTRAINTS shards_name_unique DEFERRED"
+ set_qualifier(shard_id, "")
+ set_qualifier(old_shard_id, shard.qualifier)
+ end
+
+ archive_shard(old_shard_id, merged_with: shard_id)
+ end
+
+ private def set_qualifier(shard_id, qualifier)
+ result = @db.connection.exec <<-SQL, shard_id, qualifier
+ UPDATE shards
+ SET qualifier = $2
+ WHERE id = $1
+ SQL
+
+ if result.rows_affected != 1
+ raise "set_qualifier could not be applied to #{shard_id}"
+ end
+ end
+
  def archive_unreferenced_shards
  unreferenced_shards = @db.connection.query_all <<-SQL, as: Int64
  SELECT
@@ -265,21 +294,29 @@ struct Service::ImportCatalog
  SQL
 
  unreferenced_shards.each do |shard_id|
- @db.log_activity("import_catalog:shard:archived", nil, shard_id)
- @db.connection.exec <<-SQL, shard_id
- UPDATE
- shards
- SET
- archived_at = NOW(),
- categories = '{}'
- WHERE
- id = $1
- SQL
+ archive_shard(shard_id)
  end
 
  @unreferenced_shards = unreferenced_shards
  end
 
+ private def archive_shard(shard_id, merged_with = nil)
+ @db.log_activity("import_catalog:shard:archived", nil, shard_id)
+ result = @db.connection.exec <<-SQL, shard_id, merged_with
+ UPDATE
+ shards
+ SET
+ archived_at = NOW(),
+ categories = '{}',
+ merged_with = $2
+ WHERE id = $1
+ AND archived_at IS NULL
+ SQL
+ if result.rows_affected != 1
+ raise "archive_shard could not be applied to #{shard_id}"
+ end
+ end
+
  private def obsolete_removed_repos(valid_refs)
  resolvers = Array(String).new(valid_refs.size)
  urls = Array(String).new(valid_refs.size)

src/shard.cr

@@ -4,8 +4,9 @@ class Shard
  property description : String?
  property! id : Int64
  property archived_at : Time?
+ property merged_with : Int64?
 
- def initialize(@name : String, @qualifier : String = "", @description : String? = nil, @archived_at : Time? = nil, @id : Int64? = nil)
+ def initialize(@name : String, @qualifier : String = "", @description : String? = nil, @archived_at : Time? = nil, @merged_with : Int64? = nil, @id : Int64? = nil)
  end
 
  def_equals_and_hash name, qualifier, description, archived_at