From be9624b23e2eca05b0c79a4db851a43835e102bb Mon Sep 17 00:00:00 2001 From: dmitrytrager Date: Tue, 3 Feb 2026 15:44:44 +0100 Subject: [PATCH 1/5] Add manifest API with versioning and change detection Implement GET /api/v1/beacons/manifest to build and return the full manifest JSON (language, region, tags, providers with nested topics and files, totals, checksum). Implement HEAD support with If-None-Match (304 Not Modified) and If-Match (412 Precondition Failed) for lightweight change detection during sync. Manifest versioning uses lazy evaluation: content checksum is compared with stored value on the beacon, and the version is incremented only when content actually changes. Conditional requests use the stored version directly, avoiding a full manifest build for 304/412 responses. Co-Authored-By: Claude Opus 4.5 --- .../api/v1/beacons/manifests_controller.rb | 52 +++++ app/models/beacon.rb | 20 +- app/services/beacons/manifest_builder.rb | 134 ++++++++++++ config/routes.rb | 1 + ...0005_add_manifest_versioning_to_beacons.rb | 6 + db/schema.rb | 4 +- spec/factories/beacons.rb | 20 +- spec/models/beacon_spec.rb | 20 +- .../requests/api/v1/beacons/manifests_spec.rb | 155 ++++++++++++++ .../services/beacons/manifest_builder_spec.rb | 201 ++++++++++++++++++ 10 files changed, 585 insertions(+), 28 deletions(-) create mode 100644 app/controllers/api/v1/beacons/manifests_controller.rb create mode 100644 app/services/beacons/manifest_builder.rb create mode 100644 db/migrate/20260203100005_add_manifest_versioning_to_beacons.rb create mode 100644 spec/requests/api/v1/beacons/manifests_spec.rb create mode 100644 spec/services/beacons/manifest_builder_spec.rb diff --git a/app/controllers/api/v1/beacons/manifests_controller.rb b/app/controllers/api/v1/beacons/manifests_controller.rb new file mode 100644 index 00000000..de5beb0f --- /dev/null +++ b/app/controllers/api/v1/beacons/manifests_controller.rb @@ -0,0 +1,52 @@ +module Api + module V1 + module Beacons + class ManifestsController < Beacons::BaseController + def show + stored_etag = "v#{Current.beacon.manifest_version}" + + # During sync, beacon sends If-Match with its current version. + # If the manifest changed since sync started, return 412 so beacon aborts and restarts. + if stale_by_match?(stored_etag) + response.headers["ETag"] = stored_etag + head :precondition_failed + return + end + + # Beacon sends If-None-Match with its cached version to check for updates. + # If versions match, return 304 — no sync needed. + if fresh_by_none_match?(stored_etag) + response.headers["ETag"] = stored_etag + head :not_modified + return + end + + # Build full manifest only when we need to return the body + manifest = manifest_builder.call + response.headers["ETag"] = manifest[:manifest_version] + render json: manifest + end + + private + + def manifest_builder + ::Beacons::ManifestBuilder.new(Current.beacon) + end + + def stale_by_match?(etag) + if_match = request.headers["If-Match"] + return false if if_match.blank? + + if_match != etag + end + + def fresh_by_none_match?(etag) + if_none_match = request.headers["If-None-Match"] + return false if if_none_match.blank? + + if_none_match == etag + end + end + end + end +end diff --git a/app/models/beacon.rb b/app/models/beacon.rb index d613ae38..f96530c8 100644 --- a/app/models/beacon.rb +++ b/app/models/beacon.rb @@ -3,15 +3,17 @@ # Table name: beacons # Database name: primary # -# id :bigint not null, primary key -# api_key_digest :string not null -# api_key_prefix :string not null -# name :string not null -# revoked_at :datetime -# created_at :datetime not null -# updated_at :datetime not null -# language_id :bigint not null -# region_id :bigint not null +# id :bigint not null, primary key +# api_key_digest :string not null +# api_key_prefix :string not null +# manifest_checksum :string +# manifest_version :integer default(0), not null +# name :string not null +# revoked_at :datetime +# created_at :datetime not null +# updated_at :datetime not null +# language_id :bigint not null +# region_id :bigint not null # # Indexes # diff --git a/app/services/beacons/manifest_builder.rb b/app/services/beacons/manifest_builder.rb new file mode 100644 index 00000000..5fd17a67 --- /dev/null +++ b/app/services/beacons/manifest_builder.rb @@ -0,0 +1,134 @@ +module Beacons + class ManifestBuilder + def initialize(beacon) + @beacon = beacon + end + + def call + content = build_content + checksum = "sha256:#{compute_checksum(content)}" + version = resolve_version(checksum) + + content.merge( + manifest_version: "v#{version}", + manifest_checksum: checksum, + generated_at: Time.current.iso8601, + ) + end + + private + + attr_reader :beacon + + def resolve_version(checksum) + return beacon.manifest_version if beacon.manifest_checksum == checksum + + beacon.update!( + manifest_version: beacon.manifest_version + 1, + manifest_checksum: checksum, + ) + + beacon.manifest_version + end + + def build_content + providers_data = build_providers + + { + language: build_language, + region: build_region, + tags: build_tags, + providers: providers_data, + total_size_bytes: compute_total_size(providers_data), + total_files: compute_total_files(providers_data), + } + end + + def build_language + language = beacon.language + + { + id: language.id, + code: language.code, + name: language.name, + } + end + + def build_region + region = beacon.region + + { + id: region.id, + name: region.name, + } + end + + def build_tags + topics + .flat_map(&:tags) + .uniq(&:id) + .sort_by(&:id) + .map { |tag| { id: tag.id, name: tag.name } } + end + + def build_providers + topics_by_provider = topics.group_by(&:provider_id) + + beacon.providers.sort_by(&:id).map do |provider| + provider_topics = (topics_by_provider[provider.id] || []).sort_by(&:id) + + { + id: provider.id, + name: provider.name, + topics: provider_topics.map { |topic| build_topic(topic) }, + } + end + end + + def build_topic(topic) + { + id: topic.id, + name: topic.title, + tag_ids: topic.tags.map(&:id).sort, + files: topic.documents.sort_by { |d| d.blob.id }.map { |doc| build_file(topic, doc) }, + } + end + + def build_file(topic, document) + blob = document.blob + filename = topic.custom_file_name(document) + + { + id: blob.id, + filename: filename, + path: "providers/#{topic.provider_id}/topics/#{topic.id}/#{filename}", + checksum: blob.checksum, + size_bytes: blob.byte_size, + content_type: blob.content_type, + updated_at: blob.created_at.iso8601, + } + end + + def topics + @topics ||= beacon.topics.active.includes(:tags, :provider, documents_attachments: :blob) + end + + def compute_total_size(providers_data) + providers_data.sum do |provider| + provider[:topics].sum do |topic| + topic[:files].sum { |file| file[:size_bytes] } + end + end + end + + def compute_total_files(providers_data) + providers_data.sum do |provider| + provider[:topics].sum { |topic| topic[:files].size } + end + end + + def compute_checksum(content) + OpenSSL::Digest::SHA256.hexdigest(content.to_json) + end + end +end diff --git a/config/routes.rb b/config/routes.rb index bfa18754..89f2fc11 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -42,6 +42,7 @@ namespace :beacons do resource :status, only: :show + resource :manifest, only: :show end end end diff --git a/db/migrate/20260203100005_add_manifest_versioning_to_beacons.rb b/db/migrate/20260203100005_add_manifest_versioning_to_beacons.rb new file mode 100644 index 00000000..4457e3c0 --- /dev/null +++ b/db/migrate/20260203100005_add_manifest_versioning_to_beacons.rb @@ -0,0 +1,6 @@ +class AddManifestVersioningToBeacons < ActiveRecord::Migration[8.0] + def change + add_column :beacons, :manifest_version, :integer, default: 0, null: false + add_column :beacons, :manifest_checksum, :string + end +end diff --git a/db/schema.rb b/db/schema.rb index 46aaf864..edcf6fcb 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.1].define(version: 2026_02_03_100004) do +ActiveRecord::Schema[8.1].define(version: 2026_02_03_100005) do # These are extensions that must be enabled in order to support this database enable_extension "pg_catalog.plpgsql" @@ -67,6 +67,8 @@ t.string "api_key_prefix", null: false t.datetime "created_at", null: false t.bigint "language_id", null: false + t.string "manifest_checksum" + t.integer "manifest_version", default: 0, null: false t.string "name", null: false t.bigint "region_id", null: false t.datetime "revoked_at" diff --git a/spec/factories/beacons.rb b/spec/factories/beacons.rb index 41753b49..31351f08 100644 --- a/spec/factories/beacons.rb +++ b/spec/factories/beacons.rb @@ -3,15 +3,17 @@ # Table name: beacons # Database name: primary # -# id :bigint not null, primary key -# api_key_digest :string not null -# api_key_prefix :string not null -# name :string not null -# revoked_at :datetime -# created_at :datetime not null -# updated_at :datetime not null -# language_id :bigint not null -# region_id :bigint not null +# id :bigint not null, primary key +# api_key_digest :string not null +# api_key_prefix :string not null +# manifest_checksum :string +# manifest_version :integer default(0), not null +# name :string not null +# revoked_at :datetime +# created_at :datetime not null +# updated_at :datetime not null +# language_id :bigint not null +# region_id :bigint not null # # Indexes # diff --git a/spec/models/beacon_spec.rb b/spec/models/beacon_spec.rb index f397a629..b8a8c60f 100644 --- a/spec/models/beacon_spec.rb +++ b/spec/models/beacon_spec.rb @@ -3,15 +3,17 @@ # Table name: beacons # Database name: primary # -# id :bigint not null, primary key -# api_key_digest :string not null -# api_key_prefix :string not null -# name :string not null -# revoked_at :datetime -# created_at :datetime not null -# updated_at :datetime not null -# language_id :bigint not null -# region_id :bigint not null +# id :bigint not null, primary key +# api_key_digest :string not null +# api_key_prefix :string not null +# manifest_checksum :string +# manifest_version :integer default(0), not null +# name :string not null +# revoked_at :datetime +# created_at :datetime not null +# updated_at :datetime not null +# language_id :bigint not null +# region_id :bigint not null # # Indexes # diff --git a/spec/requests/api/v1/beacons/manifests_spec.rb b/spec/requests/api/v1/beacons/manifests_spec.rb new file mode 100644 index 00000000..dbd018cf --- /dev/null +++ b/spec/requests/api/v1/beacons/manifests_spec.rb @@ -0,0 +1,155 @@ +require "rails_helper" + +RSpec.describe "Beacons Manifest API", type: :request do + let(:language) { create(:language, name: "English") } + let(:region) { create(:region, name: "East Region") } + let(:provider) { create(:provider, name: "Health Ministry") } + + let(:beacon) do + b, @raw_key = create_beacon_with_key(language: language, region: region) + b.providers << provider + b + end + + let(:raw_key) { beacon; @raw_key } + + let(:topic) do + create(:topic, :with_documents, title: "Maternal Health", provider: provider, language: language).tap do |t| + t.tag_list.add("Prenatal") + t.save! + beacon.topics << t + end + end + + before { topic } + + describe "GET /api/v1/beacons/manifest" do + it "returns the full manifest with correct structure" do + get "/api/v1/beacons/manifest", headers: beacon_auth_headers(raw_key) + + expect(response).to have_http_status(:ok) + + body = response.parsed_body + expect(body["manifest_version"]).to eq("v1") + expect(body["manifest_checksum"]).to start_with("sha256:") + expect(body["generated_at"]).to be_present + expect(body["language"]["name"]).to eq("English") + expect(body["language"]["code"]).to eq("en") + expect(body["region"]["name"]).to eq("East Region") + expect(body["tags"].first["name"]).to eq("prenatal") + expect(body["providers"].first["name"]).to eq("Health Ministry") + expect(body["providers"].first["topics"].first["name"]).to eq("Maternal Health") + expect(body["total_files"]).to eq(1) + expect(body["total_size_bytes"]).to be > 0 + end + + it "returns ETag header with manifest version" do + get "/api/v1/beacons/manifest", headers: beacon_auth_headers(raw_key) + + expect(response.headers["ETag"]).to eq("v1") + end + + it "increments version when content changes" do + get "/api/v1/beacons/manifest", headers: beacon_auth_headers(raw_key) + expect(response.parsed_body["manifest_version"]).to eq("v1") + + new_topic = create(:topic, :with_documents, title: "New Topic", provider: provider, language: language) + beacon.topics << new_topic + + get "/api/v1/beacons/manifest", headers: beacon_auth_headers(raw_key) + expect(response.parsed_body["manifest_version"]).to eq("v2") + expect(response.headers["ETag"]).to eq("v2") + end + + it "requires authentication" do + get "/api/v1/beacons/manifest" + + expect(response).to have_http_status(:unauthorized) + end + end + + describe "HEAD /api/v1/beacons/manifest" do + context "with If-None-Match" do + it "returns 304 when ETag matches current version" do + get "/api/v1/beacons/manifest", headers: beacon_auth_headers(raw_key) + etag = response.headers["ETag"] + + head "/api/v1/beacons/manifest", headers: beacon_auth_headers(raw_key).merge("If-None-Match" => etag) + + expect(response).to have_http_status(:not_modified) + expect(response.headers["ETag"]).to eq(etag) + end + + it "returns 200 when ETag does not match current version" do + get "/api/v1/beacons/manifest", headers: beacon_auth_headers(raw_key) + + head "/api/v1/beacons/manifest", headers: beacon_auth_headers(raw_key).merge("If-None-Match" => "v999") + + expect(response).to have_http_status(:ok) + expect(response.headers["ETag"]).to be_present + end + end + + context "with If-Match" do + it "returns 200 when ETag matches current version" do + get "/api/v1/beacons/manifest", headers: beacon_auth_headers(raw_key) + etag = response.headers["ETag"] + + head "/api/v1/beacons/manifest", headers: beacon_auth_headers(raw_key).merge("If-Match" => etag) + + expect(response).to have_http_status(:ok) + expect(response.headers["ETag"]).to eq(etag) + end + + it "returns 412 when ETag does not match current version" do + get "/api/v1/beacons/manifest", headers: beacon_auth_headers(raw_key) + etag = response.headers["ETag"] + + head "/api/v1/beacons/manifest", headers: beacon_auth_headers(raw_key).merge("If-Match" => "v999") + + expect(response).to have_http_status(:precondition_failed) + expect(response.headers["ETag"]).to eq(etag) + end + end + + context "using stored beacon version" do + it "returns 304 without a prior GET when beacon version is pre-set" do + beacon.update!(manifest_version: 5, manifest_checksum: "sha256:precomputed") + + head "/api/v1/beacons/manifest", headers: beacon_auth_headers(raw_key).merge("If-None-Match" => "v5") + + expect(response).to have_http_status(:not_modified) + expect(response.headers["ETag"]).to eq("v5") + end + + it "returns 412 without a prior GET when beacon version is pre-set" do + beacon.update!(manifest_version: 5, manifest_checksum: "sha256:precomputed") + + head "/api/v1/beacons/manifest", headers: beacon_auth_headers(raw_key).merge("If-Match" => "v3") + + expect(response).to have_http_status(:precondition_failed) + expect(response.headers["ETag"]).to eq("v5") + end + + it "does not build the manifest for 304 responses" do + beacon.update!(manifest_version: 5, manifest_checksum: "sha256:precomputed") + + expect(Beacons::ManifestBuilder).not_to receive(:new) + + head "/api/v1/beacons/manifest", headers: beacon_auth_headers(raw_key).merge("If-None-Match" => "v5") + + expect(response).to have_http_status(:not_modified) + end + + it "does not build the manifest for 412 responses" do + beacon.update!(manifest_version: 5, manifest_checksum: "sha256:precomputed") + + expect(Beacons::ManifestBuilder).not_to receive(:new) + + head "/api/v1/beacons/manifest", headers: beacon_auth_headers(raw_key).merge("If-Match" => "v3") + + expect(response).to have_http_status(:precondition_failed) + end + end + end +end diff --git a/spec/services/beacons/manifest_builder_spec.rb b/spec/services/beacons/manifest_builder_spec.rb new file mode 100644 index 00000000..7c062d2c --- /dev/null +++ b/spec/services/beacons/manifest_builder_spec.rb @@ -0,0 +1,201 @@ +require "rails_helper" + +RSpec.describe Beacons::ManifestBuilder do + subject(:builder) { described_class.new(beacon) } + + let(:language) { create(:language, name: "English") } + let(:region) { create(:region, name: "East Region") } + let(:provider) { create(:provider, name: "Health Ministry") } + let(:beacon) do + create(:beacon, language: language, region: region).tap do |b| + b.providers << provider + end + end + + describe "#call" do + context "with a beacon that has no topics" do + it "returns manifest with empty providers topics" do + result = builder.call + + expect(result[:language]).to eq({ id: language.id, code: "en", name: "English" }) + expect(result[:region]).to eq({ id: region.id, name: "East Region" }) + expect(result[:tags]).to eq([]) + expect(result[:providers].size).to eq(1) + expect(result[:providers].first[:topics]).to eq([]) + expect(result[:total_size_bytes]).to eq(0) + expect(result[:total_files]).to eq(0) + end + + it "includes manifest metadata" do + result = builder.call + + expect(result[:manifest_version]).to match(/\Av\d+\z/) + expect(result[:manifest_checksum]).to start_with("sha256:") + expect(result[:generated_at]).to be_present + end + end + + context "with topics and documents" do + let(:topic) do + create(:topic, :with_documents, title: "Maternal Health", provider: provider, language: language) + end + + before do + beacon.topics << topic + end + + it "includes topic under its provider" do + result = builder.call + provider_data = result[:providers].first + topic_data = provider_data[:topics].first + + expect(topic_data[:id]).to eq(topic.id) + expect(topic_data[:name]).to eq("Maternal Health") + expect(topic_data[:files]).not_to be_empty + end + + it "includes file details" do + result = builder.call + file_data = result[:providers].first[:topics].first[:files].first + blob = topic.documents.first.blob + + expect(file_data[:id]).to eq(blob.id) + expect(file_data[:checksum]).to eq(blob.checksum) + expect(file_data[:size_bytes]).to eq(blob.byte_size) + expect(file_data[:content_type]).to eq(blob.content_type) + expect(file_data[:path]).to start_with("providers/#{provider.id}/topics/#{topic.id}/") + end + + it "computes correct totals" do + result = builder.call + blob = topic.documents.first.blob + + expect(result[:total_files]).to eq(1) + expect(result[:total_size_bytes]).to eq(blob.byte_size) + end + end + + context "with tagged topics" do + let(:topic) { create(:topic, title: "Tagged Topic", provider: provider, language: language) } + + before do + topic.tag_list.add("Prenatal", "Emergency") + topic.save! + beacon.topics << topic + end + + it "collects unique tags from topics" do + result = builder.call + + tag_names = result[:tags].map { |t| t[:name] } + expect(tag_names).to include("prenatal", "emergency") + end + + it "includes tag_ids on topics" do + result = builder.call + topic_data = result[:providers].first[:topics].first + + expect(topic_data[:tag_ids]).to match_array(topic.tags.pluck(:id)) + end + end + + context "with archived topics" do + let(:active_topic) { create(:topic, title: "Active", provider: provider, language: language) } + let(:archived_topic) { create(:topic, :archived, title: "Archived", provider: provider, language: language) } + + before do + beacon.topics << [ active_topic, archived_topic ] + end + + it "excludes archived topics" do + result = builder.call + topic_names = result[:providers].first[:topics].map { |t| t[:name] } + + expect(topic_names).to include("Active") + expect(topic_names).not_to include("Archived") + end + end + + context "with multiple providers" do + let(:provider2) { create(:provider, name: "WHO") } + let(:topic1) { create(:topic, title: "Topic A", provider: provider, language: language) } + let(:topic2) { create(:topic, title: "Topic B", provider: provider2, language: language) } + + before do + beacon.providers << provider2 + beacon.topics << [ topic1, topic2 ] + end + + it "groups topics under their respective providers" do + result = builder.call + + provider_names = result[:providers].map { |p| p[:name] } + expect(provider_names).to contain_exactly("Health Ministry", "WHO") + + health_provider = result[:providers].find { |p| p[:name] == "Health Ministry" } + who_provider = result[:providers].find { |p| p[:name] == "WHO" } + + expect(health_provider[:topics].map { |t| t[:name] }).to eq([ "Topic A" ]) + expect(who_provider[:topics].map { |t| t[:name] }).to eq([ "Topic B" ]) + end + end + + context "with a topic whose provider is not assigned to the beacon" do + let(:unassigned_provider) { create(:provider, name: "Unassigned") } + let(:orphaned_topic) { create(:topic, title: "Orphaned", provider: unassigned_provider, language: language) } + + before do + beacon.topics << orphaned_topic + end + + it "does not include orphaned topics in any provider" do + result = builder.call + + all_topic_names = result[:providers].flat_map { |p| p[:topics].map { |t| t[:name] } } + expect(all_topic_names).not_to include("Orphaned") + end + end + end + + describe "versioning" do + it "sets version to v1 on first call" do + result = builder.call + + expect(result[:manifest_version]).to eq("v1") + expect(beacon.reload.manifest_version).to eq(1) + end + + it "keeps the same version when content has not changed" do + builder.call + result = described_class.new(beacon.reload).call + + expect(result[:manifest_version]).to eq("v1") + expect(beacon.reload.manifest_version).to eq(1) + end + + it "increments version when content changes" do + builder.call + + topic = create(:topic, :with_documents, title: "New Topic", provider: provider, language: language) + beacon.topics << topic + + result = described_class.new(beacon.reload).call + + expect(result[:manifest_version]).to eq("v2") + expect(beacon.reload.manifest_version).to eq(2) + end + + it "stores the checksum on the beacon" do + result = builder.call + + expect(beacon.reload.manifest_checksum).to eq(result[:manifest_checksum]) + end + + it "produces stable checksum for unchanged content" do + result1 = builder.call + result2 = described_class.new(beacon.reload).call + + expect(result1[:manifest_checksum]).to eq(result2[:manifest_checksum]) + end + end +end From 0dea3e0fb1ff3f6a7c54771bdcf5ce925e903898 Mon Sep 17 00:00:00 2001 From: dmitrytrager Date: Tue, 3 Feb 2026 16:48:28 +0100 Subject: [PATCH 2/5] Add proactive manifest rebuilding on content changes Introduce Beacons::RebuildManifestJob and wire it into Topics::Mutator (create, update, archive, unarchive, destroy), BeaconTopic/BeaconProvider join model callbacks, and SynchronizeCognatesOnTopicsJob so that beacon manifests stay current without waiting for a GET request. Co-Authored-By: Claude Opus 4.5 --- app/jobs/beacons/rebuild_manifest_job.rb | 12 ++++++++ .../synchronize_cognates_on_topics_job.rb | 17 ++++++++++- app/models/beacon_provider.rb | 8 ++++++ app/models/beacon_topic.rb | 8 ++++++ app/services/topics/mutator.rb | 23 +++++++++++++-- .../jobs/beacons/rebuild_manifest_job_spec.rb | 27 ++++++++++++++++++ ...synchronize_cognates_on_topics_job_spec.rb | 9 ++++++ spec/models/beacon_provider_spec.rb | 19 +++++++++++++ spec/models/beacon_topic_spec.rb | 19 +++++++++++++ spec/services/topics/mutator_spec.rb | 28 +++++++++++++++++++ 10 files changed, 166 insertions(+), 4 deletions(-) create mode 100644 app/jobs/beacons/rebuild_manifest_job.rb create mode 100644 spec/jobs/beacons/rebuild_manifest_job_spec.rb diff --git a/app/jobs/beacons/rebuild_manifest_job.rb b/app/jobs/beacons/rebuild_manifest_job.rb new file mode 100644 index 00000000..3063381b --- /dev/null +++ b/app/jobs/beacons/rebuild_manifest_job.rb @@ -0,0 +1,12 @@ +module Beacons + class RebuildManifestJob < ApplicationJob + queue_as :default + + def perform(beacon_id) + beacon = Beacon.find_by(id: beacon_id) + return unless beacon + + ManifestBuilder.new(beacon).call + end + end +end diff --git a/app/jobs/synchronize_cognates_on_topics_job.rb b/app/jobs/synchronize_cognates_on_topics_job.rb index 3797adc9..9e1ff327 100644 --- a/app/jobs/synchronize_cognates_on_topics_job.rb +++ b/app/jobs/synchronize_cognates_on_topics_job.rb @@ -1,9 +1,24 @@ class SynchronizeCognatesOnTopicsJob < ApplicationJob def perform(tag) + modified_topic_ids = [] + Topic.where(id: tag.taggings.select(:taggable_id)).each do |topic| tags = topic.tag_list << tag.cognates_tags.uniq.pluck(:name) topic.tag_list.add(tags) - topic.save + modified_topic_ids << topic.id if topic.save + end + + rebuild_beacon_manifests(modified_topic_ids) + end + + private + + def rebuild_beacon_manifests(topic_ids) + return if topic_ids.empty? + + beacon_ids = BeaconTopic.where(topic_id: topic_ids).distinct.pluck(:beacon_id) + beacon_ids.each do |beacon_id| + Beacons::RebuildManifestJob.perform_later(beacon_id) end end end diff --git a/app/models/beacon_provider.rb b/app/models/beacon_provider.rb index 4dd8ec96..5a9680d7 100644 --- a/app/models/beacon_provider.rb +++ b/app/models/beacon_provider.rb @@ -23,4 +23,12 @@ class BeaconProvider < ApplicationRecord belongs_to :beacon belongs_to :provider + + after_commit :rebuild_beacon_manifest, on: [ :create, :destroy ] + + private + + def rebuild_beacon_manifest + Beacons::RebuildManifestJob.perform_later(beacon_id) + end end diff --git a/app/models/beacon_topic.rb b/app/models/beacon_topic.rb index 1d67ff44..83bb1e75 100644 --- a/app/models/beacon_topic.rb +++ b/app/models/beacon_topic.rb @@ -23,4 +23,12 @@ class BeaconTopic < ApplicationRecord belongs_to :beacon belongs_to :topic + + after_commit :rebuild_beacon_manifest, on: [ :create, :destroy ] + + private + + def rebuild_beacon_manifest + Beacons::RebuildManifestJob.perform_later(beacon_id) + end end diff --git a/app/services/topics/mutator.rb b/app/services/topics/mutator.rb index fd80ffd8..df88c93a 100644 --- a/app/services/topics/mutator.rb +++ b/app/services/topics/mutator.rb @@ -7,16 +7,17 @@ def initialize(topic:, params: nil, document_signed_ids: nil) end def create - mutate + mutate.tap { |status, _| rebuild_beacon_manifests if status == :ok } end def update - mutate + mutate.tap { |status, _| rebuild_beacon_manifests if status == :ok } end def archive if @topic.archived! sync_docs_for_topic_archive if topic.documents.any? + rebuild_beacon_manifests return [ :ok, topic ] end @@ -26,6 +27,7 @@ def archive def unarchive if @topic.active! sync_docs_for_topic_archive if topic.documents.any? + rebuild_beacon_manifests return [ :ok, topic ] end @@ -33,10 +35,15 @@ def unarchive end def destroy + beacon_ids = topic.beacon_topics.pluck(:beacon_id) + # If topic deletion fails for some reason, documents deletion still will happen # This case is unlikely, and only admins can delete topics sync_docs_for_topic_deletion if topic.documents.any? - return [ :ok, topic ] if topic.destroy + if topic.destroy + dispatch_rebuild_jobs(beacon_ids) + return [ :ok, topic ] + end [ :error, topic.errors.full_messages ] end @@ -148,4 +155,14 @@ def extract_document_ids documents = params && params[:documents] || [] documents.map { |doc| doc.is_a?(String) ? doc : doc.signed_id } end + + def rebuild_beacon_manifests + dispatch_rebuild_jobs(topic.beacon_topics.pluck(:beacon_id)) + end + + def dispatch_rebuild_jobs(beacon_ids) + beacon_ids.each do |beacon_id| + Beacons::RebuildManifestJob.perform_later(beacon_id) + end + end end diff --git a/spec/jobs/beacons/rebuild_manifest_job_spec.rb b/spec/jobs/beacons/rebuild_manifest_job_spec.rb new file mode 100644 index 00000000..6c34f5dd --- /dev/null +++ b/spec/jobs/beacons/rebuild_manifest_job_spec.rb @@ -0,0 +1,27 @@ +require "rails_helper" + +RSpec.describe Beacons::RebuildManifestJob, type: :job do + describe "#perform" do + let(:beacon) { create(:beacon) } + let(:manifest_builder) { instance_double(Beacons::ManifestBuilder, call: {}) } + + before do + allow(Beacons::ManifestBuilder).to receive(:new).and_return(manifest_builder) + end + + it "rebuilds the manifest for the given beacon" do + described_class.perform_now(beacon.id) + + expect(Beacons::ManifestBuilder).to have_received(:new).with(beacon) + expect(manifest_builder).to have_received(:call) + end + + context "when the beacon does not exist" do + it "returns early without building a manifest" do + described_class.perform_now(-1) + + expect(Beacons::ManifestBuilder).not_to have_received(:new) + end + end + end +end diff --git a/spec/jobs/synchronize_cognates_on_topics_job_spec.rb b/spec/jobs/synchronize_cognates_on_topics_job_spec.rb index b6373ad8..8ac00f6f 100644 --- a/spec/jobs/synchronize_cognates_on_topics_job_spec.rb +++ b/spec/jobs/synchronize_cognates_on_topics_job_spec.rb @@ -37,5 +37,14 @@ expect(Topic.find_by(id: english_topic_1.id).tag_list) .to match_array([ "tag", "english cognate", "english reverse cognate", "spanish cognate", "spanish reverse cognate" ]) end + + it "dispatches rebuild manifest jobs for beacons associated with modified topics" do + beacon = create(:beacon) + create(:beacon_topic, beacon: beacon, topic: english_topic_1) + + expect { + SynchronizeCognatesOnTopicsJob.perform_now(tag) + }.to have_enqueued_job(Beacons::RebuildManifestJob).with(beacon.id) + end end end diff --git a/spec/models/beacon_provider_spec.rb b/spec/models/beacon_provider_spec.rb index 0c282236..332666e5 100644 --- a/spec/models/beacon_provider_spec.rb +++ b/spec/models/beacon_provider_spec.rb @@ -27,4 +27,23 @@ it { is_expected.to belong_to(:beacon) } it { is_expected.to belong_to(:provider) } end + + describe "callbacks" do + let(:beacon) { create(:beacon) } + let(:provider) { create(:provider) } + + it "enqueues a rebuild manifest job on create" do + expect { + create(:beacon_provider, beacon: beacon, provider: provider) + }.to have_enqueued_job(Beacons::RebuildManifestJob).with(beacon.id) + end + + it "enqueues a rebuild manifest job on destroy" do + beacon_provider = create(:beacon_provider, beacon: beacon, provider: provider) + + expect { + beacon_provider.destroy! + }.to have_enqueued_job(Beacons::RebuildManifestJob).with(beacon.id) + end + end end diff --git a/spec/models/beacon_topic_spec.rb b/spec/models/beacon_topic_spec.rb index 1e8a90b1..ebe6cb68 100644 --- a/spec/models/beacon_topic_spec.rb +++ b/spec/models/beacon_topic_spec.rb @@ -27,4 +27,23 @@ it { is_expected.to belong_to(:beacon) } it { is_expected.to belong_to(:topic) } end + + describe "callbacks" do + let(:beacon) { create(:beacon) } + let(:topic) { create(:topic) } + + it "enqueues a rebuild manifest job on create" do + expect { + create(:beacon_topic, beacon: beacon, topic: topic) + }.to have_enqueued_job(Beacons::RebuildManifestJob).with(beacon.id) + end + + it "enqueues a rebuild manifest job on destroy" do + beacon_topic = create(:beacon_topic, beacon: beacon, topic: topic) + + expect { + beacon_topic.destroy! + }.to have_enqueued_job(Beacons::RebuildManifestJob).with(beacon.id) + end + end end diff --git a/spec/services/topics/mutator_spec.rb b/spec/services/topics/mutator_spec.rb index 39ae0ea8..7b795e43 100644 --- a/spec/services/topics/mutator_spec.rb +++ b/spec/services/topics/mutator_spec.rb @@ -46,6 +46,7 @@ end describe "#update" do + let(:beacon) { create(:beacon) } let(:topic) { create(:topic, :with_documents, description: "topic details") } let(:document_signed_ids) do [ @@ -75,6 +76,12 @@ ) end + it "dispatches a manifest rebuild job for each associated beacon" do + create(:beacon_topic, beacon: beacon, topic: topic) + + expect { subject.update }.to have_enqueued_job(Beacons::RebuildManifestJob).with(beacon.id) + end + context "when existing document is not removed" do let(:document_ids) { [ topic.documents.first.signed_id ] } let(:document_signed_ids) { [] } @@ -96,6 +103,7 @@ end describe "#archive" do + let(:beacon) { create(:beacon) } let(:topic) { create(:topic, :with_documents) } it "archives the topic and runs the sync job for documents" do @@ -111,9 +119,16 @@ expect(archived_topic).to be_persisted expect(archived_topic.state).to eq("archived") end + + it "dispatches a manifest rebuild job for each associated beacon" do + create(:beacon_topic, beacon: beacon, topic: topic) + + expect { subject.archive }.to have_enqueued_job(Beacons::RebuildManifestJob).with(beacon.id) + end end describe "#unarchive" do + let(:beacon) { create(:beacon) } let(:topic) { create(:topic, :with_documents, state: "archived") } it "unarchive the topics" do @@ -129,9 +144,16 @@ expect(unarchived_topic).to be_persisted expect(unarchived_topic.state).to eq("active") end + + it "dispatches a manifest rebuild job for each associated beacon" do + create(:beacon_topic, beacon: beacon, topic: topic) + + expect { subject.unarchive }.to have_enqueued_job(Beacons::RebuildManifestJob).with(beacon.id) + end end describe "#destroy" do + let(:beacon) { create(:beacon) } let(:topic) { create(:topic, :with_documents) } it "deletes the topic and runs the sync job for documents" do @@ -145,5 +167,11 @@ action: "delete", ) end + + it "dispatches a manifest rebuild job for each associated beacon" do + create(:beacon_topic, beacon: beacon, topic: topic) + + expect { subject.destroy }.to have_enqueued_job(Beacons::RebuildManifestJob).with(beacon.id).at_least(:once) + end end end From d88a5bed7deb9468ffe393a7cd8210148a9a4d88 Mon Sep 17 00:00:00 2001 From: dmitrytrager Date: Wed, 4 Feb 2026 12:03:35 +0100 Subject: [PATCH 3/5] Store manifest content on beacon for admin visibility Save the manifest content (without version/checksum metadata) as jsonb on the beacon record so admins can inspect the full manifest state. Co-Authored-By: Claude Opus 4.5 --- app/models/beacon.rb | 1 + app/services/beacons/manifest_builder.rb | 5 +++-- ...0203100006_add_manifest_data_to_beacons.rb | 5 +++++ db/schema.rb | 3 ++- spec/factories/beacons.rb | 1 + spec/models/beacon_spec.rb | 1 + .../services/beacons/manifest_builder_spec.rb | 21 +++++++++++++++++++ 7 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 db/migrate/20260203100006_add_manifest_data_to_beacons.rb diff --git a/app/models/beacon.rb b/app/models/beacon.rb index f96530c8..5991648b 100644 --- a/app/models/beacon.rb +++ b/app/models/beacon.rb @@ -7,6 +7,7 @@ # api_key_digest :string not null # api_key_prefix :string not null # manifest_checksum :string +# manifest_data :jsonb # manifest_version :integer default(0), not null # name :string not null # revoked_at :datetime diff --git a/app/services/beacons/manifest_builder.rb b/app/services/beacons/manifest_builder.rb index 5fd17a67..309a8f63 100644 --- a/app/services/beacons/manifest_builder.rb +++ b/app/services/beacons/manifest_builder.rb @@ -7,7 +7,7 @@ def initialize(beacon) def call content = build_content checksum = "sha256:#{compute_checksum(content)}" - version = resolve_version(checksum) + version = resolve_version(checksum, content) content.merge( manifest_version: "v#{version}", @@ -20,12 +20,13 @@ def call attr_reader :beacon - def resolve_version(checksum) + def resolve_version(checksum, content) return beacon.manifest_version if beacon.manifest_checksum == checksum beacon.update!( manifest_version: beacon.manifest_version + 1, manifest_checksum: checksum, + manifest_data: content, ) beacon.manifest_version diff --git a/db/migrate/20260203100006_add_manifest_data_to_beacons.rb b/db/migrate/20260203100006_add_manifest_data_to_beacons.rb new file mode 100644 index 00000000..1d4227b0 --- /dev/null +++ b/db/migrate/20260203100006_add_manifest_data_to_beacons.rb @@ -0,0 +1,5 @@ +class AddManifestDataToBeacons < ActiveRecord::Migration[8.0] + def change + add_column :beacons, :manifest_data, :jsonb + end +end diff --git a/db/schema.rb b/db/schema.rb index edcf6fcb..7caaff66 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.1].define(version: 2026_02_03_100005) do +ActiveRecord::Schema[8.1].define(version: 2026_02_03_100006) do # These are extensions that must be enabled in order to support this database enable_extension "pg_catalog.plpgsql" @@ -68,6 +68,7 @@ t.datetime "created_at", null: false t.bigint "language_id", null: false t.string "manifest_checksum" + t.jsonb "manifest_data" t.integer "manifest_version", default: 0, null: false t.string "name", null: false t.bigint "region_id", null: false diff --git a/spec/factories/beacons.rb b/spec/factories/beacons.rb index 31351f08..4aff52d9 100644 --- a/spec/factories/beacons.rb +++ b/spec/factories/beacons.rb @@ -7,6 +7,7 @@ # api_key_digest :string not null # api_key_prefix :string not null # manifest_checksum :string +# manifest_data :jsonb # manifest_version :integer default(0), not null # name :string not null # revoked_at :datetime diff --git a/spec/models/beacon_spec.rb b/spec/models/beacon_spec.rb index b8a8c60f..c12b1b08 100644 --- a/spec/models/beacon_spec.rb +++ b/spec/models/beacon_spec.rb @@ -7,6 +7,7 @@ # api_key_digest :string not null # api_key_prefix :string not null # manifest_checksum :string +# manifest_data :jsonb # manifest_version :integer default(0), not null # name :string not null # revoked_at :datetime diff --git a/spec/services/beacons/manifest_builder_spec.rb b/spec/services/beacons/manifest_builder_spec.rb index 7c062d2c..b2da2ad3 100644 --- a/spec/services/beacons/manifest_builder_spec.rb +++ b/spec/services/beacons/manifest_builder_spec.rb @@ -191,6 +191,27 @@ expect(beacon.reload.manifest_checksum).to eq(result[:manifest_checksum]) end + it "stores the manifest content on the beacon" do + result = builder.call + stored = beacon.reload.manifest_data + + expect(stored["language"]["id"]).to eq(result[:language][:id]) + expect(stored["region"]["id"]).to eq(result[:region][:id]) + expect(stored["providers"].size).to eq(result[:providers].size) + expect(stored).not_to have_key("manifest_version") + expect(stored).not_to have_key("manifest_checksum") + expect(stored).not_to have_key("generated_at") + end + + it "does not update manifest_data when content is unchanged" do + builder.call + original_data = beacon.reload.manifest_data + + described_class.new(beacon.reload).call + + expect(beacon.reload.manifest_data).to eq(original_data) + end + it "produces stable checksum for unchanged content" do result1 = builder.call result2 = described_class.new(beacon.reload).call From f67712920d78b19e4005f6fcf599e0fdc55fd6fd Mon Sep 17 00:00:00 2001 From: dmitrytrager Date: Wed, 4 Feb 2026 12:29:29 +0100 Subject: [PATCH 4/5] Preserve previous manifest data for admin diff visibility Rotate manifest_data into previous_manifest_data before saving new content, so admins can compare current and previous manifests. Co-Authored-By: Claude Opus 4.5 --- app/models/beacon.rb | 25 ++++++++------- app/services/beacons/manifest_builder.rb | 1 + ...7_add_previous_manifest_data_to_beacons.rb | 5 +++ db/schema.rb | 3 +- spec/factories/beacons.rb | 25 ++++++++------- spec/models/beacon_spec.rb | 25 ++++++++------- .../services/beacons/manifest_builder_spec.rb | 31 +++++++++++++++++++ 7 files changed, 78 insertions(+), 37 deletions(-) create mode 100644 db/migrate/20260203100007_add_previous_manifest_data_to_beacons.rb diff --git a/app/models/beacon.rb b/app/models/beacon.rb index 5991648b..c7cf9a7f 100644 --- a/app/models/beacon.rb +++ b/app/models/beacon.rb @@ -3,18 +3,19 @@ # Table name: beacons # Database name: primary # -# id :bigint not null, primary key -# api_key_digest :string not null -# api_key_prefix :string not null -# manifest_checksum :string -# manifest_data :jsonb -# manifest_version :integer default(0), not null -# name :string not null -# revoked_at :datetime -# created_at :datetime not null -# updated_at :datetime not null -# language_id :bigint not null -# region_id :bigint not null +# id :bigint not null, primary key +# api_key_digest :string not null +# api_key_prefix :string not null +# manifest_checksum :string +# manifest_data :jsonb +# manifest_version :integer default(0), not null +# name :string not null +# previous_manifest_data :jsonb +# revoked_at :datetime +# created_at :datetime not null +# updated_at :datetime not null +# language_id :bigint not null +# region_id :bigint not null # # Indexes # diff --git a/app/services/beacons/manifest_builder.rb b/app/services/beacons/manifest_builder.rb index 309a8f63..b525850c 100644 --- a/app/services/beacons/manifest_builder.rb +++ b/app/services/beacons/manifest_builder.rb @@ -26,6 +26,7 @@ def resolve_version(checksum, content) beacon.update!( manifest_version: beacon.manifest_version + 1, manifest_checksum: checksum, + previous_manifest_data: beacon.manifest_data, manifest_data: content, ) diff --git a/db/migrate/20260203100007_add_previous_manifest_data_to_beacons.rb b/db/migrate/20260203100007_add_previous_manifest_data_to_beacons.rb new file mode 100644 index 00000000..fe70237f --- /dev/null +++ b/db/migrate/20260203100007_add_previous_manifest_data_to_beacons.rb @@ -0,0 +1,5 @@ +class AddPreviousManifestDataToBeacons < ActiveRecord::Migration[8.0] + def change + add_column :beacons, :previous_manifest_data, :jsonb + end +end diff --git a/db/schema.rb b/db/schema.rb index 7caaff66..c89e0c89 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.1].define(version: 2026_02_03_100006) do +ActiveRecord::Schema[8.1].define(version: 2026_02_03_100007) do # These are extensions that must be enabled in order to support this database enable_extension "pg_catalog.plpgsql" @@ -71,6 +71,7 @@ t.jsonb "manifest_data" t.integer "manifest_version", default: 0, null: false t.string "name", null: false + t.jsonb "previous_manifest_data" t.bigint "region_id", null: false t.datetime "revoked_at" t.datetime "updated_at", null: false diff --git a/spec/factories/beacons.rb b/spec/factories/beacons.rb index 4aff52d9..4e699f17 100644 --- a/spec/factories/beacons.rb +++ b/spec/factories/beacons.rb @@ -3,18 +3,19 @@ # Table name: beacons # Database name: primary # -# id :bigint not null, primary key -# api_key_digest :string not null -# api_key_prefix :string not null -# manifest_checksum :string -# manifest_data :jsonb -# manifest_version :integer default(0), not null -# name :string not null -# revoked_at :datetime -# created_at :datetime not null -# updated_at :datetime not null -# language_id :bigint not null -# region_id :bigint not null +# id :bigint not null, primary key +# api_key_digest :string not null +# api_key_prefix :string not null +# manifest_checksum :string +# manifest_data :jsonb +# manifest_version :integer default(0), not null +# name :string not null +# previous_manifest_data :jsonb +# revoked_at :datetime +# created_at :datetime not null +# updated_at :datetime not null +# language_id :bigint not null +# region_id :bigint not null # # Indexes # diff --git a/spec/models/beacon_spec.rb b/spec/models/beacon_spec.rb index c12b1b08..063474cc 100644 --- a/spec/models/beacon_spec.rb +++ b/spec/models/beacon_spec.rb @@ -3,18 +3,19 @@ # Table name: beacons # Database name: primary # -# id :bigint not null, primary key -# api_key_digest :string not null -# api_key_prefix :string not null -# manifest_checksum :string -# manifest_data :jsonb -# manifest_version :integer default(0), not null -# name :string not null -# revoked_at :datetime -# created_at :datetime not null -# updated_at :datetime not null -# language_id :bigint not null -# region_id :bigint not null +# id :bigint not null, primary key +# api_key_digest :string not null +# api_key_prefix :string not null +# manifest_checksum :string +# manifest_data :jsonb +# manifest_version :integer default(0), not null +# name :string not null +# previous_manifest_data :jsonb +# revoked_at :datetime +# created_at :datetime not null +# updated_at :datetime not null +# language_id :bigint not null +# region_id :bigint not null # # Indexes # diff --git a/spec/services/beacons/manifest_builder_spec.rb b/spec/services/beacons/manifest_builder_spec.rb index b2da2ad3..43688c08 100644 --- a/spec/services/beacons/manifest_builder_spec.rb +++ b/spec/services/beacons/manifest_builder_spec.rb @@ -212,6 +212,37 @@ expect(beacon.reload.manifest_data).to eq(original_data) end + it "keeps previous_manifest_data nil on first build" do + builder.call + + expect(beacon.reload.previous_manifest_data).to be_nil + end + + it "rotates manifest_data into previous_manifest_data when content changes" do + builder.call + first_data = beacon.reload.manifest_data + + topic = create(:topic, :with_documents, title: "New Topic", provider: provider, language: language) + beacon.topics << topic + + described_class.new(beacon.reload).call + + expect(beacon.reload.previous_manifest_data).to eq(first_data) + end + + it "does not update previous_manifest_data when content is unchanged" do + builder.call + + topic = create(:topic, :with_documents, title: "New Topic", provider: provider, language: language) + beacon.topics << topic + described_class.new(beacon.reload).call + previous_data = beacon.reload.previous_manifest_data + + described_class.new(beacon.reload).call + + expect(beacon.reload.previous_manifest_data).to eq(previous_data) + end + it "produces stable checksum for unchanged content" do result1 = builder.call result2 = described_class.new(beacon.reload).call From da357ccb6c8461e8989a0108897e99f685d88c80 Mon Sep 17 00:00:00 2001 From: dmitrytrager Date: Wed, 4 Feb 2026 16:01:19 +0100 Subject: [PATCH 5/5] Rename header checking method names --- app/controllers/api/v1/beacons/manifests_controller.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/v1/beacons/manifests_controller.rb b/app/controllers/api/v1/beacons/manifests_controller.rb index de5beb0f..466df596 100644 --- a/app/controllers/api/v1/beacons/manifests_controller.rb +++ b/app/controllers/api/v1/beacons/manifests_controller.rb @@ -7,7 +7,7 @@ def show # During sync, beacon sends If-Match with its current version. # If the manifest changed since sync started, return 412 so beacon aborts and restarts. - if stale_by_match?(stored_etag) + if sync_version_stale?(stored_etag) response.headers["ETag"] = stored_etag head :precondition_failed return @@ -15,7 +15,7 @@ def show # Beacon sends If-None-Match with its cached version to check for updates. # If versions match, return 304 — no sync needed. - if fresh_by_none_match?(stored_etag) + if cached_version_fresh?(stored_etag) response.headers["ETag"] = stored_etag head :not_modified return @@ -33,14 +33,14 @@ def manifest_builder ::Beacons::ManifestBuilder.new(Current.beacon) end - def stale_by_match?(etag) + def sync_version_stale?(etag) if_match = request.headers["If-Match"] return false if if_match.blank? if_match != etag end - def fresh_by_none_match?(etag) + def cached_version_fresh?(etag) if_none_match = request.headers["If-None-Match"] return false if if_none_match.blank?