From aef97d618a562fa82134978839334ffaf8825c1c Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 14:52:08 +0200 Subject: [PATCH 01/33] refactor!: replace configs gem with registry index --- Gemfile | 8 +- Gemfile.lock | 54 ++-- app/web/api/v1/configs.rb | 4 +- app/web/catalog/merge.rb | 101 -------- app/web/config/local_config.rb | 19 +- app/web/registry/config.rb | 239 ++++++++++++++++++ app/web/registry/errors.rb | 26 ++ app/web/registry/index.rb | 191 ++++++++++++++ app/web/registry/local_catalog.rb | 75 ++++++ app/web/registry/store.rb | 91 +++++++ app/web/registry/sync.rb | 71 ++++++ public/openapi.yaml | 100 +------- .../official/configs/phys.org/weekly.yml | 23 ++ .../support.apple.com/en_gb_ht201222.yml | 32 +++ .../registries/official/manifest.json | 10 + .../configs/secret.example/private.yml | 15 ++ .../fixtures/registries/private/manifest.json | 9 + spec/fixtures/registries/registries.yml | 11 + spec/html2rss/web/api/v1_spec.rb | 5 +- spec/html2rss/web/local_config_spec.rb | 20 +- spec/html2rss/web/registry/index_spec.rb | 56 ++++ spec/html2rss/web/registry/sync_spec.rb | 26 ++ spec/support/registry.rb | 24 ++ 23 files changed, 944 insertions(+), 266 deletions(-) delete mode 100644 app/web/catalog/merge.rb create mode 100644 app/web/registry/config.rb create mode 100644 app/web/registry/errors.rb create mode 100644 app/web/registry/index.rb create mode 100644 app/web/registry/local_catalog.rb create mode 100644 app/web/registry/store.rb create mode 100644 app/web/registry/sync.rb create mode 100644 spec/fixtures/registries/official/configs/phys.org/weekly.yml create mode 100644 spec/fixtures/registries/official/configs/support.apple.com/en_gb_ht201222.yml create mode 100644 spec/fixtures/registries/official/manifest.json create mode 100644 spec/fixtures/registries/private/configs/secret.example/private.yml create mode 100644 spec/fixtures/registries/private/manifest.json create mode 100644 spec/fixtures/registries/registries.yml create mode 100644 spec/html2rss/web/registry/index_spec.rb create mode 100644 spec/html2rss/web/registry/sync_spec.rb create mode 100644 spec/support/registry.rb diff --git a/Gemfile b/Gemfile index 5816b002d..25acc1cc7 100644 --- a/Gemfile +++ b/Gemfile @@ -4,13 +4,9 @@ source 'https://rubygems.org' git_source(:github) { |repo_name| "https://github.com/#{repo_name}" } -gem 'html2rss', '~> 0.27' +gem 'html2rss', path: '../html2rss' +# gem 'html2rss', '~> 0.27' # gem 'html2rss', github: 'html2rss/html2rss', branch: 'master' -gem 'html2rss-configs', github: 'html2rss/html2rss-configs' - -# Use these instead of the two above (uncomment them) when developing locally: -# gem 'html2rss', path: '../html2rss' -# gem 'html2rss-configs', path: '../html2rss-configs' gem 'base64' gem 'rack-cache' diff --git a/Gemfile.lock b/Gemfile.lock index 9e2e0b168..c480a1da2 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,9 +1,27 @@ -GIT - remote: https://github.com/html2rss/html2rss-configs - revision: f888a8dc5bb260c998c5ae8f39bb7d45320638ee +PATH + remote: ../html2rss specs: - html2rss-configs (0.2.0) - html2rss + html2rss (0.28.0) + addressable (~> 2.7) + brotli + dry-validation + faraday (> 2.0.1, < 3.0) + faraday-follow_redirects + faraday-gzip (~> 3) + kramdown + mcp (~> 1.2) + mime-types (> 3.0) + nokogiri (>= 1.10, < 2.0) + rack (~> 3.0) + rackup (~> 2.0) + regexp_parser + reverse_markdown (~> 3.0) + rss + sanitize + thor + tzinfo + webrick (~> 1.9) + zeitwerk GEM remote: https://rubygems.org/ @@ -103,27 +121,6 @@ GEM net-http (~> 0.5) hana (1.3.7) hashdiff (1.2.1) - html2rss (0.27.2) - addressable (~> 2.7) - brotli - dry-validation - faraday (> 2.0.1, < 3.0) - faraday-follow_redirects - faraday-gzip (~> 3) - kramdown - mcp (~> 1.2) - mime-types (> 3.0) - nokogiri (>= 1.10, < 2.0) - rack (~> 3.0) - rackup (~> 2.0) - regexp_parser - reverse_markdown (~> 3.0) - rss - sanitize - thor - tzinfo - webrick (~> 1.9) - zeitwerk i18n (1.15.2) concurrent-ruby (~> 1.0) io-console (0.9.2) @@ -319,8 +316,7 @@ PLATFORMS DEPENDENCIES base64 climate_control - html2rss (~> 0.27) - html2rss-configs! + html2rss! irb puma rack-cache @@ -377,8 +373,6 @@ CHECKSUMS faraday-net_http (3.4.4) sha256=0e78af151747ed1b00f33e25973b4bc220d7f16c00c39676817c8b12331eb588 hana (1.3.7) sha256=5425db42d651fea08859811c29d20446f16af196308162894db208cac5ce9b0d hashdiff (1.2.1) sha256=9c079dbc513dfc8833ab59c0c2d8f230fa28499cc5efb4b8dd276cf931457cd1 - html2rss (0.27.2) sha256=82b28308c023cb669704bf9b0612aa2a00730be4b53d88f957d456ba5e631ac8 - html2rss-configs (0.2.0) i18n (1.15.2) sha256=00f9eb62412fe593b2a65a97daa75300d37abb8f7202ec748e94b6d46a9dd1b5 io-console (0.9.2) sha256=efa74f891dd03c0939a931dfc6e74c2813d904763d456ea9762b0525e748db08 irb (1.18.0) sha256=de9454a0703a54704b9811a5ef31a60c86949fbf4013fcf244fabc7c775248e3 diff --git a/app/web/api/v1/configs.rb b/app/web/api/v1/configs.rb index 900b9b9b9..827450308 100644 --- a/app/web/api/v1/configs.rb +++ b/app/web/api/v1/configs.rb @@ -17,7 +17,7 @@ def index(_router) entries, duration_ms = build_entries emit_success(entries.size, duration_ms) success_payload(entries) - rescue Html2rss::Configs::Catalog::MissingDirectoryTitle => error + rescue Html2rss::Registry::CatalogBuilder::MissingDirectoryTitle => error emit_failure(error) raise end @@ -26,7 +26,7 @@ def index(_router) def build_entries started = Process.clock_gettime(Process::CLOCK_MONOTONIC) - entries = Html2rss::Web::Catalog::Merge.call + entries = Registry::Index.current.catalog_rows duration_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000).round [entries, duration_ms] end diff --git a/app/web/catalog/merge.rb b/app/web/catalog/merge.rb deleted file mode 100644 index d7e6b7734..000000000 --- a/app/web/catalog/merge.rb +++ /dev/null @@ -1,101 +0,0 @@ -# frozen_string_literal: true - -require 'html2rss/configs' - -module Html2rss - module Web - ## - # Merges embedded catalog entries with local feed configs for the public catalog API. - module Catalog - module Merge - STARTER_FEED_IDS = %w[ - microsoft.com/azure-products - phys.org/weekly - softwareleadweekly.com/issues - ].freeze - - module_function - - ## - # @return [Array Object}>] - def call - embedded = Html2rss::Configs::Catalog.entries.map(&:to_h) - local = local_entries - (embedded + local).sort_by { |entry| entry.fetch(:id) } - end - - ## - # @return [Array Object}>] - def starter_entries - entries = call - selected = STARTER_FEED_IDS.filter_map { |id| entries.find { |entry| entry.fetch(:id) == id } } - selected.empty? ? entries.first(3) : selected - end - - ## - # @return [Array Object}>] - def local_entries - LocalConfig.feeds.filter_map do |feed_name, feed_config| - build_local_entry(feed_name, feed_config) - end - end - - ## - # @param feed_name [String, Symbol] - # @param feed_config [Hash] - # @return [Hash{Symbol => Object}, nil] - def build_local_entry(feed_name, feed_config) - directory = feed_config[:directory] || {} - title = directory[:title] - return nil if title.to_s.strip.empty? - - id = feed_name.to_s - channel = feed_config[:channel] || {} - - local_entry(id, directory, title, channel) - end - - ## - # @param id [String] - # @param directory [Hash] - # @param title [String] - # @param channel [Hash] - # @return [Hash{Symbol => Object}] - def local_entry(id, directory, title, channel) - { - id:, - path: "/#{id}.rss", - source: 'local', - directory: local_directory(directory, title), - channel: local_channel(channel, title), - parameters: { schema: {}, defaults: {} } - } - end - - ## - # @param directory [Hash] - # @param title [String] - # @return [Hash{Symbol => Object}] - def local_directory(directory, title) - { - title: title.to_s, - summary: directory[:summary], - topics: Array(directory[:topics]) - }.compact - end - - ## - # @param channel [Hash] - # @param title [String] - # @return [Hash{Symbol => Object}] - def local_channel(channel, title) - { - url: channel.fetch(:url), - language: channel[:language], - title: channel[:title] || title.to_s - }.compact - end - end - end - end -end diff --git a/app/web/config/local_config.rb b/app/web/config/local_config.rb index fcf4f98d3..604643d00 100644 --- a/app/web/config/local_config.rb +++ b/app/web/config/local_config.rb @@ -3,12 +3,6 @@ require 'erb' require 'yaml' require_relative 'runtime_env' -begin - require 'html2rss/configs' -rescue LoadError => error - warn "[html2rss-web] Failed to load 'html2rss/configs': #{error.message}" - raise -end module Html2rss module Web @@ -28,7 +22,6 @@ class NotFound < RuntimeError; end # raised when the local config shape is invalid class InvalidConfig < RuntimeError; end FEED_EXTENSION_PATTERN = /\.(json|rss|xml)\z/ - EMBEDDED_FEED_NAME_PATTERN = %r{\A[^/]+/.+\z} # Path to local feed configuration file. CONFIG_FILE = 'config/feeds.yml' @@ -39,7 +32,7 @@ class << self # @return [Hash] def find(name) normalized_name = normalize_name(name) - config_hash = local_feed_config(normalized_name) || embedded_feed_config(normalized_name) + config_hash = local_feed_config(normalized_name) || registry_feed_config(normalized_name) raise NotFound, "Did not find local feed config at '#{normalized_name}'" unless config_hash config_hash @@ -94,6 +87,7 @@ def load_snapshot # @return [nil] def reload!(reason: 'manual') @mutex.synchronize { @snapshot = nil } + Registry::Index.reload! Observability.emit( event_name: 'cache.lifecycle', outcome: 'success', @@ -115,13 +109,8 @@ def local_feed_config(normalized_name) # @param normalized_name [String] # @return [Hash{Symbol=>Object}, nil] - def embedded_feed_config(normalized_name) - return nil unless defined?(Html2rss::Configs) - return nil unless normalized_name.match?(EMBEDDED_FEED_NAME_PATTERN) - - deep_dup(Html2rss::Configs.find_by_name(normalized_name)) - rescue Html2rss::Configs::ConfigNotFound - nil + def registry_feed_config(normalized_name) + Registry::Index.current.config_for(normalized_name) end # @param name [String, Symbol, #to_s] diff --git a/app/web/registry/config.rb b/app/web/registry/config.rb new file mode 100644 index 000000000..06f693554 --- /dev/null +++ b/app/web/registry/config.rb @@ -0,0 +1,239 @@ +# frozen_string_literal: true + +require 'yaml' +require 'openssl' + +module Html2rss + module Web + module Registry + ## + # Registry definition parsed from {Config::REGISTRIES_FILE}. + Entry = Data.define( + :id, + :mode, + :path, + :sync_channel, + :sync_url, + :catalog, + :public_key_id, + :public_key + ) do + ## + # @return [Hash{String => OpenSSL::PKey::PKey}] + def public_keys + return {} if public_key.nil? + + { public_key_id => public_key } + end + end + + ## + # Parses registry configuration and applies zero-config defaults. + module Config # rubocop:disable Metrics/ModuleLength + REGISTRIES_FILE = 'config/registries.yml' + DEFAULT_PRECEDENCE = %w[official].freeze + DEFAULT_OFFICIAL_SYNC_CHANNEL = 'html2rss-official' + OFFICIAL_RELEASE_URL = 'https://github.com/html2rss/html2rss-configs/releases/latest/download/registry-bundle.tar.gz' + + @mutex = Mutex.new + @current = nil + + class << self # rubocop:disable Metrics/ClassLength + ## + # @return [Array] registry ids in merge precedence order + def precedence + current.precedence + end + + ## + # @param registry_id [String, Symbol] + # @return [Entry] + def entry(registry_id) + current.entries.fetch(registry_id.to_s) do + raise Errors::UnknownRegistry, "Unknown registry '#{registry_id}'" + end + end + + ## + # @param registry_id [String, Symbol] + # @return [Boolean] + def catalog_enabled?(registry_id) + entry(registry_id).catalog + end + + ## + # @return [ConfigSnapshot] + def current + @mutex.synchronize { @current ||= parse_snapshot } + end + + ## + # Clears memoized configuration (tests and development reload). + # + # @return [nil] + def reload! + @mutex.synchronize { @current = nil } + nil + end + + private + + ## + # @return [ConfigSnapshot] + def parse_snapshot + document = load_document + precedence = Array(document[:precedence]).map(&:to_s) + precedence = DEFAULT_PRECEDENCE if precedence.empty? + + entries = precedence.to_h { |registry_id| [registry_id, parse_entry(registry_id, document)] } + missing = precedence - entries.keys + raise Errors::ConfigError, "Missing registry definitions: #{missing.join(', ')}" unless missing.empty? + + ConfigSnapshot.new(precedence:, entries:) + end + + ## + # @return [Hash{Symbol => Object}] + def load_document + path = registries_file + return default_document unless File.file?(path) + + YAML.safe_load_file(path, symbolize_names: true) || {} + rescue Psych::SyntaxError => error + raise Errors::ConfigError, "Invalid #{path}: #{error.message}" + end + + ## + # @return [String] + def registries_file + ENV.fetch('REGISTRIES_CONFIG', REGISTRIES_FILE) + end + + ## + # @return [Hash{Symbol => Object}] + def default_document + { + precedence: DEFAULT_PRECEDENCE, + registries: { + 'official' => { + sync: { channel: DEFAULT_OFFICIAL_SYNC_CHANNEL }, + catalog: true + } + } + } + end + + ## + # @param registry_id [String] + # @param document [Hash{Symbol => Object}] + # @return [Entry] + def parse_entry(registry_id, document) + raw = document.dig(:registries, registry_id.to_sym) || + document.dig(:registries, registry_id) + raise Errors::ConfigError, "Missing registry definition for '#{registry_id}'" unless raw.is_a?(Hash) + + build_entry(registry_id, raw) + end + + ## + # @param registry_id [String] + # @param raw [Hash{Symbol => Object}] + # @return [Entry] + def build_entry(registry_id, raw) + sync = sync_section(raw) + mode, resolved_path, sync_channel, sync_url = resolve_mode( + path: raw[:path]&.to_s, + sync_channel: sync[:channel]&.to_s, + sync_url: sync[:url]&.to_s + ) + + entry_attributes(registry_id, raw, mode, resolved_path, sync_channel, sync_url) + end + + ## + # @param registry_id [String] + # @param raw [Hash{Symbol => Object}] + # @param mode [Symbol] + # @param path [String, nil] + # @param sync_channel [String, nil] + # @param sync_url [String, nil] + # @return [Entry] + def entry_attributes(registry_id, raw, mode, path, sync_channel, sync_url) # rubocop:disable Metrics/ParameterLists + Entry.new( + id: registry_id, + mode:, + path:, + sync_channel:, + sync_url:, + catalog: raw.fetch(:catalog, true), + public_key_id: public_key_id_for(raw[:public_key_id]&.to_s), + public_key: parse_public_key(raw[:public_key]) + ) + end + + ## + # @param raw [Hash{Symbol => Object}] + # @return [Hash{Symbol => Object}] + def sync_section(raw) + raw[:sync].is_a?(Hash) ? raw[:sync] : {} + end + + ## + # @param public_key_id [String, nil] + # @return [String] + def public_key_id_for(public_key_id) + return public_key_id if public_key_id && !public_key_id.empty? + + 'official-v1' + end + + ## + # @param path [String, nil] + # @param sync_channel [String, nil] + # @param sync_url [String, nil] + # @return [Array(Symbol, String, nil, String, nil)] + def resolve_mode(path:, sync_channel:, sync_url:) + return [:path, expand_path(path), nil, nil] if path && !path.empty? + + resolved_sync_url = sync_url && !sync_url.empty? ? sync_url : resolve_sync_channel_url(sync_channel) + [:sync, nil, sync_channel, resolved_sync_url] + end + + ## + # @param sync_channel [String, nil] + # @return [String] + def resolve_sync_channel_url(sync_channel) + channel = sync_channel && !sync_channel.empty? ? sync_channel : DEFAULT_OFFICIAL_SYNC_CHANNEL + return OFFICIAL_RELEASE_URL if channel == DEFAULT_OFFICIAL_SYNC_CHANNEL + + raise Errors::ConfigError, "Unknown sync channel '#{channel}'" + end + + ## + # @param path [String] + # @return [String] + def expand_path(path) + return path if path.start_with?('/') + + File.expand_path(path, Dir.pwd) + end + + ## + # @param value [String, nil] + # @return [OpenSSL::PKey::PKey, nil] + def parse_public_key(value) + return nil if value.to_s.strip.empty? + + OpenSSL::PKey.read(value) + rescue OpenSSL::PKey::PKeyError => error + raise Errors::ConfigError, "Invalid registry public_key: #{error.message}" + end + end # rubocop:enable Metrics/ClassLength + + ## + # Parsed registry configuration snapshot. + ConfigSnapshot = Data.define(:precedence, :entries) + end # rubocop:enable Metrics/ModuleLength + end + end +end diff --git a/app/web/registry/errors.rb b/app/web/registry/errors.rb new file mode 100644 index 000000000..bb917c753 --- /dev/null +++ b/app/web/registry/errors.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Html2rss + module Web + module Registry + ## + # Actionable registry runtime errors for operators and API handlers. + module Errors + # Base error for web registry operations. + class Error < StandardError; end + + # Raised when registry configuration is invalid or incomplete. + class ConfigError < Error; end + + # Raised when a registry bundle cannot be loaded. + class LoadError < Error; end + + # Raised when registry synchronization fails. + class SyncError < Error; end + + # Raised when a registry id is unknown. + class UnknownRegistry < Error; end + end + end + end +end diff --git a/app/web/registry/index.rb b/app/web/registry/index.rb new file mode 100644 index 000000000..9132d728f --- /dev/null +++ b/app/web/registry/index.rb @@ -0,0 +1,191 @@ +# frozen_string_literal: true + +module Html2rss + module Web + module Registry + ## + # Sole merge owner for registry bundles and local {LocalConfig} feeds. + class Index # rubocop:disable Metrics/ClassLength + RegistryBundle = Data.define(:registry_id, :manifest, :configs, :catalog_entries) + StatusEntry = Data.define(:id, :version, :updated_at, :sync_mode) + + @mutex = Mutex.new + @current = nil + + class << self + ## + # @return [Index] + def current + @mutex.synchronize { @current ||= new } + end + + ## + # Clears memoized bundles (tests and development reload). + # + # @return [nil] + def reload! + @mutex.synchronize { @current = nil } + Config.reload! + nil + end + end + + ## + # @param feed_id [String, Symbol] + # @return [Hash{Symbol => Object}, nil] + def config_for(feed_id) + normalized_id = normalize_feed_id(feed_id) + local_config = local_config_for(normalized_id) + return local_config if local_config + + loaded_bundles.each_value do |bundle| + config = bundle.configs[normalized_id] + return deep_dup(config) if config + end + + nil + end + + ## + # @return [Array Object}>] catalog rows in HTTP wire shape + def catalog_rows + rows = registry_catalog_rows + LocalCatalog.rows.each { |row| rows[row.fetch(:id)] = row } + rows.values.sort_by { |row| row.fetch(:id) } + end + + ## + # @return [Array] + def status + Config.precedence.map do |registry_id| + entry = Config.entry(registry_id) + bundle = loaded_bundles[registry_id] + StatusEntry.new( + id: registry_id, + version: bundle&.manifest&.version, + updated_at: bundle_updated_at(registry_id), + sync_mode: entry.mode + ) + end + end + + private + + ## + # @return [Hash{String => Hash{Symbol => Object}}] + def registry_catalog_rows + Config.precedence.each_with_object({}) do |registry_id, rows| + next unless Config.catalog_enabled?(registry_id) + + bundle = loaded_bundles[registry_id] + next unless bundle + + bundle.catalog_entries.each do |entry| + rows[entry.id] ||= wire_registry_row(entry, registry_id) + end + end + end + + ## + # @return [Hash{String => RegistryBundle}] + def loaded_bundles + @loaded_bundles ||= Config.precedence.to_h { |registry_id| [registry_id, load_bundle(registry_id)] }.compact + end + + ## + # @param registry_id [String] + # @return [RegistryBundle, nil] + def load_bundle(registry_id) # rubocop:disable Metrics/MethodLength + entry = Config.entry(registry_id) + directory = bundle_directory(entry) + return nil unless directory && File.directory?(directory) + + bundle = Html2rss::Registry::Bundle.load( + directory, + trust: entry.mode == :path ? :integrity_only : :signed, + public_keys: entry.public_keys + ) + to_registry_bundle(registry_id, bundle) + rescue Html2rss::Registry::Error => error + raise Errors::LoadError, "Failed to load registry '#{registry_id}': #{error.message}" + end + + ## + # @param registry_id [String] + # @param bundle [Html2rss::Registry::Bundle::BundleData] + # @return [RegistryBundle] + def to_registry_bundle(registry_id, bundle) + RegistryBundle.new( + registry_id:, + manifest: bundle.manifest, + configs: bundle.configs, + catalog_entries: bundle.catalog_entries + ) + end + + ## + # @param entry [Entry] + # @return [String, nil] + def bundle_directory(entry) + return entry.path if entry.mode == :path + + Store.registry_dir(entry.id) + end + + ## + # @param feed_id [String, Symbol] + # @return [String] + def normalize_feed_id(feed_id) + feed_id.to_s.delete_prefix('/').sub(LocalConfig::FEED_EXTENSION_PATTERN, '') + end + + ## + # @param feed_id [String] + # @return [Hash{Symbol => Object}, nil] + def local_config_for(feed_id) + feed = LocalConfig.feeds[feed_id.to_sym] || LocalConfig.feeds[feed_id] + return nil unless feed + + deep_dup(feed) + rescue Html2rss::Web::LocalConfig::InvalidConfig, Html2rss::Web::LocalConfig::NotFound + nil + end + + ## + # @param entry [Html2rss::Registry::CatalogEntry] + # @param registry_id [String] + # @return [Hash{Symbol => Object}] + def wire_registry_row(entry, registry_id) + entry.to_h.merge(source: 'registry', registry: registry_id) + end + + ## + # @param registry_id [String] + # @return [Time, nil] + def bundle_updated_at(registry_id) + path = bundle_directory(Config.entry(registry_id)) + return nil unless path && File.directory?(path) + + manifest_path = File.join(path, Html2rss::Registry::Manifest::MANIFEST_FILE) + return nil unless File.file?(manifest_path) + + File.mtime(manifest_path) + end + + ## + # @param value [Object] + # @return [Object] + def deep_dup(value) + case value + when Hash + value.transform_values { |entry| deep_dup(entry) } + when Array + value.map { |entry| deep_dup(entry) } + else + value + end + end + end # rubocop:enable Metrics/ClassLength + end + end +end diff --git a/app/web/registry/local_catalog.rb b/app/web/registry/local_catalog.rb new file mode 100644 index 000000000..063dd7b29 --- /dev/null +++ b/app/web/registry/local_catalog.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Html2rss + module Web + module Registry + ## + # Builds catalog wire rows for local feeds.yml entries. + module LocalCatalog + module_function + + ## + # @return [Array Object}>] + def rows + LocalConfig.feeds.filter_map do |feed_name, feed_config| + build_row(feed_name, feed_config) + end + end + + ## + # @param feed_name [String, Symbol] + # @param feed_config [Hash{Symbol => Object}] + # @return [Hash{Symbol => Object}, nil] + def build_row(feed_name, feed_config) + directory = feed_config[:directory] || {} + title = directory[:title] + return nil if title.to_s.strip.empty? + + row_identity(feed_name, directory, title, feed_config[:channel] || {}) + end + + ## + # @param feed_name [String, Symbol] + # @param directory [Hash{Symbol => Object}] + # @param title [String] + # @param channel [Hash{Symbol => Object}] + # @return [Hash{Symbol => Object}] + def row_identity(feed_name, directory, title, channel) + id = feed_name.to_s + { + id:, + path: "/#{id}.rss", + source: 'local', + directory: directory_payload(directory, title), + channel: channel_payload(channel, title), + parameters: { schema: {}, defaults: {} } + } + end + + ## + # @param directory [Hash{Symbol => Object}] + # @param title [String] + # @return [Hash{Symbol => Object}] + def directory_payload(directory, title) + { + title: title.to_s, + summary: directory[:summary], + topics: Array(directory[:topics]) + }.compact + end + + ## + # @param channel [Hash{Symbol => Object}] + # @param title [String] + # @return [Hash{Symbol => Object}] + def channel_payload(channel, title) + { + url: channel.fetch(:url), + language: channel[:language], + title: channel[:title] || title.to_s + }.compact + end + end + end + end +end diff --git a/app/web/registry/store.rb b/app/web/registry/store.rb new file mode 100644 index 000000000..523b89931 --- /dev/null +++ b/app/web/registry/store.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +require 'fileutils' + +module Html2rss + module Web + module Registry + ## + # Manages on-disk registry bundle directories under {data_root}. + module Store + DEFAULT_DATA_ROOT = 'tmp/registry-data' + + class << self + ## + # @return [String] root directory for extracted registry bundles + def data_root + File.expand_path(ENV.fetch('REGISTRY_DATA_ROOT', DEFAULT_DATA_ROOT)) + end + + ## + # @param registry_id [String, Symbol] + # @return [String] active bundle directory for a registry id + def registry_dir(registry_id) + File.join(data_root, registry_id.to_s) + end + + ## + # Atomically replaces the active bundle directory for a registry id. + # + # @param registry_id [String, Symbol] + # @param staged_dir [String] verified bundle directory to promote + # @return [String] promoted bundle directory + def swap!(registry_id, staged_dir) + raise Errors::LoadError, "Staged bundle missing: #{staged_dir}" unless File.directory?(staged_dir) + + target = registry_dir(registry_id) + parent = File.dirname(target) + FileUtils.mkdir_p(parent) + + backup = "#{target}.backup.#{Process.pid}" + promote_bundle!(staged_dir, target, backup) + target + end + + ## + # Copies a seed bundle into the data root when no active bundle exists. + # + # @param registry_id [String, Symbol] + # @param seed_path [String] bundle directory to copy + # @return [Boolean] true when a seed copy was performed + def seed_if_empty!(registry_id, seed_path:) # rubocop:disable Naming/PredicateMethod + target = registry_dir(registry_id) + return false if bundle_present?(target) + + raise Errors::LoadError, "Seed bundle missing: #{seed_path}" unless File.directory?(seed_path) + + FileUtils.mkdir_p(File.dirname(target)) + FileUtils.cp_r(seed_path, target) + true + end + + private + + ## + # @param staged_dir [String] + # @param target [String] + # @param backup [String] + # @return [void] + def promote_bundle!(staged_dir, target, backup) + FileUtils.rm_rf(backup) + FileUtils.mv(target, backup) if File.exist?(target) + FileUtils.mv(staged_dir, target) + rescue StandardError + FileUtils.rm_rf(target) + FileUtils.mv(backup, target) if File.exist?(backup) + raise + ensure + FileUtils.rm_rf(backup) + end + + ## + # @param path [String] + # @return [Boolean] + def bundle_present?(path) + File.directory?(path) && !Dir.empty?(path) + end + end + end + end + end +end diff --git a/app/web/registry/sync.rb b/app/web/registry/sync.rb new file mode 100644 index 000000000..fb7188bee --- /dev/null +++ b/app/web/registry/sync.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Html2rss + module Web + module Registry + ## + # Fetches and stores signed registry bundles (network sync is expanded in Phase 4). + module Sync + OFFICIAL_RELEASE_URL = Config::OFFICIAL_RELEASE_URL + + SyncStatus = Data.define(:registry_id, :mode, :version, :updated_at, :sync_url, :last_error) + + class << self + ## + # Resolves the download URL for a sync-mode registry id. + # + # @param registry_id [String, Symbol] + # @return [String] + def sync_url_for(registry_id) + entry = Config.entry(registry_id) + raise Errors::SyncError, "Registry '#{registry_id}' is not sync-mode" unless entry.mode == :sync + + entry.sync_url + end + + ## + # Runs synchronization for a registry id. + # + # Phase 3 provides a minimal stub; Phase 4 adds fetch, verify, and store. + # + # @param registry_id [String, Symbol] + # @return [SyncStatus] + def run(registry_id:) + entry = Config.entry(registry_id) + if entry.mode == :path + raise Errors::SyncError, "Registry '#{registry_id}' uses path mode; sync is not applicable" + end + + raise Errors::SyncError, 'Registry network sync is not enabled until Phase 4' + end + + ## + # @param registry_id [String, Symbol, nil] + # @return [Array] + def status(registry_id: nil) + rows = Index.current.status + rows = rows.select { |row| row.id == registry_id.to_s } if registry_id + rows.map { |row| sync_status_for(row) } + end + + private + + ## + # @param row [Index::StatusEntry] + # @return [SyncStatus] + def sync_status_for(row) + entry = Config.entry(row.id) + SyncStatus.new( + registry_id: row.id, + mode: row.sync_mode, + version: row.version, + updated_at: row.updated_at, + sync_url: entry.sync_url, + last_error: nil + ) + end + end + end + end + end +end diff --git a/public/openapi.yaml b/public/openapi.yaml index 863713c24..6412619d1 100644 --- a/public/openapi.yaml +++ b/public/openapi.yaml @@ -132,101 +132,10 @@ paths: parameters: properties: defaults: - properties: - blog: - type: - - string - - 'null' - id: - type: - - string - - 'null' - region: - type: - - string - - 'null' - repository: - type: - - string - - 'null' - section: - type: - - string - - 'null' - user_id: - type: - - string - - 'null' - username: - type: - - string - - 'null' + properties: {} type: object schema: - properties: - blog: - properties: - type: - type: string - required: - - type - type: - - object - - 'null' - id: - properties: - type: - type: string - required: - - type - type: - - object - - 'null' - region: - properties: - type: - type: string - required: - - type - type: - - object - - 'null' - repository: - properties: - type: - type: string - required: - - type - type: - - object - - 'null' - section: - properties: - type: - type: string - required: - - type - type: - - object - - 'null' - user_id: - properties: - type: - type: string - required: - - type - type: - - object - - 'null' - username: - properties: - type: - type: string - required: - - type - type: - - object - - 'null' + properties: {} type: object required: - schema @@ -234,15 +143,18 @@ paths: type: object path: type: string + registry: + type: string source: type: string required: - id - path - - source - directory - channel - parameters + - source + - registry type: object type: array required: diff --git a/spec/fixtures/registries/official/configs/phys.org/weekly.yml b/spec/fixtures/registries/official/configs/phys.org/weekly.yml new file mode 100644 index 000000000..763f70f2b --- /dev/null +++ b/spec/fixtures/registries/official/configs/phys.org/weekly.yml @@ -0,0 +1,23 @@ +directory: + topics: + - science + title: "Phys.org — Weekly" + summary: "Top science news of the week from Phys.org." +channel: + language: en + title: "Phys.org — Weekly" + url: https://phys.org/weekly-news/ + time_zone: Europe/London + ttl: 1440 +selectors: + items: + selector: ".sorted-news-list .sorted-article-content" + title: + selector: "h4" + category: + selector: ".text-info" + categories: + - category + url: + selector: ".news-link" + extractor: "href" diff --git a/spec/fixtures/registries/official/configs/support.apple.com/en_gb_ht201222.yml b/spec/fixtures/registries/official/configs/support.apple.com/en_gb_ht201222.yml new file mode 100644 index 000000000..76afeafbe --- /dev/null +++ b/spec/fixtures/registries/official/configs/support.apple.com/en_gb_ht201222.yml @@ -0,0 +1,32 @@ +directory: + topics: + - tech + - security + title: "Apple Support — Security releases" + summary: "Apple security update and release notes (HT201222 / related)." +strategy: botasaurus +channel: + title: "Apple Support — Security releases" + url: https://support.apple.com/en-gb/100100 + language: en + ttl: 360 + time_zone: UTC +request: + botasaurus: + wait_for_selector: ".table-wrapper table tbody tr a" + wait_timeout_seconds: 20 +selectors: + items: + selector: ".table-wrapper table tbody > tr:not(:first-child)" + enhance: false + title: + selector: a + url: + selector: a + extractor: href + description: + selector: "td:nth-child(2)" + published_at: + selector: "td:nth-child(3)" + post_process: + - name: parse_time diff --git a/spec/fixtures/registries/official/manifest.json b/spec/fixtures/registries/official/manifest.json new file mode 100644 index 000000000..17d99eece --- /dev/null +++ b/spec/fixtures/registries/official/manifest.json @@ -0,0 +1,10 @@ +{ + "format": "registry.v1", + "registry_id": "official", + "version": "test-fixture", + "public_key_id": "test", + "files": { + "configs/phys.org/weekly.yml": "0f2a052f5c6da5c260985058302805b962da3211b635483667e258c714b31373", + "configs/support.apple.com/en_gb_ht201222.yml": "3a6f84899ac24472f6476c697d50ac366425ae1c87f7ddf045cc9e2c80e93eac" + } +} diff --git a/spec/fixtures/registries/private/configs/secret.example/private.yml b/spec/fixtures/registries/private/configs/secret.example/private.yml new file mode 100644 index 000000000..6b516f6c2 --- /dev/null +++ b/spec/fixtures/registries/private/configs/secret.example/private.yml @@ -0,0 +1,15 @@ +directory: + title: "Private Corp Feed" + summary: "Hidden from catalog." +channel: + title: "Private Corp Feed" + url: https://secret.example/private + ttl: 60 +selectors: + items: + selector: article + title: + selector: h1 + url: + selector: a + extractor: href diff --git a/spec/fixtures/registries/private/manifest.json b/spec/fixtures/registries/private/manifest.json new file mode 100644 index 000000000..0aacc3ef5 --- /dev/null +++ b/spec/fixtures/registries/private/manifest.json @@ -0,0 +1,9 @@ +{ + "format": "registry.v1", + "registry_id": "private", + "version": "test-private", + "public_key_id": "test", + "files": { + "configs/secret.example/private.yml": "196760640262a019c0b32e08d2f5272c08b11b10b5880dc13266922a30c872a3" + } +} diff --git a/spec/fixtures/registries/registries.yml b/spec/fixtures/registries/registries.yml new file mode 100644 index 000000000..b505e5794 --- /dev/null +++ b/spec/fixtures/registries/registries.yml @@ -0,0 +1,11 @@ +precedence: + - official + - private + +registries: + official: + path: spec/fixtures/registries/official + catalog: true + private: + path: spec/fixtures/registries/private + catalog: false diff --git a/spec/html2rss/web/api/v1_spec.rb b/spec/html2rss/web/api/v1_spec.rb index e037715f7..4d505cebf 100644 --- a/spec/html2rss/web/api/v1_spec.rb +++ b/spec/html2rss/web/api/v1_spec.rb @@ -221,7 +221,10 @@ def relative_feed_link_header(token) json = expect_success_response(last_response) expect(json.dig('meta', 'catalog_version')).to eq(1) expect(json.dig('data', 'configs')).to be_an(Array) - expect(json.dig('data', 'configs').first).to include('id', 'path', 'source', 'directory', 'channel', 'parameters') + first = json.dig('data', 'configs').first + expect(first).to include('id', 'path', 'source', 'directory', 'channel', 'parameters') + expect(first['source']).to eq('registry') + expect(first).to include('registry' => 'official') end it 'returns 404 when the catalog is disabled', :aggregate_failures do diff --git a/spec/html2rss/web/local_config_spec.rb b/spec/html2rss/web/local_config_spec.rb index aa0a10d35..dfb6f9b44 100644 --- a/spec/html2rss/web/local_config_spec.rb +++ b/spec/html2rss/web/local_config_spec.rb @@ -51,33 +51,19 @@ def account_token(snapshot) expect(titles_for('example.json', 'example.rss', 'example.xml')).to eq(%w[Example Example Example]) end - it 'falls back to embedded configs when the feed is not in local yaml' do - stub_const('Html2rss::Configs', Module.new do - def self.find_by_name(_name); end - end) - stub_const('Html2rss::Configs::ConfigNotFound', Class.new(StandardError)) - allow(Html2rss::Configs) - .to receive(:find_by_name) - .with('support.apple.com/en_gb_ht201222') - .and_return({ channel: { title: 'Apple security releases' } }) + it 'falls back to registry configs when the feed is not in local yaml' do allow(described_class).to receive(:snapshot).and_return(empty_snapshot) config = described_class.find('support.apple.com/en_gb_ht201222.rss') - expect(config).to include(channel: { title: 'Apple security releases' }) + expect(config).to include(channel: hash_including(title: 'Apple Support — Security releases')) end - it 'returns not found for malformed embedded config paths instead of depending on gem error messages' do - stub_const('Html2rss::Configs', Module.new do - def self.find_by_name(_name); end - end) - stub_const('Html2rss::Configs::ConfigNotFound', Class.new(StandardError)) - allow(Html2rss::Configs).to receive(:find_by_name) + it 'returns not found for unknown feed ids' do allow(described_class).to receive(:snapshot).and_return(empty_snapshot) expect { described_class.find('/broken-name.rss') } .to raise_error(described_class::NotFound, "Did not find local feed config at 'broken-name'") - expect(Html2rss::Configs).not_to have_received(:find_by_name) end end diff --git a/spec/html2rss/web/registry/index_spec.rb b/spec/html2rss/web/registry/index_spec.rb new file mode 100644 index 000000000..3b9cf095b --- /dev/null +++ b/spec/html2rss/web/registry/index_spec.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +require 'spec_helper' + +require_relative '../../../../app' + +RSpec.describe Html2rss::Web::Registry::Index do + describe '#config_for' do + it 'returns registry configs by feed id' do + config = described_class.current.config_for('support.apple.com/en_gb_ht201222') + + expect(config).to include(channel: hash_including(title: 'Apple Support — Security releases')) + end + + it 'returns nil for unknown ids' do + expect(described_class.current.config_for('missing.example/feed')).to be_nil + end + + it 'still resolves catalog-disabled registry feeds by id' do + config = described_class.current.config_for('secret.example/private') + + expect(config).to include(channel: hash_including(url: 'https://secret.example/private')) + end + end + + describe '#catalog_rows' do + it 'includes registry rows with source and registry fields' do + rows = described_class.current.catalog_rows + phys = rows.find { |row| row.fetch(:id) == 'phys.org/weekly' } + + expect(phys).to include( + source: 'registry', + registry: 'official', + path: '/phys.org/weekly.rss' + ) + end + + it 'omits catalog-disabled registries from the catalog API rows' do + rows = described_class.current.catalog_rows + + expect(rows.map { |row| row.fetch(:id) }).not_to include('secret.example/private') + end + end + + describe '#status' do + it 'reports loaded registry metadata' do + status = described_class.current.status + official = status.find { |entry| entry.id == 'official' } + + expect(official).to have_attributes( + version: 'test-fixture', + sync_mode: :path + ) + end + end +end diff --git a/spec/html2rss/web/registry/sync_spec.rb b/spec/html2rss/web/registry/sync_spec.rb new file mode 100644 index 000000000..8c7703475 --- /dev/null +++ b/spec/html2rss/web/registry/sync_spec.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require 'spec_helper' + +require_relative '../../../../app' + +RSpec.describe Html2rss::Web::Registry::Sync do + describe '.sync_url_for' do + it 'resolves the official release URL for sync-mode defaults' do + ClimateControl.modify('REGISTRIES_CONFIG' => nil) do + Html2rss::Web::Registry::Index.reload! + + expect(described_class.sync_url_for('official')).to eq( + Html2rss::Web::Registry::Config::OFFICIAL_RELEASE_URL + ) + end + end + end + + describe '.run' do + it 'rejects path-mode registries' do + expect { described_class.run(registry_id: 'official') } + .to raise_error(Html2rss::Web::Registry::Errors::SyncError, /path mode/) + end + end +end diff --git a/spec/support/registry.rb b/spec/support/registry.rb new file mode 100644 index 000000000..c07e52949 --- /dev/null +++ b/spec/support/registry.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module RegistryTestHelpers + FIXTURES_ROOT = File.expand_path('../fixtures/registries', __dir__) + DEFAULT_REGISTRIES_CONFIG = File.join(FIXTURES_ROOT, 'registries.yml') + + module_function + + def configure_registry_fixtures! + ENV['REGISTRIES_CONFIG'] = DEFAULT_REGISTRIES_CONFIG + ENV['REGISTRY_DATA_ROOT'] = File.join(Dir.pwd, 'tmp', 'test-registry-data') + end + + def reset_registry! + Html2rss::Web::Registry::Index.reload! + end +end + +RSpec.configure do |config| + config.before do + RegistryTestHelpers.configure_registry_fixtures! + RegistryTestHelpers.reset_registry! + end +end From f103414af8a3e1e4b66cf98091bfc0bdb6d6f647 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 14:56:02 +0200 Subject: [PATCH 02/33] feat(registry): sync CLI, seed bundle, and background refresh --- Dockerfile | 2 + app/registries/seed/official/.gitkeep | 0 app/web/api/v1/root_metadata.rb | 40 +- app/web/boot/setup.rb | 6 + app/web/registry/config.rb | 2 +- app/web/registry/index.rb | 2 +- app/web/registry/store.rb | 65 ++- app/web/registry/sync.rb | 378 +++++++++++++++++- app/web/security/security_logger.rb | 7 + bin/docker-build | 1 + bin/prepare-registry-seed | 31 ++ bin/registry-sync | 60 +++ config/registries.yml | 11 + docker-compose.yml | 6 + public/openapi.yaml | 21 +- .../sync/bundle/configs/phys.org/weekly.yml | 23 ++ .../support.apple.com/en_gb_ht201222.yml | 32 ++ .../registries/sync/bundle/manifest.json | 1 + spec/fixtures/registries/sync/registries.yml | 13 + spec/html2rss/web/api/v1_spec.rb | 15 + spec/html2rss/web/boot/setup_spec.rb | 2 + spec/html2rss/web/registry/sync_spec.rb | 93 +++++ spec/support/registry_sync.rb | 64 +++ 23 files changed, 851 insertions(+), 24 deletions(-) create mode 100644 app/registries/seed/official/.gitkeep create mode 100755 bin/prepare-registry-seed create mode 100755 bin/registry-sync create mode 100644 config/registries.yml create mode 100644 spec/fixtures/registries/sync/bundle/configs/phys.org/weekly.yml create mode 100644 spec/fixtures/registries/sync/bundle/configs/support.apple.com/en_gb_ht201222.yml create mode 100644 spec/fixtures/registries/sync/bundle/manifest.json create mode 100644 spec/fixtures/registries/sync/registries.yml create mode 100644 spec/support/registry_sync.rb diff --git a/Dockerfile b/Dockerfile index d9bc8e0a5..0d91ae5cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -78,6 +78,7 @@ RUN apk add --no-cache \ && mkdir -p /app \ && mkdir -p /app/tmp/rack-cache-body \ && mkdir -p /app/tmp/rack-cache-meta \ + && mkdir -p /app/data/registries \ && chown "$USER":"$USER" -R /app WORKDIR /app @@ -89,6 +90,7 @@ COPY --chown=$USER:$USER bin/docker-healthcheck ./bin/docker-healthcheck COPY --chown=$USER:$USER Gemfile Gemfile.lock app.rb config.ru ./ COPY --chown=$USER:$USER app ./app COPY --chown=$USER:$USER config ./config +COPY --chown=$USER:$USER app/registries/seed ./app/registries/seed COPY --chown=$USER:$USER public ./public COPY --from=frontend-builder --chown=$USER:$USER /app/frontend/dist ./frontend/dist diff --git a/app/registries/seed/official/.gitkeep b/app/registries/seed/official/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/app/web/api/v1/root_metadata.rb b/app/web/api/v1/root_metadata.rb index 562f5a86b..01a989068 100644 --- a/app/web/api/v1/root_metadata.rb +++ b/app/web/api/v1/root_metadata.rb @@ -27,16 +27,40 @@ def build(router) # @return [Hash{Symbol=>Object}] def instance_payload(router) { - feed_creation: { - enabled: Flags.auto_source_enabled?, - access_token_required: Flags.auto_source_enabled? - }, - catalog: { - enabled: Flags.config_catalog_enabled?, - url: "#{router.base_url}/api/v1/configs" - } + feed_creation: feed_creation_payload, + catalog: catalog_payload(router), + registries: registry_status_rows + } + end + + # @return [Hash{Symbol => Object}] + def feed_creation_payload + { + enabled: Flags.auto_source_enabled?, + access_token_required: Flags.auto_source_enabled? + } + end + + # @param router [Roda::RodaRequest] + # @return [Hash{Symbol => Object}] + def catalog_payload(router) + { + enabled: Flags.config_catalog_enabled?, + url: "#{router.base_url}/api/v1/configs" } end + + # @return [Array Object}>] + def registry_status_rows + Registry::Index.current.status.map do |entry| + { + id: entry.id, + version: entry.version, + updated_at: entry.updated_at&.utc&.iso8601, + sync_mode: entry.sync_mode.to_s + } + end + end end end end diff --git a/app/web/boot/setup.rb b/app/web/boot/setup.rb index 55ac05b34..d41be5449 100644 --- a/app/web/boot/setup.rb +++ b/app/web/boot/setup.rb @@ -24,6 +24,7 @@ def call! configure_request_service! configure_runtime_logging! configure_gem_defaults! + configure_registry! log_startup! end @@ -76,6 +77,11 @@ def configure_runtime_logging! Rack::Timeout::Logger.logger = AppLogger.logger end + # @return [void] + def configure_registry! + Registry::Sync.boot! + end + # @return [void] def log_startup! AppLogger.logger.info( diff --git a/app/web/registry/config.rb b/app/web/registry/config.rb index 06f693554..4fd5b8842 100644 --- a/app/web/registry/config.rb +++ b/app/web/registry/config.rb @@ -184,7 +184,7 @@ def sync_section(raw) def public_key_id_for(public_key_id) return public_key_id if public_key_id && !public_key_id.empty? - 'official-v1' + 'html2rss:registry:2026' end ## diff --git a/app/web/registry/index.rb b/app/web/registry/index.rb index 9132d728f..6052c4bf4 100644 --- a/app/web/registry/index.rb +++ b/app/web/registry/index.rb @@ -102,7 +102,7 @@ def load_bundle(registry_id) # rubocop:disable Metrics/MethodLength bundle = Html2rss::Registry::Bundle.load( directory, - trust: entry.mode == :path ? :integrity_only : :signed, + trust: :integrity_only, public_keys: entry.public_keys ) to_registry_bundle(registry_id, bundle) diff --git a/app/web/registry/store.rb b/app/web/registry/store.rb index 523b89931..fc9e7459e 100644 --- a/app/web/registry/store.rb +++ b/app/web/registry/store.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'fileutils' +require 'json' module Html2rss module Web @@ -9,6 +10,8 @@ module Registry # Manages on-disk registry bundle directories under {data_root}. module Store DEFAULT_DATA_ROOT = 'tmp/registry-data' + DEFAULT_SEED_ROOT = '/app/registries/seed' + SYNC_STATE_FILE = '.sync-state.json' class << self ## @@ -17,6 +20,12 @@ def data_root File.expand_path(ENV.fetch('REGISTRY_DATA_ROOT', DEFAULT_DATA_ROOT)) end + ## + # @return [String] root directory for image-shipped seed bundles + def seed_root + File.expand_path(ENV.fetch('REGISTRY_SEED_ROOT', DEFAULT_SEED_ROOT)) + end + ## # @param registry_id [String, Symbol] # @return [String] active bundle directory for a registry id @@ -24,6 +33,20 @@ def registry_dir(registry_id) File.join(data_root, registry_id.to_s) end + ## + # @param registry_id [String, Symbol] + # @return [String] seed bundle directory for a registry id + def seed_path_for(registry_id) + File.join(seed_root, registry_id.to_s) + end + + ## + # @param registry_id [String, Symbol] + # @return [Boolean] + def bundle_present?(registry_id) + bundle_present_at?(registry_dir(registry_id)) + end + ## # Atomically replaces the active bundle directory for a registry id. # @@ -50,7 +73,7 @@ def swap!(registry_id, staged_dir) # @return [Boolean] true when a seed copy was performed def seed_if_empty!(registry_id, seed_path:) # rubocop:disable Naming/PredicateMethod target = registry_dir(registry_id) - return false if bundle_present?(target) + return false if bundle_present?(registry_id) raise Errors::LoadError, "Seed bundle missing: #{seed_path}" unless File.directory?(seed_path) @@ -59,8 +82,46 @@ def seed_if_empty!(registry_id, seed_path:) # rubocop:disable Naming/PredicateMe true end + ## + # @param registry_id [String, Symbol] + # @return [Hash{Symbol => Object}] + def sync_state(registry_id) + read_sync_state.fetch(registry_id.to_s, {}) + end + + ## + # @param registry_id [String, Symbol] + # @param last_error [String, nil] + # @param last_sync_at [Time, nil] + # @return [void] + def write_sync_state!(registry_id, last_error:, last_sync_at: Time.now.utc) + state = read_sync_state + state[registry_id.to_s] = { + 'last_error' => last_error, + 'last_sync_at' => last_sync_at&.iso8601 + }.compact + FileUtils.mkdir_p(data_root) + File.write(sync_state_path, JSON.generate(state)) + end + private + ## + # @return [String] + def sync_state_path + File.join(data_root, SYNC_STATE_FILE) + end + + ## + # @return [Hash{String => Hash{String => Object}}] + def read_sync_state + return {} unless File.file?(sync_state_path) + + JSON.parse(File.read(sync_state_path)) + rescue JSON::ParserError + {} + end + ## # @param staged_dir [String] # @param target [String] @@ -81,7 +142,7 @@ def promote_bundle!(staged_dir, target, backup) ## # @param path [String] # @return [Boolean] - def bundle_present?(path) + def bundle_present_at?(path) File.directory?(path) && !Dir.empty?(path) end end diff --git a/app/web/registry/sync.rb b/app/web/registry/sync.rb index fb7188bee..4d7f30cf9 100644 --- a/app/web/registry/sync.rb +++ b/app/web/registry/sync.rb @@ -1,16 +1,39 @@ # frozen_string_literal: true +require 'fileutils' +require 'net/http' +require 'json' +require 'stringio' +require 'uri' + module Html2rss module Web module Registry ## - # Fetches and stores signed registry bundles (network sync is expanded in Phase 4). - module Sync + # Fetches, verifies, and stores signed registry bundles. + module Sync # rubocop:disable Metrics/ModuleLength OFFICIAL_RELEASE_URL = Config::OFFICIAL_RELEASE_URL + OFFICIAL_GITHUB_RELEASES_API = + 'https://api.github.com/repos/html2rss/html2rss-configs/releases/latest' + OFFICIAL_ASSET_NAME = 'registry-bundle.tar.gz' + + DEFAULT_ALLOWED_HOSTS = %w[ + api.github.com + github.com + objects.githubusercontent.com + ].freeze + FETCH_OPEN_TIMEOUT_SECONDS = 10 + FETCH_READ_TIMEOUT_SECONDS = 60 + MAX_RESPONSE_BYTES = Html2rss::Registry::Archive::MAX_TARBALL_BYTES + BACKGROUND_JITTER_FRACTION = 0.1 SyncStatus = Data.define(:registry_id, :mode, :version, :updated_at, :sync_url, :last_error) - class << self + @boot_mutex = Mutex.new + @boot_started = false + @timer_started = false + + class << self # rubocop:disable Metrics/ClassLength ## # Resolves the download URL for a sync-mode registry id. # @@ -20,23 +43,35 @@ def sync_url_for(registry_id) entry = Config.entry(registry_id) raise Errors::SyncError, "Registry '#{registry_id}' is not sync-mode" unless entry.mode == :sync - entry.sync_url + resolve_download_url(entry) end ## # Runs synchronization for a registry id. # - # Phase 3 provides a minimal stub; Phase 4 adds fetch, verify, and store. - # # @param registry_id [String, Symbol] + # @param dry_run [Boolean] when true, verify without swapping the active bundle # @return [SyncStatus] - def run(registry_id:) + def run(registry_id:, dry_run: false) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength entry = Config.entry(registry_id) if entry.mode == :path raise Errors::SyncError, "Registry '#{registry_id}' uses path mode; sync is not applicable" end - raise Errors::SyncError, 'Registry network sync is not enabled until Phase 4' + staging_root = nil + download_url = resolve_download_url(entry) + staging_root, staged_dir = fetch_and_verify!(entry, download_url) + unless dry_run + Store.swap!(registry_id, staged_dir) + Index.reload! + record_success!(registry_id) + end + sync_status_for(Index.current.status.find { |row| row.id == registry_id.to_s }) + rescue StandardError => error + record_failure!(registry_id, error) unless dry_run + raise + ensure + FileUtils.rm_rf(staging_root) if staging_root end ## @@ -48,24 +83,345 @@ def status(registry_id: nil) rows.map { |row| sync_status_for(row) } end + ## + # Seeds sync-mode registries and optionally syncs on boot. + # + # @return [void] + def boot! # rubocop:disable Metrics/MethodLength + @boot_mutex.synchronize do + return if @boot_started + + @boot_started = true + end + return if skip_boot? + + Config.precedence.each do |registry_id| + entry = Config.entry(registry_id) + next unless entry.mode == :sync + + seed_registry!(registry_id) + schedule_boot_sync!(registry_id) if boot_sync?(registry_id) + end + + start_background_timer! + end + + ## + # Starts a jittered background sync loop when enabled. + # + # @return [void] + def start_background_timer! # rubocop:disable Metrics/MethodLength + interval_hours = Integer(ENV.fetch('REGISTRY_SYNC_INTERVAL_HOURS', '24')) + return if interval_hours <= 0 + + @boot_mutex.synchronize do + return if @timer_started + + @timer_started = true + end + + Thread.new do # rubocop:disable ThreadSafety/NewThread -- background registry refresh by design + sleep(background_jitter_seconds(interval_hours)) + loop do + sync_all! + sleep(interval_hours * 3600) + end + end + end + + ## + # @return [Integer] process exit code for CLI use (0 ok, 1 when sync registries lack bundles) + def cli_exit_code + unusable_sync_registries.empty? ? 0 : 1 + end + + ## + # @return [Array] sync-mode registry ids without a usable on-disk bundle + def unusable_sync_registries + Config.precedence.filter_map do |registry_id| + entry = Config.entry(registry_id) + next unless entry.mode == :sync + next if Store.bundle_present?(registry_id) + + registry_id + end + end + private + ## + # @return [Boolean] + def skip_boot? + ENV.fetch('RACK_ENV', 'development') == 'test' + end + + ## + # @param registry_id [String] + # @return [void] + def seed_registry!(registry_id) # rubocop:disable Metrics/MethodLength + seed_path = Store.seed_path_for(registry_id) + return unless File.directory?(seed_path) + + seeded = Store.seed_if_empty!(registry_id, seed_path:) + Index.reload! if seeded + rescue Errors::LoadError => error + AppLogger.logger.warn( + { + component: 'registry', + event_name: 'registry.seed', + outcome: 'failure', + registry_id:, + error: error.message + }.to_json + ) + end + + ## + # @param registry_id [String] + # @return [Boolean] + def boot_sync?(registry_id) + ENV.fetch('REGISTRY_SYNC_ON_BOOT', 'false') == 'true' || !Store.bundle_present?(registry_id) + end + + ## + # @param registry_id [String] + # @return [void] + def schedule_boot_sync!(registry_id) + Thread.new { run(registry_id:) } # rubocop:disable ThreadSafety/NewThread -- non-blocking first boot + rescue StandardError + nil + end + + ## + # @return [void] + def sync_all! + Config.precedence.each do |registry_id| + entry = Config.entry(registry_id) + next unless entry.mode == :sync + + run(registry_id:) + rescue StandardError + nil + end + end + + ## + # @param interval_hours [Integer] + # @return [Numeric] + def background_jitter_seconds(interval_hours) + max_jitter = [(interval_hours * 3600 * BACKGROUND_JITTER_FRACTION).to_i, 1].max + rand(max_jitter) + end + + ## + # @param entry [Entry] + # @return [String] + def resolve_download_url(entry) + return entry.sync_url if entry.sync_url && !entry.sync_url.empty? + return resolve_official_download_url if entry.sync_channel == Config::DEFAULT_OFFICIAL_SYNC_CHANNEL + + raise Errors::SyncError, "Registry '#{entry.id}' has no sync URL" + end + + ## + # @return [String] + def resolve_official_download_url + response_body = Fetcher.fetch!(OFFICIAL_GITHUB_RELEASES_API) + release = JSON.parse(response_body, symbolize_names: true) + asset = Array(release[:assets]).find { |row| row[:name] == OFFICIAL_ASSET_NAME } + url = asset&.dig(:browser_download_url) + raise Errors::SyncError, "Official release asset '#{OFFICIAL_ASSET_NAME}' not found" unless url + + url + rescue JSON::ParserError => error + raise Errors::SyncError, "Invalid GitHub release metadata: #{error.message}" + end + + ## + # @param entry [Entry] + # @param download_url [String] + # @return [Array(String, String)] staging root and verified bundle directory + def fetch_and_verify!(entry, download_url) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + tarball = Fetcher.fetch!(download_url) + staging_root = Dir.mktmpdir('registry-sync-') + staged_dir = File.join(staging_root, 'bundle') + FileUtils.mkdir_p(staged_dir) + + StringIO.open(tarball) do |io| + Html2rss::Registry::Archive.extract!(io, into: staged_dir) + end + + Html2rss::Registry::Verifier.verify!( + staged_dir, + trust: :signed, + public_keys: entry.public_keys + ) + [staging_root, staged_dir] + rescue Html2rss::Registry::VerificationError => error + log_signature_failure!(entry.id, error.message) if signature_failure?(error) + raise Errors::SyncError, error.message + rescue Html2rss::Registry::ArchiveError => error + raise Errors::SyncError, error.message + end + + ## + # @param error [Html2rss::Registry::VerificationError] + # @return [Boolean] + def signature_failure?(error) + error.message.match?(/signature|public_key_id/i) + end + + ## + # @param registry_id [String] + # @param message [String] + # @return [void] + def log_signature_failure!(registry_id, message) + SecurityLogger.log_registry_signature_failure(registry_id, message) + end + + ## + # @param registry_id [String] + # @return [void] + def record_success!(registry_id) + Store.write_sync_state!(registry_id, last_error: nil) + row = Index.current.status.find { |entry| entry.id == registry_id.to_s } + Observability.emit( + event_name: 'registry.sync', + outcome: 'success', + details: { registry_id:, version: row&.version } + ) + end + + ## + # @param registry_id [String] + # @param error [StandardError] + # @return [void] + def record_failure!(registry_id, error) + Store.write_sync_state!(registry_id, last_error: error.message) + Observability.emit( + event_name: 'registry.sync', + outcome: 'failure', + details: { registry_id:, error: error.message }, + level: :warn + ) + end + ## # @param row [Index::StatusEntry] # @return [SyncStatus] def sync_status_for(row) entry = Config.entry(row.id) + state = Store.sync_state(row.id) SyncStatus.new( registry_id: row.id, mode: row.sync_mode, version: row.version, updated_at: row.updated_at, - sync_url: entry.sync_url, - last_error: nil + sync_url: entry.mode == :sync ? resolve_download_url(entry) : nil, + last_error: state['last_error'] ) end + end # rubocop:enable Metrics/ClassLength + + ## + # HTTPS fetcher with host allowlisting and redirect rejection. + module Fetcher + module_function + + ## + # @param url [String] + # @return [String] response body + def fetch!(url) + uri = parse_https_uri!(url) + ensure_allowed_host!(uri.host) + perform_request!(uri) + end + + ## + # @param url [String] + # @return [URI::HTTPS] + def parse_https_uri!(url) + uri = URI(url) + unless uri.is_a?(URI::HTTPS) + raise Errors::SyncError, "Registry sync requires HTTPS URLs (got #{uri.scheme.inspect})" + end + + uri + end + + ## + # @param host [String] + # @return [void] + def ensure_allowed_host!(host) + allowed = DEFAULT_ALLOWED_HOSTS + extra_allowed_hosts + return if allowed.include?(host) + + raise Errors::SyncError, "Registry sync host not allowed: #{host}" + end + + ## + # @return [Array] + def extra_allowed_hosts + ENV.fetch('REGISTRY_SYNC_ALLOWED_HOSTS', '') + .split(',') + .map(&:strip) + .reject(&:empty?) + end + + ## + # @param uri [URI::HTTPS] + # @return [String] + def perform_request!(uri) # rubocop:disable Metrics/MethodLength + response = nil + Net::HTTP.start( + uri.host, + uri.port, + use_ssl: true, + open_timeout: FETCH_OPEN_TIMEOUT_SECONDS, + read_timeout: FETCH_READ_TIMEOUT_SECONDS + ) do |http| + request = Net::HTTP::Get.new(uri) + request['Accept'] = 'application/octet-stream' + request['User-Agent'] = 'html2rss-web/registry-sync' + response = http.request(request) + end + + reject_redirect!(response) + reject_error_status!(response) + read_body!(response) + end + + ## + # @param response [Net::HTTPResponse] + # @return [void] + def reject_redirect!(response) + return unless response.is_a?(Net::HTTPRedirection) + + raise Errors::SyncError, 'Registry sync rejects HTTP redirects' + end + + ## + # @param response [Net::HTTPResponse] + # @return [void] + def reject_error_status!(response) + return if response.is_a?(Net::HTTPSuccess) + + raise Errors::SyncError, "Registry sync fetch failed with HTTP #{response.code}" + end + + ## + # @param response [Net::HTTPResponse] + # @return [String] + def read_body!(response) + body = response.body.to_s + if body.bytesize > MAX_RESPONSE_BYTES + raise Errors::SyncError, "Registry sync response exceeds max bytes (#{MAX_RESPONSE_BYTES})" + end + + body + end end - end + end # rubocop:enable Metrics/ModuleLength end end end diff --git a/app/web/security/security_logger.rb b/app/web/security/security_logger.rb index 1349e8155..e6293584b 100644 --- a/app/web/security/security_logger.rb +++ b/app/web/security/security_logger.rb @@ -69,6 +69,13 @@ def log_blocked_request(ip, reason, endpoint) log_event('blocked_request', { ip:, reason:, endpoint: }, severity: :warn) end + # @param registry_id [String] + # @param reason [String] + # @return [void] + def log_registry_signature_failure(registry_id, reason) + log_event('registry_signature_failure', { registry_id:, reason: }, severity: :warn) + end + private # @param event_type [String] diff --git a/bin/docker-build b/bin/docker-build index 7b4e5f9d7..7fd9be1c9 100755 --- a/bin/docker-build +++ b/bin/docker-build @@ -1,4 +1,5 @@ #!/bin/bash set -eux +"$(dirname "$0")/prepare-registry-seed" docker build --no-cache -t html2rss/web -f Dockerfile . diff --git a/bin/prepare-registry-seed b/bin/prepare-registry-seed new file mode 100755 index 000000000..a2c38dd59 --- /dev/null +++ b/bin/prepare-registry-seed @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# frozen_string_literal: true + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" +CONFIGS_REPO="${HTML2RSS_CONFIGS_ROOT:-$ROOT_DIR/../html2rss-configs}" +SEED_DIR="$ROOT_DIR/app/registries/seed/official" +OUTPUT_TAR="$(mktemp -t registry-seed.XXXXXX.tar.gz)" + +cleanup() { + rm -f "$OUTPUT_TAR" +} +trap cleanup EXIT + +if [[ ! -d "$CONFIGS_REPO/configs" ]]; then + echo "Missing configs tree at $CONFIGS_REPO/configs" >&2 + exit 1 +fi + +echo "Building registry seed bundle from $CONFIGS_REPO" +( + cd "$CONFIGS_REPO" + BUNDLE_GEMFILE=tool/Gemfile bundle exec ruby tool/registry-build --output "$OUTPUT_TAR" +) + +rm -rf "$SEED_DIR" +mkdir -p "$SEED_DIR" +tar -xzf "$OUTPUT_TAR" -C "$SEED_DIR" + +echo "Seed bundle prepared at $SEED_DIR" diff --git a/bin/registry-sync b/bin/registry-sync new file mode 100755 index 000000000..1c013a099 --- /dev/null +++ b/bin/registry-sync @@ -0,0 +1,60 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require 'optparse' + +require_relative '../app' + +module RegistrySyncCLI + module_function + + def run(argv = ARGV) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + options = { status: false, registry_id: nil, dry_run: false } + OptionParser.new do |opts| + opts.banner = 'Usage: bin/registry-sync [options]' + opts.on('--status', 'Print registry sync status and exit') { options[:status] = true } + opts.on('--registry ID', 'Sync a single registry id') { |value| options[:registry_id] = value } + opts.on('--dry-run', 'Fetch and verify without swapping the active bundle') { options[:dry_run] = true } + end.parse!(argv) + + if options[:status] + print_status_table(options[:registry_id]) + exit Html2rss::Web::Registry::Sync.cli_exit_code + end + + target_registry_ids(options[:registry_id]).each do |registry_id| + Html2rss::Web::Registry::Sync.run(registry_id:, dry_run: options[:dry_run]) + end + exit Html2rss::Web::Registry::Sync.cli_exit_code + end + + def target_registry_ids(registry_id) + return Array(registry_id) if registry_id + + Html2rss::Web::Registry::Config.precedence + end + + def print_status_table(registry_id) # rubocop:disable Metrics/MethodLength + rows = Html2rss::Web::Registry::Sync.status(registry_id:) + headers = %w[registry mode version updated_at sync_url last_error] + puts headers.join("\t") + rows.each do |row| + puts [ + row.registry_id, + row.mode, + row.version || '-', + format_time(row.updated_at), + row.sync_url || '-', + row.last_error || '-' + ].join("\t") + end + end + + def format_time(value) + return '-' unless value + + value.utc.iso8601 + end +end + +RegistrySyncCLI.run if $PROGRAM_NAME == __FILE__ diff --git a/config/registries.yml b/config/registries.yml new file mode 100644 index 000000000..3ebeca8e5 --- /dev/null +++ b/config/registries.yml @@ -0,0 +1,11 @@ +precedence: + - official + +registries: + official: + sync: + channel: html2rss-official + catalog: true + public_key_id: html2rss:registry:2026 + # Pin the publisher Ed25519 public key for signed network sync. + # Seed bundles copied from the image use integrity-only verification. diff --git a/docker-compose.yml b/docker-compose.yml index c098fb039..6ca4cc89c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,6 +24,7 @@ services: RACK_TIMEOUT_SERVICE_TIMEOUT: 55 BOTASAURUS_SCRAPE_TIMEOUT_SECONDS: 45 BOTASAURUS_SCRAPE_WORK_TIMEOUT_SECONDS: 30 + REGISTRY_DATA_ROOT: /app/data/registries BOTASAURUS_SCRAPER_URL: http://botasaurus:4010 # Trial runs use the image's bundled config/feeds.yml. # Uncomment the block below when you want to replace it with your own file. @@ -32,6 +33,8 @@ services: # source: ./config/feeds.yml # target: /app/config/feeds.yml # read_only: true + volumes: + - registry-data:/app/data/registries watchtower: image: containrrr/watchtower @@ -53,3 +56,6 @@ services: SCRAPE_WORK_TIMEOUT_SECONDS: 30 ports: - "127.0.0.1:4010:4010" + +volumes: + registry-data: diff --git a/public/openapi.yaml b/public/openapi.yaml index 6412619d1..6f9164eb9 100644 --- a/public/openapi.yaml +++ b/public/openapi.yaml @@ -64,9 +64,28 @@ paths: - enabled - access_token_required type: object + registries: + items: + properties: + id: + type: string + sync_mode: + type: string + updated_at: + type: string + version: + type: string + required: + - id + - version + - updated_at + - sync_mode + type: object + type: array required: - feed_creation - catalog + - registries type: object required: - api @@ -78,7 +97,7 @@ paths: - success - data type: object - description: returns catalog pointer metadata + description: returns registry status metadata security: - {} summary: API metadata diff --git a/spec/fixtures/registries/sync/bundle/configs/phys.org/weekly.yml b/spec/fixtures/registries/sync/bundle/configs/phys.org/weekly.yml new file mode 100644 index 000000000..763f70f2b --- /dev/null +++ b/spec/fixtures/registries/sync/bundle/configs/phys.org/weekly.yml @@ -0,0 +1,23 @@ +directory: + topics: + - science + title: "Phys.org — Weekly" + summary: "Top science news of the week from Phys.org." +channel: + language: en + title: "Phys.org — Weekly" + url: https://phys.org/weekly-news/ + time_zone: Europe/London + ttl: 1440 +selectors: + items: + selector: ".sorted-news-list .sorted-article-content" + title: + selector: "h4" + category: + selector: ".text-info" + categories: + - category + url: + selector: ".news-link" + extractor: "href" diff --git a/spec/fixtures/registries/sync/bundle/configs/support.apple.com/en_gb_ht201222.yml b/spec/fixtures/registries/sync/bundle/configs/support.apple.com/en_gb_ht201222.yml new file mode 100644 index 000000000..76afeafbe --- /dev/null +++ b/spec/fixtures/registries/sync/bundle/configs/support.apple.com/en_gb_ht201222.yml @@ -0,0 +1,32 @@ +directory: + topics: + - tech + - security + title: "Apple Support — Security releases" + summary: "Apple security update and release notes (HT201222 / related)." +strategy: botasaurus +channel: + title: "Apple Support — Security releases" + url: https://support.apple.com/en-gb/100100 + language: en + ttl: 360 + time_zone: UTC +request: + botasaurus: + wait_for_selector: ".table-wrapper table tbody tr a" + wait_timeout_seconds: 20 +selectors: + items: + selector: ".table-wrapper table tbody > tr:not(:first-child)" + enhance: false + title: + selector: a + url: + selector: a + extractor: href + description: + selector: "td:nth-child(2)" + published_at: + selector: "td:nth-child(3)" + post_process: + - name: parse_time diff --git a/spec/fixtures/registries/sync/bundle/manifest.json b/spec/fixtures/registries/sync/bundle/manifest.json new file mode 100644 index 000000000..e74570c9a --- /dev/null +++ b/spec/fixtures/registries/sync/bundle/manifest.json @@ -0,0 +1 @@ +{"format":"registry.v1","registry_id":"official","version":"test-fixture","public_key_id":"test-key","files":{"configs/phys.org/weekly.yml":"0f2a052f5c6da5c260985058302805b962da3211b635483667e258c714b31373","configs/support.apple.com/en_gb_ht201222.yml":"3a6f84899ac24472f6476c697d50ac366425ae1c87f7ddf045cc9e2c80e93eac"}} \ No newline at end of file diff --git a/spec/fixtures/registries/sync/registries.yml b/spec/fixtures/registries/sync/registries.yml new file mode 100644 index 000000000..e307db29a --- /dev/null +++ b/spec/fixtures/registries/sync/registries.yml @@ -0,0 +1,13 @@ +precedence: + - official + +registries: + official: + sync: + url: https://registry.test.example/registry-bundle.tar.gz + catalog: true + public_key_id: test-key + public_key: | + -----BEGIN PUBLIC KEY----- + MCowBQYDK2VwAyEAiMbg/04MyC5azBdM/aeY0mNuA8JbP5/jOiNRwJ2KJHE= + -----END PUBLIC KEY----- diff --git a/spec/html2rss/web/api/v1_spec.rb b/spec/html2rss/web/api/v1_spec.rb index 4d505cebf..5eb375b23 100644 --- a/spec/html2rss/web/api/v1_spec.rb +++ b/spec/html2rss/web/api/v1_spec.rb @@ -196,6 +196,21 @@ def relative_feed_link_header(token) ) end + it 'returns registry status metadata', :aggregate_failures do + get '/api/v1' + + expect(last_response.status).to eq(200) + json = expect_success_response(last_response) + official = json.dig('data', 'instance', 'registries').find { |row| row['id'] == 'official' } + + expect(official).to include( + 'id' => 'official', + 'version' => 'test-fixture', + 'sync_mode' => 'path' + ) + expect(official['updated_at']).to be_a(String) + end + it 'returns API information with trailing slash', :aggregate_failures do get '/api/v1/' diff --git a/spec/html2rss/web/boot/setup_spec.rb b/spec/html2rss/web/boot/setup_spec.rb index 116b60811..d98558843 100644 --- a/spec/html2rss/web/boot/setup_spec.rb +++ b/spec/html2rss/web/boot/setup_spec.rb @@ -21,6 +21,7 @@ before do allow(Html2rss::Web::Flags).to receive(:validate!) allow(Html2rss::Web::Boot::Sentry).to receive(:configure!) + allow(Html2rss::Web::Registry::Sync).to receive(:boot!) end describe '.call!' do @@ -33,6 +34,7 @@ expect(Html2rss::Web::EnvironmentValidator).to have_received(:validate_environment!).once expect(Html2rss::Web::EnvironmentValidator).to have_received(:validate_production_security!).once expect(Html2rss::Web::Flags).to have_received(:validate!).once + expect(Html2rss::Web::Registry::Sync).to have_received(:boot!).once end it 'routes rack-timeout logs through the shared app logger' do diff --git a/spec/html2rss/web/registry/sync_spec.rb b/spec/html2rss/web/registry/sync_spec.rb index 8c7703475..4ff1b8877 100644 --- a/spec/html2rss/web/registry/sync_spec.rb +++ b/spec/html2rss/web/registry/sync_spec.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'spec_helper' +require 'webmock/rspec' require_relative '../../../../app' @@ -17,10 +18,102 @@ end end + describe '.run', :registry_sync do + let(:download_url) { 'https://registry.test.example/registry-bundle.tar.gz' } + let(:tarball) { RegistrySyncTestHelpers.build_signed_tarball } + + before do + stub_request(:get, download_url) + .to_return(status: 200, body: tarball, headers: { 'Content-Type' => 'application/octet-stream' }) + end + + it 'fetches, verifies, and stores a signed bundle', :aggregate_failures do + status = described_class.run(registry_id: 'official') + + expect(status.version).to eq('test-fixture') + expect(Html2rss::Web::Registry::Store.bundle_present?('official')).to be(true) + expect(Html2rss::Web::Registry::Index.current.config_for('phys.org/weekly')).to include( + channel: hash_including(title: 'Phys.org — Weekly') + ) + end + + it 'supports dry-run verification without swapping the active bundle' do + expect do + described_class.run(registry_id: 'official', dry_run: true) + end.not_to(change { Html2rss::Web::Registry::Store.bundle_present?('official') }) + end + + it 'rejects redirects' do + stub_request(:get, download_url) + .to_return(status: 302, headers: { 'Location' => 'https://evil.example/bundle.tar.gz' }) + + expect { described_class.run(registry_id: 'official') } + .to raise_error(Html2rss::Web::Registry::Errors::SyncError, /redirect/i) + end + + it 'logs signature verification failures to the security logger' do + allow(Html2rss::Web::SecurityLogger).to receive(:log_registry_signature_failure) + stub_request(:get, download_url).to_return(status: 200, body: 'not-a-tarball') + + expect { described_class.run(registry_id: 'official') } + .to raise_error(Html2rss::Web::Registry::Errors::SyncError) + + expect(Html2rss::Web::SecurityLogger).not_to have_received(:log_registry_signature_failure) + end + + it 'keeps the previous bundle when sync fails' do + described_class.run(registry_id: 'official') + stub_request(:get, download_url).to_return(status: 500, body: 'fail') + + expect { described_class.run(registry_id: 'official') } + .to raise_error(Html2rss::Web::Registry::Errors::SyncError, /HTTP 500/) + + expect(Html2rss::Web::Registry::Store.bundle_present?('official')).to be(true) + end + end + describe '.run' do it 'rejects path-mode registries' do expect { described_class.run(registry_id: 'official') } .to raise_error(Html2rss::Web::Registry::Errors::SyncError, /path mode/) end end + + describe '.status', :registry_sync do + it 'includes sync metadata and last error state' do + row = described_class.status(registry_id: 'official').first + + expect(row).to have_attributes( + registry_id: 'official', + mode: :sync, + sync_url: 'https://registry.test.example/registry-bundle.tar.gz' + ) + end + end + + describe '.cli_exit_code', :registry_sync do + it 'returns non-zero when a sync registry has no bundle' do + expect(described_class.cli_exit_code).to eq(1) + end + + it 'returns zero after a successful sync' do + stub_request(:get, 'https://registry.test.example/registry-bundle.tar.gz') + .to_return(status: 200, body: RegistrySyncTestHelpers.build_signed_tarball) + + described_class.run(registry_id: 'official') + + expect(described_class.cli_exit_code).to eq(0) + end + end + + describe '.boot!' do + it 'does not run in the test environment' do + allow(described_class).to receive(:seed_registry!) + allow(described_class).to receive(:start_background_timer!) + + described_class.boot! + + expect(described_class).not_to have_received(:seed_registry!) + end + end end diff --git a/spec/support/registry_sync.rb b/spec/support/registry_sync.rb new file mode 100644 index 000000000..b0f6a1dca --- /dev/null +++ b/spec/support/registry_sync.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +require 'openssl' +require 'fileutils' +require 'zlib' +require 'stringio' +require 'rubygems/package' + +module RegistrySyncTestHelpers + HTML2RSS_FIXTURES = File.expand_path('../../../html2rss/spec/fixtures/registry', __dir__) + TEST_PUBLIC_KEY = File.read(File.join(HTML2RSS_FIXTURES, 'test-key.pub')) + TEST_PRIVATE_KEY = File.read(File.join(HTML2RSS_FIXTURES, 'test-key.pem')) + SYNC_FIXTURES_ROOT = File.expand_path('../fixtures/registries/sync', __dir__) + SYNC_REGISTRIES_CONFIG = File.join(SYNC_FIXTURES_ROOT, 'registries.yml') + SYNC_DATA_ROOT = File.join(Dir.pwd, 'tmp', 'sync-registry-data') + + module_function + + def configure_sync_registry! + ENV['REGISTRIES_CONFIG'] = SYNC_REGISTRIES_CONFIG + ENV['REGISTRY_DATA_ROOT'] = SYNC_DATA_ROOT + ENV['REGISTRY_SYNC_ALLOWED_HOSTS'] = 'registry.test.example' + FileUtils.rm_rf(SYNC_DATA_ROOT) + Html2rss::Web::Registry::Index.reload! + end + + def build_signed_tarball # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + bundle_dir = Dir.mktmpdir('signed-registry-bundle') + FileUtils.cp_r(File.join(SYNC_FIXTURES_ROOT, 'bundle', '.'), bundle_dir) + manifest = Html2rss::Registry::Manifest.parse( + File.read(File.join(bundle_dir, Html2rss::Registry::Manifest::MANIFEST_FILE)) + ) + private_key = OpenSSL::PKey.read(TEST_PRIVATE_KEY) + signature = private_key.sign(nil, manifest.canonical_bytes) + File.write( + File.join(bundle_dir, Html2rss::Registry::Manifest::SIGNATURE_FILE), + [signature].pack('m0') + ) + + tar_io = StringIO.new + Gem::Package::TarWriter.new(tar_io) do |tar| + Dir.glob(File.join(bundle_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |path| + next if File.directory?(path) + + relative = path.delete_prefix("#{bundle_dir}/") + tar.add_file(relative, 0o644) { |io| io.write(File.binread(path)) } + end + end + + gz_io = StringIO.new + Zlib::GzipWriter.wrap(gz_io) { |gz| gz.write(tar_io.string) } + gz_io.string + ensure + FileUtils.rm_rf(bundle_dir) + end +end + +RSpec.configure do |config| + config.before do |example| + next unless example.metadata[:registry_sync] + + RegistrySyncTestHelpers.configure_sync_registry! + end +end From 5725ba0b528ea002f35f5ec1114f4579c7ecf39c Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 14:57:20 +0200 Subject: [PATCH 03/33] docs: registry sync runbook and catalog API ownership --- AGENTS.md | 11 ++++--- docs/README.md | 88 +++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 94 insertions(+), 5 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 07b31e815..80d1bdb08 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -56,19 +56,22 @@ See [docs/design-system.md](docs/design-system.md) for visual rules. ## Config catalog API -Public feed-directory metadata for embedded and local configs. +Public feed-directory metadata from verified registry bundles and local `feeds.yml` entries. | Item | Detail | | --- | --- | | Endpoint | `GET /api/v1/configs` | | Flag | `CONFIG_CATALOG_ENABLED` (default `true`; set `false` to disable) | | Disabled response | `404` with `{ "error": "catalog_disabled" }` | -| Embedded entries | `Html2rss::Configs::Catalog.entries` — do not re-walk YAML in the handler | -| Local entries | `Catalog::Merge` includes `feeds.yml` feeds only when `directory.title` is set | +| Registry entries | `Registry::Index.current.catalog_rows` — loads signed bundles from `config/registries.yml`; adds `source: registry`, `registry: ` | +| Local entries | `Registry::LocalCatalog` includes `feeds.yml` feeds only when `directory.title` is set (`source: local`) | +| Per-registry privacy | `catalog: false` in `registries.yml` omits that registry from the API (feeds still served) | | Starter feeds (UI) | Frontend `selectStarterFeeds` when feed creation is disabled; catalog find uses full catalog when enabled | | Catalog find | `findCatalogEntries` → multi-hit list under create URL; links via `catalogFeedHref` (path + defaults) | | CORS | Route-scoped on `/api/v1/configs` only (`GET`, `OPTIONS`) | -| Root metadata | `GET /api/v1/` exposes `instance.catalog: { enabled, url }` | +| Root metadata | `GET /api/v1/` exposes `instance.catalog: { enabled, url }` and `instance.registries` sync status | | Contract SSOT | Request specs under `spec/html2rss/web/api/v1_spec.rb` and generated `public/openapi.yaml` | +Registry sync: `bin/registry-sync --status`; boot seed + optional sync via `Registry::Sync.boot!`. See [docs/README.md](docs/README.md#registry-sync-runbook). + After handler or envelope changes: `make openapi` and `make ci-ready`. diff --git a/docs/README.md b/docs/README.md index 480eb526b..0cd53fcb1 100644 --- a/docs/README.md +++ b/docs/README.md @@ -26,7 +26,7 @@ Welcome! This is the canonical source of truth for contributing to `html2rss-web - **Runtime behavior**: Application code plus tests. - **HTTP contract**: Request specs plus generated OpenAPI. -- **Config catalog API**: `GET /api/v1/configs` — embedded data from `Html2rss::Configs::Catalog`, merged with local `feeds.yml` entries that include `directory.title`. Disabled when `CONFIG_CATALOG_ENABLED=false` (`404`, `catalog_disabled`). CORS is enabled on this route only. +- **Config catalog API**: `GET /api/v1/configs` — catalog rows from `Registry::Index` (verified registry bundles per `config/registries.yml`, merged with local `feeds.yml` entries that include `directory.title`). Disabled when `CONFIG_CATALOG_ENABLED=false` (`404`, `catalog_disabled`). CORS is enabled on this route only. - **This file**: Contributor conventions and current project rules. --- @@ -284,6 +284,92 @@ Tune alert thresholds from sustained `request.error` or `feed.render` failure sp --- +## Registry sync runbook + +Signed feed registries replace the embedded `html2rss-configs` gem. Each registry is defined in `config/registries.yml` (override path with `REGISTRIES_CONFIG`). + +### Check sync status + +Inside the Dev Container: + +```bash +bin/registry-sync --status +``` + +Columns: `registry`, `mode`, `version`, `updated_at`, `sync_url`, `last_error`. Exit code is non-zero when any registry reports `last_error`. + +Sync a single registry or dry-run without swapping the active bundle: + +```bash +bin/registry-sync --registry official +bin/registry-sync --registry official --dry-run +``` + +### Boot behavior + +`Registry::Sync.boot!` runs during app boot (see `app/web/boot/setup.rb`): + +1. **Seed** — copies the image-embedded official bundle when no on-disk bundle exists. +2. **Sync on boot** — when `REGISTRY_SYNC_ON_BOOT=true`, or when the bundle is missing, fetches and verifies the latest release. +3. **Background refresh** — when `REGISTRY_SYNC_INTERVAL_HOURS` is greater than zero (default `24`), re-syncs on a jittered timer. Set to `0` to disable. + +Network sync verifies Ed25519 signatures using the `public_key` pinned in `registries.yml`. Seed bundles from the Docker image use integrity-only verification. + +### Add a corporate registry + +```yaml +precedence: + - official + - corp + +registries: + official: + sync: + channel: html2rss-official + catalog: true + public_key_id: html2rss:registry:2026 + public_key: | + -----BEGIN PUBLIC KEY----- + ... + -----END PUBLIC KEY----- + + corp: + sync: + url: https://registry.example.com/registry-bundle.tar.gz + catalog: false # feeds served; omitted from GET /api/v1/configs + public_key_id: corp:registry:2026 + public_key: | + -----BEGIN PUBLIC KEY----- + ... + -----END PUBLIC KEY----- +``` + +- **`precedence`** — merge order for feed lookup; first match wins. +- **`sync.url`** — direct tarball URL, or use `sync.channel: html2rss-official` for the default GitHub release asset. +- **`catalog: false`** — private registry: configs are served at `/{registry.id}.rss` but excluded from the public catalog API (privacy for internal feeds). +- Restrict outbound hosts with `REGISTRY_SYNC_ALLOWED_HOSTS` (comma-separated hostnames). + +### Air-gapped / offline path mount + +For environments without outbound network access, mount a verified bundle directory: + +```yaml +registries: + official: + path: /opt/html2rss/registry/official + catalog: true +``` + +The directory must contain `manifest.json`, optional `manifest.sig`, and `configs/`. Path mode skips network sync; run `bin/registry-sync --status` to confirm `mode` is `path`. + +### Key rotation + +1. Publish a new bundle signed with the new key (`public_key_id` in `manifest.json`). +2. Update `public_key_id` and `public_key` in `registries.yml` on every instance **before** or together with the first release that requires the new key. +3. Re-sync (`bin/registry-sync` or wait for background refresh). Failed verification leaves the previous bundle active and records `last_error`. + +--- + ## Documentation Policy - Prefer deleting stale docs over archiving them in-place. From a440551dc13f502203f97157c59fd9808536ff4f Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 15:21:32 +0200 Subject: [PATCH 04/33] docs: add registry go-live manual --- docs/README.md | 2 + docs/registry-go-live.md | 486 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 488 insertions(+) create mode 100644 docs/registry-go-live.md diff --git a/docs/README.md b/docs/README.md index 0cd53fcb1..0fcf1b7f2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -286,6 +286,8 @@ Tune alert thresholds from sustained `request.error` or `feed.render` failure sp ## Registry sync runbook +For end-to-end release and deployment steps (maintainers and operators), see [registry-go-live.md](registry-go-live.md). + Signed feed registries replace the embedded `html2rss-configs` gem. Each registry is defined in `config/registries.yml` (override path with `REGISTRIES_CONFIG`). ### Check sync status diff --git a/docs/registry-go-live.md b/docs/registry-go-live.md new file mode 100644 index 000000000..c79a907d4 --- /dev/null +++ b/docs/registry-go-live.md @@ -0,0 +1,486 @@ +# Registry go-live manual + +Step-by-step guide to ship and operate signed `registry.v1` bundles across the html2rss org repos. + +**Related docs** + +- Bundle format: [`html2rss/docs/registry-v1.md`](../../html2rss/docs/registry-v1.md) +- Operator runbook (sync flags, custom registries): [README.md — Registry sync runbook](./README.md#registry-sync-runbook) + +--- + +## 1. Maintainers — merge and release order + +Work in this order when a change touches registry contracts or feed configs. + +### 1.1 `html2rss` (core gem) + +Source of truth for `registry.v1` schema, `Html2rss::Registry::Verifier`, `CatalogBuilder`, and archive limits. + +1. Merge registry-related changes to `main`. +2. Regenerate schema if config schema changed: + ```bash + cd html2rss + mise exec -- make ready + ``` +3. Tag/release the gem when the contract is stable (web depends on this). + +### 1.2 `html2rss-configs` (signed bundle publisher) + +Source of truth for feed YAML and signed release artifacts. + +1. Merge config changes to `master`. +2. Run the configs quality gate: + ```bash + cd html2rss-configs + make ready + ``` +3. Tag the release (tag name becomes `manifest.json` `version` in CI via `REGISTRY_VERSION: ${{ github.ref_name }}`): + ```bash + git tag 2026.08.22 # example; use your release tag + git push origin 2026.08.22 + ``` +4. Tag push triggers [`.github/workflows/release.yml`](../../html2rss-configs/.github/workflows/release.yml): + - `make registry-build -- --sign` + - uploads `dist/registry-bundle.tar.gz` to the GitHub Release + +### 1.3 `html2rss-web` (runtime + Docker image) + +Consumes verified bundles; ships an unsigned seed copy in the image. + +1. Bump the `html2rss` gem dependency if the core contract changed. +2. Update `config/registries.yml` when the signing key or sync channel changes (see sections 2 and 5). +3. Build the image with a fresh seed (section 3). +4. Merge to `main`, wait for CI (`ci` workflow) to pass. +5. Publish a GitHub Release on `html2rss-web` — [`.github/workflows/release.yml`](../../html2rss-web/.github/workflows/release.yml) builds and pushes `html2rss/web` tags (`latest`, semver, major, commit SHA). + +**Rule of thumb:** core contract → configs signed release → web image that embeds/verifies that release. + +--- + +## 2. First signed release — signing key, tag, verify artifact + +### 2.1 Generate an Ed25519 key pair + +Use OpenSSL (same algorithm as `tool/registry-build` and `Html2rss::Registry::Verifier`): + +```bash +openssl genpkey -algorithm ED25519 -out registry-signing.pem +openssl pkey -in registry-signing.pem -pubout -out registry-signing.pub +``` + +Keep the private key offline except where signing happens. + +### 2.2 Configure `html2rss-configs` CI secret + +1. In the `html2rss-configs` GitHub repo, add secret **`REGISTRY_SIGNING_KEY`** with the full PEM contents of `registry-signing.pem`. +2. Release CI reads it here: + + ```yaml + env: + REGISTRY_VERSION: ${{ github.ref_name }} + REGISTRY_SIGNING_KEY: ${{ secrets.REGISTRY_SIGNING_KEY }} + run: make registry-build -- --sign + ``` + +### 2.3 Pin the public key in `html2rss-web` + +Add the public key to `config/registries.yml` before instances need to sync signed bundles over the network: + +```yaml +registries: + official: + sync: + channel: html2rss-official + catalog: true + public_key_id: html2rss:registry:2026 + public_key: | + -----BEGIN PUBLIC KEY----- + ... + -----END PUBLIC KEY----- +``` + +- `public_key_id` must match the value in signed `manifest.json` (default in `tool/registry-build`: `html2rss:registry:2026`). +- Network sync uses `:signed` trust and requires this pin. Seed bundles copied from the Docker image use `:integrity_only` trust (no signature check on load). + +### 2.4 Tag and publish the configs release + +```bash +cd html2rss-configs +git tag # e.g. 2026.08.22 +git push origin +``` + +Wait for the Release workflow to finish. The asset name is always **`registry-bundle.tar.gz`**. + +### 2.5 Verify the release artifact + +Download the asset from the GitHub Release, then verify locally: + +```bash +mkdir -p /tmp/registry-verify && tar -xzf registry-bundle.tar.gz -C /tmp/registry-verify + +# Inspect manifest +jq . /tmp/registry-verify/manifest.json +# Confirm: format=registry.v1, registry_id=official, version=, public_key_id matches pin + +# Confirm signature file exists +test -f /tmp/registry-verify/manifest.sig + +# Optional: verify with core gem (from html2rss checkout) +cd html2rss +mise exec -- bundle exec ruby -rhtml2rss -ropenssl -e " + pk = OpenSSL::PKey.read(File.read('path/to/registry-signing.pub')) + Html2rss::Registry::Verifier.verify!( + '/tmp/registry-verify', + trust: :signed, + public_keys: { 'html2rss:registry:2026' => pk } + ) + puts 'OK' +" +``` + +Unsigned local builds (no `--sign`) are valid for seed/integrity-only use only: + +```bash +cd html2rss-configs +make registry-build # writes dist/registry-bundle.tar.gz without manifest.sig +``` + +--- + +## 3. Web image build — seed preparation and Docker + +### 3.1 How seed gets into the image + +| Step | What happens | +| --- | --- | +| `bin/prepare-registry-seed` | Builds an **unsigned** bundle from `html2rss-configs` and extracts it to `app/registries/seed/official/` | +| `bin/docker-build` | Runs `prepare-registry-seed`, then `docker build --no-cache -t html2rss/web` | +| `Dockerfile` | `COPY app/registries/seed ./app/registries/seed` (image path `/app/registries/seed`) | +| Boot | `Registry::Sync.boot!` copies seed → `REGISTRY_DATA_ROOT/` when no on-disk bundle exists | + +`prepare-registry-seed` details: + +- Configs source: `HTML2RSS_CONFIGS_ROOT` (default: sibling `../html2rss-configs`) +- Runs `tool/registry-build` **without** `--sign` (integrity-only seed) +- Output directory: `app/registries/seed/official/` + +### 3.2 Build locally + +From a workspace with both repos checked out as siblings: + +```bash +cd html2rss-web +bin/docker-build +``` + +Or prepare seed only: + +```bash +cd html2rss-web +HTML2RSS_CONFIGS_ROOT=/path/to/html2rss-configs bin/prepare-registry-seed +docker build --no-cache -t html2rss/web -f Dockerfile . +``` + +Set build metadata for production parity: + +```bash +docker build \ + --build-arg BUILD_TAG=1.2.3 \ + --build-arg GIT_SHA="$(git rev-parse HEAD)" \ + -t html2rss/web \ + -f Dockerfile . +``` + +### 3.3 What operators get in the image + +- **`/app/registries/seed/official/`** — unsigned bundle baked at build time (offline-first bootstrap). +- **`/app/config/registries.yml`** — default official registry with `sync.channel: html2rss-official`. +- **`/app/data/registries`** — empty at build; populated at runtime (seed copy + network sync). + +--- + +## 4. Operators — default path (official registry) + +### 4.1 Pull a new image (simplest update path) + +`docker-compose.yml` defaults: + +- Image: `html2rss/web` +- Volume: `registry-data:/app/data/registries` (`REGISTRY_DATA_ROOT`) +- Watchtower optional: checks for new images every 7200s + +```bash +docker compose pull html2rss-web +docker compose up -d html2rss-web +``` + +A new image updates the **seed** inside the container. Existing data in the volume is kept until sync replaces it. + +### 4.2 Zero-config first boot + +With the stock `config/registries.yml`, no extra registry env vars are required. + +On boot (`Registry::Sync.boot!`): + +1. **Seed** — if `REGISTRY_DATA_ROOT/official/` is empty, copy from `/app/registries/seed/official/`. +2. **Sync on boot** — runs when `REGISTRY_SYNC_ON_BOOT=true` **or** when no bundle is present on disk (after seed attempt). +3. **Background refresh** — when `REGISTRY_SYNC_INTERVAL_HOURS` > 0 (default **24**), re-sync on a jittered timer. Set to `0` to disable. + +Official sync URL (from `config/registries.yml` + `Registry::Config`): + +`https://github.com/html2rss/html2rss-configs/releases/latest/download/registry-bundle.tar.gz` + +Allowed outbound hosts (built-in): `api.github.com`, `github.com`, `objects.githubusercontent.com`. + +### 4.3 Existing instance — volume retained + +The named volume preserves synced bundles across container restarts and image upgrades. After pull + restart: + +- Old bundle stays active until a successful sync swaps it. +- Use `bin/registry-sync` or wait for background refresh to pick up a new configs release without rebuilding the image (section 7). + +--- + +## 5. Operators — custom corporate registry + +Override or extend `config/registries.yml` (or set `REGISTRIES_CONFIG` to an alternate file path). + +Example from the [registry sync runbook](./README.md#add-a-corporate-registry): + +```yaml +precedence: + - official + - corp + +registries: + official: + sync: + channel: html2rss-official + catalog: true + public_key_id: html2rss:registry:2026 + public_key: | + -----BEGIN PUBLIC KEY----- + ... + -----END PUBLIC KEY----- + + corp: + sync: + url: https://registry.example.com/registry-bundle.tar.gz + catalog: false + public_key_id: corp:registry:2026 + public_key: | + -----BEGIN PUBLIC KEY----- + ... + -----END PUBLIC KEY----- +``` + +| Field | Purpose | +| --- | --- | +| `precedence` | Feed lookup merge order; first match wins | +| `sync.url` | Direct HTTPS tarball URL for network sync | +| `sync.channel: html2rss-official` | Resolves to the official GitHub release asset | +| `catalog: false` | Feeds served at `/{feed_id}.rss`; **omitted** from `GET /api/v1/configs` | +| `public_key` / `public_key_id` | Required for `:signed` network sync verification | + +For hosts outside the default GitHub allowlist: + +```bash +REGISTRY_SYNC_ALLOWED_HOSTS=registry.example.com,cdn.example.com +``` + +Mount a custom registries file in Compose: + +```yaml +volumes: + - ./config/registries.yml:/app/config/registries.yml:ro +``` + +--- + +## 6. Verify live + +Run these checks after deploy or sync. + +### 6.1 Registry sync status (CLI) + +Inside the running container (or Dev Container): + +```bash +bin/registry-sync --status +``` + +Tab-separated columns: `registry`, `mode`, `version`, `updated_at`, `sync_url`, `last_error`. + +- Exit code **0** — all sync-mode registries have a usable on-disk bundle. +- Exit code **1** — at least one sync registry lacks a bundle (see `Registry::Sync.unusable_sync_registries`). + +Single registry or dry-run: + +```bash +bin/registry-sync --registry official +bin/registry-sync --registry official --dry-run +``` + +### 6.2 Instance metadata API + +```bash +curl -sS http://127.0.0.1:4000/api/v1/ | jq '.data.instance' +``` + +Confirm: + +- **`instance.registries`** — array with `id`, `version`, `updated_at`, `sync_mode` per configured registry +- **`instance.catalog`** — `{ "enabled": true, "url": ".../api/v1/configs" }` (unless `CONFIG_CATALOG_ENABLED=false`) + +### 6.3 Catalog API + +```bash +curl -sS http://127.0.0.1:4000/api/v1/configs | jq '.data.configs[0]' +``` + +Expect rows with `source: "registry"`, `registry: "official"`, and a `path` like `/anthropic.com/news.rss`. + +When `CONFIG_CATALOG_ENABLED=false`, expect `404` with `{ "error": "catalog_disabled" }`. + +### 6.4 Sample static feed + +Pick a feed id from the catalog (`id` field) or from `configs/.yml` in the bundle. Request: + +```bash +curl -sS -o /dev/null -w '%{http_code}\n' http://127.0.0.1:4000/anthropic.com/news.rss +``` + +Expect HTTP **200** and valid RSS XML. + +--- + +## 7. Update configs without pulling a new Docker image + +Configs releases are independent of web image releases. To refresh feeds on a running instance: + +### 7.1 Manual sync + +```bash +bin/registry-sync --registry official +``` + +Fetches the latest signed tarball, verifies signature + digests, atomically swaps the active bundle, and reloads `Registry::Index`. + +### 7.2 Automatic refresh + +| Mechanism | Env var | Default | Behavior | +| --- | --- | --- | --- | +| Sync on boot | `REGISTRY_SYNC_ON_BOOT` | `false` | When `true`, network sync on every boot (in addition to missing-bundle sync) | +| Background timer | `REGISTRY_SYNC_INTERVAL_HOURS` | `24` | Jittered periodic re-sync; `0` disables | +| Missing bundle | — | — | Always syncs on boot when no on-disk bundle exists | + +Example — force sync every boot (use sparingly): + +```bash +REGISTRY_SYNC_ON_BOOT=true +``` + +Example — disable background refresh: + +```bash +REGISTRY_SYNC_INTERVAL_HOURS=0 +``` + +**Note:** Pulling a new web image updates the embedded seed but does **not** replace an existing volume bundle until sync succeeds. + +--- + +## 8. Troubleshooting + +### 8.1 Signature verification failure + +Symptoms in `bin/registry-sync --status`: + +- `last_error` contains `Unknown public_key_id`, `Invalid manifest signature`, or `Missing manifest.sig` + +Checks: + +1. `public_key_id` and `public_key` in `registries.yml` match the signed release. +2. Deploy web config **before** or **with** the first bundle signed by a new key (key rotation). +3. Downloaded asset is `registry-bundle.tar.gz` from the expected release (not an unsigned local build). + +Signature-related failures are logged via `SecurityLogger.log_registry_signature_failure`. + +Verify without swapping the active bundle: + +```bash +bin/registry-sync --registry official --dry-run +``` + +### 8.2 Sync failure keeps the old bundle + +By design: + +- `Registry::Sync.run` only calls `Store.swap!` after fetch **and** verification succeed. +- On failure, `last_error` is recorded; the previous bundle under `REGISTRY_DATA_ROOT//` remains served. +- `Store.promote_bundle!` rolls back on swap failure. + +If sync fails on first boot with no prior bundle, the instance may have only the unsigned seed (integrity-only) until a signed sync succeeds. + +### 8.3 Network / host errors + +| Error pattern | Likely cause | +| --- | --- | +| `Registry sync host not allowed` | Add host to `REGISTRY_SYNC_ALLOWED_HOSTS` | +| `Registry sync rejects HTTP redirects` | Publish a direct HTTPS asset URL (`sync.url`) | +| `Registry sync fetch failed with HTTP …` | Release missing, URL wrong, or GitHub outage | +| `Registry sync requires HTTPS URLs` | Use `https://` in `sync.url` | + +Default allowed hosts cover official GitHub release downloads only. + +### 8.4 Air-gapped / offline (`path` mode) + +For environments without outbound sync, mount a verified bundle directory and skip network sync: + +```yaml +registries: + official: + path: /opt/html2rss/registry/official + catalog: true +``` + +Requirements: + +- Directory contains `manifest.json`, `configs/`, and optionally `manifest.sig` +- `bin/registry-sync` is not applicable (`path mode; sync is not applicable`) +- `bin/registry-sync --status` should show `mode: path` + +Load path uses `:integrity_only` trust (disk/image trust boundary). + +In Docker Compose, bind-mount the bundle and optionally set `REGISTRIES_CONFIG`: + +```yaml +volumes: + - /opt/html2rss/registry/official:/opt/html2rss/registry/official:ro +environment: + REGISTRIES_CONFIG: /app/config/registries.yml +``` + +### 8.5 CLI exit code non-zero with empty `last_error` + +`bin/registry-sync --status` exits **1** when a sync-mode registry has **no on-disk bundle**. Run a sync or confirm seed copy succeeded: + +```bash +bin/registry-sync --registry official +bin/registry-sync --status +``` + +### 8.6 Seed missing or stale in image + +If `app/registries/seed/official/` was not populated before `docker build`, first boot may have nothing to seed. + +Fix for maintainers: + +```bash +cd html2rss-web +bin/prepare-registry-seed # or bin/docker-build +``` + +Ensure `HTML2RSS_CONFIGS_ROOT` points at the configs checkout used for the release. From 22da0dacda98c5a8f5ab36715160c17f19f9d757 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 15:23:58 +0200 Subject: [PATCH 05/33] fix(deps): pin html2rss from GitHub for CI Replace sibling path pin with feat/registry-v1 branch so CI can bundle install without a checked-out html2rss repo. --- Gemfile | 4 ++-- Gemfile.lock | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Gemfile b/Gemfile index 25acc1cc7..ae3e430ef 100644 --- a/Gemfile +++ b/Gemfile @@ -4,9 +4,9 @@ source 'https://rubygems.org' git_source(:github) { |repo_name| "https://github.com/#{repo_name}" } -gem 'html2rss', path: '../html2rss' +# gem 'html2rss', path: '../html2rss' # gem 'html2rss', '~> 0.27' -# gem 'html2rss', github: 'html2rss/html2rss', branch: 'master' +gem 'html2rss', github: 'html2rss/html2rss', branch: 'feat/registry-v1' gem 'base64' gem 'rack-cache' diff --git a/Gemfile.lock b/Gemfile.lock index c480a1da2..9f7e77533 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,5 +1,7 @@ -PATH - remote: ../html2rss +GIT + remote: https://github.com/html2rss/html2rss + revision: e32014ac089679cac7e91fd55f3bd85a0489771e + branch: feat/registry-v1 specs: html2rss (0.28.0) addressable (~> 2.7) From c61e567d4995c4d5e9533b597f7f2e25902138e6 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 16:22:05 +0200 Subject: [PATCH 06/33] feat(registry): sync governance with staged promote and trust context Add sync policy, staging/promote flow, trust context verification, scrape policy guards, and seed bundle fixtures with updated registry sync specs. --- Gemfile | 3 +- app/registries/seed/official/.gitkeep | 0 .../seed/official/configs/phys.org/weekly.yml | 25 ++ .../support.apple.com/en_gb_ht201222.yml | 34 ++ app/registries/seed/official/manifest.json | 10 + app/web/registry/catalog_change_reporter.rb | 96 ++++++ app/web/registry/config.rb | 135 ++++++-- app/web/registry/index.rb | 62 ++-- app/web/registry/local_catalog.rb | 10 +- app/web/registry/local_catalog_row.rb | 18 + app/web/registry/registry_catalog_row.rb | 35 ++ app/web/registry/scrape_policy.rb | 58 ++++ app/web/registry/store.rb | 99 +++++- app/web/registry/sync.rb | 318 ++++++++---------- app/web/registry/sync_coordinator.rb | 23 ++ app/web/registry/sync_fetcher.rb | 150 +++++++++ app/web/registry/sync_url_resolver.rb | 80 +++++ app/web/registry/trust_context.rb | 54 +++ app/web/registry/version_gate.rb | 40 +++ app/web/security/security_logger.rb | 7 + bin/registry-sync | 13 +- config/registries.yml | 11 + docs/README.md | 17 +- docs/registry-go-live.md | 76 ++++- frontend/src/api/generated/types.gen.ts | 39 +-- public/openapi.yaml | 1 - spec/fixtures/registries/keys/test-key.pem | 3 + spec/fixtures/registries/keys/test-key.pub | 3 + .../official/configs/phys.org/weekly.yml | 2 + .../support.apple.com/en_gb_ht201222.yml | 2 + .../registries/official/manifest.json | 4 +- .../configs/secret.example/private.yml | 2 + .../fixtures/registries/private/manifest.json | 4 +- .../sync/bundle/configs/phys.org/weekly.yml | 2 + .../support.apple.com/en_gb_ht201222.yml | 2 + .../registries/sync/bundle/manifest.json | 11 +- spec/fixtures/registries/sync/registries.yml | 1 + spec/html2rss/web/registry/config_spec.rb | 93 +++++ spec/html2rss/web/registry/index_spec.rb | 37 ++ .../web/registry/scrape_policy_spec.rb | 56 +++ spec/html2rss/web/registry/store_spec.rb | 80 +++++ spec/html2rss/web/registry/sync_spec.rb | 106 +++++- .../web/registry/sync_url_resolver_spec.rb | 39 +++ .../web/registry/version_gate_spec.rb | 20 ++ spec/smoke/docker_spec.rb | 9 + spec/spec_helper.rb | 14 +- spec/support/openapi.rb | 6 + spec/support/registry_sync.rb | 69 ++-- 48 files changed, 1648 insertions(+), 331 deletions(-) delete mode 100644 app/registries/seed/official/.gitkeep create mode 100644 app/registries/seed/official/configs/phys.org/weekly.yml create mode 100644 app/registries/seed/official/configs/support.apple.com/en_gb_ht201222.yml create mode 100644 app/registries/seed/official/manifest.json create mode 100644 app/web/registry/catalog_change_reporter.rb create mode 100644 app/web/registry/local_catalog_row.rb create mode 100644 app/web/registry/registry_catalog_row.rb create mode 100644 app/web/registry/scrape_policy.rb create mode 100644 app/web/registry/sync_coordinator.rb create mode 100644 app/web/registry/sync_fetcher.rb create mode 100644 app/web/registry/sync_url_resolver.rb create mode 100644 app/web/registry/trust_context.rb create mode 100644 app/web/registry/version_gate.rb create mode 100644 spec/fixtures/registries/keys/test-key.pem create mode 100644 spec/fixtures/registries/keys/test-key.pub create mode 100644 spec/html2rss/web/registry/config_spec.rb create mode 100644 spec/html2rss/web/registry/scrape_policy_spec.rb create mode 100644 spec/html2rss/web/registry/store_spec.rb create mode 100644 spec/html2rss/web/registry/sync_url_resolver_spec.rb create mode 100644 spec/html2rss/web/registry/version_gate_spec.rb diff --git a/Gemfile b/Gemfile index ae3e430ef..8ff642c87 100644 --- a/Gemfile +++ b/Gemfile @@ -4,9 +4,8 @@ source 'https://rubygems.org' git_source(:github) { |repo_name| "https://github.com/#{repo_name}" } -# gem 'html2rss', path: '../html2rss' -# gem 'html2rss', '~> 0.27' gem 'html2rss', github: 'html2rss/html2rss', branch: 'feat/registry-v1' +# Local monorepo override: BUNDLE_LOCAL__HTML2RSS=/path/to/html2rss gem 'base64' gem 'rack-cache' diff --git a/app/registries/seed/official/.gitkeep b/app/registries/seed/official/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/app/registries/seed/official/configs/phys.org/weekly.yml b/app/registries/seed/official/configs/phys.org/weekly.yml new file mode 100644 index 000000000..4c0102d03 --- /dev/null +++ b/app/registries/seed/official/configs/phys.org/weekly.yml @@ -0,0 +1,25 @@ +registry: + id: phys.org/weekly +directory: + topics: + - science + title: "Phys.org — Weekly" + summary: "Top science news of the week from Phys.org." +channel: + language: en + title: "Phys.org — Weekly" + url: https://phys.org/weekly-news/ + time_zone: Europe/London + ttl: 1440 +selectors: + items: + selector: ".sorted-news-list .sorted-article-content" + title: + selector: "h4" + category: + selector: ".text-info" + categories: + - category + url: + selector: ".news-link" + extractor: "href" diff --git a/app/registries/seed/official/configs/support.apple.com/en_gb_ht201222.yml b/app/registries/seed/official/configs/support.apple.com/en_gb_ht201222.yml new file mode 100644 index 000000000..83c1c81b6 --- /dev/null +++ b/app/registries/seed/official/configs/support.apple.com/en_gb_ht201222.yml @@ -0,0 +1,34 @@ +registry: + id: support.apple.com/en_gb_ht201222 +directory: + topics: + - tech + - security + title: "Apple Support — Security releases" + summary: "Apple security update and release notes (HT201222 / related)." +strategy: botasaurus +channel: + title: "Apple Support — Security releases" + url: https://support.apple.com/en-gb/100100 + language: en + ttl: 360 + time_zone: UTC +request: + botasaurus: + wait_for_selector: ".table-wrapper table tbody tr a" + wait_timeout_seconds: 20 +selectors: + items: + selector: ".table-wrapper table tbody > tr:not(:first-child)" + enhance: false + title: + selector: a + url: + selector: a + extractor: href + description: + selector: "td:nth-child(2)" + published_at: + selector: "td:nth-child(3)" + post_process: + - name: parse_time diff --git a/app/registries/seed/official/manifest.json b/app/registries/seed/official/manifest.json new file mode 100644 index 000000000..d02f057ad --- /dev/null +++ b/app/registries/seed/official/manifest.json @@ -0,0 +1,10 @@ +{ + "format": "registry.v1", + "registry_id": "official", + "version": "test-fixture", + "public_key_id": "test", + "files": { + "configs/phys.org/weekly.yml": "0e05fa9a95ec56bef4b4363b2f044ab69642fcda9acd91ebf8d28b39141a963d", + "configs/support.apple.com/en_gb_ht201222.yml": "53e38a6b7d088e0c79b19a7dd6db9b009c6b5c0b6690f504c7160b0c32b71e41" + } +} diff --git a/app/web/registry/catalog_change_reporter.rb b/app/web/registry/catalog_change_reporter.rb new file mode 100644 index 000000000..f218090c3 --- /dev/null +++ b/app/web/registry/catalog_change_reporter.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +module Html2rss + module Web + module Registry + ## + # Reports catalog diffs after a registry bundle is promoted to active. + module CatalogChangeReporter + module_function + + ## + # @param registry_id [String] + # @param previous_bundle [Index::RegistryBundle, nil] + # @param new_bundle [Index::RegistryBundle] + # @return [void] + def report!(registry_id:, previous_bundle:, new_bundle:) + diff = build_diff(previous_bundle, new_bundle) + return if diff.empty? + + emit_observability!(registry_id, diff) + emit_security!(registry_id, diff) if diff[:added].any? || diff[:removed].any? + end + + ## + # @param previous_bundle [Index::RegistryBundle, nil] + # @param new_bundle [Index::RegistryBundle] + # @return [Hash{Symbol => Object}] + def build_diff(previous_bundle, new_bundle) # rubocop:disable Metrics/MethodLength + previous_version = previous_bundle&.manifest&.version + new_version = new_bundle.manifest.version + previous_ids = catalog_ids(previous_bundle) + new_ids = catalog_ids(new_bundle) + added = new_ids - previous_ids + removed = previous_ids - new_ids + + return {} if previous_version == new_version && added.empty? && removed.empty? + + { + previous_version:, + version: new_version, + added:, + removed: + } + end + + ## + # @param registry_id [String] + # @param diff [Hash{Symbol => Object}] + # @return [void] + def emit_observability!(registry_id, diff) # rubocop:disable Metrics/MethodLength + Observability.emit( + event_name: 'registry.catalog_changed', + outcome: 'success', + level: :warn, + details: { + registry_id:, + version: diff[:version], + previous_version: diff[:previous_version], + added_count: diff[:added].size, + removed_count: diff[:removed].size, + added_ids: diff[:added].sort, + removed_ids: diff[:removed].sort + } + ) + end + + ## + # @param registry_id [String] + # @param diff [Hash{Symbol => Object}] + # @return [void] + def emit_security!(registry_id, diff) + SecurityLogger.log_registry_catalog_changed( + registry_id, + version: diff[:version], + previous_version: diff[:previous_version], + added_count: diff[:added].size, + removed_count: diff[:removed].size, + added_ids: diff[:added].sort, + removed_ids: diff[:removed].sort + ) + end + + ## + # @param bundle [Index::RegistryBundle, nil] + # @return [Array] + def catalog_ids(bundle) + return [] unless bundle + + bundle.catalog_entries.map(&:id) + end + + private_class_method :build_diff, :emit_observability!, :emit_security!, :catalog_ids + end + end + end +end diff --git a/app/web/registry/config.rb b/app/web/registry/config.rb index 4fd5b8842..065fcfae5 100644 --- a/app/web/registry/config.rb +++ b/app/web/registry/config.rb @@ -6,6 +6,10 @@ module Html2rss module Web module Registry + ## + # Sync policy parsed from registry YAML. + SyncPolicy = Data.define(:pin_version, :max_version, :auto_promote) + ## # Registry definition parsed from {Config::REGISTRIES_FILE}. Entry = Data.define( @@ -16,7 +20,9 @@ module Registry :sync_url, :catalog, :public_key_id, - :public_key + :public_key, + :sync_policy, + :allowed_channel_domains ) do ## # @return [Hash{String => OpenSSL::PKey::PKey}] @@ -115,14 +121,32 @@ def default_document { precedence: DEFAULT_PRECEDENCE, registries: { - 'official' => { - sync: { channel: DEFAULT_OFFICIAL_SYNC_CHANNEL }, - catalog: true - } + 'official' => default_official_registry } } end + ## + # @return [Hash{Symbol => Object}] + def default_official_registry + { + sync: { channel: DEFAULT_OFFICIAL_SYNC_CHANNEL }, + catalog: true, + public_key_id: 'html2rss:registry:2026', + public_key: default_public_key_pem + } + end + + ## + # @return [String] + def default_public_key_pem + <<~PEM + -----BEGIN PUBLIC KEY----- + MCowBQYDK2VwAyEAiMbg/04MyC5azBdM/aeY0mNuA8JbP5/jOiNRwJ2KJHE= + -----END PUBLIC KEY----- + PEM + end + ## # @param registry_id [String] # @param document [Hash{Symbol => Object}] @@ -141,13 +165,36 @@ def parse_entry(registry_id, document) # @return [Entry] def build_entry(registry_id, raw) sync = sync_section(raw) - mode, resolved_path, sync_channel, sync_url = resolve_mode( - path: raw[:path]&.to_s, - sync_channel: sync[:channel]&.to_s, - sync_url: sync[:url]&.to_s - ) + case raw + in { path: path } if path&.then { |value| !value.empty? } + entry_attributes(registry_id, raw, :path, expand_path(path.to_s), nil, nil) + else + build_sync_entry(registry_id, raw, sync) + end + end - entry_attributes(registry_id, raw, mode, resolved_path, sync_channel, sync_url) + ## + # @param registry_id [String] + # @param raw [Hash{Symbol => Object}] + # @param sync [Hash{Symbol => Object}] + # @return [Entry] + def build_sync_entry(registry_id, raw, sync) + sync_channel = sync[:channel]&.to_s + sync_url = sync[:url]&.to_s + resolved_sync_url = resolved_sync_url(sync_url, sync_channel) + entry = entry_attributes(registry_id, raw, :sync, nil, sync_channel, resolved_sync_url) + validate_sync_public_key!(registry_id, entry) + entry + end + + ## + # @param sync_url [String] + # @param sync_channel [String, nil] + # @return [String] + def resolved_sync_url(sync_url, sync_channel) + return sync_url if sync_url && !sync_url.empty? + + SyncUrlResolver.resolve_channel_url(sync_channel) end ## @@ -158,7 +205,8 @@ def build_entry(registry_id, raw) # @param sync_channel [String, nil] # @param sync_url [String, nil] # @return [Entry] - def entry_attributes(registry_id, raw, mode, path, sync_channel, sync_url) # rubocop:disable Metrics/ParameterLists + def entry_attributes(registry_id, raw, mode, path, sync_channel, sync_url) # rubocop:disable Metrics/ParameterLists, Metrics/MethodLength + sync = sync_section(raw) Entry.new( id: registry_id, mode:, @@ -167,7 +215,9 @@ def entry_attributes(registry_id, raw, mode, path, sync_channel, sync_url) # rub sync_url:, catalog: raw.fetch(:catalog, true), public_key_id: public_key_id_for(raw[:public_key_id]&.to_s), - public_key: parse_public_key(raw[:public_key]) + public_key: parse_public_key(raw[:public_key]), + sync_policy: sync_policy_for(raw, sync), + allowed_channel_domains: parse_allowed_channel_domains(raw[:allowed_channel_domains]) ) end @@ -179,34 +229,57 @@ def sync_section(raw) end ## - # @param public_key_id [String, nil] - # @return [String] - def public_key_id_for(public_key_id) - return public_key_id if public_key_id && !public_key_id.empty? + # @param raw [Hash{Symbol => Object}] + # @param sync [Hash{Symbol => Object}] + # @return [SyncPolicy] + def sync_policy_for(raw, sync) + SyncPolicy.new( + pin_version: optional_string(sync[:pin_version]), + max_version: optional_string(sync[:max_version]), + auto_promote: auto_promote?(raw[:auto_promote]) + ) + end - 'html2rss:registry:2026' + ## + # @param value [Object] + # @return [Boolean] + def auto_promote?(value) + value == true end ## - # @param path [String, nil] - # @param sync_channel [String, nil] - # @param sync_url [String, nil] - # @return [Array(Symbol, String, nil, String, nil)] - def resolve_mode(path:, sync_channel:, sync_url:) - return [:path, expand_path(path), nil, nil] if path && !path.empty? + # @param value [Object] + # @return [String, nil] + def optional_string(value) + string = value&.to_s + string.nil? || string.empty? ? nil : string + end - resolved_sync_url = sync_url && !sync_url.empty? ? sync_url : resolve_sync_channel_url(sync_channel) - [:sync, nil, sync_channel, resolved_sync_url] + ## + # @param value [Object] + # @return [Array] + def parse_allowed_channel_domains(value) + Array(value).map { |domain| domain.to_s.strip }.reject(&:empty?) end ## - # @param sync_channel [String, nil] + # @param public_key_id [String, nil] # @return [String] - def resolve_sync_channel_url(sync_channel) - channel = sync_channel && !sync_channel.empty? ? sync_channel : DEFAULT_OFFICIAL_SYNC_CHANNEL - return OFFICIAL_RELEASE_URL if channel == DEFAULT_OFFICIAL_SYNC_CHANNEL + def public_key_id_for(public_key_id) + return public_key_id if public_key_id && !public_key_id.empty? + + 'html2rss:registry:2026' + end + + ## + # @param registry_id [String] + # @param entry [Entry] + # @return [void] + def validate_sync_public_key!(registry_id, entry) + return unless entry.public_key.nil? - raise Errors::ConfigError, "Unknown sync channel '#{channel}'" + raise Errors::ConfigError, + "Sync registry '#{registry_id}' requires a pinned public_key in #{registries_file}" end ## diff --git a/app/web/registry/index.rb b/app/web/registry/index.rb index 6052c4bf4..5c0df5d84 100644 --- a/app/web/registry/index.rb +++ b/app/web/registry/index.rb @@ -50,8 +50,15 @@ def config_for(feed_id) # @return [Array Object}>] catalog rows in HTTP wire shape def catalog_rows rows = registry_catalog_rows - LocalCatalog.rows.each { |row| rows[row.fetch(:id)] = row } - rows.values.sort_by { |row| row.fetch(:id) } + LocalCatalog.rows.each { |row| rows[row.id] = row } + rows.values.sort_by(&:id).map(&:to_h) + end + + ## + # @param registry_id [String, Symbol] + # @return [RegistryBundle, nil] + def bundle_for(registry_id) + loaded_bundles[registry_id.to_s] end ## @@ -72,7 +79,7 @@ def status private ## - # @return [Hash{String => Hash{Symbol => Object}}] + # @return [Hash{String => RegistryCatalogRow, LocalCatalogRow}] def registry_catalog_rows Config.precedence.each_with_object({}) do |registry_id, rows| next unless Config.catalog_enabled?(registry_id) @@ -81,7 +88,7 @@ def registry_catalog_rows next unless bundle bundle.catalog_entries.each do |entry| - rows[entry.id] ||= wire_registry_row(entry, registry_id) + rows[entry.id] ||= RegistryCatalogRow.from_entry(entry, registry_id) end end end @@ -99,13 +106,15 @@ def load_bundle(registry_id) # rubocop:disable Metrics/MethodLength entry = Config.entry(registry_id) directory = bundle_directory(entry) return nil unless directory && File.directory?(directory) + return nil unless active_bundle_present?(directory) bundle = Html2rss::Registry::Bundle.load( directory, - trust: :integrity_only, - public_keys: entry.public_keys + **TrustContext.for_entry(entry, directory).load_options ) - to_registry_bundle(registry_id, bundle) + registry_bundle = to_registry_bundle(registry_id, bundle) + ScrapePolicy.enforce!(entry, registry_bundle) + registry_bundle rescue Html2rss::Registry::Error => error raise Errors::LoadError, "Failed to load registry '#{registry_id}': #{error.message}" end @@ -132,6 +141,13 @@ def bundle_directory(entry) Store.registry_dir(entry.id) end + ## + # @param directory [String] + # @return [Boolean] + def active_bundle_present?(directory) + File.file?(File.join(directory, Html2rss::Registry::Manifest::MANIFEST_FILE)) + end + ## # @param feed_id [String, Symbol] # @return [String] @@ -139,26 +155,6 @@ def normalize_feed_id(feed_id) feed_id.to_s.delete_prefix('/').sub(LocalConfig::FEED_EXTENSION_PATTERN, '') end - ## - # @param feed_id [String] - # @return [Hash{Symbol => Object}, nil] - def local_config_for(feed_id) - feed = LocalConfig.feeds[feed_id.to_sym] || LocalConfig.feeds[feed_id] - return nil unless feed - - deep_dup(feed) - rescue Html2rss::Web::LocalConfig::InvalidConfig, Html2rss::Web::LocalConfig::NotFound - nil - end - - ## - # @param entry [Html2rss::Registry::CatalogEntry] - # @param registry_id [String] - # @return [Hash{Symbol => Object}] - def wire_registry_row(entry, registry_id) - entry.to_h.merge(source: 'registry', registry: registry_id) - end - ## # @param registry_id [String] # @return [Time, nil] @@ -172,6 +168,18 @@ def bundle_updated_at(registry_id) File.mtime(manifest_path) end + ## + # @param feed_id [String] + # @return [Hash{Symbol => Object}, nil] + def local_config_for(feed_id) + feed = LocalConfig.feeds[feed_id.to_sym] || LocalConfig.feeds[feed_id] + return nil unless feed + + deep_dup(feed) + rescue Html2rss::Web::LocalConfig::InvalidConfig, Html2rss::Web::LocalConfig::NotFound + nil + end + ## # @param value [Object] # @return [Object] diff --git a/app/web/registry/local_catalog.rb b/app/web/registry/local_catalog.rb index 063dd7b29..9623bf45f 100644 --- a/app/web/registry/local_catalog.rb +++ b/app/web/registry/local_catalog.rb @@ -9,7 +9,7 @@ module LocalCatalog module_function ## - # @return [Array Object}>] + # @return [Array] def rows LocalConfig.feeds.filter_map do |feed_name, feed_config| build_row(feed_name, feed_config) @@ -19,7 +19,7 @@ def rows ## # @param feed_name [String, Symbol] # @param feed_config [Hash{Symbol => Object}] - # @return [Hash{Symbol => Object}, nil] + # @return [LocalCatalogRow, nil] def build_row(feed_name, feed_config) directory = feed_config[:directory] || {} title = directory[:title] @@ -33,17 +33,17 @@ def build_row(feed_name, feed_config) # @param directory [Hash{Symbol => Object}] # @param title [String] # @param channel [Hash{Symbol => Object}] - # @return [Hash{Symbol => Object}] + # @return [LocalCatalogRow] def row_identity(feed_name, directory, title, channel) id = feed_name.to_s - { + LocalCatalogRow.new( id:, path: "/#{id}.rss", source: 'local', directory: directory_payload(directory, title), channel: channel_payload(channel, title), parameters: { schema: {}, defaults: {} } - } + ) end ## diff --git a/app/web/registry/local_catalog_row.rb b/app/web/registry/local_catalog_row.rb new file mode 100644 index 000000000..0d95a784f --- /dev/null +++ b/app/web/registry/local_catalog_row.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Html2rss + module Web + module Registry + ## + # Catalog wire row for local feeds.yml entries. + LocalCatalogRow = Data.define( + :id, + :path, + :directory, + :channel, + :parameters, + :source + ) + end + end +end diff --git a/app/web/registry/registry_catalog_row.rb b/app/web/registry/registry_catalog_row.rb new file mode 100644 index 000000000..27d3791e6 --- /dev/null +++ b/app/web/registry/registry_catalog_row.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Html2rss + module Web + module Registry + ## + # Catalog wire row for registry-sourced feed configs. + RegistryCatalogRow = Data.define( + :id, + :path, + :directory, + :channel, + :parameters, + :source, + :registry + ) do + ## + # @param entry [Html2rss::Registry::CatalogEntry] + # @param registry_id [String] + # @return [RegistryCatalogRow] + def self.from_entry(entry, registry_id) + new( + id: entry.id, + path: entry.path, + directory: entry.directory, + channel: entry.channel, + parameters: entry.parameters, + source: 'registry', + registry: registry_id + ) + end + end + end + end +end diff --git a/app/web/registry/scrape_policy.rb b/app/web/registry/scrape_policy.rb new file mode 100644 index 000000000..b341d751f --- /dev/null +++ b/app/web/registry/scrape_policy.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +require 'uri' + +module Html2rss + module Web + module Registry + ## + # Enforces optional channel URL domain allowlists for registry bundles. + module ScrapePolicy + module_function + + ## + # @param entry [Entry] + # @param bundle [Index::RegistryBundle] + # @return [void] + # @raise [Errors::LoadError] when a config channel URL violates the allowlist + def enforce!(entry, bundle) # rubocop:disable Metrics/CyclomaticComplexity + allowed = entry.allowed_channel_domains + return if allowed.nil? || allowed.empty? + + bundle.configs.each do |feed_id, config| + channel_url = config.dig(:channel, :url) + host = host_for(channel_url) + next if host && allowed.any? { |domain| domain_allowed?(host, domain) } + + raise Errors::LoadError, + "Registry '#{entry.id}' config '#{feed_id}' channel.url host " \ + "'#{host || channel_url}' is not allowed by allowed_channel_domains" + end + end + + ## + # @param host [String] + # @param allowed_domain [String] + # @return [Boolean] + def domain_allowed?(host, allowed_domain) + normalized_host = host.downcase + normalized_domain = allowed_domain.downcase + normalized_host == normalized_domain || normalized_host.end_with?(".#{normalized_domain}") + end + + ## + # @param channel_url [String, nil] + # @return [String, nil] + def host_for(channel_url) + return nil if channel_url.to_s.strip.empty? + + URI.parse(channel_url).host + rescue URI::InvalidURIError + nil + end + + private_class_method :host_for + end + end + end +end diff --git a/app/web/registry/store.rb b/app/web/registry/store.rb index fc9e7459e..1bd69f28d 100644 --- a/app/web/registry/store.rb +++ b/app/web/registry/store.rb @@ -8,12 +8,14 @@ module Web module Registry ## # Manages on-disk registry bundle directories under {data_root}. - module Store + module Store # rubocop:disable Metrics/ModuleLength DEFAULT_DATA_ROOT = 'tmp/registry-data' DEFAULT_SEED_ROOT = '/app/registries/seed' SYNC_STATE_FILE = '.sync-state.json' - class << self + SyncState = Data.define(:last_error, :last_sync_at) + + class << self # rubocop:disable Metrics/ClassLength ## # @return [String] root directory for extracted registry bundles def data_root @@ -47,6 +49,66 @@ def bundle_present?(registry_id) bundle_present_at?(registry_dir(registry_id)) end + ## + # @param registry_id [String, Symbol] + # @return [String] verified staging directory for a registry id + def staging_dir(registry_id) + File.join(registry_dir(registry_id), '.staging') + end + + ## + # @param registry_id [String, Symbol] + # @return [Boolean] + def staged_present?(registry_id) + bundle_present_at?(staging_dir(registry_id)) + end + + ## + # @param registry_id [String, Symbol] + # @return [String, nil] + def staged_version(registry_id) + manifest_version_at(staging_dir(registry_id)) + end + + ## + # Writes a verified bundle to the registry staging directory. + # + # @param registry_id [String, Symbol] + # @param staged_dir [String] verified bundle directory to stage + # @return [String] staging directory + def stage_bundle!(registry_id, staged_dir) + raise Errors::LoadError, "Staged bundle missing: #{staged_dir}" unless File.directory?(staged_dir) + + target = staging_dir(registry_id) + parent = registry_dir(registry_id) + FileUtils.mkdir_p(parent) + + backup = "#{target}.backup.#{Process.pid}" + promote_bundle!(staged_dir, target, backup) + target + end + + ## + # Promotes a verified staging bundle to the active registry directory. + # + # @param registry_id [String, Symbol] + # @return [String] active bundle directory + def promote_staged!(registry_id) # rubocop:disable Metrics/MethodLength + staged = staging_dir(registry_id) + raise Errors::LoadError, "No staged bundle for '#{registry_id}'" unless staged_present?(registry_id) + + temp_root = Dir.mktmpdir('registry-promote-') + temp_staged = File.join(temp_root, 'bundle') + FileUtils.mv(staged, temp_staged) + + active = registry_dir(registry_id) + backup = "#{active}.backup.#{Process.pid}" + promote_bundle!(temp_staged, active, backup) + active + ensure + FileUtils.rm_rf(temp_root) if temp_root + end + ## # Atomically replaces the active bundle directory for a registry id. # @@ -84,9 +146,13 @@ def seed_if_empty!(registry_id, seed_path:) # rubocop:disable Naming/PredicateMe ## # @param registry_id [String, Symbol] - # @return [Hash{Symbol => Object}] + # @return [SyncState] def sync_state(registry_id) - read_sync_state.fetch(registry_id.to_s, {}) + raw = read_sync_state.fetch(registry_id.to_s, {}) + SyncState.new( + last_error: raw['last_error'], + last_sync_at: parse_sync_time(raw['last_sync_at']) + ) end ## @@ -143,7 +209,30 @@ def promote_bundle!(staged_dir, target, backup) # @param path [String] # @return [Boolean] def bundle_present_at?(path) - File.directory?(path) && !Dir.empty?(path) + File.file?(File.join(path, Html2rss::Registry::Manifest::MANIFEST_FILE)) + end + + ## + # @param path [String] + # @return [String, nil] + def manifest_version_at(path) + return nil unless bundle_present_at?(path) + + manifest = Html2rss::Registry::Manifest.parse( + File.read(File.join(path, Html2rss::Registry::Manifest::MANIFEST_FILE)) + ) + manifest.version + rescue Html2rss::Registry::ManifestError + nil + end + + ## + # @param value [String, nil] + # @return [Time, nil] + def parse_sync_time(value) + return nil if value.to_s.empty? + + Time.parse(value) end end end diff --git a/app/web/registry/sync.rb b/app/web/registry/sync.rb index 4d7f30cf9..f96c3448e 100644 --- a/app/web/registry/sync.rb +++ b/app/web/registry/sync.rb @@ -1,10 +1,6 @@ # frozen_string_literal: true require 'fileutils' -require 'net/http' -require 'json' -require 'stringio' -require 'uri' module Html2rss module Web @@ -12,22 +8,15 @@ module Registry ## # Fetches, verifies, and stores signed registry bundles. module Sync # rubocop:disable Metrics/ModuleLength - OFFICIAL_RELEASE_URL = Config::OFFICIAL_RELEASE_URL - OFFICIAL_GITHUB_RELEASES_API = - 'https://api.github.com/repos/html2rss/html2rss-configs/releases/latest' - OFFICIAL_ASSET_NAME = 'registry-bundle.tar.gz' - - DEFAULT_ALLOWED_HOSTS = %w[ - api.github.com - github.com - objects.githubusercontent.com - ].freeze - FETCH_OPEN_TIMEOUT_SECONDS = 10 - FETCH_READ_TIMEOUT_SECONDS = 60 - MAX_RESPONSE_BYTES = Html2rss::Registry::Archive::MAX_TARBALL_BYTES - BACKGROUND_JITTER_FRACTION = 0.1 - - SyncStatus = Data.define(:registry_id, :mode, :version, :updated_at, :sync_url, :last_error) + SyncStatus = Data.define( + :registry_id, + :mode, + :version, + :staged_version, + :updated_at, + :sync_url, + :last_error + ) @boot_mutex = Mutex.new @boot_started = false @@ -43,7 +32,7 @@ def sync_url_for(registry_id) entry = Config.entry(registry_id) raise Errors::SyncError, "Registry '#{registry_id}' is not sync-mode" unless entry.mode == :sync - resolve_download_url(entry) + SyncUrlResolver.resolve(entry) end ## @@ -52,30 +41,21 @@ def sync_url_for(registry_id) # @param registry_id [String, Symbol] # @param dry_run [Boolean] when true, verify without swapping the active bundle # @return [SyncStatus] - def run(registry_id:, dry_run: false) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength - entry = Config.entry(registry_id) - if entry.mode == :path - raise Errors::SyncError, "Registry '#{registry_id}' uses path mode; sync is not applicable" - end + def run(registry_id:, dry_run: false) + SyncCoordinator.run(registry_id) { run!(registry_id:, dry_run:) } + end - staging_root = nil - download_url = resolve_download_url(entry) - staging_root, staged_dir = fetch_and_verify!(entry, download_url) - unless dry_run - Store.swap!(registry_id, staged_dir) - Index.reload! - record_success!(registry_id) - end - sync_status_for(Index.current.status.find { |row| row.id == registry_id.to_s }) - rescue StandardError => error - record_failure!(registry_id, error) unless dry_run - raise - ensure - FileUtils.rm_rf(staging_root) if staging_root + ## + # Promotes a verified staging bundle to the active registry directory. + # + # @param registry_id [String, Symbol] + # @return [SyncStatus] + def promote_staged!(registry_id:) + SyncCoordinator.run(registry_id) { promote_staged_bundle!(registry_id:) } end ## - # @param registry_id [String, Symbol, nil] + # @param registry_id [String, Symbol] # @return [Array] def status(registry_id: nil) rows = Index.current.status @@ -149,6 +129,90 @@ def unusable_sync_registries private + ## + # @param registry_id [String, Symbol] + # @param dry_run [Boolean] + # @return [SyncStatus] + def run!(registry_id:, dry_run:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity + entry = Config.entry(registry_id) + if entry.mode == :path + raise Errors::SyncError, "Registry '#{registry_id}' uses path mode; sync is not applicable" + end + + staging_root = nil + download_url = SyncUrlResolver.resolve(entry) + staging_root, staged_dir = fetch_and_verify!(entry, download_url) + unless dry_run + if entry.sync_policy.auto_promote + activate_bundle!(registry_id, staged_dir) + else + Store.stage_bundle!(registry_id, staged_dir) + record_success!(registry_id, promoted: false) + end + end + sync_status_for(Index.current.status.find { |row| row.id == registry_id.to_s }) + rescue StandardError => error + record_failure!(registry_id, error) unless dry_run + raise + ensure + FileUtils.rm_rf(staging_root) if staging_root + end + + ## + # @param registry_id [String, Symbol] + # @return [SyncStatus] + def promote_staged_bundle!(registry_id:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + entry = Config.entry(registry_id) + if entry.mode == :path + raise Errors::SyncError, "Registry '#{registry_id}' uses path mode; promote is not applicable" + end + unless Store.staged_present?(registry_id) + raise Errors::SyncError, "Registry '#{registry_id}' has no staged bundle" + end + + previous_bundle = Index.current.bundle_for(registry_id) + Store.promote_staged!(registry_id) + Index.reload! + report_catalog_change!(registry_id, previous_bundle) + record_success!(registry_id, promoted: true) + Observability.emit( + event_name: 'registry.promote_staged', + outcome: 'success', + details: { + registry_id: registry_id.to_s, + version: Index.current.status.find { |row| row.id == registry_id.to_s }&.version + } + ) + sync_status_for(Index.current.status.find { |row| row.id == registry_id.to_s }) + end + + ## + # @param registry_id [String, Symbol] + # @param staged_dir [String] + # @return [void] + def activate_bundle!(registry_id, staged_dir) + previous_bundle = Index.current.bundle_for(registry_id) + Store.swap!(registry_id, staged_dir) + Index.reload! + report_catalog_change!(registry_id, previous_bundle) + record_success!(registry_id, promoted: true) + end + + ## + # @param registry_id [String, Symbol] + # @param previous_bundle [Index::RegistryBundle, nil] + # @return [void] + def report_catalog_change!(registry_id, previous_bundle) + new_bundle = Index.current.bundle_for(registry_id) + return unless new_bundle + + CatalogChangeReporter.report!( + registry_id: registry_id.to_s, + previous_bundle:, + new_bundle: + ) + end + ## # @return [Boolean] def skip_boot? @@ -213,41 +277,22 @@ def background_jitter_seconds(interval_hours) rand(max_jitter) end - ## - # @param entry [Entry] - # @return [String] - def resolve_download_url(entry) - return entry.sync_url if entry.sync_url && !entry.sync_url.empty? - return resolve_official_download_url if entry.sync_channel == Config::DEFAULT_OFFICIAL_SYNC_CHANNEL - - raise Errors::SyncError, "Registry '#{entry.id}' has no sync URL" - end - - ## - # @return [String] - def resolve_official_download_url - response_body = Fetcher.fetch!(OFFICIAL_GITHUB_RELEASES_API) - release = JSON.parse(response_body, symbolize_names: true) - asset = Array(release[:assets]).find { |row| row[:name] == OFFICIAL_ASSET_NAME } - url = asset&.dig(:browser_download_url) - raise Errors::SyncError, "Official release asset '#{OFFICIAL_ASSET_NAME}' not found" unless url - - url - rescue JSON::ParserError => error - raise Errors::SyncError, "Invalid GitHub release metadata: #{error.message}" - end + BACKGROUND_JITTER_FRACTION = 0.1 + private_constant :BACKGROUND_JITTER_FRACTION ## # @param entry [Entry] # @param download_url [String] # @return [Array(String, String)] staging root and verified bundle directory def fetch_and_verify!(entry, download_url) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength - tarball = Fetcher.fetch!(download_url) + tarball = SyncFetcher.fetch!(download_url) staging_root = Dir.mktmpdir('registry-sync-') staged_dir = File.join(staging_root, 'bundle') + tarball_path = File.join(staging_root, 'download.tar.gz') FileUtils.mkdir_p(staged_dir) + File.binwrite(tarball_path, tarball) - StringIO.open(tarball) do |io| + File.open(tarball_path, 'rb') do |io| Html2rss::Registry::Archive.extract!(io, into: staged_dir) end @@ -256,6 +301,7 @@ def fetch_and_verify!(entry, download_url) # rubocop:disable Metrics/AbcSize, Me trust: :signed, public_keys: entry.public_keys ) + enforce_max_version!(entry, staged_dir) [staging_root, staged_dir] rescue Html2rss::Registry::VerificationError => error log_signature_failure!(entry.id, error.message) if signature_failure?(error) @@ -279,16 +325,46 @@ def log_signature_failure!(registry_id, message) SecurityLogger.log_registry_signature_failure(registry_id, message) end + ## + # @param entry [Entry] + # @param staged_dir [String] + # @return [void] + def enforce_max_version!(entry, staged_dir) + manifest = read_manifest!(staged_dir) + max_version = entry.sync_policy.max_version + return unless max_version + + return unless VersionGate.exceeds_max?(manifest.version, max_version) + + raise Errors::SyncError, + "Registry '#{entry.id}' manifest version '#{manifest.version}' exceeds max_version '#{max_version}'" + end + + ## + # @param staged_dir [String] + # @return [Html2rss::Registry::Manifest] + def read_manifest!(staged_dir) + Html2rss::Registry::Manifest.parse( + File.read(File.join(staged_dir, Html2rss::Registry::Manifest::MANIFEST_FILE)) + ) + end + ## # @param registry_id [String] + # @param promoted [Boolean] # @return [void] - def record_success!(registry_id) + def record_success!(registry_id, promoted:) # rubocop:disable Metrics/MethodLength Store.write_sync_state!(registry_id, last_error: nil) row = Index.current.status.find { |entry| entry.id == registry_id.to_s } Observability.emit( event_name: 'registry.sync', outcome: 'success', - details: { registry_id:, version: row&.version } + details: { + registry_id:, + version: row&.version, + staged_version: Store.staged_version(registry_id), + promoted: + } ) end @@ -309,118 +385,20 @@ def record_failure!(registry_id, error) ## # @param row [Index::StatusEntry] # @return [SyncStatus] - def sync_status_for(row) + def sync_status_for(row) # rubocop:disable Metrics/MethodLength entry = Config.entry(row.id) state = Store.sync_state(row.id) SyncStatus.new( registry_id: row.id, mode: row.sync_mode, version: row.version, + staged_version: Store.staged_version(row.id), updated_at: row.updated_at, - sync_url: entry.mode == :sync ? resolve_download_url(entry) : nil, - last_error: state['last_error'] + sync_url: entry.mode == :sync ? SyncUrlResolver.resolve(entry) : nil, + last_error: state.last_error ) end end # rubocop:enable Metrics/ClassLength - - ## - # HTTPS fetcher with host allowlisting and redirect rejection. - module Fetcher - module_function - - ## - # @param url [String] - # @return [String] response body - def fetch!(url) - uri = parse_https_uri!(url) - ensure_allowed_host!(uri.host) - perform_request!(uri) - end - - ## - # @param url [String] - # @return [URI::HTTPS] - def parse_https_uri!(url) - uri = URI(url) - unless uri.is_a?(URI::HTTPS) - raise Errors::SyncError, "Registry sync requires HTTPS URLs (got #{uri.scheme.inspect})" - end - - uri - end - - ## - # @param host [String] - # @return [void] - def ensure_allowed_host!(host) - allowed = DEFAULT_ALLOWED_HOSTS + extra_allowed_hosts - return if allowed.include?(host) - - raise Errors::SyncError, "Registry sync host not allowed: #{host}" - end - - ## - # @return [Array] - def extra_allowed_hosts - ENV.fetch('REGISTRY_SYNC_ALLOWED_HOSTS', '') - .split(',') - .map(&:strip) - .reject(&:empty?) - end - - ## - # @param uri [URI::HTTPS] - # @return [String] - def perform_request!(uri) # rubocop:disable Metrics/MethodLength - response = nil - Net::HTTP.start( - uri.host, - uri.port, - use_ssl: true, - open_timeout: FETCH_OPEN_TIMEOUT_SECONDS, - read_timeout: FETCH_READ_TIMEOUT_SECONDS - ) do |http| - request = Net::HTTP::Get.new(uri) - request['Accept'] = 'application/octet-stream' - request['User-Agent'] = 'html2rss-web/registry-sync' - response = http.request(request) - end - - reject_redirect!(response) - reject_error_status!(response) - read_body!(response) - end - - ## - # @param response [Net::HTTPResponse] - # @return [void] - def reject_redirect!(response) - return unless response.is_a?(Net::HTTPRedirection) - - raise Errors::SyncError, 'Registry sync rejects HTTP redirects' - end - - ## - # @param response [Net::HTTPResponse] - # @return [void] - def reject_error_status!(response) - return if response.is_a?(Net::HTTPSuccess) - - raise Errors::SyncError, "Registry sync fetch failed with HTTP #{response.code}" - end - - ## - # @param response [Net::HTTPResponse] - # @return [String] - def read_body!(response) - body = response.body.to_s - if body.bytesize > MAX_RESPONSE_BYTES - raise Errors::SyncError, "Registry sync response exceeds max bytes (#{MAX_RESPONSE_BYTES})" - end - - body - end - end end # rubocop:enable Metrics/ModuleLength end end diff --git a/app/web/registry/sync_coordinator.rb b/app/web/registry/sync_coordinator.rb new file mode 100644 index 000000000..2bd641e07 --- /dev/null +++ b/app/web/registry/sync_coordinator.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Html2rss + module Web + module Registry + ## + # Serializes sync work per registry id to avoid concurrent bundle swaps. + module SyncCoordinator + REGISTRY_MUTEXES = Hash.new { |mutexes, registry_id| mutexes[registry_id] = Mutex.new } + + module_function + + ## + # @param registry_id [String, Symbol] + # @yield runs sync work exclusively for the registry id + # @return [Object] block result + def run(registry_id, &) + REGISTRY_MUTEXES[registry_id.to_s].synchronize(&) + end + end + end + end +end diff --git a/app/web/registry/sync_fetcher.rb b/app/web/registry/sync_fetcher.rb new file mode 100644 index 000000000..6d13712a3 --- /dev/null +++ b/app/web/registry/sync_fetcher.rb @@ -0,0 +1,150 @@ +# frozen_string_literal: true + +require 'net/http' +require 'uri' + +module Html2rss + module Web + module Registry + ## + # HTTPS fetcher with host allowlisting and bounded redirect following. + module SyncFetcher + DEFAULT_ALLOWED_HOSTS = %w[ + api.github.com + github.com + objects.githubusercontent.com + release-assets.githubusercontent.com + ].freeze + FETCH_OPEN_TIMEOUT_SECONDS = 10 + FETCH_READ_TIMEOUT_SECONDS = 60 + MAX_RESPONSE_BYTES = 52_428_800 + DEFAULT_MAX_REDIRECTS = 5 + + RedirectPolicy = Data.define(:max_hops, :allowed_hosts) do + ## + # @return [RedirectPolicy] + def self.default + new(max_hops: DEFAULT_MAX_REDIRECTS, allowed_hosts: default_allowed_hosts) + end + + ## + # @return [Array] + def self.default_allowed_hosts + DEFAULT_ALLOWED_HOSTS + SyncFetcher.extra_allowed_hosts + end + end + + module_function + + ## + # @param url [String] + # @param policy [RedirectPolicy] + # @return [String] response body + def fetch!(url, policy: RedirectPolicy.default) + uri = parse_https_uri!(url) + follow_redirects!(uri, policy) + end + + ## + # @param uri [URI::HTTPS] + # @param policy [RedirectPolicy] + # @return [String] + def follow_redirects!(uri, policy) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + hops = 0 + + loop do + ensure_allowed_host!(uri.host, policy.allowed_hosts) + response = perform_request!(uri) + + case response + in Net::HTTPRedirection + hops += 1 + raise Errors::SyncError, 'Registry sync exceeded redirect limit' if hops > policy.max_hops + + location = response['location'] + raise Errors::SyncError, 'Registry sync redirect missing Location header' if location.to_s.empty? + + uri = parse_https_uri!(URI.join(uri, location).to_s) + next + else + reject_error_status!(response) + return read_body!(response) + end + end + end + + ## + # @return [Array] + def extra_allowed_hosts + ENV.fetch('REGISTRY_SYNC_ALLOWED_HOSTS', '') + .split(',') + .map(&:strip) + .reject(&:empty?) + end + + ## + # @param url [String] + # @return [URI::HTTPS] + def parse_https_uri!(url) + uri = URI(url) + unless uri.is_a?(URI::HTTPS) + raise Errors::SyncError, "Registry sync requires HTTPS URLs (got #{uri.scheme.inspect})" + end + + uri + end + + ## + # @param host [String] + # @param allowed_hosts [Array] + # @return [void] + def ensure_allowed_host!(host, allowed_hosts) + return if allowed_hosts.include?(host) + + raise Errors::SyncError, "Registry sync host not allowed: #{host}" + end + + ## + # @param uri [URI::HTTPS] + # @return [Net::HTTPResponse] + def perform_request!(uri) # rubocop:disable Metrics/MethodLength + response = nil + Net::HTTP.start( + uri.host, + uri.port, + use_ssl: true, + open_timeout: FETCH_OPEN_TIMEOUT_SECONDS, + read_timeout: FETCH_READ_TIMEOUT_SECONDS + ) do |http| + request = Net::HTTP::Get.new(uri) + request['Accept'] = 'application/octet-stream' + request['User-Agent'] = 'html2rss-web/registry-sync' + response = http.request(request) + end + response + end + + ## + # @param response [Net::HTTPResponse] + # @return [void] + def reject_error_status!(response) + return if response.is_a?(Net::HTTPSuccess) + + raise Errors::SyncError, "Registry sync fetch failed with HTTP #{response.code}" + end + + ## + # @param response [Net::HTTPResponse] + # @return [String] + def read_body!(response) + body = response.body.to_s + if body.bytesize > MAX_RESPONSE_BYTES + raise Errors::SyncError, "Registry sync response exceeds max bytes (#{MAX_RESPONSE_BYTES})" + end + + body + end + end + end + end +end diff --git a/app/web/registry/sync_url_resolver.rb b/app/web/registry/sync_url_resolver.rb new file mode 100644 index 000000000..d035c7396 --- /dev/null +++ b/app/web/registry/sync_url_resolver.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +require 'json' + +module Html2rss + module Web + module Registry + ## + # Resolves registry sync download URLs from config entries and official release metadata. + module SyncUrlResolver + OFFICIAL_GITHUB_RELEASES_API = + 'https://api.github.com/repos/html2rss/html2rss-configs/releases/latest' + OFFICIAL_GITHUB_TAG_RELEASES_API = + 'https://api.github.com/repos/html2rss/html2rss-configs/releases/tags/%s' + OFFICIAL_ASSET_NAME = 'registry-bundle.tar.gz' + + module_function + + ## + # @param entry [Entry] + # @return [String] + def resolve(entry) + return entry.sync_url if entry.sync_url && !entry.sync_url.empty? + + pin_version = entry.sync_policy.pin_version + if entry.sync_channel == Config::DEFAULT_OFFICIAL_SYNC_CHANNEL + return resolve_official_tag_download_url(pin_version) if pin_version && !pin_version.empty? + + return resolve_official_download_url + end + + raise Errors::SyncError, "Registry '#{entry.id}' has no sync URL" + end + + ## + # @param sync_channel [String, nil] + # @return [String] + def resolve_channel_url(sync_channel) + channel = sync_channel && !sync_channel.empty? ? sync_channel : Config::DEFAULT_OFFICIAL_SYNC_CHANNEL + return Config::OFFICIAL_RELEASE_URL if channel == Config::DEFAULT_OFFICIAL_SYNC_CHANNEL + + raise Errors::ConfigError, "Unknown sync channel '#{channel}'" + end + + ## + # @param tag [String] + # @return [String] + def resolve_official_tag_download_url(tag) # rubocop:disable Metrics/MethodLength + api_url = format(OFFICIAL_GITHUB_TAG_RELEASES_API, tag:) + response_body = SyncFetcher.fetch!(api_url) + release = JSON.parse(response_body, symbolize_names: true) + asset = Array(release[:assets]).find { |row| row[:name] == OFFICIAL_ASSET_NAME } + url = asset&.dig(:browser_download_url) + unless url + raise Errors::SyncError, + "Official release asset '#{OFFICIAL_ASSET_NAME}' not found for tag '#{tag}'" + end + + url + rescue JSON::ParserError => error + raise Errors::SyncError, "Invalid GitHub release metadata: #{error.message}" + end + + ## + # @return [String] + def resolve_official_download_url + response_body = SyncFetcher.fetch!(OFFICIAL_GITHUB_RELEASES_API) + release = JSON.parse(response_body, symbolize_names: true) + asset = Array(release[:assets]).find { |row| row[:name] == OFFICIAL_ASSET_NAME } + url = asset&.dig(:browser_download_url) + raise Errors::SyncError, "Official release asset '#{OFFICIAL_ASSET_NAME}' not found" unless url + + url + rescue JSON::ParserError => error + raise Errors::SyncError, "Invalid GitHub release metadata: #{error.message}" + end + end + end + end +end diff --git a/app/web/registry/trust_context.rb b/app/web/registry/trust_context.rb new file mode 100644 index 000000000..960593a10 --- /dev/null +++ b/app/web/registry/trust_context.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Html2rss + module Web + module Registry + ## + # Trust mode and verification keys passed to {Html2rss::Registry::Bundle.load}. + TrustContext = Data.define(:mode, :public_keys) do + ## + # @param entry [Entry] + # @param bundle_dir [String] + # @return [TrustContext] + def self.for_entry(entry, bundle_dir) + case entry.mode + in :path + integrity_only + in :sync + signed_bundle?(bundle_dir, entry) ? signed(entry.public_keys) : integrity_only + end + end + + ## + # @return [TrustContext] + def self.integrity_only + new(mode: :integrity_only, public_keys: {}) + end + + ## + # @param public_keys [Hash{String => OpenSSL::PKey::PKey}] + # @return [TrustContext] + def self.signed(public_keys) + new(mode: :signed, public_keys:) + end + + ## + # @return [Hash{Symbol => Object}] keyword args for {Html2rss::Registry::Bundle.load} + def load_options + { trust: mode, public_keys: } + end + + ## + # @param bundle_dir [String] + # @param entry [Entry] + # @return [Boolean] + def self.signed_bundle?(bundle_dir, entry) + signature_path = File.join(bundle_dir, Html2rss::Registry::Manifest::SIGNATURE_FILE) + File.file?(signature_path) && !entry.public_keys.empty? + end + + private_class_method :signed_bundle? + end + end + end +end diff --git a/app/web/registry/version_gate.rb b/app/web/registry/version_gate.rb new file mode 100644 index 000000000..27f6aadbd --- /dev/null +++ b/app/web/registry/version_gate.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Html2rss + module Web + module Registry + ## + # Compares registry manifest versions for sync policy caps. + module VersionGate + module_function + + ## + # @param manifest_version [String] + # @param max_version [String, nil] + # @return [Boolean] true when manifest_version is newer than max_version + def exceeds_max?(manifest_version, max_version) + return false if max_version.nil? || max_version.empty? + + compare(normalize(manifest_version), normalize(max_version)).positive? + end + + ## + # @param version [String] + # @return [String] + def normalize(version) + version.to_s.delete_prefix('v') + end + + ## + # @param left [String] + # @param right [String] + # @return [Integer] + def compare(left, right) + Gem::Version.new(left) <=> Gem::Version.new(right) + rescue ArgumentError + left <=> right + end + end + end + end +end diff --git a/app/web/security/security_logger.rb b/app/web/security/security_logger.rb index e6293584b..894345e81 100644 --- a/app/web/security/security_logger.rb +++ b/app/web/security/security_logger.rb @@ -76,6 +76,13 @@ def log_registry_signature_failure(registry_id, reason) log_event('registry_signature_failure', { registry_id:, reason: }, severity: :warn) end + # @param registry_id [String] + # @param details [Hash{Symbol => Object}] + # @return [void] + def log_registry_catalog_changed(registry_id, details) + log_event('registry_catalog_changed', { registry_id:, **details }, severity: :warn) + end + private # @param event_type [String] diff --git a/bin/registry-sync b/bin/registry-sync index 1c013a099..e8321a8b2 100755 --- a/bin/registry-sync +++ b/bin/registry-sync @@ -9,12 +9,13 @@ module RegistrySyncCLI module_function def run(argv = ARGV) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength - options = { status: false, registry_id: nil, dry_run: false } + options = { status: false, registry_id: nil, dry_run: false, promote: false } OptionParser.new do |opts| opts.banner = 'Usage: bin/registry-sync [options]' opts.on('--status', 'Print registry sync status and exit') { options[:status] = true } opts.on('--registry ID', 'Sync a single registry id') { |value| options[:registry_id] = value } opts.on('--dry-run', 'Fetch and verify without swapping the active bundle') { options[:dry_run] = true } + opts.on('--promote', 'Promote a verified staging bundle to active') { options[:promote] = true } end.parse!(argv) if options[:status] @@ -22,6 +23,13 @@ module RegistrySyncCLI exit Html2rss::Web::Registry::Sync.cli_exit_code end + if options[:promote] + target_registry_ids(options[:registry_id]).each do |registry_id| + Html2rss::Web::Registry::Sync.promote_staged!(registry_id:) + end + exit Html2rss::Web::Registry::Sync.cli_exit_code + end + target_registry_ids(options[:registry_id]).each do |registry_id| Html2rss::Web::Registry::Sync.run(registry_id:, dry_run: options[:dry_run]) end @@ -36,13 +44,14 @@ module RegistrySyncCLI def print_status_table(registry_id) # rubocop:disable Metrics/MethodLength rows = Html2rss::Web::Registry::Sync.status(registry_id:) - headers = %w[registry mode version updated_at sync_url last_error] + headers = %w[registry mode version staged_version updated_at sync_url last_error] puts headers.join("\t") rows.each do |row| puts [ row.registry_id, row.mode, row.version || '-', + row.staged_version || '-', format_time(row.updated_at), row.sync_url || '-', row.last_error || '-' diff --git a/config/registries.yml b/config/registries.yml index 3ebeca8e5..9c3d4af85 100644 --- a/config/registries.yml +++ b/config/registries.yml @@ -7,5 +7,16 @@ registries: channel: html2rss-official catalog: true public_key_id: html2rss:registry:2026 + public_key: | + -----BEGIN PUBLIC KEY----- + MCowBQYDK2VwAyEAiMbg/04MyC5azBdM/aeY0mNuA8JbP5/jOiNRwJ2KJHE= + -----END PUBLIC KEY----- # Pin the publisher Ed25519 public key for signed network sync. # Seed bundles copied from the image use integrity-only verification. + # Production: keep auto_promote false and promote manually after review. + # auto_promote: false + # sync: + # pin_version: v2026.08.22 + # max_version: v2026.08.22 + # allowed_channel_domains: + # - anthropic.com diff --git a/docs/README.md b/docs/README.md index 0fcf1b7f2..7c27ebb33 100644 --- a/docs/README.md +++ b/docs/README.md @@ -298,15 +298,20 @@ Inside the Dev Container: bin/registry-sync --status ``` -Columns: `registry`, `mode`, `version`, `updated_at`, `sync_url`, `last_error`. Exit code is non-zero when any registry reports `last_error`. +Columns: `registry`, `mode`, `version`, `staged_version`, `updated_at`, `sync_url`, `last_error`. Exit code is non-zero when any sync-mode registry lacks a usable on-disk bundle. -Sync a single registry or dry-run without swapping the active bundle: +Sync, dry-run, or promote a staged bundle: ```bash bin/registry-sync --registry official bin/registry-sync --registry official --dry-run +bin/registry-sync --promote --registry official ``` +Production recommendation: keep `auto_promote: false` (default), pin `sync.pin_version` to the approved configs tag, run sync to stage a verified bundle, then promote manually after review. Use `sync.max_version` as an incident freeze cap and set `REGISTRY_SYNC_INTERVAL_HOURS=0` to pause background refresh. + +Optional hardening: `allowed_channel_domains` suffix-matches every registry config `channel.url` host at bundle load time. + ### Boot behavior `Registry::Sync.boot!` runs during app boot (see `app/web/boot/setup.rb`): @@ -328,6 +333,9 @@ registries: official: sync: channel: html2rss-official + pin_version: v2026.08.22 # optional + max_version: v2026.08.22 # optional incident freeze + auto_promote: false # default; verified bundles stage until --promote catalog: true public_key_id: html2rss:registry:2026 public_key: | @@ -348,6 +356,9 @@ registries: - **`precedence`** — merge order for feed lookup; first match wins. - **`sync.url`** — direct tarball URL, or use `sync.channel: html2rss-official` for the default GitHub release asset. +- **`sync.pin_version` / `sync.max_version`** — fetch a specific tag or reject manifests newer than the cap. +- **`auto_promote: false`** — default; verified bundles land in `REGISTRY_DATA_ROOT//.staging/` until `bin/registry-sync --promote`. +- **`allowed_channel_domains`** — optional suffix allowlist enforced when the bundle loads. - **`catalog: false`** — private registry: configs are served at `/{registry.id}.rss` but excluded from the public catalog API (privacy for internal feeds). - Restrict outbound hosts with `REGISTRY_SYNC_ALLOWED_HOSTS` (comma-separated hostnames). @@ -368,7 +379,7 @@ The directory must contain `manifest.json`, optional `manifest.sig`, and `config 1. Publish a new bundle signed with the new key (`public_key_id` in `manifest.json`). 2. Update `public_key_id` and `public_key` in `registries.yml` on every instance **before** or together with the first release that requires the new key. -3. Re-sync (`bin/registry-sync` or wait for background refresh). Failed verification leaves the previous bundle active and records `last_error`. +3. Re-sync (`bin/registry-sync`) and promote when `auto_promote: false` (`bin/registry-sync --promote`). Failed verification leaves the previous bundle active and records `last_error`. --- diff --git a/docs/registry-go-live.md b/docs/registry-go-live.md index c79a907d4..eebc337a7 100644 --- a/docs/registry-go-live.md +++ b/docs/registry-go-live.md @@ -37,12 +37,14 @@ Source of truth for feed YAML and signed release artifacts. ``` 3. Tag the release (tag name becomes `manifest.json` `version` in CI via `REGISTRY_VERSION: ${{ github.ref_name }}`): ```bash - git tag 2026.08.22 # example; use your release tag - git push origin 2026.08.22 + git tag v2026.08.22 # example; use your release tag + git push origin v2026.08.22 ``` 4. Tag push triggers [`.github/workflows/release.yml`](../../html2rss-configs/.github/workflows/release.yml): - `make registry-build -- --sign` - - uploads `dist/registry-bundle.tar.gz` to the GitHub Release + - uploads `dist/registry-bundle.tar.gz` to a **draft** GitHub Release (human publishes after review) + +**Draft → publish:** CI creates a draft release. Maintainers review the asset, release notes, and tag diff, then click **Publish release** in GitHub. Draft releases are invisible to `/releases/latest` until published. ### 1.3 `html2rss-web` (runtime + Docker image) @@ -71,17 +73,27 @@ openssl pkey -in registry-signing.pem -pubout -out registry-signing.pub Keep the private key offline except where signing happens. -### 2.2 Configure `html2rss-configs` CI secret +### 2.2 Configure `html2rss-configs` CI secrets -1. In the `html2rss-configs` GitHub repo, add secret **`REGISTRY_SIGNING_KEY`** with the full PEM contents of `registry-signing.pem`. -2. Release CI reads it here: +In the `html2rss-configs` GitHub repo, add secrets on the **`registry-release`** environment: - ```yaml - env: - REGISTRY_VERSION: ${{ github.ref_name }} - REGISTRY_SIGNING_KEY: ${{ secrets.REGISTRY_SIGNING_KEY }} - run: make registry-build -- --sign - ``` +| Secret | Purpose | +| --- | --- | +| `REGISTRY_SIGNING_KEY` | Full PEM contents of `registry-signing.pem` (private key; used by `make registry-build -- --sign`) | +| `REGISTRY_PUBLIC_KEY_PEM` | Full PEM contents of `registry-signing.pub` (public key; used by the release verify step — not a repo fixture) | + +Release CI reads them here: + +```yaml +env: + REGISTRY_VERSION: ${{ github.ref_name }} + REGISTRY_SIGNING_KEY: ${{ secrets.REGISTRY_SIGNING_KEY }} +run: make registry-build -- --sign +``` + +The verify step loads the public key from `REGISTRY_PUBLIC_KEY_PEM` in-process (`OpenSSL::PKey.read(ENV.fetch('REGISTRY_PUBLIC_KEY_PEM'))`); no committed PEM file in the configs repo. + +At go-live, the same public key must appear in `html2rss-web/config/registries.yml` (`public_key` / `public_key_id`) so runtime sync verification matches CI. ### 2.3 Pin the public key in `html2rss-web` @@ -92,16 +104,26 @@ registries: official: sync: channel: html2rss-official + pin_version: v2026.08.22 # optional: fetch this tag only + max_version: v2026.08.22 # optional: reject newer manifests (incident freeze) + auto_promote: false # default false; verified bundles stage until manual promote catalog: true public_key_id: html2rss:registry:2026 public_key: | -----BEGIN PUBLIC KEY----- ... -----END PUBLIC KEY----- + allowed_channel_domains: # optional suffix allowlist for channel.url domains + - anthropic.com + - github.com ``` - `public_key_id` must match the value in signed `manifest.json` (default in `tool/registry-build`: `html2rss:registry:2026`). - Network sync uses `:signed` trust and requires this pin. Seed bundles copied from the Docker image use `:integrity_only` trust (no signature check on load). +- **`auto_promote: false`** (default) writes verified bundles to `REGISTRY_DATA_ROOT//.staging/` without changing the active catalog. Promote after review with `bin/registry-sync --promote --registry official`. +- **`pin_version`** resolves the GitHub tag release API instead of `/releases/latest`. +- **`max_version`** rejects sync when the verified manifest version is newer than the cap (incident freeze). +- **`allowed_channel_domains`** rejects bundle load when any config `channel.url` host is outside the suffix allowlist. ### 2.4 Tag and publish the configs release @@ -311,16 +333,17 @@ Inside the running container (or Dev Container): bin/registry-sync --status ``` -Tab-separated columns: `registry`, `mode`, `version`, `updated_at`, `sync_url`, `last_error`. +Tab-separated columns: `registry`, `mode`, `version`, `staged_version`, `updated_at`, `sync_url`, `last_error`. - Exit code **0** — all sync-mode registries have a usable on-disk bundle. - Exit code **1** — at least one sync registry lacks a bundle (see `Registry::Sync.unusable_sync_registries`). -Single registry or dry-run: +Single registry, dry-run, or promote staged bundle: ```bash bin/registry-sync --registry official bin/registry-sync --registry official --dry-run +bin/registry-sync --promote --registry official ``` ### 6.2 Instance metadata API @@ -360,15 +383,34 @@ Expect HTTP **200** and valid RSS XML. Configs releases are independent of web image releases. To refresh feeds on a running instance: -### 7.1 Manual sync +### 7.1 Manual sync and promote + +Production recommendation: keep `auto_promote: false`, set `sync.pin_version` to the approved tag, fetch with sync, then promote manually after review. ```bash bin/registry-sync --registry official +bin/registry-sync --status # staged_version shows verified bundle +bin/registry-sync --promote --registry official +``` + +Fetches the pinned or latest signed tarball, verifies signature + digests, and either stages (`auto_promote: false`) or atomically swaps the active bundle (`auto_promote: true`). + +### 7.2 Incident freeze + +To block uptake of a newer configs release without disabling sync entirely: + +```yaml +registries: + official: + sync: + channel: html2rss-official + max_version: v2026.08.21 + auto_promote: false ``` -Fetches the latest signed tarball, verifies signature + digests, atomically swaps the active bundle, and reloads `Registry::Index`. +Set `REGISTRY_SYNC_INTERVAL_HOURS=0` to pause background refresh while investigating. -### 7.2 Automatic refresh +### 7.3 Automatic refresh | Mechanism | Env var | Default | Behavior | | --- | --- | --- | --- | diff --git a/frontend/src/api/generated/types.gen.ts b/frontend/src/api/generated/types.gen.ts index 2939a0d7b..395c56397 100644 --- a/frontend/src/api/generated/types.gen.ts +++ b/frontend/src/api/generated/types.gen.ts @@ -13,7 +13,7 @@ export type GetApiMetadataData = { export type GetApiMetadataResponses = { /** - * returns catalog pointer metadata + * returns registry status metadata */ 200: { data: { @@ -31,6 +31,12 @@ export type GetApiMetadataResponses = { access_token_required: boolean; enabled: boolean; }; + registries: Array<{ + id: string; + sync_mode: string; + updated_at: string; + version: string; + }>; }; }; success: boolean; @@ -77,39 +83,14 @@ export type GetConfigCatalogResponses = { id: string; parameters: { defaults: { - blog?: string | null; - id?: string | null; - region?: string | null; - repository?: string | null; - section?: string | null; - user_id?: string | null; - username?: string | null; + [key: string]: unknown; }; schema: { - blog?: { - type: string; - } | null; - id?: { - type: string; - } | null; - region?: { - type: string; - } | null; - repository?: { - type: string; - } | null; - section?: { - type: string; - } | null; - user_id?: { - type: string; - } | null; - username?: { - type: string; - } | null; + [key: string]: unknown; }; }; path: string; + registry?: string; source: string; }>; }; diff --git a/public/openapi.yaml b/public/openapi.yaml index 6f9164eb9..8e8e54cac 100644 --- a/public/openapi.yaml +++ b/public/openapi.yaml @@ -173,7 +173,6 @@ paths: - channel - parameters - source - - registry type: object type: array required: diff --git a/spec/fixtures/registries/keys/test-key.pem b/spec/fixtures/registries/keys/test-key.pem new file mode 100644 index 000000000..d1583ef45 --- /dev/null +++ b/spec/fixtures/registries/keys/test-key.pem @@ -0,0 +1,3 @@ +-----BEGIN PRIVATE KEY----- +MC4CAQAwBQYDK2VwBCIEIKWA7CdQvmMCa06H6jojjHE30xRV9Ps823kjgqWKvfSE +-----END PRIVATE KEY----- diff --git a/spec/fixtures/registries/keys/test-key.pub b/spec/fixtures/registries/keys/test-key.pub new file mode 100644 index 000000000..236aab386 --- /dev/null +++ b/spec/fixtures/registries/keys/test-key.pub @@ -0,0 +1,3 @@ +-----BEGIN PUBLIC KEY----- +MCowBQYDK2VwAyEAiMbg/04MyC5azBdM/aeY0mNuA8JbP5/jOiNRwJ2KJHE= +-----END PUBLIC KEY----- diff --git a/spec/fixtures/registries/official/configs/phys.org/weekly.yml b/spec/fixtures/registries/official/configs/phys.org/weekly.yml index 763f70f2b..4c0102d03 100644 --- a/spec/fixtures/registries/official/configs/phys.org/weekly.yml +++ b/spec/fixtures/registries/official/configs/phys.org/weekly.yml @@ -1,3 +1,5 @@ +registry: + id: phys.org/weekly directory: topics: - science diff --git a/spec/fixtures/registries/official/configs/support.apple.com/en_gb_ht201222.yml b/spec/fixtures/registries/official/configs/support.apple.com/en_gb_ht201222.yml index 76afeafbe..83c1c81b6 100644 --- a/spec/fixtures/registries/official/configs/support.apple.com/en_gb_ht201222.yml +++ b/spec/fixtures/registries/official/configs/support.apple.com/en_gb_ht201222.yml @@ -1,3 +1,5 @@ +registry: + id: support.apple.com/en_gb_ht201222 directory: topics: - tech diff --git a/spec/fixtures/registries/official/manifest.json b/spec/fixtures/registries/official/manifest.json index 17d99eece..d02f057ad 100644 --- a/spec/fixtures/registries/official/manifest.json +++ b/spec/fixtures/registries/official/manifest.json @@ -4,7 +4,7 @@ "version": "test-fixture", "public_key_id": "test", "files": { - "configs/phys.org/weekly.yml": "0f2a052f5c6da5c260985058302805b962da3211b635483667e258c714b31373", - "configs/support.apple.com/en_gb_ht201222.yml": "3a6f84899ac24472f6476c697d50ac366425ae1c87f7ddf045cc9e2c80e93eac" + "configs/phys.org/weekly.yml": "0e05fa9a95ec56bef4b4363b2f044ab69642fcda9acd91ebf8d28b39141a963d", + "configs/support.apple.com/en_gb_ht201222.yml": "53e38a6b7d088e0c79b19a7dd6db9b009c6b5c0b6690f504c7160b0c32b71e41" } } diff --git a/spec/fixtures/registries/private/configs/secret.example/private.yml b/spec/fixtures/registries/private/configs/secret.example/private.yml index 6b516f6c2..14a119e7f 100644 --- a/spec/fixtures/registries/private/configs/secret.example/private.yml +++ b/spec/fixtures/registries/private/configs/secret.example/private.yml @@ -1,3 +1,5 @@ +registry: + id: secret.example/private directory: title: "Private Corp Feed" summary: "Hidden from catalog." diff --git a/spec/fixtures/registries/private/manifest.json b/spec/fixtures/registries/private/manifest.json index 0aacc3ef5..6d901dbbf 100644 --- a/spec/fixtures/registries/private/manifest.json +++ b/spec/fixtures/registries/private/manifest.json @@ -1,9 +1,9 @@ { "format": "registry.v1", "registry_id": "private", - "version": "test-private", + "version": "test-fixture", "public_key_id": "test", "files": { - "configs/secret.example/private.yml": "196760640262a019c0b32e08d2f5272c08b11b10b5880dc13266922a30c872a3" + "configs/secret.example/private.yml": "f3b4f6d567e44960be89c171a915a5451266998090ca9ffdb394050bcbbd0dd2" } } diff --git a/spec/fixtures/registries/sync/bundle/configs/phys.org/weekly.yml b/spec/fixtures/registries/sync/bundle/configs/phys.org/weekly.yml index 763f70f2b..4c0102d03 100644 --- a/spec/fixtures/registries/sync/bundle/configs/phys.org/weekly.yml +++ b/spec/fixtures/registries/sync/bundle/configs/phys.org/weekly.yml @@ -1,3 +1,5 @@ +registry: + id: phys.org/weekly directory: topics: - science diff --git a/spec/fixtures/registries/sync/bundle/configs/support.apple.com/en_gb_ht201222.yml b/spec/fixtures/registries/sync/bundle/configs/support.apple.com/en_gb_ht201222.yml index 76afeafbe..83c1c81b6 100644 --- a/spec/fixtures/registries/sync/bundle/configs/support.apple.com/en_gb_ht201222.yml +++ b/spec/fixtures/registries/sync/bundle/configs/support.apple.com/en_gb_ht201222.yml @@ -1,3 +1,5 @@ +registry: + id: support.apple.com/en_gb_ht201222 directory: topics: - tech diff --git a/spec/fixtures/registries/sync/bundle/manifest.json b/spec/fixtures/registries/sync/bundle/manifest.json index e74570c9a..4e20cdcbf 100644 --- a/spec/fixtures/registries/sync/bundle/manifest.json +++ b/spec/fixtures/registries/sync/bundle/manifest.json @@ -1 +1,10 @@ -{"format":"registry.v1","registry_id":"official","version":"test-fixture","public_key_id":"test-key","files":{"configs/phys.org/weekly.yml":"0f2a052f5c6da5c260985058302805b962da3211b635483667e258c714b31373","configs/support.apple.com/en_gb_ht201222.yml":"3a6f84899ac24472f6476c697d50ac366425ae1c87f7ddf045cc9e2c80e93eac"}} \ No newline at end of file +{ + "format": "registry.v1", + "registry_id": "official", + "version": "test-fixture", + "public_key_id": "test-key", + "files": { + "configs/phys.org/weekly.yml": "0e05fa9a95ec56bef4b4363b2f044ab69642fcda9acd91ebf8d28b39141a963d", + "configs/support.apple.com/en_gb_ht201222.yml": "53e38a6b7d088e0c79b19a7dd6db9b009c6b5c0b6690f504c7160b0c32b71e41" + } +} diff --git a/spec/fixtures/registries/sync/registries.yml b/spec/fixtures/registries/sync/registries.yml index e307db29a..4f1ab3dff 100644 --- a/spec/fixtures/registries/sync/registries.yml +++ b/spec/fixtures/registries/sync/registries.yml @@ -5,6 +5,7 @@ registries: official: sync: url: https://registry.test.example/registry-bundle.tar.gz + auto_promote: true catalog: true public_key_id: test-key public_key: | diff --git a/spec/html2rss/web/registry/config_spec.rb b/spec/html2rss/web/registry/config_spec.rb new file mode 100644 index 000000000..e5edd9788 --- /dev/null +++ b/spec/html2rss/web/registry/config_spec.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +require 'fileutils' +require 'climate_control' +require 'spec_helper' + +require_relative '../../../../app' + +RSpec.describe Html2rss::Web::Registry::Config do + describe '.entry' do + let(:config_path) { File.join(Dir.pwd, 'tmp', 'missing-public-key-registries.yml') } + + before do + FileUtils.mkdir_p(File.dirname(config_path)) + File.write(config_path, <<~YAML) + precedence: + - official + registries: + official: + sync: + url: https://registry.test.example/registry-bundle.tar.gz + catalog: true + YAML + end + + after do + FileUtils.rm_f(config_path) + end + + it 'requires a pinned public key for sync-mode registries' do + ClimateControl.modify('REGISTRIES_CONFIG' => config_path) do + expect do + described_class.reload! + described_class.entry('official') + end.to raise_error(Html2rss::Web::Registry::Errors::ConfigError, /requires a pinned public_key/) + end + end + end + + describe 'sync policy parsing' do + let(:config_path) { File.join(Dir.pwd, 'tmp', 'sync-policy-registries.yml') } + + before do + FileUtils.mkdir_p(File.dirname(config_path)) + File.write(config_path, <<~YAML) + precedence: + - official + registries: + official: + sync: + channel: html2rss-official + pin_version: v2026.08.22 + max_version: v2026.08.21 + auto_promote: true + allowed_channel_domains: + - phys.org + catalog: true + public_key_id: html2rss:registry:2026 + public_key: | + -----BEGIN PUBLIC KEY----- + MCowBQYDK2VwAyEAiMbg/04MyC5azBdM/aeY0mNuA8JbP5/jOiNRwJ2KJHE= + -----END PUBLIC KEY----- + YAML + end + + after do + FileUtils.rm_f(config_path) + end + + it 'parses sync policy and domain allowlist fields', :aggregate_failures do + ClimateControl.modify('REGISTRIES_CONFIG' => config_path) do + described_class.reload! + entry = described_class.entry('official') + + expect(entry.sync_policy).to have_attributes( + pin_version: 'v2026.08.22', + max_version: 'v2026.08.21', + auto_promote: true + ) + expect(entry.allowed_channel_domains).to eq(['phys.org']) + end + end + + it 'defaults auto_promote to false for security' do + ClimateControl.modify('REGISTRIES_CONFIG' => nil) do + described_class.reload! + entry = described_class.entry('official') + + expect(entry.sync_policy.auto_promote).to be(false) + end + end + end +end diff --git a/spec/html2rss/web/registry/index_spec.rb b/spec/html2rss/web/registry/index_spec.rb index 3b9cf095b..cf8d91c1f 100644 --- a/spec/html2rss/web/registry/index_spec.rb +++ b/spec/html2rss/web/registry/index_spec.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +require 'fileutils' require 'spec_helper' require_relative '../../../../app' @@ -40,6 +41,13 @@ expect(rows.map { |row| row.fetch(:id) }).not_to include('secret.example/private') end + + it 'prefers the first registry in precedence for duplicate feed ids' do + rows = described_class.current.catalog_rows + apple = rows.find { |row| row.fetch(:id) == 'support.apple.com/en_gb_ht201222' } + + expect(apple.fetch(:registry)).to eq('official') + end end describe '#status' do @@ -53,4 +61,33 @@ ) end end + + describe 'allowed_channel_domains' do + let(:config_path) { File.join(Dir.pwd, 'tmp', 'domain-allowlist-registries.yml') } + + before do + FileUtils.mkdir_p(File.dirname(config_path)) + File.write(config_path, <<~YAML) + precedence: + - official + registries: + official: + path: spec/fixtures/registries/official + catalog: true + allowed_channel_domains: + - blocked.example + YAML + ENV['REGISTRIES_CONFIG'] = config_path + described_class.reload! + end + + after do + FileUtils.rm_f(config_path) + end + + it 'rejects bundles with channel URLs outside the allowlist' do + expect { described_class.current.config_for('phys.org/weekly') } + .to raise_error(Html2rss::Web::Registry::Errors::LoadError, /phys.org/) + end + end end diff --git a/spec/html2rss/web/registry/scrape_policy_spec.rb b/spec/html2rss/web/registry/scrape_policy_spec.rb new file mode 100644 index 000000000..681e88cd8 --- /dev/null +++ b/spec/html2rss/web/registry/scrape_policy_spec.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +require 'fileutils' +require 'climate_control' +require 'spec_helper' + +require_relative '../../../../app' + +RSpec.describe Html2rss::Web::Registry::ScrapePolicy do + let(:allowed_entry) do + Html2rss::Web::Registry::Entry.new( + id: 'official', + mode: :path, + path: nil, + sync_channel: nil, + sync_url: nil, + catalog: true, + public_key_id: 'test', + public_key: nil, + sync_policy: Html2rss::Web::Registry::SyncPolicy.new(nil, nil, false), + allowed_channel_domains: ['phys.org'] + ) + end + + let(:allowed_bundle) do + Html2rss::Web::Registry::Index::RegistryBundle.new( + registry_id: 'official', + manifest: instance_double(Html2rss::Registry::Manifest, version: '1.0.0'), + configs: { + 'phys.org/weekly' => { channel: { url: 'https://phys.org/weekly-news/' } } + }, + catalog_entries: [] + ) + end + + let(:blocked_bundle) do + Html2rss::Web::Registry::Index::RegistryBundle.new( + registry_id: 'official', + manifest: instance_double(Html2rss::Registry::Manifest, version: '1.0.0'), + configs: { + 'phys.org/weekly' => { channel: { url: 'https://phys.org/weekly-news/' } }, + 'blocked.example/feed' => { channel: { url: 'https://blocked.example/private' } } + }, + catalog_entries: [] + ) + end + + it 'allows suffix-matching channel domains' do + expect { described_class.enforce!(allowed_entry, allowed_bundle) }.not_to raise_error + end + + it 'raises when a config channel domain is not allowlisted' do + expect { described_class.enforce!(allowed_entry, blocked_bundle) } + .to raise_error(Html2rss::Web::Registry::Errors::LoadError, /blocked.example/) + end +end diff --git a/spec/html2rss/web/registry/store_spec.rb b/spec/html2rss/web/registry/store_spec.rb new file mode 100644 index 000000000..8a1d92508 --- /dev/null +++ b/spec/html2rss/web/registry/store_spec.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +require 'fileutils' +require 'json' +require 'digest' +require 'spec_helper' + +require_relative '../../../../app' + +RSpec.describe Html2rss::Web::Registry::Store do + let(:registry_id) { 'store-test' } + let(:data_root) { File.join(Dir.pwd, 'tmp', 'store-spec-data') } + + before do + ENV['REGISTRY_DATA_ROOT'] = data_root + FileUtils.rm_rf(data_root) + end + + describe '.stage_bundle! and .promote_staged!' do + it 'stages a verified bundle and promotes it to active', :aggregate_failures do + source = build_bundle_dir('stage-source') + active_before = build_bundle_dir('active-before') + + FileUtils.mkdir_p(File.dirname(described_class.registry_dir(registry_id))) + FileUtils.cp_r(active_before, described_class.registry_dir(registry_id)) + + described_class.stage_bundle!(registry_id, source) + + expect(described_class.staged_present?(registry_id)).to be(true) + expect(described_class.staged_version(registry_id)).to eq('stage-source') + expect(described_class.bundle_present?(registry_id)).to be(true) + expect(read_manifest_version(described_class.registry_dir(registry_id))).to eq('active-before') + + described_class.promote_staged!(registry_id) + + expect(described_class.staged_present?(registry_id)).to be(false) + expect(read_manifest_version(described_class.registry_dir(registry_id))).to eq('stage-source') + end + end + + describe '.bundle_present?' do + it 'requires manifest.json instead of any non-empty directory' do + path = described_class.registry_dir(registry_id) + + FileUtils.rm_rf(path) + FileUtils.mkdir_p(path) + File.write(File.join(path, 'placeholder.txt'), 'seed') + + expect(described_class.bundle_present?(registry_id)).to be(false) + + File.write(File.join(path, Html2rss::Registry::Manifest::MANIFEST_FILE), '{}') + expect(described_class.bundle_present?(registry_id)).to be(true) + end + end + + def build_bundle_dir(version) # rubocop:disable Metrics/MethodLength + dir = Dir.mktmpdir("registry-store-#{version}") + config_path = File.join(dir, 'configs', 'example.com', 'feed.yml') + FileUtils.mkdir_p(File.dirname(config_path)) + File.write(config_path, "channel:\n url: https://example.com/\n") + digest = Digest::SHA256.file(config_path).hexdigest + File.write( + File.join(dir, Html2rss::Registry::Manifest::MANIFEST_FILE), + { + format: 'registry.v1', + registry_id: 'store-test', + version:, + public_key_id: 'test', + files: { + 'configs/example.com/feed.yml' => digest + } + }.to_json + ) + dir + end + + def read_manifest_version(path) + JSON.parse(File.read(File.join(path, Html2rss::Registry::Manifest::MANIFEST_FILE))).fetch('version') + end +end diff --git a/spec/html2rss/web/registry/sync_spec.rb b/spec/html2rss/web/registry/sync_spec.rb index 4ff1b8877..0630807ec 100644 --- a/spec/html2rss/web/registry/sync_spec.rb +++ b/spec/html2rss/web/registry/sync_spec.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'fileutils' +require 'climate_control' require 'spec_helper' require 'webmock/rspec' @@ -43,12 +45,36 @@ end.not_to(change { Html2rss::Web::Registry::Store.bundle_present?('official') }) end - it 'rejects redirects' do + it 'follows bounded redirects to allowed CDN hosts', :aggregate_failures do + cdn_url = 'https://release-assets.githubusercontent.com/registry-bundle.tar.gz' + stub_request(:get, download_url) + .to_return(status: 302, headers: { 'Location' => cdn_url }) + stub_request(:get, cdn_url) + .to_return(status: 200, body: tarball, headers: { 'Content-Type' => 'application/octet-stream' }) + + status = described_class.run(registry_id: 'official') + + expect(status.version).to eq('test-fixture') + expect(Html2rss::Web::Registry::Store.bundle_present?('official')).to be(true) + end + + it 'rejects redirects to disallowed hosts' do stub_request(:get, download_url) .to_return(status: 302, headers: { 'Location' => 'https://evil.example/bundle.tar.gz' }) expect { described_class.run(registry_id: 'official') } - .to raise_error(Html2rss::Web::Registry::Errors::SyncError, /redirect/i) + .to raise_error(Html2rss::Web::Registry::Errors::SyncError, /host not allowed/i) + end + + it 'rejects excessive redirect chains' do + (1..6).each do |hop| + from = hop == 1 ? download_url : "#{download_url}?hop=#{hop - 1}" + to = "#{download_url}?hop=#{hop}" + stub_request(:get, from).to_return(status: 302, headers: { 'Location' => to }) + end + + expect { described_class.run(registry_id: 'official') } + .to raise_error(Html2rss::Web::Registry::Errors::SyncError, /redirect limit/i) end it 'logs signature verification failures to the security logger' do @@ -86,8 +112,82 @@ expect(row).to have_attributes( registry_id: 'official', mode: :sync, - sync_url: 'https://registry.test.example/registry-bundle.tar.gz' + sync_url: 'https://registry.test.example/registry-bundle.tar.gz', + staged_version: nil + ) + end + end + + describe 'sync policy', :registry_sync do + let(:download_url) { 'https://registry.test.example/registry-bundle.tar.gz' } + let(:tarball) { RegistrySyncTestHelpers.build_signed_tarball } + let(:policy_config_path) { File.join(Dir.pwd, 'tmp', 'sync-policy-official.yml') } + + before do + FileUtils.mkdir_p(File.dirname(policy_config_path)) + stub_request(:get, download_url) + .to_return(status: 200, body: tarball, headers: { 'Content-Type' => 'application/octet-stream' }) + end + + after do + FileUtils.rm_f(policy_config_path) + end + + def write_policy_config(yaml) + File.write(policy_config_path, yaml) + ENV['REGISTRIES_CONFIG'] = policy_config_path + Html2rss::Web::Registry::Index.reload! + end + + it 'stages verified bundles when auto_promote is false', :aggregate_failures do + write_policy_config( + RegistrySyncTestHelpers.policy_registry_yaml( + download_url:, + auto_promote: false, + sync_extra: { max_version: 'test-fixture' } + ) + ) + + status = described_class.run(registry_id: 'official') + + expect(Html2rss::Web::Registry::Store.staged_present?('official')).to be(true) + expect(Html2rss::Web::Registry::Store.bundle_present?('official')).to be(false) + expect(status.staged_version).to eq('test-fixture') + end + + it 'promotes staged bundles and emits catalog change telemetry', :aggregate_failures do # rubocop:disable RSpec/ExampleLength + allow(Html2rss::Web::Observability).to receive(:emit) + allow(Html2rss::Web::SecurityLogger).to receive(:log_registry_catalog_changed) + + write_policy_config( + RegistrySyncTestHelpers.policy_registry_yaml(download_url:, auto_promote: false) ) + + described_class.run(registry_id: 'official') + status = described_class.promote_staged!(registry_id: 'official') + + expect(status.version).to eq('test-fixture') + expect(Html2rss::Web::Registry::Store.staged_present?('official')).to be(false) + expect(Html2rss::Web::Observability).to have_received(:emit).with( + hash_including(event_name: 'registry.promote_staged', outcome: 'success') + ) + expect(Html2rss::Web::Observability).to have_received(:emit).with( + hash_including(event_name: 'registry.catalog_changed', outcome: 'success') + ) + expect(Html2rss::Web::SecurityLogger).to have_received(:log_registry_catalog_changed) + end + + it 'rejects manifests newer than max_version' do + write_policy_config( + RegistrySyncTestHelpers.policy_registry_yaml( + download_url:, + auto_promote: true, + sync_extra: { max_version: '0.0.1' } + ) + ) + + expect { described_class.run(registry_id: 'official') } + .to raise_error(Html2rss::Web::Registry::Errors::SyncError, /exceeds max_version/) end end diff --git a/spec/html2rss/web/registry/sync_url_resolver_spec.rb b/spec/html2rss/web/registry/sync_url_resolver_spec.rb new file mode 100644 index 000000000..1736a8e45 --- /dev/null +++ b/spec/html2rss/web/registry/sync_url_resolver_spec.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'webmock/rspec' + +require_relative '../../../../app' + +RSpec.describe Html2rss::Web::Registry::SyncUrlResolver do + describe '.resolve' do + it 'uses the GitHub tag release API when pin_version is set', :aggregate_failures do # rubocop:disable RSpec/ExampleLength + tag_api = format( + described_class::OFFICIAL_GITHUB_TAG_RELEASES_API, + tag: 'v2026.08.22' + ) + download_url = 'https://release-assets.githubusercontent.com/registry-bundle.tar.gz' + stub_request(:get, tag_api).to_return( + status: 200, + body: { + assets: [{ name: described_class::OFFICIAL_ASSET_NAME, browser_download_url: download_url }] + }.to_json + ) + + entry = Html2rss::Web::Registry::Entry.new( + id: 'official', + mode: :sync, + path: nil, + sync_channel: Html2rss::Web::Registry::Config::DEFAULT_OFFICIAL_SYNC_CHANNEL, + sync_url: nil, + catalog: true, + public_key_id: 'html2rss:registry:2026', + public_key: nil, + sync_policy: Html2rss::Web::Registry::SyncPolicy.new('v2026.08.22', nil, false), + allowed_channel_domains: [] + ) + + expect(described_class.resolve(entry)).to eq(download_url) + end + end +end diff --git a/spec/html2rss/web/registry/version_gate_spec.rb b/spec/html2rss/web/registry/version_gate_spec.rb new file mode 100644 index 000000000..09b7d43dd --- /dev/null +++ b/spec/html2rss/web/registry/version_gate_spec.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +require 'spec_helper' + +require_relative '../../../../app' + +RSpec.describe Html2rss::Web::Registry::VersionGate do + it 'compares dotted numeric versions' do + expect(described_class.exceeds_max?('2026.08.22', '2026.08.21')).to be(true) + expect(described_class.exceeds_max?('2026.08.21', '2026.08.22')).to be(false) + end + + it 'strips a leading v prefix before comparing' do + expect(described_class.exceeds_max?('v2026.08.22', '2026.08.21')).to be(true) + end + + it 'returns false when max_version is unset' do + expect(described_class.exceeds_max?('2026.08.22', nil)).to be(false) + end +end diff --git a/spec/smoke/docker_spec.rb b/spec/smoke/docker_spec.rb index 115d2b76b..c056a2624 100644 --- a/spec/smoke/docker_spec.rb +++ b/spec/smoke/docker_spec.rb @@ -112,4 +112,13 @@ def expect_json_feed_response(path) expect(body.dig('error', 'code')).to eq('FORBIDDEN') expect(body.dig('error', 'message')).to eq('Auto source feature is disabled') end + + it 'exposes the config catalog without authentication', :aggregate_failures do + response, payload = get_json('/api/v1/configs') + + expect(response).to be_a(Net::HTTPOK) + expect(payload.fetch('success')).to be(true) + expect(payload.dig('data', 'configs')).to be_an(Array) + expect(payload.dig('meta', 'catalog_version')).to eq(1) + end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 4a7811d74..383238f13 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -8,13 +8,17 @@ require 'simplecov' SimpleCov.start do - add_filter '/spec/' - add_filter '/config/' + enable_coverage :branch + primary_coverage :branch - track_files '**/*.rb' + add_group 'App', 'app' + add_group 'Config', 'config' - minimum_coverage 80 unless ENV['OPENAPI'] - maximum_coverage_drop 5 unless ENV['OPENAPI'] + skip_filter %r{/spec/} + skip_filter %r{/config/} + + minimum_coverage line: 80, branch: 70 unless ENV['OPENAPI'] + maximum_coverage_drop line: 5, branch: 5 unless ENV['OPENAPI'] end end diff --git a/spec/support/openapi.rb b/spec/support/openapi.rb index 30ea6e7e1..13fa7edf0 100644 --- a/spec/support/openapi.rb +++ b/spec/support/openapi.rb @@ -233,5 +233,11 @@ else spec[:tags] = tags end + + paths = spec['paths'] || spec[:paths] + catalog_configs_required = paths&.dig('/configs', 'get', 'responses', '200', 'content', + 'application/json', 'schema', 'properties', 'data', 'properties', 'configs', + 'items', 'required') + catalog_configs_required&.delete('registry') if catalog_configs_required.is_a?(Array) end end diff --git a/spec/support/registry_sync.rb b/spec/support/registry_sync.rb index b0f6a1dca..839d9e13a 100644 --- a/spec/support/registry_sync.rb +++ b/spec/support/registry_sync.rb @@ -1,15 +1,14 @@ # frozen_string_literal: true -require 'openssl' require 'fileutils' -require 'zlib' +require 'openssl' require 'stringio' -require 'rubygems/package' +require 'yaml' module RegistrySyncTestHelpers - HTML2RSS_FIXTURES = File.expand_path('../../../html2rss/spec/fixtures/registry', __dir__) - TEST_PUBLIC_KEY = File.read(File.join(HTML2RSS_FIXTURES, 'test-key.pub')) - TEST_PRIVATE_KEY = File.read(File.join(HTML2RSS_FIXTURES, 'test-key.pem')) + FIXTURE_KEYS_ROOT = File.expand_path('../fixtures/registries/keys', __dir__) + TEST_PUBLIC_KEY = File.read(File.join(FIXTURE_KEYS_ROOT, 'test-key.pub')) + TEST_PRIVATE_KEY = File.read(File.join(FIXTURE_KEYS_ROOT, 'test-key.pem')) SYNC_FIXTURES_ROOT = File.expand_path('../fixtures/registries/sync', __dir__) SYNC_REGISTRIES_CONFIG = File.join(SYNC_FIXTURES_ROOT, 'registries.yml') SYNC_DATA_ROOT = File.join(Dir.pwd, 'tmp', 'sync-registry-data') @@ -19,39 +18,57 @@ module RegistrySyncTestHelpers def configure_sync_registry! ENV['REGISTRIES_CONFIG'] = SYNC_REGISTRIES_CONFIG ENV['REGISTRY_DATA_ROOT'] = SYNC_DATA_ROOT - ENV['REGISTRY_SYNC_ALLOWED_HOSTS'] = 'registry.test.example' + ENV['REGISTRY_SYNC_ALLOWED_HOSTS'] = 'registry.test.example,release-assets.githubusercontent.com' FileUtils.rm_rf(SYNC_DATA_ROOT) Html2rss::Web::Registry::Index.reload! end - def build_signed_tarball # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def build_signed_tarball bundle_dir = Dir.mktmpdir('signed-registry-bundle') FileUtils.cp_r(File.join(SYNC_FIXTURES_ROOT, 'bundle', '.'), bundle_dir) manifest = Html2rss::Registry::Manifest.parse( File.read(File.join(bundle_dir, Html2rss::Registry::Manifest::MANIFEST_FILE)) ) - private_key = OpenSSL::PKey.read(TEST_PRIVATE_KEY) - signature = private_key.sign(nil, manifest.canonical_bytes) - File.write( - File.join(bundle_dir, Html2rss::Registry::Manifest::SIGNATURE_FILE), - [signature].pack('m0') - ) + sign_manifest!(manifest, key_pem: TEST_PRIVATE_KEY, bundle_dir:) - tar_io = StringIO.new - Gem::Package::TarWriter.new(tar_io) do |tar| - Dir.glob(File.join(bundle_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |path| - next if File.directory?(path) + pack_bundle_dir(bundle_dir) + ensure + FileUtils.rm_rf(bundle_dir) + end - relative = path.delete_prefix("#{bundle_dir}/") - tar.add_file(relative, 0o644) { |io| io.write(File.binread(path)) } - end - end + def pack_bundle_dir(bundle_dir) + dir = File.dirname(tarball_path = File.join(Dir.mktmpdir('registry-sync-tarball'), 'bundle.tar.gz')) + env = { 'COPYFILE_DISABLE' => '1' } + success = system(env, 'tar', '--format=ustar', '-czf', tarball_path, '-C', bundle_dir, '.', exception: false) + raise "Failed to pack registry test bundle from #{bundle_dir}" unless success - gz_io = StringIO.new - Zlib::GzipWriter.wrap(gz_io) { |gz| gz.write(tar_io.string) } - gz_io.string + File.binread(tarball_path) ensure - FileUtils.rm_rf(bundle_dir) + FileUtils.rm_rf(dir) if dir + end + + def policy_registry_yaml(download_url:, auto_promote:, sync_extra: {}) # rubocop:disable Metrics/MethodLength + sync = { 'url' => download_url }.merge(sync_extra.transform_keys(&:to_s)) + YAML.dump( + { + 'precedence' => ['official'], + 'registries' => { + 'official' => { + 'sync' => sync, + 'auto_promote' => auto_promote, + 'catalog' => true, + 'public_key_id' => 'test-key', + 'public_key' => TEST_PUBLIC_KEY + } + } + } + ) + end + + def sign_manifest!(manifest, key_pem:, bundle_dir:) + private_key = OpenSSL::PKey.read(key_pem) + signature = private_key.sign(nil, manifest.canonical_bytes) + File.write(File.join(bundle_dir, Html2rss::Registry::Manifest::SIGNATURE_FILE), [signature].pack('m0')) end end From 4ba73860d209fb9b2ca1b623f217e955ef19bf52 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 16:23:23 +0200 Subject: [PATCH 07/33] chore(deps): pin html2rss to registry hardening commit Align Gemfile.lock with feat/registry-v1 b32ec095 for signed bundle verify and manifest-only catalog APIs. --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 9f7e77533..24d62d03a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/html2rss/html2rss - revision: e32014ac089679cac7e91fd55f3bd85a0489771e + revision: b32ec095bd366075297f9b7f59cf55db6b665d2a branch: feat/registry-v1 specs: html2rss (0.28.0) From 8ff3bfabb946ce696147ca599e167805eb9fb83a Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 17:16:06 +0200 Subject: [PATCH 08/33] fix(ci): use SimpleCov add_filter and git html2rss pin Replace invalid skip_filter with add_filter so ruby and openapi CI jobs load spec_helper; keep html2rss on feat/registry-v1 until gem release. --- Gemfile | 2 +- spec/spec_helper.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index 8ff642c87..85e42a7e8 100644 --- a/Gemfile +++ b/Gemfile @@ -4,8 +4,8 @@ source 'https://rubygems.org' git_source(:github) { |repo_name| "https://github.com/#{repo_name}" } +# Until rubygems 0.28.0: git branch; local monorepo: BUNDLE_LOCAL__HTML2RSS=/path/to/html2rss gem 'html2rss', github: 'html2rss/html2rss', branch: 'feat/registry-v1' -# Local monorepo override: BUNDLE_LOCAL__HTML2RSS=/path/to/html2rss gem 'base64' gem 'rack-cache' diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 383238f13..25b4b1d90 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -14,8 +14,8 @@ add_group 'App', 'app' add_group 'Config', 'config' - skip_filter %r{/spec/} - skip_filter %r{/config/} + add_filter %r{/spec/} + add_filter %r{/config/} minimum_coverage line: 80, branch: 70 unless ENV['OPENAPI'] maximum_coverage_drop line: 5, branch: 5 unless ENV['OPENAPI'] From 45dc4f58610ac6810c87d57a76eb685b9a6a5b67 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 17:49:18 +0200 Subject: [PATCH 09/33] refactor(registry): fold sync satellites into sync_transport --- app/web/registry/catalog_change_reporter.rb | 96 ----------------- app/web/registry/config.rb | 2 +- app/web/registry/sync.rb | 101 +++++++++++++++--- app/web/registry/sync_coordinator.rb | 23 ---- .../{sync_fetcher.rb => sync_transport.rb} | 101 +++++++++++++++++- app/web/registry/sync_url_resolver.rb | 80 -------------- app/web/registry/version_gate.rb | 40 ------- spec/html2rss/web/registry/sync_spec.rb | 48 +++++++++ .../web/registry/sync_url_resolver_spec.rb | 39 ------- .../web/registry/version_gate_spec.rb | 20 ---- 10 files changed, 235 insertions(+), 315 deletions(-) delete mode 100644 app/web/registry/catalog_change_reporter.rb delete mode 100644 app/web/registry/sync_coordinator.rb rename app/web/registry/{sync_fetcher.rb => sync_transport.rb} (52%) delete mode 100644 app/web/registry/sync_url_resolver.rb delete mode 100644 app/web/registry/version_gate.rb delete mode 100644 spec/html2rss/web/registry/sync_url_resolver_spec.rb delete mode 100644 spec/html2rss/web/registry/version_gate_spec.rb diff --git a/app/web/registry/catalog_change_reporter.rb b/app/web/registry/catalog_change_reporter.rb deleted file mode 100644 index f218090c3..000000000 --- a/app/web/registry/catalog_change_reporter.rb +++ /dev/null @@ -1,96 +0,0 @@ -# frozen_string_literal: true - -module Html2rss - module Web - module Registry - ## - # Reports catalog diffs after a registry bundle is promoted to active. - module CatalogChangeReporter - module_function - - ## - # @param registry_id [String] - # @param previous_bundle [Index::RegistryBundle, nil] - # @param new_bundle [Index::RegistryBundle] - # @return [void] - def report!(registry_id:, previous_bundle:, new_bundle:) - diff = build_diff(previous_bundle, new_bundle) - return if diff.empty? - - emit_observability!(registry_id, diff) - emit_security!(registry_id, diff) if diff[:added].any? || diff[:removed].any? - end - - ## - # @param previous_bundle [Index::RegistryBundle, nil] - # @param new_bundle [Index::RegistryBundle] - # @return [Hash{Symbol => Object}] - def build_diff(previous_bundle, new_bundle) # rubocop:disable Metrics/MethodLength - previous_version = previous_bundle&.manifest&.version - new_version = new_bundle.manifest.version - previous_ids = catalog_ids(previous_bundle) - new_ids = catalog_ids(new_bundle) - added = new_ids - previous_ids - removed = previous_ids - new_ids - - return {} if previous_version == new_version && added.empty? && removed.empty? - - { - previous_version:, - version: new_version, - added:, - removed: - } - end - - ## - # @param registry_id [String] - # @param diff [Hash{Symbol => Object}] - # @return [void] - def emit_observability!(registry_id, diff) # rubocop:disable Metrics/MethodLength - Observability.emit( - event_name: 'registry.catalog_changed', - outcome: 'success', - level: :warn, - details: { - registry_id:, - version: diff[:version], - previous_version: diff[:previous_version], - added_count: diff[:added].size, - removed_count: diff[:removed].size, - added_ids: diff[:added].sort, - removed_ids: diff[:removed].sort - } - ) - end - - ## - # @param registry_id [String] - # @param diff [Hash{Symbol => Object}] - # @return [void] - def emit_security!(registry_id, diff) - SecurityLogger.log_registry_catalog_changed( - registry_id, - version: diff[:version], - previous_version: diff[:previous_version], - added_count: diff[:added].size, - removed_count: diff[:removed].size, - added_ids: diff[:added].sort, - removed_ids: diff[:removed].sort - ) - end - - ## - # @param bundle [Index::RegistryBundle, nil] - # @return [Array] - def catalog_ids(bundle) - return [] unless bundle - - bundle.catalog_entries.map(&:id) - end - - private_class_method :build_diff, :emit_observability!, :emit_security!, :catalog_ids - end - end - end -end diff --git a/app/web/registry/config.rb b/app/web/registry/config.rb index 065fcfae5..789bd07d6 100644 --- a/app/web/registry/config.rb +++ b/app/web/registry/config.rb @@ -194,7 +194,7 @@ def build_sync_entry(registry_id, raw, sync) def resolved_sync_url(sync_url, sync_channel) return sync_url if sync_url && !sync_url.empty? - SyncUrlResolver.resolve_channel_url(sync_channel) + SyncTransport.resolve_channel_url(sync_channel) end ## diff --git a/app/web/registry/sync.rb b/app/web/registry/sync.rb index f96c3448e..9c110e915 100644 --- a/app/web/registry/sync.rb +++ b/app/web/registry/sync.rb @@ -21,6 +21,7 @@ module Sync # rubocop:disable Metrics/ModuleLength @boot_mutex = Mutex.new @boot_started = false @timer_started = false + REGISTRY_MUTEXES = Hash.new { |mutexes, registry_id| mutexes[registry_id] = Mutex.new } class << self # rubocop:disable Metrics/ClassLength ## @@ -32,7 +33,7 @@ def sync_url_for(registry_id) entry = Config.entry(registry_id) raise Errors::SyncError, "Registry '#{registry_id}' is not sync-mode" unless entry.mode == :sync - SyncUrlResolver.resolve(entry) + SyncTransport.resolve(entry) end ## @@ -42,7 +43,7 @@ def sync_url_for(registry_id) # @param dry_run [Boolean] when true, verify without swapping the active bundle # @return [SyncStatus] def run(registry_id:, dry_run: false) - SyncCoordinator.run(registry_id) { run!(registry_id:, dry_run:) } + with_registry_lock(registry_id) { run!(registry_id:, dry_run:) } end ## @@ -51,7 +52,7 @@ def run(registry_id:, dry_run: false) # @param registry_id [String, Symbol] # @return [SyncStatus] def promote_staged!(registry_id:) - SyncCoordinator.run(registry_id) { promote_staged_bundle!(registry_id:) } + with_registry_lock(registry_id) { promote_staged_bundle!(registry_id:) } end ## @@ -129,6 +130,14 @@ def unusable_sync_registries private + ## + # @param registry_id [String, Symbol] + # @yield runs sync work exclusively for the registry id + # @return [Object] block result + def with_registry_lock(registry_id, &) + REGISTRY_MUTEXES[registry_id.to_s].synchronize(&) + end + ## # @param registry_id [String, Symbol] # @param dry_run [Boolean] @@ -140,7 +149,7 @@ def run!(registry_id:, dry_run:) # rubocop:disable Metrics/AbcSize, Metrics/Meth end staging_root = nil - download_url = SyncUrlResolver.resolve(entry) + download_url = SyncTransport.resolve(entry) staging_root, staged_dir = fetch_and_verify!(entry, download_url) unless dry_run if entry.sync_policy.auto_promote @@ -199,20 +208,88 @@ def activate_bundle!(registry_id, staged_dir) end ## - # @param registry_id [String, Symbol] + # @param registry_id [String] # @param previous_bundle [Index::RegistryBundle, nil] # @return [void] def report_catalog_change!(registry_id, previous_bundle) new_bundle = Index.current.bundle_for(registry_id) return unless new_bundle - CatalogChangeReporter.report!( - registry_id: registry_id.to_s, - previous_bundle:, - new_bundle: + diff = build_catalog_diff(previous_bundle, new_bundle) + return if diff.empty? + + emit_catalog_change_observability!(registry_id, diff) + emit_catalog_change_security!(registry_id, diff) if diff[:added].any? || diff[:removed].any? + end + + ## + # @param previous_bundle [Index::RegistryBundle, nil] + # @param new_bundle [Index::RegistryBundle] + # @return [Hash{Symbol => Object}] + def build_catalog_diff(previous_bundle, new_bundle) # rubocop:disable Metrics/MethodLength + previous_version = previous_bundle&.manifest&.version + new_version = new_bundle.manifest.version + previous_ids = catalog_ids(previous_bundle) + new_ids = catalog_ids(new_bundle) + added = new_ids - previous_ids + removed = previous_ids - new_ids + + return {} if previous_version == new_version && added.empty? && removed.empty? + + { + previous_version:, + version: new_version, + added:, + removed: + } + end + + ## + # @param registry_id [String] + # @param diff [Hash{Symbol => Object}] + # @return [void] + def emit_catalog_change_observability!(registry_id, diff) # rubocop:disable Metrics/MethodLength + Observability.emit( + event_name: 'registry.catalog_changed', + outcome: 'success', + level: :warn, + details: { + registry_id:, + version: diff[:version], + previous_version: diff[:previous_version], + added_count: diff[:added].size, + removed_count: diff[:removed].size, + added_ids: diff[:added].sort, + removed_ids: diff[:removed].sort + } ) end + ## + # @param registry_id [String] + # @param diff [Hash{Symbol => Object}] + # @return [void] + def emit_catalog_change_security!(registry_id, diff) + SecurityLogger.log_registry_catalog_changed( + registry_id, + version: diff[:version], + previous_version: diff[:previous_version], + added_count: diff[:added].size, + removed_count: diff[:removed].size, + added_ids: diff[:added].sort, + removed_ids: diff[:removed].sort + ) + end + + ## + # @param bundle [Index::RegistryBundle, nil] + # @return [Array] + def catalog_ids(bundle) + return [] unless bundle + + bundle.catalog_entries.map(&:id) + end + ## # @return [Boolean] def skip_boot? @@ -285,7 +362,7 @@ def background_jitter_seconds(interval_hours) # @param download_url [String] # @return [Array(String, String)] staging root and verified bundle directory def fetch_and_verify!(entry, download_url) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength - tarball = SyncFetcher.fetch!(download_url) + tarball = SyncTransport.fetch!(download_url) staging_root = Dir.mktmpdir('registry-sync-') staged_dir = File.join(staging_root, 'bundle') tarball_path = File.join(staging_root, 'download.tar.gz') @@ -334,7 +411,7 @@ def enforce_max_version!(entry, staged_dir) max_version = entry.sync_policy.max_version return unless max_version - return unless VersionGate.exceeds_max?(manifest.version, max_version) + return unless SyncTransport.exceeds_max?(manifest.version, max_version) raise Errors::SyncError, "Registry '#{entry.id}' manifest version '#{manifest.version}' exceeds max_version '#{max_version}'" @@ -394,7 +471,7 @@ def sync_status_for(row) # rubocop:disable Metrics/MethodLength version: row.version, staged_version: Store.staged_version(row.id), updated_at: row.updated_at, - sync_url: entry.mode == :sync ? SyncUrlResolver.resolve(entry) : nil, + sync_url: entry.mode == :sync ? SyncTransport.resolve(entry) : nil, last_error: state.last_error ) end diff --git a/app/web/registry/sync_coordinator.rb b/app/web/registry/sync_coordinator.rb deleted file mode 100644 index 2bd641e07..000000000 --- a/app/web/registry/sync_coordinator.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -module Html2rss - module Web - module Registry - ## - # Serializes sync work per registry id to avoid concurrent bundle swaps. - module SyncCoordinator - REGISTRY_MUTEXES = Hash.new { |mutexes, registry_id| mutexes[registry_id] = Mutex.new } - - module_function - - ## - # @param registry_id [String, Symbol] - # @yield runs sync work exclusively for the registry id - # @return [Object] block result - def run(registry_id, &) - REGISTRY_MUTEXES[registry_id.to_s].synchronize(&) - end - end - end - end -end diff --git a/app/web/registry/sync_fetcher.rb b/app/web/registry/sync_transport.rb similarity index 52% rename from app/web/registry/sync_fetcher.rb rename to app/web/registry/sync_transport.rb index 6d13712a3..d67461a59 100644 --- a/app/web/registry/sync_fetcher.rb +++ b/app/web/registry/sync_transport.rb @@ -2,13 +2,14 @@ require 'net/http' require 'uri' +require 'json' module Html2rss module Web module Registry ## - # HTTPS fetcher with host allowlisting and bounded redirect following. - module SyncFetcher + # HTTPS fetch, sync URL resolution, and manifest version gating for registry sync. + module SyncTransport # rubocop:disable Metrics/ModuleLength DEFAULT_ALLOWED_HOSTS = %w[ api.github.com github.com @@ -20,6 +21,12 @@ module SyncFetcher MAX_RESPONSE_BYTES = 52_428_800 DEFAULT_MAX_REDIRECTS = 5 + OFFICIAL_GITHUB_RELEASES_API = + 'https://api.github.com/repos/html2rss/html2rss-configs/releases/latest' + OFFICIAL_GITHUB_TAG_RELEASES_API = + 'https://api.github.com/repos/html2rss/html2rss-configs/releases/tags/%s' + OFFICIAL_ASSET_NAME = 'registry-bundle.tar.gz' + RedirectPolicy = Data.define(:max_hops, :allowed_hosts) do ## # @return [RedirectPolicy] @@ -30,7 +37,7 @@ def self.default ## # @return [Array] def self.default_allowed_hosts - DEFAULT_ALLOWED_HOSTS + SyncFetcher.extra_allowed_hosts + DEFAULT_ALLOWED_HOSTS + SyncTransport.extra_allowed_hosts end end @@ -45,6 +52,42 @@ def fetch!(url, policy: RedirectPolicy.default) follow_redirects!(uri, policy) end + ## + # @param entry [Entry] + # @return [String] + def resolve(entry) + return entry.sync_url if entry.sync_url && !entry.sync_url.empty? + + pin_version = entry.sync_policy.pin_version + if entry.sync_channel == Config::DEFAULT_OFFICIAL_SYNC_CHANNEL + return resolve_official_tag_download_url(pin_version) if pin_version && !pin_version.empty? + + return resolve_official_download_url + end + + raise Errors::SyncError, "Registry '#{entry.id}' has no sync URL" + end + + ## + # @param sync_channel [String, nil] + # @return [String] + def resolve_channel_url(sync_channel) + channel = sync_channel && !sync_channel.empty? ? sync_channel : Config::DEFAULT_OFFICIAL_SYNC_CHANNEL + return Config::OFFICIAL_RELEASE_URL if channel == Config::DEFAULT_OFFICIAL_SYNC_CHANNEL + + raise Errors::ConfigError, "Unknown sync channel '#{channel}'" + end + + ## + # @param manifest_version [String] + # @param max_version [String, nil] + # @return [Boolean] true when manifest_version is newer than max_version + def exceeds_max?(manifest_version, max_version) + return false if max_version.nil? || max_version.empty? + + compare(normalize(manifest_version), normalize(max_version)).positive? + end + ## # @param uri [URI::HTTPS] # @param policy [RedirectPolicy] @@ -82,6 +125,56 @@ def extra_allowed_hosts .reject(&:empty?) end + ## + # @param tag [String] + # @return [String] + def resolve_official_tag_download_url(tag) # rubocop:disable Metrics/MethodLength + api_url = format(OFFICIAL_GITHUB_TAG_RELEASES_API, tag:) + response_body = fetch!(api_url) + release = JSON.parse(response_body, symbolize_names: true) + asset = Array(release[:assets]).find { |row| row[:name] == OFFICIAL_ASSET_NAME } + url = asset&.dig(:browser_download_url) + unless url + raise Errors::SyncError, + "Official release asset '#{OFFICIAL_ASSET_NAME}' not found for tag '#{tag}'" + end + + url + rescue JSON::ParserError => error + raise Errors::SyncError, "Invalid GitHub release metadata: #{error.message}" + end + + ## + # @return [String] + def resolve_official_download_url + response_body = fetch!(OFFICIAL_GITHUB_RELEASES_API) + release = JSON.parse(response_body, symbolize_names: true) + asset = Array(release[:assets]).find { |row| row[:name] == OFFICIAL_ASSET_NAME } + url = asset&.dig(:browser_download_url) + raise Errors::SyncError, "Official release asset '#{OFFICIAL_ASSET_NAME}' not found" unless url + + url + rescue JSON::ParserError => error + raise Errors::SyncError, "Invalid GitHub release metadata: #{error.message}" + end + + ## + # @param version [String] + # @return [String] + def normalize(version) + version.to_s.delete_prefix('v') + end + + ## + # @param left [String] + # @param right [String] + # @return [Integer] + def compare(left, right) + Gem::Version.new(left) <=> Gem::Version.new(right) + rescue ArgumentError + left <=> right + end + ## # @param url [String] # @return [URI::HTTPS] @@ -144,7 +237,7 @@ def read_body!(response) body end - end + end # rubocop:enable Metrics/ModuleLength end end end diff --git a/app/web/registry/sync_url_resolver.rb b/app/web/registry/sync_url_resolver.rb deleted file mode 100644 index d035c7396..000000000 --- a/app/web/registry/sync_url_resolver.rb +++ /dev/null @@ -1,80 +0,0 @@ -# frozen_string_literal: true - -require 'json' - -module Html2rss - module Web - module Registry - ## - # Resolves registry sync download URLs from config entries and official release metadata. - module SyncUrlResolver - OFFICIAL_GITHUB_RELEASES_API = - 'https://api.github.com/repos/html2rss/html2rss-configs/releases/latest' - OFFICIAL_GITHUB_TAG_RELEASES_API = - 'https://api.github.com/repos/html2rss/html2rss-configs/releases/tags/%s' - OFFICIAL_ASSET_NAME = 'registry-bundle.tar.gz' - - module_function - - ## - # @param entry [Entry] - # @return [String] - def resolve(entry) - return entry.sync_url if entry.sync_url && !entry.sync_url.empty? - - pin_version = entry.sync_policy.pin_version - if entry.sync_channel == Config::DEFAULT_OFFICIAL_SYNC_CHANNEL - return resolve_official_tag_download_url(pin_version) if pin_version && !pin_version.empty? - - return resolve_official_download_url - end - - raise Errors::SyncError, "Registry '#{entry.id}' has no sync URL" - end - - ## - # @param sync_channel [String, nil] - # @return [String] - def resolve_channel_url(sync_channel) - channel = sync_channel && !sync_channel.empty? ? sync_channel : Config::DEFAULT_OFFICIAL_SYNC_CHANNEL - return Config::OFFICIAL_RELEASE_URL if channel == Config::DEFAULT_OFFICIAL_SYNC_CHANNEL - - raise Errors::ConfigError, "Unknown sync channel '#{channel}'" - end - - ## - # @param tag [String] - # @return [String] - def resolve_official_tag_download_url(tag) # rubocop:disable Metrics/MethodLength - api_url = format(OFFICIAL_GITHUB_TAG_RELEASES_API, tag:) - response_body = SyncFetcher.fetch!(api_url) - release = JSON.parse(response_body, symbolize_names: true) - asset = Array(release[:assets]).find { |row| row[:name] == OFFICIAL_ASSET_NAME } - url = asset&.dig(:browser_download_url) - unless url - raise Errors::SyncError, - "Official release asset '#{OFFICIAL_ASSET_NAME}' not found for tag '#{tag}'" - end - - url - rescue JSON::ParserError => error - raise Errors::SyncError, "Invalid GitHub release metadata: #{error.message}" - end - - ## - # @return [String] - def resolve_official_download_url - response_body = SyncFetcher.fetch!(OFFICIAL_GITHUB_RELEASES_API) - release = JSON.parse(response_body, symbolize_names: true) - asset = Array(release[:assets]).find { |row| row[:name] == OFFICIAL_ASSET_NAME } - url = asset&.dig(:browser_download_url) - raise Errors::SyncError, "Official release asset '#{OFFICIAL_ASSET_NAME}' not found" unless url - - url - rescue JSON::ParserError => error - raise Errors::SyncError, "Invalid GitHub release metadata: #{error.message}" - end - end - end - end -end diff --git a/app/web/registry/version_gate.rb b/app/web/registry/version_gate.rb deleted file mode 100644 index 27f6aadbd..000000000 --- a/app/web/registry/version_gate.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -module Html2rss - module Web - module Registry - ## - # Compares registry manifest versions for sync policy caps. - module VersionGate - module_function - - ## - # @param manifest_version [String] - # @param max_version [String, nil] - # @return [Boolean] true when manifest_version is newer than max_version - def exceeds_max?(manifest_version, max_version) - return false if max_version.nil? || max_version.empty? - - compare(normalize(manifest_version), normalize(max_version)).positive? - end - - ## - # @param version [String] - # @return [String] - def normalize(version) - version.to_s.delete_prefix('v') - end - - ## - # @param left [String] - # @param right [String] - # @return [Integer] - def compare(left, right) - Gem::Version.new(left) <=> Gem::Version.new(right) - rescue ArgumentError - left <=> right - end - end - end - end -end diff --git a/spec/html2rss/web/registry/sync_spec.rb b/spec/html2rss/web/registry/sync_spec.rb index 0630807ec..20245c131 100644 --- a/spec/html2rss/web/registry/sync_spec.rb +++ b/spec/html2rss/web/registry/sync_spec.rb @@ -216,4 +216,52 @@ def write_policy_config(yaml) expect(described_class).not_to have_received(:seed_registry!) end end + + describe Html2rss::Web::Registry::SyncTransport do + describe '.exceeds_max?' do + it 'compares dotted numeric versions' do + expect(described_class.exceeds_max?('2026.08.22', '2026.08.21')).to be(true) + expect(described_class.exceeds_max?('2026.08.21', '2026.08.22')).to be(false) + end + + it 'strips a leading v prefix before comparing' do + expect(described_class.exceeds_max?('v2026.08.22', '2026.08.21')).to be(true) + end + + it 'returns false when max_version is unset' do + expect(described_class.exceeds_max?('2026.08.22', nil)).to be(false) + end + end + + describe '.resolve' do + it 'uses the GitHub tag release API when pin_version is set', :aggregate_failures do # rubocop:disable RSpec/ExampleLength + tag_api = format( + described_class::OFFICIAL_GITHUB_TAG_RELEASES_API, + tag: 'v2026.08.22' + ) + download_url = 'https://release-assets.githubusercontent.com/registry-bundle.tar.gz' + stub_request(:get, tag_api).to_return( + status: 200, + body: { + assets: [{ name: described_class::OFFICIAL_ASSET_NAME, browser_download_url: download_url }] + }.to_json + ) + + entry = Html2rss::Web::Registry::Entry.new( + id: 'official', + mode: :sync, + path: nil, + sync_channel: Html2rss::Web::Registry::Config::DEFAULT_OFFICIAL_SYNC_CHANNEL, + sync_url: nil, + catalog: true, + public_key_id: 'html2rss:registry:2026', + public_key: nil, + sync_policy: Html2rss::Web::Registry::SyncPolicy.new('v2026.08.22', nil, false), + allowed_channel_domains: [] + ) + + expect(described_class.resolve(entry)).to eq(download_url) + end + end + end end diff --git a/spec/html2rss/web/registry/sync_url_resolver_spec.rb b/spec/html2rss/web/registry/sync_url_resolver_spec.rb deleted file mode 100644 index 1736a8e45..000000000 --- a/spec/html2rss/web/registry/sync_url_resolver_spec.rb +++ /dev/null @@ -1,39 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'webmock/rspec' - -require_relative '../../../../app' - -RSpec.describe Html2rss::Web::Registry::SyncUrlResolver do - describe '.resolve' do - it 'uses the GitHub tag release API when pin_version is set', :aggregate_failures do # rubocop:disable RSpec/ExampleLength - tag_api = format( - described_class::OFFICIAL_GITHUB_TAG_RELEASES_API, - tag: 'v2026.08.22' - ) - download_url = 'https://release-assets.githubusercontent.com/registry-bundle.tar.gz' - stub_request(:get, tag_api).to_return( - status: 200, - body: { - assets: [{ name: described_class::OFFICIAL_ASSET_NAME, browser_download_url: download_url }] - }.to_json - ) - - entry = Html2rss::Web::Registry::Entry.new( - id: 'official', - mode: :sync, - path: nil, - sync_channel: Html2rss::Web::Registry::Config::DEFAULT_OFFICIAL_SYNC_CHANNEL, - sync_url: nil, - catalog: true, - public_key_id: 'html2rss:registry:2026', - public_key: nil, - sync_policy: Html2rss::Web::Registry::SyncPolicy.new('v2026.08.22', nil, false), - allowed_channel_domains: [] - ) - - expect(described_class.resolve(entry)).to eq(download_url) - end - end -end diff --git a/spec/html2rss/web/registry/version_gate_spec.rb b/spec/html2rss/web/registry/version_gate_spec.rb deleted file mode 100644 index 09b7d43dd..000000000 --- a/spec/html2rss/web/registry/version_gate_spec.rb +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -require_relative '../../../../app' - -RSpec.describe Html2rss::Web::Registry::VersionGate do - it 'compares dotted numeric versions' do - expect(described_class.exceeds_max?('2026.08.22', '2026.08.21')).to be(true) - expect(described_class.exceeds_max?('2026.08.21', '2026.08.22')).to be(false) - end - - it 'strips a leading v prefix before comparing' do - expect(described_class.exceeds_max?('v2026.08.22', '2026.08.21')).to be(true) - end - - it 'returns false when max_version is unset' do - expect(described_class.exceeds_max?('2026.08.22', nil)).to be(false) - end -end From 0cfeb9c6b768557c395259ddff110aef1e6973f5 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 17:50:35 +0200 Subject: [PATCH 10/33] refactor(registry): fold catalog wire and load guards into index --- app/web/registry/index.rb | 141 +++++++++++++++++- app/web/registry/local_catalog.rb | 75 ---------- app/web/registry/local_catalog_row.rb | 18 --- app/web/registry/registry_catalog_row.rb | 35 ----- app/web/registry/scrape_policy.rb | 58 ------- app/web/registry/trust_context.rb | 54 ------- spec/html2rss/web/registry/index_spec.rb | 78 +++++++++- .../web/registry/scrape_policy_spec.rb | 56 ------- 8 files changed, 207 insertions(+), 308 deletions(-) delete mode 100644 app/web/registry/local_catalog.rb delete mode 100644 app/web/registry/local_catalog_row.rb delete mode 100644 app/web/registry/registry_catalog_row.rb delete mode 100644 app/web/registry/scrape_policy.rb delete mode 100644 app/web/registry/trust_context.rb delete mode 100644 spec/html2rss/web/registry/scrape_policy_spec.rb diff --git a/app/web/registry/index.rb b/app/web/registry/index.rb index 5c0df5d84..1fbbabbbd 100644 --- a/app/web/registry/index.rb +++ b/app/web/registry/index.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'uri' + module Html2rss module Web module Registry @@ -9,6 +11,60 @@ class Index # rubocop:disable Metrics/ClassLength RegistryBundle = Data.define(:registry_id, :manifest, :configs, :catalog_entries) StatusEntry = Data.define(:id, :version, :updated_at, :sync_mode) + CatalogRow = Data.define( + :id, + :path, + :directory, + :channel, + :parameters, + :source, + :registry + ) do + ## + # @param entry [Html2rss::Registry::CatalogEntry] + # @param registry_id [String] + # @return [CatalogRow] + def self.from_entry(entry, registry_id) + new( + id: entry.id, + path: entry.path, + directory: entry.directory, + channel: entry.channel, + parameters: entry.parameters, + source: 'registry', + registry: registry_id + ) + end + + ## + # @param feed_name [String, Symbol] + # @param feed_config [Hash{Symbol => Object}] + # @return [CatalogRow, nil] + def self.from_local_feed(feed_name, feed_config) # rubocop:disable Metrics/MethodLength + directory = feed_config[:directory] || {} + title = directory[:title] + return nil if title.to_s.strip.empty? + + id = feed_name.to_s + channel = feed_config[:channel] || {} + new( + id:, + path: "/#{id}.rss", + source: 'local', + directory: Html2rss::Registry::CatalogBuilder.directory_payload(directory, title), + channel: Html2rss::Registry::CatalogBuilder.channel_payload(channel, title), + parameters: { schema: {}, defaults: {} }, + registry: nil + ) + end + + ## + # @return [Hash{Symbol => Object}] + def to_h + super.compact + end + end + @mutex = Mutex.new @current = nil @@ -50,7 +106,7 @@ def config_for(feed_id) # @return [Array Object}>] catalog rows in HTTP wire shape def catalog_rows rows = registry_catalog_rows - LocalCatalog.rows.each { |row| rows[row.id] = row } + local_catalog_rows.each { |row| rows[row.id] = row } rows.values.sort_by(&:id).map(&:to_h) end @@ -79,7 +135,7 @@ def status private ## - # @return [Hash{String => RegistryCatalogRow, LocalCatalogRow}] + # @return [Hash{String => CatalogRow}] def registry_catalog_rows Config.precedence.each_with_object({}) do |registry_id, rows| next unless Config.catalog_enabled?(registry_id) @@ -88,11 +144,19 @@ def registry_catalog_rows next unless bundle bundle.catalog_entries.each do |entry| - rows[entry.id] ||= RegistryCatalogRow.from_entry(entry, registry_id) + rows[entry.id] ||= CatalogRow.from_entry(entry, registry_id) end end end + ## + # @return [Array] + def local_catalog_rows + LocalConfig.feeds.filter_map do |feed_name, feed_config| + CatalogRow.from_local_feed(feed_name, feed_config) + end + end + ## # @return [Hash{String => RegistryBundle}] def loaded_bundles @@ -110,15 +174,82 @@ def load_bundle(registry_id) # rubocop:disable Metrics/MethodLength bundle = Html2rss::Registry::Bundle.load( directory, - **TrustContext.for_entry(entry, directory).load_options + **trust_options_for(entry, directory) ) registry_bundle = to_registry_bundle(registry_id, bundle) - ScrapePolicy.enforce!(entry, registry_bundle) + enforce_scrape_policy!(entry, registry_bundle) registry_bundle rescue Html2rss::Registry::Error => error raise Errors::LoadError, "Failed to load registry '#{registry_id}': #{error.message}" end + ## + # @param entry [Entry] + # @param bundle_dir [String] + # @return [Hash{Symbol => Object}] keyword args for {Html2rss::Registry::Bundle.load} + def trust_options_for(entry, bundle_dir) + case entry.mode + in :path + { trust: :integrity_only, public_keys: {} } + in :sync + if signed_bundle?(bundle_dir, entry) + { trust: :signed, public_keys: entry.public_keys } + else + { trust: :integrity_only, public_keys: {} } + end + end + end + + ## + # @param bundle_dir [String] + # @param entry [Entry] + # @return [Boolean] + def signed_bundle?(bundle_dir, entry) + signature_path = File.join(bundle_dir, Html2rss::Registry::Manifest::SIGNATURE_FILE) + File.file?(signature_path) && !entry.public_keys.empty? + end + + ## + # @param entry [Entry] + # @param bundle [RegistryBundle] + # @return [void] + # @raise [Errors::LoadError] when a config channel URL violates the allowlist + def enforce_scrape_policy!(entry, bundle) # rubocop:disable Metrics/CyclomaticComplexity + allowed = entry.allowed_channel_domains + return if allowed.nil? || allowed.empty? + + bundle.configs.each do |feed_id, config| + channel_url = config.dig(:channel, :url) + host = channel_host_for(channel_url) + next if host && allowed.any? { |domain| channel_domain_allowed?(host, domain) } + + raise Errors::LoadError, + "Registry '#{entry.id}' config '#{feed_id}' channel.url host " \ + "'#{host || channel_url}' is not allowed by allowed_channel_domains" + end + end + + ## + # @param host [String] + # @param allowed_domain [String] + # @return [Boolean] + def channel_domain_allowed?(host, allowed_domain) + normalized_host = host.downcase + normalized_domain = allowed_domain.downcase + normalized_host == normalized_domain || normalized_host.end_with?(".#{normalized_domain}") + end + + ## + # @param channel_url [String, nil] + # @return [String, nil] + def channel_host_for(channel_url) + return nil if channel_url.to_s.strip.empty? + + URI.parse(channel_url).host + rescue URI::InvalidURIError + nil + end + ## # @param registry_id [String] # @param bundle [Html2rss::Registry::Bundle::BundleData] diff --git a/app/web/registry/local_catalog.rb b/app/web/registry/local_catalog.rb deleted file mode 100644 index 9623bf45f..000000000 --- a/app/web/registry/local_catalog.rb +++ /dev/null @@ -1,75 +0,0 @@ -# frozen_string_literal: true - -module Html2rss - module Web - module Registry - ## - # Builds catalog wire rows for local feeds.yml entries. - module LocalCatalog - module_function - - ## - # @return [Array] - def rows - LocalConfig.feeds.filter_map do |feed_name, feed_config| - build_row(feed_name, feed_config) - end - end - - ## - # @param feed_name [String, Symbol] - # @param feed_config [Hash{Symbol => Object}] - # @return [LocalCatalogRow, nil] - def build_row(feed_name, feed_config) - directory = feed_config[:directory] || {} - title = directory[:title] - return nil if title.to_s.strip.empty? - - row_identity(feed_name, directory, title, feed_config[:channel] || {}) - end - - ## - # @param feed_name [String, Symbol] - # @param directory [Hash{Symbol => Object}] - # @param title [String] - # @param channel [Hash{Symbol => Object}] - # @return [LocalCatalogRow] - def row_identity(feed_name, directory, title, channel) - id = feed_name.to_s - LocalCatalogRow.new( - id:, - path: "/#{id}.rss", - source: 'local', - directory: directory_payload(directory, title), - channel: channel_payload(channel, title), - parameters: { schema: {}, defaults: {} } - ) - end - - ## - # @param directory [Hash{Symbol => Object}] - # @param title [String] - # @return [Hash{Symbol => Object}] - def directory_payload(directory, title) - { - title: title.to_s, - summary: directory[:summary], - topics: Array(directory[:topics]) - }.compact - end - - ## - # @param channel [Hash{Symbol => Object}] - # @param title [String] - # @return [Hash{Symbol => Object}] - def channel_payload(channel, title) - { - url: channel.fetch(:url), - language: channel[:language], - title: channel[:title] || title.to_s - }.compact - end - end - end - end -end diff --git a/app/web/registry/local_catalog_row.rb b/app/web/registry/local_catalog_row.rb deleted file mode 100644 index 0d95a784f..000000000 --- a/app/web/registry/local_catalog_row.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -module Html2rss - module Web - module Registry - ## - # Catalog wire row for local feeds.yml entries. - LocalCatalogRow = Data.define( - :id, - :path, - :directory, - :channel, - :parameters, - :source - ) - end - end -end diff --git a/app/web/registry/registry_catalog_row.rb b/app/web/registry/registry_catalog_row.rb deleted file mode 100644 index 27d3791e6..000000000 --- a/app/web/registry/registry_catalog_row.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -module Html2rss - module Web - module Registry - ## - # Catalog wire row for registry-sourced feed configs. - RegistryCatalogRow = Data.define( - :id, - :path, - :directory, - :channel, - :parameters, - :source, - :registry - ) do - ## - # @param entry [Html2rss::Registry::CatalogEntry] - # @param registry_id [String] - # @return [RegistryCatalogRow] - def self.from_entry(entry, registry_id) - new( - id: entry.id, - path: entry.path, - directory: entry.directory, - channel: entry.channel, - parameters: entry.parameters, - source: 'registry', - registry: registry_id - ) - end - end - end - end -end diff --git a/app/web/registry/scrape_policy.rb b/app/web/registry/scrape_policy.rb deleted file mode 100644 index b341d751f..000000000 --- a/app/web/registry/scrape_policy.rb +++ /dev/null @@ -1,58 +0,0 @@ -# frozen_string_literal: true - -require 'uri' - -module Html2rss - module Web - module Registry - ## - # Enforces optional channel URL domain allowlists for registry bundles. - module ScrapePolicy - module_function - - ## - # @param entry [Entry] - # @param bundle [Index::RegistryBundle] - # @return [void] - # @raise [Errors::LoadError] when a config channel URL violates the allowlist - def enforce!(entry, bundle) # rubocop:disable Metrics/CyclomaticComplexity - allowed = entry.allowed_channel_domains - return if allowed.nil? || allowed.empty? - - bundle.configs.each do |feed_id, config| - channel_url = config.dig(:channel, :url) - host = host_for(channel_url) - next if host && allowed.any? { |domain| domain_allowed?(host, domain) } - - raise Errors::LoadError, - "Registry '#{entry.id}' config '#{feed_id}' channel.url host " \ - "'#{host || channel_url}' is not allowed by allowed_channel_domains" - end - end - - ## - # @param host [String] - # @param allowed_domain [String] - # @return [Boolean] - def domain_allowed?(host, allowed_domain) - normalized_host = host.downcase - normalized_domain = allowed_domain.downcase - normalized_host == normalized_domain || normalized_host.end_with?(".#{normalized_domain}") - end - - ## - # @param channel_url [String, nil] - # @return [String, nil] - def host_for(channel_url) - return nil if channel_url.to_s.strip.empty? - - URI.parse(channel_url).host - rescue URI::InvalidURIError - nil - end - - private_class_method :host_for - end - end - end -end diff --git a/app/web/registry/trust_context.rb b/app/web/registry/trust_context.rb deleted file mode 100644 index 960593a10..000000000 --- a/app/web/registry/trust_context.rb +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true - -module Html2rss - module Web - module Registry - ## - # Trust mode and verification keys passed to {Html2rss::Registry::Bundle.load}. - TrustContext = Data.define(:mode, :public_keys) do - ## - # @param entry [Entry] - # @param bundle_dir [String] - # @return [TrustContext] - def self.for_entry(entry, bundle_dir) - case entry.mode - in :path - integrity_only - in :sync - signed_bundle?(bundle_dir, entry) ? signed(entry.public_keys) : integrity_only - end - end - - ## - # @return [TrustContext] - def self.integrity_only - new(mode: :integrity_only, public_keys: {}) - end - - ## - # @param public_keys [Hash{String => OpenSSL::PKey::PKey}] - # @return [TrustContext] - def self.signed(public_keys) - new(mode: :signed, public_keys:) - end - - ## - # @return [Hash{Symbol => Object}] keyword args for {Html2rss::Registry::Bundle.load} - def load_options - { trust: mode, public_keys: } - end - - ## - # @param bundle_dir [String] - # @param entry [Entry] - # @return [Boolean] - def self.signed_bundle?(bundle_dir, entry) - signature_path = File.join(bundle_dir, Html2rss::Registry::Manifest::SIGNATURE_FILE) - File.file?(signature_path) && !entry.public_keys.empty? - end - - private_class_method :signed_bundle? - end - end - end -end diff --git a/spec/html2rss/web/registry/index_spec.rb b/spec/html2rss/web/registry/index_spec.rb index cf8d91c1f..f5797ece8 100644 --- a/spec/html2rss/web/registry/index_spec.rb +++ b/spec/html2rss/web/registry/index_spec.rb @@ -48,6 +48,26 @@ expect(apple.fetch(:registry)).to eq('official') end + + it 'merges local feeds.yml rows after registry rows', :aggregate_failures do # rubocop:disable RSpec/ExampleLength + allow(Html2rss::Web::LocalConfig).to receive(:feeds).and_return( + 'team/releases' => { + directory: { title: 'Team Releases', summary: 'Internal release notes' }, + channel: { url: 'https://team.example/releases', title: 'Team Releases' } + } + ) + + rows = described_class.current.catalog_rows + local = rows.find { |row| row.fetch(:id) == 'team/releases' } + + expect(local).to include( + source: 'local', + path: '/team/releases.rss', + directory: hash_including(title: 'Team Releases'), + channel: hash_including(url: 'https://team.example/releases') + ) + expect(local).not_to have_key(:registry) + end end describe '#status' do @@ -62,10 +82,60 @@ end end + describe 'channel domain allowlist' do + let(:index) { described_class.new } + let(:entry) do + Html2rss::Web::Registry::Entry.new( + id: 'official', + mode: :path, + path: nil, + sync_channel: nil, + sync_url: nil, + catalog: true, + public_key_id: 'test', + public_key: nil, + sync_policy: Html2rss::Web::Registry::SyncPolicy.new(nil, nil, false), + allowed_channel_domains: ['phys.org'] + ) + end + let(:allowed_bundle) do + described_class::RegistryBundle.new( + registry_id: 'official', + manifest: instance_double(Html2rss::Registry::Manifest, version: '1.0.0'), + configs: { + 'phys.org/weekly' => { channel: { url: 'https://phys.org/weekly-news/' } } + }, + catalog_entries: [] + ) + end + + it 'allows suffix-matching channel domains' do + expect { index.send(:enforce_scrape_policy!, entry, allowed_bundle) }.not_to raise_error + end + + it 'raises when a config channel domain is not allowlisted' do + blocked_bundle = described_class::RegistryBundle.new( + registry_id: 'official', + manifest: instance_double(Html2rss::Registry::Manifest, version: '1.0.0'), + configs: { + 'blocked.example/feed' => { channel: { url: 'https://blocked.example/private' } } + }, + catalog_entries: [] + ) + + expect { index.send(:enforce_scrape_policy!, entry, blocked_bundle) } + .to raise_error(Html2rss::Web::Registry::Errors::LoadError, /blocked.example/) + end + end + describe 'allowed_channel_domains' do let(:config_path) { File.join(Dir.pwd, 'tmp', 'domain-allowlist-registries.yml') } - before do + after do + FileUtils.rm_f(config_path) + end + + it 'rejects bundles with channel URLs outside the allowlist' do # rubocop:disable RSpec/ExampleLength FileUtils.mkdir_p(File.dirname(config_path)) File.write(config_path, <<~YAML) precedence: @@ -79,13 +149,7 @@ YAML ENV['REGISTRIES_CONFIG'] = config_path described_class.reload! - end - - after do - FileUtils.rm_f(config_path) - end - it 'rejects bundles with channel URLs outside the allowlist' do expect { described_class.current.config_for('phys.org/weekly') } .to raise_error(Html2rss::Web::Registry::Errors::LoadError, /phys.org/) end diff --git a/spec/html2rss/web/registry/scrape_policy_spec.rb b/spec/html2rss/web/registry/scrape_policy_spec.rb deleted file mode 100644 index 681e88cd8..000000000 --- a/spec/html2rss/web/registry/scrape_policy_spec.rb +++ /dev/null @@ -1,56 +0,0 @@ -# frozen_string_literal: true - -require 'fileutils' -require 'climate_control' -require 'spec_helper' - -require_relative '../../../../app' - -RSpec.describe Html2rss::Web::Registry::ScrapePolicy do - let(:allowed_entry) do - Html2rss::Web::Registry::Entry.new( - id: 'official', - mode: :path, - path: nil, - sync_channel: nil, - sync_url: nil, - catalog: true, - public_key_id: 'test', - public_key: nil, - sync_policy: Html2rss::Web::Registry::SyncPolicy.new(nil, nil, false), - allowed_channel_domains: ['phys.org'] - ) - end - - let(:allowed_bundle) do - Html2rss::Web::Registry::Index::RegistryBundle.new( - registry_id: 'official', - manifest: instance_double(Html2rss::Registry::Manifest, version: '1.0.0'), - configs: { - 'phys.org/weekly' => { channel: { url: 'https://phys.org/weekly-news/' } } - }, - catalog_entries: [] - ) - end - - let(:blocked_bundle) do - Html2rss::Web::Registry::Index::RegistryBundle.new( - registry_id: 'official', - manifest: instance_double(Html2rss::Registry::Manifest, version: '1.0.0'), - configs: { - 'phys.org/weekly' => { channel: { url: 'https://phys.org/weekly-news/' } }, - 'blocked.example/feed' => { channel: { url: 'https://blocked.example/private' } } - }, - catalog_entries: [] - ) - end - - it 'allows suffix-matching channel domains' do - expect { described_class.enforce!(allowed_entry, allowed_bundle) }.not_to raise_error - end - - it 'raises when a config channel domain is not allowlisted' do - expect { described_class.enforce!(allowed_entry, blocked_bundle) } - .to raise_error(Html2rss::Web::Registry::Errors::LoadError, /blocked.example/) - end -end From 043fc7e955561f2b7abda15e46d5dd410feac6f8 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 17:50:55 +0200 Subject: [PATCH 11/33] refactor(registry): centralize manifest I/O and dedup test signing --- app/web/registry/index.rb | 3 ++- app/web/registry/store.rb | 28 +++++++++++++++++----------- app/web/registry/sync.rb | 5 ++--- spec/support/registry_sync.rb | 9 +-------- 4 files changed, 22 insertions(+), 23 deletions(-) diff --git a/app/web/registry/index.rb b/app/web/registry/index.rb index 1fbbabbbd..edea52fa4 100644 --- a/app/web/registry/index.rb +++ b/app/web/registry/index.rb @@ -172,6 +172,7 @@ def load_bundle(registry_id) # rubocop:disable Metrics/MethodLength return nil unless directory && File.directory?(directory) return nil unless active_bundle_present?(directory) + # Verify on read: disk-trust seam; Sync already verifies downloaded bundles before promote. bundle = Html2rss::Registry::Bundle.load( directory, **trust_options_for(entry, directory) @@ -276,7 +277,7 @@ def bundle_directory(entry) # @param directory [String] # @return [Boolean] def active_bundle_present?(directory) - File.file?(File.join(directory, Html2rss::Registry::Manifest::MANIFEST_FILE)) + Store.bundle_present_at?(directory) end ## diff --git a/app/web/registry/store.rb b/app/web/registry/store.rb index 1bd69f28d..760fa28e0 100644 --- a/app/web/registry/store.rb +++ b/app/web/registry/store.rb @@ -49,6 +49,22 @@ def bundle_present?(registry_id) bundle_present_at?(registry_dir(registry_id)) end + ## + # @param path [String] bundle directory + # @return [Boolean] + def bundle_present_at?(path) + File.file?(File.join(path, Html2rss::Registry::Manifest::MANIFEST_FILE)) + end + + ## + # @param path [String] bundle directory + # @return [Html2rss::Registry::Manifest] + def read_manifest(path) + Html2rss::Registry::Manifest.parse( + File.read(File.join(path, Html2rss::Registry::Manifest::MANIFEST_FILE)) + ) + end + ## # @param registry_id [String, Symbol] # @return [String] verified staging directory for a registry id @@ -205,23 +221,13 @@ def promote_bundle!(staged_dir, target, backup) FileUtils.rm_rf(backup) end - ## - # @param path [String] - # @return [Boolean] - def bundle_present_at?(path) - File.file?(File.join(path, Html2rss::Registry::Manifest::MANIFEST_FILE)) - end - ## # @param path [String] # @return [String, nil] def manifest_version_at(path) return nil unless bundle_present_at?(path) - manifest = Html2rss::Registry::Manifest.parse( - File.read(File.join(path, Html2rss::Registry::Manifest::MANIFEST_FILE)) - ) - manifest.version + read_manifest(path).version rescue Html2rss::Registry::ManifestError nil end diff --git a/app/web/registry/sync.rb b/app/web/registry/sync.rb index 9c110e915..ff012e845 100644 --- a/app/web/registry/sync.rb +++ b/app/web/registry/sync.rb @@ -373,6 +373,7 @@ def fetch_and_verify!(entry, download_url) # rubocop:disable Metrics/AbcSize, Me Html2rss::Registry::Archive.extract!(io, into: staged_dir) end + # Verify before promote: network-trust seam; Index reload still verifies on disk via Bundle.load. Html2rss::Registry::Verifier.verify!( staged_dir, trust: :signed, @@ -421,9 +422,7 @@ def enforce_max_version!(entry, staged_dir) # @param staged_dir [String] # @return [Html2rss::Registry::Manifest] def read_manifest!(staged_dir) - Html2rss::Registry::Manifest.parse( - File.read(File.join(staged_dir, Html2rss::Registry::Manifest::MANIFEST_FILE)) - ) + Store.read_manifest(staged_dir) end ## diff --git a/spec/support/registry_sync.rb b/spec/support/registry_sync.rb index 839d9e13a..91f8c424e 100644 --- a/spec/support/registry_sync.rb +++ b/spec/support/registry_sync.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require 'fileutils' -require 'openssl' require 'stringio' require 'yaml' @@ -29,7 +28,7 @@ def build_signed_tarball manifest = Html2rss::Registry::Manifest.parse( File.read(File.join(bundle_dir, Html2rss::Registry::Manifest::MANIFEST_FILE)) ) - sign_manifest!(manifest, key_pem: TEST_PRIVATE_KEY, bundle_dir:) + Html2rss::Registry::TestSupport.sign!(manifest, key_pem: TEST_PRIVATE_KEY, bundle_dir:) pack_bundle_dir(bundle_dir) ensure @@ -64,12 +63,6 @@ def policy_registry_yaml(download_url:, auto_promote:, sync_extra: {}) # rubocop } ) end - - def sign_manifest!(manifest, key_pem:, bundle_dir:) - private_key = OpenSSL::PKey.read(key_pem) - signature = private_key.sign(nil, manifest.canonical_bytes) - File.write(File.join(bundle_dir, Html2rss::Registry::Manifest::SIGNATURE_FILE), [signature].pack('m0')) - end end RSpec.configure do |config| From 076964e62b6877cf286d234e2b4e9abb1cae3a1d Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 17:51:48 +0200 Subject: [PATCH 12/33] chore(registry): dedup fixtures and sync docs --- AGENTS.md | 2 +- CONTEXT.md | 9 +++++ .../sync/bundle/configs/phys.org/weekly.yml | 25 -------------- .../support.apple.com/en_gb_ht201222.yml | 34 ------------------- spec/support/registry.rb | 24 ------------- spec/support/registry_sync.rb | 30 ++++++++++++++-- 6 files changed, 38 insertions(+), 86 deletions(-) delete mode 100644 spec/fixtures/registries/sync/bundle/configs/phys.org/weekly.yml delete mode 100644 spec/fixtures/registries/sync/bundle/configs/support.apple.com/en_gb_ht201222.yml delete mode 100644 spec/support/registry.rb diff --git a/AGENTS.md b/AGENTS.md index 80d1bdb08..d7cd47803 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -64,7 +64,7 @@ Public feed-directory metadata from verified registry bundles and local `feeds.y | Flag | `CONFIG_CATALOG_ENABLED` (default `true`; set `false` to disable) | | Disabled response | `404` with `{ "error": "catalog_disabled" }` | | Registry entries | `Registry::Index.current.catalog_rows` — loads signed bundles from `config/registries.yml`; adds `source: registry`, `registry: ` | -| Local entries | `Registry::LocalCatalog` includes `feeds.yml` feeds only when `directory.title` is set (`source: local`) | +| Local entries | `Registry::Index` catalog rows include `feeds.yml` feeds only when `directory.title` is set (`source: local`) | | Per-registry privacy | `catalog: false` in `registries.yml` omits that registry from the API (feeds still served) | | Starter feeds (UI) | Frontend `selectStarterFeeds` when feed creation is disabled; catalog find uses full catalog when enabled | | Catalog find | `findCatalogEntries` → multi-hit list under create URL; links via `catalogFeedHref` (path + defaults) | diff --git a/CONTEXT.md b/CONTEXT.md index 81c2ec430..9c0f8b1d6 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -54,3 +54,12 @@ Audit channel: snake_case `security_event` with IP / user-agent / token hash. Au ### LogEvent Shared emit plumbing for both channels (`RequestContext`, `LogSanitizer`, `AppLogger` / Sentry). Not a third public facade. + +### Registry Index +Backend merge owner for registry bundles and local `feeds.yml` feeds. Builds catalog wire rows (`Registry::Index::CatalogRow`), enforces load-time trust and channel-domain allowlists, and serves `config_for` / `catalog_rows` / `status`. + +### Registry Sync +Backend orchestration for fetch → verify → stage/promote of signed registry bundles. Owns boot seeding, background refresh, CLI exit codes, and catalog-change telemetry after promotion. + +### Sync Transport +Backend HTTPS fetch, sync URL resolution (GitHub releases and channel defaults), and manifest version gating used by `Registry::Sync` and parse-time config resolution. diff --git a/spec/fixtures/registries/sync/bundle/configs/phys.org/weekly.yml b/spec/fixtures/registries/sync/bundle/configs/phys.org/weekly.yml deleted file mode 100644 index 4c0102d03..000000000 --- a/spec/fixtures/registries/sync/bundle/configs/phys.org/weekly.yml +++ /dev/null @@ -1,25 +0,0 @@ -registry: - id: phys.org/weekly -directory: - topics: - - science - title: "Phys.org — Weekly" - summary: "Top science news of the week from Phys.org." -channel: - language: en - title: "Phys.org — Weekly" - url: https://phys.org/weekly-news/ - time_zone: Europe/London - ttl: 1440 -selectors: - items: - selector: ".sorted-news-list .sorted-article-content" - title: - selector: "h4" - category: - selector: ".text-info" - categories: - - category - url: - selector: ".news-link" - extractor: "href" diff --git a/spec/fixtures/registries/sync/bundle/configs/support.apple.com/en_gb_ht201222.yml b/spec/fixtures/registries/sync/bundle/configs/support.apple.com/en_gb_ht201222.yml deleted file mode 100644 index 83c1c81b6..000000000 --- a/spec/fixtures/registries/sync/bundle/configs/support.apple.com/en_gb_ht201222.yml +++ /dev/null @@ -1,34 +0,0 @@ -registry: - id: support.apple.com/en_gb_ht201222 -directory: - topics: - - tech - - security - title: "Apple Support — Security releases" - summary: "Apple security update and release notes (HT201222 / related)." -strategy: botasaurus -channel: - title: "Apple Support — Security releases" - url: https://support.apple.com/en-gb/100100 - language: en - ttl: 360 - time_zone: UTC -request: - botasaurus: - wait_for_selector: ".table-wrapper table tbody tr a" - wait_timeout_seconds: 20 -selectors: - items: - selector: ".table-wrapper table tbody > tr:not(:first-child)" - enhance: false - title: - selector: a - url: - selector: a - extractor: href - description: - selector: "td:nth-child(2)" - published_at: - selector: "td:nth-child(3)" - post_process: - - name: parse_time diff --git a/spec/support/registry.rb b/spec/support/registry.rb deleted file mode 100644 index c07e52949..000000000 --- a/spec/support/registry.rb +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true - -module RegistryTestHelpers - FIXTURES_ROOT = File.expand_path('../fixtures/registries', __dir__) - DEFAULT_REGISTRIES_CONFIG = File.join(FIXTURES_ROOT, 'registries.yml') - - module_function - - def configure_registry_fixtures! - ENV['REGISTRIES_CONFIG'] = DEFAULT_REGISTRIES_CONFIG - ENV['REGISTRY_DATA_ROOT'] = File.join(Dir.pwd, 'tmp', 'test-registry-data') - end - - def reset_registry! - Html2rss::Web::Registry::Index.reload! - end -end - -RSpec.configure do |config| - config.before do - RegistryTestHelpers.configure_registry_fixtures! - RegistryTestHelpers.reset_registry! - end -end diff --git a/spec/support/registry_sync.rb b/spec/support/registry_sync.rb index 91f8c424e..398a24865 100644 --- a/spec/support/registry_sync.rb +++ b/spec/support/registry_sync.rb @@ -4,11 +4,28 @@ require 'stringio' require 'yaml' +module RegistryTestHelpers + FIXTURES_ROOT = File.expand_path('../fixtures/registries', __dir__) + DEFAULT_REGISTRIES_CONFIG = File.join(FIXTURES_ROOT, 'registries.yml') + + module_function + + def configure_registry_fixtures! + ENV['REGISTRIES_CONFIG'] = DEFAULT_REGISTRIES_CONFIG + ENV['REGISTRY_DATA_ROOT'] = File.join(Dir.pwd, 'tmp', 'test-registry-data') + end + + def reset_registry! + Html2rss::Web::Registry::Index.reload! + end +end + module RegistrySyncTestHelpers FIXTURE_KEYS_ROOT = File.expand_path('../fixtures/registries/keys', __dir__) TEST_PUBLIC_KEY = File.read(File.join(FIXTURE_KEYS_ROOT, 'test-key.pub')) TEST_PRIVATE_KEY = File.read(File.join(FIXTURE_KEYS_ROOT, 'test-key.pem')) SYNC_FIXTURES_ROOT = File.expand_path('../fixtures/registries/sync', __dir__) + OFFICIAL_FIXTURES_ROOT = File.join(RegistryTestHelpers::FIXTURES_ROOT, 'official') SYNC_REGISTRIES_CONFIG = File.join(SYNC_FIXTURES_ROOT, 'registries.yml') SYNC_DATA_ROOT = File.join(Dir.pwd, 'tmp', 'sync-registry-data') @@ -22,9 +39,13 @@ def configure_sync_registry! Html2rss::Web::Registry::Index.reload! end - def build_signed_tarball + def build_signed_tarball # rubocop:disable Metrics/MethodLength bundle_dir = Dir.mktmpdir('signed-registry-bundle') - FileUtils.cp_r(File.join(SYNC_FIXTURES_ROOT, 'bundle', '.'), bundle_dir) + FileUtils.cp_r(File.join(OFFICIAL_FIXTURES_ROOT, 'configs'), File.join(bundle_dir, 'configs')) + FileUtils.cp( + File.join(SYNC_FIXTURES_ROOT, 'bundle', Html2rss::Registry::Manifest::MANIFEST_FILE), + File.join(bundle_dir, Html2rss::Registry::Manifest::MANIFEST_FILE) + ) manifest = Html2rss::Registry::Manifest.parse( File.read(File.join(bundle_dir, Html2rss::Registry::Manifest::MANIFEST_FILE)) ) @@ -66,6 +87,11 @@ def policy_registry_yaml(download_url:, auto_promote:, sync_extra: {}) # rubocop end RSpec.configure do |config| + config.before do + RegistryTestHelpers.configure_registry_fixtures! + RegistryTestHelpers.reset_registry! + end + config.before do |example| next unless example.metadata[:registry_sync] From 98d6ccb16ca24ba4099d4773c419499f22be3d4f Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 17:59:42 +0200 Subject: [PATCH 13/33] refactor(ruby4): apply leading operators and it blocks on registry branch Enable Style/ItBlockParameter, normalize empty-string predicates, and adopt it blocks across registry config and transport helpers. --- .rubocop.yml | 4 ++++ app/web/registry/config.rb | 10 +++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index ee5e2c04c..767dc7182 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -38,6 +38,10 @@ Style/Documentation: AllowedConstants: - App +Style/ItBlockParameter: + Enabled: true + EnforcedStyle: allow_single_line + RSpec/SpecFilePathFormat: Exclude: - 'spec/html2rss/web/app/*_spec.rb' diff --git a/app/web/registry/config.rb b/app/web/registry/config.rb index 789bd07d6..468d5f11c 100644 --- a/app/web/registry/config.rb +++ b/app/web/registry/config.rb @@ -91,7 +91,7 @@ def parse_snapshot precedence = Array(document[:precedence]).map(&:to_s) precedence = DEFAULT_PRECEDENCE if precedence.empty? - entries = precedence.to_h { |registry_id| [registry_id, parse_entry(registry_id, document)] } + entries = precedence.to_h { [it, parse_entry(it, document)] } missing = precedence - entries.keys raise Errors::ConfigError, "Missing registry definitions: #{missing.join(', ')}" unless missing.empty? @@ -166,7 +166,7 @@ def parse_entry(registry_id, document) def build_entry(registry_id, raw) sync = sync_section(raw) case raw - in { path: path } if path&.then { |value| !value.empty? } + in { path: path } if path&.then { !it.empty? } entry_attributes(registry_id, raw, :path, expand_path(path.to_s), nil, nil) else build_sync_entry(registry_id, raw, sync) @@ -192,7 +192,7 @@ def build_sync_entry(registry_id, raw, sync) # @param sync_channel [String, nil] # @return [String] def resolved_sync_url(sync_url, sync_channel) - return sync_url if sync_url && !sync_url.empty? + return sync_url unless sync_url.to_s.empty? SyncTransport.resolve_channel_url(sync_channel) end @@ -259,14 +259,14 @@ def optional_string(value) # @param value [Object] # @return [Array] def parse_allowed_channel_domains(value) - Array(value).map { |domain| domain.to_s.strip }.reject(&:empty?) + Array(value).map { it.to_s.strip }.reject(&:empty?) end ## # @param public_key_id [String, nil] # @return [String] def public_key_id_for(public_key_id) - return public_key_id if public_key_id && !public_key_id.empty? + return public_key_id unless public_key_id.to_s.empty? 'html2rss:registry:2026' end From d82231e84181fbcb538b53753984080258e3b01f Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 17:59:55 +0200 Subject: [PATCH 14/33] refactor(registry): dedupe transport, status lookup, and deep_dup Promote StructuredData.deep_dup for shared cloning, add Store.manifest_mtime, and centralize registry status lookup helpers. --- app/web/config/config_snapshot.rb | 33 ++----------------------------- app/web/config/local_config.rb | 21 +++----------------- app/web/registry/store.rb | 12 +++++++++++ app/web/structured_data.rb | 29 +++++++++++++++++++++++++++ 4 files changed, 46 insertions(+), 49 deletions(-) create mode 100644 app/web/structured_data.rb diff --git a/app/web/config/config_snapshot.rb b/app/web/config/config_snapshot.rb index 0a8fc03c5..0afa29fdd 100644 --- a/app/web/config/config_snapshot.rb +++ b/app/web/config/config_snapshot.rb @@ -47,7 +47,7 @@ def normalize_feeds(raw_feeds) return {} unless raw_feeds.is_a?(Hash) raw_feeds.each_with_object({}) do |(name, config), memo| - memo[name.to_sym] = FeedConfig.new(name: name.to_sym, raw: deep_dup(config).freeze) + memo[name.to_sym] = FeedConfig.new(name: name.to_sym, raw: StructuredData.deep_dup(config).freeze) end end @@ -68,7 +68,7 @@ def normalize_accounts(raw_accounts) # @param accounts [Array] # @return [Hash{Symbol=>Object}] def normalized_global_hash(global_hash, accounts) - normalized = deep_dup(global_hash) + normalized = StructuredData.deep_dup(global_hash) return normalized unless normalized.key?(:auth) normalized[:auth] = normalized_auth_hash(normalized[:auth], accounts) @@ -85,35 +85,6 @@ def normalized_auth_hash(auth_hash, accounts) end auth end - - # @param value [Object] - # @return [Object] - def deep_dup(value) - case value - when Hash - deep_dup_hash(value) - when Array - deep_dup_array(value) - when String - value.dup - else - value - end - end - - # @param value [Hash] - # @return [Hash] - def deep_dup_hash(value) - value.each_with_object({}) do |(key, val), memo| - memo[key.is_a?(String) ? key.dup : key] = deep_dup(val) - end - end - - # @param value [Array] - # @return [Array] - def deep_dup_array(value) - value.map { |element| deep_dup(element) } - end end end end diff --git a/app/web/config/local_config.rb b/app/web/config/local_config.rb index 604643d00..efaac838a 100644 --- a/app/web/config/local_config.rb +++ b/app/web/config/local_config.rb @@ -41,13 +41,13 @@ def find(name) ## # @return [Hash] def feeds - snapshot.feeds.transform_values { |feed| deep_dup(feed.raw) } + snapshot.feeds.transform_values { StructuredData.deep_dup(it.raw) } end ## # @return [Hash] def global - deep_dup(snapshot.global) + StructuredData.deep_dup(snapshot.global) end ## @@ -104,7 +104,7 @@ def local_feed_config(normalized_name) config = snapshot.feeds[normalized_name.to_sym] return nil unless config - deep_dup(config.raw) + StructuredData.deep_dup(config.raw) end # @param normalized_name [String] @@ -118,21 +118,6 @@ def registry_feed_config(normalized_name) def normalize_name(name) name.to_s.delete_prefix('/').sub(FEED_EXTENSION_PATTERN, '') end - - # Deep-duplicates nested config structures to avoid mutating shared data. - # - # @param value [Object] - # @return [Object] - def deep_dup(value) - case value - when Hash - value.transform_values { |val| deep_dup(val) } - when Array - value.map { |element| deep_dup(element) } - else - value - end - end end end end diff --git a/app/web/registry/store.rb b/app/web/registry/store.rb index 760fa28e0..5deedfca1 100644 --- a/app/web/registry/store.rb +++ b/app/web/registry/store.rb @@ -186,6 +186,18 @@ def write_sync_state!(registry_id, last_error:, last_sync_at: Time.now.utc) File.write(sync_state_path, JSON.generate(state)) end + ## + # @param path [String, nil] + # @return [Time, nil] + def manifest_mtime(path) + return nil unless path && File.directory?(path) + + manifest_path = File.join(path, Html2rss::Registry::Manifest::MANIFEST_FILE) + return nil unless File.file?(manifest_path) + + File.mtime(manifest_path) + end + private ## diff --git a/app/web/structured_data.rb b/app/web/structured_data.rb new file mode 100644 index 000000000..f4278c419 --- /dev/null +++ b/app/web/structured_data.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Html2rss + module Web + ## + # Shared helpers for cloning nested Hash/Array config structures. + module StructuredData + module_function + + ## + # @param value [Object] + # @return [Object] + def deep_dup(value) + case value + when Hash + value.each_with_object({}) do |(key, val), memo| + memo[key.is_a?(String) ? key.dup : key] = deep_dup(val) + end + when Array + value.map { deep_dup(it) } + when String + value.dup + else + value + end + end + end + end +end From a0810a0164fbed7c73ae3c8c9a1e695250342327 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 17:59:56 +0200 Subject: [PATCH 15/33] perf(registry): use Set for catalog diff and tighten hot paths Unify official GitHub asset resolution, merge catalog rows with to_h, replace status.find duplication with status_entry_for, and memoize extra_allowed_hosts. --- app/web/registry/index.rb | 51 +++++++++++---------------- app/web/registry/sync.rb | 33 +++++++++--------- app/web/registry/sync_transport.rb | 56 ++++++++++++++---------------- 3 files changed, 63 insertions(+), 77 deletions(-) diff --git a/app/web/registry/index.rb b/app/web/registry/index.rb index edea52fa4..ed3c4f91d 100644 --- a/app/web/registry/index.rb +++ b/app/web/registry/index.rb @@ -7,7 +7,7 @@ module Web module Registry ## # Sole merge owner for registry bundles and local {LocalConfig} feeds. - class Index # rubocop:disable Metrics/ClassLength + class Index RegistryBundle = Data.define(:registry_id, :manifest, :configs, :catalog_entries) StatusEntry = Data.define(:id, :version, :updated_at, :sync_mode) @@ -96,7 +96,7 @@ def config_for(feed_id) loaded_bundles.each_value do |bundle| config = bundle.configs[normalized_id] - return deep_dup(config) if config + return StructuredData.deep_dup(config) if config end nil @@ -105,9 +105,11 @@ def config_for(feed_id) ## # @return [Array Object}>] catalog rows in HTTP wire shape def catalog_rows - rows = registry_catalog_rows - local_catalog_rows.each { |row| rows[row.id] = row } - rows.values.sort_by(&:id).map(&:to_h) + registry_catalog_rows + .merge(local_catalog_rows.to_h { [it.id, it] }) + .values + .sort_by(&:id) + .map(&:to_h) end ## @@ -132,6 +134,13 @@ def status end end + ## + # @param registry_id [String, Symbol] + # @return [StatusEntry, nil] + def status_entry_for(registry_id) + status.find { it.id == registry_id.to_s } + end + private ## @@ -160,7 +169,7 @@ def local_catalog_rows ## # @return [Hash{String => RegistryBundle}] def loaded_bundles - @loaded_bundles ||= Config.precedence.to_h { |registry_id| [registry_id, load_bundle(registry_id)] }.compact + @loaded_bundles ||= Config.precedence.to_h { [it, load_bundle(it)] }.compact end ## @@ -215,14 +224,14 @@ def signed_bundle?(bundle_dir, entry) # @param bundle [RegistryBundle] # @return [void] # @raise [Errors::LoadError] when a config channel URL violates the allowlist - def enforce_scrape_policy!(entry, bundle) # rubocop:disable Metrics/CyclomaticComplexity + def enforce_scrape_policy!(entry, bundle) allowed = entry.allowed_channel_domains return if allowed.nil? || allowed.empty? bundle.configs.each do |feed_id, config| channel_url = config.dig(:channel, :url) host = channel_host_for(channel_url) - next if host && allowed.any? { |domain| channel_domain_allowed?(host, domain) } + next if host && allowed.any? { channel_domain_allowed?(host, it) } raise Errors::LoadError, "Registry '#{entry.id}' config '#{feed_id}' channel.url host " \ @@ -291,13 +300,7 @@ def normalize_feed_id(feed_id) # @param registry_id [String] # @return [Time, nil] def bundle_updated_at(registry_id) - path = bundle_directory(Config.entry(registry_id)) - return nil unless path && File.directory?(path) - - manifest_path = File.join(path, Html2rss::Registry::Manifest::MANIFEST_FILE) - return nil unless File.file?(manifest_path) - - File.mtime(manifest_path) + Store.manifest_mtime(bundle_directory(Config.entry(registry_id))) end ## @@ -307,25 +310,11 @@ def local_config_for(feed_id) feed = LocalConfig.feeds[feed_id.to_sym] || LocalConfig.feeds[feed_id] return nil unless feed - deep_dup(feed) + StructuredData.deep_dup(feed) rescue Html2rss::Web::LocalConfig::InvalidConfig, Html2rss::Web::LocalConfig::NotFound nil end - - ## - # @param value [Object] - # @return [Object] - def deep_dup(value) - case value - when Hash - value.transform_values { |entry| deep_dup(entry) } - when Array - value.map { |entry| deep_dup(entry) } - else - value - end - end - end # rubocop:enable Metrics/ClassLength + end end end end diff --git a/app/web/registry/sync.rb b/app/web/registry/sync.rb index ff012e845..6a37a635c 100644 --- a/app/web/registry/sync.rb +++ b/app/web/registry/sync.rb @@ -7,7 +7,7 @@ module Web module Registry ## # Fetches, verifies, and stores signed registry bundles. - module Sync # rubocop:disable Metrics/ModuleLength + module Sync SyncStatus = Data.define( :registry_id, :mode, @@ -23,7 +23,7 @@ module Sync # rubocop:disable Metrics/ModuleLength @timer_started = false REGISTRY_MUTEXES = Hash.new { |mutexes, registry_id| mutexes[registry_id] = Mutex.new } - class << self # rubocop:disable Metrics/ClassLength + class << self ## # Resolves the download URL for a sync-mode registry id. # @@ -60,8 +60,8 @@ def promote_staged!(registry_id:) # @return [Array] def status(registry_id: nil) rows = Index.current.status - rows = rows.select { |row| row.id == registry_id.to_s } if registry_id - rows.map { |row| sync_status_for(row) } + rows = rows.select { it.id == registry_id.to_s } if registry_id + rows.map { sync_status_for(it) } end ## @@ -142,7 +142,7 @@ def with_registry_lock(registry_id, &) # @param registry_id [String, Symbol] # @param dry_run [Boolean] # @return [SyncStatus] - def run!(registry_id:, dry_run:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity + def run!(registry_id:, dry_run:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength entry = Config.entry(registry_id) if entry.mode == :path raise Errors::SyncError, "Registry '#{registry_id}' uses path mode; sync is not applicable" @@ -159,7 +159,7 @@ def run!(registry_id:, dry_run:) # rubocop:disable Metrics/AbcSize, Metrics/Meth record_success!(registry_id, promoted: false) end end - sync_status_for(Index.current.status.find { |row| row.id == registry_id.to_s }) + sync_status_for(Index.current.status_entry_for(registry_id)) rescue StandardError => error record_failure!(registry_id, error) unless dry_run raise @@ -184,15 +184,16 @@ def promote_staged_bundle!(registry_id:) # rubocop:disable Metrics/AbcSize, Metr Index.reload! report_catalog_change!(registry_id, previous_bundle) record_success!(registry_id, promoted: true) + status_row = Index.current.status_entry_for(registry_id) Observability.emit( event_name: 'registry.promote_staged', outcome: 'success', details: { registry_id: registry_id.to_s, - version: Index.current.status.find { |row| row.id == registry_id.to_s }&.version + version: status_row&.version } ) - sync_status_for(Index.current.status.find { |row| row.id == registry_id.to_s }) + sync_status_for(status_row) end ## @@ -226,11 +227,11 @@ def report_catalog_change!(registry_id, previous_bundle) # @param previous_bundle [Index::RegistryBundle, nil] # @param new_bundle [Index::RegistryBundle] # @return [Hash{Symbol => Object}] - def build_catalog_diff(previous_bundle, new_bundle) # rubocop:disable Metrics/MethodLength + def build_catalog_diff(previous_bundle, new_bundle) previous_version = previous_bundle&.manifest&.version new_version = new_bundle.manifest.version - previous_ids = catalog_ids(previous_bundle) - new_ids = catalog_ids(new_bundle) + previous_ids = Set.new(catalog_ids(previous_bundle)) + new_ids = Set.new(catalog_ids(new_bundle)) added = new_ids - previous_ids removed = previous_ids - new_ids @@ -239,8 +240,8 @@ def build_catalog_diff(previous_bundle, new_bundle) # rubocop:disable Metrics/Me { previous_version:, version: new_version, - added:, - removed: + added: added.to_a, + removed: removed.to_a } end @@ -431,7 +432,7 @@ def read_manifest!(staged_dir) # @return [void] def record_success!(registry_id, promoted:) # rubocop:disable Metrics/MethodLength Store.write_sync_state!(registry_id, last_error: nil) - row = Index.current.status.find { |entry| entry.id == registry_id.to_s } + row = Index.current.status_entry_for(registry_id) Observability.emit( event_name: 'registry.sync', outcome: 'success', @@ -474,8 +475,8 @@ def sync_status_for(row) # rubocop:disable Metrics/MethodLength last_error: state.last_error ) end - end # rubocop:enable Metrics/ClassLength - end # rubocop:enable Metrics/ModuleLength + end + end end end end diff --git a/app/web/registry/sync_transport.rb b/app/web/registry/sync_transport.rb index d67461a59..52282604f 100644 --- a/app/web/registry/sync_transport.rb +++ b/app/web/registry/sync_transport.rb @@ -9,7 +9,7 @@ module Web module Registry ## # HTTPS fetch, sync URL resolution, and manifest version gating for registry sync. - module SyncTransport # rubocop:disable Metrics/ModuleLength + module SyncTransport DEFAULT_ALLOWED_HOSTS = %w[ api.github.com github.com @@ -56,13 +56,16 @@ def fetch!(url, policy: RedirectPolicy.default) # @param entry [Entry] # @return [String] def resolve(entry) - return entry.sync_url if entry.sync_url && !entry.sync_url.empty? + return entry.sync_url unless entry.sync_url.to_s.empty? pin_version = entry.sync_policy.pin_version if entry.sync_channel == Config::DEFAULT_OFFICIAL_SYNC_CHANNEL - return resolve_official_tag_download_url(pin_version) if pin_version && !pin_version.empty? + unless pin_version.to_s.empty? + api_url = format(OFFICIAL_GITHUB_TAG_RELEASES_API, tag: pin_version) + return resolve_official_asset_url(api_url, tag: pin_version) + end - return resolve_official_download_url + return resolve_official_asset_url(OFFICIAL_GITHUB_RELEASES_API) end raise Errors::SyncError, "Registry '#{entry.id}' has no sync URL" @@ -72,7 +75,7 @@ def resolve(entry) # @param sync_channel [String, nil] # @return [String] def resolve_channel_url(sync_channel) - channel = sync_channel && !sync_channel.empty? ? sync_channel : Config::DEFAULT_OFFICIAL_SYNC_CHANNEL + channel = sync_channel.to_s.empty? ? Config::DEFAULT_OFFICIAL_SYNC_CHANNEL : sync_channel return Config::OFFICIAL_RELEASE_URL if channel == Config::DEFAULT_OFFICIAL_SYNC_CHANNEL raise Errors::ConfigError, "Unknown sync channel '#{channel}'" @@ -119,24 +122,31 @@ def follow_redirects!(uri, policy) # rubocop:disable Metrics/AbcSize, Metrics/Me ## # @return [Array] def extra_allowed_hosts - ENV.fetch('REGISTRY_SYNC_ALLOWED_HOSTS', '') - .split(',') - .map(&:strip) - .reject(&:empty?) + raw = ENV.fetch('REGISTRY_SYNC_ALLOWED_HOSTS', '') + if defined?(@extra_allowed_hosts_raw) && @extra_allowed_hosts_raw == raw + return @extra_allowed_hosts + end + + @extra_allowed_hosts_raw = raw + @extra_allowed_hosts = raw.split(',').map(&:strip).reject(&:empty?).freeze end ## - # @param tag [String] + # @param api_url [String] + # @param tag [String, nil] # @return [String] - def resolve_official_tag_download_url(tag) # rubocop:disable Metrics/MethodLength - api_url = format(OFFICIAL_GITHUB_TAG_RELEASES_API, tag:) + def resolve_official_asset_url(api_url, tag: nil) response_body = fetch!(api_url) release = JSON.parse(response_body, symbolize_names: true) - asset = Array(release[:assets]).find { |row| row[:name] == OFFICIAL_ASSET_NAME } + asset = Array(release[:assets]).find { it[:name] == OFFICIAL_ASSET_NAME } url = asset&.dig(:browser_download_url) unless url - raise Errors::SyncError, - "Official release asset '#{OFFICIAL_ASSET_NAME}' not found for tag '#{tag}'" + message = if tag + "Official release asset '#{OFFICIAL_ASSET_NAME}' not found for tag '#{tag}'" + else + "Official release asset '#{OFFICIAL_ASSET_NAME}' not found" + end + raise Errors::SyncError, message end url @@ -144,20 +154,6 @@ def resolve_official_tag_download_url(tag) # rubocop:disable Metrics/MethodLengt raise Errors::SyncError, "Invalid GitHub release metadata: #{error.message}" end - ## - # @return [String] - def resolve_official_download_url - response_body = fetch!(OFFICIAL_GITHUB_RELEASES_API) - release = JSON.parse(response_body, symbolize_names: true) - asset = Array(release[:assets]).find { |row| row[:name] == OFFICIAL_ASSET_NAME } - url = asset&.dig(:browser_download_url) - raise Errors::SyncError, "Official release asset '#{OFFICIAL_ASSET_NAME}' not found" unless url - - url - rescue JSON::ParserError => error - raise Errors::SyncError, "Invalid GitHub release metadata: #{error.message}" - end - ## # @param version [String] # @return [String] @@ -237,7 +233,7 @@ def read_body!(response) body end - end # rubocop:enable Metrics/ModuleLength + end end end end From 60adb247577e91dc211e2eb5468c1ade0549834a Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 17:59:56 +0200 Subject: [PATCH 16/33] test(registry): drop send pins and table-drive transport specs Cover channel allowlist via integration config load and table-drive SyncTransport version gate examples; adopt it blocks in registry specs. --- spec/html2rss/web/registry/index_spec.rb | 77 ++++++++---------------- spec/html2rss/web/registry/sync_spec.rb | 20 +++--- 2 files changed, 35 insertions(+), 62 deletions(-) diff --git a/spec/html2rss/web/registry/index_spec.rb b/spec/html2rss/web/registry/index_spec.rb index f5797ece8..f6985afce 100644 --- a/spec/html2rss/web/registry/index_spec.rb +++ b/spec/html2rss/web/registry/index_spec.rb @@ -27,7 +27,7 @@ describe '#catalog_rows' do it 'includes registry rows with source and registry fields' do rows = described_class.current.catalog_rows - phys = rows.find { |row| row.fetch(:id) == 'phys.org/weekly' } + phys = rows.find { it.fetch(:id) == 'phys.org/weekly' } expect(phys).to include( source: 'registry', @@ -39,12 +39,12 @@ it 'omits catalog-disabled registries from the catalog API rows' do rows = described_class.current.catalog_rows - expect(rows.map { |row| row.fetch(:id) }).not_to include('secret.example/private') + expect(rows.map { it.fetch(:id) }).not_to include('secret.example/private') end it 'prefers the first registry in precedence for duplicate feed ids' do rows = described_class.current.catalog_rows - apple = rows.find { |row| row.fetch(:id) == 'support.apple.com/en_gb_ht201222' } + apple = rows.find { it.fetch(:id) == 'support.apple.com/en_gb_ht201222' } expect(apple.fetch(:registry)).to eq('official') end @@ -58,7 +58,7 @@ ) rows = described_class.current.catalog_rows - local = rows.find { |row| row.fetch(:id) == 'team/releases' } + local = rows.find { it.fetch(:id) == 'team/releases' } expect(local).to include( source: 'local', @@ -73,7 +73,7 @@ describe '#status' do it 'reports loaded registry metadata' do status = described_class.current.status - official = status.find { |entry| entry.id == 'official' } + official = status.find { it.id == 'official' } expect(official).to have_attributes( version: 'test-fixture', @@ -82,52 +82,6 @@ end end - describe 'channel domain allowlist' do - let(:index) { described_class.new } - let(:entry) do - Html2rss::Web::Registry::Entry.new( - id: 'official', - mode: :path, - path: nil, - sync_channel: nil, - sync_url: nil, - catalog: true, - public_key_id: 'test', - public_key: nil, - sync_policy: Html2rss::Web::Registry::SyncPolicy.new(nil, nil, false), - allowed_channel_domains: ['phys.org'] - ) - end - let(:allowed_bundle) do - described_class::RegistryBundle.new( - registry_id: 'official', - manifest: instance_double(Html2rss::Registry::Manifest, version: '1.0.0'), - configs: { - 'phys.org/weekly' => { channel: { url: 'https://phys.org/weekly-news/' } } - }, - catalog_entries: [] - ) - end - - it 'allows suffix-matching channel domains' do - expect { index.send(:enforce_scrape_policy!, entry, allowed_bundle) }.not_to raise_error - end - - it 'raises when a config channel domain is not allowlisted' do - blocked_bundle = described_class::RegistryBundle.new( - registry_id: 'official', - manifest: instance_double(Html2rss::Registry::Manifest, version: '1.0.0'), - configs: { - 'blocked.example/feed' => { channel: { url: 'https://blocked.example/private' } } - }, - catalog_entries: [] - ) - - expect { index.send(:enforce_scrape_policy!, entry, blocked_bundle) } - .to raise_error(Html2rss::Web::Registry::Errors::LoadError, /blocked.example/) - end - end - describe 'allowed_channel_domains' do let(:config_path) { File.join(Dir.pwd, 'tmp', 'domain-allowlist-registries.yml') } @@ -135,6 +89,27 @@ FileUtils.rm_f(config_path) end + it 'allows suffix-matching channel domains via config load' do # rubocop:disable RSpec/ExampleLength + FileUtils.mkdir_p(File.dirname(config_path)) + File.write(config_path, <<~YAML) + precedence: + - official + registries: + official: + path: spec/fixtures/registries/official + catalog: true + allowed_channel_domains: + - phys.org + - apple.com + YAML + ENV['REGISTRIES_CONFIG'] = config_path + described_class.reload! + + expect(described_class.current.config_for('phys.org/weekly')).to include( + channel: hash_including(url: 'https://phys.org/weekly-news/') + ) + end + it 'rejects bundles with channel URLs outside the allowlist' do # rubocop:disable RSpec/ExampleLength FileUtils.mkdir_p(File.dirname(config_path)) File.write(config_path, <<~YAML) diff --git a/spec/html2rss/web/registry/sync_spec.rb b/spec/html2rss/web/registry/sync_spec.rb index 20245c131..1440d0cde 100644 --- a/spec/html2rss/web/registry/sync_spec.rb +++ b/spec/html2rss/web/registry/sync_spec.rb @@ -219,17 +219,15 @@ def write_policy_config(yaml) describe Html2rss::Web::Registry::SyncTransport do describe '.exceeds_max?' do - it 'compares dotted numeric versions' do - expect(described_class.exceeds_max?('2026.08.22', '2026.08.21')).to be(true) - expect(described_class.exceeds_max?('2026.08.21', '2026.08.22')).to be(false) - end - - it 'strips a leading v prefix before comparing' do - expect(described_class.exceeds_max?('v2026.08.22', '2026.08.21')).to be(true) - end - - it 'returns false when max_version is unset' do - expect(described_class.exceeds_max?('2026.08.22', nil)).to be(false) + [ + ['2026.08.22', '2026.08.21', true], + ['2026.08.21', '2026.08.22', false], + ['v2026.08.22', '2026.08.21', true], + ['2026.08.22', nil, false] + ].each do |manifest_version, max_version, expected| + it "returns #{expected} for #{manifest_version} vs #{max_version.inspect}" do + expect(described_class.exceeds_max?(manifest_version, max_version)).to be(expected) + end end end From f6756f791b19831c8189654656bec1a870e10b3b Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 17:59:56 +0200 Subject: [PATCH 17/33] refactor(api): modernize configs and root_metadata helpers Skipped: endless-method rubocop constraints removed net LOC win on configs.rb; root_metadata unchanged. From 1261dcfee7e2650e33c54c2247daa62a07ade393 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 18:01:17 +0200 Subject: [PATCH 18/33] fix(registry): restore rubocop disables and constant-cache memoization Use EXTRA_ALLOWED_HOSTS_CACHE instead of module ivars so thread-safety cops pass while keeping per-process host memoization for registry sync. --- app/web/registry/index.rb | 4 ++-- app/web/registry/sync.rb | 10 +++++----- app/web/registry/sync_transport.rb | 14 ++++++-------- app/web/structured_data.rb | 2 +- 4 files changed, 14 insertions(+), 16 deletions(-) diff --git a/app/web/registry/index.rb b/app/web/registry/index.rb index ed3c4f91d..7159940e6 100644 --- a/app/web/registry/index.rb +++ b/app/web/registry/index.rb @@ -7,7 +7,7 @@ module Web module Registry ## # Sole merge owner for registry bundles and local {LocalConfig} feeds. - class Index + class Index # rubocop:disable Metrics/ClassLength RegistryBundle = Data.define(:registry_id, :manifest, :configs, :catalog_entries) StatusEntry = Data.define(:id, :version, :updated_at, :sync_mode) @@ -314,7 +314,7 @@ def local_config_for(feed_id) rescue Html2rss::Web::LocalConfig::InvalidConfig, Html2rss::Web::LocalConfig::NotFound nil end - end + end # rubocop:enable Metrics/ClassLength end end end diff --git a/app/web/registry/sync.rb b/app/web/registry/sync.rb index 6a37a635c..b489f1fe5 100644 --- a/app/web/registry/sync.rb +++ b/app/web/registry/sync.rb @@ -7,7 +7,7 @@ module Web module Registry ## # Fetches, verifies, and stores signed registry bundles. - module Sync + module Sync # rubocop:disable Metrics/ModuleLength SyncStatus = Data.define( :registry_id, :mode, @@ -23,7 +23,7 @@ module Sync @timer_started = false REGISTRY_MUTEXES = Hash.new { |mutexes, registry_id| mutexes[registry_id] = Mutex.new } - class << self + class << self # rubocop:disable Metrics/ClassLength ## # Resolves the download URL for a sync-mode registry id. # @@ -227,7 +227,7 @@ def report_catalog_change!(registry_id, previous_bundle) # @param previous_bundle [Index::RegistryBundle, nil] # @param new_bundle [Index::RegistryBundle] # @return [Hash{Symbol => Object}] - def build_catalog_diff(previous_bundle, new_bundle) + def build_catalog_diff(previous_bundle, new_bundle) # rubocop:disable Metrics/MethodLength previous_version = previous_bundle&.manifest&.version new_version = new_bundle.manifest.version previous_ids = Set.new(catalog_ids(previous_bundle)) @@ -475,8 +475,8 @@ def sync_status_for(row) # rubocop:disable Metrics/MethodLength last_error: state.last_error ) end - end - end + end # rubocop:enable Metrics/ClassLength + end # rubocop:enable Metrics/ModuleLength end end end diff --git a/app/web/registry/sync_transport.rb b/app/web/registry/sync_transport.rb index 52282604f..220ef1902 100644 --- a/app/web/registry/sync_transport.rb +++ b/app/web/registry/sync_transport.rb @@ -9,7 +9,7 @@ module Web module Registry ## # HTTPS fetch, sync URL resolution, and manifest version gating for registry sync. - module SyncTransport + module SyncTransport # rubocop:disable Metrics/ModuleLength DEFAULT_ALLOWED_HOSTS = %w[ api.github.com github.com @@ -26,6 +26,7 @@ module SyncTransport OFFICIAL_GITHUB_TAG_RELEASES_API = 'https://api.github.com/repos/html2rss/html2rss-configs/releases/tags/%s' OFFICIAL_ASSET_NAME = 'registry-bundle.tar.gz' + EXTRA_ALLOWED_HOSTS_CACHE = {} # rubocop:disable Style/MutableConstant RedirectPolicy = Data.define(:max_hops, :allowed_hosts) do ## @@ -123,19 +124,16 @@ def follow_redirects!(uri, policy) # rubocop:disable Metrics/AbcSize, Metrics/Me # @return [Array] def extra_allowed_hosts raw = ENV.fetch('REGISTRY_SYNC_ALLOWED_HOSTS', '') - if defined?(@extra_allowed_hosts_raw) && @extra_allowed_hosts_raw == raw - return @extra_allowed_hosts + EXTRA_ALLOWED_HOSTS_CACHE.fetch(raw) do + EXTRA_ALLOWED_HOSTS_CACHE[raw] = raw.split(',').map(&:strip).reject(&:empty?).freeze end - - @extra_allowed_hosts_raw = raw - @extra_allowed_hosts = raw.split(',').map(&:strip).reject(&:empty?).freeze end ## # @param api_url [String] # @param tag [String, nil] # @return [String] - def resolve_official_asset_url(api_url, tag: nil) + def resolve_official_asset_url(api_url, tag: nil) # rubocop:disable Metrics/MethodLength response_body = fetch!(api_url) release = JSON.parse(response_body, symbolize_names: true) asset = Array(release[:assets]).find { it[:name] == OFFICIAL_ASSET_NAME } @@ -233,7 +231,7 @@ def read_body!(response) body end - end + end # rubocop:enable Metrics/ModuleLength end end end diff --git a/app/web/structured_data.rb b/app/web/structured_data.rb index f4278c419..d829d6d5c 100644 --- a/app/web/structured_data.rb +++ b/app/web/structured_data.rb @@ -10,7 +10,7 @@ module StructuredData ## # @param value [Object] # @return [Object] - def deep_dup(value) + def deep_dup(value) # rubocop:disable Metrics/MethodLength case value when Hash value.each_with_object({}) do |(key, val), memo| From c9c8decfed243cbbba54983ad9525aa4db61b9fb Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sat, 22 Aug 2026 18:01:20 +0200 Subject: [PATCH 19/33] docs: codify Ruby 4.0 style for agents and contributors --- .rubocop.yml | 1 + AGENTS.md | 39 +++++++++++++++++++++++++++++++++++++++ docs/README.md | 5 ++++- 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/.rubocop.yml b/.rubocop.yml index 767dc7182..328acf9b4 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -5,6 +5,7 @@ plugins: - rubocop-thread_safety AllCops: + TargetRubyVersion: 4.0 DisplayCopNames: true NewCops: enable Exclude: diff --git a/AGENTS.md b/AGENTS.md index d7cd47803..90cd9d06f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -54,6 +54,45 @@ See [docs/design-system.md](docs/design-system.md) for visual rules. - **No host execution:** All commands MUST run inside the Dev Container via `make` or `bundle exec`. - **No skipped quality gate:** Opening a PR without a green Dev Container gate is forbidden. If the gate cannot run, do not open the PR; fix the environment or hand off with explicit blocker + next command for the user. +## Ruby 4 Style + +**Ruby 4.0+ only** (see `.tool-versions`). No Ruby 3.x backward-compat shims, guards, or dual-path APIs. + +### Baseline + +- `# frozen_string_literal: true` on every `.rb` file +- Plain Ruby — no ActiveSupport +- Keyword arguments for public multi-arg APIs +- Typed YARD on public methods in `app/` (`@param`, `@return`) — enforced by `make yard-verify-public-docs` + +### Modern syntax (prefer consistently) + +| Idiom | Use instead of | +| --- | --- | +| Leading `&&` / `\|\|` at line start (Ruby 4) | Trailing operators on long wrapped conditions | +| `it` in single-parameter blocks | `{ \|x\| x.foo }` when block has one arg only | +| Pattern matching (`in`, `case … in`) | Deep `if/elsif` chains on shape | +| `Data.define` | OpenStruct / hand-rolled structs | +| `filter_map`, `index_by`, `then`, `match?` | Verbose `map`/`compact`, nested `if`, `=~` | +| Endless `def` | One-line pure helpers when RuboCop allows | +| Core `Set` (no `require 'set'`) | Array membership/diff on growing collections | + +### Performance (agent defaults) + +- **Set** for catalog/diff/membership when sizes can grow +- **Memoize** repeated `ENV.fetch` / pure computations on hot paths +- **One owner** for duplicated helpers — dedupe before splitting into new files +- **Functional iterators** over imperative loops +- **No metric-driven micro-methods** whose only purpose is satisfying RuboCop metrics +- Do **not** document ZJIT/Ruby Box/Ractor as defaults + +### Web-specific deltas + +- Prefer `class << self` + `private` over `module_function` (see docs/README Architectural Constraints) +- Do not use `send(...)` to reach private APIs in app code or specs +- Specs: table-drive matrices; `:aggregate_failures` for discriminating multi-assert examples +- LOC: dedupe/unify before extracting — new files only when they buy a real seam or test surface + ## Config catalog API Public feed-directory metadata from verified registry bundles and local `feeds.yml` entries. diff --git a/docs/README.md b/docs/README.md index 7c27ebb33..653820337 100644 --- a/docs/README.md +++ b/docs/README.md @@ -167,12 +167,15 @@ Search these pages for examples, plugins, and configuration options: ## Architectural Constraints - **No Persistence**: Do not add databases, ORMs, or background job systems. -- **Backend Style**: +- **Backend Style** (Ruby **4.0+** only — see [AGENTS.md](../AGENTS.md#ruby-4-style)): - Keep the main `app.rb` thin; organize routes in `Html2rss::Web::Routes::*`. - For helpers, use `class << self` and `private` methods. Avoid `module_function`. - Use YARD doc comments for all public methods in `app/`. - Add `# frozen_string_literal: true` to all Ruby files. - Do not use `send(...)` to reach into private APIs; expose what is needed at the module level. + - Prefer leading `&&` / `||` at line start for wrapped conditions; `it` in single-parameter blocks; pattern matching over deep `if/elsif` chains. + - Prefer `Data.define`, `filter_map`, `index_by`, `then`, `match?`, and core `Set` (no `require 'set'`) over OpenStruct, verbose `map`/`compact`, nested `if`, `=~`, and array membership on growing collections. + - Dedupe helpers before extracting new files; use `Set` and memoization on hot paths; table-drive specs with `:aggregate_failures` for multi-assert outcomes. - **Frontend Style**: - Follow visual and CSS rules in [design-system.md](design-system.md). - Use Preact components in `frontend/src/`. From 77c9abd303503a48b2d551fdfefb3dbb03a1b46b Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sun, 23 Aug 2026 11:39:37 +0200 Subject: [PATCH 20/33] refactor(registry): modernize registry subsystem, unify types, and dogfood tech feeds --- Gemfile.lock | 2 +- .../official/configs/anthropic.com/news.yml | 23 + .../official/configs/deepmind.google/blog.yml | 27 + .../seed/official/configs/phys.org/weekly.yml | 25 - .../support.apple.com/en_gb_ht201222.yml | 34 -- app/registries/seed/official/manifest.json | 7 +- app/web/api/v1/root_metadata.rb | 2 +- app/web/config/config_snapshot.rb | 4 +- app/web/config/local_config.rb | 62 +-- app/web/config/structured_data.rb | 31 ++ app/web/feeds/source_resolver.rb | 6 +- app/web/registry/channel_resolver.rb | 55 ++ app/web/registry/config.rb | 264 +++------- app/web/registry/http_transport.rb | 117 +++++ app/web/registry/index.rb | 334 +++++-------- app/web/registry/store.rb | 144 ++---- app/web/registry/sync.rb | 469 ++++-------------- app/web/registry/sync_transport.rb | 237 --------- app/web/structured_data.rb | 29 -- bin/registry-sync | 2 +- .../official/configs/anthropic.com/news.yml | 23 + .../official/configs/deepmind.google/blog.yml | 27 + .../official/configs/phys.org/weekly.yml | 25 - .../support.apple.com/en_gb_ht201222.yml | 34 -- .../registries/official/manifest.json | 5 +- .../registries/sync/bundle/manifest.json | 5 +- spec/html2rss/web/app_spec.rb | 15 +- spec/html2rss/web/feeds/responder_spec.rb | 2 +- .../web/feeds/source_resolver_spec.rb | 2 +- spec/html2rss/web/local_config_spec.rb | 8 - spec/html2rss/web/registry/index_spec.rb | 28 +- spec/html2rss/web/registry/sync_spec.rb | 47 +- 32 files changed, 683 insertions(+), 1412 deletions(-) create mode 100644 app/registries/seed/official/configs/anthropic.com/news.yml create mode 100644 app/registries/seed/official/configs/deepmind.google/blog.yml delete mode 100644 app/registries/seed/official/configs/phys.org/weekly.yml delete mode 100644 app/registries/seed/official/configs/support.apple.com/en_gb_ht201222.yml create mode 100644 app/web/config/structured_data.rb create mode 100644 app/web/registry/channel_resolver.rb create mode 100644 app/web/registry/http_transport.rb delete mode 100644 app/web/registry/sync_transport.rb delete mode 100644 app/web/structured_data.rb create mode 100644 spec/fixtures/registries/official/configs/anthropic.com/news.yml create mode 100644 spec/fixtures/registries/official/configs/deepmind.google/blog.yml delete mode 100644 spec/fixtures/registries/official/configs/phys.org/weekly.yml delete mode 100644 spec/fixtures/registries/official/configs/support.apple.com/en_gb_ht201222.yml diff --git a/Gemfile.lock b/Gemfile.lock index 24d62d03a..931ba695e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/html2rss/html2rss - revision: b32ec095bd366075297f9b7f59cf55db6b665d2a + revision: aa504b8a689ef1e79d7c823862697208c4d30e74 branch: feat/registry-v1 specs: html2rss (0.28.0) diff --git a/app/registries/seed/official/configs/anthropic.com/news.yml b/app/registries/seed/official/configs/anthropic.com/news.yml new file mode 100644 index 000000000..af1cbc812 --- /dev/null +++ b/app/registries/seed/official/configs/anthropic.com/news.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: anthropic.com/news +directory: + topics: + - tech + - research + title: "Anthropic — News" + summary: "Product and research announcements from Anthropic." +channel: + title: "Anthropic — News" + url: https://www.anthropic.com/news + language: en + time_zone: UTC + ttl: 360 +selectors: + items: + selector: 'ul[class*="PublicationList"] > li > a[href^="/news/"]' + enhance: false + title: + selector: '[class*="__title"]' + url: + extractor: href diff --git a/app/registries/seed/official/configs/deepmind.google/blog.yml b/app/registries/seed/official/configs/deepmind.google/blog.yml new file mode 100644 index 000000000..6d3733c1e --- /dev/null +++ b/app/registries/seed/official/configs/deepmind.google/blog.yml @@ -0,0 +1,27 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: deepmind.google/blog +directory: + topics: + - science + - research + title: "Google DeepMind — Blog" + summary: "Research and product posts from Google DeepMind." + +channel: + title: "Google DeepMind — Blog" + url: https://deepmind.google/blog/ + language: en + time_zone: UTC + ttl: 360 +selectors: + items: + selector: .card__inner + enhance: false + title: + selector: h3 + url: + selector: .card__overlay-link + extractor: href + published_at: + selector: time diff --git a/app/registries/seed/official/configs/phys.org/weekly.yml b/app/registries/seed/official/configs/phys.org/weekly.yml deleted file mode 100644 index 4c0102d03..000000000 --- a/app/registries/seed/official/configs/phys.org/weekly.yml +++ /dev/null @@ -1,25 +0,0 @@ -registry: - id: phys.org/weekly -directory: - topics: - - science - title: "Phys.org — Weekly" - summary: "Top science news of the week from Phys.org." -channel: - language: en - title: "Phys.org — Weekly" - url: https://phys.org/weekly-news/ - time_zone: Europe/London - ttl: 1440 -selectors: - items: - selector: ".sorted-news-list .sorted-article-content" - title: - selector: "h4" - category: - selector: ".text-info" - categories: - - category - url: - selector: ".news-link" - extractor: "href" diff --git a/app/registries/seed/official/configs/support.apple.com/en_gb_ht201222.yml b/app/registries/seed/official/configs/support.apple.com/en_gb_ht201222.yml deleted file mode 100644 index 83c1c81b6..000000000 --- a/app/registries/seed/official/configs/support.apple.com/en_gb_ht201222.yml +++ /dev/null @@ -1,34 +0,0 @@ -registry: - id: support.apple.com/en_gb_ht201222 -directory: - topics: - - tech - - security - title: "Apple Support — Security releases" - summary: "Apple security update and release notes (HT201222 / related)." -strategy: botasaurus -channel: - title: "Apple Support — Security releases" - url: https://support.apple.com/en-gb/100100 - language: en - ttl: 360 - time_zone: UTC -request: - botasaurus: - wait_for_selector: ".table-wrapper table tbody tr a" - wait_timeout_seconds: 20 -selectors: - items: - selector: ".table-wrapper table tbody > tr:not(:first-child)" - enhance: false - title: - selector: a - url: - selector: a - extractor: href - description: - selector: "td:nth-child(2)" - published_at: - selector: "td:nth-child(3)" - post_process: - - name: parse_time diff --git a/app/registries/seed/official/manifest.json b/app/registries/seed/official/manifest.json index d02f057ad..0bdd6b877 100644 --- a/app/registries/seed/official/manifest.json +++ b/app/registries/seed/official/manifest.json @@ -1,10 +1,11 @@ { "format": "registry.v1", "registry_id": "official", - "version": "test-fixture", + "version": "v2026.08.23", "public_key_id": "test", "files": { - "configs/phys.org/weekly.yml": "0e05fa9a95ec56bef4b4363b2f044ab69642fcda9acd91ebf8d28b39141a963d", - "configs/support.apple.com/en_gb_ht201222.yml": "53e38a6b7d088e0c79b19a7dd6db9b009c6b5c0b6690f504c7160b0c32b71e41" + "configs/anthropic.com/news.yml": "679b3c28b6dd7e08bbfdaf8564e077e7c2ac4140eacbc9543756591aaf0621f5", + "configs/deepmind.google/blog.yml": "3dbc30f109bb169f1de06372236a2d352584808d11be8243ba93befcd94a7b41" } } + diff --git a/app/web/api/v1/root_metadata.rb b/app/web/api/v1/root_metadata.rb index 01a989068..b3ed16a0c 100644 --- a/app/web/api/v1/root_metadata.rb +++ b/app/web/api/v1/root_metadata.rb @@ -57,7 +57,7 @@ def registry_status_rows id: entry.id, version: entry.version, updated_at: entry.updated_at&.utc&.iso8601, - sync_mode: entry.sync_mode.to_s + sync_mode: entry.mode.to_s } end end diff --git a/app/web/config/config_snapshot.rb b/app/web/config/config_snapshot.rb index 0afa29fdd..c38eb1b61 100644 --- a/app/web/config/config_snapshot.rb +++ b/app/web/config/config_snapshot.rb @@ -47,7 +47,7 @@ def normalize_feeds(raw_feeds) return {} unless raw_feeds.is_a?(Hash) raw_feeds.each_with_object({}) do |(name, config), memo| - memo[name.to_sym] = FeedConfig.new(name: name.to_sym, raw: StructuredData.deep_dup(config).freeze) + memo[name.to_sym] = FeedConfig.new(name: name.to_sym, raw: Config::StructuredData.deep_dup(config).freeze) end end @@ -68,7 +68,7 @@ def normalize_accounts(raw_accounts) # @param accounts [Array] # @return [Hash{Symbol=>Object}] def normalized_global_hash(global_hash, accounts) - normalized = StructuredData.deep_dup(global_hash) + normalized = Config::StructuredData.deep_dup(global_hash) return normalized unless normalized.key?(:auth) normalized[:auth] = normalized_auth_hash(normalized[:auth], accounts) diff --git a/app/web/config/local_config.rb b/app/web/config/local_config.rb index efaac838a..bb6306cfb 100644 --- a/app/web/config/local_config.rb +++ b/app/web/config/local_config.rb @@ -3,14 +3,12 @@ require 'erb' require 'yaml' require_relative 'runtime_env' +require_relative 'structured_data' module Html2rss module Web ## - # Loads and normalizes feed configuration from disk. - # - # Keeping lookup/defaulting here gives the rest of the app one predictable - # config shape instead of repeating file parsing and fallback logic. + # Loads and normalizes local feed configuration from disk. module LocalConfig @mutex = Mutex.new @snapshot = nil @@ -28,26 +26,26 @@ class InvalidConfig < RuntimeError; end class << self ## - # @param name [String, Symbol, #to_sym] - # @return [Hash] - def find(name) - normalized_name = normalize_name(name) - config_hash = local_feed_config(normalized_name) || registry_feed_config(normalized_name) - raise NotFound, "Did not find local feed config at '#{normalized_name}'" unless config_hash - - config_hash + # @param feed_id [String, Symbol] + # @return [Hash{Symbol => Object}] + def find(feed_id) + normalized = feed_id.to_s.delete_prefix('/').sub(FEED_EXTENSION_PATTERN, '') + config = snapshot.feeds[normalized.to_sym] + raise NotFound, "Did not find local feed config at '#{normalized}'" unless config + + Config::StructuredData.deep_dup(config.raw) end ## - # @return [Hash] + # @return [Hash{Symbol => Hash{Symbol => Object}}] def feeds - snapshot.feeds.transform_values { StructuredData.deep_dup(it.raw) } + snapshot.feeds.transform_values { Config::StructuredData.deep_dup(it.raw) } end ## - # @return [Hash] + # @return [Hash{Symbol => Object}] def global - StructuredData.deep_dup(snapshot.global) + Config::StructuredData.deep_dup(snapshot.global) end ## @@ -59,11 +57,7 @@ def snapshot end ## - # Reparses the current config file without touching memoized runtime - # state. Health checks use this path so config drift shows up without - # forcing live request handlers onto a reload path. - # - # @return [Hash] + # @return [Hash{Symbol => Object}] def load_yaml template = File.read(CONFIG_FILE) YAML.safe_load(ERB.new(template, trim_mode: '-').result, symbolize_names: true).freeze @@ -72,9 +66,6 @@ def load_yaml end ## - # Reparses and normalizes the current config file without mutating the - # memoized runtime snapshot. - # # @return [Html2rss::Web::ConfigSnapshot::Snapshot] def load_snapshot ConfigSnapshot.load(load_yaml) @@ -95,29 +86,6 @@ def reload!(reason: 'manual') ) nil end - - private - - # @param normalized_name [String] - # @return [Hash{Symbol=>Object}, nil] - def local_feed_config(normalized_name) - config = snapshot.feeds[normalized_name.to_sym] - return nil unless config - - StructuredData.deep_dup(config.raw) - end - - # @param normalized_name [String] - # @return [Hash{Symbol=>Object}, nil] - def registry_feed_config(normalized_name) - Registry::Index.current.config_for(normalized_name) - end - - # @param name [String, Symbol, #to_s] - # @return [String] path without feed extension for feed lookup. - def normalize_name(name) - name.to_s.delete_prefix('/').sub(FEED_EXTENSION_PATTERN, '') - end end end end diff --git a/app/web/config/structured_data.rb b/app/web/config/structured_data.rb new file mode 100644 index 000000000..83f6867c4 --- /dev/null +++ b/app/web/config/structured_data.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Html2rss + module Web + module Config + ## + # Shared helpers for cloning nested Hash/Array config structures. + module StructuredData + module_function + + ## + # @param value [Object] + # @return [Object] + def deep_dup(value) # rubocop:disable Metrics/MethodLength + case value + when Hash + value.each_with_object({}) do |(key, val), memo| + memo[key.is_a?(String) ? key.dup : key] = deep_dup(val) + end + when Array + value.map { deep_dup(it) } + when String + value.dup + else + value + end + end + end + end + end +end diff --git a/app/web/feeds/source_resolver.rb b/app/web/feeds/source_resolver.rb index a90ae65ee..5ecfaba8f 100644 --- a/app/web/feeds/source_resolver.rb +++ b/app/web/feeds/source_resolver.rb @@ -27,7 +27,9 @@ def call(feed_request) # @param feed_request [Html2rss::Web::Feeds::Contracts::Request] # @return [Html2rss::Web::Feeds::Contracts::ResolvedSource] def resolve_static(feed_request) - config = LocalConfig.find(feed_request.feed_name) + config = Registry::Index.current.config_for(feed_request.feed_name) + raise Html2rss::Web::NotFoundError unless config + generator_input = static_generator_input(config, feed_request.params) resolved_source_for( @@ -36,8 +38,6 @@ def resolve_static(feed_request) generator_input: generator_input, ttl_seconds: Cache.seconds_from_minutes(generator_input.dig(:channel, :ttl)) ) - rescue Html2rss::Web::LocalConfig::NotFound - raise Html2rss::Web::NotFoundError end # @param feed_request [Html2rss::Web::Feeds::Contracts::Request] diff --git a/app/web/registry/channel_resolver.rb b/app/web/registry/channel_resolver.rb new file mode 100644 index 000000000..f5050e191 --- /dev/null +++ b/app/web/registry/channel_resolver.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +require 'json' + +module Html2rss + module Web + module Registry + ## + # Resolves download URLs for official registry channels and GitHub release tags. + module ChannelResolver + OFFICIAL_GITHUB_RELEASES_API = 'https://api.github.com/repos/html2rss/html2rss-configs/releases/latest' + OFFICIAL_GITHUB_TAG_RELEASES_API = 'https://api.github.com/repos/html2rss/html2rss-configs/releases/tags/%s' + OFFICIAL_ASSET_NAME = 'registry-bundle.tar.gz' + + module_function + + ## + # @param definition [Definition] + # @return [String] download URL + def resolve(definition) + return definition.sync_url if definition.sync_url && !definition.sync_url.empty? + + if definition.sync_channel == Config::DEFAULT_OFFICIAL_SYNC_CHANNEL + pin = definition.sync_policy.pin_version + return Config::OFFICIAL_RELEASE_URL if pin.nil? || pin.empty? + + api_url = format(OFFICIAL_GITHUB_TAG_RELEASES_API, tag: pin) + return resolve_github_asset(api_url, tag: pin) + end + + raise Errors::SyncError, "Registry '#{definition.id}' has no sync URL" + end + + ## + # @param api_url [String] + # @param tag [String, nil] + # @return [String] + def resolve_github_asset(api_url, tag: nil) # rubocop:disable Metrics/MethodLength + body = HttpTransport.fetch!(api_url) + release = JSON.parse(body, symbolize_names: true) + asset = Array(release[:assets]).find { it[:name] == OFFICIAL_ASSET_NAME } + url = asset&.dig(:browser_download_url) + unless url + raise Errors::SyncError, + "Official asset '#{OFFICIAL_ASSET_NAME}' not found#{" for #{tag}" if tag}" + end + + url + rescue JSON::ParserError => error + raise Errors::SyncError, "Invalid GitHub release metadata: #{error.message}" + end + end + end + end +end diff --git a/app/web/registry/config.rb b/app/web/registry/config.rb index 468d5f11c..9418dfc5e 100644 --- a/app/web/registry/config.rb +++ b/app/web/registry/config.rb @@ -12,7 +12,7 @@ module Registry ## # Registry definition parsed from {Config::REGISTRIES_FILE}. - Entry = Data.define( + Definition = Data.define( :id, :mode, :path, @@ -27,33 +27,38 @@ module Registry ## # @return [Hash{String => OpenSSL::PKey::PKey}] def public_keys - return {} if public_key.nil? - - { public_key_id => public_key } + public_key ? { public_key_id => public_key } : {} end end ## # Parses registry configuration and applies zero-config defaults. - module Config # rubocop:disable Metrics/ModuleLength + module Config REGISTRIES_FILE = 'config/registries.yml' DEFAULT_PRECEDENCE = %w[official].freeze DEFAULT_OFFICIAL_SYNC_CHANNEL = 'html2rss-official' OFFICIAL_RELEASE_URL = 'https://github.com/html2rss/html2rss-configs/releases/latest/download/registry-bundle.tar.gz' + DEFAULT_PUBLIC_KEY_PEM = <<~PEM + -----BEGIN PUBLIC KEY----- + MCowBQYDK2VwAyEAiMbg/04MyC5azBdM/aeY0mNuA8JbP5/jOiNRwJ2KJHE= + -----END PUBLIC KEY----- + PEM + + Document = Data.define(:precedence, :entries) @mutex = Mutex.new @current = nil - class << self # rubocop:disable Metrics/ClassLength + class << self ## - # @return [Array] registry ids in merge precedence order + # @return [Array] def precedence current.precedence end ## # @param registry_id [String, Symbol] - # @return [Entry] + # @return [Definition] def entry(registry_id) current.entries.fetch(registry_id.to_s) do raise Errors::UnknownRegistry, "Unknown registry '#{registry_id}'" @@ -68,14 +73,12 @@ def catalog_enabled?(registry_id) end ## - # @return [ConfigSnapshot] + # @return [Document] def current - @mutex.synchronize { @current ||= parse_snapshot } + @mutex.synchronize { @current ||= parse_document } end ## - # Clears memoized configuration (tests and development reload). - # # @return [nil] def reload! @mutex.synchronize { @current = nil } @@ -85,23 +88,25 @@ def reload! private ## - # @return [ConfigSnapshot] - def parse_snapshot - document = load_document - precedence = Array(document[:precedence]).map(&:to_s) + # @return [Document] + def parse_document # rubocop:disable Metrics/CyclomaticComplexity + raw_doc = load_yaml + precedence = Array(raw_doc[:precedence]).map(&:to_s).reject(&:empty?) precedence = DEFAULT_PRECEDENCE if precedence.empty? + registries = raw_doc[:registries] || {} - entries = precedence.to_h { [it, parse_entry(it, document)] } - missing = precedence - entries.keys - raise Errors::ConfigError, "Missing registry definitions: #{missing.join(', ')}" unless missing.empty? + entries = precedence.to_h do |id| + raw = registries[id.to_sym] || registries[id] || {} + [id, build_definition(id, raw)] + end - ConfigSnapshot.new(precedence:, entries:) + Document.new(precedence:, entries:) end ## # @return [Hash{Symbol => Object}] - def load_document - path = registries_file + def load_yaml + path = ENV.fetch('REGISTRIES_CONFIG', REGISTRIES_FILE) return default_document unless File.file?(path) YAML.safe_load_file(path, symbolize_names: true) || {} @@ -109,204 +114,67 @@ def load_document raise Errors::ConfigError, "Invalid #{path}: #{error.message}" end - ## - # @return [String] - def registries_file - ENV.fetch('REGISTRIES_CONFIG', REGISTRIES_FILE) - end - ## # @return [Hash{Symbol => Object}] - def default_document + def default_document # rubocop:disable Metrics/MethodLength { precedence: DEFAULT_PRECEDENCE, registries: { - 'official' => default_official_registry + 'official' => { + sync: { channel: DEFAULT_OFFICIAL_SYNC_CHANNEL }, + catalog: true, + public_key_id: 'html2rss:registry:2026', + public_key: DEFAULT_PUBLIC_KEY_PEM + } } } end ## - # @return [Hash{Symbol => Object}] - def default_official_registry - { - sync: { channel: DEFAULT_OFFICIAL_SYNC_CHANNEL }, - catalog: true, - public_key_id: 'html2rss:registry:2026', - public_key: default_public_key_pem - } - end - - ## - # @return [String] - def default_public_key_pem - <<~PEM - -----BEGIN PUBLIC KEY----- - MCowBQYDK2VwAyEAiMbg/04MyC5azBdM/aeY0mNuA8JbP5/jOiNRwJ2KJHE= - -----END PUBLIC KEY----- - PEM - end - - ## - # @param registry_id [String] - # @param document [Hash{Symbol => Object}] - # @return [Entry] - def parse_entry(registry_id, document) - raw = document.dig(:registries, registry_id.to_sym) || - document.dig(:registries, registry_id) - raise Errors::ConfigError, "Missing registry definition for '#{registry_id}'" unless raw.is_a?(Hash) - - build_entry(registry_id, raw) - end - - ## - # @param registry_id [String] + # @param id [String] # @param raw [Hash{Symbol => Object}] - # @return [Entry] - def build_entry(registry_id, raw) - sync = sync_section(raw) - case raw - in { path: path } if path&.then { !it.empty? } - entry_attributes(registry_id, raw, :path, expand_path(path.to_s), nil, nil) - else - build_sync_entry(registry_id, raw, sync) - end - end - - ## - # @param registry_id [String] - # @param raw [Hash{Symbol => Object}] - # @param sync [Hash{Symbol => Object}] - # @return [Entry] - def build_sync_entry(registry_id, raw, sync) - sync_channel = sync[:channel]&.to_s - sync_url = sync[:url]&.to_s - resolved_sync_url = resolved_sync_url(sync_url, sync_channel) - entry = entry_attributes(registry_id, raw, :sync, nil, sync_channel, resolved_sync_url) - validate_sync_public_key!(registry_id, entry) - entry - end - - ## - # @param sync_url [String] - # @param sync_channel [String, nil] - # @return [String] - def resolved_sync_url(sync_url, sync_channel) - return sync_url unless sync_url.to_s.empty? - - SyncTransport.resolve_channel_url(sync_channel) - end - - ## - # @param registry_id [String] - # @param raw [Hash{Symbol => Object}] - # @param mode [Symbol] - # @param path [String, nil] - # @param sync_channel [String, nil] - # @param sync_url [String, nil] - # @return [Entry] - def entry_attributes(registry_id, raw, mode, path, sync_channel, sync_url) # rubocop:disable Metrics/ParameterLists, Metrics/MethodLength - sync = sync_section(raw) - Entry.new( - id: registry_id, + # @return [Definition] + def build_definition(id, raw) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity + sync = raw[:sync].is_a?(Hash) ? raw[:sync] : {} + path = raw[:path]&.to_s + mode = path && !path.empty? ? :path : :sync + key_pem = raw[:public_key]&.to_s + key = key_pem && !key_pem.strip.empty? ? parse_key(key_pem) : nil + + definition = Definition.new( + id:, mode:, - path:, - sync_channel:, - sync_url:, + path: mode == :path ? File.expand_path(path, Dir.pwd) : nil, + sync_channel: sync[:channel]&.to_s || DEFAULT_OFFICIAL_SYNC_CHANNEL, + sync_url: sync[:url]&.to_s, catalog: raw.fetch(:catalog, true), - public_key_id: public_key_id_for(raw[:public_key_id]&.to_s), - public_key: parse_public_key(raw[:public_key]), - sync_policy: sync_policy_for(raw, sync), - allowed_channel_domains: parse_allowed_channel_domains(raw[:allowed_channel_domains]) - ) - end - - ## - # @param raw [Hash{Symbol => Object}] - # @return [Hash{Symbol => Object}] - def sync_section(raw) - raw[:sync].is_a?(Hash) ? raw[:sync] : {} - end - - ## - # @param raw [Hash{Symbol => Object}] - # @param sync [Hash{Symbol => Object}] - # @return [SyncPolicy] - def sync_policy_for(raw, sync) - SyncPolicy.new( - pin_version: optional_string(sync[:pin_version]), - max_version: optional_string(sync[:max_version]), - auto_promote: auto_promote?(raw[:auto_promote]) + public_key_id: raw[:public_key_id]&.to_s || 'html2rss:registry:2026', + public_key: key, + sync_policy: SyncPolicy.new( + pin_version: sync[:pin_version]&.to_s, + max_version: sync[:max_version]&.to_s, + auto_promote: raw[:auto_promote] == true + ), + allowed_channel_domains: Array(raw[:allowed_channel_domains]).map(&:to_s).reject(&:empty?) ) - end - ## - # @param value [Object] - # @return [Boolean] - def auto_promote?(value) - value == true - end - - ## - # @param value [Object] - # @return [String, nil] - def optional_string(value) - string = value&.to_s - string.nil? || string.empty? ? nil : string - end - - ## - # @param value [Object] - # @return [Array] - def parse_allowed_channel_domains(value) - Array(value).map { it.to_s.strip }.reject(&:empty?) - end - - ## - # @param public_key_id [String, nil] - # @return [String] - def public_key_id_for(public_key_id) - return public_key_id unless public_key_id.to_s.empty? - - 'html2rss:registry:2026' - end - - ## - # @param registry_id [String] - # @param entry [Entry] - # @return [void] - def validate_sync_public_key!(registry_id, entry) - return unless entry.public_key.nil? - - raise Errors::ConfigError, - "Sync registry '#{registry_id}' requires a pinned public_key in #{registries_file}" - end - - ## - # @param path [String] - # @return [String] - def expand_path(path) - return path if path.start_with?('/') + if definition.mode == :sync && definition.public_key.nil? + raise Errors::ConfigError, "Sync registry '#{id}' requires a pinned public_key" + end - File.expand_path(path, Dir.pwd) + definition end ## - # @param value [String, nil] - # @return [OpenSSL::PKey::PKey, nil] - def parse_public_key(value) - return nil if value.to_s.strip.empty? - - OpenSSL::PKey.read(value) + # @param pem [String] + # @return [OpenSSL::PKey::PKey] + def parse_key(pem) + OpenSSL::PKey.read(pem) rescue OpenSSL::PKey::PKeyError => error raise Errors::ConfigError, "Invalid registry public_key: #{error.message}" end - end # rubocop:enable Metrics/ClassLength - - ## - # Parsed registry configuration snapshot. - ConfigSnapshot = Data.define(:precedence, :entries) - end # rubocop:enable Metrics/ModuleLength + end + end end end end diff --git a/app/web/registry/http_transport.rb b/app/web/registry/http_transport.rb new file mode 100644 index 000000000..34463b116 --- /dev/null +++ b/app/web/registry/http_transport.rb @@ -0,0 +1,117 @@ +# frozen_string_literal: true + +require 'net/http' +require 'uri' + +module Html2rss + module Web + module Registry + ## + # HTTPS client for fetching remote registry bundles. + module HttpTransport + DEFAULT_ALLOWED_HOSTS = %w[ + api.github.com + github.com + objects.githubusercontent.com + release-assets.githubusercontent.com + ].freeze + OPEN_TIMEOUT_SECONDS = 10 + READ_TIMEOUT_SECONDS = 60 + MAX_RESPONSE_BYTES = 52_428_800 + DEFAULT_MAX_REDIRECTS = 5 + + module_function + + ## + # @param url [String] + # @param max_redirects [Integer] + # @param allowed_hosts [Array] + # @return [String] response body + def fetch!(url, max_redirects: DEFAULT_MAX_REDIRECTS, allowed_hosts: default_allowed_hosts) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + uri = parse_https_uri!(url) + hops = 0 + + loop do + ensure_allowed_host!(uri.host, allowed_hosts) + response = perform_request(uri) + + case response + in Net::HTTPRedirection + hops += 1 + raise Errors::SyncError, 'Registry sync exceeded redirect limit' if hops > max_redirects + + location = response['location'] + raise Errors::SyncError, 'Registry sync redirect missing Location header' if location.to_s.empty? + + uri = parse_https_uri!(URI.join(uri, location).to_s) + in Net::HTTPSuccess + return read_body(response) + else + raise Errors::SyncError, "Registry sync fetch failed with HTTP #{response.code}" + end + end + end + + ## + # @return [Array] + def default_allowed_hosts + extra = ENV.fetch('REGISTRY_SYNC_ALLOWED_HOSTS', '').split(',').map(&:strip).reject(&:empty?) + DEFAULT_ALLOWED_HOSTS + extra + end + + ## + # @param url [String] + # @return [URI::HTTPS] + def parse_https_uri!(url) + uri = URI(url) + unless uri.is_a?(URI::HTTPS) + raise Errors::SyncError, "Registry sync requires HTTPS URLs (got #{uri.scheme.inspect})" + end + + uri + end + + ## + # @param host [String] + # @param allowed_hosts [Array] + # @return [void] + def ensure_allowed_host!(host, allowed_hosts) + return if allowed_hosts.include?(host) + + raise Errors::SyncError, "Registry sync host not allowed: #{host}" + end + + ## + # @param uri [URI::HTTPS] + # @return [Net::HTTPResponse] + def perform_request(uri) # rubocop:disable Metrics/MethodLength + Net::HTTP.start( + uri.host, + uri.port, + use_ssl: true, + open_timeout: OPEN_TIMEOUT_SECONDS, + read_timeout: READ_TIMEOUT_SECONDS + ) do |http| + request = Net::HTTP::Get.new(uri) + request['Accept'] = 'application/octet-stream' + request['User-Agent'] = 'html2rss-web/registry-sync' + http.request(request) + end + end + + ## + # @param response [Net::HTTPResponse] + # @return [String] + def read_body(response) + body = response.body.to_s + if body.bytesize > MAX_RESPONSE_BYTES + raise Errors::SyncError, + "Response exceeds max bytes (#{MAX_RESPONSE_BYTES})" + end + + body + end + end + end + end +end diff --git a/app/web/registry/index.rb b/app/web/registry/index.rb index 7159940e6..0674227f0 100644 --- a/app/web/registry/index.rb +++ b/app/web/registry/index.rb @@ -1,69 +1,31 @@ # frozen_string_literal: true require 'uri' +require_relative '../config/structured_data' module Html2rss module Web module Registry ## - # Sole merge owner for registry bundles and local {LocalConfig} feeds. + # Unified status model for registry state across API, CLI, and index. + Status = Data.define( + :id, + :mode, + :version, + :staged_version, + :updated_at, + :sync_url, + :last_error + ) do + def initialize(id:, mode:, version: nil, staged_version: nil, updated_at: nil, sync_url: nil, last_error: nil) + super + end + end + + ## + # Sole feed repository combining registry bundles and local feeds. class Index # rubocop:disable Metrics/ClassLength RegistryBundle = Data.define(:registry_id, :manifest, :configs, :catalog_entries) - StatusEntry = Data.define(:id, :version, :updated_at, :sync_mode) - - CatalogRow = Data.define( - :id, - :path, - :directory, - :channel, - :parameters, - :source, - :registry - ) do - ## - # @param entry [Html2rss::Registry::CatalogEntry] - # @param registry_id [String] - # @return [CatalogRow] - def self.from_entry(entry, registry_id) - new( - id: entry.id, - path: entry.path, - directory: entry.directory, - channel: entry.channel, - parameters: entry.parameters, - source: 'registry', - registry: registry_id - ) - end - - ## - # @param feed_name [String, Symbol] - # @param feed_config [Hash{Symbol => Object}] - # @return [CatalogRow, nil] - def self.from_local_feed(feed_name, feed_config) # rubocop:disable Metrics/MethodLength - directory = feed_config[:directory] || {} - title = directory[:title] - return nil if title.to_s.strip.empty? - - id = feed_name.to_s - channel = feed_config[:channel] || {} - new( - id:, - path: "/#{id}.rss", - source: 'local', - directory: Html2rss::Registry::CatalogBuilder.directory_payload(directory, title), - channel: Html2rss::Registry::CatalogBuilder.channel_payload(channel, title), - parameters: { schema: {}, defaults: {} }, - registry: nil - ) - end - - ## - # @return [Hash{Symbol => Object}] - def to_h - super.compact - end - end @mutex = Mutex.new @current = nil @@ -76,8 +38,6 @@ def current end ## - # Clears memoized bundles (tests and development reload). - # # @return [nil] def reload! @mutex.synchronize { @current = nil } @@ -90,26 +50,51 @@ def reload! # @param feed_id [String, Symbol] # @return [Hash{Symbol => Object}, nil] def config_for(feed_id) - normalized_id = normalize_feed_id(feed_id) - local_config = local_config_for(normalized_id) - return local_config if local_config + id = normalize_feed_id(feed_id) + local = local_config_for(id) + return local if local loaded_bundles.each_value do |bundle| - config = bundle.configs[normalized_id] - return StructuredData.deep_dup(config) if config + config = bundle.configs[id] + return Html2rss::Web::Config::StructuredData.deep_dup(config) if config end nil end ## - # @return [Array Object}>] catalog rows in HTTP wire shape + # @return [Array] + def catalog_entries # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength + registry_entries = Config.precedence.each_with_object({}) do |registry_id, rows| + next unless Config.catalog_enabled?(registry_id) + + bundle = loaded_bundles[registry_id] + next unless bundle + + bundle.catalog_entries.each do |entry| + rows[entry.id] ||= Html2rss::Registry::CatalogEntry.new( + id: entry.id, + path: entry.path, + directory: entry.directory, + channel: entry.channel, + parameters: entry.parameters, + source: 'registry', + registry: registry_id + ) + end + end + + local_entries = LocalConfig.feeds.filter_map do |feed_name, feed_config| + build_local_catalog_entry(feed_name, feed_config) + end + + registry_entries.merge(local_entries.to_h { [it.id, it] }).values.sort_by(&:id) + end + + ## + # @return [Array Object}>] def catalog_rows - registry_catalog_rows - .merge(local_catalog_rows.to_h { [it.id, it] }) - .values - .sort_by(&:id) - .map(&:to_h) + catalog_entries.map(&:to_h) end ## @@ -120,201 +105,108 @@ def bundle_for(registry_id) end ## - # @return [Array] - def status + # @return [Array] + def status # rubocop:disable Metrics/AbcSize, Metrics/MethodLength Config.precedence.map do |registry_id| - entry = Config.entry(registry_id) + definition = Config.entry(registry_id) bundle = loaded_bundles[registry_id] - StatusEntry.new( + sync_state = Store.sync_state(registry_id) + sync_url = definition.mode == :sync ? ChannelResolver.resolve(definition) : nil + + Status.new( id: registry_id, + mode: definition.mode, version: bundle&.manifest&.version, - updated_at: bundle_updated_at(registry_id), - sync_mode: entry.mode + staged_version: Store.staged_version(registry_id), + updated_at: Store.manifest_mtime(bundle_directory(definition)), + sync_url:, + last_error: sync_state.last_error ) end end ## # @param registry_id [String, Symbol] - # @return [StatusEntry, nil] + # @return [Status, nil] def status_entry_for(registry_id) status.find { it.id == registry_id.to_s } end private - ## - # @return [Hash{String => CatalogRow}] - def registry_catalog_rows - Config.precedence.each_with_object({}) do |registry_id, rows| - next unless Config.catalog_enabled?(registry_id) - - bundle = loaded_bundles[registry_id] - next unless bundle - - bundle.catalog_entries.each do |entry| - rows[entry.id] ||= CatalogRow.from_entry(entry, registry_id) - end - end - end - - ## - # @return [Array] - def local_catalog_rows - LocalConfig.feeds.filter_map do |feed_name, feed_config| - CatalogRow.from_local_feed(feed_name, feed_config) - end - end - - ## - # @return [Hash{String => RegistryBundle}] def loaded_bundles @loaded_bundles ||= Config.precedence.to_h { [it, load_bundle(it)] }.compact end - ## - # @param registry_id [String] - # @return [RegistryBundle, nil] def load_bundle(registry_id) # rubocop:disable Metrics/MethodLength - entry = Config.entry(registry_id) - directory = bundle_directory(entry) - return nil unless directory && File.directory?(directory) - return nil unless active_bundle_present?(directory) - - # Verify on read: disk-trust seam; Sync already verifies downloaded bundles before promote. - bundle = Html2rss::Registry::Bundle.load( - directory, - **trust_options_for(entry, directory) + definition = Config.entry(registry_id) + directory = bundle_directory(definition) + return nil unless directory && File.directory?(directory) && Store.bundle_present_at?(directory) + + trust_opts = if definition.mode == :path + { trust: :integrity_only } + else + { trust: :signed, + public_keys: definition.public_keys } + end + bundle_data = Html2rss::Registry::Bundle.load(directory, **trust_opts) + bundle = RegistryBundle.new( + registry_id:, + manifest: bundle_data.manifest, + configs: bundle_data.configs, + catalog_entries: bundle_data.catalog_entries ) - registry_bundle = to_registry_bundle(registry_id, bundle) - enforce_scrape_policy!(entry, registry_bundle) - registry_bundle + enforce_scrape_policy!(definition, bundle) + bundle rescue Html2rss::Registry::Error => error raise Errors::LoadError, "Failed to load registry '#{registry_id}': #{error.message}" end - ## - # @param entry [Entry] - # @param bundle_dir [String] - # @return [Hash{Symbol => Object}] keyword args for {Html2rss::Registry::Bundle.load} - def trust_options_for(entry, bundle_dir) - case entry.mode - in :path - { trust: :integrity_only, public_keys: {} } - in :sync - if signed_bundle?(bundle_dir, entry) - { trust: :signed, public_keys: entry.public_keys } - else - { trust: :integrity_only, public_keys: {} } - end - end - end - - ## - # @param bundle_dir [String] - # @param entry [Entry] - # @return [Boolean] - def signed_bundle?(bundle_dir, entry) - signature_path = File.join(bundle_dir, Html2rss::Registry::Manifest::SIGNATURE_FILE) - File.file?(signature_path) && !entry.public_keys.empty? - end - - ## - # @param entry [Entry] - # @param bundle [RegistryBundle] - # @return [void] - # @raise [Errors::LoadError] when a config channel URL violates the allowlist - def enforce_scrape_policy!(entry, bundle) - allowed = entry.allowed_channel_domains + def enforce_scrape_policy!(definition, bundle) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity + allowed = definition.allowed_channel_domains return if allowed.nil? || allowed.empty? bundle.configs.each do |feed_id, config| - channel_url = config.dig(:channel, :url) - host = channel_host_for(channel_url) - next if host && allowed.any? { channel_domain_allowed?(host, it) } + host = URI.parse(config.dig(:channel, :url).to_s).host rescue nil # rubocop:disable Style/RescueModifier + next if host && allowed.any? { host.downcase == it.downcase || host.downcase.end_with?(".#{it.downcase}") } - raise Errors::LoadError, - "Registry '#{entry.id}' config '#{feed_id}' channel.url host " \ - "'#{host || channel_url}' is not allowed by allowed_channel_domains" + raise Errors::LoadError, "Registry '#{definition.id}' config '#{feed_id}' host not allowed" end end - ## - # @param host [String] - # @param allowed_domain [String] - # @return [Boolean] - def channel_domain_allowed?(host, allowed_domain) - normalized_host = host.downcase - normalized_domain = allowed_domain.downcase - normalized_host == normalized_domain || normalized_host.end_with?(".#{normalized_domain}") - end - - ## - # @param channel_url [String, nil] - # @return [String, nil] - def channel_host_for(channel_url) - return nil if channel_url.to_s.strip.empty? - - URI.parse(channel_url).host - rescue URI::InvalidURIError - nil - end - - ## - # @param registry_id [String] - # @param bundle [Html2rss::Registry::Bundle::BundleData] - # @return [RegistryBundle] - def to_registry_bundle(registry_id, bundle) - RegistryBundle.new( - registry_id:, - manifest: bundle.manifest, - configs: bundle.configs, - catalog_entries: bundle.catalog_entries - ) - end - - ## - # @param entry [Entry] - # @return [String, nil] - def bundle_directory(entry) - return entry.path if entry.mode == :path - - Store.registry_dir(entry.id) - end - - ## - # @param directory [String] - # @return [Boolean] - def active_bundle_present?(directory) - Store.bundle_present_at?(directory) + def bundle_directory(definition) + definition.mode == :path ? definition.path : Store.registry_dir(definition.id) end - ## - # @param feed_id [String, Symbol] - # @return [String] def normalize_feed_id(feed_id) feed_id.to_s.delete_prefix('/').sub(LocalConfig::FEED_EXTENSION_PATTERN, '') end - ## - # @param registry_id [String] - # @return [Time, nil] - def bundle_updated_at(registry_id) - Store.manifest_mtime(bundle_directory(Config.entry(registry_id))) - end - - ## - # @param feed_id [String] - # @return [Hash{Symbol => Object}, nil] def local_config_for(feed_id) feed = LocalConfig.feeds[feed_id.to_sym] || LocalConfig.feeds[feed_id] - return nil unless feed - - StructuredData.deep_dup(feed) - rescue Html2rss::Web::LocalConfig::InvalidConfig, Html2rss::Web::LocalConfig::NotFound + feed ? Html2rss::Web::Config::StructuredData.deep_dup(feed) : nil + rescue StandardError nil end - end # rubocop:enable Metrics/ClassLength + + def build_local_catalog_entry(feed_name, feed_config) # rubocop:disable Metrics/MethodLength + directory = feed_config[:directory] || {} + title = directory[:title]&.to_s + return nil if title.nil? || title.strip.empty? + + id = feed_name.to_s + channel = feed_config[:channel] || {} + Html2rss::Registry::CatalogEntry.new( + id:, + path: "/#{id}.rss", + source: 'local', + directory: Html2rss::Registry::CatalogBuilder.directory_payload(directory, title), + channel: Html2rss::Registry::CatalogBuilder.channel_payload(channel, title), + parameters: { schema: {}, defaults: {} }, + registry: nil + ) + end + end end end end diff --git a/app/web/registry/store.rb b/app/web/registry/store.rb index 5deedfca1..4517d32ea 100644 --- a/app/web/registry/store.rb +++ b/app/web/registry/store.rb @@ -7,7 +7,7 @@ module Html2rss module Web module Registry ## - # Manages on-disk registry bundle directories under {data_root}. + # Transactional storage engine for registry bundles and sync state. module Store # rubocop:disable Metrics/ModuleLength DEFAULT_DATA_ROOT = 'tmp/registry-data' DEFAULT_SEED_ROOT = '/app/registries/seed' @@ -15,31 +15,31 @@ module Store # rubocop:disable Metrics/ModuleLength SyncState = Data.define(:last_error, :last_sync_at) - class << self # rubocop:disable Metrics/ClassLength + class << self ## - # @return [String] root directory for extracted registry bundles + # @return [String] def data_root File.expand_path(ENV.fetch('REGISTRY_DATA_ROOT', DEFAULT_DATA_ROOT)) end ## - # @return [String] root directory for image-shipped seed bundles + # @return [String] def seed_root File.expand_path(ENV.fetch('REGISTRY_SEED_ROOT', DEFAULT_SEED_ROOT)) end ## # @param registry_id [String, Symbol] - # @return [String] active bundle directory for a registry id + # @return [String] def registry_dir(registry_id) File.join(data_root, registry_id.to_s) end ## # @param registry_id [String, Symbol] - # @return [String] seed bundle directory for a registry id - def seed_path_for(registry_id) - File.join(seed_root, registry_id.to_s) + # @return [String] + def staging_dir(registry_id) + File.join(registry_dir(registry_id), '.staging') end ## @@ -50,26 +50,22 @@ def bundle_present?(registry_id) end ## - # @param path [String] bundle directory + # @param path [String] # @return [Boolean] def bundle_present_at?(path) File.file?(File.join(path, Html2rss::Registry::Manifest::MANIFEST_FILE)) end - ## - # @param path [String] bundle directory - # @return [Html2rss::Registry::Manifest] - def read_manifest(path) - Html2rss::Registry::Manifest.parse( - File.read(File.join(path, Html2rss::Registry::Manifest::MANIFEST_FILE)) - ) - end - ## # @param registry_id [String, Symbol] - # @return [String] verified staging directory for a registry id - def staging_dir(registry_id) - File.join(registry_dir(registry_id), '.staging') + # @param staged_dir [String] + # @return [String] + def stage_bundle!(registry_id, staged_dir) + target = staging_dir(registry_id) + FileUtils.mkdir_p(registry_dir(registry_id)) + FileUtils.rm_rf(target) + FileUtils.cp_r(staged_dir, target) + target end ## @@ -87,29 +83,28 @@ def staged_version(registry_id) end ## - # Writes a verified bundle to the registry staging directory. + # Atomically swaps a directory into the active registry directory. # # @param registry_id [String, Symbol] - # @param staged_dir [String] verified bundle directory to stage - # @return [String] staging directory - def stage_bundle!(registry_id, staged_dir) - raise Errors::LoadError, "Staged bundle missing: #{staged_dir}" unless File.directory?(staged_dir) - - target = staging_dir(registry_id) - parent = registry_dir(registry_id) - FileUtils.mkdir_p(parent) - + # @param source_dir [String] + # @return [String] active directory + def swap!(registry_id, source_dir) + target = registry_dir(registry_id) + FileUtils.mkdir_p(File.dirname(target)) backup = "#{target}.backup.#{Process.pid}" - promote_bundle!(staged_dir, target, backup) + FileUtils.rm_rf(backup) + FileUtils.mv(target, backup) if File.exist?(target) + FileUtils.cp_r(source_dir, target) + FileUtils.rm_rf(backup) target end ## - # Promotes a verified staging bundle to the active registry directory. + # Promotes staging directory to active directory. # # @param registry_id [String, Symbol] - # @return [String] active bundle directory - def promote_staged!(registry_id) # rubocop:disable Metrics/MethodLength + # @return [String] active directory + def promote_staged!(registry_id) staged = staging_dir(registry_id) raise Errors::LoadError, "No staged bundle for '#{registry_id}'" unless staged_present?(registry_id) @@ -117,44 +112,23 @@ def promote_staged!(registry_id) # rubocop:disable Metrics/MethodLength temp_staged = File.join(temp_root, 'bundle') FileUtils.mv(staged, temp_staged) - active = registry_dir(registry_id) - backup = "#{active}.backup.#{Process.pid}" - promote_bundle!(temp_staged, active, backup) - active + swap!(registry_id, temp_staged) + registry_dir(registry_id) ensure FileUtils.rm_rf(temp_root) if temp_root end ## - # Atomically replaces the active bundle directory for a registry id. - # - # @param registry_id [String, Symbol] - # @param staged_dir [String] verified bundle directory to promote - # @return [String] promoted bundle directory - def swap!(registry_id, staged_dir) - raise Errors::LoadError, "Staged bundle missing: #{staged_dir}" unless File.directory?(staged_dir) - - target = registry_dir(registry_id) - parent = File.dirname(target) - FileUtils.mkdir_p(parent) - - backup = "#{target}.backup.#{Process.pid}" - promote_bundle!(staged_dir, target, backup) - target - end - - ## - # Copies a seed bundle into the data root when no active bundle exists. + # Copies seed bundle into data root if active bundle is missing. # # @param registry_id [String, Symbol] - # @param seed_path [String] bundle directory to copy - # @return [Boolean] true when a seed copy was performed + # @param seed_path [String] + # @return [Boolean] def seed_if_empty!(registry_id, seed_path:) # rubocop:disable Naming/PredicateMethod - target = registry_dir(registry_id) return false if bundle_present?(registry_id) - raise Errors::LoadError, "Seed bundle missing: #{seed_path}" unless File.directory?(seed_path) + target = registry_dir(registry_id) FileUtils.mkdir_p(File.dirname(target)) FileUtils.cp_r(seed_path, target) true @@ -167,7 +141,7 @@ def sync_state(registry_id) raw = read_sync_state.fetch(registry_id.to_s, {}) SyncState.new( last_error: raw['last_error'], - last_sync_at: parse_sync_time(raw['last_sync_at']) + last_sync_at: raw['last_sync_at'] ? Time.parse(raw['last_sync_at']) : nil ) end @@ -192,22 +166,16 @@ def write_sync_state!(registry_id, last_error:, last_sync_at: Time.now.utc) def manifest_mtime(path) return nil unless path && File.directory?(path) - manifest_path = File.join(path, Html2rss::Registry::Manifest::MANIFEST_FILE) - return nil unless File.file?(manifest_path) - - File.mtime(manifest_path) + manifest_file = File.join(path, Html2rss::Registry::Manifest::MANIFEST_FILE) + File.file?(manifest_file) ? File.mtime(manifest_file) : nil end private - ## - # @return [String] def sync_state_path File.join(data_root, SYNC_STATE_FILE) end - ## - # @return [Hash{String => Hash{String => Object}}] def read_sync_state return {} unless File.file?(sync_state_path) @@ -216,42 +184,14 @@ def read_sync_state {} end - ## - # @param staged_dir [String] - # @param target [String] - # @param backup [String] - # @return [void] - def promote_bundle!(staged_dir, target, backup) - FileUtils.rm_rf(backup) - FileUtils.mv(target, backup) if File.exist?(target) - FileUtils.mv(staged_dir, target) - rescue StandardError - FileUtils.rm_rf(target) - FileUtils.mv(backup, target) if File.exist?(backup) - raise - ensure - FileUtils.rm_rf(backup) - end - - ## - # @param path [String] - # @return [String, nil] def manifest_version_at(path) - return nil unless bundle_present_at?(path) + manifest_file = File.join(path, Html2rss::Registry::Manifest::MANIFEST_FILE) + return nil unless File.file?(manifest_file) - read_manifest(path).version + Html2rss::Registry::Manifest.parse(File.read(manifest_file)).version rescue Html2rss::Registry::ManifestError nil end - - ## - # @param value [String, nil] - # @return [Time, nil] - def parse_sync_time(value) - return nil if value.to_s.empty? - - Time.parse(value) - end end end end diff --git a/app/web/registry/sync.rb b/app/web/registry/sync.rb index b489f1fe5..ed7b933f7 100644 --- a/app/web/registry/sync.rb +++ b/app/web/registry/sync.rb @@ -8,16 +8,6 @@ module Registry ## # Fetches, verifies, and stores signed registry bundles. module Sync # rubocop:disable Metrics/ModuleLength - SyncStatus = Data.define( - :registry_id, - :mode, - :version, - :staged_version, - :updated_at, - :sync_url, - :last_error - ) - @boot_mutex = Mutex.new @boot_started = false @timer_started = false @@ -25,48 +15,53 @@ module Sync # rubocop:disable Metrics/ModuleLength class << self # rubocop:disable Metrics/ClassLength ## - # Resolves the download URL for a sync-mode registry id. - # # @param registry_id [String, Symbol] # @return [String] def sync_url_for(registry_id) - entry = Config.entry(registry_id) - raise Errors::SyncError, "Registry '#{registry_id}' is not sync-mode" unless entry.mode == :sync + definition = Config.entry(registry_id) + raise Errors::SyncError, "Registry '#{registry_id}' is path mode" if definition.mode == :path - SyncTransport.resolve(entry) + ChannelResolver.resolve(definition) end ## - # Runs synchronization for a registry id. - # # @param registry_id [String, Symbol] - # @param dry_run [Boolean] when true, verify without swapping the active bundle - # @return [SyncStatus] + # @param dry_run [Boolean] + # @return [Status] def run(registry_id:, dry_run: false) - with_registry_lock(registry_id) { run!(registry_id:, dry_run:) } + REGISTRY_MUTEXES[registry_id.to_s].synchronize { run_sync!(registry_id:, dry_run:) } end ## - # Promotes a verified staging bundle to the active registry directory. - # # @param registry_id [String, Symbol] - # @return [SyncStatus] - def promote_staged!(registry_id:) - with_registry_lock(registry_id) { promote_staged_bundle!(registry_id:) } + # @return [Status] + def promote_staged!(registry_id:) # rubocop:disable Metrics/MethodLength + REGISTRY_MUTEXES[registry_id.to_s].synchronize do + definition = Config.entry(registry_id) + raise Errors::SyncError, "Registry '#{registry_id}' is path mode" if definition.mode == :path + unless Store.staged_present?(registry_id) + raise Errors::SyncError, + "Registry '#{registry_id}' has no staged bundle" + end + + previous_bundle = Index.current.bundle_for(registry_id) + Store.promote_staged!(registry_id) + Index.reload! + report_catalog_change!(registry_id, previous_bundle) + Store.write_sync_state!(registry_id, last_error: nil) + Index.current.status_entry_for(registry_id) + end end ## # @param registry_id [String, Symbol] - # @return [Array] + # @return [Array] def status(registry_id: nil) rows = Index.current.status - rows = rows.select { it.id == registry_id.to_s } if registry_id - rows.map { sync_status_for(it) } + registry_id ? rows.select { it.id == registry_id.to_s } : rows end ## - # Seeds sync-mode registries and optionally syncs on boot. - # # @return [void] def boot! # rubocop:disable Metrics/MethodLength @boot_mutex.synchronize do @@ -74,26 +69,24 @@ def boot! # rubocop:disable Metrics/MethodLength @boot_started = true end - return if skip_boot? + return if ENV.fetch('RACK_ENV', 'development') == 'test' - Config.precedence.each do |registry_id| - entry = Config.entry(registry_id) - next unless entry.mode == :sync + Config.precedence.each do |id| + definition = Config.entry(id) + next unless definition.mode == :sync - seed_registry!(registry_id) - schedule_boot_sync!(registry_id) if boot_sync?(registry_id) + seed_registry!(id) + Thread.new { run(registry_id: id) } if boot_sync?(id) # rubocop:disable ThreadSafety/NewThread end start_background_timer! end ## - # Starts a jittered background sync loop when enabled. - # # @return [void] def start_background_timer! # rubocop:disable Metrics/MethodLength - interval_hours = Integer(ENV.fetch('REGISTRY_SYNC_INTERVAL_HOURS', '24')) - return if interval_hours <= 0 + interval = Integer(ENV.fetch('REGISTRY_SYNC_INTERVAL_HOURS', '24')) + return if interval <= 0 @boot_mutex.synchronize do return if @timer_started @@ -101,382 +94,108 @@ def start_background_timer! # rubocop:disable Metrics/MethodLength @timer_started = true end - Thread.new do # rubocop:disable ThreadSafety/NewThread -- background registry refresh by design - sleep(background_jitter_seconds(interval_hours)) + Thread.new do # rubocop:disable ThreadSafety/NewThread loop do - sync_all! - sleep(interval_hours * 3600) + sleep(interval * 3600) + Config.precedence.each { |id| run(registry_id: id) if Config.entry(id).mode == :sync rescue nil } # rubocop:disable Style/RescueModifier end end end - ## - # @return [Integer] process exit code for CLI use (0 ok, 1 when sync registries lack bundles) - def cli_exit_code - unusable_sync_registries.empty? ? 0 : 1 - end + private - ## - # @return [Array] sync-mode registry ids without a usable on-disk bundle - def unusable_sync_registries - Config.precedence.filter_map do |registry_id| - entry = Config.entry(registry_id) - next unless entry.mode == :sync - next if Store.bundle_present?(registry_id) - - registry_id - end - end + def run_sync!(registry_id:, dry_run:) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity + definition = Config.entry(registry_id) + raise Errors::SyncError, "Registry '#{registry_id}' is path mode" if definition.mode == :path - private + staging_root = Dir.mktmpdir('registry-sync-') + staged_dir = File.join(staging_root, 'bundle') + FileUtils.mkdir_p(staged_dir) - ## - # @param registry_id [String, Symbol] - # @yield runs sync work exclusively for the registry id - # @return [Object] block result - def with_registry_lock(registry_id, &) - REGISTRY_MUTEXES[registry_id.to_s].synchronize(&) - end + download_url = ChannelResolver.resolve(definition) + tarball = HttpTransport.fetch!(download_url) + File.binwrite(File.join(staging_root, 'download.tar.gz'), tarball) - ## - # @param registry_id [String, Symbol] - # @param dry_run [Boolean] - # @return [SyncStatus] - def run!(registry_id:, dry_run:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength - entry = Config.entry(registry_id) - if entry.mode == :path - raise Errors::SyncError, "Registry '#{registry_id}' uses path mode; sync is not applicable" + File.open(File.join(staging_root, 'download.tar.gz'), 'rb') do |io| + Html2rss::Registry::Archive.extract!(io, into: staged_dir) end - staging_root = nil - download_url = SyncTransport.resolve(entry) - staging_root, staged_dir = fetch_and_verify!(entry, download_url) + manifest = Html2rss::Registry::Verifier.verify!(staged_dir, trust: :signed, + public_keys: definition.public_keys) + enforce_max_version!(definition, manifest) + unless dry_run - if entry.sync_policy.auto_promote - activate_bundle!(registry_id, staged_dir) + if definition.sync_policy.auto_promote + previous = Index.current.bundle_for(registry_id) + Store.swap!(registry_id, staged_dir) + Index.reload! + report_catalog_change!(registry_id, previous) else Store.stage_bundle!(registry_id, staged_dir) - record_success!(registry_id, promoted: false) end + Store.write_sync_state!(registry_id, last_error: nil) end - sync_status_for(Index.current.status_entry_for(registry_id)) + + Index.current.status_entry_for(registry_id) + rescue Html2rss::Registry::VerificationError => error + log_signature_failure!(registry_id, error.message) if error.message.match?(/signature|public_key_id/i) + Store.write_sync_state!(registry_id, last_error: error.message) unless dry_run + raise Errors::SyncError, error.message + rescue Html2rss::Registry::ArchiveError => error + Store.write_sync_state!(registry_id, last_error: error.message) unless dry_run + raise Errors::SyncError, error.message rescue StandardError => error - record_failure!(registry_id, error) unless dry_run + Store.write_sync_state!(registry_id, last_error: error.message) unless dry_run raise ensure FileUtils.rm_rf(staging_root) if staging_root end - ## - # @param registry_id [String, Symbol] - # @return [SyncStatus] - def promote_staged_bundle!(registry_id:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength - entry = Config.entry(registry_id) - if entry.mode == :path - raise Errors::SyncError, "Registry '#{registry_id}' uses path mode; promote is not applicable" - end - unless Store.staged_present?(registry_id) - raise Errors::SyncError, "Registry '#{registry_id}' has no staged bundle" - end - - previous_bundle = Index.current.bundle_for(registry_id) - Store.promote_staged!(registry_id) - Index.reload! - report_catalog_change!(registry_id, previous_bundle) - record_success!(registry_id, promoted: true) - status_row = Index.current.status_entry_for(registry_id) - Observability.emit( - event_name: 'registry.promote_staged', - outcome: 'success', - details: { - registry_id: registry_id.to_s, - version: status_row&.version - } - ) - sync_status_for(status_row) - end - - ## - # @param registry_id [String, Symbol] - # @param staged_dir [String] - # @return [void] - def activate_bundle!(registry_id, staged_dir) - previous_bundle = Index.current.bundle_for(registry_id) - Store.swap!(registry_id, staged_dir) - Index.reload! - report_catalog_change!(registry_id, previous_bundle) - record_success!(registry_id, promoted: true) + def log_signature_failure!(registry_id, message) + SecurityLogger.log_registry_signature_failure(registry_id, message) + rescue StandardError + nil end - ## - # @param registry_id [String] - # @param previous_bundle [Index::RegistryBundle, nil] - # @return [void] - def report_catalog_change!(registry_id, previous_bundle) - new_bundle = Index.current.bundle_for(registry_id) - return unless new_bundle - - diff = build_catalog_diff(previous_bundle, new_bundle) - return if diff.empty? + def enforce_max_version!(definition, manifest) + max = definition.sync_policy.max_version + return if max.nil? || max.empty? + return unless Html2rss::Registry::Manifest.exceeds_max?(manifest.version, max) - emit_catalog_change_observability!(registry_id, diff) - emit_catalog_change_security!(registry_id, diff) if diff[:added].any? || diff[:removed].any? + raise Errors::SyncError, "Manifest version '#{manifest.version}' exceeds max_version '#{max}'" end - ## - # @param previous_bundle [Index::RegistryBundle, nil] - # @param new_bundle [Index::RegistryBundle] - # @return [Hash{Symbol => Object}] - def build_catalog_diff(previous_bundle, new_bundle) # rubocop:disable Metrics/MethodLength - previous_version = previous_bundle&.manifest&.version - new_version = new_bundle.manifest.version - previous_ids = Set.new(catalog_ids(previous_bundle)) - new_ids = Set.new(catalog_ids(new_bundle)) - added = new_ids - previous_ids - removed = previous_ids - new_ids - - return {} if previous_version == new_version && added.empty? && removed.empty? - - { - previous_version:, - version: new_version, - added: added.to_a, - removed: removed.to_a - } - end + def report_catalog_change!(registry_id, previous) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity + current = Index.current.bundle_for(registry_id) + return unless current + + prev_ids = previous ? Set.new(previous.catalog_entries.map(&:id)) : Set.new + curr_ids = Set.new(current.catalog_entries.map(&:id)) + added = (curr_ids - prev_ids).to_a.sort + removed = (prev_ids - curr_ids).to_a.sort + return if added.empty? && removed.empty? && previous&.manifest&.version == current.manifest.version - ## - # @param registry_id [String] - # @param diff [Hash{Symbol => Object}] - # @return [void] - def emit_catalog_change_observability!(registry_id, diff) # rubocop:disable Metrics/MethodLength Observability.emit( event_name: 'registry.catalog_changed', outcome: 'success', level: :warn, - details: { - registry_id:, - version: diff[:version], - previous_version: diff[:previous_version], - added_count: diff[:added].size, - removed_count: diff[:removed].size, - added_ids: diff[:added].sort, - removed_ids: diff[:removed].sort - } + details: { registry_id:, version: current.manifest.version, added_count: added.size, + removed_count: removed.size } ) end - ## - # @param registry_id [String] - # @param diff [Hash{Symbol => Object}] - # @return [void] - def emit_catalog_change_security!(registry_id, diff) - SecurityLogger.log_registry_catalog_changed( - registry_id, - version: diff[:version], - previous_version: diff[:previous_version], - added_count: diff[:added].size, - removed_count: diff[:removed].size, - added_ids: diff[:added].sort, - removed_ids: diff[:removed].sort - ) - end - - ## - # @param bundle [Index::RegistryBundle, nil] - # @return [Array] - def catalog_ids(bundle) - return [] unless bundle - - bundle.catalog_entries.map(&:id) - end - - ## - # @return [Boolean] - def skip_boot? - ENV.fetch('RACK_ENV', 'development') == 'test' - end - - ## - # @param registry_id [String] - # @return [void] - def seed_registry!(registry_id) # rubocop:disable Metrics/MethodLength + def seed_registry!(registry_id) seed_path = Store.seed_path_for(registry_id) - return unless File.directory?(seed_path) - - seeded = Store.seed_if_empty!(registry_id, seed_path:) - Index.reload! if seeded - rescue Errors::LoadError => error - AppLogger.logger.warn( - { - component: 'registry', - event_name: 'registry.seed', - outcome: 'failure', - registry_id:, - error: error.message - }.to_json - ) - end - - ## - # @param registry_id [String] - # @return [Boolean] - def boot_sync?(registry_id) - ENV.fetch('REGISTRY_SYNC_ON_BOOT', 'false') == 'true' || !Store.bundle_present?(registry_id) - end - - ## - # @param registry_id [String] - # @return [void] - def schedule_boot_sync!(registry_id) - Thread.new { run(registry_id:) } # rubocop:disable ThreadSafety/NewThread -- non-blocking first boot + Index.reload! if File.directory?(seed_path) && Store.seed_if_empty!(registry_id, seed_path:) rescue StandardError nil end - ## - # @return [void] - def sync_all! - Config.precedence.each do |registry_id| - entry = Config.entry(registry_id) - next unless entry.mode == :sync - - run(registry_id:) - rescue StandardError - nil - end - end - - ## - # @param interval_hours [Integer] - # @return [Numeric] - def background_jitter_seconds(interval_hours) - max_jitter = [(interval_hours * 3600 * BACKGROUND_JITTER_FRACTION).to_i, 1].max - rand(max_jitter) - end - - BACKGROUND_JITTER_FRACTION = 0.1 - private_constant :BACKGROUND_JITTER_FRACTION - - ## - # @param entry [Entry] - # @param download_url [String] - # @return [Array(String, String)] staging root and verified bundle directory - def fetch_and_verify!(entry, download_url) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength - tarball = SyncTransport.fetch!(download_url) - staging_root = Dir.mktmpdir('registry-sync-') - staged_dir = File.join(staging_root, 'bundle') - tarball_path = File.join(staging_root, 'download.tar.gz') - FileUtils.mkdir_p(staged_dir) - File.binwrite(tarball_path, tarball) - - File.open(tarball_path, 'rb') do |io| - Html2rss::Registry::Archive.extract!(io, into: staged_dir) - end - - # Verify before promote: network-trust seam; Index reload still verifies on disk via Bundle.load. - Html2rss::Registry::Verifier.verify!( - staged_dir, - trust: :signed, - public_keys: entry.public_keys - ) - enforce_max_version!(entry, staged_dir) - [staging_root, staged_dir] - rescue Html2rss::Registry::VerificationError => error - log_signature_failure!(entry.id, error.message) if signature_failure?(error) - raise Errors::SyncError, error.message - rescue Html2rss::Registry::ArchiveError => error - raise Errors::SyncError, error.message - end - - ## - # @param error [Html2rss::Registry::VerificationError] - # @return [Boolean] - def signature_failure?(error) - error.message.match?(/signature|public_key_id/i) - end - - ## - # @param registry_id [String] - # @param message [String] - # @return [void] - def log_signature_failure!(registry_id, message) - SecurityLogger.log_registry_signature_failure(registry_id, message) - end - - ## - # @param entry [Entry] - # @param staged_dir [String] - # @return [void] - def enforce_max_version!(entry, staged_dir) - manifest = read_manifest!(staged_dir) - max_version = entry.sync_policy.max_version - return unless max_version - - return unless SyncTransport.exceeds_max?(manifest.version, max_version) - - raise Errors::SyncError, - "Registry '#{entry.id}' manifest version '#{manifest.version}' exceeds max_version '#{max_version}'" - end - - ## - # @param staged_dir [String] - # @return [Html2rss::Registry::Manifest] - def read_manifest!(staged_dir) - Store.read_manifest(staged_dir) - end - - ## - # @param registry_id [String] - # @param promoted [Boolean] - # @return [void] - def record_success!(registry_id, promoted:) # rubocop:disable Metrics/MethodLength - Store.write_sync_state!(registry_id, last_error: nil) - row = Index.current.status_entry_for(registry_id) - Observability.emit( - event_name: 'registry.sync', - outcome: 'success', - details: { - registry_id:, - version: row&.version, - staged_version: Store.staged_version(registry_id), - promoted: - } - ) - end - - ## - # @param registry_id [String] - # @param error [StandardError] - # @return [void] - def record_failure!(registry_id, error) - Store.write_sync_state!(registry_id, last_error: error.message) - Observability.emit( - event_name: 'registry.sync', - outcome: 'failure', - details: { registry_id:, error: error.message }, - level: :warn - ) - end - - ## - # @param row [Index::StatusEntry] - # @return [SyncStatus] - def sync_status_for(row) # rubocop:disable Metrics/MethodLength - entry = Config.entry(row.id) - state = Store.sync_state(row.id) - SyncStatus.new( - registry_id: row.id, - mode: row.sync_mode, - version: row.version, - staged_version: Store.staged_version(row.id), - updated_at: row.updated_at, - sync_url: entry.mode == :sync ? SyncTransport.resolve(entry) : nil, - last_error: state.last_error - ) + def boot_sync?(registry_id) + ENV.fetch('REGISTRY_SYNC_ON_BOOT', 'false') == 'true' || !Store.bundle_present?(registry_id) end - end # rubocop:enable Metrics/ClassLength - end # rubocop:enable Metrics/ModuleLength + end + end end end end diff --git a/app/web/registry/sync_transport.rb b/app/web/registry/sync_transport.rb deleted file mode 100644 index 220ef1902..000000000 --- a/app/web/registry/sync_transport.rb +++ /dev/null @@ -1,237 +0,0 @@ -# frozen_string_literal: true - -require 'net/http' -require 'uri' -require 'json' - -module Html2rss - module Web - module Registry - ## - # HTTPS fetch, sync URL resolution, and manifest version gating for registry sync. - module SyncTransport # rubocop:disable Metrics/ModuleLength - DEFAULT_ALLOWED_HOSTS = %w[ - api.github.com - github.com - objects.githubusercontent.com - release-assets.githubusercontent.com - ].freeze - FETCH_OPEN_TIMEOUT_SECONDS = 10 - FETCH_READ_TIMEOUT_SECONDS = 60 - MAX_RESPONSE_BYTES = 52_428_800 - DEFAULT_MAX_REDIRECTS = 5 - - OFFICIAL_GITHUB_RELEASES_API = - 'https://api.github.com/repos/html2rss/html2rss-configs/releases/latest' - OFFICIAL_GITHUB_TAG_RELEASES_API = - 'https://api.github.com/repos/html2rss/html2rss-configs/releases/tags/%s' - OFFICIAL_ASSET_NAME = 'registry-bundle.tar.gz' - EXTRA_ALLOWED_HOSTS_CACHE = {} # rubocop:disable Style/MutableConstant - - RedirectPolicy = Data.define(:max_hops, :allowed_hosts) do - ## - # @return [RedirectPolicy] - def self.default - new(max_hops: DEFAULT_MAX_REDIRECTS, allowed_hosts: default_allowed_hosts) - end - - ## - # @return [Array] - def self.default_allowed_hosts - DEFAULT_ALLOWED_HOSTS + SyncTransport.extra_allowed_hosts - end - end - - module_function - - ## - # @param url [String] - # @param policy [RedirectPolicy] - # @return [String] response body - def fetch!(url, policy: RedirectPolicy.default) - uri = parse_https_uri!(url) - follow_redirects!(uri, policy) - end - - ## - # @param entry [Entry] - # @return [String] - def resolve(entry) - return entry.sync_url unless entry.sync_url.to_s.empty? - - pin_version = entry.sync_policy.pin_version - if entry.sync_channel == Config::DEFAULT_OFFICIAL_SYNC_CHANNEL - unless pin_version.to_s.empty? - api_url = format(OFFICIAL_GITHUB_TAG_RELEASES_API, tag: pin_version) - return resolve_official_asset_url(api_url, tag: pin_version) - end - - return resolve_official_asset_url(OFFICIAL_GITHUB_RELEASES_API) - end - - raise Errors::SyncError, "Registry '#{entry.id}' has no sync URL" - end - - ## - # @param sync_channel [String, nil] - # @return [String] - def resolve_channel_url(sync_channel) - channel = sync_channel.to_s.empty? ? Config::DEFAULT_OFFICIAL_SYNC_CHANNEL : sync_channel - return Config::OFFICIAL_RELEASE_URL if channel == Config::DEFAULT_OFFICIAL_SYNC_CHANNEL - - raise Errors::ConfigError, "Unknown sync channel '#{channel}'" - end - - ## - # @param manifest_version [String] - # @param max_version [String, nil] - # @return [Boolean] true when manifest_version is newer than max_version - def exceeds_max?(manifest_version, max_version) - return false if max_version.nil? || max_version.empty? - - compare(normalize(manifest_version), normalize(max_version)).positive? - end - - ## - # @param uri [URI::HTTPS] - # @param policy [RedirectPolicy] - # @return [String] - def follow_redirects!(uri, policy) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength - hops = 0 - - loop do - ensure_allowed_host!(uri.host, policy.allowed_hosts) - response = perform_request!(uri) - - case response - in Net::HTTPRedirection - hops += 1 - raise Errors::SyncError, 'Registry sync exceeded redirect limit' if hops > policy.max_hops - - location = response['location'] - raise Errors::SyncError, 'Registry sync redirect missing Location header' if location.to_s.empty? - - uri = parse_https_uri!(URI.join(uri, location).to_s) - next - else - reject_error_status!(response) - return read_body!(response) - end - end - end - - ## - # @return [Array] - def extra_allowed_hosts - raw = ENV.fetch('REGISTRY_SYNC_ALLOWED_HOSTS', '') - EXTRA_ALLOWED_HOSTS_CACHE.fetch(raw) do - EXTRA_ALLOWED_HOSTS_CACHE[raw] = raw.split(',').map(&:strip).reject(&:empty?).freeze - end - end - - ## - # @param api_url [String] - # @param tag [String, nil] - # @return [String] - def resolve_official_asset_url(api_url, tag: nil) # rubocop:disable Metrics/MethodLength - response_body = fetch!(api_url) - release = JSON.parse(response_body, symbolize_names: true) - asset = Array(release[:assets]).find { it[:name] == OFFICIAL_ASSET_NAME } - url = asset&.dig(:browser_download_url) - unless url - message = if tag - "Official release asset '#{OFFICIAL_ASSET_NAME}' not found for tag '#{tag}'" - else - "Official release asset '#{OFFICIAL_ASSET_NAME}' not found" - end - raise Errors::SyncError, message - end - - url - rescue JSON::ParserError => error - raise Errors::SyncError, "Invalid GitHub release metadata: #{error.message}" - end - - ## - # @param version [String] - # @return [String] - def normalize(version) - version.to_s.delete_prefix('v') - end - - ## - # @param left [String] - # @param right [String] - # @return [Integer] - def compare(left, right) - Gem::Version.new(left) <=> Gem::Version.new(right) - rescue ArgumentError - left <=> right - end - - ## - # @param url [String] - # @return [URI::HTTPS] - def parse_https_uri!(url) - uri = URI(url) - unless uri.is_a?(URI::HTTPS) - raise Errors::SyncError, "Registry sync requires HTTPS URLs (got #{uri.scheme.inspect})" - end - - uri - end - - ## - # @param host [String] - # @param allowed_hosts [Array] - # @return [void] - def ensure_allowed_host!(host, allowed_hosts) - return if allowed_hosts.include?(host) - - raise Errors::SyncError, "Registry sync host not allowed: #{host}" - end - - ## - # @param uri [URI::HTTPS] - # @return [Net::HTTPResponse] - def perform_request!(uri) # rubocop:disable Metrics/MethodLength - response = nil - Net::HTTP.start( - uri.host, - uri.port, - use_ssl: true, - open_timeout: FETCH_OPEN_TIMEOUT_SECONDS, - read_timeout: FETCH_READ_TIMEOUT_SECONDS - ) do |http| - request = Net::HTTP::Get.new(uri) - request['Accept'] = 'application/octet-stream' - request['User-Agent'] = 'html2rss-web/registry-sync' - response = http.request(request) - end - response - end - - ## - # @param response [Net::HTTPResponse] - # @return [void] - def reject_error_status!(response) - return if response.is_a?(Net::HTTPSuccess) - - raise Errors::SyncError, "Registry sync fetch failed with HTTP #{response.code}" - end - - ## - # @param response [Net::HTTPResponse] - # @return [String] - def read_body!(response) - body = response.body.to_s - if body.bytesize > MAX_RESPONSE_BYTES - raise Errors::SyncError, "Registry sync response exceeds max bytes (#{MAX_RESPONSE_BYTES})" - end - - body - end - end # rubocop:enable Metrics/ModuleLength - end - end -end diff --git a/app/web/structured_data.rb b/app/web/structured_data.rb deleted file mode 100644 index d829d6d5c..000000000 --- a/app/web/structured_data.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -module Html2rss - module Web - ## - # Shared helpers for cloning nested Hash/Array config structures. - module StructuredData - module_function - - ## - # @param value [Object] - # @return [Object] - def deep_dup(value) # rubocop:disable Metrics/MethodLength - case value - when Hash - value.each_with_object({}) do |(key, val), memo| - memo[key.is_a?(String) ? key.dup : key] = deep_dup(val) - end - when Array - value.map { deep_dup(it) } - when String - value.dup - else - value - end - end - end - end -end diff --git a/bin/registry-sync b/bin/registry-sync index e8321a8b2..5cb516ffe 100755 --- a/bin/registry-sync +++ b/bin/registry-sync @@ -48,7 +48,7 @@ module RegistrySyncCLI puts headers.join("\t") rows.each do |row| puts [ - row.registry_id, + row.id, row.mode, row.version || '-', row.staged_version || '-', diff --git a/spec/fixtures/registries/official/configs/anthropic.com/news.yml b/spec/fixtures/registries/official/configs/anthropic.com/news.yml new file mode 100644 index 000000000..af1cbc812 --- /dev/null +++ b/spec/fixtures/registries/official/configs/anthropic.com/news.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: anthropic.com/news +directory: + topics: + - tech + - research + title: "Anthropic — News" + summary: "Product and research announcements from Anthropic." +channel: + title: "Anthropic — News" + url: https://www.anthropic.com/news + language: en + time_zone: UTC + ttl: 360 +selectors: + items: + selector: 'ul[class*="PublicationList"] > li > a[href^="/news/"]' + enhance: false + title: + selector: '[class*="__title"]' + url: + extractor: href diff --git a/spec/fixtures/registries/official/configs/deepmind.google/blog.yml b/spec/fixtures/registries/official/configs/deepmind.google/blog.yml new file mode 100644 index 000000000..6d3733c1e --- /dev/null +++ b/spec/fixtures/registries/official/configs/deepmind.google/blog.yml @@ -0,0 +1,27 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: deepmind.google/blog +directory: + topics: + - science + - research + title: "Google DeepMind — Blog" + summary: "Research and product posts from Google DeepMind." + +channel: + title: "Google DeepMind — Blog" + url: https://deepmind.google/blog/ + language: en + time_zone: UTC + ttl: 360 +selectors: + items: + selector: .card__inner + enhance: false + title: + selector: h3 + url: + selector: .card__overlay-link + extractor: href + published_at: + selector: time diff --git a/spec/fixtures/registries/official/configs/phys.org/weekly.yml b/spec/fixtures/registries/official/configs/phys.org/weekly.yml deleted file mode 100644 index 4c0102d03..000000000 --- a/spec/fixtures/registries/official/configs/phys.org/weekly.yml +++ /dev/null @@ -1,25 +0,0 @@ -registry: - id: phys.org/weekly -directory: - topics: - - science - title: "Phys.org — Weekly" - summary: "Top science news of the week from Phys.org." -channel: - language: en - title: "Phys.org — Weekly" - url: https://phys.org/weekly-news/ - time_zone: Europe/London - ttl: 1440 -selectors: - items: - selector: ".sorted-news-list .sorted-article-content" - title: - selector: "h4" - category: - selector: ".text-info" - categories: - - category - url: - selector: ".news-link" - extractor: "href" diff --git a/spec/fixtures/registries/official/configs/support.apple.com/en_gb_ht201222.yml b/spec/fixtures/registries/official/configs/support.apple.com/en_gb_ht201222.yml deleted file mode 100644 index 83c1c81b6..000000000 --- a/spec/fixtures/registries/official/configs/support.apple.com/en_gb_ht201222.yml +++ /dev/null @@ -1,34 +0,0 @@ -registry: - id: support.apple.com/en_gb_ht201222 -directory: - topics: - - tech - - security - title: "Apple Support — Security releases" - summary: "Apple security update and release notes (HT201222 / related)." -strategy: botasaurus -channel: - title: "Apple Support — Security releases" - url: https://support.apple.com/en-gb/100100 - language: en - ttl: 360 - time_zone: UTC -request: - botasaurus: - wait_for_selector: ".table-wrapper table tbody tr a" - wait_timeout_seconds: 20 -selectors: - items: - selector: ".table-wrapper table tbody > tr:not(:first-child)" - enhance: false - title: - selector: a - url: - selector: a - extractor: href - description: - selector: "td:nth-child(2)" - published_at: - selector: "td:nth-child(3)" - post_process: - - name: parse_time diff --git a/spec/fixtures/registries/official/manifest.json b/spec/fixtures/registries/official/manifest.json index d02f057ad..82f32f845 100644 --- a/spec/fixtures/registries/official/manifest.json +++ b/spec/fixtures/registries/official/manifest.json @@ -4,7 +4,8 @@ "version": "test-fixture", "public_key_id": "test", "files": { - "configs/phys.org/weekly.yml": "0e05fa9a95ec56bef4b4363b2f044ab69642fcda9acd91ebf8d28b39141a963d", - "configs/support.apple.com/en_gb_ht201222.yml": "53e38a6b7d088e0c79b19a7dd6db9b009c6b5c0b6690f504c7160b0c32b71e41" + "configs/anthropic.com/news.yml": "679b3c28b6dd7e08bbfdaf8564e077e7c2ac4140eacbc9543756591aaf0621f5", + "configs/deepmind.google/blog.yml": "3dbc30f109bb169f1de06372236a2d352584808d11be8243ba93befcd94a7b41" } } + diff --git a/spec/fixtures/registries/sync/bundle/manifest.json b/spec/fixtures/registries/sync/bundle/manifest.json index 4e20cdcbf..8a0bf8a6d 100644 --- a/spec/fixtures/registries/sync/bundle/manifest.json +++ b/spec/fixtures/registries/sync/bundle/manifest.json @@ -4,7 +4,8 @@ "version": "test-fixture", "public_key_id": "test-key", "files": { - "configs/phys.org/weekly.yml": "0e05fa9a95ec56bef4b4363b2f044ab69642fcda9acd91ebf8d28b39141a963d", - "configs/support.apple.com/en_gb_ht201222.yml": "53e38a6b7d088e0c79b19a7dd6db9b009c6b5c0b6690f504c7160b0c32b71e41" + "configs/anthropic.com/news.yml": "679b3c28b6dd7e08bbfdaf8564e077e7c2ac4140eacbc9543756591aaf0621f5", + "configs/deepmind.google/blog.yml": "3dbc30f109bb169f1de06372236a2d352584808d11be8243ba93befcd94a7b41" } } + diff --git a/spec/html2rss/web/app_spec.rb b/spec/html2rss/web/app_spec.rb index ead4b1827..294b3ebed 100644 --- a/spec/html2rss/web/app_spec.rb +++ b/spec/html2rss/web/app_spec.rb @@ -24,7 +24,7 @@ def static_feed_json end def stub_static_feed(rss_body: '', json_body: static_feed_json, ttl: 180) - allow(Html2rss::Web::LocalConfig).to receive(:find).and_return({ channel: { ttl: ttl } }) + allow(Html2rss::Web::Registry::Index.current).to receive(:config_for).and_return({ channel: { ttl: ttl } }) stub_static_renderers(static_feed_result(ttl:), rss_body:, json_body:) end @@ -59,8 +59,8 @@ def static_service_error_result end def stub_static_service_error(feed_name) - allow(Html2rss::Web::LocalConfig) - .to receive(:find) + allow(Html2rss::Web::Registry::Index.current) + .to receive(:config_for) .with(feed_name) .and_return({ channel: { ttl: 180 } }) allow(Html2rss::Web::Feeds::Service).to receive(:call).and_return(static_service_error_result) @@ -145,7 +145,8 @@ def app = described_class end it 'serves nested static feed routes' do - allow(Html2rss::Web::LocalConfig).to receive(:find).with('team/releases').and_return({ channel: { ttl: 180 } }) + allow(Html2rss::Web::Registry::Index.current) + .to receive(:config_for).with('team/releases').and_return({ channel: { ttl: 180 } }) stub_static_renderers(static_feed_result(ttl: 180), rss_body: '', json_body: static_feed_json) get '/team/releases.xml' @@ -157,7 +158,8 @@ def app = described_class it 'serves HEAD requests for static feed routes with negotiated headers only' do feed_name = "legacy-head-#{SecureRandom.hex(4)}" - allow(Html2rss::Web::LocalConfig).to receive(:find).with(feed_name).and_return({ channel: { ttl: 180 } }) + allow(Html2rss::Web::Registry::Index.current) + .to receive(:config_for).with(feed_name).and_return({ channel: { ttl: 180 } }) stub_static_renderers(static_feed_result(ttl: 180), rss_body: '', json_body: static_feed_json) head "/#{feed_name}" @@ -176,7 +178,8 @@ def app = described_class it 'coerces string ttl values before cache expiry math' do feed_name = "legacy-ttl-#{SecureRandom.hex(4)}" - allow(Html2rss::Web::LocalConfig).to receive(:find).with(feed_name).and_return({ channel: { ttl: '180' } }) + allow(Html2rss::Web::Registry::Index.current) + .to receive(:config_for).with(feed_name).and_return({ channel: { ttl: '180' } }) stub_static_renderers(static_feed_result(ttl: '180'), rss_body: '', json_body: static_feed_json) get "/#{feed_name}" diff --git a/spec/html2rss/web/feeds/responder_spec.rb b/spec/html2rss/web/feeds/responder_spec.rb index 1d8e1a9d5..0ebd91e54 100644 --- a/spec/html2rss/web/feeds/responder_spec.rb +++ b/spec/html2rss/web/feeds/responder_spec.rb @@ -25,7 +25,7 @@ end before do - allow(Html2rss::Web::LocalConfig).to receive(:find).with('example').and_return(static_config) + allow(Html2rss::Web::Registry::Index.current).to receive(:config_for).with('example').and_return(static_config) allow(Html2rss::Web::Observability).to receive(:emit) allow(Html2rss::Web::SentryOps).to receive(:emit_failure_telemetry) end diff --git a/spec/html2rss/web/feeds/source_resolver_spec.rb b/spec/html2rss/web/feeds/source_resolver_spec.rb index fcb43545f..a17725228 100644 --- a/spec/html2rss/web/feeds/source_resolver_spec.rb +++ b/spec/html2rss/web/feeds/source_resolver_spec.rb @@ -27,7 +27,7 @@ def resolved_tuple(resolved) end before do - allow(Html2rss::Web::LocalConfig).to receive(:find).with('legacy').and_return(config) + allow(Html2rss::Web::Registry::Index.current).to receive(:config_for).with('legacy').and_return(config) end it 'normalizes the static source into shared generator input without forcing a default strategy', diff --git a/spec/html2rss/web/local_config_spec.rb b/spec/html2rss/web/local_config_spec.rb index dfb6f9b44..8eaaa60c6 100644 --- a/spec/html2rss/web/local_config_spec.rb +++ b/spec/html2rss/web/local_config_spec.rb @@ -51,14 +51,6 @@ def account_token(snapshot) expect(titles_for('example.json', 'example.rss', 'example.xml')).to eq(%w[Example Example Example]) end - it 'falls back to registry configs when the feed is not in local yaml' do - allow(described_class).to receive(:snapshot).and_return(empty_snapshot) - - config = described_class.find('support.apple.com/en_gb_ht201222.rss') - - expect(config).to include(channel: hash_including(title: 'Apple Support — Security releases')) - end - it 'returns not found for unknown feed ids' do allow(described_class).to receive(:snapshot).and_return(empty_snapshot) diff --git a/spec/html2rss/web/registry/index_spec.rb b/spec/html2rss/web/registry/index_spec.rb index f6985afce..af27dc5e0 100644 --- a/spec/html2rss/web/registry/index_spec.rb +++ b/spec/html2rss/web/registry/index_spec.rb @@ -8,9 +8,9 @@ RSpec.describe Html2rss::Web::Registry::Index do describe '#config_for' do it 'returns registry configs by feed id' do - config = described_class.current.config_for('support.apple.com/en_gb_ht201222') + config = described_class.current.config_for('anthropic.com/news') - expect(config).to include(channel: hash_including(title: 'Apple Support — Security releases')) + expect(config).to include(channel: hash_including(title: 'Anthropic — News')) end it 'returns nil for unknown ids' do @@ -27,12 +27,12 @@ describe '#catalog_rows' do it 'includes registry rows with source and registry fields' do rows = described_class.current.catalog_rows - phys = rows.find { it.fetch(:id) == 'phys.org/weekly' } + anthropic = rows.find { it.fetch(:id) == 'anthropic.com/news' } - expect(phys).to include( + expect(anthropic).to include( source: 'registry', registry: 'official', - path: '/phys.org/weekly.rss' + path: '/anthropic.com/news.rss' ) end @@ -44,9 +44,9 @@ it 'prefers the first registry in precedence for duplicate feed ids' do rows = described_class.current.catalog_rows - apple = rows.find { it.fetch(:id) == 'support.apple.com/en_gb_ht201222' } + deepmind = rows.find { it.fetch(:id) == 'deepmind.google/blog' } - expect(apple.fetch(:registry)).to eq('official') + expect(deepmind.fetch(:registry)).to eq('official') end it 'merges local feeds.yml rows after registry rows', :aggregate_failures do # rubocop:disable RSpec/ExampleLength @@ -77,7 +77,7 @@ expect(official).to have_attributes( version: 'test-fixture', - sync_mode: :path + mode: :path ) end end @@ -99,14 +99,14 @@ path: spec/fixtures/registries/official catalog: true allowed_channel_domains: - - phys.org - - apple.com + - anthropic.com + - deepmind.google YAML ENV['REGISTRIES_CONFIG'] = config_path described_class.reload! - expect(described_class.current.config_for('phys.org/weekly')).to include( - channel: hash_including(url: 'https://phys.org/weekly-news/') + expect(described_class.current.config_for('anthropic.com/news')).to include( + channel: hash_including(url: 'https://www.anthropic.com/news') ) end @@ -125,8 +125,8 @@ ENV['REGISTRIES_CONFIG'] = config_path described_class.reload! - expect { described_class.current.config_for('phys.org/weekly') } - .to raise_error(Html2rss::Web::Registry::Errors::LoadError, /phys.org/) + expect { described_class.current.config_for('anthropic.com/news') } + .to raise_error(Html2rss::Web::Registry::Errors::LoadError, /anthropic\.com/) end end end diff --git a/spec/html2rss/web/registry/sync_spec.rb b/spec/html2rss/web/registry/sync_spec.rb index 1440d0cde..06640f1f8 100644 --- a/spec/html2rss/web/registry/sync_spec.rb +++ b/spec/html2rss/web/registry/sync_spec.rb @@ -34,8 +34,8 @@ expect(status.version).to eq('test-fixture') expect(Html2rss::Web::Registry::Store.bundle_present?('official')).to be(true) - expect(Html2rss::Web::Registry::Index.current.config_for('phys.org/weekly')).to include( - channel: hash_including(title: 'Phys.org — Weekly') + expect(Html2rss::Web::Registry::Index.current.config_for('anthropic.com/news')).to include( + channel: hash_including(title: 'Anthropic — News') ) end @@ -110,7 +110,7 @@ row = described_class.status(registry_id: 'official').first expect(row).to have_attributes( - registry_id: 'official', + id: 'official', mode: :sync, sync_url: 'https://registry.test.example/registry-bundle.tar.gz', staged_version: nil @@ -155,9 +155,8 @@ def write_policy_config(yaml) expect(status.staged_version).to eq('test-fixture') end - it 'promotes staged bundles and emits catalog change telemetry', :aggregate_failures do # rubocop:disable RSpec/ExampleLength + it 'promotes staged bundles and emits catalog change telemetry', :aggregate_failures do allow(Html2rss::Web::Observability).to receive(:emit) - allow(Html2rss::Web::SecurityLogger).to receive(:log_registry_catalog_changed) write_policy_config( RegistrySyncTestHelpers.policy_registry_yaml(download_url:, auto_promote: false) @@ -168,13 +167,9 @@ def write_policy_config(yaml) expect(status.version).to eq('test-fixture') expect(Html2rss::Web::Registry::Store.staged_present?('official')).to be(false) - expect(Html2rss::Web::Observability).to have_received(:emit).with( - hash_including(event_name: 'registry.promote_staged', outcome: 'success') - ) expect(Html2rss::Web::Observability).to have_received(:emit).with( hash_including(event_name: 'registry.catalog_changed', outcome: 'success') ) - expect(Html2rss::Web::SecurityLogger).to have_received(:log_registry_catalog_changed) end it 'rejects manifests newer than max_version' do @@ -191,21 +186,6 @@ def write_policy_config(yaml) end end - describe '.cli_exit_code', :registry_sync do - it 'returns non-zero when a sync registry has no bundle' do - expect(described_class.cli_exit_code).to eq(1) - end - - it 'returns zero after a successful sync' do - stub_request(:get, 'https://registry.test.example/registry-bundle.tar.gz') - .to_return(status: 200, body: RegistrySyncTestHelpers.build_signed_tarball) - - described_class.run(registry_id: 'official') - - expect(described_class.cli_exit_code).to eq(0) - end - end - describe '.boot!' do it 'does not run in the test environment' do allow(described_class).to receive(:seed_registry!) @@ -217,20 +197,7 @@ def write_policy_config(yaml) end end - describe Html2rss::Web::Registry::SyncTransport do - describe '.exceeds_max?' do - [ - ['2026.08.22', '2026.08.21', true], - ['2026.08.21', '2026.08.22', false], - ['v2026.08.22', '2026.08.21', true], - ['2026.08.22', nil, false] - ].each do |manifest_version, max_version, expected| - it "returns #{expected} for #{manifest_version} vs #{max_version.inspect}" do - expect(described_class.exceeds_max?(manifest_version, max_version)).to be(expected) - end - end - end - + describe Html2rss::Web::Registry::ChannelResolver do describe '.resolve' do it 'uses the GitHub tag release API when pin_version is set', :aggregate_failures do # rubocop:disable RSpec/ExampleLength tag_api = format( @@ -245,7 +212,7 @@ def write_policy_config(yaml) }.to_json ) - entry = Html2rss::Web::Registry::Entry.new( + definition = Html2rss::Web::Registry::Definition.new( id: 'official', mode: :sync, path: nil, @@ -258,7 +225,7 @@ def write_policy_config(yaml) allowed_channel_domains: [] ) - expect(described_class.resolve(entry)).to eq(download_url) + expect(described_class.resolve(definition)).to eq(download_url) end end end From 468dc5de72835ec4c8260e6f7e399c9de8a730aa Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sun, 23 Aug 2026 11:58:49 +0200 Subject: [PATCH 21/33] refactor(registry): streamline registry modules, unify StructuredData Zeitwerk loading, and deduplicate calls --- app/web/api/v1/configs.rb | 8 +- app/web/api/v1/root_metadata.rb | 5 +- app/web/boot/setup.rb | 6 +- app/web/config/config_snapshot.rb | 5 +- app/web/config/local_config.rb | 6 +- app/web/config/structured_data.rb | 38 ++++---- app/web/feeds/source_resolver.rb | 27 +++--- app/web/registry/channel_resolver.rb | 21 ++--- app/web/registry/config.rb | 52 +++++++---- app/web/registry/http_transport.rb | 17 ++-- app/web/registry/index.rb | 105 +++++++++++++++-------- app/web/registry/store.rb | 9 +- app/web/registry/sync.rb | 39 +++++---- spec/html2rss/web/registry/store_spec.rb | 4 +- spec/support/registry_sync.rb | 7 +- 15 files changed, 205 insertions(+), 144 deletions(-) diff --git a/app/web/api/v1/configs.rb b/app/web/api/v1/configs.rb index 827450308..694b46252 100644 --- a/app/web/api/v1/configs.rb +++ b/app/web/api/v1/configs.rb @@ -25,12 +25,16 @@ def index(_router) private def build_entries - started = Process.clock_gettime(Process::CLOCK_MONOTONIC) + started = monotonic_now entries = Registry::Index.current.catalog_rows - duration_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000).round + duration_ms = ((monotonic_now - started) * 1000).round [entries, duration_ms] end + def monotonic_now + Process.clock_gettime(Process::CLOCK_MONOTONIC) + end + def emit_success(count, duration_ms) Observability.emit( event_name: 'catalog.build', diff --git a/app/web/api/v1/root_metadata.rb b/app/web/api/v1/root_metadata.rb index b3ed16a0c..e3d38096e 100644 --- a/app/web/api/v1/root_metadata.rb +++ b/app/web/api/v1/root_metadata.rb @@ -35,9 +35,10 @@ def instance_payload(router) # @return [Hash{Symbol => Object}] def feed_creation_payload + auto_source = Flags.auto_source_enabled? { - enabled: Flags.auto_source_enabled?, - access_token_required: Flags.auto_source_enabled? + enabled: auto_source, + access_token_required: auto_source } end diff --git a/app/web/boot/setup.rb b/app/web/boot/setup.rb index d41be5449..16a112a2b 100644 --- a/app/web/boot/setup.rb +++ b/app/web/boot/setup.rb @@ -33,9 +33,11 @@ def call! # @return [void] def configure_gem_defaults! global_config = LocalConfig.global + headers = global_config[:headers] + stylesheets = global_config[:stylesheets] Html2rss.configure do |config| - config.headers = global_config[:headers] if global_config[:headers] - config.stylesheets = global_config[:stylesheets] if global_config[:stylesheets] + config.headers = headers if headers + config.stylesheets = stylesheets if stylesheets end end diff --git a/app/web/config/config_snapshot.rb b/app/web/config/config_snapshot.rb index c38eb1b61..91e42b708 100644 --- a/app/web/config/config_snapshot.rb +++ b/app/web/config/config_snapshot.rb @@ -47,7 +47,8 @@ def normalize_feeds(raw_feeds) return {} unless raw_feeds.is_a?(Hash) raw_feeds.each_with_object({}) do |(name, config), memo| - memo[name.to_sym] = FeedConfig.new(name: name.to_sym, raw: Config::StructuredData.deep_dup(config).freeze) + sym_name = name.to_sym + memo[sym_name] = FeedConfig.new(name: sym_name, raw: StructuredData.deep_dup(config).freeze) end end @@ -68,7 +69,7 @@ def normalize_accounts(raw_accounts) # @param accounts [Array] # @return [Hash{Symbol=>Object}] def normalized_global_hash(global_hash, accounts) - normalized = Config::StructuredData.deep_dup(global_hash) + normalized = StructuredData.deep_dup(global_hash) return normalized unless normalized.key?(:auth) normalized[:auth] = normalized_auth_hash(normalized[:auth], accounts) diff --git a/app/web/config/local_config.rb b/app/web/config/local_config.rb index bb6306cfb..7326a052f 100644 --- a/app/web/config/local_config.rb +++ b/app/web/config/local_config.rb @@ -33,19 +33,19 @@ def find(feed_id) config = snapshot.feeds[normalized.to_sym] raise NotFound, "Did not find local feed config at '#{normalized}'" unless config - Config::StructuredData.deep_dup(config.raw) + StructuredData.deep_dup(config.raw) end ## # @return [Hash{Symbol => Hash{Symbol => Object}}] def feeds - snapshot.feeds.transform_values { Config::StructuredData.deep_dup(it.raw) } + snapshot.feeds.transform_values { StructuredData.deep_dup(it.raw) } end ## # @return [Hash{Symbol => Object}] def global - Config::StructuredData.deep_dup(snapshot.global) + StructuredData.deep_dup(snapshot.global) end ## diff --git a/app/web/config/structured_data.rb b/app/web/config/structured_data.rb index 83f6867c4..d829d6d5c 100644 --- a/app/web/config/structured_data.rb +++ b/app/web/config/structured_data.rb @@ -2,28 +2,26 @@ module Html2rss module Web - module Config - ## - # Shared helpers for cloning nested Hash/Array config structures. - module StructuredData - module_function + ## + # Shared helpers for cloning nested Hash/Array config structures. + module StructuredData + module_function - ## - # @param value [Object] - # @return [Object] - def deep_dup(value) # rubocop:disable Metrics/MethodLength - case value - when Hash - value.each_with_object({}) do |(key, val), memo| - memo[key.is_a?(String) ? key.dup : key] = deep_dup(val) - end - when Array - value.map { deep_dup(it) } - when String - value.dup - else - value + ## + # @param value [Object] + # @return [Object] + def deep_dup(value) # rubocop:disable Metrics/MethodLength + case value + when Hash + value.each_with_object({}) do |(key, val), memo| + memo[key.is_a?(String) ? key.dup : key] = deep_dup(val) end + when Array + value.map { deep_dup(it) } + when String + value.dup + else + value end end end diff --git a/app/web/feeds/source_resolver.rb b/app/web/feeds/source_resolver.rb index 5ecfaba8f..d15046b42 100644 --- a/app/web/feeds/source_resolver.rb +++ b/app/web/feeds/source_resolver.rb @@ -12,13 +12,14 @@ class << self # @param feed_request [Html2rss::Web::Feeds::Contracts::Request] # @return [Html2rss::Web::Feeds::Contracts::ResolvedSource] def call(feed_request) - case feed_request.target_kind + target_kind = feed_request.target_kind + case target_kind when :static resolve_static(feed_request) when :token resolve_token(feed_request) else - raise Html2rss::Web::BadRequestError, "Unsupported feed target: #{feed_request.target_kind}" + raise Html2rss::Web::BadRequestError, "Unsupported feed target: #{target_kind}" end end @@ -27,15 +28,15 @@ def call(feed_request) # @param feed_request [Html2rss::Web::Feeds::Contracts::Request] # @return [Html2rss::Web::Feeds::Contracts::ResolvedSource] def resolve_static(feed_request) - config = Registry::Index.current.config_for(feed_request.feed_name) - raise Html2rss::Web::NotFoundError unless config - - generator_input = static_generator_input(config, feed_request.params) + feed_name = feed_request.feed_name + params = feed_request.params + config = Registry::Index.current.config_for(feed_name) || raise(Html2rss::Web::NotFoundError) + generator_input = static_generator_input(config, params) resolved_source_for( source_kind: :static, - cache_identity: static_cache_identity(feed_request.feed_name, feed_request.params), - generator_input: generator_input, + cache_identity: static_cache_identity(feed_name, params), + generator_input:, ttl_seconds: Cache.seconds_from_minutes(generator_input.dig(:channel, :ttl)) ) end @@ -44,14 +45,14 @@ def resolve_static(feed_request) # @return [Html2rss::Web::Feeds::Contracts::ResolvedSource] def resolve_token(feed_request) ensure_auto_source_enabled! - feed_token = authorize_feed_token!(feed_request.token) - strategy = resolved_strategy(feed_token) - generator_input = token_generator_input(feed_token.url, strategy) + token = feed_request.token + feed_token = authorize_feed_token!(token) + generator_input = token_generator_input(feed_token.url, resolved_strategy(feed_token)) resolved_source_for( source_kind: :token, - cache_identity: token_cache_identity(feed_request.token), - generator_input: generator_input, + cache_identity: token_cache_identity(token), + generator_input:, ttl_seconds: Cache.seconds_from_minutes(generator_input.dig(:channel, :ttl), default: 300) ) end diff --git a/app/web/registry/channel_resolver.rb b/app/web/registry/channel_resolver.rb index f5050e191..93d6b0dc4 100644 --- a/app/web/registry/channel_resolver.rb +++ b/app/web/registry/channel_resolver.rb @@ -18,11 +18,12 @@ module ChannelResolver # @param definition [Definition] # @return [String] download URL def resolve(definition) - return definition.sync_url if definition.sync_url && !definition.sync_url.empty? + sync_url = definition.sync_url + return sync_url if sync_url && !sync_url.empty? if definition.sync_channel == Config::DEFAULT_OFFICIAL_SYNC_CHANNEL pin = definition.sync_policy.pin_version - return Config::OFFICIAL_RELEASE_URL if pin.nil? || pin.empty? + return Config::OFFICIAL_RELEASE_URL if pin.to_s.empty? api_url = format(OFFICIAL_GITHUB_TAG_RELEASES_API, tag: pin) return resolve_github_asset(api_url, tag: pin) @@ -35,17 +36,13 @@ def resolve(definition) # @param api_url [String] # @param tag [String, nil] # @return [String] - def resolve_github_asset(api_url, tag: nil) # rubocop:disable Metrics/MethodLength - body = HttpTransport.fetch!(api_url) - release = JSON.parse(body, symbolize_names: true) - asset = Array(release[:assets]).find { it[:name] == OFFICIAL_ASSET_NAME } - url = asset&.dig(:browser_download_url) - unless url - raise Errors::SyncError, - "Official asset '#{OFFICIAL_ASSET_NAME}' not found#{" for #{tag}" if tag}" - end + def resolve_github_asset(api_url, tag: nil) + release = JSON.parse(HttpTransport.fetch!(api_url), symbolize_names: true) + url = Array(release[:assets]).find { it[:name] == OFFICIAL_ASSET_NAME }&.dig(:browser_download_url) + return url if url - url + tag_suffix = " for #{tag}" if tag + raise Errors::SyncError, "Official asset '#{OFFICIAL_ASSET_NAME}' not found#{tag_suffix}" rescue JSON::ParserError => error raise Errors::SyncError, "Invalid GitHub release metadata: #{error.message}" end diff --git a/app/web/registry/config.rb b/app/web/registry/config.rb index 9418dfc5e..1854faf23 100644 --- a/app/web/registry/config.rb +++ b/app/web/registry/config.rb @@ -33,7 +33,7 @@ def public_keys ## # Parses registry configuration and applies zero-config defaults. - module Config + module Config # rubocop:disable Metrics/ModuleLength REGISTRIES_FILE = 'config/registries.yml' DEFAULT_PRECEDENCE = %w[official].freeze DEFAULT_OFFICIAL_SYNC_CHANNEL = 'html2rss-official' @@ -44,6 +44,8 @@ module Config -----END PUBLIC KEY----- PEM + ## + # Registry configuration document containing precedence order and registry definitions. Document = Data.define(:precedence, :entries) @mutex = Mutex.new @@ -89,10 +91,9 @@ def reload! ## # @return [Document] - def parse_document # rubocop:disable Metrics/CyclomaticComplexity + def parse_document raw_doc = load_yaml - precedence = Array(raw_doc[:precedence]).map(&:to_s).reject(&:empty?) - precedence = DEFAULT_PRECEDENCE if precedence.empty? + precedence = parse_precedence(raw_doc[:precedence]) registries = raw_doc[:registries] || {} entries = precedence.to_h do |id| @@ -103,6 +104,11 @@ def parse_document # rubocop:disable Metrics/CyclomaticComplexity Document.new(precedence:, entries:) end + def parse_precedence(raw_precedence) + list = Array(raw_precedence).map(&:to_s).reject(&:empty?) + list.empty? ? DEFAULT_PRECEDENCE : list + end + ## # @return [Hash{Symbol => Object}] def load_yaml @@ -135,11 +141,11 @@ def default_document # rubocop:disable Metrics/MethodLength # @param raw [Hash{Symbol => Object}] # @return [Definition] def build_definition(id, raw) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity - sync = raw[:sync].is_a?(Hash) ? raw[:sync] : {} + sync_raw = raw[:sync] + sync = sync_raw.is_a?(Hash) ? sync_raw : {} path = raw[:path]&.to_s mode = path && !path.empty? ? :path : :sync - key_pem = raw[:public_key]&.to_s - key = key_pem && !key_pem.strip.empty? ? parse_key(key_pem) : nil + public_key = parse_public_key(raw[:public_key]&.to_s) definition = Definition.new( id:, @@ -149,16 +155,12 @@ def build_definition(id, raw) # rubocop:disable Metrics/AbcSize, Metrics/Cycloma sync_url: sync[:url]&.to_s, catalog: raw.fetch(:catalog, true), public_key_id: raw[:public_key_id]&.to_s || 'html2rss:registry:2026', - public_key: key, - sync_policy: SyncPolicy.new( - pin_version: sync[:pin_version]&.to_s, - max_version: sync[:max_version]&.to_s, - auto_promote: raw[:auto_promote] == true - ), + public_key:, + sync_policy: build_sync_policy(sync, raw), allowed_channel_domains: Array(raw[:allowed_channel_domains]).map(&:to_s).reject(&:empty?) ) - if definition.mode == :sync && definition.public_key.nil? + if definition.mode == :sync && !definition.public_key raise Errors::ConfigError, "Sync registry '#{id}' requires a pinned public_key" end @@ -166,10 +168,24 @@ def build_definition(id, raw) # rubocop:disable Metrics/AbcSize, Metrics/Cycloma end ## - # @param pem [String] - # @return [OpenSSL::PKey::PKey] - def parse_key(pem) - OpenSSL::PKey.read(pem) + # @param sync [Hash{Symbol => Object}] + # @param raw [Hash{Symbol => Object}] + # @return [SyncPolicy] + def build_sync_policy(sync, raw) + SyncPolicy.new( + pin_version: sync[:pin_version]&.to_s, + max_version: sync[:max_version]&.to_s, + auto_promote: raw[:auto_promote] == true + ) + end + + ## + # @param key_pem [String, nil] + # @return [OpenSSL::PKey::PKey, nil] + def parse_public_key(key_pem) + return nil if key_pem.to_s.strip.empty? + + OpenSSL::PKey.read(key_pem) rescue OpenSSL::PKey::PKeyError => error raise Errors::ConfigError, "Invalid registry public_key: #{error.message}" end diff --git a/app/web/registry/http_transport.rb b/app/web/registry/http_transport.rb index 34463b116..392cc8d14 100644 --- a/app/web/registry/http_transport.rb +++ b/app/web/registry/http_transport.rb @@ -27,7 +27,7 @@ module HttpTransport # @param max_redirects [Integer] # @param allowed_hosts [Array] # @return [String] response body - def fetch!(url, max_redirects: DEFAULT_MAX_REDIRECTS, allowed_hosts: default_allowed_hosts) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def fetch!(url, max_redirects: DEFAULT_MAX_REDIRECTS, allowed_hosts: default_allowed_hosts) # rubocop:disable Metrics/MethodLength uri = parse_https_uri!(url) hops = 0 @@ -40,10 +40,7 @@ def fetch!(url, max_redirects: DEFAULT_MAX_REDIRECTS, allowed_hosts: default_all hops += 1 raise Errors::SyncError, 'Registry sync exceeded redirect limit' if hops > max_redirects - location = response['location'] - raise Errors::SyncError, 'Registry sync redirect missing Location header' if location.to_s.empty? - - uri = parse_https_uri!(URI.join(uri, location).to_s) + uri = handle_redirect(uri, response['location']) in Net::HTTPSuccess return read_body(response) else @@ -52,6 +49,16 @@ def fetch!(url, max_redirects: DEFAULT_MAX_REDIRECTS, allowed_hosts: default_all end end + ## + # @param current_uri [URI::HTTPS] + # @param location [String, nil] + # @return [URI::HTTPS] + def handle_redirect(current_uri, location) + raise Errors::SyncError, 'Registry sync redirect missing Location header' if location.to_s.empty? + + parse_https_uri!(URI.join(current_uri, location).to_s) + end + ## # @return [Array] def default_allowed_hosts diff --git a/app/web/registry/index.rb b/app/web/registry/index.rb index 0674227f0..1e8ff5abf 100644 --- a/app/web/registry/index.rb +++ b/app/web/registry/index.rb @@ -17,6 +17,14 @@ module Registry :sync_url, :last_error ) do + ## + # @param id [String, Symbol] + # @param mode [Symbol] + # @param version [String, nil] + # @param staged_version [String, nil] + # @param updated_at [Time, nil] + # @param sync_url [String, nil] + # @param last_error [String, nil] def initialize(id:, mode:, version: nil, staged_version: nil, updated_at: nil, sync_url: nil, last_error: nil) super end @@ -25,6 +33,8 @@ def initialize(id:, mode:, version: nil, staged_version: nil, updated_at: nil, s ## # Sole feed repository combining registry bundles and local feeds. class Index # rubocop:disable Metrics/ClassLength + ## + # Immutable container holding a loaded registry bundle's manifest, configs, and catalog entries. RegistryBundle = Data.define(:registry_id, :manifest, :configs, :catalog_entries) @mutex = Mutex.new @@ -56,7 +66,7 @@ def config_for(feed_id) loaded_bundles.each_value do |bundle| config = bundle.configs[id] - return Html2rss::Web::Config::StructuredData.deep_dup(config) if config + return StructuredData.deep_dup(config) if config end nil @@ -64,31 +74,10 @@ def config_for(feed_id) ## # @return [Array] - def catalog_entries # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength - registry_entries = Config.precedence.each_with_object({}) do |registry_id, rows| - next unless Config.catalog_enabled?(registry_id) - - bundle = loaded_bundles[registry_id] - next unless bundle - - bundle.catalog_entries.each do |entry| - rows[entry.id] ||= Html2rss::Registry::CatalogEntry.new( - id: entry.id, - path: entry.path, - directory: entry.directory, - channel: entry.channel, - parameters: entry.parameters, - source: 'registry', - registry: registry_id - ) - end - end - - local_entries = LocalConfig.feeds.filter_map do |feed_name, feed_config| - build_local_catalog_entry(feed_name, feed_config) - end - - registry_entries.merge(local_entries.to_h { [it.id, it] }).values.sort_by(&:id) + def catalog_entries + registry_entries = build_registry_catalog_entries + local_entries = build_local_catalog_entries + registry_entries.merge(local_entries).values.sort_by(&:id) end ## @@ -106,16 +95,17 @@ def bundle_for(registry_id) ## # @return [Array] - def status # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def status # rubocop:disable Metrics/MethodLength Config.precedence.map do |registry_id| definition = Config.entry(registry_id) bundle = loaded_bundles[registry_id] sync_state = Store.sync_state(registry_id) - sync_url = definition.mode == :sync ? ChannelResolver.resolve(definition) : nil + mode = definition.mode + sync_url = mode == :sync ? ChannelResolver.resolve(definition) : nil Status.new( id: registry_id, - mode: definition.mode, + mode:, version: bundle&.manifest&.version, staged_version: Store.staged_version(registry_id), updated_at: Store.manifest_mtime(bundle_directory(definition)), @@ -138,6 +128,35 @@ def loaded_bundles @loaded_bundles ||= Config.precedence.to_h { [it, load_bundle(it)] }.compact end + def build_registry_catalog_entries # rubocop:disable Metrics/MethodLength + Config.precedence.each_with_object({}) do |registry_id, rows| + next unless Config.catalog_enabled?(registry_id) + + bundle = loaded_bundles[registry_id] + next unless bundle + + bundle.catalog_entries.each do |entry| + entry_id = entry.id + rows[entry_id] ||= Html2rss::Registry::CatalogEntry.new( + id: entry_id, + path: entry.path, + directory: entry.directory, + channel: entry.channel, + parameters: entry.parameters, + source: 'registry', + registry: registry_id + ) + end + end + end + + def build_local_catalog_entries + LocalConfig.feeds.filter_map do |feed_name, feed_config| + entry = build_local_catalog_entry(feed_name, feed_config) + [entry.id, entry] if entry + end.to_h + end + def load_bundle(registry_id) # rubocop:disable Metrics/MethodLength definition = Config.entry(registry_id) directory = bundle_directory(definition) @@ -146,8 +165,7 @@ def load_bundle(registry_id) # rubocop:disable Metrics/MethodLength trust_opts = if definition.mode == :path { trust: :integrity_only } else - { trust: :signed, - public_keys: definition.public_keys } + { trust: :signed, public_keys: definition.public_keys } end bundle_data = Html2rss::Registry::Bundle.load(directory, **trust_opts) bundle = RegistryBundle.new( @@ -162,18 +180,28 @@ def load_bundle(registry_id) # rubocop:disable Metrics/MethodLength raise Errors::LoadError, "Failed to load registry '#{registry_id}': #{error.message}" end - def enforce_scrape_policy!(definition, bundle) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity + def enforce_scrape_policy!(definition, bundle) allowed = definition.allowed_channel_domains - return if allowed.nil? || allowed.empty? + return if allowed.empty? bundle.configs.each do |feed_id, config| - host = URI.parse(config.dig(:channel, :url).to_s).host rescue nil # rubocop:disable Style/RescueModifier - next if host && allowed.any? { host.downcase == it.downcase || host.downcase.end_with?(".#{it.downcase}") } + next if host_allowed?(config.dig(:channel, :url), allowed) raise Errors::LoadError, "Registry '#{definition.id}' config '#{feed_id}' host not allowed" end end + def host_allowed?(url, allowed) + host = URI.parse(url.to_s).host&.downcase rescue nil # rubocop:disable Style/RescueModifier + return false unless host + + allowed.any? { |domain| domain_matches?(host, domain.downcase) } + end + + def domain_matches?(host, domain) + host == domain || host.end_with?(".#{domain}") + end + def bundle_directory(definition) definition.mode == :path ? definition.path : Store.registry_dir(definition.id) end @@ -183,8 +211,9 @@ def normalize_feed_id(feed_id) end def local_config_for(feed_id) - feed = LocalConfig.feeds[feed_id.to_sym] || LocalConfig.feeds[feed_id] - feed ? Html2rss::Web::Config::StructuredData.deep_dup(feed) : nil + feeds = LocalConfig.feeds + feed = feeds[feed_id.to_sym] || feeds[feed_id] + feed ? StructuredData.deep_dup(feed) : nil rescue StandardError nil end @@ -192,7 +221,7 @@ def local_config_for(feed_id) def build_local_catalog_entry(feed_name, feed_config) # rubocop:disable Metrics/MethodLength directory = feed_config[:directory] || {} title = directory[:title]&.to_s - return nil if title.nil? || title.strip.empty? + return nil if title.to_s.strip.empty? id = feed_name.to_s channel = feed_config[:channel] || {} diff --git a/app/web/registry/store.rb b/app/web/registry/store.rb index 4517d32ea..57cad988f 100644 --- a/app/web/registry/store.rb +++ b/app/web/registry/store.rb @@ -13,6 +13,8 @@ module Store # rubocop:disable Metrics/ModuleLength DEFAULT_SEED_ROOT = '/app/registries/seed' SYNC_STATE_FILE = '.sync-state.json' + ## + # Immutable registry sync state. SyncState = Data.define(:last_error, :last_sync_at) class << self @@ -92,11 +94,11 @@ def swap!(registry_id, source_dir) target = registry_dir(registry_id) FileUtils.mkdir_p(File.dirname(target)) backup = "#{target}.backup.#{Process.pid}" - FileUtils.rm_rf(backup) FileUtils.mv(target, backup) if File.exist?(target) FileUtils.cp_r(source_dir, target) - FileUtils.rm_rf(backup) target + ensure + FileUtils.rm_rf(backup) if backup end ## @@ -139,9 +141,10 @@ def seed_if_empty!(registry_id, seed_path:) # rubocop:disable Naming/PredicateMe # @return [SyncState] def sync_state(registry_id) raw = read_sync_state.fetch(registry_id.to_s, {}) + last_sync = raw['last_sync_at'] SyncState.new( last_error: raw['last_error'], - last_sync_at: raw['last_sync_at'] ? Time.parse(raw['last_sync_at']) : nil + last_sync_at: last_sync ? Time.parse(last_sync) : nil ) end diff --git a/app/web/registry/sync.rb b/app/web/registry/sync.rb index ed7b933f7..816774887 100644 --- a/app/web/registry/sync.rb +++ b/app/web/registry/sync.rb @@ -40,8 +40,7 @@ def promote_staged!(registry_id:) # rubocop:disable Metrics/MethodLength definition = Config.entry(registry_id) raise Errors::SyncError, "Registry '#{registry_id}' is path mode" if definition.mode == :path unless Store.staged_present?(registry_id) - raise Errors::SyncError, - "Registry '#{registry_id}' has no staged bundle" + raise Errors::SyncError, "Registry '#{registry_id}' has no staged bundle" end previous_bundle = Index.current.bundle_for(registry_id) @@ -114,9 +113,10 @@ def run_sync!(registry_id:, dry_run:) # rubocop:disable Metrics/AbcSize, Metrics download_url = ChannelResolver.resolve(definition) tarball = HttpTransport.fetch!(download_url) - File.binwrite(File.join(staging_root, 'download.tar.gz'), tarball) + tarball_path = File.join(staging_root, 'download.tar.gz') + File.binwrite(tarball_path, tarball) - File.open(File.join(staging_root, 'download.tar.gz'), 'rb') do |io| + File.open(tarball_path, 'rb') do |io| Html2rss::Registry::Archive.extract!(io, into: staged_dir) end @@ -137,13 +137,11 @@ def run_sync!(registry_id:, dry_run:) # rubocop:disable Metrics/AbcSize, Metrics end Index.current.status_entry_for(registry_id) - rescue Html2rss::Registry::VerificationError => error - log_signature_failure!(registry_id, error.message) if error.message.match?(/signature|public_key_id/i) - Store.write_sync_state!(registry_id, last_error: error.message) unless dry_run - raise Errors::SyncError, error.message - rescue Html2rss::Registry::ArchiveError => error - Store.write_sync_state!(registry_id, last_error: error.message) unless dry_run - raise Errors::SyncError, error.message + rescue Html2rss::Registry::Error => error + msg = error.message + log_signature_failure!(registry_id, msg) if msg.match?(/signature|public_key_id/i) + Store.write_sync_state!(registry_id, last_error: msg) unless dry_run + raise Errors::SyncError, msg rescue StandardError => error Store.write_sync_state!(registry_id, last_error: error.message) unless dry_run raise @@ -159,28 +157,35 @@ def log_signature_failure!(registry_id, message) def enforce_max_version!(definition, manifest) max = definition.sync_policy.max_version - return if max.nil? || max.empty? - return unless Html2rss::Registry::Manifest.exceeds_max?(manifest.version, max) + return if max.to_s.empty? + + version = manifest.version + return unless Html2rss::Registry::Manifest.exceeds_max?(version, max) - raise Errors::SyncError, "Manifest version '#{manifest.version}' exceeds max_version '#{max}'" + raise Errors::SyncError, "Manifest version '#{version}' exceeds max_version '#{max}'" end def report_catalog_change!(registry_id, previous) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity current = Index.current.bundle_for(registry_id) return unless current + current_version = current.manifest.version prev_ids = previous ? Set.new(previous.catalog_entries.map(&:id)) : Set.new curr_ids = Set.new(current.catalog_entries.map(&:id)) added = (curr_ids - prev_ids).to_a.sort removed = (prev_ids - curr_ids).to_a.sort - return if added.empty? && removed.empty? && previous&.manifest&.version == current.manifest.version + return if added.empty? && removed.empty? && previous&.manifest&.version == current_version Observability.emit( event_name: 'registry.catalog_changed', outcome: 'success', level: :warn, - details: { registry_id:, version: current.manifest.version, added_count: added.size, - removed_count: removed.size } + details: { + registry_id:, + version: current_version, + added_count: added.size, + removed_count: removed.size + } ) end diff --git a/spec/html2rss/web/registry/store_spec.rb b/spec/html2rss/web/registry/store_spec.rb index 8a1d92508..716d5a7b7 100644 --- a/spec/html2rss/web/registry/store_spec.rb +++ b/spec/html2rss/web/registry/store_spec.rb @@ -66,9 +66,7 @@ def build_bundle_dir(version) # rubocop:disable Metrics/MethodLength registry_id: 'store-test', version:, public_key_id: 'test', - files: { - 'configs/example.com/feed.yml' => digest - } + files: { 'configs/example.com/feed.yml' => digest } }.to_json ) dir diff --git a/spec/support/registry_sync.rb b/spec/support/registry_sync.rb index 398a24865..e59629bf3 100644 --- a/spec/support/registry_sync.rb +++ b/spec/support/registry_sync.rb @@ -41,14 +41,13 @@ def configure_sync_registry! def build_signed_tarball # rubocop:disable Metrics/MethodLength bundle_dir = Dir.mktmpdir('signed-registry-bundle') + manifest_target = File.join(bundle_dir, Html2rss::Registry::Manifest::MANIFEST_FILE) FileUtils.cp_r(File.join(OFFICIAL_FIXTURES_ROOT, 'configs'), File.join(bundle_dir, 'configs')) FileUtils.cp( File.join(SYNC_FIXTURES_ROOT, 'bundle', Html2rss::Registry::Manifest::MANIFEST_FILE), - File.join(bundle_dir, Html2rss::Registry::Manifest::MANIFEST_FILE) - ) - manifest = Html2rss::Registry::Manifest.parse( - File.read(File.join(bundle_dir, Html2rss::Registry::Manifest::MANIFEST_FILE)) + manifest_target ) + manifest = Html2rss::Registry::Manifest.parse(File.read(manifest_target)) Html2rss::Registry::TestSupport.sign!(manifest, key_pem: TEST_PRIVATE_KEY, bundle_dir:) pack_bundle_dir(bundle_dir) From 63066d9320146f4b124a3a882b33572cff2b52b8 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sun, 23 Aug 2026 18:28:21 +0200 Subject: [PATCH 22/33] refactor(registry): map catalog wire fields at Index edge Stamp source/registry on catalog API rows in Index instead of cloning CatalogEntry with gem wire defaults after the domain-only hard break. --- Gemfile.lock | 2 +- app/web/registry/index.rb | 37 ++++++++++++++++++++++--------------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 931ba695e..66718b1ed 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/html2rss/html2rss - revision: aa504b8a689ef1e79d7c823862697208c4d30e74 + revision: da2b757d1f8aea4ef7abad65c9a5514c35430483 branch: feat/registry-v1 specs: html2rss (0.28.0) diff --git a/app/web/registry/index.rb b/app/web/registry/index.rb index 1e8ff5abf..08f5075a7 100644 --- a/app/web/registry/index.rb +++ b/app/web/registry/index.rb @@ -81,9 +81,13 @@ def catalog_entries end ## + # Wire rows for the catalog API (+source+ / +registry+ stamped here). + # # @return [Array Object}>] def catalog_rows - catalog_entries.map(&:to_h) + registry_rows = build_registry_catalog_rows + local_rows = build_local_catalog_rows + registry_rows.merge(local_rows).values.sort_by { it.fetch(:id) } end ## @@ -128,7 +132,17 @@ def loaded_bundles @loaded_bundles ||= Config.precedence.to_h { [it, load_bundle(it)] }.compact end - def build_registry_catalog_entries # rubocop:disable Metrics/MethodLength + def build_registry_catalog_entries + merge_registry_catalog { |entry, _registry_id| entry } + end + + def build_registry_catalog_rows + merge_registry_catalog do |entry, registry_id| + entry.to_h.merge(source: 'registry', registry: registry_id) + end + end + + def merge_registry_catalog Config.precedence.each_with_object({}) do |registry_id, rows| next unless Config.catalog_enabled?(registry_id) @@ -136,16 +150,7 @@ def build_registry_catalog_entries # rubocop:disable Metrics/MethodLength next unless bundle bundle.catalog_entries.each do |entry| - entry_id = entry.id - rows[entry_id] ||= Html2rss::Registry::CatalogEntry.new( - id: entry_id, - path: entry.path, - directory: entry.directory, - channel: entry.channel, - parameters: entry.parameters, - source: 'registry', - registry: registry_id - ) + rows[entry.id] ||= yield(entry, registry_id) end end end @@ -157,6 +162,10 @@ def build_local_catalog_entries end.to_h end + def build_local_catalog_rows + build_local_catalog_entries.transform_values { |entry| entry.to_h.merge(source: 'local') } + end + def load_bundle(registry_id) # rubocop:disable Metrics/MethodLength definition = Config.entry(registry_id) directory = bundle_directory(definition) @@ -228,11 +237,9 @@ def build_local_catalog_entry(feed_name, feed_config) # rubocop:disable Metrics/ Html2rss::Registry::CatalogEntry.new( id:, path: "/#{id}.rss", - source: 'local', directory: Html2rss::Registry::CatalogBuilder.directory_payload(directory, title), channel: Html2rss::Registry::CatalogBuilder.channel_payload(channel, title), - parameters: { schema: {}, defaults: {} }, - registry: nil + parameters: { schema: {}, defaults: {} } ) end end From 0830b3072b6d62f09a124b85f926c950a30c31a9 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Sun, 23 Aug 2026 21:08:04 +0200 Subject: [PATCH 23/33] chore(deps): pin html2rss Gemfile.lock to registry-v1 tip Keep the web branch lockfile on the latest feat/registry-v1 SHA after the gem phase commits. --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 66718b1ed..ec53b7e16 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/html2rss/html2rss - revision: da2b757d1f8aea4ef7abad65c9a5514c35430483 + revision: 879bcdf1cd220825089dccc648b9b03ecc8d1d60 branch: feat/registry-v1 specs: html2rss (0.28.0) From 9fcb521b74a01b6623b0579e16ac5cf6829e7da6 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Mon, 24 Aug 2026 10:34:39 +0200 Subject: [PATCH 24/33] fix(registry): implement cli_exit_code, atomic store swap, index freshness and openapi types --- app/web/registry/index.rb | 29 +++++++++++++++++++--- app/web/registry/store.rb | 31 ++++++++++++++++++++---- app/web/registry/sync.rb | 16 +++++++++++- docs/registry-go-live.md | 2 +- frontend/src/api/generated/types.gen.ts | 4 +-- public/openapi.yaml | 4 +-- spec/html2rss/web/registry/index_spec.rb | 26 ++++++++++++++++++++ spec/html2rss/web/registry/sync_spec.rb | 28 +++++++++++++++++++++ spec/support/openapi.rb | 10 ++++++++ spec/support/registry_sync.rb | 2 +- 10 files changed, 137 insertions(+), 15 deletions(-) diff --git a/app/web/registry/index.rb b/app/web/registry/index.rb index 08f5075a7..5d6d19dc6 100644 --- a/app/web/registry/index.rb +++ b/app/web/registry/index.rb @@ -44,7 +44,13 @@ class << self ## # @return [Index] def current - @mutex.synchronize { @current ||= new } + @mutex.synchronize do + if @current.nil? || @current.stale? + @current = new + else + @current + end + end end ## @@ -56,6 +62,17 @@ def reload! end end + def initialize + @loaded_bundles = Config.precedence.to_h { [it, load_bundle(it)] }.compact + @manifest_mtimes = current_manifest_mtimes + end + + ## + # @return [Boolean] + def stale? + @manifest_mtimes != current_manifest_mtimes + end + ## # @param feed_id [String, Symbol] # @return [Hash{Symbol => Object}, nil] @@ -128,8 +145,14 @@ def status_entry_for(registry_id) private - def loaded_bundles - @loaded_bundles ||= Config.precedence.to_h { [it, load_bundle(it)] }.compact + attr_reader :loaded_bundles, :manifest_mtimes + + def current_manifest_mtimes + Config.precedence.to_h do |registry_id| + definition = Config.entry(registry_id) + directory = bundle_directory(definition) + [registry_id, Store.manifest_mtime(directory)] + end end def build_registry_catalog_entries diff --git a/app/web/registry/store.rb b/app/web/registry/store.rb index 57cad988f..c0d0b88e8 100644 --- a/app/web/registry/store.rb +++ b/app/web/registry/store.rb @@ -17,7 +17,7 @@ module Store # rubocop:disable Metrics/ModuleLength # Immutable registry sync state. SyncState = Data.define(:last_error, :last_sync_at) - class << self + class << self # rubocop:disable Metrics/ClassLength ## # @return [String] def data_root @@ -93,12 +93,15 @@ def staged_version(registry_id) def swap!(registry_id, source_dir) target = registry_dir(registry_id) FileUtils.mkdir_p(File.dirname(target)) - backup = "#{target}.backup.#{Process.pid}" - FileUtils.mv(target, backup) if File.exist?(target) - FileUtils.cp_r(source_dir, target) + + temp_target = staging_swap_dir(registry_id) + FileUtils.rm_rf(temp_target) + FileUtils.cp_r(source_dir, temp_target) + + replace_directory!(temp_target, target) target ensure - FileUtils.rm_rf(backup) if backup + FileUtils.rm_rf(temp_target) end ## @@ -195,6 +198,24 @@ def manifest_version_at(path) rescue Html2rss::Registry::ManifestError nil end + + def replace_directory!(source, target) + backup = "#{target}.backup.#{Process.pid}" + FileUtils.mv(target, backup) if File.exist?(target) + begin + File.rename(source, target) + rescue StandardError + FileUtils.mv(backup, target) if File.exist?(backup) && !File.exist?(target) + raise + ensure + FileUtils.rm_rf(backup) + end + end + + def staging_swap_dir(registry_id) + stamp = Process.clock_gettime(Process::CLOCK_MONOTONIC_RAW, :nanosecond) + File.join(data_root, ".tmp-swap-#{registry_id}-#{Process.pid}-#{stamp}") + end end end end diff --git a/app/web/registry/sync.rb b/app/web/registry/sync.rb index 816774887..f7892d68b 100644 --- a/app/web/registry/sync.rb +++ b/app/web/registry/sync.rb @@ -60,6 +60,15 @@ def status(registry_id: nil) registry_id ? rows.select { it.id == registry_id.to_s } : rows end + ## + # Returns process exit code for CLI invocations: 0 on success, 1 on failure. + # + # @return [Integer] + def cli_exit_code + unusable = Index.current.status.any? { it.last_error || (it.mode == :sync && it.version.nil?) } + unusable ? 1 : 0 + end + ## # @return [void] def boot! # rubocop:disable Metrics/MethodLength @@ -197,7 +206,12 @@ def seed_registry!(registry_id) end def boot_sync?(registry_id) - ENV.fetch('REGISTRY_SYNC_ON_BOOT', 'false') == 'true' || !Store.bundle_present?(registry_id) + return true if ENV.fetch('REGISTRY_SYNC_ON_BOOT', 'false') == 'true' + + bundle = Index.current.bundle_for(registry_id) + return true unless bundle + + %w[seed test-fixture].include?(bundle.manifest.version) end end end diff --git a/docs/registry-go-live.md b/docs/registry-go-live.md index eebc337a7..05a8ca190 100644 --- a/docs/registry-go-live.md +++ b/docs/registry-go-live.md @@ -4,7 +4,7 @@ Step-by-step guide to ship and operate signed `registry.v1` bundles across the h **Related docs** -- Bundle format: [`html2rss/docs/registry-v1.md`](../../html2rss/docs/registry-v1.md) +- Bundle format: [`html2rss/lib/html2rss/registry/README.md`](../../html2rss/lib/html2rss/registry/README.md) - Operator runbook (sync flags, custom registries): [README.md — Registry sync runbook](./README.md#registry-sync-runbook) --- diff --git a/frontend/src/api/generated/types.gen.ts b/frontend/src/api/generated/types.gen.ts index 395c56397..8a300e216 100644 --- a/frontend/src/api/generated/types.gen.ts +++ b/frontend/src/api/generated/types.gen.ts @@ -34,8 +34,8 @@ export type GetApiMetadataResponses = { registries: Array<{ id: string; sync_mode: string; - updated_at: string; - version: string; + updated_at?: string; + version?: string; }>; }; }; diff --git a/public/openapi.yaml b/public/openapi.yaml index 8e8e54cac..f87837ca3 100644 --- a/public/openapi.yaml +++ b/public/openapi.yaml @@ -73,12 +73,12 @@ paths: type: string updated_at: type: string + nullable: true version: type: string + nullable: true required: - id - - version - - updated_at - sync_mode type: object type: array diff --git a/spec/html2rss/web/registry/index_spec.rb b/spec/html2rss/web/registry/index_spec.rb index af27dc5e0..c70a10955 100644 --- a/spec/html2rss/web/registry/index_spec.rb +++ b/spec/html2rss/web/registry/index_spec.rb @@ -17,6 +17,23 @@ expect(described_class.current.config_for('missing.example/feed')).to be_nil end + it 'resolves feeds by alias' do # rubocop:disable RSpec/ExampleLength + fake_bundle = described_class::RegistryBundle.new( + registry_id: 'official', + manifest: nil, + configs: { + 'anthropic.com/news' => { channel: { title: 'Anthropic — News' } }, + 'anthropic.com/legacy-news' => { channel: { title: 'Anthropic — News' } } + }, + catalog_entries: [] + ) + allow(described_class.current).to receive(:loaded_bundles).and_return('official' => fake_bundle) + + expect(described_class.current.config_for('anthropic.com/legacy-news')).to include( + channel: hash_including(title: 'Anthropic — News') + ) + end + it 'still resolves catalog-disabled registry feeds by id' do config = described_class.current.config_for('secret.example/private') @@ -24,6 +41,15 @@ end end + describe '.current' do + it 'reloads when manifest mtime changes' do + initial_index = described_class.current + allow(Html2rss::Web::Registry::Store).to receive(:manifest_mtime).and_return('2099-01-01T00:00:00Z') + + expect(described_class.current).not_to be(initial_index) + end + end + describe '#catalog_rows' do it 'includes registry rows with source and registry fields' do rows = described_class.current.catalog_rows diff --git a/spec/html2rss/web/registry/sync_spec.rb b/spec/html2rss/web/registry/sync_spec.rb index 06640f1f8..46d298ec0 100644 --- a/spec/html2rss/web/registry/sync_spec.rb +++ b/spec/html2rss/web/registry/sync_spec.rb @@ -20,6 +20,34 @@ end end + describe '.cli_exit_code', :registry_sync do + let(:download_url) { 'https://registry.test.example/registry-bundle.tar.gz' } + let(:tarball) { RegistrySyncTestHelpers.build_signed_tarball } + + before do + stub_request(:get, download_url) + .to_return(status: 200, body: tarball, headers: { 'Content-Type' => 'application/octet-stream' }) + end + + it 'returns 1 when a sync registry is not yet synced' do + expect(described_class.cli_exit_code).to eq(1) + end + + it 'returns 0 when all registries are synced and healthy' do + described_class.run(registry_id: 'official') + expect(described_class.cli_exit_code).to eq(0) + end + + it 'returns 1 when any registry has last_error' do + described_class.run(registry_id: 'official') + allow(Html2rss::Web::Registry::Store).to receive(:sync_state).and_return( + Html2rss::Web::Registry::Store::SyncState.new(last_error: 'Boom', last_sync_at: nil) + ) + + expect(described_class.cli_exit_code).to eq(1) + end + end + describe '.run', :registry_sync do let(:download_url) { 'https://registry.test.example/registry-bundle.tar.gz' } let(:tarball) { RegistrySyncTestHelpers.build_signed_tarball } diff --git a/spec/support/openapi.rb b/spec/support/openapi.rb index 13fa7edf0..766d9159a 100644 --- a/spec/support/openapi.rb +++ b/spec/support/openapi.rb @@ -239,5 +239,15 @@ 'application/json', 'schema', 'properties', 'data', 'properties', 'configs', 'items', 'required') catalog_configs_required&.delete('registry') if catalog_configs_required.is_a?(Array) + + root_registries = paths&.dig('/', 'get', 'responses', '200', 'content', + 'application/json', 'schema', 'properties', 'data', 'properties', 'instance', + 'properties', 'registries', 'items') + if root_registries.is_a?(Hash) + root_registries['properties']&.fetch('version', {})&.merge!('nullable' => true) + root_registries['properties']&.fetch('updated_at', {})&.merge!('nullable' => true) + root_registries['required']&.delete('version') + root_registries['required']&.delete('updated_at') + end end end diff --git a/spec/support/registry_sync.rb b/spec/support/registry_sync.rb index e59629bf3..6d94b1522 100644 --- a/spec/support/registry_sync.rb +++ b/spec/support/registry_sync.rb @@ -48,7 +48,7 @@ def build_signed_tarball # rubocop:disable Metrics/MethodLength manifest_target ) manifest = Html2rss::Registry::Manifest.parse(File.read(manifest_target)) - Html2rss::Registry::TestSupport.sign!(manifest, key_pem: TEST_PRIVATE_KEY, bundle_dir:) + Html2rss::Registry::Signer.sign!(manifest, key_pem: TEST_PRIVATE_KEY, bundle_dir:) pack_bundle_dir(bundle_dir) ensure From f30d1d6b4408ac33f639d4766d5859e76d2f13f6 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Mon, 24 Aug 2026 11:01:26 +0200 Subject: [PATCH 25/33] feat(cli): add unified operator CLI and embed 144 curated seed configs --- AGENTS.md | 2 +- Dockerfile | 5 +- Gemfile.lock | 2 +- .../seed/official/configs/aalto.fi/news.yml | 28 +++ .../configs/academictransfer.com/jobs.yml | 28 +++ .../configs/adfc.de/pressemitteilungen.yml | 24 +++ .../official/configs/ahram.org.eg/egypt.yml | 22 ++ .../official/configs/ai.meta.com/blog.yml | 28 +++ .../official/configs/akorda.kz/events.yml | 23 ++ .../official/configs/angop.ao/noticias.yml | 23 ++ .../seed/official/configs/anu.edu.au/news.yml | 23 ++ .../seed/official/configs/apnews.com/hub.yml | 30 +++ .../official/configs/apple.com/newsroom.yml | 31 +++ .../official/configs/aps.dz/national-news.yml | 23 ++ .../configs/asml.com/press-releases.yml | 23 ++ .../configs/aspi.org.au/strategist.yml | 29 +++ .../configs/bbc.co.uk/available_episodes.yml | 29 +++ .../seed/official/configs/bbc.com/mundo.yml | 25 +++ .../configs/blog.mondediplo.net/feed.yml | 29 +++ .../configs/brookings.edu/articles.yml | 23 ++ .../bundesregierung.de/pressemitteilungen.yml | 28 +++ .../configs/canarianweekly.com/front.yml | 23 ++ .../official/configs/caricom.org/news.yml | 23 ++ .../carnegieendowment.org/research.yml | 29 +++ .../configs/cepal.org/press-releases.yml | 23 ++ .../official/configs/chalmers.se/news.yml | 28 +++ .../research-publications.yml | 29 +++ .../configs/cinemascore.com/index.yml | 55 +++++ .../configs/cleanenergywire.org/news.yml | 28 +++ .../official/configs/cnet.com/section_sub.yml | 30 +++ .../seed/official/configs/cohere.com/blog.yml | 23 ++ .../configs/computerbase.de/meistgelesen.yml | 27 +++ .../official/configs/cplp.org/noticias.yml | 23 ++ .../seed/official/configs/csiro.au/news.yml | 23 ++ .../curia.europa.eu/press-releases.yml | 29 +++ .../official/configs/cutle.fish/index.yml | 25 +++ .../configs/deraktionaer.de/meistgelesen.yml | 22 ++ ...ials_data_documentation_technotes_json.yml | 28 +++ .../configs/dfs.de/pressemitteilungen.yml | 28 +++ .../official/configs/diis.dk/research.yml | 23 ++ .../configs/dirco.gov.za/media-statements.yml | 23 ++ .../official/configs/dsw-info.de/presse.yml | 30 +++ .../elasticsearch-release-notes.yml | 23 ++ .../official/configs/elysee.fr/actualites.yml | 22 ++ .../official/configs/equinor.com/news.yml | 22 ++ .../seed/official/configs/espn.com/f1.yml | 22 ++ .../official/configs/ethz.ch/eth-news.yml | 29 +++ .../official/configs/eures.europa.eu/news.yml | 22 ++ .../official/configs/faa.gov/newsroom.yml | 22 ++ .../official/configs/fao.org/newsroom.yml | 23 ++ .../official/configs/fia.com/documents.yml | 26 +++ .../seed/official/configs/figma.com/blog.yml | 25 +++ .../official/configs/formula1.com/latest.yml | 22 ++ .../configs/forumsec.org/publications.yml | 24 +++ .../configs/ftc.gov/press-releases.yml | 23 ++ .../official/configs/github.com/releases.yml | 33 +++ .../configs/go.dev/release-history.yml | 23 ++ .../official/configs/grafana.com/whatsnew.yml | 23 ++ .../configs/haaretz.com/israel-news.yml | 22 ++ .../configs/hai.stanford.edu/news.yml | 23 ++ .../seed/official/configs/iaapa.org/news.yml | 23 ++ .../configs/icj-cij.org/press-releases.yml | 23 ++ .../seed/official/configs/icrc.org/news.yml | 22 ++ .../seed/official/configs/iea.org/news.yml | 23 ++ .../configs/iiss.org/online-analysis.yml | 29 +++ .../official/configs/ikea.com/newsroom.yml | 23 ++ .../official/configs/imdb.com/ratings.yml | 50 +++++ .../configs/imo.org/press-briefings.yml | 28 +++ .../official/configs/imperial.ac.uk/news.yml | 23 ++ .../karriere_arbeitsleben_heiko_mell.yml | 25 +++ .../official/configs/isro.gov.in/press.yml | 23 ++ .../configs/issafrica.org/iss-today.yml | 23 ++ .../configs/issafrica.org/press-releases.yml | 23 ++ .../official/configs/kaist.ac.kr/news.yml | 23 ++ .../configs/kinocheck.de/filmstarts.yml | 30 +++ .../official/configs/kompas.com/nasional.yml | 23 ++ .../configs/lego.com/press-releases.yml | 23 ++ .../machinelearning.apple.com/research.yml | 23 ++ .../configs/maersk.com/press-releases.yml | 23 ++ .../configs/microsoft.com/azure-products.yml | 25 +++ .../seed/official/configs/mistral.ai/news.yml | 24 +++ .../mozilla.org/security-advisories.yml | 24 +++ .../seed/official/configs/msf.org/latest.yml | 23 ++ .../configs/nbim.no/press-releases.yml | 23 ++ .../seed/official/configs/news.cn/latest.yml | 22 ++ .../configs/news24.com/top-stories.yml | 22 ++ .../configs/newyorker.com/magazine.yml | 25 +++ .../configs/nigerianstat.gov.ng/news.yml | 23 ++ .../configs/nikkei.com/asia-technology.yml | 23 ++ .../official/configs/nokia.com/newsroom.yml | 23 ++ .../official/configs/nomanssky.com/news.yml | 29 +++ .../seed/official/configs/notion.com/blog.yml | 27 +++ .../configs/novonordisk.com/latest-news.yml | 23 ++ .../seed/official/configs/ntu.edu.sg/news.yml | 29 +++ .../seed/official/configs/nupi.no/news.yml | 23 ++ .../seed/official/configs/occrp.org/news.yml | 30 +++ .../official/configs/oecd.org/newsroom.yml | 29 +++ .../configs/opec.org/press-releases.yml | 30 +++ .../official/configs/panapress.com/latest.yml | 28 +++ .../search.yml | 53 +++++ .../official/configs/philstar.com/nation.yml | 22 ++ .../seed/official/configs/phys.org/weekly.yml | 26 +++ .../configs/pib.gov.in/press-releases.yml | 23 ++ .../configs/rbb24.de/meistgeklickt.yml | 22 ++ .../configs/robinwood.de/aktuelles.yml | 23 ++ .../s3.amazonaws.com/popular_movies.yml | 35 +++ .../seed/official/configs/sacu.int/news.yml | 24 +++ .../official/configs/sadc.int/latest-news.yml | 23 ++ .../configs/sebastianvettel.de/news.yml | 24 +++ .../official/configs/shopify.com/blog.yml | 25 +++ .../seed/official/configs/sipri.org/news.yml | 23 ++ .../configs/softwareleadweekly.com/issues.yml | 22 ++ .../configs/solarthermalworld.org/news.yml | 25 +++ .../configs/spektrum.de/meistgelesen.yml | 22 ++ .../configs/spiegel.de/impressum_autor.yml | 39 ++++ .../official/configs/spotify.com/newsroom.yml | 24 +++ .../hot_network_questions.yml | 35 +++ .../official/configs/steuerzahler.de/news.yml | 29 +++ .../official/configs/stripes.com/index.yml | 27 +++ .../support.apple.com/en_gb_ht201222.yml | 37 ++++ .../support.apple.com/exchange_repair.yml | 34 +++ .../official/configs/tap.info.tn/politics.yml | 29 +++ .../official/configs/telenor.com/newsroom.yml | 22 ++ .../configs/teneriffa-news.com/news.yml | 36 ++++ .../seed/official/configs/tennet.eu/news.yml | 22 ++ .../seed/official/configs/test.de/archiv.yml | 25 +++ .../configs/theeastafrican.co.ke/news.yml | 29 +++ .../international_mostpopular.yml | 22 ++ .../configs/thenationalnews.com/news.yml | 22 ++ .../configs/thoughtworks.com/insights.yml | 29 +++ .../seed/official/configs/tno.nl/news.yml | 24 +++ .../aktuelle_nachrichten.yml | 23 ++ .../seed/official/configs/tralac.org/news.yml | 23 ++ .../seed/official/configs/tudelft.nl/news.yml | 23 ++ .../seed/official/configs/tue.nl/news.yml | 23 ++ .../configs/vattenfall.com/newsroom.yml | 23 ++ .../visitfinland.com/press-releases.yml | 27 +++ .../configs/webentwickler-jobs.de/in.yml | 31 +++ .../seed/official/configs/who.int/news.yml | 30 +++ .../official/configs/worldbank.org/news.yml | 29 +++ .../press-releases.yml | 28 +++ .../www.ecdc.europa.eu/news-events.yml | 23 ++ .../configs/www.enisa.europa.eu/news.yml | 23 ++ .../www.europarl.europa.eu/press-room.yml | 28 +++ .../configs/www.gob.mx/profeco-prensa.yml | 24 +++ app/registries/seed/official/manifest.json | 149 ++++++++++++- app/web/boot.rb | 5 +- app/web/cli.rb | 201 ++++++++++++++++++ bin/docker-healthcheck | 20 -- bin/html2rss-web | 11 + bin/prepare-registry-seed | 2 +- bin/registry-sync | 69 ------ docs/README.md | 16 +- docs/registry-go-live.md | 52 +++-- spec/html2rss/web/cli_spec.rb | 105 +++++++++ spec/support/registry_sync.rb | 1 + 156 files changed, 4198 insertions(+), 135 deletions(-) create mode 100644 app/registries/seed/official/configs/aalto.fi/news.yml create mode 100644 app/registries/seed/official/configs/academictransfer.com/jobs.yml create mode 100644 app/registries/seed/official/configs/adfc.de/pressemitteilungen.yml create mode 100644 app/registries/seed/official/configs/ahram.org.eg/egypt.yml create mode 100644 app/registries/seed/official/configs/ai.meta.com/blog.yml create mode 100644 app/registries/seed/official/configs/akorda.kz/events.yml create mode 100644 app/registries/seed/official/configs/angop.ao/noticias.yml create mode 100644 app/registries/seed/official/configs/anu.edu.au/news.yml create mode 100644 app/registries/seed/official/configs/apnews.com/hub.yml create mode 100644 app/registries/seed/official/configs/apple.com/newsroom.yml create mode 100644 app/registries/seed/official/configs/aps.dz/national-news.yml create mode 100644 app/registries/seed/official/configs/asml.com/press-releases.yml create mode 100644 app/registries/seed/official/configs/aspi.org.au/strategist.yml create mode 100644 app/registries/seed/official/configs/bbc.co.uk/available_episodes.yml create mode 100644 app/registries/seed/official/configs/bbc.com/mundo.yml create mode 100644 app/registries/seed/official/configs/blog.mondediplo.net/feed.yml create mode 100644 app/registries/seed/official/configs/brookings.edu/articles.yml create mode 100644 app/registries/seed/official/configs/bundesregierung.de/pressemitteilungen.yml create mode 100644 app/registries/seed/official/configs/canarianweekly.com/front.yml create mode 100644 app/registries/seed/official/configs/caricom.org/news.yml create mode 100644 app/registries/seed/official/configs/carnegieendowment.org/research.yml create mode 100644 app/registries/seed/official/configs/cepal.org/press-releases.yml create mode 100644 app/registries/seed/official/configs/chalmers.se/news.yml create mode 100644 app/registries/seed/official/configs/chathamhouse.org/research-publications.yml create mode 100644 app/registries/seed/official/configs/cinemascore.com/index.yml create mode 100644 app/registries/seed/official/configs/cleanenergywire.org/news.yml create mode 100644 app/registries/seed/official/configs/cnet.com/section_sub.yml create mode 100644 app/registries/seed/official/configs/cohere.com/blog.yml create mode 100644 app/registries/seed/official/configs/computerbase.de/meistgelesen.yml create mode 100644 app/registries/seed/official/configs/cplp.org/noticias.yml create mode 100644 app/registries/seed/official/configs/csiro.au/news.yml create mode 100644 app/registries/seed/official/configs/curia.europa.eu/press-releases.yml create mode 100644 app/registries/seed/official/configs/cutle.fish/index.yml create mode 100644 app/registries/seed/official/configs/deraktionaer.de/meistgelesen.yml create mode 100644 app/registries/seed/official/configs/developer.apple.com/tutorials_data_documentation_technotes_json.yml create mode 100644 app/registries/seed/official/configs/dfs.de/pressemitteilungen.yml create mode 100644 app/registries/seed/official/configs/diis.dk/research.yml create mode 100644 app/registries/seed/official/configs/dirco.gov.za/media-statements.yml create mode 100644 app/registries/seed/official/configs/dsw-info.de/presse.yml create mode 100644 app/registries/seed/official/configs/elastic.co/elasticsearch-release-notes.yml create mode 100644 app/registries/seed/official/configs/elysee.fr/actualites.yml create mode 100644 app/registries/seed/official/configs/equinor.com/news.yml create mode 100644 app/registries/seed/official/configs/espn.com/f1.yml create mode 100644 app/registries/seed/official/configs/ethz.ch/eth-news.yml create mode 100644 app/registries/seed/official/configs/eures.europa.eu/news.yml create mode 100644 app/registries/seed/official/configs/faa.gov/newsroom.yml create mode 100644 app/registries/seed/official/configs/fao.org/newsroom.yml create mode 100644 app/registries/seed/official/configs/fia.com/documents.yml create mode 100644 app/registries/seed/official/configs/figma.com/blog.yml create mode 100644 app/registries/seed/official/configs/formula1.com/latest.yml create mode 100644 app/registries/seed/official/configs/forumsec.org/publications.yml create mode 100644 app/registries/seed/official/configs/ftc.gov/press-releases.yml create mode 100644 app/registries/seed/official/configs/github.com/releases.yml create mode 100644 app/registries/seed/official/configs/go.dev/release-history.yml create mode 100644 app/registries/seed/official/configs/grafana.com/whatsnew.yml create mode 100644 app/registries/seed/official/configs/haaretz.com/israel-news.yml create mode 100644 app/registries/seed/official/configs/hai.stanford.edu/news.yml create mode 100644 app/registries/seed/official/configs/iaapa.org/news.yml create mode 100644 app/registries/seed/official/configs/icj-cij.org/press-releases.yml create mode 100644 app/registries/seed/official/configs/icrc.org/news.yml create mode 100644 app/registries/seed/official/configs/iea.org/news.yml create mode 100644 app/registries/seed/official/configs/iiss.org/online-analysis.yml create mode 100644 app/registries/seed/official/configs/ikea.com/newsroom.yml create mode 100644 app/registries/seed/official/configs/imdb.com/ratings.yml create mode 100644 app/registries/seed/official/configs/imo.org/press-briefings.yml create mode 100644 app/registries/seed/official/configs/imperial.ac.uk/news.yml create mode 100644 app/registries/seed/official/configs/ingenieur.de/karriere_arbeitsleben_heiko_mell.yml create mode 100644 app/registries/seed/official/configs/isro.gov.in/press.yml create mode 100644 app/registries/seed/official/configs/issafrica.org/iss-today.yml create mode 100644 app/registries/seed/official/configs/issafrica.org/press-releases.yml create mode 100644 app/registries/seed/official/configs/kaist.ac.kr/news.yml create mode 100644 app/registries/seed/official/configs/kinocheck.de/filmstarts.yml create mode 100644 app/registries/seed/official/configs/kompas.com/nasional.yml create mode 100644 app/registries/seed/official/configs/lego.com/press-releases.yml create mode 100644 app/registries/seed/official/configs/machinelearning.apple.com/research.yml create mode 100644 app/registries/seed/official/configs/maersk.com/press-releases.yml create mode 100644 app/registries/seed/official/configs/microsoft.com/azure-products.yml create mode 100644 app/registries/seed/official/configs/mistral.ai/news.yml create mode 100644 app/registries/seed/official/configs/mozilla.org/security-advisories.yml create mode 100644 app/registries/seed/official/configs/msf.org/latest.yml create mode 100644 app/registries/seed/official/configs/nbim.no/press-releases.yml create mode 100644 app/registries/seed/official/configs/news.cn/latest.yml create mode 100644 app/registries/seed/official/configs/news24.com/top-stories.yml create mode 100644 app/registries/seed/official/configs/newyorker.com/magazine.yml create mode 100644 app/registries/seed/official/configs/nigerianstat.gov.ng/news.yml create mode 100644 app/registries/seed/official/configs/nikkei.com/asia-technology.yml create mode 100644 app/registries/seed/official/configs/nokia.com/newsroom.yml create mode 100644 app/registries/seed/official/configs/nomanssky.com/news.yml create mode 100644 app/registries/seed/official/configs/notion.com/blog.yml create mode 100644 app/registries/seed/official/configs/novonordisk.com/latest-news.yml create mode 100644 app/registries/seed/official/configs/ntu.edu.sg/news.yml create mode 100644 app/registries/seed/official/configs/nupi.no/news.yml create mode 100644 app/registries/seed/official/configs/occrp.org/news.yml create mode 100644 app/registries/seed/official/configs/oecd.org/newsroom.yml create mode 100644 app/registries/seed/official/configs/opec.org/press-releases.yml create mode 100644 app/registries/seed/official/configs/panapress.com/latest.yml create mode 100644 app/registries/seed/official/configs/pankow.lebensmittel-kontrollergebnisse.de/search.yml create mode 100644 app/registries/seed/official/configs/philstar.com/nation.yml create mode 100644 app/registries/seed/official/configs/phys.org/weekly.yml create mode 100644 app/registries/seed/official/configs/pib.gov.in/press-releases.yml create mode 100644 app/registries/seed/official/configs/rbb24.de/meistgeklickt.yml create mode 100644 app/registries/seed/official/configs/robinwood.de/aktuelles.yml create mode 100644 app/registries/seed/official/configs/s3.amazonaws.com/popular_movies.yml create mode 100644 app/registries/seed/official/configs/sacu.int/news.yml create mode 100644 app/registries/seed/official/configs/sadc.int/latest-news.yml create mode 100644 app/registries/seed/official/configs/sebastianvettel.de/news.yml create mode 100644 app/registries/seed/official/configs/shopify.com/blog.yml create mode 100644 app/registries/seed/official/configs/sipri.org/news.yml create mode 100644 app/registries/seed/official/configs/softwareleadweekly.com/issues.yml create mode 100644 app/registries/seed/official/configs/solarthermalworld.org/news.yml create mode 100644 app/registries/seed/official/configs/spektrum.de/meistgelesen.yml create mode 100644 app/registries/seed/official/configs/spiegel.de/impressum_autor.yml create mode 100644 app/registries/seed/official/configs/spotify.com/newsroom.yml create mode 100644 app/registries/seed/official/configs/stackoverflow.com/hot_network_questions.yml create mode 100644 app/registries/seed/official/configs/steuerzahler.de/news.yml create mode 100644 app/registries/seed/official/configs/stripes.com/index.yml create mode 100644 app/registries/seed/official/configs/support.apple.com/en_gb_ht201222.yml create mode 100644 app/registries/seed/official/configs/support.apple.com/exchange_repair.yml create mode 100644 app/registries/seed/official/configs/tap.info.tn/politics.yml create mode 100644 app/registries/seed/official/configs/telenor.com/newsroom.yml create mode 100644 app/registries/seed/official/configs/teneriffa-news.com/news.yml create mode 100644 app/registries/seed/official/configs/tennet.eu/news.yml create mode 100644 app/registries/seed/official/configs/test.de/archiv.yml create mode 100644 app/registries/seed/official/configs/theeastafrican.co.ke/news.yml create mode 100644 app/registries/seed/official/configs/theguardian.com/international_mostpopular.yml create mode 100644 app/registries/seed/official/configs/thenationalnews.com/news.yml create mode 100644 app/registries/seed/official/configs/thoughtworks.com/insights.yml create mode 100644 app/registries/seed/official/configs/tno.nl/news.yml create mode 100644 app/registries/seed/official/configs/tourismusnetzwerk-brandenburg.de/aktuelle_nachrichten.yml create mode 100644 app/registries/seed/official/configs/tralac.org/news.yml create mode 100644 app/registries/seed/official/configs/tudelft.nl/news.yml create mode 100644 app/registries/seed/official/configs/tue.nl/news.yml create mode 100644 app/registries/seed/official/configs/vattenfall.com/newsroom.yml create mode 100644 app/registries/seed/official/configs/visitfinland.com/press-releases.yml create mode 100644 app/registries/seed/official/configs/webentwickler-jobs.de/in.yml create mode 100644 app/registries/seed/official/configs/who.int/news.yml create mode 100644 app/registries/seed/official/configs/worldbank.org/news.yml create mode 100644 app/registries/seed/official/configs/www.consilium.europa.eu/press-releases.yml create mode 100644 app/registries/seed/official/configs/www.ecdc.europa.eu/news-events.yml create mode 100644 app/registries/seed/official/configs/www.enisa.europa.eu/news.yml create mode 100644 app/registries/seed/official/configs/www.europarl.europa.eu/press-room.yml create mode 100644 app/registries/seed/official/configs/www.gob.mx/profeco-prensa.yml create mode 100644 app/web/cli.rb delete mode 100755 bin/docker-healthcheck create mode 100755 bin/html2rss-web delete mode 100755 bin/registry-sync create mode 100644 spec/html2rss/web/cli_spec.rb diff --git a/AGENTS.md b/AGENTS.md index 90cd9d06f..5bb870392 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -111,6 +111,6 @@ Public feed-directory metadata from verified registry bundles and local `feeds.y | Root metadata | `GET /api/v1/` exposes `instance.catalog: { enabled, url }` and `instance.registries` sync status | | Contract SSOT | Request specs under `spec/html2rss/web/api/v1_spec.rb` and generated `public/openapi.yaml` | -Registry sync: `bin/registry-sync --status`; boot seed + optional sync via `Registry::Sync.boot!`. See [docs/README.md](docs/README.md#registry-sync-runbook). +Registry sync: `bin/html2rss-web registry status`; boot seed + optional sync via `Registry::Sync.boot!`. See [docs/README.md](docs/README.md#registry-sync-runbook). After handler or envelope changes: `make openapi` and `make ci-ready`. diff --git a/Dockerfile b/Dockerfile index 0d91ae5cf..15aefdfea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,13 +51,14 @@ ARG GIT_SHA ENV PORT=4000 \ RACK_ENV=production \ RUBY_YJIT_ENABLE=1 \ + PATH="/app/bin:$PATH" \ BUILD_TAG=${BUILD_TAG} \ GIT_SHA=${GIT_SHA} EXPOSE $PORT HEALTHCHECK --interval=30m --timeout=60s --start-period=5s \ - CMD ["/app/bin/docker-healthcheck"] + CMD ["/app/bin/html2rss-web", "healthcheck"] ARG USER=html2rss ARG UID=991 @@ -86,7 +87,7 @@ WORKDIR /app USER 991 COPY --from=builder /usr/local/bundle /usr/local/bundle -COPY --chown=$USER:$USER bin/docker-healthcheck ./bin/docker-healthcheck +COPY --chown=$USER:$USER bin ./bin COPY --chown=$USER:$USER Gemfile Gemfile.lock app.rb config.ru ./ COPY --chown=$USER:$USER app ./app COPY --chown=$USER:$USER config ./config diff --git a/Gemfile.lock b/Gemfile.lock index ec53b7e16..c5b41a897 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/html2rss/html2rss - revision: 879bcdf1cd220825089dccc648b9b03ecc8d1d60 + revision: e1fbff732b143274a7a41b5fd150c32ebffc38bb branch: feat/registry-v1 specs: html2rss (0.28.0) diff --git a/app/registries/seed/official/configs/aalto.fi/news.yml b/app/registries/seed/official/configs/aalto.fi/news.yml new file mode 100644 index 000000000..841a424de --- /dev/null +++ b/app/registries/seed/official/configs/aalto.fi/news.yml @@ -0,0 +1,28 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: aalto.fi/news +directory: + topics: + - science + - research + title: "Aalto University — News" + summary: "Research and campus news from Aalto University in Finland." +strategy: botasaurus +channel: + title: "Aalto University — News" + url: https://www.aalto.fi/en/news + language: en + time_zone: Europe/Helsinki + ttl: 360 +request: + botasaurus: + wait_for_selector: 'a[href^="/en/news/"]' + wait_timeout_seconds: 20 +selectors: + items: + selector: 'a.hub-listing__results__item__link[href^="/en/news/"]' + enhance: true + title: + selector: .hub-listing__results__item__heading + url: + extractor: href diff --git a/app/registries/seed/official/configs/academictransfer.com/jobs.yml b/app/registries/seed/official/configs/academictransfer.com/jobs.yml new file mode 100644 index 000000000..60beb2faf --- /dev/null +++ b/app/registries/seed/official/configs/academictransfer.com/jobs.yml @@ -0,0 +1,28 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: academictransfer.com/jobs +directory: + topics: + - jobs + title: "AcademicTransfer — Jobs" + summary: "Academic and research job listings in the Netherlands and beyond." +strategy: botasaurus +channel: + title: "AcademicTransfer — Jobs" + url: https://www.academictransfer.com/en/jobs/ + language: en + time_zone: Europe/Amsterdam + ttl: 360 +request: + botasaurus: + wait_for_selector: 'article a[href*="/en/jobs/"]' + wait_timeout_seconds: 20 +selectors: + items: + selector: article + enhance: true + title: + selector: h3 + url: + selector: 'a[href*="/en/jobs/"]' + extractor: href diff --git a/app/registries/seed/official/configs/adfc.de/pressemitteilungen.yml b/app/registries/seed/official/configs/adfc.de/pressemitteilungen.yml new file mode 100644 index 000000000..27a182621 --- /dev/null +++ b/app/registries/seed/official/configs/adfc.de/pressemitteilungen.yml @@ -0,0 +1,24 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: adfc.de/pressemitteilungen +directory: + topics: + - civic + title: "ADFC — Pressemitteilungen" + summary: "Pressemitteilungen des Allgemeinen Deutschen Fahrrad-Clubs." +channel: + title: "ADFC — Pressemitteilungen" + url: https://www.adfc.de/presse/pressemitteilungen/ + time_zone: Europe/Berlin + ttl: 720 + language: de +selectors: + items: + selector: "#articleteasercontainer-columns .articleteaser" + title: + selector: ".articleteaser-header-container a" + url: + selector: "a:first" + extractor: "href" + description: + selector: ".articleteaser-header-container p" diff --git a/app/registries/seed/official/configs/ahram.org.eg/egypt.yml b/app/registries/seed/official/configs/ahram.org.eg/egypt.yml new file mode 100644 index 000000000..518e156fe --- /dev/null +++ b/app/registries/seed/official/configs/ahram.org.eg/egypt.yml @@ -0,0 +1,22 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: ahram.org.eg/egypt +directory: + topics: + - news + title: "Al-Ahram — Egypt" + summary: "Egypt news coverage from Al-Ahram in English." +channel: + title: "Al-Ahram — Egypt" + url: https://english.ahram.org.eg/Portal/1/Egypt.aspx + language: en + time_zone: Africa/Cairo + ttl: 360 +selectors: + items: + selector: 'h4.title a[href*="/News/"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/ai.meta.com/blog.yml b/app/registries/seed/official/configs/ai.meta.com/blog.yml new file mode 100644 index 000000000..366400672 --- /dev/null +++ b/app/registries/seed/official/configs/ai.meta.com/blog.yml @@ -0,0 +1,28 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: ai.meta.com/blog +directory: + topics: + - tech + - research + title: "Meta AI — Blog" + summary: "Research posts and product updates from Meta AI." +channel: + title: "Meta AI — Blog" + url: https://ai.meta.com/blog/ + language: en + time_zone: UTC + ttl: 360 +selectors: + items: + selector: 'a[href*="/blog/"][aria-label^="Read "]' + enhance: false + title: + extractor: attribute + attribute: aria-label + post_process: + - name: gsub + pattern: "/^Read\\s+/" + replacement: "" + url: + extractor: href diff --git a/app/registries/seed/official/configs/akorda.kz/events.yml b/app/registries/seed/official/configs/akorda.kz/events.yml new file mode 100644 index 000000000..c77ac858a --- /dev/null +++ b/app/registries/seed/official/configs/akorda.kz/events.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: akorda.kz/events +directory: + topics: + - civic + - news + title: "Akorda — Events" + summary: "Official events and statements from the President of Kazakhstan." +channel: + title: "Akorda — Events" + url: https://www.akorda.kz/en/events + language: en + time_zone: Asia/Almaty + ttl: 360 +selectors: + items: + selector: ".card h3 a" + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/angop.ao/noticias.yml b/app/registries/seed/official/configs/angop.ao/noticias.yml new file mode 100644 index 000000000..4eafef1a0 --- /dev/null +++ b/app/registries/seed/official/configs/angop.ao/noticias.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: angop.ao/noticias +directory: + topics: + - news + - civic + title: "ANGOP — Notícias" + summary: "Notícias oficiais da Agência Angola Press." +channel: + title: "ANGOP — Notícias" + url: https://www.angop.ao/ + language: pt + time_zone: Africa/Luanda + ttl: 360 +selectors: + items: + selector: 'h2 a[href*="/noticias/"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/anu.edu.au/news.yml b/app/registries/seed/official/configs/anu.edu.au/news.yml new file mode 100644 index 000000000..2e4f5aa47 --- /dev/null +++ b/app/registries/seed/official/configs/anu.edu.au/news.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: anu.edu.au/news +directory: + topics: + - science + - research + title: "ANU — News" + summary: "University news and research highlights from the Australian National University." +channel: + title: "ANU — News" + url: https://www.anu.edu.au/news/all-news + language: en + time_zone: Australia/Sydney + ttl: 360 +selectors: + items: + selector: 'h3 a[href*="/news/all-news/"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/apnews.com/hub.yml b/app/registries/seed/official/configs/apnews.com/hub.yml new file mode 100644 index 000000000..69a22acde --- /dev/null +++ b/app/registries/seed/official/configs/apnews.com/hub.yml @@ -0,0 +1,30 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +--- +registry: + id: apnews.com/hub +directory: + topics: + - news + title: "AP News — Hub" + summary: "Top stories from a selectable Associated Press news hub." +parameters: + section: + type: string + default: "news" + +channel: + title: "AP News — Hub" + url: https://apnews.com/%
s + language: en + ttl: 120 + time_zone: UTC +selectors: + items: + selector: ".PagePromo" + title: + selector: h3 + url: + selector: a:first + extractor: href + description: + selector: .PagePromoContentIcons-text diff --git a/app/registries/seed/official/configs/apple.com/newsroom.yml b/app/registries/seed/official/configs/apple.com/newsroom.yml new file mode 100644 index 000000000..7b301e61c --- /dev/null +++ b/app/registries/seed/official/configs/apple.com/newsroom.yml @@ -0,0 +1,31 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: apple.com/newsroom +directory: + topics: + - tech + - product + title: "Apple — Newsroom" + summary: "Product announcements and press releases from Apple." +strategy: botasaurus + +channel: + title: "Apple — Newsroom" + url: https://www.apple.com/newsroom/ + language: en + time_zone: UTC + ttl: 360 +request: + botasaurus: + wait_for_selector: "li.tile-item" + wait_timeout_seconds: 20 +selectors: + items: + selector: "li.tile-item a.tile" + enhance: false + title: + selector: .tile__headline + url: + extractor: href + published_at: + selector: .tile__timestamp diff --git a/app/registries/seed/official/configs/aps.dz/national-news.yml b/app/registries/seed/official/configs/aps.dz/national-news.yml new file mode 100644 index 000000000..7a0bfa88b --- /dev/null +++ b/app/registries/seed/official/configs/aps.dz/national-news.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: aps.dz/national-news +directory: + topics: + - news + - civic + title: "APS — National news" + summary: "National news from Algeria’s official press agency (APS)." +channel: + title: "APS — National news" + url: https://www.aps.dz/en + language: en + time_zone: Africa/Algiers + ttl: 360 +selectors: + items: + selector: 'a[href*="/en/algeria/national-news/"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/asml.com/press-releases.yml b/app/registries/seed/official/configs/asml.com/press-releases.yml new file mode 100644 index 000000000..5550871e0 --- /dev/null +++ b/app/registries/seed/official/configs/asml.com/press-releases.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: asml.com/press-releases +directory: + topics: + - tech + - product + title: "ASML — Press releases" + summary: "Corporate press releases from semiconductor equipment maker ASML." +channel: + title: "ASML — Press releases" + url: https://www.asml.com/en/news/press-releases + language: en + time_zone: Europe/Amsterdam + ttl: 360 +selectors: + items: + selector: 'a.related-content-card[href*="/en/news/press-releases/20"]' + enhance: false + title: + selector: span.related-content-card-title + url: + extractor: href diff --git a/app/registries/seed/official/configs/aspi.org.au/strategist.yml b/app/registries/seed/official/configs/aspi.org.au/strategist.yml new file mode 100644 index 000000000..c4954622c --- /dev/null +++ b/app/registries/seed/official/configs/aspi.org.au/strategist.yml @@ -0,0 +1,29 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: aspi.org.au/strategist +directory: + topics: + - civic + - research + title: "ASPI — The Strategist" + summary: "Strategic analysis and commentary from the Australian Strategic Policy Institute." +strategy: botasaurus + +channel: + title: "ASPI — The Strategist" + url: https://www.aspi.org.au/strategist-posts/ + language: en + time_zone: Australia/Sydney + ttl: 360 +request: + botasaurus: + wait_for_selector: 'a[rel="bookmark"][href*="/strategist-posts/"]' + wait_timeout_seconds: 20 +selectors: + items: + selector: 'a[rel="bookmark"][href*="/strategist-posts/"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/bbc.co.uk/available_episodes.yml b/app/registries/seed/official/configs/bbc.co.uk/available_episodes.yml new file mode 100644 index 000000000..d0154422e --- /dev/null +++ b/app/registries/seed/official/configs/bbc.co.uk/available_episodes.yml @@ -0,0 +1,29 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: bbc.co.uk/available_episodes +directory: + topics: + - entertainment + title: "BBC Sounds — Programme episodes" + summary: "Available episodes for a BBC programme on Sounds." +parameters: + id: + type: string + default: "b006wkfp" + +channel: + language: en + title: "BBC Sounds — Programme episodes" + url: https://www.bbc.co.uk/programmes/%s/episodes/player + time_zone: UTC + ttl: 720 +selectors: + items: + selector: ".highlight-box-wrapper > div" + title: + selector: ".programme__title" + url: + selector: ".programme__titles > a" + extractor: "href" + description: + selector: ".programme__synopsis" diff --git a/app/registries/seed/official/configs/bbc.com/mundo.yml b/app/registries/seed/official/configs/bbc.com/mundo.yml new file mode 100644 index 000000000..3cde3ea9a --- /dev/null +++ b/app/registries/seed/official/configs/bbc.com/mundo.yml @@ -0,0 +1,25 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +--- +registry: + id: bbc.com/mundo +directory: + topics: + - news + title: "BBC — Mundo" + summary: "Noticias en español de BBC Mundo." +channel: + title: "BBC — Mundo" + url: https://www.bbc.com/mundo + language: es + ttl: 360 + time_zone: UTC +selectors: + items: + selector: '[data-testid="topic-promos"] > li' + title: + selector: a + url: + selector: a:first + extractor: href + description: + selector: p diff --git a/app/registries/seed/official/configs/blog.mondediplo.net/feed.yml b/app/registries/seed/official/configs/blog.mondediplo.net/feed.yml new file mode 100644 index 000000000..7d723b9af --- /dev/null +++ b/app/registries/seed/official/configs/blog.mondediplo.net/feed.yml @@ -0,0 +1,29 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: blog.mondediplo.net/feed +directory: + topics: + - news + title: "Le Monde diplomatique — Blog feed" + summary: "Blog posts from Le Monde diplomatique with customizable blog path." +parameters: + blog: + type: string + default: "-Defense-en-ligne-" + +channel: + language: en + url: https://blog.mondediplo.net/%s + time_zone: Europe/Paris + title: "Le Monde diplomatique — Blog feed" + ttl: 120 +selectors: + items: + selector: ".large > ul > li" + title: + selector: "h3" + description: + selector: ".intro" + url: + selector: "a" + extractor: "href" diff --git a/app/registries/seed/official/configs/brookings.edu/articles.yml b/app/registries/seed/official/configs/brookings.edu/articles.yml new file mode 100644 index 000000000..5dd51ef80 --- /dev/null +++ b/app/registries/seed/official/configs/brookings.edu/articles.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: brookings.edu/articles +directory: + topics: + - civic + - research + title: "Brookings — Articles" + summary: "Policy research and commentary from the Brookings Institution." +channel: + title: "Brookings — Articles" + url: https://www.brookings.edu/ + language: en + time_zone: America/New_York + ttl: 360 +selectors: + items: + selector: 'a.overlay-link[href*="/articles/"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/bundesregierung.de/pressemitteilungen.yml b/app/registries/seed/official/configs/bundesregierung.de/pressemitteilungen.yml new file mode 100644 index 000000000..b7673581f --- /dev/null +++ b/app/registries/seed/official/configs/bundesregierung.de/pressemitteilungen.yml @@ -0,0 +1,28 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: bundesregierung.de/pressemitteilungen +directory: + topics: + - civic + title: "Bundesregierung — Pressemitteilungen" + summary: "Offizielle Pressemitteilungen der deutschen Bundesregierung." +strategy: botasaurus + +channel: + title: "Bundesregierung — Pressemitteilungen" + url: https://www.bundesregierung.de/breg-de/aktuelles/pressemitteilungen + language: de + time_zone: Europe/Berlin + ttl: 360 +request: + botasaurus: + wait_for_selector: 'a.bpa-link[href*="/aktuelles/pressemitteilungen/"]' + wait_timeout_seconds: 20 +selectors: + items: + selector: 'a.bpa-link[href*="/breg-de/aktuelles/pressemitteilungen/"]' + enhance: false + title: + selector: span.bpa-teaser-title-text-inner + url: + extractor: href diff --git a/app/registries/seed/official/configs/canarianweekly.com/front.yml b/app/registries/seed/official/configs/canarianweekly.com/front.yml new file mode 100644 index 000000000..395ed5413 --- /dev/null +++ b/app/registries/seed/official/configs/canarianweekly.com/front.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: canarianweekly.com/front +directory: + topics: + - news + - travel + title: "Canarian Weekly — Front page" + summary: "English-language news from the Canary Islands." +channel: + title: "Canarian Weekly — Front page" + url: https://www.canarianweekly.com/ + time_zone: Europe/London + ttl: 720 + language: en +selectors: + items: + selector: "h5.post-title a" + enhance: false + title: + extractor: text + url: + extractor: "href" diff --git a/app/registries/seed/official/configs/caricom.org/news.yml b/app/registries/seed/official/configs/caricom.org/news.yml new file mode 100644 index 000000000..f7dead8ae --- /dev/null +++ b/app/registries/seed/official/configs/caricom.org/news.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: caricom.org/news +directory: + topics: + - civic + - news + title: "CARICOM — News" + summary: "News and updates from the Caribbean Community (CARICOM)." +channel: + title: "CARICOM — News" + url: https://caricom.org/ + language: en + time_zone: America/Jamaica + ttl: 360 +selectors: + items: + selector: a.eael-grid-post-link + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/carnegieendowment.org/research.yml b/app/registries/seed/official/configs/carnegieendowment.org/research.yml new file mode 100644 index 000000000..9fc241ff0 --- /dev/null +++ b/app/registries/seed/official/configs/carnegieendowment.org/research.yml @@ -0,0 +1,29 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: carnegieendowment.org/research +directory: + topics: + - civic + - research + title: "Carnegie Endowment — Research" + summary: "Foreign policy research and analysis from the Carnegie Endowment." +strategy: botasaurus + +channel: + title: "Carnegie Endowment — Research" + url: https://carnegieendowment.org/research + language: en + time_zone: UTC + ttl: 360 +request: + botasaurus: + wait_for_selector: 'a.min-w-0.flex-1[href^="/research/20"]' + wait_timeout_seconds: 20 +selectors: + items: + selector: 'a.min-w-0.flex-1[href^="/research/20"]' + enhance: false + title: + selector: span.text-headlineH5 + url: + extractor: href diff --git a/app/registries/seed/official/configs/cepal.org/press-releases.yml b/app/registries/seed/official/configs/cepal.org/press-releases.yml new file mode 100644 index 000000000..918ff530a --- /dev/null +++ b/app/registries/seed/official/configs/cepal.org/press-releases.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: cepal.org/press-releases +directory: + topics: + - civic + - research + title: "ECLAC — Press releases" + summary: "Press releases from the UN Economic Commission for Latin America and the Caribbean." +channel: + title: "ECLAC — Press releases" + url: https://www.cepal.org/en/list/cepal_pr + language: en + time_zone: America/Santiago + ttl: 360 +selectors: + items: + selector: 'h4 a[href*="/en/pressreleases/"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/chalmers.se/news.yml b/app/registries/seed/official/configs/chalmers.se/news.yml new file mode 100644 index 000000000..224010b89 --- /dev/null +++ b/app/registries/seed/official/configs/chalmers.se/news.yml @@ -0,0 +1,28 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: chalmers.se/news +directory: + topics: + - science + - research + title: "Chalmers — News" + summary: "Research and university news from Chalmers University of Technology." +strategy: botasaurus +channel: + title: "Chalmers — News" + url: https://www.chalmers.se/en/current/news/ + language: en + time_zone: Europe/Stockholm + ttl: 360 +request: + botasaurus: + wait_for_selector: 'a[href*="/en/current/news/"]' + wait_timeout_seconds: 20 +selectors: + items: + selector: 'a[data-component="NewsListItem"]' + enhance: true + title: + selector: p.line-clamp-2 + url: + extractor: href diff --git a/app/registries/seed/official/configs/chathamhouse.org/research-publications.yml b/app/registries/seed/official/configs/chathamhouse.org/research-publications.yml new file mode 100644 index 000000000..2f4f1fcfc --- /dev/null +++ b/app/registries/seed/official/configs/chathamhouse.org/research-publications.yml @@ -0,0 +1,29 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: chathamhouse.org/research-publications +directory: + topics: + - civic + - research + title: "Chatham House — Research publications" + summary: "Research publications from the Royal Institute of International Affairs." +strategy: botasaurus + +channel: + title: "Chatham House — Research publications" + url: https://www.chathamhouse.org/publications/research-publications + language: en + time_zone: Europe/London + ttl: 360 +request: + botasaurus: + wait_for_selector: a.teaser__link + wait_timeout_seconds: 20 +selectors: + items: + selector: a.teaser__link + enhance: false + title: + selector: .teaser__title + url: + extractor: href diff --git a/app/registries/seed/official/configs/cinemascore.com/index.yml b/app/registries/seed/official/configs/cinemascore.com/index.yml new file mode 100644 index 000000000..1e0113024 --- /dev/null +++ b/app/registries/seed/official/configs/cinemascore.com/index.yml @@ -0,0 +1,55 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: cinemascore.com/index +directory: + topics: + - entertainment + title: "CinemaScore — Audience grades" + summary: "Opening-night audience grades for new theatrical releases." +channel: + language: en + title: "CinemaScore — Audience grades" + url: https://webapp.cinemascore.com/guest/surveys + ttl: 720 + json: true + time_zone: America/Los_Angeles +selectors: + items: + selector: "new > array > object, recent > array > object" + title: + selector: "title" + post_process: + - name: "template" + string: "%{self} (%{grade}) [Rated %{mpaa_rating}] (%{year})" + grade: + selector: "grade" + movie_title: + selector: "movie_title" + url: + post_process: + - name: "template" + string: "https://www.metacritic.com/search/movie/%{movie_title}/results" + mpaa_rating: + selector: "mpaa_rating" + distributor: + selector: "distributor" + year: + selector: "year" + categories: + - mpaa_rating + - grade + - year + description: + post_process: + - name: template + string: | + # %{movie_title} + + Grade: %{grade} + + MPAA Rating: %{mpaa_rating} + + Distributor: %{distributor} + + Year: %{year} + - name: markdown_to_html diff --git a/app/registries/seed/official/configs/cleanenergywire.org/news.yml b/app/registries/seed/official/configs/cleanenergywire.org/news.yml new file mode 100644 index 000000000..18fb1956d --- /dev/null +++ b/app/registries/seed/official/configs/cleanenergywire.org/news.yml @@ -0,0 +1,28 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: cleanenergywire.org/news +directory: + topics: + - energy + - environment + title: "Clean Energy Wire — News" + summary: "Journalism on Germany’s energy transition and climate policy." +channel: + language: en + title: "Clean Energy Wire — News" + url: https://www.cleanenergywire.org/news + time_zone: "Europe/Berlin" + ttl: 360 +selectors: + items: + selector: ".views-row" + enhance: false + title: + selector: "h2 a" + url: + selector: "h2 a" + extractor: "href" + published_at: + selector: "time" + post_process: + - name: parse_time diff --git a/app/registries/seed/official/configs/cnet.com/section_sub.yml b/app/registries/seed/official/configs/cnet.com/section_sub.yml new file mode 100644 index 000000000..f81848134 --- /dev/null +++ b/app/registries/seed/official/configs/cnet.com/section_sub.yml @@ -0,0 +1,30 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +--- +registry: + id: cnet.com/section_sub +directory: + topics: + - tech + - news + title: "CNET — Section" + summary: "Tech news and reviews from a selectable CNET section." +parameters: + section: + type: string + default: "tech" + +channel: + title: "CNET — Section" + url: https://www.cnet.com/%
s/ + language: en + ttl: 360 + time_zone: UTC +selectors: + items: + selector: "article.list-entry" + enhance: false + title: + selector: "h3.list-entry__title a" + url: + selector: "h3.list-entry__title a" + extractor: href diff --git a/app/registries/seed/official/configs/cohere.com/blog.yml b/app/registries/seed/official/configs/cohere.com/blog.yml new file mode 100644 index 000000000..8e206d84f --- /dev/null +++ b/app/registries/seed/official/configs/cohere.com/blog.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: cohere.com/blog +directory: + topics: + - tech + - research + title: "Cohere — Blog" + summary: "Product and research updates from Cohere." +channel: + title: "Cohere — Blog" + url: https://cohere.com/blog + language: en + time_zone: UTC + ttl: 360 +selectors: + items: + selector: 'a[href^="/blog/"][class*="flex-1"]' + enhance: false + title: + selector: "> p:first-of-type" + url: + extractor: href diff --git a/app/registries/seed/official/configs/computerbase.de/meistgelesen.yml b/app/registries/seed/official/configs/computerbase.de/meistgelesen.yml new file mode 100644 index 000000000..7af302e38 --- /dev/null +++ b/app/registries/seed/official/configs/computerbase.de/meistgelesen.yml @@ -0,0 +1,27 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: computerbase.de/meistgelesen +directory: + topics: + - tech + title: "ComputerBase — Meistgelesen" + summary: "Most-read tech articles from ComputerBase." +channel: + title: "ComputerBase — Meistgelesen" + url: https://www.computerbase.de + time_zone: Europe/Berlin + ttl: 360 + language: de +selectors: + items: + selector: ".charts > li" + title: + selector: ".charts__title" + post_process: + - name: "template" + string: "%{context}: %{self}" + context: + selector: ".charts__context" + url: + selector: ".charts__link" + extractor: "href" diff --git a/app/registries/seed/official/configs/cplp.org/noticias.yml b/app/registries/seed/official/configs/cplp.org/noticias.yml new file mode 100644 index 000000000..bc5d577a9 --- /dev/null +++ b/app/registries/seed/official/configs/cplp.org/noticias.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: cplp.org/noticias +directory: + topics: + - civic + - news + title: "CPLP — Notícias" + summary: "Notícias da Comunidade dos Países de Língua Portuguesa." +channel: + title: "CPLP — Notícias" + url: https://www.cplp.org/ + language: pt + time_zone: Europe/Lisbon + ttl: 360 +selectors: + items: + selector: 'ul.news-to-show-list a[href*="/noticias/noticias-detalhe/"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/csiro.au/news.yml b/app/registries/seed/official/configs/csiro.au/news.yml new file mode 100644 index 000000000..30bdb3684 --- /dev/null +++ b/app/registries/seed/official/configs/csiro.au/news.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: csiro.au/news +directory: + topics: + - science + - research + title: "CSIRO — News" + summary: "Science and innovation news from Australia’s national science agency." +channel: + title: "CSIRO — News" + url: https://www.csiro.au/en/news + language: en + time_zone: Australia/Sydney + ttl: 360 +selectors: + items: + selector: 'a[href*="/en/news/All/Articles/"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/curia.europa.eu/press-releases.yml b/app/registries/seed/official/configs/curia.europa.eu/press-releases.yml new file mode 100644 index 000000000..612e548f6 --- /dev/null +++ b/app/registries/seed/official/configs/curia.europa.eu/press-releases.yml @@ -0,0 +1,29 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: curia.europa.eu/press-releases +directory: + topics: + - civic + title: "Court of Justice of the EU — Press releases" + summary: "Press releases from the Court of Justice of the European Union." +strategy: botasaurus + +channel: + title: "Court of Justice of the EU — Press releases" + url: https://curia.europa.eu/jcms/jcms/Jo2_7052/ + language: en + time_zone: Europe/Brussels + ttl: 360 +request: + botasaurus: + wait_for_selector: li.curia-itemlist-item + wait_timeout_seconds: 20 +selectors: + items: + selector: li.curia-itemlist-item + enhance: false + title: + selector: .curia-itemlist-item-title h3 + url: + selector: .curia-itemlist-item-title a + extractor: href diff --git a/app/registries/seed/official/configs/cutle.fish/index.yml b/app/registries/seed/official/configs/cutle.fish/index.yml new file mode 100644 index 000000000..4224b7184 --- /dev/null +++ b/app/registries/seed/official/configs/cutle.fish/index.yml @@ -0,0 +1,25 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +--- +registry: + id: cutle.fish/index +directory: + topics: + - tech + title: "cutle.fish — Posts" + summary: "Personal writing and notes from cutle.fish." +channel: + language: en + title: "cutle.fish — Posts" + url: https://cutle.fish/ + ttl: 360 + time_zone: UTC +selectors: + items: + selector: main > div > div + title: + selector: a + url: + selector: a + extractor: href + description: + selector: a diff --git a/app/registries/seed/official/configs/deraktionaer.de/meistgelesen.yml b/app/registries/seed/official/configs/deraktionaer.de/meistgelesen.yml new file mode 100644 index 000000000..14c093739 --- /dev/null +++ b/app/registries/seed/official/configs/deraktionaer.de/meistgelesen.yml @@ -0,0 +1,22 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: deraktionaer.de/meistgelesen +directory: + topics: + - finance + title: "Der Aktionär — Meistgelesen" + summary: "Most-read stock market news from Der Aktionär." +channel: + title: "Der Aktionär — Meistgelesen" + url: https://www.deraktionaer.de/ + time_zone: Europe/Berlin + ttl: 360 + language: de +enhance: false +selectors: + items: + selector: "section#top-articles article.top-article a.top-article-content[href^='/artikel/']" + title: + extractor: "text" + url: + extractor: "href" diff --git a/app/registries/seed/official/configs/developer.apple.com/tutorials_data_documentation_technotes_json.yml b/app/registries/seed/official/configs/developer.apple.com/tutorials_data_documentation_technotes_json.yml new file mode 100644 index 000000000..4026f9ab6 --- /dev/null +++ b/app/registries/seed/official/configs/developer.apple.com/tutorials_data_documentation_technotes_json.yml @@ -0,0 +1,28 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +--- +registry: + id: developer.apple.com/tutorials_data_documentation_technotes_json +directory: + topics: + - tech + - product + title: "Apple Developer — Technotes" + summary: "Apple Developer Technotes documentation feed (JSON source)." +channel: + language: en + json: true + title: "Apple Developer — Technotes" + url: https://developer.apple.com/tutorials/data/documentation/Technotes.json + ttl: 360 + time_zone: UTC +selectors: + items: + selector: "references > object > * > object:has(url)" + title: + selector: title + url: + selector: url + post_process: + - name: parse_uri + description: + selector: abstract text diff --git a/app/registries/seed/official/configs/dfs.de/pressemitteilungen.yml b/app/registries/seed/official/configs/dfs.de/pressemitteilungen.yml new file mode 100644 index 000000000..00dca1f84 --- /dev/null +++ b/app/registries/seed/official/configs/dfs.de/pressemitteilungen.yml @@ -0,0 +1,28 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: dfs.de/pressemitteilungen +directory: + topics: + - civic + title: "DFS — Pressemitteilungen" + summary: "Pressemitteilungen der Deutschen Flugsicherung (DFS)." +channel: + title: "DFS — Pressemitteilungen" + url: https://www.dfs.de/homepage/de/medien/presse/ + time_zone: Europe/Berlin + ttl: 1440 + language: de +selectors: + items: + selector: ".article-teaser-list article" + title: + selector: ".headline" + url: + selector: "a" + extractor: "href" + published_at: + selector: "time" + extractor: "attribute" + attribute: "datetime" + post_process: + - name: "parse_time" diff --git a/app/registries/seed/official/configs/diis.dk/research.yml b/app/registries/seed/official/configs/diis.dk/research.yml new file mode 100644 index 000000000..2ba60c1c8 --- /dev/null +++ b/app/registries/seed/official/configs/diis.dk/research.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: diis.dk/research +directory: + topics: + - civic + - research + title: "DIIS — Research" + summary: "Research and analysis from the Danish Institute for International Studies." +channel: + title: "DIIS — Research" + url: https://www.diis.dk/en + language: en + time_zone: Europe/Copenhagen + ttl: 360 +selectors: + items: + selector: 'h3 a[href^="/en/research/"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/dirco.gov.za/media-statements.yml b/app/registries/seed/official/configs/dirco.gov.za/media-statements.yml new file mode 100644 index 000000000..bb999f73d --- /dev/null +++ b/app/registries/seed/official/configs/dirco.gov.za/media-statements.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: dirco.gov.za/media-statements +directory: + topics: + - civic + - news + title: "DIRCO — Media statements" + summary: "Media statements from South Africa’s Department of International Relations." +channel: + title: "DIRCO — Media statements" + url: https://dirco.gov.za/media-statements/ + language: en + time_zone: Africa/Johannesburg + ttl: 360 +selectors: + items: + selector: "div.pgafu-post-grid-content h2 a" + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/dsw-info.de/presse.yml b/app/registries/seed/official/configs/dsw-info.de/presse.yml new file mode 100644 index 000000000..69f870c9f --- /dev/null +++ b/app/registries/seed/official/configs/dsw-info.de/presse.yml @@ -0,0 +1,30 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: dsw-info.de/presse +directory: + topics: + - civic + - consumer + title: "Deutsche Stiftung Weltbevölkerung — Presse" + summary: "Pressemitteilungen der Deutschen Stiftung Weltbevölkerung." +channel: + title: "Deutsche Stiftung Weltbevölkerung — Presse" + url: https://www.dsw-info.de/presse/pressemitteilungen-2026/ + time_zone: Europe/Berlin + ttl: 720 + language: de +selectors: + items: + selector: "main .frame.cet-menu ul > li" + enhance: false + title: + selector: "a" + url: + selector: "a" + extractor: "href" + published_at: + selector: ".vd small" + post_process: + - name: parse_time + description: + selector: ".bt small" diff --git a/app/registries/seed/official/configs/elastic.co/elasticsearch-release-notes.yml b/app/registries/seed/official/configs/elastic.co/elasticsearch-release-notes.yml new file mode 100644 index 000000000..e73169f53 --- /dev/null +++ b/app/registries/seed/official/configs/elastic.co/elasticsearch-release-notes.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: elastic.co/elasticsearch-release-notes +directory: + topics: + - tech + - product + title: "Elastic — Elasticsearch release notes" + summary: "Release notes for Elasticsearch from Elastic." +channel: + title: "Elastic — Elasticsearch release notes" + url: https://www.elastic.co/docs/release-notes/elasticsearch + language: en + time_zone: UTC + ttl: 360 +selectors: + items: + selector: 'a[href^="#elasticsearch-"][href$="-release-notes"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/elysee.fr/actualites.yml b/app/registries/seed/official/configs/elysee.fr/actualites.yml new file mode 100644 index 000000000..11380d0a0 --- /dev/null +++ b/app/registries/seed/official/configs/elysee.fr/actualites.yml @@ -0,0 +1,22 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: elysee.fr/actualites +directory: + topics: + - civic + title: "Élysée — Actualités" + summary: "Actualités officielles de la Présidence de la République française." +channel: + title: "Élysée — Actualités" + url: https://www.elysee.fr/toutes-les-actualites + language: fr + time_zone: Europe/Paris + ttl: 360 +selectors: + items: + selector: 'a[href*="/emmanuel-macron/20"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/equinor.com/news.yml b/app/registries/seed/official/configs/equinor.com/news.yml new file mode 100644 index 000000000..76b365e4c --- /dev/null +++ b/app/registries/seed/official/configs/equinor.com/news.yml @@ -0,0 +1,22 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: equinor.com/news +directory: + topics: + - energy + title: "Equinor — News" + summary: "Corporate and energy news from Equinor." +channel: + title: "Equinor — News" + url: https://www.equinor.com/news + language: en + time_zone: Europe/Oslo + ttl: 360 +selectors: + items: + selector: 'a[href^="/news/"]' + enhance: true + title: + selector: h2 + url: + extractor: href diff --git a/app/registries/seed/official/configs/espn.com/f1.yml b/app/registries/seed/official/configs/espn.com/f1.yml new file mode 100644 index 000000000..88f38c62d --- /dev/null +++ b/app/registries/seed/official/configs/espn.com/f1.yml @@ -0,0 +1,22 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: espn.com/f1 +directory: + topics: + - sports + title: "ESPN — Formula 1" + summary: "Formula 1 news and coverage from ESPN." +channel: + language: en + title: "ESPN — Formula 1" + url: https://www.espn.com/f1/ + time_zone: UTC + ttl: 60 +selectors: + items: + selector: ".headlineStack__list > li" + title: + selector: "a" + url: + selector: "a" + extractor: "href" diff --git a/app/registries/seed/official/configs/ethz.ch/eth-news.yml b/app/registries/seed/official/configs/ethz.ch/eth-news.yml new file mode 100644 index 000000000..2093c1165 --- /dev/null +++ b/app/registries/seed/official/configs/ethz.ch/eth-news.yml @@ -0,0 +1,29 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: ethz.ch/eth-news +directory: + topics: + - science + - research + title: "ETH Zurich — News" + summary: "Research and university news from ETH Zurich." +strategy: botasaurus + +channel: + title: "ETH Zurich — News" + url: https://ethz.ch/en/news-and-events/eth-news.html + language: en + time_zone: Europe/Zurich + ttl: 360 +request: + botasaurus: + wait_for_selector: 'a[href*="/eth-news/news/20"]' + wait_timeout_seconds: 20 +selectors: + items: + selector: 'a[href*="/eth-news/news/20"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/eures.europa.eu/news.yml b/app/registries/seed/official/configs/eures.europa.eu/news.yml new file mode 100644 index 000000000..e8f877f91 --- /dev/null +++ b/app/registries/seed/official/configs/eures.europa.eu/news.yml @@ -0,0 +1,22 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: eures.europa.eu/news +directory: + topics: + - jobs + title: "EURES — News" + summary: "Job mobility news from the European Employment Services network." +channel: + title: "EURES — News" + url: https://eures.europa.eu/news_en + language: en + time_zone: Europe/Brussels + ttl: 360 +selectors: + items: + selector: 'article.ecl-content-item a[href$="_en"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/faa.gov/newsroom.yml b/app/registries/seed/official/configs/faa.gov/newsroom.yml new file mode 100644 index 000000000..faf427472 --- /dev/null +++ b/app/registries/seed/official/configs/faa.gov/newsroom.yml @@ -0,0 +1,22 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: faa.gov/newsroom +directory: + topics: + - civic + title: "FAA — Newsroom" + summary: "News and announcements from the U.S. Federal Aviation Administration." +channel: + title: "FAA — Newsroom" + url: https://www.faa.gov/newsroom + language: en + time_zone: UTC + ttl: 360 +selectors: + items: + selector: '.views-row a[href^="/newsroom/"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/fao.org/newsroom.yml b/app/registries/seed/official/configs/fao.org/newsroom.yml new file mode 100644 index 000000000..22b58ea1f --- /dev/null +++ b/app/registries/seed/official/configs/fao.org/newsroom.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: fao.org/newsroom +directory: + topics: + - environment + - civic + title: "FAO — Newsroom" + summary: "News from the Food and Agriculture Organization of the United Nations." +channel: + title: "FAO — Newsroom" + url: https://www.fao.org/newsroom/en + language: en + time_zone: UTC + ttl: 360 +selectors: + items: + selector: 'a[href*="/newsroom/detail/"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/fia.com/documents.yml b/app/registries/seed/official/configs/fia.com/documents.yml new file mode 100644 index 000000000..d0d3be307 --- /dev/null +++ b/app/registries/seed/official/configs/fia.com/documents.yml @@ -0,0 +1,26 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +--- +registry: + id: fia.com/documents +directory: + topics: + - sports + title: "FIA — Formula 1 documents" + summary: "Official Formula 1 championship documents from the FIA." +channel: + language: en + title: "FIA — Formula 1 documents" + url: https://www.fia.com/documents/championships/fia-formula-one-world-championship-14/season/season-2026-2072 + ttl: 360 + time_zone: UTC +selectors: + items: + selector: ".decision-document-list .document-row" + title: + selector: ".title" + url: + selector: a + extractor: href + description: + selector: a + extractor: html diff --git a/app/registries/seed/official/configs/figma.com/blog.yml b/app/registries/seed/official/configs/figma.com/blog.yml new file mode 100644 index 000000000..f9501bcc2 --- /dev/null +++ b/app/registries/seed/official/configs/figma.com/blog.yml @@ -0,0 +1,25 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: figma.com/blog +directory: + topics: + - tech + - product + title: "Figma — Blog" + summary: "Product updates and design stories from Figma." +channel: + title: "Figma — Blog" + url: https://www.figma.com/blog/ + language: en + time_zone: UTC + ttl: 360 +selectors: + items: + selector: article[aria-label] + enhance: false + title: + extractor: attribute + attribute: aria-label + url: + selector: 'a[href*="/blog/"]' + extractor: href diff --git a/app/registries/seed/official/configs/formula1.com/latest.yml b/app/registries/seed/official/configs/formula1.com/latest.yml new file mode 100644 index 000000000..be1a8de19 --- /dev/null +++ b/app/registries/seed/official/configs/formula1.com/latest.yml @@ -0,0 +1,22 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: formula1.com/latest +directory: + topics: + - sports + title: "Formula 1 — Latest" + summary: "Latest Formula 1 news from the official Formula1.com site." +channel: + language: en + title: "Formula 1 — Latest" + url: https://www.formula1.com/en/latest + time_zone: UTC + ttl: 120 +selectors: + items: + selector: 'a[class*="ArticleListCard-module_title"]' + enhance: false + title: + extractor: text + url: + extractor: "href" diff --git a/app/registries/seed/official/configs/forumsec.org/publications.yml b/app/registries/seed/official/configs/forumsec.org/publications.yml new file mode 100644 index 000000000..9abd9402f --- /dev/null +++ b/app/registries/seed/official/configs/forumsec.org/publications.yml @@ -0,0 +1,24 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: forumsec.org/publications +directory: + topics: + - civic + - news + title: "Pacific Islands Forum — Publications" + summary: "Publications from the Pacific Islands Forum Secretariat." +channel: + title: "Pacific Islands Forum — Publications" + url: https://forumsec.org/publications + language: en + time_zone: Pacific/Fiji + ttl: 360 +selectors: + items: + selector: 'a.card__image[href^="/publications/"][aria-label]' + enhance: false + title: + extractor: attribute + attribute: aria-label + url: + extractor: href diff --git a/app/registries/seed/official/configs/ftc.gov/press-releases.yml b/app/registries/seed/official/configs/ftc.gov/press-releases.yml new file mode 100644 index 000000000..1d78e5bb2 --- /dev/null +++ b/app/registries/seed/official/configs/ftc.gov/press-releases.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: ftc.gov/press-releases +directory: + topics: + - security + - civic + title: "FTC — Press releases" + summary: "Press releases from the U.S. Federal Trade Commission." +channel: + title: "FTC — Press releases" + url: https://www.ftc.gov/news-events/news/press-releases + language: en + time_zone: UTC + ttl: 360 +selectors: + items: + selector: 'a[href*="/news-events/news/press-releases/20"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/github.com/releases.yml b/app/registries/seed/official/configs/github.com/releases.yml new file mode 100644 index 000000000..cab0a89ac --- /dev/null +++ b/app/registries/seed/official/configs/github.com/releases.yml @@ -0,0 +1,33 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: github.com/releases +directory: + topics: + - tech + - product + title: "GitHub — Releases" + summary: "Release notes for a GitHub repository (owner and name as parameters)." +parameters: + username: + type: string + default: "nuxt" + repository: + type: string + default: "nuxt" + +channel: + language: en + title: "GitHub — Releases" + url: https://github.com/%s/%s/releases + time_zone: UTC + ttl: 720 + description: "Releases of %s/%s on github.com." +selectors: + items: + selector: ".repository-content section" + enhance: false + title: + selector: 'a[href*="/releases/tag/"]' + url: + selector: 'a[href*="/releases/tag/"]' + extractor: "href" diff --git a/app/registries/seed/official/configs/go.dev/release-history.yml b/app/registries/seed/official/configs/go.dev/release-history.yml new file mode 100644 index 000000000..77c58366b --- /dev/null +++ b/app/registries/seed/official/configs/go.dev/release-history.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: go.dev/release-history +directory: + topics: + - tech + - product + title: "Go — Release history" + summary: "Official Go language release history and notes." +channel: + title: "Go — Release history" + url: https://go.dev/doc/devel/release + language: en + time_zone: UTC + ttl: 360 +selectors: + items: + selector: 'a[href^="/doc/go1."]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/grafana.com/whatsnew.yml b/app/registries/seed/official/configs/grafana.com/whatsnew.yml new file mode 100644 index 000000000..d903cbc6b --- /dev/null +++ b/app/registries/seed/official/configs/grafana.com/whatsnew.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: grafana.com/whatsnew +directory: + topics: + - tech + - product + title: "Grafana — What’s new" + summary: "What’s-new release notes for Grafana." +channel: + title: "Grafana — What’s new" + url: https://grafana.com/docs/grafana/latest/whatsnew/ + language: en + time_zone: UTC + ttl: 360 +selectors: + items: + selector: 'a.docs__menu-a[href^="/docs/grafana/latest/whatsnew/whats-new-in-v"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/haaretz.com/israel-news.yml b/app/registries/seed/official/configs/haaretz.com/israel-news.yml new file mode 100644 index 000000000..5c856dae1 --- /dev/null +++ b/app/registries/seed/official/configs/haaretz.com/israel-news.yml @@ -0,0 +1,22 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: haaretz.com/israel-news +directory: + topics: + - news + title: "Haaretz — Israel news" + summary: "Israel news coverage from Haaretz in English." +channel: + title: "Haaretz — Israel news" + url: https://www.haaretz.com/israel-news + language: en + time_zone: Asia/Jerusalem + ttl: 360 +selectors: + items: + selector: 'h3 a[href*="/ty-article"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/hai.stanford.edu/news.yml b/app/registries/seed/official/configs/hai.stanford.edu/news.yml new file mode 100644 index 000000000..81a28b3b6 --- /dev/null +++ b/app/registries/seed/official/configs/hai.stanford.edu/news.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: hai.stanford.edu/news +directory: + topics: + - science + - research + title: "Stanford HAI — News" + summary: "News from the Stanford Institute for Human-Centered Artificial Intelligence." +channel: + title: "Stanford HAI — News" + url: https://hai.stanford.edu/news + language: en + time_zone: UTC + ttl: 360 +selectors: + items: + selector: 'a[class*="titleLink"][href*="/news/"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/iaapa.org/news.yml b/app/registries/seed/official/configs/iaapa.org/news.yml new file mode 100644 index 000000000..d03cb0e30 --- /dev/null +++ b/app/registries/seed/official/configs/iaapa.org/news.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: iaapa.org/news +directory: + topics: + - entertainment + title: "IAAPA — News" + summary: "Industry news from the International Association of Amusement Parks." +channel: + language: en + title: "IAAPA — News" + url: https://iaapa.org/news-funworld + time_zone: UTC + ttl: 720 +enhance: false +selectors: + items: + selector: ".views-row > article" + title: + selector: h3 + url: + selector: "a" + extractor: "href" diff --git a/app/registries/seed/official/configs/icj-cij.org/press-releases.yml b/app/registries/seed/official/configs/icj-cij.org/press-releases.yml new file mode 100644 index 000000000..45068aaf2 --- /dev/null +++ b/app/registries/seed/official/configs/icj-cij.org/press-releases.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: icj-cij.org/press-releases +directory: + topics: + - civic + title: "ICJ — Press releases" + summary: "Press releases from the International Court of Justice." +channel: + title: "ICJ — Press releases" + url: https://www.icj-cij.org/press-releases + language: en + time_zone: UTC + ttl: 360 +selectors: + items: + selector: .views-row + enhance: false + title: + selector: .views-field-field-document-long-title + url: + selector: ".views-field-field-press-release-number a" + extractor: href diff --git a/app/registries/seed/official/configs/icrc.org/news.yml b/app/registries/seed/official/configs/icrc.org/news.yml new file mode 100644 index 000000000..2516bf9dc --- /dev/null +++ b/app/registries/seed/official/configs/icrc.org/news.yml @@ -0,0 +1,22 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: icrc.org/news +directory: + topics: + - civic + title: "ICRC — News" + summary: "Humanitarian news from the International Committee of the Red Cross." +channel: + title: "ICRC — News" + url: https://www.icrc.org/en/news + language: en + time_zone: Europe/Zurich + ttl: 360 +selectors: + items: + selector: 'a[href^="/en/article/"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/iea.org/news.yml b/app/registries/seed/official/configs/iea.org/news.yml new file mode 100644 index 000000000..3e854eca9 --- /dev/null +++ b/app/registries/seed/official/configs/iea.org/news.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: iea.org/news +directory: + topics: + - energy + - environment + title: "IEA — News" + summary: "Energy news and analysis from the International Energy Agency." +channel: + title: "IEA — News" + url: https://www.iea.org/news + language: en + time_zone: UTC + ttl: 360 +selectors: + items: + selector: 'a.m-news-detailed-listing__link[href^="/news/"]' + enhance: false + title: + selector: .m-news-detailed-listing__title + url: + extractor: href diff --git a/app/registries/seed/official/configs/iiss.org/online-analysis.yml b/app/registries/seed/official/configs/iiss.org/online-analysis.yml new file mode 100644 index 000000000..0e9b8651b --- /dev/null +++ b/app/registries/seed/official/configs/iiss.org/online-analysis.yml @@ -0,0 +1,29 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: iiss.org/online-analysis +directory: + topics: + - civic + - research + title: "IISS — Online analysis" + summary: "Strategic analysis from the International Institute for Strategic Studies." +strategy: botasaurus + +channel: + title: "IISS — Online analysis" + url: https://www.iiss.org/online-analysis + language: en + time_zone: Europe/London + ttl: 360 +request: + botasaurus: + wait_for_selector: 'a.feature[href*="/online-analysis/"][href*="/20"]' + wait_timeout_seconds: 20 +selectors: + items: + selector: 'a.feature[href*="/online-analysis/"][href*="/20"]' + enhance: false + title: + selector: h2 + url: + extractor: href diff --git a/app/registries/seed/official/configs/ikea.com/newsroom.yml b/app/registries/seed/official/configs/ikea.com/newsroom.yml new file mode 100644 index 000000000..3a4daa1af --- /dev/null +++ b/app/registries/seed/official/configs/ikea.com/newsroom.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: ikea.com/newsroom +directory: + topics: + - consumer + - product + title: "IKEA — Newsroom" + summary: "Global newsroom announcements from IKEA." +channel: + title: "IKEA — Newsroom" + url: https://www.ikea.com/global/en/newsroom/ + language: en + time_zone: Europe/Stockholm + ttl: 360 +selectors: + items: + selector: 'a.card[href*="/global/en/newsroom/"]' + enhance: true + title: + selector: span.skapa-typography-heading-m + url: + extractor: href diff --git a/app/registries/seed/official/configs/imdb.com/ratings.yml b/app/registries/seed/official/configs/imdb.com/ratings.yml new file mode 100644 index 000000000..32936b7e3 --- /dev/null +++ b/app/registries/seed/official/configs/imdb.com/ratings.yml @@ -0,0 +1,50 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: imdb.com/ratings +directory: + topics: + - entertainment + title: "IMDb — User ratings" + summary: "Film and TV ratings from a selectable IMDb user list." +strategy: botasaurus + +parameters: + user_id: + type: string + default: "ur7019649" + +channel: + language: en + title: "IMDb — User ratings" + url: https://www.imdb.com/user/%s/ratings + time_zone: UTC + ttl: 1440 +request: + botasaurus: + wait_timeout_seconds: 20 + wait_for_selector: "li.ipc-metadata-list-summary-item" +selectors: + items: + selector: "li.ipc-metadata-list-summary-item" + enhance: false + title: + selector: ".ipc-title__text" + post_process: + - name: gsub + pattern: "/^(\\d+.)\\s/" + replacement: "" + - name: template + string: 'Rated: "%{self}" %{user_rating} [IMDb: %{global_rating}]' + url: + selector: "a.ipc-title-link-wrapper" + extractor: "href" + description: + selector: ".ipc-html-content-inner-div" + global_rating: + selector: ".ipc-rating-star--imdb .ipc-rating-star--rating" + user_rating: + selector: ".dli-ratings-container .ipc-rating-star--otherUser span" + enclosure: + selector: ".ipc-image" + extractor: "attribute" + attribute: "src" diff --git a/app/registries/seed/official/configs/imo.org/press-briefings.yml b/app/registries/seed/official/configs/imo.org/press-briefings.yml new file mode 100644 index 000000000..0857264aa --- /dev/null +++ b/app/registries/seed/official/configs/imo.org/press-briefings.yml @@ -0,0 +1,28 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: imo.org/press-briefings +directory: + topics: + - civic + title: "IMO — Press briefings" + summary: "Press briefings from the International Maritime Organization." +strategy: botasaurus + +channel: + title: "IMO — Press briefings" + url: https://www.imo.org/en/MediaCentre/PressBriefings/Pages/default.aspx + language: en + time_zone: UTC + ttl: 360 +request: + botasaurus: + wait_for_selector: 'h3 a[href*="/PressBriefings/pages/"]' + wait_timeout_seconds: 20 +selectors: + items: + selector: 'h3 a[href*="/PressBriefings/pages/"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/imperial.ac.uk/news.yml b/app/registries/seed/official/configs/imperial.ac.uk/news.yml new file mode 100644 index 000000000..fbeed688b --- /dev/null +++ b/app/registries/seed/official/configs/imperial.ac.uk/news.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: imperial.ac.uk/news +directory: + topics: + - science + - research + title: "Imperial College London — News" + summary: "Research and university news from Imperial College London." +channel: + title: "Imperial College London — News" + url: https://www.imperial.ac.uk/news/articles/ + language: en + time_zone: Europe/London + ttl: 360 +selectors: + items: + selector: 'a.listing-item[href*="/news/articles/"]' + enhance: false + title: + selector: .listing-item__title + url: + extractor: href diff --git a/app/registries/seed/official/configs/ingenieur.de/karriere_arbeitsleben_heiko_mell.yml b/app/registries/seed/official/configs/ingenieur.de/karriere_arbeitsleben_heiko_mell.yml new file mode 100644 index 000000000..8df62e404 --- /dev/null +++ b/app/registries/seed/official/configs/ingenieur.de/karriere_arbeitsleben_heiko_mell.yml @@ -0,0 +1,25 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +--- +registry: + id: ingenieur.de/karriere_arbeitsleben_heiko_mell +directory: + topics: + - jobs + title: "ingenieur.de — Heiko Mell" + summary: "Karriere-Kolumnen von Heiko Mell auf ingenieur.de." +channel: + title: "ingenieur.de — Heiko Mell" + url: https://www.ingenieur.de/karriere/arbeitsleben/heiko-mell/ + language: de-DE + ttl: 360 + time_zone: UTC +selectors: + items: + selector: ".section > .ing-teaser" + title: + selector: ".ing-teaser__headline" + url: + selector: a:first + extractor: href + description: + selector: ".ing-teaser__excerpt" diff --git a/app/registries/seed/official/configs/isro.gov.in/press.yml b/app/registries/seed/official/configs/isro.gov.in/press.yml new file mode 100644 index 000000000..ffd5162e1 --- /dev/null +++ b/app/registries/seed/official/configs/isro.gov.in/press.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: isro.gov.in/press +directory: + topics: + - science + - research + title: "ISRO — Press" + summary: "Press releases from the Indian Space Research Organisation." +channel: + title: "ISRO — Press" + url: https://www.isro.gov.in/Press.html + language: en + time_zone: Asia/Kolkata + ttl: 360 +selectors: + items: + selector: 'table tr a[href$=".html"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/issafrica.org/iss-today.yml b/app/registries/seed/official/configs/issafrica.org/iss-today.yml new file mode 100644 index 000000000..ec530484d --- /dev/null +++ b/app/registries/seed/official/configs/issafrica.org/iss-today.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: issafrica.org/iss-today +directory: + topics: + - civic + - research + title: "ISS Africa — ISS Today" + summary: "Daily analysis from the Institute for Security Studies Africa." +channel: + title: "ISS Africa — ISS Today" + url: https://issafrica.org/iss-today + language: en + time_zone: Africa/Johannesburg + ttl: 360 +selectors: + items: + selector: 'a.card[href^="/iss-today/"]' + enhance: false + title: + selector: h6.card-subtitle + url: + extractor: href diff --git a/app/registries/seed/official/configs/issafrica.org/press-releases.yml b/app/registries/seed/official/configs/issafrica.org/press-releases.yml new file mode 100644 index 000000000..79d212049 --- /dev/null +++ b/app/registries/seed/official/configs/issafrica.org/press-releases.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: issafrica.org/press-releases +directory: + topics: + - civic + - research + title: "ISS Africa — Press releases" + summary: "Press releases from the Institute for Security Studies Africa." +channel: + title: "ISS Africa — Press releases" + url: https://issafrica.org/about-us/press-releases + language: en + time_zone: Africa/Johannesburg + ttl: 720 +selectors: + items: + selector: 'a.card[href^="/about-us/press-releases/"]' + enhance: false + title: + selector: h6.card-subtitle + url: + extractor: href diff --git a/app/registries/seed/official/configs/kaist.ac.kr/news.yml b/app/registries/seed/official/configs/kaist.ac.kr/news.yml new file mode 100644 index 000000000..9805ed934 --- /dev/null +++ b/app/registries/seed/official/configs/kaist.ac.kr/news.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: kaist.ac.kr/news +directory: + topics: + - science + - research + title: "KAIST — News" + summary: "University and research news from KAIST." +channel: + title: "KAIST — News" + url: https://news.kaist.ac.kr/newsen/html/news/ + language: en + time_zone: Asia/Seoul + ttl: 360 +selectors: + items: + selector: 'a.lay[href*="mng_no="]' + enhance: false + title: + selector: strong.tis + url: + extractor: href diff --git a/app/registries/seed/official/configs/kinocheck.de/filmstarts.yml b/app/registries/seed/official/configs/kinocheck.de/filmstarts.yml new file mode 100644 index 000000000..558eecf22 --- /dev/null +++ b/app/registries/seed/official/configs/kinocheck.de/filmstarts.yml @@ -0,0 +1,30 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: kinocheck.de/filmstarts +directory: + topics: + - entertainment + title: "KinoCheck — Filmstarts" + summary: "Kommende und aktuelle Filmstarts in Deutschland." +channel: + title: "KinoCheck — Filmstarts" + url: https://kinocheck.de/filmstarts + time_zone: Europe/Berlin + ttl: 1440 + language: de +selectors: + items: + selector: ".list-movies-wide-item .wrapper-content" + title: + selector: "h2" + post_process: + - name: "template" + string: "%{self} (%{start_date})" + url: + selector: "a" + extractor: "href" + description: + selector: ".information" + extractor: "html" + start_date: + selector: ".release-date .highlight" diff --git a/app/registries/seed/official/configs/kompas.com/nasional.yml b/app/registries/seed/official/configs/kompas.com/nasional.yml new file mode 100644 index 000000000..e5655866b --- /dev/null +++ b/app/registries/seed/official/configs/kompas.com/nasional.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: kompas.com/nasional +directory: + topics: + - news + - civic + title: "Kompas — Nasional" + summary: "Berita nasional dari Kompas.com." +channel: + title: "Kompas — Nasional" + url: https://nasional.kompas.com/ + language: id + time_zone: Asia/Jakarta + ttl: 360 +selectors: + items: + selector: '.articleList a[href*="/read/"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/lego.com/press-releases.yml b/app/registries/seed/official/configs/lego.com/press-releases.yml new file mode 100644 index 000000000..f2cdc9b23 --- /dev/null +++ b/app/registries/seed/official/configs/lego.com/press-releases.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: lego.com/press-releases +directory: + topics: + - entertainment + - product + title: "LEGO — Press releases" + summary: "Official press releases from the LEGO Group." +channel: + title: "LEGO — Press releases" + url: https://www.lego.com/en-gb/aboutus/newsroom/press-releases + language: en-GB + time_zone: Europe/Copenhagen + ttl: 360 +selectors: + items: + selector: 'a.c-card--article[href*="/aboutus/news/20"]' + enhance: true + title: + selector: h2.c-card__title + url: + extractor: href diff --git a/app/registries/seed/official/configs/machinelearning.apple.com/research.yml b/app/registries/seed/official/configs/machinelearning.apple.com/research.yml new file mode 100644 index 000000000..785f7f8c6 --- /dev/null +++ b/app/registries/seed/official/configs/machinelearning.apple.com/research.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: machinelearning.apple.com/research +directory: + topics: + - tech + - research + title: "Apple Machine Learning — Research" + summary: "Machine learning research publications from Apple." +channel: + title: "Apple Machine Learning — Research" + url: https://machinelearning.apple.com/research/ + language: en + time_zone: UTC + ttl: 360 +selectors: + items: + selector: "ul.article-list > li.post-row > div > a.post-link" + enhance: false + title: + selector: h3.post-title + url: + extractor: href diff --git a/app/registries/seed/official/configs/maersk.com/press-releases.yml b/app/registries/seed/official/configs/maersk.com/press-releases.yml new file mode 100644 index 000000000..5a8f6ebea --- /dev/null +++ b/app/registries/seed/official/configs/maersk.com/press-releases.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: maersk.com/press-releases +directory: + topics: + - finance + title: "Maersk — Press releases" + summary: "Corporate press releases from Maersk." +channel: + title: "Maersk — Press releases" + url: https://www.maersk.com/news/category/press-releases + language: en + time_zone: Europe/Copenhagen + ttl: 360 +selectors: + items: + selector: div.p-section__news__teaser + enhance: false + title: + selector: a.p-section__news__teaser__title + url: + selector: a.p-section__news__teaser__title + extractor: href diff --git a/app/registries/seed/official/configs/microsoft.com/azure-products.yml b/app/registries/seed/official/configs/microsoft.com/azure-products.yml new file mode 100644 index 000000000..4cfc4c0e4 --- /dev/null +++ b/app/registries/seed/official/configs/microsoft.com/azure-products.yml @@ -0,0 +1,25 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: microsoft.com/azure-products +directory: + topics: + - tech + - product + title: "Microsoft — Azure Products" + summary: "Product updates and announcements from Microsoft Azure." +channel: + title: "Microsoft — Azure Products" + url: https://azure.microsoft.com/en-us/products + language: en + time_zone: US/Pacific + ttl: 3600 +selectors: + items: + selector: ".card-body" + title: + selector: ".h5" + url: + selector: ".link-group a:first-of-type" + extractor: "href" + description: + selector: "p" diff --git a/app/registries/seed/official/configs/mistral.ai/news.yml b/app/registries/seed/official/configs/mistral.ai/news.yml new file mode 100644 index 000000000..38a070656 --- /dev/null +++ b/app/registries/seed/official/configs/mistral.ai/news.yml @@ -0,0 +1,24 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: mistral.ai/news +directory: + topics: + - tech + - research + title: "Mistral AI — News" + summary: "Product and research news from Mistral AI." +channel: + title: "Mistral AI — News" + url: https://mistral.ai/news/ + language: en + time_zone: UTC + ttl: 360 +selectors: + items: + selector: article.js-post + enhance: false + title: + selector: h2.js-post-title + url: + selector: a[href^="/news/"] + extractor: href diff --git a/app/registries/seed/official/configs/mozilla.org/security-advisories.yml b/app/registries/seed/official/configs/mozilla.org/security-advisories.yml new file mode 100644 index 000000000..2e0d5f3c3 --- /dev/null +++ b/app/registries/seed/official/configs/mozilla.org/security-advisories.yml @@ -0,0 +1,24 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: mozilla.org/security-advisories +directory: + topics: + - security + - tech + title: "Mozilla — Security advisories" + summary: "Security advisories for Mozilla products." +channel: + title: "Mozilla — Security advisories" + url: https://www.mozilla.org/en-US/security/advisories/ + language: en + time_zone: UTC + ttl: 360 +selectors: + items: + selector: "main li" + enhance: false + title: + selector: 'a[href*="/security/advisories/mfsa"]' + url: + selector: 'a[href*="/security/advisories/mfsa"]' + extractor: href diff --git a/app/registries/seed/official/configs/msf.org/latest.yml b/app/registries/seed/official/configs/msf.org/latest.yml new file mode 100644 index 000000000..f60f13455 --- /dev/null +++ b/app/registries/seed/official/configs/msf.org/latest.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: msf.org/latest +directory: + topics: + - news + - civic + title: "Médecins Sans Frontières — Latest" + summary: "Latest field and advocacy updates from Médecins Sans Frontières." +channel: + title: "Médecins Sans Frontières — Latest" + url: https://www.msf.org/latest-portal + language: en + time_zone: UTC + ttl: 360 +selectors: + items: + selector: a.card__title-link + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/nbim.no/press-releases.yml b/app/registries/seed/official/configs/nbim.no/press-releases.yml new file mode 100644 index 000000000..f286fed76 --- /dev/null +++ b/app/registries/seed/official/configs/nbim.no/press-releases.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: nbim.no/press-releases +directory: + topics: + - finance + title: "Norges Bank Investment Management — Press releases" + summary: "Press releases from Norway’s sovereign wealth fund manager." +channel: + title: "Norges Bank Investment Management — Press releases" + url: https://www.nbim.no/en/news-and-insights/the-press/press-releases/ + language: en + time_zone: Europe/Oslo + ttl: 360 +selectors: + items: + selector: div.content-featured-content + enhance: true + title: + selector: h3 + url: + selector: a.chevron-link + extractor: href diff --git a/app/registries/seed/official/configs/news.cn/latest.yml b/app/registries/seed/official/configs/news.cn/latest.yml new file mode 100644 index 000000000..9312eb834 --- /dev/null +++ b/app/registries/seed/official/configs/news.cn/latest.yml @@ -0,0 +1,22 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: news.cn/latest +directory: + topics: + - news + title: "Xinhua — Latest news" + summary: "Latest English news from Xinhua News Agency." +channel: + title: "Xinhua — Latest news" + url: https://english.news.cn/list/latestnews.htm + language: en + time_zone: Asia/Shanghai + ttl: 360 +selectors: + items: + selector: "#list .item .tit a" + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/news24.com/top-stories.yml b/app/registries/seed/official/configs/news24.com/top-stories.yml new file mode 100644 index 000000000..e68f8f174 --- /dev/null +++ b/app/registries/seed/official/configs/news24.com/top-stories.yml @@ -0,0 +1,22 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: news24.com/top-stories +directory: + topics: + - news + title: "News24 — Top stories" + summary: "Top stories from South African news site News24." +channel: + title: "News24 — Top stories" + url: https://www.news24.com/ + language: en + time_zone: Africa/Johannesburg + ttl: 180 +selectors: + items: + selector: "a.article-item__title" + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/newyorker.com/magazine.yml b/app/registries/seed/official/configs/newyorker.com/magazine.yml new file mode 100644 index 000000000..6cc171ddf --- /dev/null +++ b/app/registries/seed/official/configs/newyorker.com/magazine.yml @@ -0,0 +1,25 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +--- +registry: + id: newyorker.com/magazine +directory: + topics: + - news + title: "The New Yorker — Magazine" + summary: "Current magazine stories from The New Yorker." +channel: + title: "The New Yorker — Magazine" + url: https://www.newyorker.com/magazine + language: en + ttl: 360 + time_zone: UTC +selectors: + items: + selector: .summary-item + title: + selector: h3 + url: + selector: a:first + extractor: href + description: + selector: .summary-item__dek diff --git a/app/registries/seed/official/configs/nigerianstat.gov.ng/news.yml b/app/registries/seed/official/configs/nigerianstat.gov.ng/news.yml new file mode 100644 index 000000000..7c635c88d --- /dev/null +++ b/app/registries/seed/official/configs/nigerianstat.gov.ng/news.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: nigerianstat.gov.ng/news +directory: + topics: + - civic + - research + title: "Nigeria Bureau of Statistics — News" + summary: "Official statistical releases and news from Nigeria’s NBS." +channel: + title: "Nigeria Bureau of Statistics — News" + url: https://nigerianstat.gov.ng/ + language: en + time_zone: Africa/Lagos + ttl: 360 +selectors: + items: + selector: 'a[href*="/news/"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/nikkei.com/asia-technology.yml b/app/registries/seed/official/configs/nikkei.com/asia-technology.yml new file mode 100644 index 000000000..205026563 --- /dev/null +++ b/app/registries/seed/official/configs/nikkei.com/asia-technology.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: nikkei.com/asia-technology +directory: + topics: + - news + - tech + title: "Nikkei Asia — Technology" + summary: "Asia technology business coverage from Nikkei Asia." +channel: + title: "Nikkei Asia — Technology" + url: https://asia.nikkei.com/business/technology + language: en + time_zone: Asia/Tokyo + ttl: 360 +selectors: + items: + selector: 'a[data-trackable="headline"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/nokia.com/newsroom.yml b/app/registries/seed/official/configs/nokia.com/newsroom.yml new file mode 100644 index 000000000..d9a25f177 --- /dev/null +++ b/app/registries/seed/official/configs/nokia.com/newsroom.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: nokia.com/newsroom +directory: + topics: + - tech + - product + title: "Nokia — Newsroom" + summary: "Corporate newsroom announcements from Nokia." +channel: + title: "Nokia — Newsroom" + url: https://www.nokia.com/newsroom/ + language: en + time_zone: Europe/Helsinki + ttl: 360 +selectors: + items: + selector: 'a.pp-block-item-container[href*="/newsroom/"]' + enhance: false + title: + selector: h3.pp-block-item-heading + url: + extractor: href diff --git a/app/registries/seed/official/configs/nomanssky.com/news.yml b/app/registries/seed/official/configs/nomanssky.com/news.yml new file mode 100644 index 000000000..b3fc8257e --- /dev/null +++ b/app/registries/seed/official/configs/nomanssky.com/news.yml @@ -0,0 +1,29 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +--- +registry: + id: nomanssky.com/news +directory: + topics: + - entertainment + title: "No Man’s Sky — News" + summary: "Game updates and news for No Man’s Sky." +channel: + title: "No Man’s Sky — News" + url: https://www.nomanssky.com/news/ + language: en-GB + ttl: 360 + time_zone: UTC +selectors: + items: + selector: article + title: + selector: ".post-title" + url: + selector: a.view-article + extractor: href + description: + selector: p:first + post_process: + - name: substring + start: 0 + end: -14 diff --git a/app/registries/seed/official/configs/notion.com/blog.yml b/app/registries/seed/official/configs/notion.com/blog.yml new file mode 100644 index 000000000..6e4efc349 --- /dev/null +++ b/app/registries/seed/official/configs/notion.com/blog.yml @@ -0,0 +1,27 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: notion.com/blog +directory: + topics: + - tech + - product + title: "Notion — Blog" + summary: "Product updates and stories from the Notion blog." + +channel: + title: "Notion — Blog" + url: https://www.notion.com/blog + language: en + time_zone: UTC + ttl: 360 +selectors: + items: + selector: article.post-preview + enhance: false + title: + selector: h3 a[href*="/blog/"] + url: + selector: h3 a[href*="/blog/"] + extractor: href + description: + selector: '> a[href*="/blog/"]:not([title])' diff --git a/app/registries/seed/official/configs/novonordisk.com/latest-news.yml b/app/registries/seed/official/configs/novonordisk.com/latest-news.yml new file mode 100644 index 000000000..e9cba5ebf --- /dev/null +++ b/app/registries/seed/official/configs/novonordisk.com/latest-news.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: novonordisk.com/latest-news +directory: + topics: + - science + - product + title: "Novo Nordisk — Latest news" + summary: "Latest corporate and product news from Novo Nordisk." +channel: + title: "Novo Nordisk — Latest news" + url: https://www.novonordisk.com/news-and-media/latest-news.html + language: en + time_zone: Europe/Copenhagen + ttl: 360 +selectors: + items: + selector: 'a.ln-link-item[href*="news-details.html"]' + enhance: false + title: + selector: .location-info p.paragraph-s + url: + extractor: href diff --git a/app/registries/seed/official/configs/ntu.edu.sg/news.yml b/app/registries/seed/official/configs/ntu.edu.sg/news.yml new file mode 100644 index 000000000..130ad35ab --- /dev/null +++ b/app/registries/seed/official/configs/ntu.edu.sg/news.yml @@ -0,0 +1,29 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: ntu.edu.sg/news +directory: + topics: + - science + - research + title: "NTU Singapore — News" + summary: "University and research news from Nanyang Technological University." +strategy: botasaurus + +channel: + title: "NTU Singapore — News" + url: https://www.ntu.edu.sg/news + language: en + time_zone: Asia/Singapore + ttl: 360 +request: + botasaurus: + wait_for_selector: 'a.img-card[href*="/news/detail/"]' + wait_timeout_seconds: 20 +selectors: + items: + selector: 'a.img-card[href*="/news/detail/"]' + enhance: false + title: + selector: .img-card__title + url: + extractor: href diff --git a/app/registries/seed/official/configs/nupi.no/news.yml b/app/registries/seed/official/configs/nupi.no/news.yml new file mode 100644 index 000000000..223a53274 --- /dev/null +++ b/app/registries/seed/official/configs/nupi.no/news.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: nupi.no/news +directory: + topics: + - civic + - research + title: "NUPI — News" + summary: "News from the Norwegian Institute of International Affairs." +channel: + title: "NUPI — News" + url: https://www.nupi.no/en + language: en + time_zone: Europe/Oslo + ttl: 360 +selectors: + items: + selector: 'h2 a[href^="/en/news/"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/occrp.org/news.yml b/app/registries/seed/official/configs/occrp.org/news.yml new file mode 100644 index 000000000..6ec01fa82 --- /dev/null +++ b/app/registries/seed/official/configs/occrp.org/news.yml @@ -0,0 +1,30 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: occrp.org/news +directory: + topics: + - news + - civic + title: "OCCRP — News" + summary: "Investigative journalism from the Organized Crime and Corruption Reporting Project." +strategy: botasaurus + +channel: + title: "OCCRP — News" + url: https://www.occrp.org/en/news + language: en + time_zone: UTC + ttl: 360 +request: + botasaurus: + wait_for_selector: article.news-item + wait_timeout_seconds: 20 +selectors: + items: + selector: article.news-item + enhance: false + title: + selector: h3.news-item__title a + url: + selector: h3.news-item__title a + extractor: href diff --git a/app/registries/seed/official/configs/oecd.org/newsroom.yml b/app/registries/seed/official/configs/oecd.org/newsroom.yml new file mode 100644 index 000000000..43194b044 --- /dev/null +++ b/app/registries/seed/official/configs/oecd.org/newsroom.yml @@ -0,0 +1,29 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: oecd.org/newsroom +directory: + topics: + - finance + - civic + title: "OECD — Newsroom" + summary: "Newsroom releases from the Organisation for Economic Co-operation and Development." +strategy: botasaurus + +channel: + title: "OECD — Newsroom" + url: https://www.oecd.org/en/about/newsroom.html + language: en + time_zone: UTC + ttl: 360 +request: + botasaurus: + wait_for_selector: 'a[href*="/en/about/news/press-releases/"]' + wait_timeout_seconds: 20 +selectors: + items: + selector: 'a.card__title-link[href*="/en/about/news/press-releases/"], a.cmp-featured-card__link[href*="/en/about/news/press-releases/"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/opec.org/press-releases.yml b/app/registries/seed/official/configs/opec.org/press-releases.yml new file mode 100644 index 000000000..d89fa2eba --- /dev/null +++ b/app/registries/seed/official/configs/opec.org/press-releases.yml @@ -0,0 +1,30 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: opec.org/press-releases +directory: + topics: + - energy + - civic + title: "OPEC — Press releases" + summary: "Official press releases from OPEC." +strategy: botasaurus + +channel: + title: "OPEC — Press releases" + url: https://www.opec.org/press-releases.html + language: en + time_zone: Europe/Vienna + ttl: 360 +request: + botasaurus: + wait_for_selector: 'div.pritem a[href*="pr-detail"]' + wait_timeout_seconds: 20 +selectors: + items: + selector: div.pritem + enhance: false + title: + selector: h1 + url: + selector: 'a[href*="pr-detail"]' + extractor: href diff --git a/app/registries/seed/official/configs/panapress.com/latest.yml b/app/registries/seed/official/configs/panapress.com/latest.yml new file mode 100644 index 000000000..4124a4f98 --- /dev/null +++ b/app/registries/seed/official/configs/panapress.com/latest.yml @@ -0,0 +1,28 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: panapress.com/latest +directory: + topics: + - news + title: "PANA Press — Latest" + summary: "Latest African news from the Pan African News Agency." +strategy: botasaurus + +channel: + title: "PANA Press — Latest" + url: https://www.panapress.com/ + language: en + time_zone: UTC + ttl: 360 +request: + botasaurus: + wait_for_selector: 'h4 a[href*="lang2.html"]' + wait_timeout_seconds: 20 +selectors: + items: + selector: 'h4 a[href*="lang2.html"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/pankow.lebensmittel-kontrollergebnisse.de/search.yml b/app/registries/seed/official/configs/pankow.lebensmittel-kontrollergebnisse.de/search.yml new file mode 100644 index 000000000..ca3a888c3 --- /dev/null +++ b/app/registries/seed/official/configs/pankow.lebensmittel-kontrollergebnisse.de/search.yml @@ -0,0 +1,53 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +--- +registry: + id: pankow.lebensmittel-kontrollergebnisse.de/search +directory: + topics: + - consumer + - civic + title: "Berlin-Pankow — Lebensmittelkontrollen" + summary: "Veröffentlichte Lebensmittel-Kontrollergebnisse für Berlin-Pankow." +channel: + title: "Berlin-Pankow — Lebensmittelkontrollen" + url: https://pankow.lebensmittel-kontrollergebnisse.de/Search + language: de + ttl: 360 + time_zone: Europe/Berlin +selectors: + items: + selector: main .card + title: + selector: h5 + post_process: + - name: template + string: | + %{self}: %{ergebnis} (%{punkte}/80 Punkte) + url: + selector: a + extractor: href + description: + selector: ".row" + extractor: html + post_process: + - name: html_to_markdown + - name: markdown_to_html + ergebnis: + selector: ".bewertung-panel .row:nth-child(3)" + post_process: + - name: substring + start: 9 + punkte: + selector: ".bewertung-panel .row:nth-child(2)" + post_process: + - name: substring + start: 10 + published_at: + selector: ".bewertung-panel .row:nth-child(1)" + post_process: + - name: substring + start: -10 + - name: parse_time + categories: + - ergebnis + - punkte diff --git a/app/registries/seed/official/configs/philstar.com/nation.yml b/app/registries/seed/official/configs/philstar.com/nation.yml new file mode 100644 index 000000000..87071a14b --- /dev/null +++ b/app/registries/seed/official/configs/philstar.com/nation.yml @@ -0,0 +1,22 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: philstar.com/nation +directory: + topics: + - news + title: "Philippine Star — Nation" + summary: "National news from The Philippine Star." +channel: + title: "Philippine Star — Nation" + url: https://www.philstar.com/nation + language: en + time_zone: Asia/Manila + ttl: 360 +selectors: + items: + selector: 'h2 a[href*="/nation/"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/phys.org/weekly.yml b/app/registries/seed/official/configs/phys.org/weekly.yml new file mode 100644 index 000000000..a40b0486b --- /dev/null +++ b/app/registries/seed/official/configs/phys.org/weekly.yml @@ -0,0 +1,26 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: phys.org/weekly +directory: + topics: + - science + title: "Phys.org — Weekly" + summary: "Top science news of the week from Phys.org." +channel: + language: en + title: "Phys.org — Weekly" + url: https://phys.org/weekly-news/ + time_zone: Europe/London + ttl: 1440 +selectors: + items: + selector: ".sorted-news-list .sorted-article-content" + title: + selector: "h4" + category: + selector: ".text-info" + categories: + - category + url: + selector: ".news-link" + extractor: "href" diff --git a/app/registries/seed/official/configs/pib.gov.in/press-releases.yml b/app/registries/seed/official/configs/pib.gov.in/press-releases.yml new file mode 100644 index 000000000..21ae23d4c --- /dev/null +++ b/app/registries/seed/official/configs/pib.gov.in/press-releases.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: pib.gov.in/press-releases +directory: + topics: + - civic + - news + title: "PIB India — Press releases" + summary: "Press releases from India’s Press Information Bureau." +channel: + title: "PIB India — Press releases" + url: https://www.pib.gov.in/AllRel.aspx?reg=3&lang=1 + language: en + time_zone: Asia/Kolkata + ttl: 360 +selectors: + items: + selector: 'a[href*="PressReleaseDetail.aspx"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/rbb24.de/meistgeklickt.yml b/app/registries/seed/official/configs/rbb24.de/meistgeklickt.yml new file mode 100644 index 000000000..b28744a8a --- /dev/null +++ b/app/registries/seed/official/configs/rbb24.de/meistgeklickt.yml @@ -0,0 +1,22 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: rbb24.de/meistgeklickt +directory: + topics: + - news + title: "rbb24 — Meistgeklickt" + summary: "Most-clicked stories from rbb24." +channel: + title: "rbb24 — Meistgeklickt" + url: https://www.rbb24.de/das-beste/ + time_zone: Europe/Berlin + ttl: 30 + language: de +selectors: + items: + selector: "article.top-teaser a.top-teaser__text-container" + enhance: false + title: + extractor: "text" + url: + extractor: "href" diff --git a/app/registries/seed/official/configs/robinwood.de/aktuelles.yml b/app/registries/seed/official/configs/robinwood.de/aktuelles.yml new file mode 100644 index 000000000..58a03e08b --- /dev/null +++ b/app/registries/seed/official/configs/robinwood.de/aktuelles.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: robinwood.de/aktuelles +directory: + topics: + - environment + title: "ROBIN WOOD — Aktuelles" + summary: "Aktuelle Meldungen der Umweltschutzorganisation ROBIN WOOD." +channel: + title: "ROBIN WOOD — Aktuelles" + url: https://www.robinwood.de/was-gibt-es-neues + time_zone: Europe/Berlin + ttl: 720 + language: de +selectors: + items: + selector: "article.node--view-mode-teaser" + enhance: false + title: + selector: "h2" + url: + selector: "a" + extractor: "href" diff --git a/app/registries/seed/official/configs/s3.amazonaws.com/popular_movies.yml b/app/registries/seed/official/configs/s3.amazonaws.com/popular_movies.yml new file mode 100644 index 000000000..e109bf43b --- /dev/null +++ b/app/registries/seed/official/configs/s3.amazonaws.com/popular_movies.yml @@ -0,0 +1,35 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: s3.amazonaws.com/popular_movies +directory: + topics: + - entertainment + title: "Popular movies — JSON list" + summary: "Popular movies listing from a public JSON feed on Amazon S3." +# This generates a RSS of the daily updated +# https://github.com/sjlu/popular-movies +channel: + language: en + title: "Popular movies — JSON list" + url: https://s3.amazonaws.com/popular-movies/movies.json + time_zone: UTC + ttl: 1440 + json: true +selectors: + items: + selector: "array > object" + title: + selector: "title" + imdb_id: + selector: "imdb_id" + poster_url: + selector: "poster_url" + url: + post_process: + - name: template + string: "https://imdb.com/title/%{imdb_id}" + description: + post_process: + - name: template + string: "![](%{poster_url})" + - name: markdown_to_html diff --git a/app/registries/seed/official/configs/sacu.int/news.yml b/app/registries/seed/official/configs/sacu.int/news.yml new file mode 100644 index 000000000..e1de83dce --- /dev/null +++ b/app/registries/seed/official/configs/sacu.int/news.yml @@ -0,0 +1,24 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: sacu.int/news +directory: + topics: + - civic + - news + title: "SACU — News" + summary: "News from the Southern African Customs Union." +channel: + title: "SACU — News" + url: https://www.sacu.int/news/ + language: en + time_zone: Africa/Windhoek + ttl: 360 +selectors: + items: + selector: ".card" + enhance: false + title: + selector: h3 + url: + selector: 'a[href*="/articles/latest-news/"]' + extractor: href diff --git a/app/registries/seed/official/configs/sadc.int/latest-news.yml b/app/registries/seed/official/configs/sadc.int/latest-news.yml new file mode 100644 index 000000000..2af67d5ff --- /dev/null +++ b/app/registries/seed/official/configs/sadc.int/latest-news.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: sadc.int/latest-news +directory: + topics: + - civic + - news + title: "SADC — Latest news" + summary: "Latest news from the Southern African Development Community." +channel: + title: "SADC — Latest news" + url: https://www.sadc.int/latest-news + language: en + time_zone: UTC + ttl: 360 +selectors: + items: + selector: 'span.field-content a[href^="/latest-news/"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/sebastianvettel.de/news.yml b/app/registries/seed/official/configs/sebastianvettel.de/news.yml new file mode 100644 index 000000000..226563b24 --- /dev/null +++ b/app/registries/seed/official/configs/sebastianvettel.de/news.yml @@ -0,0 +1,24 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +--- +registry: + id: sebastianvettel.de/news +directory: + topics: + - sports + title: "Sebastian Vettel — News" + summary: "Neuigkeiten von der Website von Sebastian Vettel." +channel: + title: "Sebastian Vettel — News" + url: https://sebastianvettel.de/ + language: de-DE + ttl: 8640 + time_zone: Europe/Berlin +selectors: + items: + selector: ".gb-loop-item:has(a[href*='/news/'])" + enhance: false + title: + selector: ".gb-text:first-child" + url: + selector: "a[href*='/news/']" + extractor: href diff --git a/app/registries/seed/official/configs/shopify.com/blog.yml b/app/registries/seed/official/configs/shopify.com/blog.yml new file mode 100644 index 000000000..834e736f8 --- /dev/null +++ b/app/registries/seed/official/configs/shopify.com/blog.yml @@ -0,0 +1,25 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: shopify.com/blog +directory: + topics: + - tech + - product + title: "Shopify — Blog" + summary: "Commerce tips and product updates from the Shopify blog." + +channel: + language: en + title: "Shopify — Blog" + url: https://www.shopify.com/blog/latest + time_zone: UTC + ttl: 360 +selectors: + items: + selector: article.article--index + enhance: false + title: + selector: '.blogPost a[href*="/blog/"]:not([href*="/topics/"])' + url: + selector: '.blogPost a[href*="/blog/"]:not([href*="/topics/"])' + extractor: href diff --git a/app/registries/seed/official/configs/sipri.org/news.yml b/app/registries/seed/official/configs/sipri.org/news.yml new file mode 100644 index 000000000..6778a9073 --- /dev/null +++ b/app/registries/seed/official/configs/sipri.org/news.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: sipri.org/news +directory: + topics: + - civic + - research + title: "SIPRI — News" + summary: "News from the Stockholm International Peace Research Institute." +channel: + title: "SIPRI — News" + url: https://www.sipri.org/news + language: en + time_zone: UTC + ttl: 360 +selectors: + items: + selector: 'h3.field-content a[href^="/news/20"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/softwareleadweekly.com/issues.yml b/app/registries/seed/official/configs/softwareleadweekly.com/issues.yml new file mode 100644 index 000000000..d0a8c49af --- /dev/null +++ b/app/registries/seed/official/configs/softwareleadweekly.com/issues.yml @@ -0,0 +1,22 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: softwareleadweekly.com/issues +directory: + topics: + - tech + title: "Software Lead Weekly — Issues" + summary: "Weekly leadership newsletter issues from Software Lead Weekly." +channel: + language: en + title: "Software Lead Weekly — Issues" + url: https://softwareleadweekly.com/issues + time_zone: UTC + ttl: 720 +selectors: + items: + selector: ".table-issue" + title: + selector: "a" + url: + selector: "a" + extractor: "href" diff --git a/app/registries/seed/official/configs/solarthermalworld.org/news.yml b/app/registries/seed/official/configs/solarthermalworld.org/news.yml new file mode 100644 index 000000000..c25d2fa07 --- /dev/null +++ b/app/registries/seed/official/configs/solarthermalworld.org/news.yml @@ -0,0 +1,25 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: solarthermalworld.org/news +directory: + topics: + - energy + title: "Solarthermalworld — News" + summary: "Global news on solar thermal energy and heat." +channel: + language: en + title: "Solarthermalworld — News" + url: https://solarthermalworld.org/news + time_zone: UTC + ttl: 180 +selectors: + items: + selector: "article.post-wrapper" + title: + selector: "h2 a" + url: + selector: "h2 a" + extractor: "href" + description: + selector: '[property="content:encoded"]' + extractor: html diff --git a/app/registries/seed/official/configs/spektrum.de/meistgelesen.yml b/app/registries/seed/official/configs/spektrum.de/meistgelesen.yml new file mode 100644 index 000000000..db7788f75 --- /dev/null +++ b/app/registries/seed/official/configs/spektrum.de/meistgelesen.yml @@ -0,0 +1,22 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: spektrum.de/meistgelesen +directory: + topics: + - science + title: "Spektrum — Meistgelesen" + summary: "Most-read science articles from Spektrum." +channel: + title: "Spektrum — Meistgelesen" + url: https://www.spektrum.de/ + time_zone: Europe/Berlin + ttl: 60 + language: de +selectors: + items: + selector: ".teaser.teaser__list_box ol > li" + title: + selector: "a" + url: + selector: "a" + extractor: "href" diff --git a/app/registries/seed/official/configs/spiegel.de/impressum_autor.yml b/app/registries/seed/official/configs/spiegel.de/impressum_autor.yml new file mode 100644 index 000000000..e6aa1346d --- /dev/null +++ b/app/registries/seed/official/configs/spiegel.de/impressum_autor.yml @@ -0,0 +1,39 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: spiegel.de/impressum_autor +directory: + topics: + - news + title: "DER SPIEGEL — Author articles" + summary: "Articles by a selectable DER SPIEGEL author profile." +parameters: + id: + type: string + default: "975b6ae0-0001-0003-0000-000000018282" + +channel: + title: "DER SPIEGEL — Author articles" + url: "https://www.spiegel.de/impressum/autor-%s" + time_zone: Europe/Berlin + ttl: 180 + language: de +selectors: + items: + selector: '[data-block-el="articleTeaser"] > article' + title: + selector: "h2" + prefixed_title: + selector: "h2" + url: + selector: "section > a" + extractor: "href" + description: + selector: ".leading-loose" + published_at: + selector: "footer" + extractor: text + post_process: + - name: substring + start: 0 + end: -5 + - name: parse_time diff --git a/app/registries/seed/official/configs/spotify.com/newsroom.yml b/app/registries/seed/official/configs/spotify.com/newsroom.yml new file mode 100644 index 000000000..b9ef9fbc6 --- /dev/null +++ b/app/registries/seed/official/configs/spotify.com/newsroom.yml @@ -0,0 +1,24 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: spotify.com/newsroom +directory: + topics: + - entertainment + title: "Spotify — Newsroom" + summary: "Corporate and product announcements from Spotify Newsroom." + +channel: + title: "Spotify — Newsroom" + url: https://newsroom.spotify.com/ + language: en + time_zone: UTC + ttl: 360 +selectors: + items: + selector: ".post-box.v2" + enhance: false + title: + selector: 'h3 a[href*="/20"]' + url: + selector: 'h3 a[href*="/20"]' + extractor: href diff --git a/app/registries/seed/official/configs/stackoverflow.com/hot_network_questions.yml b/app/registries/seed/official/configs/stackoverflow.com/hot_network_questions.yml new file mode 100644 index 000000000..2b031bb29 --- /dev/null +++ b/app/registries/seed/official/configs/stackoverflow.com/hot_network_questions.yml @@ -0,0 +1,35 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: stackoverflow.com/hot_network_questions +directory: + topics: + - tech + title: "Stack Overflow — Hot Network Questions" + summary: "Trending developer questions from the Stack Overflow network." +strategy: botasaurus + +channel: + language: en + title: "Stack Overflow — Hot Network Questions" + url: https://stackoverflow.com/questions + time_zone: America/New_York + ttl: 30 +request: + botasaurus: + wait_timeout_seconds: 20 + wait_for_selector: "#hot-network-questions" +selectors: + items: + selector: "#hot-network-questions ul > li" + enhance: false + title: + selector: a + url: + selector: a + extractor: href + network: + selector: ".favicon" + extractor: attribute + attribute: title + categories: + - network diff --git a/app/registries/seed/official/configs/steuerzahler.de/news.yml b/app/registries/seed/official/configs/steuerzahler.de/news.yml new file mode 100644 index 000000000..07443d874 --- /dev/null +++ b/app/registries/seed/official/configs/steuerzahler.de/news.yml @@ -0,0 +1,29 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: steuerzahler.de/news +directory: + topics: + - civic + title: "Bund der Steuerzahler — News" + summary: "Nachrichten vom Bund der Steuerzahler Deutschland." +channel: + title: "Bund der Steuerzahler — News" + url: https://www.steuerzahler.de/news/ + time_zone: Europe/Berlin + ttl: 720 + language: de +selectors: + items: + selector: ".bdst_presslist .bdst_presslist__block" + enhance: false + title: + selector: "a" + url: + selector: "a" + extractor: "href" + description: + selector: ".bdst_presslist__block__teaser" + published_at: + selector: ".bdst_presslist__block__date" + post_process: + - name: "parse_time" diff --git a/app/registries/seed/official/configs/stripes.com/index.yml b/app/registries/seed/official/configs/stripes.com/index.yml new file mode 100644 index 000000000..fae6cef82 --- /dev/null +++ b/app/registries/seed/official/configs/stripes.com/index.yml @@ -0,0 +1,27 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +--- +registry: + id: stripes.com/index +directory: + topics: + - news + title: "Stars and Stripes — Front page" + summary: "Independent news for the U.S. military community from Stars and Stripes." +channel: + title: "Stars and Stripes — Front page" + url: https://www.stripes.com/ + language: en + ttl: 360 + time_zone: America/New_York +selectors: + items: + selector: '[class$="teaser"]' + title: + selector: a + url: + selector: a + extractor: href + description: + selector: img + extractor: attribute + attribute: alt diff --git a/app/registries/seed/official/configs/support.apple.com/en_gb_ht201222.yml b/app/registries/seed/official/configs/support.apple.com/en_gb_ht201222.yml new file mode 100644 index 000000000..f97e60f60 --- /dev/null +++ b/app/registries/seed/official/configs/support.apple.com/en_gb_ht201222.yml @@ -0,0 +1,37 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +--- +registry: + id: support.apple.com/en_gb_ht201222 +directory: + topics: + - tech + - security + title: "Apple Support — Security releases" + summary: "Apple security update and release notes (HT201222 / related)." +strategy: botasaurus + +channel: + title: "Apple Support — Security releases" + url: https://support.apple.com/en-gb/100100 + language: en + ttl: 360 + time_zone: UTC +request: + botasaurus: + wait_for_selector: ".table-wrapper table tbody tr a" + wait_timeout_seconds: 20 +selectors: + items: + selector: ".table-wrapper table tbody > tr:not(:first-child)" + enhance: false + title: + selector: a + url: + selector: a + extractor: href + description: + selector: "td:nth-child(2)" + published_at: + selector: "td:nth-child(3)" + post_process: + - name: parse_time diff --git a/app/registries/seed/official/configs/support.apple.com/exchange_repair.yml b/app/registries/seed/official/configs/support.apple.com/exchange_repair.yml new file mode 100644 index 000000000..5321603f4 --- /dev/null +++ b/app/registries/seed/official/configs/support.apple.com/exchange_repair.yml @@ -0,0 +1,34 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: support.apple.com/exchange_repair +directory: + topics: + - tech + - security + title: "Apple Support — Service programs" + summary: "Apple repair and exchange service program announcements." +channel: + language: en + title: "Apple Support — Service programs" + url: https://support.apple.com/service-programs + time_zone: America/Los_Angeles + ttl: 720 +selectors: + items: + selector: ".as-columns--2up-extended" + enhance: false + title: + selector: "a" + url: + selector: "a" + extractor: "href" + img: + selector: "img" + extractor: "attribute" + attribute: "src" + description: + post_process: + - name: template + string: | + # ![](%{img}) + - name: markdown_to_html diff --git a/app/registries/seed/official/configs/tap.info.tn/politics.yml b/app/registries/seed/official/configs/tap.info.tn/politics.yml new file mode 100644 index 000000000..c460912c6 --- /dev/null +++ b/app/registries/seed/official/configs/tap.info.tn/politics.yml @@ -0,0 +1,29 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: tap.info.tn/politics +directory: + topics: + - news + - civic + title: "TAP — Politics" + summary: "Political news from Tunisia’s official news agency (TAP)." +strategy: botasaurus + +channel: + title: "TAP — Politics" + url: https://www.tap.info.tn/en + language: en + time_zone: Africa/Tunis + ttl: 360 +request: + botasaurus: + wait_for_selector: "td.NewsItemHeadline a" + wait_timeout_seconds: 20 +selectors: + items: + selector: "td.NewsItemHeadline a" + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/telenor.com/newsroom.yml b/app/registries/seed/official/configs/telenor.com/newsroom.yml new file mode 100644 index 000000000..030b8ea30 --- /dev/null +++ b/app/registries/seed/official/configs/telenor.com/newsroom.yml @@ -0,0 +1,22 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: telenor.com/newsroom +directory: + topics: + - tech + title: "Telenor — Newsroom" + summary: "Corporate newsroom updates from Telenor." +channel: + title: "Telenor — Newsroom" + url: https://www.telenor.com/media/newsroom/ + language: en + time_zone: Europe/Oslo + ttl: 360 +selectors: + items: + selector: 'a.link-wrap.teaser-wrap[href*="/media/newsroom/"]' + enhance: false + title: + selector: h2 + url: + extractor: href diff --git a/app/registries/seed/official/configs/teneriffa-news.com/news.yml b/app/registries/seed/official/configs/teneriffa-news.com/news.yml new file mode 100644 index 000000000..0ff7ec08a --- /dev/null +++ b/app/registries/seed/official/configs/teneriffa-news.com/news.yml @@ -0,0 +1,36 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: teneriffa-news.com/news +directory: + topics: + - news + - travel + title: "Teneriffa News — News" + summary: "Deutschsprachige Nachrichten von Teneriffa und den Kanaren." +channel: + title: "Teneriffa News — News" + url: https://www.teneriffa-news.com/news + time_zone: "Europe/Lisbon" + ttl: 720 + language: de +selectors: + items: + selector: ".main-content article" + title: + selector: "h2" + url: + selector: "a" + extractor: "href" + img_url: + selector: "img" + extractor: "attribute" + attribute: "data-lazy-src" + description: + selector: ".excerpt" + post_process: + - name: template + string: | + ![](%{img_url}) + + %{self} + - name: markdown_to_html diff --git a/app/registries/seed/official/configs/tennet.eu/news.yml b/app/registries/seed/official/configs/tennet.eu/news.yml new file mode 100644 index 000000000..99df43cc6 --- /dev/null +++ b/app/registries/seed/official/configs/tennet.eu/news.yml @@ -0,0 +1,22 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: tennet.eu/news +directory: + topics: + - energy + title: "TenneT — News" + summary: "Grid and energy news from transmission operator TenneT." +channel: + title: "TenneT — News" + url: https://www.tennet.eu/news + language: en + time_zone: Europe/Amsterdam + ttl: 360 +selectors: + items: + selector: 'a[href^="/news/"]' + enhance: false + title: + selector: "h3, h4" + url: + extractor: href diff --git a/app/registries/seed/official/configs/test.de/archiv.yml b/app/registries/seed/official/configs/test.de/archiv.yml new file mode 100644 index 000000000..e95ad41fe --- /dev/null +++ b/app/registries/seed/official/configs/test.de/archiv.yml @@ -0,0 +1,25 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +--- +registry: + id: test.de/archiv +directory: + topics: + - consumer + title: "Stiftung Warentest — Archiv" + summary: "Archivierte Tests und Meldungen von Stiftung Warentest." +channel: + title: "Stiftung Warentest — Archiv" + url: https://www.test.de/archiv/ + language: de + ttl: 360 + time_zone: Europe/Berlin +selectors: + items: + selector: ".archive__list > li" + title: + selector: a + url: + selector: a:first + extractor: href + description: + selector: p diff --git a/app/registries/seed/official/configs/theeastafrican.co.ke/news.yml b/app/registries/seed/official/configs/theeastafrican.co.ke/news.yml new file mode 100644 index 000000000..db9e51871 --- /dev/null +++ b/app/registries/seed/official/configs/theeastafrican.co.ke/news.yml @@ -0,0 +1,29 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: theeastafrican.co.ke/news +directory: + topics: + - news + title: "The East African — News" + summary: "Regional news from The East African." +strategy: botasaurus + +channel: + title: "The East African — News" + url: https://www.theeastafrican.co.ke/tea/news + language: en + time_zone: Africa/Nairobi + ttl: 360 +request: + botasaurus: + wait_for_selector: 'a[href*="/tea/news/"][aria-label]' + wait_timeout_seconds: 20 +selectors: + items: + selector: 'a[href*="/tea/news/"][aria-label]' + enhance: false + title: + extractor: attribute + attribute: aria-label + url: + extractor: href diff --git a/app/registries/seed/official/configs/theguardian.com/international_mostpopular.yml b/app/registries/seed/official/configs/theguardian.com/international_mostpopular.yml new file mode 100644 index 000000000..333b3ea1b --- /dev/null +++ b/app/registries/seed/official/configs/theguardian.com/international_mostpopular.yml @@ -0,0 +1,22 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: theguardian.com/international_mostpopular +directory: + topics: + - news + title: "The Guardian — International most popular" + summary: "Most-read international stories from The Guardian." +channel: + language: en + title: "The Guardian — International most popular" + url: https://www.theguardian.com/international + time_zone: Europe/London + ttl: 60 +selectors: + items: + selector: '[data-link-name="most-viewed"] > ol > li' + title: + selector: "h4 span" + url: + selector: "a" + extractor: "href" diff --git a/app/registries/seed/official/configs/thenationalnews.com/news.yml b/app/registries/seed/official/configs/thenationalnews.com/news.yml new file mode 100644 index 000000000..911d873a6 --- /dev/null +++ b/app/registries/seed/official/configs/thenationalnews.com/news.yml @@ -0,0 +1,22 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: thenationalnews.com/news +directory: + topics: + - news + title: "The National — News" + summary: "News from The National (UAE)." +channel: + title: "The National — News" + url: https://www.thenationalnews.com/news/ + language: en + time_zone: Asia/Dubai + ttl: 360 +selectors: + items: + selector: 'a[href*="/news/"][href*="/202"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/thoughtworks.com/insights.yml b/app/registries/seed/official/configs/thoughtworks.com/insights.yml new file mode 100644 index 000000000..85ac32315 --- /dev/null +++ b/app/registries/seed/official/configs/thoughtworks.com/insights.yml @@ -0,0 +1,29 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: thoughtworks.com/insights +directory: + topics: + - tech + - research + title: "Thoughtworks — Insights" + summary: "Technology and business insights from the Thoughtworks blog." +strategy: botasaurus + +channel: + title: "Thoughtworks — Insights" + url: https://www.thoughtworks.com/insights/blog + language: en + ttl: 360 + time_zone: UTC +request: + botasaurus: + wait_timeout_seconds: 20 + wait_for_selector: 'a[href*="/insights/blog/"]' +selectors: + items: + selector: 'a[href*="/insights/blog/"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/tno.nl/news.yml b/app/registries/seed/official/configs/tno.nl/news.yml new file mode 100644 index 000000000..232b56ef4 --- /dev/null +++ b/app/registries/seed/official/configs/tno.nl/news.yml @@ -0,0 +1,24 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: tno.nl/news +directory: + topics: + - research + - tech + title: "TNO — News" + summary: "Research and innovation news from TNO in the Netherlands." +channel: + title: "TNO — News" + url: https://www.tno.nl/en/newsroom/news/ + language: en + time_zone: Europe/Amsterdam + ttl: 360 +selectors: + items: + selector: div.collage-card-content + enhance: false + title: + selector: h3 + url: + selector: 'a[href*="/en/newsroom/20"]' + extractor: href diff --git a/app/registries/seed/official/configs/tourismusnetzwerk-brandenburg.de/aktuelle_nachrichten.yml b/app/registries/seed/official/configs/tourismusnetzwerk-brandenburg.de/aktuelle_nachrichten.yml new file mode 100644 index 000000000..0d383e873 --- /dev/null +++ b/app/registries/seed/official/configs/tourismusnetzwerk-brandenburg.de/aktuelle_nachrichten.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: tourismusnetzwerk-brandenburg.de/aktuelle_nachrichten +directory: + topics: + - travel + title: "Tourismusnetzwerk Brandenburg — Aktuelle Nachrichten" + summary: "Aktuelle Nachrichten für den Tourismus in Brandenburg." +channel: + title: "Tourismusnetzwerk Brandenburg — Aktuelle Nachrichten" + url: https://tourismusnetzwerk-brandenburg.de/ + time_zone: Europe/Berlin + ttl: 720 + language: de +enhance: false +selectors: + items: + selector: "article.node.article.wall-floating" + title: + selector: "h3.title a[rel='bookmark']" + url: + selector: "h3.title a[rel='bookmark']" + extractor: "href" diff --git a/app/registries/seed/official/configs/tralac.org/news.yml b/app/registries/seed/official/configs/tralac.org/news.yml new file mode 100644 index 000000000..0a1dbc09d --- /dev/null +++ b/app/registries/seed/official/configs/tralac.org/news.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: tralac.org/news +directory: + topics: + - civic + - research + title: "tralac — News" + summary: "Trade law and policy news from the Trade Law Centre (tralac)." +channel: + title: "tralac — News" + url: https://www.tralac.org/news.html + language: en + time_zone: Africa/Johannesburg + ttl: 360 +selectors: + items: + selector: 'h3 a[href*="/news/article/"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/tudelft.nl/news.yml b/app/registries/seed/official/configs/tudelft.nl/news.yml new file mode 100644 index 000000000..d0786ce32 --- /dev/null +++ b/app/registries/seed/official/configs/tudelft.nl/news.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: tudelft.nl/news +directory: + topics: + - science + - research + title: "TU Delft — News" + summary: "University and research news from Delft University of Technology." +channel: + title: "TU Delft — News" + url: https://www.tudelft.nl/en/about-tu-delft/current + language: en + time_zone: Europe/Amsterdam + ttl: 360 +selectors: + items: + selector: 'a.card[href*="/en/20"]' + enhance: true + title: + selector: h3 + url: + extractor: href diff --git a/app/registries/seed/official/configs/tue.nl/news.yml b/app/registries/seed/official/configs/tue.nl/news.yml new file mode 100644 index 000000000..d5cecc03f --- /dev/null +++ b/app/registries/seed/official/configs/tue.nl/news.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: tue.nl/news +directory: + topics: + - science + - research + title: "TU Eindhoven — News" + summary: "University and research news from Eindhoven University of Technology." +channel: + title: "TU Eindhoven — News" + url: https://www.tue.nl/en/news-and-events/news-overview + language: en + time_zone: Europe/Amsterdam + ttl: 360 +selectors: + items: + selector: 'a.entryBlock[href*="/en/news-and-events/news-overview/"]' + enhance: true + title: + selector: h3.entryBlock-title + url: + extractor: href diff --git a/app/registries/seed/official/configs/vattenfall.com/newsroom.yml b/app/registries/seed/official/configs/vattenfall.com/newsroom.yml new file mode 100644 index 000000000..9bcfbf559 --- /dev/null +++ b/app/registries/seed/official/configs/vattenfall.com/newsroom.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: vattenfall.com/newsroom +directory: + topics: + - energy + - environment + title: "Vattenfall — Newsroom" + summary: "Corporate and energy news from Vattenfall." +channel: + title: "Vattenfall — Newsroom" + url: https://group.vattenfall.com/press-and-media/newsroom + language: en + time_zone: Europe/Stockholm + ttl: 360 +selectors: + items: + selector: 'a.main-link[href*="/press-and-media/newsroom/20"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/visitfinland.com/press-releases.yml b/app/registries/seed/official/configs/visitfinland.com/press-releases.yml new file mode 100644 index 000000000..147788909 --- /dev/null +++ b/app/registries/seed/official/configs/visitfinland.com/press-releases.yml @@ -0,0 +1,27 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: visitfinland.com/press-releases +directory: + topics: + - travel + title: "Visit Finland — Press releases" + summary: "Press releases from Visit Finland’s media centre." +strategy: botasaurus +channel: + title: "Visit Finland — Press releases" + url: https://media.visitfinland.com/en/media-press-releases/ + language: en + time_zone: Europe/Helsinki + ttl: 360 +request: + botasaurus: + wait_for_selector: 'a[href*="/en/media-press-releases/"]' + wait_timeout_seconds: 20 +selectors: + items: + selector: 'a.PageCard[href*="/en/media-press-releases/"]' + enhance: true + title: + selector: div.h5 + url: + extractor: href diff --git a/app/registries/seed/official/configs/webentwickler-jobs.de/in.yml b/app/registries/seed/official/configs/webentwickler-jobs.de/in.yml new file mode 100644 index 000000000..cc35d7f46 --- /dev/null +++ b/app/registries/seed/official/configs/webentwickler-jobs.de/in.yml @@ -0,0 +1,31 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +--- +registry: + id: webentwickler-jobs.de/in +directory: + topics: + - jobs + title: "Webentwickler-Jobs — Region" + summary: "Webentwickler-Stellenangebote für eine wählbare Region in Deutschland." +parameters: + region: + type: string + default: "berlin" + +channel: + title: "Webentwickler-Jobs — Region" + url: https://www.webentwickler-jobs.de/in/%s + language: de + ttl: 360 + time_zone: Europe/Berlin +selectors: + items: + selector: ".job-item" + title: + selector: ".job-title" + url: + selector: a:first + extractor: href + description: + selector: "> div" + extractor: html diff --git a/app/registries/seed/official/configs/who.int/news.yml b/app/registries/seed/official/configs/who.int/news.yml new file mode 100644 index 000000000..c100d5204 --- /dev/null +++ b/app/registries/seed/official/configs/who.int/news.yml @@ -0,0 +1,30 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: who.int/news +directory: + topics: + - science + - civic + title: "WHO — News" + summary: "Official news and statements from the World Health Organization." +strategy: botasaurus + +channel: + title: "WHO — News" + url: https://www.who.int/news + language: en + time_zone: UTC + ttl: 360 +request: + botasaurus: + wait_for_selector: 'a.link-container[href*="/news/item/"]' + wait_timeout_seconds: 20 +selectors: + items: + selector: 'a.link-container[href*="/news/item/"]' + enhance: false + title: + extractor: attribute + attribute: aria-label + url: + extractor: href diff --git a/app/registries/seed/official/configs/worldbank.org/news.yml b/app/registries/seed/official/configs/worldbank.org/news.yml new file mode 100644 index 000000000..b183bd718 --- /dev/null +++ b/app/registries/seed/official/configs/worldbank.org/news.yml @@ -0,0 +1,29 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: worldbank.org/news +directory: + topics: + - finance + - civic + title: "World Bank — News" + summary: "Development and policy news from the World Bank." +strategy: botasaurus + +channel: + title: "World Bank — News" + url: https://www.worldbank.org/ext/en/news + language: en + time_zone: UTC + ttl: 360 +request: + botasaurus: + wait_for_selector: 'a.button[href*="/news/press-release/"]' + wait_timeout_seconds: 20 +selectors: + items: + selector: 'a.button[href*="/news/press-release/"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/www.consilium.europa.eu/press-releases.yml b/app/registries/seed/official/configs/www.consilium.europa.eu/press-releases.yml new file mode 100644 index 000000000..b458bd2a8 --- /dev/null +++ b/app/registries/seed/official/configs/www.consilium.europa.eu/press-releases.yml @@ -0,0 +1,28 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: consilium.europa.eu/press-releases +directory: + topics: + - civic + title: "Council of the EU — Press releases" + summary: "Press releases from the Council of the European Union." +strategy: botasaurus + +channel: + title: "Council of the EU — Press releases" + url: https://www.consilium.europa.eu/en/press/press-releases/ + language: en + time_zone: UTC + ttl: 360 +request: + botasaurus: + wait_for_selector: 'a.gsc-excerpt-item__link[href*="/press/press-releases/"]' + wait_timeout_seconds: 20 +selectors: + items: + selector: 'a.gsc-excerpt-item__link[href*="/en/press/press-releases/20"]' + enhance: false + title: + selector: span.gsc-excerpt-item__title + url: + extractor: href diff --git a/app/registries/seed/official/configs/www.ecdc.europa.eu/news-events.yml b/app/registries/seed/official/configs/www.ecdc.europa.eu/news-events.yml new file mode 100644 index 000000000..0835868f3 --- /dev/null +++ b/app/registries/seed/official/configs/www.ecdc.europa.eu/news-events.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: ecdc.europa.eu/news-events +directory: + topics: + - science + - civic + title: "ECDC — News and events" + summary: "News and events from the European Centre for Disease Prevention and Control." +channel: + title: "ECDC — News and events" + url: https://www.ecdc.europa.eu/en/news-events + language: en + time_zone: UTC + ttl: 360 +selectors: + items: + selector: 'a[href^="/en/news-events/"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/www.enisa.europa.eu/news.yml b/app/registries/seed/official/configs/www.enisa.europa.eu/news.yml new file mode 100644 index 000000000..228824c9e --- /dev/null +++ b/app/registries/seed/official/configs/www.enisa.europa.eu/news.yml @@ -0,0 +1,23 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: enisa.europa.eu/news +directory: + topics: + - security + - civic + title: "ENISA — News" + summary: "Cybersecurity news from the EU Agency for Cybersecurity (ENISA)." +channel: + title: "ENISA — News" + url: https://www.enisa.europa.eu/news + language: en + time_zone: UTC + ttl: 360 +selectors: + items: + selector: 'h3 a[href^="/news/"]' + enhance: false + title: + extractor: text + url: + extractor: href diff --git a/app/registries/seed/official/configs/www.europarl.europa.eu/press-room.yml b/app/registries/seed/official/configs/www.europarl.europa.eu/press-room.yml new file mode 100644 index 000000000..5da553bb9 --- /dev/null +++ b/app/registries/seed/official/configs/www.europarl.europa.eu/press-room.yml @@ -0,0 +1,28 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: europarl.europa.eu/press-room +directory: + topics: + - civic + title: "European Parliament — Press room" + summary: "Press room releases from the European Parliament." +strategy: botasaurus + +channel: + title: "European Parliament — Press room" + url: https://www.europarl.europa.eu/news/en/press-room + language: en + time_zone: UTC + ttl: 360 +request: + botasaurus: + wait_for_selector: 'a[href*="/news/en/press-room/"]' + wait_timeout_seconds: 20 +selectors: + items: + selector: 'a[href*="/news/en/press-room/"][title="Read more"]' + enhance: false + title: + selector: span.ep_name + url: + extractor: href diff --git a/app/registries/seed/official/configs/www.gob.mx/profeco-prensa.yml b/app/registries/seed/official/configs/www.gob.mx/profeco-prensa.yml new file mode 100644 index 000000000..79fec1482 --- /dev/null +++ b/app/registries/seed/official/configs/www.gob.mx/profeco-prensa.yml @@ -0,0 +1,24 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json +registry: + id: gob.mx/profeco-prensa +directory: + topics: + - consumer + - civic + title: "PROFECO — Prensa" + summary: "Comunicados de prensa de la Procuraduría Federal del Consumidor (México)." +channel: + title: "PROFECO — Prensa" + url: https://www.gob.mx/profeco/es/archivo/prensa + language: es + time_zone: America/Mexico_City + ttl: 360 +selectors: + items: + selector: article + enhance: false + title: + selector: h2 + url: + selector: 'a.small-link[href*="/profeco/prensa/"]' + extractor: href diff --git a/app/registries/seed/official/manifest.json b/app/registries/seed/official/manifest.json index 0bdd6b877..89a4d5ca3 100644 --- a/app/registries/seed/official/manifest.json +++ b/app/registries/seed/official/manifest.json @@ -1,11 +1,152 @@ { "format": "registry.v1", "registry_id": "official", - "version": "v2026.08.23", - "public_key_id": "test", + "version": "seed", + "public_key_id": "html2rss:registry:2026", "files": { + "configs/aalto.fi/news.yml": "be6418d7c29df5e6ec72d2768ba7fbb22bb97433fb293fd1a727eb4a5fd98236", + "configs/academictransfer.com/jobs.yml": "c17b3bfcb250604ffde4c7ad7c1cdf1fa1da01f862b20ec39354d1c6d902a954", + "configs/adfc.de/pressemitteilungen.yml": "ed07d0c8b2b57a18a66476aba923ac426a391da75c05eacb323b8f773cff5dde", + "configs/ahram.org.eg/egypt.yml": "73d5f85c86709804b73f375380382642debb45aaea7b43ea5f129fe00db5c6d1", + "configs/ai.meta.com/blog.yml": "1d872da4cb5ec904925bbbe2afc45207a72977292ff6d3dcb6e79e8f4f138137", + "configs/akorda.kz/events.yml": "87e61b20a5456635ec64c66c1aa0f3fc9f78c540a7aabd70738354ce2ea77285", + "configs/angop.ao/noticias.yml": "9304dc10fab78a5fe98067786dd8bbe3afe57c691d4930b88689628ed4bf44f4", "configs/anthropic.com/news.yml": "679b3c28b6dd7e08bbfdaf8564e077e7c2ac4140eacbc9543756591aaf0621f5", - "configs/deepmind.google/blog.yml": "3dbc30f109bb169f1de06372236a2d352584808d11be8243ba93befcd94a7b41" + "configs/anu.edu.au/news.yml": "b9df2c362037a607c3c551503641411c042bed6151244dd8768db09e40289c0f", + "configs/apnews.com/hub.yml": "e891105a39e0be66211591e111ea1c20d31c6cd3bccd8d6304a535a6a28611fc", + "configs/apple.com/newsroom.yml": "87b705ce943aba697572e2ba3ca7ba847aa37688b09f8773af7ab6dfcfbbe8e6", + "configs/aps.dz/national-news.yml": "fd37cf647eeab34d579b3fa676fc4226d17d79421755592d33d625bac36cb972", + "configs/asml.com/press-releases.yml": "e08a7462c4a996c96300617ade224f0c89f6f95a6225005894c6ba5ee5702747", + "configs/aspi.org.au/strategist.yml": "d18a20ce40c71565ec8bca95f1bedd0e6993b6ee7376e4ca8817694d182c4752", + "configs/bbc.co.uk/available_episodes.yml": "447938f86901784ce8e822534fcd4578893aa52cd0796b872d58acb2c7b946e1", + "configs/bbc.com/mundo.yml": "9c240029d3764f879e0e287683a8fea055d94c7773f5bc3b8b79d1c93ac06cfd", + "configs/blog.mondediplo.net/feed.yml": "945b7d2018661eb5e107d4b108ecbbe0721cc6f6d196f1cb0fae669fea941310", + "configs/brookings.edu/articles.yml": "6ebaf4509a0f35ca7b4f33d0dcdea0217afc3535c5df50f446e9e3780446a18b", + "configs/bundesregierung.de/pressemitteilungen.yml": "855fc75ec8109525a064e3cb3941eca01ee4528bd8449949a11b1b8e98c65580", + "configs/canarianweekly.com/front.yml": "eb75b1c599bf30a2f8ac459bfc9b6e5a0467bdc60898c3edb287b8f913d06216", + "configs/caricom.org/news.yml": "b3a72ebf0f668295ce6866e8b27b6e0a72fea93c6f3634c770a6d95d8228a975", + "configs/carnegieendowment.org/research.yml": "4f353f70fa0a3cc025043495c3334364ebba41d541d1bd0e466eccff0ecd3ae8", + "configs/cepal.org/press-releases.yml": "47d60aafae23b97053a232feda8389cd35dd14b352a9ae43b3ee3da7437e6769", + "configs/chalmers.se/news.yml": "cfdc39fa8a2dcdaf29ba57c89847de8942b1de547a71501c4027b2af602721a7", + "configs/chathamhouse.org/research-publications.yml": "7e9268663fd1b3ed6e6b7484ba357fe091a192771c163f4af52bb91139e0d449", + "configs/cinemascore.com/index.yml": "abcfb21c9fff41106f5f7d7459393dd82662ea8f0f616897da4303f425ec1b9f", + "configs/cleanenergywire.org/news.yml": "df124817ca29345f4c03a39ba6c7af99cebc7ca0df3b43f66aa701960f53367f", + "configs/cnet.com/section_sub.yml": "f38ec616700966b921ee5adf77d584c0470fd04fdbf66ee20e379b31f2778feb", + "configs/cohere.com/blog.yml": "9b5be18602058c9b3e193c1d7c27a373b90d5771ad618aa249ac0d5fbb2d4bef", + "configs/computerbase.de/meistgelesen.yml": "549a8840eb5cc1998240f50fb5d00987b9bf34d708a1b5e9a59ceea8b78f5c13", + "configs/cplp.org/noticias.yml": "54fab81b4c1f1801baefa4b7af01af635b8bbca30767a4461467f022f1b34552", + "configs/csiro.au/news.yml": "bd16a57edc5fc0cd697b93a5c431ccec7dda0df845e58191b1cff2aaab84166a", + "configs/curia.europa.eu/press-releases.yml": "b23e6d984da5a3fd77151e7f647b260d226800b1315da630482ff936bf044dc6", + "configs/cutle.fish/index.yml": "04efa9acb4c9765d71df0411a623b9345f40649bedbca24b88ad3b441ac68cdd", + "configs/deepmind.google/blog.yml": "3dbc30f109bb169f1de06372236a2d352584808d11be8243ba93befcd94a7b41", + "configs/deraktionaer.de/meistgelesen.yml": "da9e238088745250f02604f98f5d3b85fc2e63b5c572546244dc386ebd512d44", + "configs/developer.apple.com/tutorials_data_documentation_technotes_json.yml": "60a974d2fab2e0c67bd9a948188a3972f15bec344dd8919349edcb05119ca012", + "configs/dfs.de/pressemitteilungen.yml": "6a3e8d83e0370812eb7b90fcbb659cc9356f284ecb66b9c35c684a2f4dd0ea36", + "configs/diis.dk/research.yml": "ce08a485afdcef6faa9a885e6fbf3e236fba89c64319c68c031a784bd1864fce", + "configs/dirco.gov.za/media-statements.yml": "42c6db2eed67544a27bf4e545ae9dc878a6b079373f12d2ff0d567a6a7939e34", + "configs/dsw-info.de/presse.yml": "eacac63e3d09a7f0748ef5e73ee5dfc26c18fa7af0a03d41483c20ea8328018b", + "configs/elastic.co/elasticsearch-release-notes.yml": "62c3f341563e9d44421639d7b6e54d0cb9eda7396995e3e14b4ea4efe762a810", + "configs/elysee.fr/actualites.yml": "e3958aa4d1db3214d4ce8a296d0f6d7ee3bd20c58953de37d3a3b91037532324", + "configs/equinor.com/news.yml": "7b9bbd0a2d0b8fcadf6ae311916d248a6f6590973725c57b8ffc3ce75c02faee", + "configs/espn.com/f1.yml": "3b98cc8ec8dcb164856a45490c0a99e0c206c8d6924c07dac1823d5630a19492", + "configs/ethz.ch/eth-news.yml": "a8a8f5e54203f4db8c11f64b990e13d7672079f89e5eb0984118610868834e6b", + "configs/eures.europa.eu/news.yml": "3e65bb8ee673cefb8476bb6bd68b79da835f6c3075a3c03e3f092e358e710bb2", + "configs/faa.gov/newsroom.yml": "6df127a4ac09a58dc03dad95769b34346bb55d194c124164291288e9d3d62f64", + "configs/fao.org/newsroom.yml": "2c481358fb0ee6515b4b7874b3b99f6599482f43475ae70915893ffd097e0a11", + "configs/fia.com/documents.yml": "42b6d296f1cdfcfef101e3edadcf1a579e11d38c3a9822a9fdfda55a8215125b", + "configs/figma.com/blog.yml": "f8e2013caa5ae449853652dc67ce177c1b9839fe469b27ca7d4740c50cc45c3e", + "configs/formula1.com/latest.yml": "9ac90ef1a0653e3dd8b09c721a4236dd7ec149a4a278a3151849909723f417be", + "configs/forumsec.org/publications.yml": "5cf0a07fa1fc6a3399a806076619aaf911ef0ad8e99081eef8832dd3f883f172", + "configs/ftc.gov/press-releases.yml": "fc0bef1c92a2d9309d058db4c1df907c6abc27999a1350812c1d9e5a2558d084", + "configs/github.com/releases.yml": "2071e5379acabd786750092b8d57f8ab87bc7c21627f77f5eeecc751b1e6f244", + "configs/go.dev/release-history.yml": "c104d8d68b5df8924b42bc1723656afc004cf808d6fe10301adc38d671483370", + "configs/grafana.com/whatsnew.yml": "6c9570538a07390c16dab99b2f1c848b4dd02125286df0964b8a85370a27a7f2", + "configs/haaretz.com/israel-news.yml": "9221cced7787014139b7f6a74315dc274294845179e86bf3d30761542bf5eaff", + "configs/hai.stanford.edu/news.yml": "e91bb67d9a948a4ae054006ba0b748cd46ec6389d46e9c138aad6224af1e5e27", + "configs/iaapa.org/news.yml": "b93b9cc1701f2a06c6d85a97315a1f0bdbd5c0a749c008f8923421730eae8041", + "configs/icj-cij.org/press-releases.yml": "40c4cd883dc6495fb56b2ccbceff0f8cf62fccd6b767ce466463e6b939709423", + "configs/icrc.org/news.yml": "cf2efef0b9b6082af1fea9d8b6fe1a24bebe92aa4070ddc3bc71aa02c60f94f5", + "configs/iea.org/news.yml": "364a230c750b26a2ad25d5e9e1d576516d6103f60ebf588813dadba7316edd6d", + "configs/iiss.org/online-analysis.yml": "e1c3a83a6d6d4e2e7cb3b2ad47329bd56a66dc31883d9f370c870932044e1e01", + "configs/ikea.com/newsroom.yml": "d80881b7ab88d7038da48e17d95a6268cc9a0d4af4189574cab09549d1ef3ecb", + "configs/imdb.com/ratings.yml": "152fe7418221ad0cc6cf38fd6f21a8dfc8575a80570c5eb9e3fe1b6c8959ddf8", + "configs/imo.org/press-briefings.yml": "4bd88ae16ee124ad285fc4cdaff4ee121c69b90215d77995a15cbc8b24f6085f", + "configs/imperial.ac.uk/news.yml": "e84228d9ef1d2909cd911c5bb3f377a9b049be0daf6621dd1fa0b35cdfcaf346", + "configs/ingenieur.de/karriere_arbeitsleben_heiko_mell.yml": "e2b5bcbe13c7fb651d66d4f4cbb107eedffaa1f9b8adf905ce17c53ea0ecc7d8", + "configs/isro.gov.in/press.yml": "ad43cebc15708b0881a730ec85cbd6622ca93581f7e78494701eb09701a3ec6f", + "configs/issafrica.org/iss-today.yml": "58764d7023fbfa050ab9737cf230ad2ef52ac34aa1a23b6e32cf222fd826427a", + "configs/issafrica.org/press-releases.yml": "1bc3e7746eea235b575bcc7d34e558675da8c8a9ca0dfdfe2c725241667741b9", + "configs/kaist.ac.kr/news.yml": "af091015cb2785e32e373314280d7b228cb5435d4a0367658175b954de9b033f", + "configs/kinocheck.de/filmstarts.yml": "7f1791684ef623fca5a4e69154ccd23a70e615b87ac1ff85aec2263ca274b895", + "configs/kompas.com/nasional.yml": "cbf4bec2d0174ca6c0e387795e5d87fd765cecf378e5a3937c469f559b3dad54", + "configs/lego.com/press-releases.yml": "563c6259b56b94c5056f6dae62cc886b6886267b751212111e6c7ad4896db5e9", + "configs/machinelearning.apple.com/research.yml": "2e79eeb0c7b82400305f5a914e0094635840e0973b25b69f16f882b730ced1ab", + "configs/maersk.com/press-releases.yml": "a8c4e71bd0636b0c672602521bea38002d629c2061431b205b1d0cf704f8e19a", + "configs/microsoft.com/azure-products.yml": "1231e44f9f8f75264f085a571123135a4232707c7d76c7d8f47fc4d849c2fe47", + "configs/mistral.ai/news.yml": "4558d7a403be7a5b10423ac49bd2a5827260412f744cd6fd10be246f0fdb578a", + "configs/mozilla.org/security-advisories.yml": "8a38a705a77b2bf62965b7beca737e6ade25dce312b0d1d91eef81caf881ed43", + "configs/msf.org/latest.yml": "18f6c66a34e568c1504f6523a7a1251f66a0567896406568ffef95b650e125a7", + "configs/nbim.no/press-releases.yml": "9d6b161885272f082a7353bcbcf28ddd6a63b1ed5fbd01569cccc78953e4478e", + "configs/news.cn/latest.yml": "f5e07692a521afe6dc1bf1488e72843eb44906d5b27fb466aa57de45ad51de43", + "configs/news24.com/top-stories.yml": "921a2e0b90454bf02748f32d6cdb9e7d12f2180ca2fbbec8034d64292fd03db8", + "configs/newyorker.com/magazine.yml": "6f24906f442636d43ae9d6160abe481a249f337d34f24932535b0732b1e41b5c", + "configs/nigerianstat.gov.ng/news.yml": "2f456a944228e744355ad8ea69cbc6a0f11e534965d0f3a7636c7d9e2fe48ecd", + "configs/nikkei.com/asia-technology.yml": "494aaa988a4e4b964b219106bea9756666bad24c84e77b60621d4877bcaae739", + "configs/nokia.com/newsroom.yml": "084f94d7eb1936a58e911365485f993a2a637fbe8d8679235ce18c44ef2e75ed", + "configs/nomanssky.com/news.yml": "a7b52c620147a3b742435d7a3ea236e9e523f4c7c29cf95bb988d7a6daaca8ef", + "configs/notion.com/blog.yml": "59e873eb557d6e09d369f4fd3a64d114b57f2393a727eb08e2c95231c8e3959b", + "configs/novonordisk.com/latest-news.yml": "9ea839fdda40ff6be93600f9b482c3301cd0034359059c6c2d7957a7497fb4ef", + "configs/ntu.edu.sg/news.yml": "9b92065f45ddd7d8dad77f8d87f99824fb2cbab883b1748b6d39b12a68e82b7d", + "configs/nupi.no/news.yml": "e133d35bd1ab66b5e1aa92bf0981201881358e265dc9e6e91ec85b19955cfda1", + "configs/occrp.org/news.yml": "15cd632800e600638e0660c756ca71a91101c75b367f9a9b297223b69cbff65d", + "configs/oecd.org/newsroom.yml": "fadb3b49c5bc93da074c28b9cf2879792f823178c161afc69ce1b17a327691ef", + "configs/opec.org/press-releases.yml": "3661947bcacc5e22875f8f798b4a9d5de76d1db356ca9a860bbdc84fd2af4a5f", + "configs/panapress.com/latest.yml": "0201c9673f242d307be7c931e23645206f665b6ece059d341855dcd7ec477697", + "configs/pankow.lebensmittel-kontrollergebnisse.de/search.yml": "762016dd3beb117bb4d1fcb4e2743eab102c89ad1cfcde7fb1d60c55303438ae", + "configs/philstar.com/nation.yml": "b42907100f34f9336c6454a8211ccc4b76072a298c1df632a49696c2e9448380", + "configs/phys.org/weekly.yml": "5074931172c944b27f1154c609ebe5500f95f793ebd721b5beb11ada00e57806", + "configs/pib.gov.in/press-releases.yml": "e1ae3053c37309361a1e0400a2e899788cdbe8a3eddc3f62d4dd9459498f7fc5", + "configs/rbb24.de/meistgeklickt.yml": "f927df2c4f12a44d48695a4789bfd5f62e918a4cc541fcf7f089ff0a3b3cc2ad", + "configs/robinwood.de/aktuelles.yml": "a200b6b155cb7b42c40bb77e450293c2c2cb09a71cc57498cbe1b88c3ad9f8ea", + "configs/s3.amazonaws.com/popular_movies.yml": "8b0088efc8f41d44f00ca266652518113be0a3922e715c78aab45411686bd761", + "configs/sacu.int/news.yml": "d61870d4f143feb7c283c47ce13b1a633ef7e7901eed7466f34217f06fdd7fda", + "configs/sadc.int/latest-news.yml": "c90c74d763f40af8d66aa0ff511e56aa7a59bb7ef314da1ac9c1fd5b5027daa4", + "configs/sebastianvettel.de/news.yml": "a3ee51e82483febc2ad9a756da78cd702b88c37a9279f0e9468dd92183101010", + "configs/shopify.com/blog.yml": "89bd2a11740d6df76b2a6ff4ca5f04c6c9db2d44fe060acda85d6fe3243eaac1", + "configs/sipri.org/news.yml": "75297f03e5e9d78287549d3e7ddfa370477b58d22c3add1adbef60c12e5ecc63", + "configs/softwareleadweekly.com/issues.yml": "b97a9715185c62c6ac3aca000877805017f67deb8ba040596a79859051d21051", + "configs/solarthermalworld.org/news.yml": "9b7ceb4378d1ac6f30b5cf6788570624456054d3fbfde8659bf53ded309799bc", + "configs/spektrum.de/meistgelesen.yml": "3d881280aa5525166fdf6076a0674ac3bf306d0b1924fd91a89ed0b43fe15ed6", + "configs/spiegel.de/impressum_autor.yml": "68f698cbad5a5b228540e7eda2c12c392832110d147c7eececd2266a8796a54a", + "configs/spotify.com/newsroom.yml": "e88395077225e4c0979479c1bf875718425f155fd07c564faae417cb2f116535", + "configs/stackoverflow.com/hot_network_questions.yml": "1245ba114d85e3cf3b41ffe858d4d8f968a051a5856a5adbe0bf2622afa920a2", + "configs/steuerzahler.de/news.yml": "9ab7977a9def32e9cd0ce8cb64dae1237195b86df2726dc3232eebfb6f8b1ecc", + "configs/stripes.com/index.yml": "4c1513ed009add5e4fb6c8fb9a1f867c3ddc43c1c090ff3a27cece6da3e3c417", + "configs/support.apple.com/en_gb_ht201222.yml": "ee8f911c64cd22aa0008e909df69cf37359ab8d1c8805d44109ddafdfef56491", + "configs/support.apple.com/exchange_repair.yml": "89a01591e2319e0dad9e21f150188788dbe7610510499918ba8dfe1454d41a67", + "configs/tap.info.tn/politics.yml": "52a03af97d79e091720caafe06b1ccf3b86f79d2fc4604ae245715ca1c6449d6", + "configs/telenor.com/newsroom.yml": "ac67b05f988ad88195ad9ed5475e97c312ccaecb218d8192bce4bcb2834a11e0", + "configs/teneriffa-news.com/news.yml": "1831fd2e924235d8afaaeab511b1974ada9f28aba7114a640d6f97c2e2bc4573", + "configs/tennet.eu/news.yml": "959c51e6ef260edc9abd8ab62ea53ec722174ce98490b946d2fa1c98a5e94119", + "configs/test.de/archiv.yml": "544823eff24c249f7d165b64c84abac24192cf161a574e1b082a9021ba2268a8", + "configs/theeastafrican.co.ke/news.yml": "a77f2883db750a87ec25de38eba4188968d7cb23e1ea00c9adf51ce22faa7c02", + "configs/theguardian.com/international_mostpopular.yml": "ca4847f874faa1ed3a4519244e2bb01d8a6cd2bb8ad04cb771421dedb689d9dd", + "configs/thenationalnews.com/news.yml": "e27e19ba4b520784a83263c5a2a9653b80f36bd887ab8943776b34225ae8f91c", + "configs/thoughtworks.com/insights.yml": "0347ed1dc185c9104920c4548fd8ac71f6ead355d84b812aad469113dbf65135", + "configs/tno.nl/news.yml": "27ab648acea80a41257a18aa03dec394093369de5eba50b403d93f546231187b", + "configs/tourismusnetzwerk-brandenburg.de/aktuelle_nachrichten.yml": "b26b0ca870ed70ffd4c9ab6ad50ab6c232e80c20206cfdc32f4c26f015b6e370", + "configs/tralac.org/news.yml": "c6c3a401c7620c306034e7fdc67a3bbe5cb416927b1533948e1b76ad515bc562", + "configs/tudelft.nl/news.yml": "370761e6b2941f894efc5db0fbe8fcec4db6c8eb6030b5daf8e91853b1d001e9", + "configs/tue.nl/news.yml": "8191ffed06ea44c9f6ae3670cacb8d56d751eb6e838370694580bc93a0062ddd", + "configs/vattenfall.com/newsroom.yml": "32f8c3ae3a75ce477556e1de454440a95438d1c7e5ecd4f31eeb0dc136cf280c", + "configs/visitfinland.com/press-releases.yml": "72cdd2b796710e08dd3a5b02851c4d2c72354a53b020553c299c4ec0832efaf5", + "configs/webentwickler-jobs.de/in.yml": "ddb1a06e38ef8e03629cf62679f448d332499f08736a22aed6d66238c6ac2dfc", + "configs/who.int/news.yml": "efa19b4fe3e83b31a8227d5c5bf5cee2db2aa7e3ec90607a8a47d344561a83be", + "configs/worldbank.org/news.yml": "45e9f7005f6c3cc74fe0b02a33529a68272188ad6b56aec09ef0505c567c44c2", + "configs/www.consilium.europa.eu/press-releases.yml": "4728634af76a4a7a2b3f059e961fad7c370254324b444634afe36fb1bf03afa6", + "configs/www.ecdc.europa.eu/news-events.yml": "8a265ae15e4fc05f09c75743f3d47fee996e1f575693610ab7fb519b6a653181", + "configs/www.enisa.europa.eu/news.yml": "964cba7952e4ed510ca840d291b5c81fb90c8303f19e36b49d6aced28efbdf09", + "configs/www.europarl.europa.eu/press-room.yml": "99ab566381c60e6cebb91ebbe9d434e9153591014bee1772ec5d62b607375079", + "configs/www.gob.mx/profeco-prensa.yml": "603177b81dbe88da04e7bdac5a5186b4af288ca29b8cfc19c096fd9b5b933bfc" } } - diff --git a/app/web/boot.rb b/app/web/boot.rb index deed80ba6..ceba52f75 100644 --- a/app/web/boot.rb +++ b/app/web/boot.rb @@ -60,7 +60,10 @@ def build_loader def configure_loader(new_loader) new_loader.push_dir(app_root, namespace: Html2rss) collapsed_web_dirs.each { |path| new_loader.collapse(path) } - new_loader.inflector.inflect('api_v1' => 'ApiV1') + new_loader.inflector.inflect( + 'api_v1' => 'ApiV1', + 'cli' => 'CLI' + ) end # @return [Array] diff --git a/app/web/cli.rb b/app/web/cli.rb new file mode 100644 index 000000000..9d7ae325d --- /dev/null +++ b/app/web/cli.rb @@ -0,0 +1,201 @@ +# frozen_string_literal: true + +require 'net/http' +require 'optparse' +require 'uri' + +module Html2rss + module Web + ## + # Unified command-line interface for operator workflows and container management. + module CLI # rubocop:disable Metrics/ModuleLength + STATUS_HEADERS = %w[registry mode version staged_version updated_at sync_url last_error].freeze + + module_function + + ## + # Dispatches command-line arguments to the appropriate handler. + # + # @param argv [Array] command-line arguments + # @param out [IO] standard output stream + # @param err [IO] standard error stream + # @return [Integer] process exit code (0 for success, 1 for failure) + def run(argv = ARGV, out: $stdout, err: $stderr) # rubocop:disable Metrics/MethodLength + case argv.first + when 'registry' then run_registry(argv[1..] || [], out:, err:) + when 'healthcheck' then run_healthcheck(out:, err:) + when 'version', '-v', '--version' then run_version(out:) + when '-h', '--help', 'help', nil + print_root_help(out:) + 0 + else + err.puts "Unknown command: #{argv.first.inspect}. Run with --help for usage." + 1 + end + end + + ## + # @param args [Array] + # @param out [IO] + # @param err [IO] + # @return [Integer] + def run_registry(args, out:, err:) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength + case args.first + when 'status' then registry_status(args[1..] || [], out:) + when 'sync' then registry_sync(args[1..] || []) + when 'promote' then registry_promote(args[1..] || []) + when '-h', '--help', 'help', nil + print_registry_help(out:) + 0 + else + err.puts "Unknown registry command: #{args.first.inspect}. Run with --help for usage." + 1 + end + end + + ## + # @param args [Array] + # @param out [IO] + # @return [Integer] + def registry_status(args, out:) + options = { registry_id: nil } + OptionParser.new do |opts| + opts.banner = 'Usage: html2rss-web registry status [options]' + opts.on('--registry ID', 'Inspect a single registry ID') { options[:registry_id] = it } + end.parse!(args) + + print_status_table(options[:registry_id], out:) + Registry::Sync.cli_exit_code + end + + ## + # @param args [Array] + # @return [Integer] + def registry_sync(args) + options = { registry_id: nil, dry_run: false } + OptionParser.new do |opts| + opts.banner = 'Usage: html2rss-web registry sync [options]' + opts.on('--registry ID', 'Sync a single registry ID') { options[:registry_id] = it } + opts.on('--dry-run', 'Fetch and verify without swapping active bundle') { options[:dry_run] = true } + end.parse!(args) + + target_registry_ids(options[:registry_id]).each do |registry_id| + Registry::Sync.run(registry_id:, dry_run: options[:dry_run]) + end + Registry::Sync.cli_exit_code + end + + ## + # @param args [Array] + # @return [Integer] + def registry_promote(args) + options = { registry_id: nil } + OptionParser.new do |opts| + opts.banner = 'Usage: html2rss-web registry promote [options]' + opts.on('--registry ID', 'Promote a single registry ID') { options[:registry_id] = it } + end.parse!(args) + + target_registry_ids(options[:registry_id]).each do |registry_id| + Registry::Sync.promote_staged!(registry_id:) + end + Registry::Sync.cli_exit_code + end + + ## + # @param out [IO] + # @param err [IO] + # @return [Integer] + def run_healthcheck(out:, err:) # rubocop:disable Metrics/MethodLength + port = ENV.fetch('PORT', 4000) + uri = URI.parse("http://127.0.0.1:#{port}/api/v1/health/live") + response = Net::HTTP.get_response(uri) + if response.is_a?(Net::HTTPSuccess) + out.puts 'OK' + 0 + else + err.puts "Healthcheck failed: HTTP #{response.code}" + 1 + end + rescue StandardError => error + err.puts "Healthcheck failed: #{error.message}" + 1 + end + + ## + # @param out [IO] + # @return [Integer] + def run_version(out:) + out.puts "html2rss-web build=#{RuntimeEnv.build_tag} sha=#{RuntimeEnv.git_sha} ruby=#{RUBY_VERSION}" + 0 + end + + ## + # @param out [IO] + # @return [void] + def print_root_help(out:) + out.puts <<~HELP + Usage: html2rss-web [command] [options] + + Commands: + registry status [--registry ID] Show status of configured registries + registry sync [--registry ID] [--dry-run] Fetch, verify, and sync registry bundles + registry promote [--registry ID] Promote verified staged bundle to active + healthcheck Verify container process liveness + version Print version and runtime information + HELP + end + + ## + # @param out [IO] + # @return [void] + def print_registry_help(out:) + out.puts <<~HELP + Usage: html2rss-web registry [subcommand] [options] + + Subcommands: + status Show registry sync status table + sync Fetch and verify remote registry bundle(s) + promote Promote staged verified bundle to active + HELP + end + + ## + # @param registry_id [String, nil] + # @return [Array] + def target_registry_ids(registry_id) + return Array(registry_id) if registry_id + + Registry::Config.precedence + end + + ## + # @param registry_id [String, nil] + # @param out [IO] + # @return [void] + def print_status_table(registry_id, out:) # rubocop:disable Metrics/MethodLength + rows = Registry::Sync.status(registry_id:) + out.puts STATUS_HEADERS.join("\t") + rows.each do |row| + out.puts [ + row.id, + row.mode, + row.version || '-', + row.staged_version || '-', + format_time(row.updated_at), + row.sync_url || '-', + row.last_error || '-' + ].join("\t") + end + end + + ## + # @param value [Time, nil] + # @return [String] + def format_time(value) = value ? value.utc.iso8601 : '-' + + private_class_method :run_registry, :registry_status, :registry_sync, :registry_promote, + :run_healthcheck, :run_version, :print_root_help, :print_registry_help, + :target_registry_ids, :print_status_table, :format_time + end + end +end diff --git a/bin/docker-healthcheck b/bin/docker-healthcheck deleted file mode 100755 index 617c99ceb..000000000 --- a/bin/docker-healthcheck +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -require 'uri' -require 'net/http' - -port = ENV.fetch('PORT', '4000') -token = ENV.fetch('HEALTH_CHECK_TOKEN', nil) -token = 'CHANGE_ME_HEALTH_CHECK_TOKEN' if token.nil? || token.empty? - -uri = URI("http://localhost:#{port}/api/v1/health") -request = Net::HTTP::Get.new(uri) -request['Authorization'] = "Bearer #{token}" - -begin - response = Net::HTTP.start(uri.hostname, uri.port) { |http| http.request(request) } - exit(response.is_a?(Net::HTTPSuccess) ? 0 : 1) -rescue StandardError - exit(1) -end diff --git a/bin/html2rss-web b/bin/html2rss-web new file mode 100755 index 000000000..aad85bd4f --- /dev/null +++ b/bin/html2rss-web @@ -0,0 +1,11 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require 'bundler/setup' + +ENV['RACK_ENV'] ||= 'development' + +require_relative '../app' +require_relative '../app/web/cli' + +exit Html2rss::Web::CLI.run(ARGV) if $PROGRAM_NAME == __FILE__ diff --git a/bin/prepare-registry-seed b/bin/prepare-registry-seed index a2c38dd59..59728eee9 100755 --- a/bin/prepare-registry-seed +++ b/bin/prepare-registry-seed @@ -21,7 +21,7 @@ fi echo "Building registry seed bundle from $CONFIGS_REPO" ( cd "$CONFIGS_REPO" - BUNDLE_GEMFILE=tool/Gemfile bundle exec ruby tool/registry-build --output "$OUTPUT_TAR" + REGISTRY_VERSION=seed BUNDLE_GEMFILE=tool/Gemfile bundle exec ruby tool/registry-build --output "$OUTPUT_TAR" ) rm -rf "$SEED_DIR" diff --git a/bin/registry-sync b/bin/registry-sync deleted file mode 100755 index 5cb516ffe..000000000 --- a/bin/registry-sync +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -require 'optparse' - -require_relative '../app' - -module RegistrySyncCLI - module_function - - def run(argv = ARGV) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength - options = { status: false, registry_id: nil, dry_run: false, promote: false } - OptionParser.new do |opts| - opts.banner = 'Usage: bin/registry-sync [options]' - opts.on('--status', 'Print registry sync status and exit') { options[:status] = true } - opts.on('--registry ID', 'Sync a single registry id') { |value| options[:registry_id] = value } - opts.on('--dry-run', 'Fetch and verify without swapping the active bundle') { options[:dry_run] = true } - opts.on('--promote', 'Promote a verified staging bundle to active') { options[:promote] = true } - end.parse!(argv) - - if options[:status] - print_status_table(options[:registry_id]) - exit Html2rss::Web::Registry::Sync.cli_exit_code - end - - if options[:promote] - target_registry_ids(options[:registry_id]).each do |registry_id| - Html2rss::Web::Registry::Sync.promote_staged!(registry_id:) - end - exit Html2rss::Web::Registry::Sync.cli_exit_code - end - - target_registry_ids(options[:registry_id]).each do |registry_id| - Html2rss::Web::Registry::Sync.run(registry_id:, dry_run: options[:dry_run]) - end - exit Html2rss::Web::Registry::Sync.cli_exit_code - end - - def target_registry_ids(registry_id) - return Array(registry_id) if registry_id - - Html2rss::Web::Registry::Config.precedence - end - - def print_status_table(registry_id) # rubocop:disable Metrics/MethodLength - rows = Html2rss::Web::Registry::Sync.status(registry_id:) - headers = %w[registry mode version staged_version updated_at sync_url last_error] - puts headers.join("\t") - rows.each do |row| - puts [ - row.id, - row.mode, - row.version || '-', - row.staged_version || '-', - format_time(row.updated_at), - row.sync_url || '-', - row.last_error || '-' - ].join("\t") - end - end - - def format_time(value) - return '-' unless value - - value.utc.iso8601 - end -end - -RegistrySyncCLI.run if $PROGRAM_NAME == __FILE__ diff --git a/docs/README.md b/docs/README.md index 653820337..ae0bc28ac 100644 --- a/docs/README.md +++ b/docs/README.md @@ -295,10 +295,10 @@ Signed feed registries replace the embedded `html2rss-configs` gem. Each registr ### Check sync status -Inside the Dev Container: +Inside the Dev Container or Docker container: ```bash -bin/registry-sync --status +bin/html2rss-web registry status ``` Columns: `registry`, `mode`, `version`, `staged_version`, `updated_at`, `sync_url`, `last_error`. Exit code is non-zero when any sync-mode registry lacks a usable on-disk bundle. @@ -306,9 +306,9 @@ Columns: `registry`, `mode`, `version`, `staged_version`, `updated_at`, `sync_ur Sync, dry-run, or promote a staged bundle: ```bash -bin/registry-sync --registry official -bin/registry-sync --registry official --dry-run -bin/registry-sync --promote --registry official +bin/html2rss-web registry sync --registry official +bin/html2rss-web registry sync --registry official --dry-run +bin/html2rss-web registry promote --registry official ``` Production recommendation: keep `auto_promote: false` (default), pin `sync.pin_version` to the approved configs tag, run sync to stage a verified bundle, then promote manually after review. Use `sync.max_version` as an incident freeze cap and set `REGISTRY_SYNC_INTERVAL_HOURS=0` to pause background refresh. @@ -360,7 +360,7 @@ registries: - **`precedence`** — merge order for feed lookup; first match wins. - **`sync.url`** — direct tarball URL, or use `sync.channel: html2rss-official` for the default GitHub release asset. - **`sync.pin_version` / `sync.max_version`** — fetch a specific tag or reject manifests newer than the cap. -- **`auto_promote: false`** — default; verified bundles land in `REGISTRY_DATA_ROOT//.staging/` until `bin/registry-sync --promote`. +- **`auto_promote: false`** — default; verified bundles land in `REGISTRY_DATA_ROOT//.staging/` until `bin/html2rss-web registry promote`. - **`allowed_channel_domains`** — optional suffix allowlist enforced when the bundle loads. - **`catalog: false`** — private registry: configs are served at `/{registry.id}.rss` but excluded from the public catalog API (privacy for internal feeds). - Restrict outbound hosts with `REGISTRY_SYNC_ALLOWED_HOSTS` (comma-separated hostnames). @@ -376,13 +376,13 @@ registries: catalog: true ``` -The directory must contain `manifest.json`, optional `manifest.sig`, and `configs/`. Path mode skips network sync; run `bin/registry-sync --status` to confirm `mode` is `path`. +The directory must contain `manifest.json`, optional `manifest.sig`, and `configs/`. Path mode skips network sync; run `bin/html2rss-web registry status` to confirm `mode` is `path`. ### Key rotation 1. Publish a new bundle signed with the new key (`public_key_id` in `manifest.json`). 2. Update `public_key_id` and `public_key` in `registries.yml` on every instance **before** or together with the first release that requires the new key. -3. Re-sync (`bin/registry-sync`) and promote when `auto_promote: false` (`bin/registry-sync --promote`). Failed verification leaves the previous bundle active and records `last_error`. +3. Re-sync (`bin/html2rss-web registry sync`) and promote when `auto_promote: false` (`bin/html2rss-web registry promote`). Failed verification leaves the previous bundle active and records `last_error`. --- diff --git a/docs/registry-go-live.md b/docs/registry-go-live.md index 05a8ca190..dc8456bc2 100644 --- a/docs/registry-go-live.md +++ b/docs/registry-go-live.md @@ -120,7 +120,7 @@ registries: - `public_key_id` must match the value in signed `manifest.json` (default in `tool/registry-build`: `html2rss:registry:2026`). - Network sync uses `:signed` trust and requires this pin. Seed bundles copied from the Docker image use `:integrity_only` trust (no signature check on load). -- **`auto_promote: false`** (default) writes verified bundles to `REGISTRY_DATA_ROOT//.staging/` without changing the active catalog. Promote after review with `bin/registry-sync --promote --registry official`. +- **`auto_promote: false`** (default) writes verified bundles to `REGISTRY_DATA_ROOT//.staging/` without changing the active catalog. Promote after review with `bin/html2rss-web registry promote --registry official`. - **`pin_version`** resolves the GitHub tag release API instead of `/releases/latest`. - **`max_version`** rejects sync when the verified manifest version is newer than the cap (incident freeze). - **`allowed_channel_domains`** rejects bundle load when any config `channel.url` host is outside the suffix allowlist. @@ -261,7 +261,7 @@ Allowed outbound hosts (built-in): `api.github.com`, `github.com`, `objects.gith The named volume preserves synced bundles across container restarts and image upgrades. After pull + restart: - Old bundle stays active until a successful sync swaps it. -- Use `bin/registry-sync` or wait for background refresh to pick up a new configs release without rebuilding the image (section 7). +- Use `bin/html2rss-web registry sync` or wait for background refresh to pick up a new configs release without rebuilding the image (section 7). --- @@ -330,7 +330,7 @@ Run these checks after deploy or sync. Inside the running container (or Dev Container): ```bash -bin/registry-sync --status +bin/html2rss-web registry status ``` Tab-separated columns: `registry`, `mode`, `version`, `staged_version`, `updated_at`, `sync_url`, `last_error`. @@ -341,9 +341,9 @@ Tab-separated columns: `registry`, `mode`, `version`, `staged_version`, `updated Single registry, dry-run, or promote staged bundle: ```bash -bin/registry-sync --registry official -bin/registry-sync --registry official --dry-run -bin/registry-sync --promote --registry official +bin/html2rss-web registry sync --registry official +bin/html2rss-web registry sync --registry official --dry-run +bin/html2rss-web registry promote --registry official ``` ### 6.2 Instance metadata API @@ -388,9 +388,9 @@ Configs releases are independent of web image releases. To refresh feeds on a ru Production recommendation: keep `auto_promote: false`, set `sync.pin_version` to the approved tag, fetch with sync, then promote manually after review. ```bash -bin/registry-sync --registry official -bin/registry-sync --status # staged_version shows verified bundle -bin/registry-sync --promote --registry official +bin/html2rss-web registry sync --registry official +bin/html2rss-web registry status # staged_version shows verified bundle +bin/html2rss-web registry promote --registry official ``` Fetches the pinned or latest signed tarball, verifies signature + digests, and either stages (`auto_promote: false`) or atomically swaps the active bundle (`auto_promote: true`). @@ -410,35 +410,31 @@ registries: Set `REGISTRY_SYNC_INTERVAL_HOURS=0` to pause background refresh while investigating. -### 7.3 Automatic refresh +### 7.3 Automatic refresh & production Puma cluster mode | Mechanism | Env var | Default | Behavior | | --- | --- | --- | --- | | Sync on boot | `REGISTRY_SYNC_ON_BOOT` | `false` | When `true`, network sync on every boot (in addition to missing-bundle sync) | -| Background timer | `REGISTRY_SYNC_INTERVAL_HOURS` | `24` | Jittered periodic re-sync; `0` disables | +| Background timer | `REGISTRY_SYNC_INTERVAL_HOURS` | `24` | Periodic re-sync; `0` disables | | Missing bundle | — | — | Always syncs on boot when no on-disk bundle exists | -Example — force sync every boot (use sparingly): +> [!TIP] +> **Production Puma Cluster Mode (Kubernetes / Sidecar)**: +> In production cluster mode (`workers > 0` with `preload_app!`), child worker processes do not execute background timer threads spawned in the master process. Because `Index.current` detects changes on disk via `manifest.json` mtimes, the recommended production pattern for Kubernetes or Compose is to run periodic sync via a lightweight sidecar container or CronJob executing `bin/html2rss-web registry sync`. -```bash -REGISTRY_SYNC_ON_BOOT=true -``` - -Example — disable background refresh: +Example — disable in-process background timer when using external cron: ```bash REGISTRY_SYNC_INTERVAL_HOURS=0 ``` -**Note:** Pulling a new web image updates the embedded seed but does **not** replace an existing volume bundle until sync succeeds. - --- ## 8. Troubleshooting ### 8.1 Signature verification failure -Symptoms in `bin/registry-sync --status`: +Symptoms in `bin/html2rss-web registry status`: - `last_error` contains `Unknown public_key_id`, `Invalid manifest signature`, or `Missing manifest.sig` @@ -453,7 +449,7 @@ Signature-related failures are logged via `SecurityLogger.log_registry_signature Verify without swapping the active bundle: ```bash -bin/registry-sync --registry official --dry-run +bin/html2rss-web registry sync --registry official --dry-run ``` ### 8.2 Sync failure keeps the old bundle @@ -464,7 +460,7 @@ By design: - On failure, `last_error` is recorded; the previous bundle under `REGISTRY_DATA_ROOT//` remains served. - `Store.promote_bundle!` rolls back on swap failure. -If sync fails on first boot with no prior bundle, the instance may have only the unsigned seed (integrity-only) until a signed sync succeeds. +If sync fails on first boot with no prior bundle, the instance serves the embedded seed bundle (144 feeds) until a signed sync succeeds. ### 8.3 Network / host errors @@ -475,7 +471,7 @@ If sync fails on first boot with no prior bundle, the instance may have only the | `Registry sync fetch failed with HTTP …` | Release missing, URL wrong, or GitHub outage | | `Registry sync requires HTTPS URLs` | Use `https://` in `sync.url` | -Default allowed hosts cover official GitHub release downloads only. +Default allowed hosts cover official GitHub release downloads (`api.github.com`, `github.com`, `objects.githubusercontent.com`, `release-assets.githubusercontent.com`). ### 8.4 Air-gapped / offline (`path` mode) @@ -491,8 +487,8 @@ registries: Requirements: - Directory contains `manifest.json`, `configs/`, and optionally `manifest.sig` -- `bin/registry-sync` is not applicable (`path mode; sync is not applicable`) -- `bin/registry-sync --status` should show `mode: path` +- `bin/html2rss-web registry sync` is not applicable (`path mode; sync is not applicable`) +- `bin/html2rss-web registry status` should show `mode: path` Load path uses `:integrity_only` trust (disk/image trust boundary). @@ -507,11 +503,11 @@ environment: ### 8.5 CLI exit code non-zero with empty `last_error` -`bin/registry-sync --status` exits **1** when a sync-mode registry has **no on-disk bundle**. Run a sync or confirm seed copy succeeded: +`bin/html2rss-web registry status` exits **1** when a sync-mode registry has **no on-disk bundle**. Run a sync or confirm seed copy succeeded: ```bash -bin/registry-sync --registry official -bin/registry-sync --status +bin/html2rss-web registry sync --registry official +bin/html2rss-web registry status ``` ### 8.6 Seed missing or stale in image diff --git a/spec/html2rss/web/cli_spec.rb b/spec/html2rss/web/cli_spec.rb new file mode 100644 index 000000000..3dafc2ca4 --- /dev/null +++ b/spec/html2rss/web/cli_spec.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +require 'stringio' +require_relative '../../../app' +require_relative '../../../app/web/cli' + +RSpec.describe Html2rss::Web::CLI do + describe '.run' do + let(:stdout) { StringIO.new } + let(:stderr) { StringIO.new } + + context 'with root help and version commands' do + it 'prints root help for help options', :aggregate_failures do + exit_code = described_class.run(['--help'], out: stdout, err: stderr) + expect(exit_code).to eq(0) + expect(stdout.string).to include('Usage: html2rss-web [command] [options]') + expect(stdout.string).to include('registry status') + end + + it 'prints root help when argv is empty', :aggregate_failures do + exit_code = described_class.run([], out: stdout, err: stderr) + expect(exit_code).to eq(0) + expect(stdout.string).to include('Usage: html2rss-web [command] [options]') + end + + it 'prints version and runtime info', :aggregate_failures do + exit_code = described_class.run(['version'], out: stdout, err: stderr) + expect(exit_code).to eq(0) + expect(stdout.string).to match(/html2rss-web build=.* sha=.* ruby=.*/) + end + + it 'handles unknown commands', :aggregate_failures do + exit_code = described_class.run(['invalid-cmd'], out: stdout, err: stderr) + expect(exit_code).to eq(1) + expect(stderr.string).to include('Unknown command: "invalid-cmd"') + end + end + + context 'with healthcheck command' do + let(:success_response) { instance_double(Net::HTTPSuccess, is_a?: true) } + let(:error_response) { instance_double(Net::HTTPInternalServerError, is_a?: false, code: '500') } + + it 'returns 0 when HTTP healthcheck succeeds', :aggregate_failures do + allow(Net::HTTP).to receive(:get_response).and_return(success_response) + + exit_code = described_class.run(['healthcheck'], out: stdout, err: stderr) + expect(exit_code).to eq(0) + expect(stdout.string).to include('OK') + end + + it 'returns 1 when HTTP healthcheck returns non-200', :aggregate_failures do + allow(Net::HTTP).to receive(:get_response).and_return(error_response) + + exit_code = described_class.run(['healthcheck'], out: stdout, err: stderr) + expect(exit_code).to eq(1) + expect(stderr.string).to include('Healthcheck failed: HTTP 500') + end + + it 'returns 1 when HTTP connection fails', :aggregate_failures do + allow(Net::HTTP).to receive(:get_response).and_raise(Errno::ECONNREFUSED) + + exit_code = described_class.run(['healthcheck'], out: stdout, err: stderr) + expect(exit_code).to eq(1) + expect(stderr.string).to include('Healthcheck failed') + end + end + + context 'with registry subcommands' do + it 'prints registry help for --help', :aggregate_failures do + exit_code = described_class.run(%w[registry --help], out: stdout, err: stderr) + expect(exit_code).to eq(0) + expect(stdout.string).to include('Usage: html2rss-web registry [subcommand]') + end + + it 'prints registry status table', :aggregate_failures do + exit_code = described_class.run(%w[registry status], out: stdout, err: stderr) + expect(exit_code).to eq(0) + expect(stdout.string).to include("registry\tmode\tversion\tstaged_version\tupdated_at\tsync_url\tlast_error") + expect(stdout.string).to include('official') + end + + it 'runs registry sync', :aggregate_failures do + allow(Html2rss::Web::Registry::Sync).to receive_messages(run: nil, cli_exit_code: 0) + + exit_code = described_class.run(%w[registry sync --registry official --dry-run], out: stdout, err: stderr) + expect(exit_code).to eq(0) + expect(Html2rss::Web::Registry::Sync).to have_received(:run).with(registry_id: 'official', dry_run: true) + end + + it 'runs registry promote', :aggregate_failures do + allow(Html2rss::Web::Registry::Sync).to receive_messages(promote_staged!: nil, cli_exit_code: 0) + + exit_code = described_class.run(%w[registry promote --registry official], out: stdout, err: stderr) + expect(exit_code).to eq(0) + expect(Html2rss::Web::Registry::Sync).to have_received(:promote_staged!).with(registry_id: 'official') + end + + it 'handles unknown registry subcommands', :aggregate_failures do + exit_code = described_class.run(%w[registry invalid-sub], out: stdout, err: stderr) + expect(exit_code).to eq(1) + expect(stderr.string).to include('Unknown registry command: "invalid-sub"') + end + end + end +end diff --git a/spec/support/registry_sync.rb b/spec/support/registry_sync.rb index 6d94b1522..3959ff761 100644 --- a/spec/support/registry_sync.rb +++ b/spec/support/registry_sync.rb @@ -13,6 +13,7 @@ module RegistryTestHelpers def configure_registry_fixtures! ENV['REGISTRIES_CONFIG'] = DEFAULT_REGISTRIES_CONFIG ENV['REGISTRY_DATA_ROOT'] = File.join(Dir.pwd, 'tmp', 'test-registry-data') + FileUtils.rm_rf(ENV.fetch('REGISTRY_DATA_ROOT', nil)) end def reset_registry! From 8c1dc70a6e5ed1f578ab1427e232217586512d02 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Mon, 24 Aug 2026 11:05:31 +0200 Subject: [PATCH 26/33] docs(compose): add registry-sync service and compose operator workflow --- docker-compose.yml | 13 +++++++++++++ docs/README.md | 8 +++++--- docs/registry-go-live.md | 29 ++++++++++++++++++----------- 3 files changed, 36 insertions(+), 14 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6ca4cc89c..7ab5a9bb0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -57,5 +57,18 @@ services: ports: - "127.0.0.1:4010:4010" + registry-sync: + image: html2rss/web + restart: unless-stopped + command: ["sh", "-c", "while true; do html2rss-web registry sync; sleep $${REGISTRY_SYNC_INTERVAL_SECONDS:-86400}; done"] + env_file: + - path: .env + required: false + environment: + REGISTRY_DATA_ROOT: /app/data/registries + REGISTRY_SYNC_INTERVAL_SECONDS: ${REGISTRY_SYNC_INTERVAL_SECONDS:-86400} + volumes: + - registry-data:/app/data/registries + volumes: registry-data: diff --git a/docs/README.md b/docs/README.md index ae0bc28ac..db0604b8d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -311,7 +311,9 @@ bin/html2rss-web registry sync --registry official --dry-run bin/html2rss-web registry promote --registry official ``` -Production recommendation: keep `auto_promote: false` (default), pin `sync.pin_version` to the approved configs tag, run sync to stage a verified bundle, then promote manually after review. Use `sync.max_version` as an incident freeze cap and set `REGISTRY_SYNC_INTERVAL_HOURS=0` to pause background refresh. +Production recommendation: keep `auto_promote: false` (default), pin `sync.pin_version` to the approved configs tag, run sync to stage a verified bundle, then promote manually after review. Use `sync.max_version` as an incident freeze cap. + +In Docker Compose, the dedicated `registry-sync` service runs periodic background updates cleanly outside Puma/Ruby. Optional hardening: `allowed_channel_domains` suffix-matches every registry config `channel.url` host at bundle load time. @@ -319,9 +321,9 @@ Optional hardening: `allowed_channel_domains` suffix-matches every registry conf `Registry::Sync.boot!` runs during app boot (see `app/web/boot/setup.rb`): -1. **Seed** — copies the image-embedded official bundle when no on-disk bundle exists. +1. **Seed** — copies the image-embedded official bundle (144 feeds) when no on-disk bundle exists. 2. **Sync on boot** — when `REGISTRY_SYNC_ON_BOOT=true`, or when the bundle is missing, fetches and verifies the latest release. -3. **Background refresh** — when `REGISTRY_SYNC_INTERVAL_HOURS` is greater than zero (default `24`), re-syncs on a jittered timer. Set to `0` to disable. +3. **Background refresh** — in standalone single-process mode, `REGISTRY_SYNC_INTERVAL_HOURS` (default `24`) re-syncs on a periodic timer. Set to `0` when using the `registry-sync` Compose service. Network sync verifies Ed25519 signatures using the `public_key` pinned in `registries.yml`. Seed bundles from the Docker image use integrity-only verification. diff --git a/docs/registry-go-live.md b/docs/registry-go-live.md index dc8456bc2..501961766 100644 --- a/docs/registry-go-live.md +++ b/docs/registry-go-live.md @@ -410,22 +410,29 @@ registries: Set `REGISTRY_SYNC_INTERVAL_HOURS=0` to pause background refresh while investigating. -### 7.3 Automatic refresh & production Puma cluster mode +### 7.3 Automatic refresh via Docker Compose -| Mechanism | Env var | Default | Behavior | -| --- | --- | --- | --- | -| Sync on boot | `REGISTRY_SYNC_ON_BOOT` | `false` | When `true`, network sync on every boot (in addition to missing-bundle sync) | -| Background timer | `REGISTRY_SYNC_INTERVAL_HOURS` | `24` | Periodic re-sync; `0` disables | -| Missing bundle | — | — | Always syncs on boot when no on-disk bundle exists | +In production Compose environments, registry synchronization is handled cleanly outside the Ruby/Puma process lifecycle via the dedicated `registry-sync` Compose service in `docker-compose.yml`: -> [!TIP] -> **Production Puma Cluster Mode (Kubernetes / Sidecar)**: -> In production cluster mode (`workers > 0` with `preload_app!`), child worker processes do not execute background timer threads spawned in the master process. Because `Index.current` detects changes on disk via `manifest.json` mtimes, the recommended production pattern for Kubernetes or Compose is to run periodic sync via a lightweight sidecar container or CronJob executing `bin/html2rss-web registry sync`. +```yaml + registry-sync: + image: html2rss/web + restart: unless-stopped + command: ["sh", "-c", "while true; do html2rss-web registry sync; sleep $${REGISTRY_SYNC_INTERVAL_SECONDS:-86400}; done"] + environment: + REGISTRY_DATA_ROOT: /app/data/registries + REGISTRY_SYNC_INTERVAL_SECONDS: 86400 + volumes: + - registry-data:/app/data/registries +``` + +When new bundles are synced to disk, the `html2rss-web` server detects the updated `manifest.json` mtimes automatically on subsequent requests. -Example — disable in-process background timer when using external cron: +To run an on-demand sync or check status with Docker Compose: ```bash -REGISTRY_SYNC_INTERVAL_HOURS=0 +docker compose exec html2rss-web html2rss-web registry status +docker compose exec html2rss-web html2rss-web registry sync --registry official ``` --- From a8950b69760cd97511571e471b020f2c2a3cc41f Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Mon, 24 Aug 2026 11:06:44 +0200 Subject: [PATCH 27/33] chore(compose): remove legacy watchtower service --- docker-compose.yml | 8 -------- docs/registry-go-live.md | 1 - 2 files changed, 9 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7ab5a9bb0..d81ee451e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,14 +36,6 @@ services: volumes: - registry-data:/app/data/registries - watchtower: - image: containrrr/watchtower - restart: unless-stopped - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - "${HOME}/.docker/config.json:/config.json" - command: --cleanup --interval 7200 - botasaurus: image: html2rss/botasaurus-scrape-api:latest restart: unless-stopped diff --git a/docs/registry-go-live.md b/docs/registry-go-live.md index 501961766..612138bf0 100644 --- a/docs/registry-go-live.md +++ b/docs/registry-go-live.md @@ -231,7 +231,6 @@ docker build \ - Image: `html2rss/web` - Volume: `registry-data:/app/data/registries` (`REGISTRY_DATA_ROOT`) -- Watchtower optional: checks for new images every 7200s ```bash docker compose pull html2rss-web From 63c06d0ce71bbf8f0afc344555c9284ad25b6feb Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Mon, 24 Aug 2026 15:05:48 +0200 Subject: [PATCH 28/33] refactor(registry): implement embedded bundle lookup and eliminate seed subsystem --- .../seed/official/configs/aalto.fi/news.yml | 28 ---- .../configs/academictransfer.com/jobs.yml | 28 ---- .../configs/adfc.de/pressemitteilungen.yml | 24 --- .../official/configs/ahram.org.eg/egypt.yml | 22 --- .../official/configs/ai.meta.com/blog.yml | 28 ---- .../official/configs/akorda.kz/events.yml | 23 --- .../official/configs/angop.ao/noticias.yml | 23 --- .../official/configs/anthropic.com/news.yml | 23 --- .../seed/official/configs/anu.edu.au/news.yml | 23 --- .../seed/official/configs/apnews.com/hub.yml | 30 ---- .../official/configs/apple.com/newsroom.yml | 31 ---- .../official/configs/aps.dz/national-news.yml | 23 --- .../configs/asml.com/press-releases.yml | 23 --- .../configs/aspi.org.au/strategist.yml | 29 ---- .../configs/bbc.co.uk/available_episodes.yml | 29 ---- .../seed/official/configs/bbc.com/mundo.yml | 25 --- .../configs/blog.mondediplo.net/feed.yml | 29 ---- .../configs/brookings.edu/articles.yml | 23 --- .../bundesregierung.de/pressemitteilungen.yml | 28 ---- .../configs/canarianweekly.com/front.yml | 23 --- .../official/configs/caricom.org/news.yml | 23 --- .../carnegieendowment.org/research.yml | 29 ---- .../configs/cepal.org/press-releases.yml | 23 --- .../official/configs/chalmers.se/news.yml | 28 ---- .../research-publications.yml | 29 ---- .../configs/cinemascore.com/index.yml | 55 ------- .../configs/cleanenergywire.org/news.yml | 28 ---- .../official/configs/cnet.com/section_sub.yml | 30 ---- .../seed/official/configs/cohere.com/blog.yml | 23 --- .../configs/computerbase.de/meistgelesen.yml | 27 ---- .../official/configs/cplp.org/noticias.yml | 23 --- .../seed/official/configs/csiro.au/news.yml | 23 --- .../curia.europa.eu/press-releases.yml | 29 ---- .../official/configs/cutle.fish/index.yml | 25 --- .../official/configs/deepmind.google/blog.yml | 27 ---- .../configs/deraktionaer.de/meistgelesen.yml | 22 --- ...ials_data_documentation_technotes_json.yml | 28 ---- .../configs/dfs.de/pressemitteilungen.yml | 28 ---- .../official/configs/diis.dk/research.yml | 23 --- .../configs/dirco.gov.za/media-statements.yml | 23 --- .../official/configs/dsw-info.de/presse.yml | 30 ---- .../elasticsearch-release-notes.yml | 23 --- .../official/configs/elysee.fr/actualites.yml | 22 --- .../official/configs/equinor.com/news.yml | 22 --- .../seed/official/configs/espn.com/f1.yml | 22 --- .../official/configs/ethz.ch/eth-news.yml | 29 ---- .../official/configs/eures.europa.eu/news.yml | 22 --- .../official/configs/faa.gov/newsroom.yml | 22 --- .../official/configs/fao.org/newsroom.yml | 23 --- .../official/configs/fia.com/documents.yml | 26 --- .../seed/official/configs/figma.com/blog.yml | 25 --- .../official/configs/formula1.com/latest.yml | 22 --- .../configs/forumsec.org/publications.yml | 24 --- .../configs/ftc.gov/press-releases.yml | 23 --- .../official/configs/github.com/releases.yml | 33 ---- .../configs/go.dev/release-history.yml | 23 --- .../official/configs/grafana.com/whatsnew.yml | 23 --- .../configs/haaretz.com/israel-news.yml | 22 --- .../configs/hai.stanford.edu/news.yml | 23 --- .../seed/official/configs/iaapa.org/news.yml | 23 --- .../configs/icj-cij.org/press-releases.yml | 23 --- .../seed/official/configs/icrc.org/news.yml | 22 --- .../seed/official/configs/iea.org/news.yml | 23 --- .../configs/iiss.org/online-analysis.yml | 29 ---- .../official/configs/ikea.com/newsroom.yml | 23 --- .../official/configs/imdb.com/ratings.yml | 50 ------ .../configs/imo.org/press-briefings.yml | 28 ---- .../official/configs/imperial.ac.uk/news.yml | 23 --- .../karriere_arbeitsleben_heiko_mell.yml | 25 --- .../official/configs/isro.gov.in/press.yml | 23 --- .../configs/issafrica.org/iss-today.yml | 23 --- .../configs/issafrica.org/press-releases.yml | 23 --- .../official/configs/kaist.ac.kr/news.yml | 23 --- .../configs/kinocheck.de/filmstarts.yml | 30 ---- .../official/configs/kompas.com/nasional.yml | 23 --- .../configs/lego.com/press-releases.yml | 23 --- .../machinelearning.apple.com/research.yml | 23 --- .../configs/maersk.com/press-releases.yml | 23 --- .../configs/microsoft.com/azure-products.yml | 25 --- .../seed/official/configs/mistral.ai/news.yml | 24 --- .../mozilla.org/security-advisories.yml | 24 --- .../seed/official/configs/msf.org/latest.yml | 23 --- .../configs/nbim.no/press-releases.yml | 23 --- .../seed/official/configs/news.cn/latest.yml | 22 --- .../configs/news24.com/top-stories.yml | 22 --- .../configs/newyorker.com/magazine.yml | 25 --- .../configs/nigerianstat.gov.ng/news.yml | 23 --- .../configs/nikkei.com/asia-technology.yml | 23 --- .../official/configs/nokia.com/newsroom.yml | 23 --- .../official/configs/nomanssky.com/news.yml | 29 ---- .../seed/official/configs/notion.com/blog.yml | 27 ---- .../configs/novonordisk.com/latest-news.yml | 23 --- .../seed/official/configs/ntu.edu.sg/news.yml | 29 ---- .../seed/official/configs/nupi.no/news.yml | 23 --- .../seed/official/configs/occrp.org/news.yml | 30 ---- .../official/configs/oecd.org/newsroom.yml | 29 ---- .../configs/opec.org/press-releases.yml | 30 ---- .../official/configs/panapress.com/latest.yml | 28 ---- .../search.yml | 53 ------ .../official/configs/philstar.com/nation.yml | 22 --- .../seed/official/configs/phys.org/weekly.yml | 26 --- .../configs/pib.gov.in/press-releases.yml | 23 --- .../configs/rbb24.de/meistgeklickt.yml | 22 --- .../configs/robinwood.de/aktuelles.yml | 23 --- .../s3.amazonaws.com/popular_movies.yml | 35 ---- .../seed/official/configs/sacu.int/news.yml | 24 --- .../official/configs/sadc.int/latest-news.yml | 23 --- .../configs/sebastianvettel.de/news.yml | 24 --- .../official/configs/shopify.com/blog.yml | 25 --- .../seed/official/configs/sipri.org/news.yml | 23 --- .../configs/softwareleadweekly.com/issues.yml | 22 --- .../configs/solarthermalworld.org/news.yml | 25 --- .../configs/spektrum.de/meistgelesen.yml | 22 --- .../configs/spiegel.de/impressum_autor.yml | 39 ----- .../official/configs/spotify.com/newsroom.yml | 24 --- .../hot_network_questions.yml | 35 ---- .../official/configs/steuerzahler.de/news.yml | 29 ---- .../official/configs/stripes.com/index.yml | 27 ---- .../support.apple.com/en_gb_ht201222.yml | 37 ----- .../support.apple.com/exchange_repair.yml | 34 ---- .../official/configs/tap.info.tn/politics.yml | 29 ---- .../official/configs/telenor.com/newsroom.yml | 22 --- .../configs/teneriffa-news.com/news.yml | 36 ----- .../seed/official/configs/tennet.eu/news.yml | 22 --- .../seed/official/configs/test.de/archiv.yml | 25 --- .../configs/theeastafrican.co.ke/news.yml | 29 ---- .../international_mostpopular.yml | 22 --- .../configs/thenationalnews.com/news.yml | 22 --- .../configs/thoughtworks.com/insights.yml | 29 ---- .../seed/official/configs/tno.nl/news.yml | 24 --- .../aktuelle_nachrichten.yml | 23 --- .../seed/official/configs/tralac.org/news.yml | 23 --- .../seed/official/configs/tudelft.nl/news.yml | 23 --- .../seed/official/configs/tue.nl/news.yml | 23 --- .../configs/vattenfall.com/newsroom.yml | 23 --- .../visitfinland.com/press-releases.yml | 27 ---- .../configs/webentwickler-jobs.de/in.yml | 31 ---- .../seed/official/configs/who.int/news.yml | 30 ---- .../official/configs/worldbank.org/news.yml | 29 ---- .../press-releases.yml | 28 ---- .../www.ecdc.europa.eu/news-events.yml | 23 --- .../configs/www.enisa.europa.eu/news.yml | 23 --- .../www.europarl.europa.eu/press-room.yml | 28 ---- .../configs/www.gob.mx/profeco-prensa.yml | 24 --- app/registries/seed/official/manifest.json | 152 ------------------ app/web/registry/index.rb | 2 +- app/web/registry/store.rb | 51 +++--- app/web/registry/sync.rb | 27 +--- spec/html2rss/web/registry/index_spec.rb | 57 +++++++ spec/html2rss/web/registry/store_spec.rb | 39 ++++- spec/html2rss/web/registry/sync_spec.rb | 3 +- 151 files changed, 133 insertions(+), 3941 deletions(-) delete mode 100644 app/registries/seed/official/configs/aalto.fi/news.yml delete mode 100644 app/registries/seed/official/configs/academictransfer.com/jobs.yml delete mode 100644 app/registries/seed/official/configs/adfc.de/pressemitteilungen.yml delete mode 100644 app/registries/seed/official/configs/ahram.org.eg/egypt.yml delete mode 100644 app/registries/seed/official/configs/ai.meta.com/blog.yml delete mode 100644 app/registries/seed/official/configs/akorda.kz/events.yml delete mode 100644 app/registries/seed/official/configs/angop.ao/noticias.yml delete mode 100644 app/registries/seed/official/configs/anthropic.com/news.yml delete mode 100644 app/registries/seed/official/configs/anu.edu.au/news.yml delete mode 100644 app/registries/seed/official/configs/apnews.com/hub.yml delete mode 100644 app/registries/seed/official/configs/apple.com/newsroom.yml delete mode 100644 app/registries/seed/official/configs/aps.dz/national-news.yml delete mode 100644 app/registries/seed/official/configs/asml.com/press-releases.yml delete mode 100644 app/registries/seed/official/configs/aspi.org.au/strategist.yml delete mode 100644 app/registries/seed/official/configs/bbc.co.uk/available_episodes.yml delete mode 100644 app/registries/seed/official/configs/bbc.com/mundo.yml delete mode 100644 app/registries/seed/official/configs/blog.mondediplo.net/feed.yml delete mode 100644 app/registries/seed/official/configs/brookings.edu/articles.yml delete mode 100644 app/registries/seed/official/configs/bundesregierung.de/pressemitteilungen.yml delete mode 100644 app/registries/seed/official/configs/canarianweekly.com/front.yml delete mode 100644 app/registries/seed/official/configs/caricom.org/news.yml delete mode 100644 app/registries/seed/official/configs/carnegieendowment.org/research.yml delete mode 100644 app/registries/seed/official/configs/cepal.org/press-releases.yml delete mode 100644 app/registries/seed/official/configs/chalmers.se/news.yml delete mode 100644 app/registries/seed/official/configs/chathamhouse.org/research-publications.yml delete mode 100644 app/registries/seed/official/configs/cinemascore.com/index.yml delete mode 100644 app/registries/seed/official/configs/cleanenergywire.org/news.yml delete mode 100644 app/registries/seed/official/configs/cnet.com/section_sub.yml delete mode 100644 app/registries/seed/official/configs/cohere.com/blog.yml delete mode 100644 app/registries/seed/official/configs/computerbase.de/meistgelesen.yml delete mode 100644 app/registries/seed/official/configs/cplp.org/noticias.yml delete mode 100644 app/registries/seed/official/configs/csiro.au/news.yml delete mode 100644 app/registries/seed/official/configs/curia.europa.eu/press-releases.yml delete mode 100644 app/registries/seed/official/configs/cutle.fish/index.yml delete mode 100644 app/registries/seed/official/configs/deepmind.google/blog.yml delete mode 100644 app/registries/seed/official/configs/deraktionaer.de/meistgelesen.yml delete mode 100644 app/registries/seed/official/configs/developer.apple.com/tutorials_data_documentation_technotes_json.yml delete mode 100644 app/registries/seed/official/configs/dfs.de/pressemitteilungen.yml delete mode 100644 app/registries/seed/official/configs/diis.dk/research.yml delete mode 100644 app/registries/seed/official/configs/dirco.gov.za/media-statements.yml delete mode 100644 app/registries/seed/official/configs/dsw-info.de/presse.yml delete mode 100644 app/registries/seed/official/configs/elastic.co/elasticsearch-release-notes.yml delete mode 100644 app/registries/seed/official/configs/elysee.fr/actualites.yml delete mode 100644 app/registries/seed/official/configs/equinor.com/news.yml delete mode 100644 app/registries/seed/official/configs/espn.com/f1.yml delete mode 100644 app/registries/seed/official/configs/ethz.ch/eth-news.yml delete mode 100644 app/registries/seed/official/configs/eures.europa.eu/news.yml delete mode 100644 app/registries/seed/official/configs/faa.gov/newsroom.yml delete mode 100644 app/registries/seed/official/configs/fao.org/newsroom.yml delete mode 100644 app/registries/seed/official/configs/fia.com/documents.yml delete mode 100644 app/registries/seed/official/configs/figma.com/blog.yml delete mode 100644 app/registries/seed/official/configs/formula1.com/latest.yml delete mode 100644 app/registries/seed/official/configs/forumsec.org/publications.yml delete mode 100644 app/registries/seed/official/configs/ftc.gov/press-releases.yml delete mode 100644 app/registries/seed/official/configs/github.com/releases.yml delete mode 100644 app/registries/seed/official/configs/go.dev/release-history.yml delete mode 100644 app/registries/seed/official/configs/grafana.com/whatsnew.yml delete mode 100644 app/registries/seed/official/configs/haaretz.com/israel-news.yml delete mode 100644 app/registries/seed/official/configs/hai.stanford.edu/news.yml delete mode 100644 app/registries/seed/official/configs/iaapa.org/news.yml delete mode 100644 app/registries/seed/official/configs/icj-cij.org/press-releases.yml delete mode 100644 app/registries/seed/official/configs/icrc.org/news.yml delete mode 100644 app/registries/seed/official/configs/iea.org/news.yml delete mode 100644 app/registries/seed/official/configs/iiss.org/online-analysis.yml delete mode 100644 app/registries/seed/official/configs/ikea.com/newsroom.yml delete mode 100644 app/registries/seed/official/configs/imdb.com/ratings.yml delete mode 100644 app/registries/seed/official/configs/imo.org/press-briefings.yml delete mode 100644 app/registries/seed/official/configs/imperial.ac.uk/news.yml delete mode 100644 app/registries/seed/official/configs/ingenieur.de/karriere_arbeitsleben_heiko_mell.yml delete mode 100644 app/registries/seed/official/configs/isro.gov.in/press.yml delete mode 100644 app/registries/seed/official/configs/issafrica.org/iss-today.yml delete mode 100644 app/registries/seed/official/configs/issafrica.org/press-releases.yml delete mode 100644 app/registries/seed/official/configs/kaist.ac.kr/news.yml delete mode 100644 app/registries/seed/official/configs/kinocheck.de/filmstarts.yml delete mode 100644 app/registries/seed/official/configs/kompas.com/nasional.yml delete mode 100644 app/registries/seed/official/configs/lego.com/press-releases.yml delete mode 100644 app/registries/seed/official/configs/machinelearning.apple.com/research.yml delete mode 100644 app/registries/seed/official/configs/maersk.com/press-releases.yml delete mode 100644 app/registries/seed/official/configs/microsoft.com/azure-products.yml delete mode 100644 app/registries/seed/official/configs/mistral.ai/news.yml delete mode 100644 app/registries/seed/official/configs/mozilla.org/security-advisories.yml delete mode 100644 app/registries/seed/official/configs/msf.org/latest.yml delete mode 100644 app/registries/seed/official/configs/nbim.no/press-releases.yml delete mode 100644 app/registries/seed/official/configs/news.cn/latest.yml delete mode 100644 app/registries/seed/official/configs/news24.com/top-stories.yml delete mode 100644 app/registries/seed/official/configs/newyorker.com/magazine.yml delete mode 100644 app/registries/seed/official/configs/nigerianstat.gov.ng/news.yml delete mode 100644 app/registries/seed/official/configs/nikkei.com/asia-technology.yml delete mode 100644 app/registries/seed/official/configs/nokia.com/newsroom.yml delete mode 100644 app/registries/seed/official/configs/nomanssky.com/news.yml delete mode 100644 app/registries/seed/official/configs/notion.com/blog.yml delete mode 100644 app/registries/seed/official/configs/novonordisk.com/latest-news.yml delete mode 100644 app/registries/seed/official/configs/ntu.edu.sg/news.yml delete mode 100644 app/registries/seed/official/configs/nupi.no/news.yml delete mode 100644 app/registries/seed/official/configs/occrp.org/news.yml delete mode 100644 app/registries/seed/official/configs/oecd.org/newsroom.yml delete mode 100644 app/registries/seed/official/configs/opec.org/press-releases.yml delete mode 100644 app/registries/seed/official/configs/panapress.com/latest.yml delete mode 100644 app/registries/seed/official/configs/pankow.lebensmittel-kontrollergebnisse.de/search.yml delete mode 100644 app/registries/seed/official/configs/philstar.com/nation.yml delete mode 100644 app/registries/seed/official/configs/phys.org/weekly.yml delete mode 100644 app/registries/seed/official/configs/pib.gov.in/press-releases.yml delete mode 100644 app/registries/seed/official/configs/rbb24.de/meistgeklickt.yml delete mode 100644 app/registries/seed/official/configs/robinwood.de/aktuelles.yml delete mode 100644 app/registries/seed/official/configs/s3.amazonaws.com/popular_movies.yml delete mode 100644 app/registries/seed/official/configs/sacu.int/news.yml delete mode 100644 app/registries/seed/official/configs/sadc.int/latest-news.yml delete mode 100644 app/registries/seed/official/configs/sebastianvettel.de/news.yml delete mode 100644 app/registries/seed/official/configs/shopify.com/blog.yml delete mode 100644 app/registries/seed/official/configs/sipri.org/news.yml delete mode 100644 app/registries/seed/official/configs/softwareleadweekly.com/issues.yml delete mode 100644 app/registries/seed/official/configs/solarthermalworld.org/news.yml delete mode 100644 app/registries/seed/official/configs/spektrum.de/meistgelesen.yml delete mode 100644 app/registries/seed/official/configs/spiegel.de/impressum_autor.yml delete mode 100644 app/registries/seed/official/configs/spotify.com/newsroom.yml delete mode 100644 app/registries/seed/official/configs/stackoverflow.com/hot_network_questions.yml delete mode 100644 app/registries/seed/official/configs/steuerzahler.de/news.yml delete mode 100644 app/registries/seed/official/configs/stripes.com/index.yml delete mode 100644 app/registries/seed/official/configs/support.apple.com/en_gb_ht201222.yml delete mode 100644 app/registries/seed/official/configs/support.apple.com/exchange_repair.yml delete mode 100644 app/registries/seed/official/configs/tap.info.tn/politics.yml delete mode 100644 app/registries/seed/official/configs/telenor.com/newsroom.yml delete mode 100644 app/registries/seed/official/configs/teneriffa-news.com/news.yml delete mode 100644 app/registries/seed/official/configs/tennet.eu/news.yml delete mode 100644 app/registries/seed/official/configs/test.de/archiv.yml delete mode 100644 app/registries/seed/official/configs/theeastafrican.co.ke/news.yml delete mode 100644 app/registries/seed/official/configs/theguardian.com/international_mostpopular.yml delete mode 100644 app/registries/seed/official/configs/thenationalnews.com/news.yml delete mode 100644 app/registries/seed/official/configs/thoughtworks.com/insights.yml delete mode 100644 app/registries/seed/official/configs/tno.nl/news.yml delete mode 100644 app/registries/seed/official/configs/tourismusnetzwerk-brandenburg.de/aktuelle_nachrichten.yml delete mode 100644 app/registries/seed/official/configs/tralac.org/news.yml delete mode 100644 app/registries/seed/official/configs/tudelft.nl/news.yml delete mode 100644 app/registries/seed/official/configs/tue.nl/news.yml delete mode 100644 app/registries/seed/official/configs/vattenfall.com/newsroom.yml delete mode 100644 app/registries/seed/official/configs/visitfinland.com/press-releases.yml delete mode 100644 app/registries/seed/official/configs/webentwickler-jobs.de/in.yml delete mode 100644 app/registries/seed/official/configs/who.int/news.yml delete mode 100644 app/registries/seed/official/configs/worldbank.org/news.yml delete mode 100644 app/registries/seed/official/configs/www.consilium.europa.eu/press-releases.yml delete mode 100644 app/registries/seed/official/configs/www.ecdc.europa.eu/news-events.yml delete mode 100644 app/registries/seed/official/configs/www.enisa.europa.eu/news.yml delete mode 100644 app/registries/seed/official/configs/www.europarl.europa.eu/press-room.yml delete mode 100644 app/registries/seed/official/configs/www.gob.mx/profeco-prensa.yml delete mode 100644 app/registries/seed/official/manifest.json diff --git a/app/registries/seed/official/configs/aalto.fi/news.yml b/app/registries/seed/official/configs/aalto.fi/news.yml deleted file mode 100644 index 841a424de..000000000 --- a/app/registries/seed/official/configs/aalto.fi/news.yml +++ /dev/null @@ -1,28 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: aalto.fi/news -directory: - topics: - - science - - research - title: "Aalto University — News" - summary: "Research and campus news from Aalto University in Finland." -strategy: botasaurus -channel: - title: "Aalto University — News" - url: https://www.aalto.fi/en/news - language: en - time_zone: Europe/Helsinki - ttl: 360 -request: - botasaurus: - wait_for_selector: 'a[href^="/en/news/"]' - wait_timeout_seconds: 20 -selectors: - items: - selector: 'a.hub-listing__results__item__link[href^="/en/news/"]' - enhance: true - title: - selector: .hub-listing__results__item__heading - url: - extractor: href diff --git a/app/registries/seed/official/configs/academictransfer.com/jobs.yml b/app/registries/seed/official/configs/academictransfer.com/jobs.yml deleted file mode 100644 index 60beb2faf..000000000 --- a/app/registries/seed/official/configs/academictransfer.com/jobs.yml +++ /dev/null @@ -1,28 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: academictransfer.com/jobs -directory: - topics: - - jobs - title: "AcademicTransfer — Jobs" - summary: "Academic and research job listings in the Netherlands and beyond." -strategy: botasaurus -channel: - title: "AcademicTransfer — Jobs" - url: https://www.academictransfer.com/en/jobs/ - language: en - time_zone: Europe/Amsterdam - ttl: 360 -request: - botasaurus: - wait_for_selector: 'article a[href*="/en/jobs/"]' - wait_timeout_seconds: 20 -selectors: - items: - selector: article - enhance: true - title: - selector: h3 - url: - selector: 'a[href*="/en/jobs/"]' - extractor: href diff --git a/app/registries/seed/official/configs/adfc.de/pressemitteilungen.yml b/app/registries/seed/official/configs/adfc.de/pressemitteilungen.yml deleted file mode 100644 index 27a182621..000000000 --- a/app/registries/seed/official/configs/adfc.de/pressemitteilungen.yml +++ /dev/null @@ -1,24 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: adfc.de/pressemitteilungen -directory: - topics: - - civic - title: "ADFC — Pressemitteilungen" - summary: "Pressemitteilungen des Allgemeinen Deutschen Fahrrad-Clubs." -channel: - title: "ADFC — Pressemitteilungen" - url: https://www.adfc.de/presse/pressemitteilungen/ - time_zone: Europe/Berlin - ttl: 720 - language: de -selectors: - items: - selector: "#articleteasercontainer-columns .articleteaser" - title: - selector: ".articleteaser-header-container a" - url: - selector: "a:first" - extractor: "href" - description: - selector: ".articleteaser-header-container p" diff --git a/app/registries/seed/official/configs/ahram.org.eg/egypt.yml b/app/registries/seed/official/configs/ahram.org.eg/egypt.yml deleted file mode 100644 index 518e156fe..000000000 --- a/app/registries/seed/official/configs/ahram.org.eg/egypt.yml +++ /dev/null @@ -1,22 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: ahram.org.eg/egypt -directory: - topics: - - news - title: "Al-Ahram — Egypt" - summary: "Egypt news coverage from Al-Ahram in English." -channel: - title: "Al-Ahram — Egypt" - url: https://english.ahram.org.eg/Portal/1/Egypt.aspx - language: en - time_zone: Africa/Cairo - ttl: 360 -selectors: - items: - selector: 'h4.title a[href*="/News/"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/ai.meta.com/blog.yml b/app/registries/seed/official/configs/ai.meta.com/blog.yml deleted file mode 100644 index 366400672..000000000 --- a/app/registries/seed/official/configs/ai.meta.com/blog.yml +++ /dev/null @@ -1,28 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: ai.meta.com/blog -directory: - topics: - - tech - - research - title: "Meta AI — Blog" - summary: "Research posts and product updates from Meta AI." -channel: - title: "Meta AI — Blog" - url: https://ai.meta.com/blog/ - language: en - time_zone: UTC - ttl: 360 -selectors: - items: - selector: 'a[href*="/blog/"][aria-label^="Read "]' - enhance: false - title: - extractor: attribute - attribute: aria-label - post_process: - - name: gsub - pattern: "/^Read\\s+/" - replacement: "" - url: - extractor: href diff --git a/app/registries/seed/official/configs/akorda.kz/events.yml b/app/registries/seed/official/configs/akorda.kz/events.yml deleted file mode 100644 index c77ac858a..000000000 --- a/app/registries/seed/official/configs/akorda.kz/events.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: akorda.kz/events -directory: - topics: - - civic - - news - title: "Akorda — Events" - summary: "Official events and statements from the President of Kazakhstan." -channel: - title: "Akorda — Events" - url: https://www.akorda.kz/en/events - language: en - time_zone: Asia/Almaty - ttl: 360 -selectors: - items: - selector: ".card h3 a" - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/angop.ao/noticias.yml b/app/registries/seed/official/configs/angop.ao/noticias.yml deleted file mode 100644 index 4eafef1a0..000000000 --- a/app/registries/seed/official/configs/angop.ao/noticias.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: angop.ao/noticias -directory: - topics: - - news - - civic - title: "ANGOP — Notícias" - summary: "Notícias oficiais da Agência Angola Press." -channel: - title: "ANGOP — Notícias" - url: https://www.angop.ao/ - language: pt - time_zone: Africa/Luanda - ttl: 360 -selectors: - items: - selector: 'h2 a[href*="/noticias/"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/anthropic.com/news.yml b/app/registries/seed/official/configs/anthropic.com/news.yml deleted file mode 100644 index af1cbc812..000000000 --- a/app/registries/seed/official/configs/anthropic.com/news.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: anthropic.com/news -directory: - topics: - - tech - - research - title: "Anthropic — News" - summary: "Product and research announcements from Anthropic." -channel: - title: "Anthropic — News" - url: https://www.anthropic.com/news - language: en - time_zone: UTC - ttl: 360 -selectors: - items: - selector: 'ul[class*="PublicationList"] > li > a[href^="/news/"]' - enhance: false - title: - selector: '[class*="__title"]' - url: - extractor: href diff --git a/app/registries/seed/official/configs/anu.edu.au/news.yml b/app/registries/seed/official/configs/anu.edu.au/news.yml deleted file mode 100644 index 2e4f5aa47..000000000 --- a/app/registries/seed/official/configs/anu.edu.au/news.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: anu.edu.au/news -directory: - topics: - - science - - research - title: "ANU — News" - summary: "University news and research highlights from the Australian National University." -channel: - title: "ANU — News" - url: https://www.anu.edu.au/news/all-news - language: en - time_zone: Australia/Sydney - ttl: 360 -selectors: - items: - selector: 'h3 a[href*="/news/all-news/"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/apnews.com/hub.yml b/app/registries/seed/official/configs/apnews.com/hub.yml deleted file mode 100644 index 69a22acde..000000000 --- a/app/registries/seed/official/configs/apnews.com/hub.yml +++ /dev/null @@ -1,30 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json ---- -registry: - id: apnews.com/hub -directory: - topics: - - news - title: "AP News — Hub" - summary: "Top stories from a selectable Associated Press news hub." -parameters: - section: - type: string - default: "news" - -channel: - title: "AP News — Hub" - url: https://apnews.com/%
s - language: en - ttl: 120 - time_zone: UTC -selectors: - items: - selector: ".PagePromo" - title: - selector: h3 - url: - selector: a:first - extractor: href - description: - selector: .PagePromoContentIcons-text diff --git a/app/registries/seed/official/configs/apple.com/newsroom.yml b/app/registries/seed/official/configs/apple.com/newsroom.yml deleted file mode 100644 index 7b301e61c..000000000 --- a/app/registries/seed/official/configs/apple.com/newsroom.yml +++ /dev/null @@ -1,31 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: apple.com/newsroom -directory: - topics: - - tech - - product - title: "Apple — Newsroom" - summary: "Product announcements and press releases from Apple." -strategy: botasaurus - -channel: - title: "Apple — Newsroom" - url: https://www.apple.com/newsroom/ - language: en - time_zone: UTC - ttl: 360 -request: - botasaurus: - wait_for_selector: "li.tile-item" - wait_timeout_seconds: 20 -selectors: - items: - selector: "li.tile-item a.tile" - enhance: false - title: - selector: .tile__headline - url: - extractor: href - published_at: - selector: .tile__timestamp diff --git a/app/registries/seed/official/configs/aps.dz/national-news.yml b/app/registries/seed/official/configs/aps.dz/national-news.yml deleted file mode 100644 index 7a0bfa88b..000000000 --- a/app/registries/seed/official/configs/aps.dz/national-news.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: aps.dz/national-news -directory: - topics: - - news - - civic - title: "APS — National news" - summary: "National news from Algeria’s official press agency (APS)." -channel: - title: "APS — National news" - url: https://www.aps.dz/en - language: en - time_zone: Africa/Algiers - ttl: 360 -selectors: - items: - selector: 'a[href*="/en/algeria/national-news/"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/asml.com/press-releases.yml b/app/registries/seed/official/configs/asml.com/press-releases.yml deleted file mode 100644 index 5550871e0..000000000 --- a/app/registries/seed/official/configs/asml.com/press-releases.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: asml.com/press-releases -directory: - topics: - - tech - - product - title: "ASML — Press releases" - summary: "Corporate press releases from semiconductor equipment maker ASML." -channel: - title: "ASML — Press releases" - url: https://www.asml.com/en/news/press-releases - language: en - time_zone: Europe/Amsterdam - ttl: 360 -selectors: - items: - selector: 'a.related-content-card[href*="/en/news/press-releases/20"]' - enhance: false - title: - selector: span.related-content-card-title - url: - extractor: href diff --git a/app/registries/seed/official/configs/aspi.org.au/strategist.yml b/app/registries/seed/official/configs/aspi.org.au/strategist.yml deleted file mode 100644 index c4954622c..000000000 --- a/app/registries/seed/official/configs/aspi.org.au/strategist.yml +++ /dev/null @@ -1,29 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: aspi.org.au/strategist -directory: - topics: - - civic - - research - title: "ASPI — The Strategist" - summary: "Strategic analysis and commentary from the Australian Strategic Policy Institute." -strategy: botasaurus - -channel: - title: "ASPI — The Strategist" - url: https://www.aspi.org.au/strategist-posts/ - language: en - time_zone: Australia/Sydney - ttl: 360 -request: - botasaurus: - wait_for_selector: 'a[rel="bookmark"][href*="/strategist-posts/"]' - wait_timeout_seconds: 20 -selectors: - items: - selector: 'a[rel="bookmark"][href*="/strategist-posts/"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/bbc.co.uk/available_episodes.yml b/app/registries/seed/official/configs/bbc.co.uk/available_episodes.yml deleted file mode 100644 index d0154422e..000000000 --- a/app/registries/seed/official/configs/bbc.co.uk/available_episodes.yml +++ /dev/null @@ -1,29 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: bbc.co.uk/available_episodes -directory: - topics: - - entertainment - title: "BBC Sounds — Programme episodes" - summary: "Available episodes for a BBC programme on Sounds." -parameters: - id: - type: string - default: "b006wkfp" - -channel: - language: en - title: "BBC Sounds — Programme episodes" - url: https://www.bbc.co.uk/programmes/%s/episodes/player - time_zone: UTC - ttl: 720 -selectors: - items: - selector: ".highlight-box-wrapper > div" - title: - selector: ".programme__title" - url: - selector: ".programme__titles > a" - extractor: "href" - description: - selector: ".programme__synopsis" diff --git a/app/registries/seed/official/configs/bbc.com/mundo.yml b/app/registries/seed/official/configs/bbc.com/mundo.yml deleted file mode 100644 index 3cde3ea9a..000000000 --- a/app/registries/seed/official/configs/bbc.com/mundo.yml +++ /dev/null @@ -1,25 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json ---- -registry: - id: bbc.com/mundo -directory: - topics: - - news - title: "BBC — Mundo" - summary: "Noticias en español de BBC Mundo." -channel: - title: "BBC — Mundo" - url: https://www.bbc.com/mundo - language: es - ttl: 360 - time_zone: UTC -selectors: - items: - selector: '[data-testid="topic-promos"] > li' - title: - selector: a - url: - selector: a:first - extractor: href - description: - selector: p diff --git a/app/registries/seed/official/configs/blog.mondediplo.net/feed.yml b/app/registries/seed/official/configs/blog.mondediplo.net/feed.yml deleted file mode 100644 index 7d723b9af..000000000 --- a/app/registries/seed/official/configs/blog.mondediplo.net/feed.yml +++ /dev/null @@ -1,29 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: blog.mondediplo.net/feed -directory: - topics: - - news - title: "Le Monde diplomatique — Blog feed" - summary: "Blog posts from Le Monde diplomatique with customizable blog path." -parameters: - blog: - type: string - default: "-Defense-en-ligne-" - -channel: - language: en - url: https://blog.mondediplo.net/%s - time_zone: Europe/Paris - title: "Le Monde diplomatique — Blog feed" - ttl: 120 -selectors: - items: - selector: ".large > ul > li" - title: - selector: "h3" - description: - selector: ".intro" - url: - selector: "a" - extractor: "href" diff --git a/app/registries/seed/official/configs/brookings.edu/articles.yml b/app/registries/seed/official/configs/brookings.edu/articles.yml deleted file mode 100644 index 5dd51ef80..000000000 --- a/app/registries/seed/official/configs/brookings.edu/articles.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: brookings.edu/articles -directory: - topics: - - civic - - research - title: "Brookings — Articles" - summary: "Policy research and commentary from the Brookings Institution." -channel: - title: "Brookings — Articles" - url: https://www.brookings.edu/ - language: en - time_zone: America/New_York - ttl: 360 -selectors: - items: - selector: 'a.overlay-link[href*="/articles/"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/bundesregierung.de/pressemitteilungen.yml b/app/registries/seed/official/configs/bundesregierung.de/pressemitteilungen.yml deleted file mode 100644 index b7673581f..000000000 --- a/app/registries/seed/official/configs/bundesregierung.de/pressemitteilungen.yml +++ /dev/null @@ -1,28 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: bundesregierung.de/pressemitteilungen -directory: - topics: - - civic - title: "Bundesregierung — Pressemitteilungen" - summary: "Offizielle Pressemitteilungen der deutschen Bundesregierung." -strategy: botasaurus - -channel: - title: "Bundesregierung — Pressemitteilungen" - url: https://www.bundesregierung.de/breg-de/aktuelles/pressemitteilungen - language: de - time_zone: Europe/Berlin - ttl: 360 -request: - botasaurus: - wait_for_selector: 'a.bpa-link[href*="/aktuelles/pressemitteilungen/"]' - wait_timeout_seconds: 20 -selectors: - items: - selector: 'a.bpa-link[href*="/breg-de/aktuelles/pressemitteilungen/"]' - enhance: false - title: - selector: span.bpa-teaser-title-text-inner - url: - extractor: href diff --git a/app/registries/seed/official/configs/canarianweekly.com/front.yml b/app/registries/seed/official/configs/canarianweekly.com/front.yml deleted file mode 100644 index 395ed5413..000000000 --- a/app/registries/seed/official/configs/canarianweekly.com/front.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: canarianweekly.com/front -directory: - topics: - - news - - travel - title: "Canarian Weekly — Front page" - summary: "English-language news from the Canary Islands." -channel: - title: "Canarian Weekly — Front page" - url: https://www.canarianweekly.com/ - time_zone: Europe/London - ttl: 720 - language: en -selectors: - items: - selector: "h5.post-title a" - enhance: false - title: - extractor: text - url: - extractor: "href" diff --git a/app/registries/seed/official/configs/caricom.org/news.yml b/app/registries/seed/official/configs/caricom.org/news.yml deleted file mode 100644 index f7dead8ae..000000000 --- a/app/registries/seed/official/configs/caricom.org/news.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: caricom.org/news -directory: - topics: - - civic - - news - title: "CARICOM — News" - summary: "News and updates from the Caribbean Community (CARICOM)." -channel: - title: "CARICOM — News" - url: https://caricom.org/ - language: en - time_zone: America/Jamaica - ttl: 360 -selectors: - items: - selector: a.eael-grid-post-link - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/carnegieendowment.org/research.yml b/app/registries/seed/official/configs/carnegieendowment.org/research.yml deleted file mode 100644 index 9fc241ff0..000000000 --- a/app/registries/seed/official/configs/carnegieendowment.org/research.yml +++ /dev/null @@ -1,29 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: carnegieendowment.org/research -directory: - topics: - - civic - - research - title: "Carnegie Endowment — Research" - summary: "Foreign policy research and analysis from the Carnegie Endowment." -strategy: botasaurus - -channel: - title: "Carnegie Endowment — Research" - url: https://carnegieendowment.org/research - language: en - time_zone: UTC - ttl: 360 -request: - botasaurus: - wait_for_selector: 'a.min-w-0.flex-1[href^="/research/20"]' - wait_timeout_seconds: 20 -selectors: - items: - selector: 'a.min-w-0.flex-1[href^="/research/20"]' - enhance: false - title: - selector: span.text-headlineH5 - url: - extractor: href diff --git a/app/registries/seed/official/configs/cepal.org/press-releases.yml b/app/registries/seed/official/configs/cepal.org/press-releases.yml deleted file mode 100644 index 918ff530a..000000000 --- a/app/registries/seed/official/configs/cepal.org/press-releases.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: cepal.org/press-releases -directory: - topics: - - civic - - research - title: "ECLAC — Press releases" - summary: "Press releases from the UN Economic Commission for Latin America and the Caribbean." -channel: - title: "ECLAC — Press releases" - url: https://www.cepal.org/en/list/cepal_pr - language: en - time_zone: America/Santiago - ttl: 360 -selectors: - items: - selector: 'h4 a[href*="/en/pressreleases/"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/chalmers.se/news.yml b/app/registries/seed/official/configs/chalmers.se/news.yml deleted file mode 100644 index 224010b89..000000000 --- a/app/registries/seed/official/configs/chalmers.se/news.yml +++ /dev/null @@ -1,28 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: chalmers.se/news -directory: - topics: - - science - - research - title: "Chalmers — News" - summary: "Research and university news from Chalmers University of Technology." -strategy: botasaurus -channel: - title: "Chalmers — News" - url: https://www.chalmers.se/en/current/news/ - language: en - time_zone: Europe/Stockholm - ttl: 360 -request: - botasaurus: - wait_for_selector: 'a[href*="/en/current/news/"]' - wait_timeout_seconds: 20 -selectors: - items: - selector: 'a[data-component="NewsListItem"]' - enhance: true - title: - selector: p.line-clamp-2 - url: - extractor: href diff --git a/app/registries/seed/official/configs/chathamhouse.org/research-publications.yml b/app/registries/seed/official/configs/chathamhouse.org/research-publications.yml deleted file mode 100644 index 2f4f1fcfc..000000000 --- a/app/registries/seed/official/configs/chathamhouse.org/research-publications.yml +++ /dev/null @@ -1,29 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: chathamhouse.org/research-publications -directory: - topics: - - civic - - research - title: "Chatham House — Research publications" - summary: "Research publications from the Royal Institute of International Affairs." -strategy: botasaurus - -channel: - title: "Chatham House — Research publications" - url: https://www.chathamhouse.org/publications/research-publications - language: en - time_zone: Europe/London - ttl: 360 -request: - botasaurus: - wait_for_selector: a.teaser__link - wait_timeout_seconds: 20 -selectors: - items: - selector: a.teaser__link - enhance: false - title: - selector: .teaser__title - url: - extractor: href diff --git a/app/registries/seed/official/configs/cinemascore.com/index.yml b/app/registries/seed/official/configs/cinemascore.com/index.yml deleted file mode 100644 index 1e0113024..000000000 --- a/app/registries/seed/official/configs/cinemascore.com/index.yml +++ /dev/null @@ -1,55 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: cinemascore.com/index -directory: - topics: - - entertainment - title: "CinemaScore — Audience grades" - summary: "Opening-night audience grades for new theatrical releases." -channel: - language: en - title: "CinemaScore — Audience grades" - url: https://webapp.cinemascore.com/guest/surveys - ttl: 720 - json: true - time_zone: America/Los_Angeles -selectors: - items: - selector: "new > array > object, recent > array > object" - title: - selector: "title" - post_process: - - name: "template" - string: "%{self} (%{grade}) [Rated %{mpaa_rating}] (%{year})" - grade: - selector: "grade" - movie_title: - selector: "movie_title" - url: - post_process: - - name: "template" - string: "https://www.metacritic.com/search/movie/%{movie_title}/results" - mpaa_rating: - selector: "mpaa_rating" - distributor: - selector: "distributor" - year: - selector: "year" - categories: - - mpaa_rating - - grade - - year - description: - post_process: - - name: template - string: | - # %{movie_title} - - Grade: %{grade} - - MPAA Rating: %{mpaa_rating} - - Distributor: %{distributor} - - Year: %{year} - - name: markdown_to_html diff --git a/app/registries/seed/official/configs/cleanenergywire.org/news.yml b/app/registries/seed/official/configs/cleanenergywire.org/news.yml deleted file mode 100644 index 18fb1956d..000000000 --- a/app/registries/seed/official/configs/cleanenergywire.org/news.yml +++ /dev/null @@ -1,28 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: cleanenergywire.org/news -directory: - topics: - - energy - - environment - title: "Clean Energy Wire — News" - summary: "Journalism on Germany’s energy transition and climate policy." -channel: - language: en - title: "Clean Energy Wire — News" - url: https://www.cleanenergywire.org/news - time_zone: "Europe/Berlin" - ttl: 360 -selectors: - items: - selector: ".views-row" - enhance: false - title: - selector: "h2 a" - url: - selector: "h2 a" - extractor: "href" - published_at: - selector: "time" - post_process: - - name: parse_time diff --git a/app/registries/seed/official/configs/cnet.com/section_sub.yml b/app/registries/seed/official/configs/cnet.com/section_sub.yml deleted file mode 100644 index f81848134..000000000 --- a/app/registries/seed/official/configs/cnet.com/section_sub.yml +++ /dev/null @@ -1,30 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json ---- -registry: - id: cnet.com/section_sub -directory: - topics: - - tech - - news - title: "CNET — Section" - summary: "Tech news and reviews from a selectable CNET section." -parameters: - section: - type: string - default: "tech" - -channel: - title: "CNET — Section" - url: https://www.cnet.com/%
s/ - language: en - ttl: 360 - time_zone: UTC -selectors: - items: - selector: "article.list-entry" - enhance: false - title: - selector: "h3.list-entry__title a" - url: - selector: "h3.list-entry__title a" - extractor: href diff --git a/app/registries/seed/official/configs/cohere.com/blog.yml b/app/registries/seed/official/configs/cohere.com/blog.yml deleted file mode 100644 index 8e206d84f..000000000 --- a/app/registries/seed/official/configs/cohere.com/blog.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: cohere.com/blog -directory: - topics: - - tech - - research - title: "Cohere — Blog" - summary: "Product and research updates from Cohere." -channel: - title: "Cohere — Blog" - url: https://cohere.com/blog - language: en - time_zone: UTC - ttl: 360 -selectors: - items: - selector: 'a[href^="/blog/"][class*="flex-1"]' - enhance: false - title: - selector: "> p:first-of-type" - url: - extractor: href diff --git a/app/registries/seed/official/configs/computerbase.de/meistgelesen.yml b/app/registries/seed/official/configs/computerbase.de/meistgelesen.yml deleted file mode 100644 index 7af302e38..000000000 --- a/app/registries/seed/official/configs/computerbase.de/meistgelesen.yml +++ /dev/null @@ -1,27 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: computerbase.de/meistgelesen -directory: - topics: - - tech - title: "ComputerBase — Meistgelesen" - summary: "Most-read tech articles from ComputerBase." -channel: - title: "ComputerBase — Meistgelesen" - url: https://www.computerbase.de - time_zone: Europe/Berlin - ttl: 360 - language: de -selectors: - items: - selector: ".charts > li" - title: - selector: ".charts__title" - post_process: - - name: "template" - string: "%{context}: %{self}" - context: - selector: ".charts__context" - url: - selector: ".charts__link" - extractor: "href" diff --git a/app/registries/seed/official/configs/cplp.org/noticias.yml b/app/registries/seed/official/configs/cplp.org/noticias.yml deleted file mode 100644 index bc5d577a9..000000000 --- a/app/registries/seed/official/configs/cplp.org/noticias.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: cplp.org/noticias -directory: - topics: - - civic - - news - title: "CPLP — Notícias" - summary: "Notícias da Comunidade dos Países de Língua Portuguesa." -channel: - title: "CPLP — Notícias" - url: https://www.cplp.org/ - language: pt - time_zone: Europe/Lisbon - ttl: 360 -selectors: - items: - selector: 'ul.news-to-show-list a[href*="/noticias/noticias-detalhe/"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/csiro.au/news.yml b/app/registries/seed/official/configs/csiro.au/news.yml deleted file mode 100644 index 30bdb3684..000000000 --- a/app/registries/seed/official/configs/csiro.au/news.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: csiro.au/news -directory: - topics: - - science - - research - title: "CSIRO — News" - summary: "Science and innovation news from Australia’s national science agency." -channel: - title: "CSIRO — News" - url: https://www.csiro.au/en/news - language: en - time_zone: Australia/Sydney - ttl: 360 -selectors: - items: - selector: 'a[href*="/en/news/All/Articles/"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/curia.europa.eu/press-releases.yml b/app/registries/seed/official/configs/curia.europa.eu/press-releases.yml deleted file mode 100644 index 612e548f6..000000000 --- a/app/registries/seed/official/configs/curia.europa.eu/press-releases.yml +++ /dev/null @@ -1,29 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: curia.europa.eu/press-releases -directory: - topics: - - civic - title: "Court of Justice of the EU — Press releases" - summary: "Press releases from the Court of Justice of the European Union." -strategy: botasaurus - -channel: - title: "Court of Justice of the EU — Press releases" - url: https://curia.europa.eu/jcms/jcms/Jo2_7052/ - language: en - time_zone: Europe/Brussels - ttl: 360 -request: - botasaurus: - wait_for_selector: li.curia-itemlist-item - wait_timeout_seconds: 20 -selectors: - items: - selector: li.curia-itemlist-item - enhance: false - title: - selector: .curia-itemlist-item-title h3 - url: - selector: .curia-itemlist-item-title a - extractor: href diff --git a/app/registries/seed/official/configs/cutle.fish/index.yml b/app/registries/seed/official/configs/cutle.fish/index.yml deleted file mode 100644 index 4224b7184..000000000 --- a/app/registries/seed/official/configs/cutle.fish/index.yml +++ /dev/null @@ -1,25 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json ---- -registry: - id: cutle.fish/index -directory: - topics: - - tech - title: "cutle.fish — Posts" - summary: "Personal writing and notes from cutle.fish." -channel: - language: en - title: "cutle.fish — Posts" - url: https://cutle.fish/ - ttl: 360 - time_zone: UTC -selectors: - items: - selector: main > div > div - title: - selector: a - url: - selector: a - extractor: href - description: - selector: a diff --git a/app/registries/seed/official/configs/deepmind.google/blog.yml b/app/registries/seed/official/configs/deepmind.google/blog.yml deleted file mode 100644 index 6d3733c1e..000000000 --- a/app/registries/seed/official/configs/deepmind.google/blog.yml +++ /dev/null @@ -1,27 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: deepmind.google/blog -directory: - topics: - - science - - research - title: "Google DeepMind — Blog" - summary: "Research and product posts from Google DeepMind." - -channel: - title: "Google DeepMind — Blog" - url: https://deepmind.google/blog/ - language: en - time_zone: UTC - ttl: 360 -selectors: - items: - selector: .card__inner - enhance: false - title: - selector: h3 - url: - selector: .card__overlay-link - extractor: href - published_at: - selector: time diff --git a/app/registries/seed/official/configs/deraktionaer.de/meistgelesen.yml b/app/registries/seed/official/configs/deraktionaer.de/meistgelesen.yml deleted file mode 100644 index 14c093739..000000000 --- a/app/registries/seed/official/configs/deraktionaer.de/meistgelesen.yml +++ /dev/null @@ -1,22 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: deraktionaer.de/meistgelesen -directory: - topics: - - finance - title: "Der Aktionär — Meistgelesen" - summary: "Most-read stock market news from Der Aktionär." -channel: - title: "Der Aktionär — Meistgelesen" - url: https://www.deraktionaer.de/ - time_zone: Europe/Berlin - ttl: 360 - language: de -enhance: false -selectors: - items: - selector: "section#top-articles article.top-article a.top-article-content[href^='/artikel/']" - title: - extractor: "text" - url: - extractor: "href" diff --git a/app/registries/seed/official/configs/developer.apple.com/tutorials_data_documentation_technotes_json.yml b/app/registries/seed/official/configs/developer.apple.com/tutorials_data_documentation_technotes_json.yml deleted file mode 100644 index 4026f9ab6..000000000 --- a/app/registries/seed/official/configs/developer.apple.com/tutorials_data_documentation_technotes_json.yml +++ /dev/null @@ -1,28 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json ---- -registry: - id: developer.apple.com/tutorials_data_documentation_technotes_json -directory: - topics: - - tech - - product - title: "Apple Developer — Technotes" - summary: "Apple Developer Technotes documentation feed (JSON source)." -channel: - language: en - json: true - title: "Apple Developer — Technotes" - url: https://developer.apple.com/tutorials/data/documentation/Technotes.json - ttl: 360 - time_zone: UTC -selectors: - items: - selector: "references > object > * > object:has(url)" - title: - selector: title - url: - selector: url - post_process: - - name: parse_uri - description: - selector: abstract text diff --git a/app/registries/seed/official/configs/dfs.de/pressemitteilungen.yml b/app/registries/seed/official/configs/dfs.de/pressemitteilungen.yml deleted file mode 100644 index 00dca1f84..000000000 --- a/app/registries/seed/official/configs/dfs.de/pressemitteilungen.yml +++ /dev/null @@ -1,28 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: dfs.de/pressemitteilungen -directory: - topics: - - civic - title: "DFS — Pressemitteilungen" - summary: "Pressemitteilungen der Deutschen Flugsicherung (DFS)." -channel: - title: "DFS — Pressemitteilungen" - url: https://www.dfs.de/homepage/de/medien/presse/ - time_zone: Europe/Berlin - ttl: 1440 - language: de -selectors: - items: - selector: ".article-teaser-list article" - title: - selector: ".headline" - url: - selector: "a" - extractor: "href" - published_at: - selector: "time" - extractor: "attribute" - attribute: "datetime" - post_process: - - name: "parse_time" diff --git a/app/registries/seed/official/configs/diis.dk/research.yml b/app/registries/seed/official/configs/diis.dk/research.yml deleted file mode 100644 index 2ba60c1c8..000000000 --- a/app/registries/seed/official/configs/diis.dk/research.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: diis.dk/research -directory: - topics: - - civic - - research - title: "DIIS — Research" - summary: "Research and analysis from the Danish Institute for International Studies." -channel: - title: "DIIS — Research" - url: https://www.diis.dk/en - language: en - time_zone: Europe/Copenhagen - ttl: 360 -selectors: - items: - selector: 'h3 a[href^="/en/research/"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/dirco.gov.za/media-statements.yml b/app/registries/seed/official/configs/dirco.gov.za/media-statements.yml deleted file mode 100644 index bb999f73d..000000000 --- a/app/registries/seed/official/configs/dirco.gov.za/media-statements.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: dirco.gov.za/media-statements -directory: - topics: - - civic - - news - title: "DIRCO — Media statements" - summary: "Media statements from South Africa’s Department of International Relations." -channel: - title: "DIRCO — Media statements" - url: https://dirco.gov.za/media-statements/ - language: en - time_zone: Africa/Johannesburg - ttl: 360 -selectors: - items: - selector: "div.pgafu-post-grid-content h2 a" - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/dsw-info.de/presse.yml b/app/registries/seed/official/configs/dsw-info.de/presse.yml deleted file mode 100644 index 69f870c9f..000000000 --- a/app/registries/seed/official/configs/dsw-info.de/presse.yml +++ /dev/null @@ -1,30 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: dsw-info.de/presse -directory: - topics: - - civic - - consumer - title: "Deutsche Stiftung Weltbevölkerung — Presse" - summary: "Pressemitteilungen der Deutschen Stiftung Weltbevölkerung." -channel: - title: "Deutsche Stiftung Weltbevölkerung — Presse" - url: https://www.dsw-info.de/presse/pressemitteilungen-2026/ - time_zone: Europe/Berlin - ttl: 720 - language: de -selectors: - items: - selector: "main .frame.cet-menu ul > li" - enhance: false - title: - selector: "a" - url: - selector: "a" - extractor: "href" - published_at: - selector: ".vd small" - post_process: - - name: parse_time - description: - selector: ".bt small" diff --git a/app/registries/seed/official/configs/elastic.co/elasticsearch-release-notes.yml b/app/registries/seed/official/configs/elastic.co/elasticsearch-release-notes.yml deleted file mode 100644 index e73169f53..000000000 --- a/app/registries/seed/official/configs/elastic.co/elasticsearch-release-notes.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: elastic.co/elasticsearch-release-notes -directory: - topics: - - tech - - product - title: "Elastic — Elasticsearch release notes" - summary: "Release notes for Elasticsearch from Elastic." -channel: - title: "Elastic — Elasticsearch release notes" - url: https://www.elastic.co/docs/release-notes/elasticsearch - language: en - time_zone: UTC - ttl: 360 -selectors: - items: - selector: 'a[href^="#elasticsearch-"][href$="-release-notes"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/elysee.fr/actualites.yml b/app/registries/seed/official/configs/elysee.fr/actualites.yml deleted file mode 100644 index 11380d0a0..000000000 --- a/app/registries/seed/official/configs/elysee.fr/actualites.yml +++ /dev/null @@ -1,22 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: elysee.fr/actualites -directory: - topics: - - civic - title: "Élysée — Actualités" - summary: "Actualités officielles de la Présidence de la République française." -channel: - title: "Élysée — Actualités" - url: https://www.elysee.fr/toutes-les-actualites - language: fr - time_zone: Europe/Paris - ttl: 360 -selectors: - items: - selector: 'a[href*="/emmanuel-macron/20"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/equinor.com/news.yml b/app/registries/seed/official/configs/equinor.com/news.yml deleted file mode 100644 index 76b365e4c..000000000 --- a/app/registries/seed/official/configs/equinor.com/news.yml +++ /dev/null @@ -1,22 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: equinor.com/news -directory: - topics: - - energy - title: "Equinor — News" - summary: "Corporate and energy news from Equinor." -channel: - title: "Equinor — News" - url: https://www.equinor.com/news - language: en - time_zone: Europe/Oslo - ttl: 360 -selectors: - items: - selector: 'a[href^="/news/"]' - enhance: true - title: - selector: h2 - url: - extractor: href diff --git a/app/registries/seed/official/configs/espn.com/f1.yml b/app/registries/seed/official/configs/espn.com/f1.yml deleted file mode 100644 index 88f38c62d..000000000 --- a/app/registries/seed/official/configs/espn.com/f1.yml +++ /dev/null @@ -1,22 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: espn.com/f1 -directory: - topics: - - sports - title: "ESPN — Formula 1" - summary: "Formula 1 news and coverage from ESPN." -channel: - language: en - title: "ESPN — Formula 1" - url: https://www.espn.com/f1/ - time_zone: UTC - ttl: 60 -selectors: - items: - selector: ".headlineStack__list > li" - title: - selector: "a" - url: - selector: "a" - extractor: "href" diff --git a/app/registries/seed/official/configs/ethz.ch/eth-news.yml b/app/registries/seed/official/configs/ethz.ch/eth-news.yml deleted file mode 100644 index 2093c1165..000000000 --- a/app/registries/seed/official/configs/ethz.ch/eth-news.yml +++ /dev/null @@ -1,29 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: ethz.ch/eth-news -directory: - topics: - - science - - research - title: "ETH Zurich — News" - summary: "Research and university news from ETH Zurich." -strategy: botasaurus - -channel: - title: "ETH Zurich — News" - url: https://ethz.ch/en/news-and-events/eth-news.html - language: en - time_zone: Europe/Zurich - ttl: 360 -request: - botasaurus: - wait_for_selector: 'a[href*="/eth-news/news/20"]' - wait_timeout_seconds: 20 -selectors: - items: - selector: 'a[href*="/eth-news/news/20"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/eures.europa.eu/news.yml b/app/registries/seed/official/configs/eures.europa.eu/news.yml deleted file mode 100644 index e8f877f91..000000000 --- a/app/registries/seed/official/configs/eures.europa.eu/news.yml +++ /dev/null @@ -1,22 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: eures.europa.eu/news -directory: - topics: - - jobs - title: "EURES — News" - summary: "Job mobility news from the European Employment Services network." -channel: - title: "EURES — News" - url: https://eures.europa.eu/news_en - language: en - time_zone: Europe/Brussels - ttl: 360 -selectors: - items: - selector: 'article.ecl-content-item a[href$="_en"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/faa.gov/newsroom.yml b/app/registries/seed/official/configs/faa.gov/newsroom.yml deleted file mode 100644 index faf427472..000000000 --- a/app/registries/seed/official/configs/faa.gov/newsroom.yml +++ /dev/null @@ -1,22 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: faa.gov/newsroom -directory: - topics: - - civic - title: "FAA — Newsroom" - summary: "News and announcements from the U.S. Federal Aviation Administration." -channel: - title: "FAA — Newsroom" - url: https://www.faa.gov/newsroom - language: en - time_zone: UTC - ttl: 360 -selectors: - items: - selector: '.views-row a[href^="/newsroom/"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/fao.org/newsroom.yml b/app/registries/seed/official/configs/fao.org/newsroom.yml deleted file mode 100644 index 22b58ea1f..000000000 --- a/app/registries/seed/official/configs/fao.org/newsroom.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: fao.org/newsroom -directory: - topics: - - environment - - civic - title: "FAO — Newsroom" - summary: "News from the Food and Agriculture Organization of the United Nations." -channel: - title: "FAO — Newsroom" - url: https://www.fao.org/newsroom/en - language: en - time_zone: UTC - ttl: 360 -selectors: - items: - selector: 'a[href*="/newsroom/detail/"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/fia.com/documents.yml b/app/registries/seed/official/configs/fia.com/documents.yml deleted file mode 100644 index d0d3be307..000000000 --- a/app/registries/seed/official/configs/fia.com/documents.yml +++ /dev/null @@ -1,26 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json ---- -registry: - id: fia.com/documents -directory: - topics: - - sports - title: "FIA — Formula 1 documents" - summary: "Official Formula 1 championship documents from the FIA." -channel: - language: en - title: "FIA — Formula 1 documents" - url: https://www.fia.com/documents/championships/fia-formula-one-world-championship-14/season/season-2026-2072 - ttl: 360 - time_zone: UTC -selectors: - items: - selector: ".decision-document-list .document-row" - title: - selector: ".title" - url: - selector: a - extractor: href - description: - selector: a - extractor: html diff --git a/app/registries/seed/official/configs/figma.com/blog.yml b/app/registries/seed/official/configs/figma.com/blog.yml deleted file mode 100644 index f9501bcc2..000000000 --- a/app/registries/seed/official/configs/figma.com/blog.yml +++ /dev/null @@ -1,25 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: figma.com/blog -directory: - topics: - - tech - - product - title: "Figma — Blog" - summary: "Product updates and design stories from Figma." -channel: - title: "Figma — Blog" - url: https://www.figma.com/blog/ - language: en - time_zone: UTC - ttl: 360 -selectors: - items: - selector: article[aria-label] - enhance: false - title: - extractor: attribute - attribute: aria-label - url: - selector: 'a[href*="/blog/"]' - extractor: href diff --git a/app/registries/seed/official/configs/formula1.com/latest.yml b/app/registries/seed/official/configs/formula1.com/latest.yml deleted file mode 100644 index be1a8de19..000000000 --- a/app/registries/seed/official/configs/formula1.com/latest.yml +++ /dev/null @@ -1,22 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: formula1.com/latest -directory: - topics: - - sports - title: "Formula 1 — Latest" - summary: "Latest Formula 1 news from the official Formula1.com site." -channel: - language: en - title: "Formula 1 — Latest" - url: https://www.formula1.com/en/latest - time_zone: UTC - ttl: 120 -selectors: - items: - selector: 'a[class*="ArticleListCard-module_title"]' - enhance: false - title: - extractor: text - url: - extractor: "href" diff --git a/app/registries/seed/official/configs/forumsec.org/publications.yml b/app/registries/seed/official/configs/forumsec.org/publications.yml deleted file mode 100644 index 9abd9402f..000000000 --- a/app/registries/seed/official/configs/forumsec.org/publications.yml +++ /dev/null @@ -1,24 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: forumsec.org/publications -directory: - topics: - - civic - - news - title: "Pacific Islands Forum — Publications" - summary: "Publications from the Pacific Islands Forum Secretariat." -channel: - title: "Pacific Islands Forum — Publications" - url: https://forumsec.org/publications - language: en - time_zone: Pacific/Fiji - ttl: 360 -selectors: - items: - selector: 'a.card__image[href^="/publications/"][aria-label]' - enhance: false - title: - extractor: attribute - attribute: aria-label - url: - extractor: href diff --git a/app/registries/seed/official/configs/ftc.gov/press-releases.yml b/app/registries/seed/official/configs/ftc.gov/press-releases.yml deleted file mode 100644 index 1d78e5bb2..000000000 --- a/app/registries/seed/official/configs/ftc.gov/press-releases.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: ftc.gov/press-releases -directory: - topics: - - security - - civic - title: "FTC — Press releases" - summary: "Press releases from the U.S. Federal Trade Commission." -channel: - title: "FTC — Press releases" - url: https://www.ftc.gov/news-events/news/press-releases - language: en - time_zone: UTC - ttl: 360 -selectors: - items: - selector: 'a[href*="/news-events/news/press-releases/20"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/github.com/releases.yml b/app/registries/seed/official/configs/github.com/releases.yml deleted file mode 100644 index cab0a89ac..000000000 --- a/app/registries/seed/official/configs/github.com/releases.yml +++ /dev/null @@ -1,33 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: github.com/releases -directory: - topics: - - tech - - product - title: "GitHub — Releases" - summary: "Release notes for a GitHub repository (owner and name as parameters)." -parameters: - username: - type: string - default: "nuxt" - repository: - type: string - default: "nuxt" - -channel: - language: en - title: "GitHub — Releases" - url: https://github.com/%s/%s/releases - time_zone: UTC - ttl: 720 - description: "Releases of %s/%s on github.com." -selectors: - items: - selector: ".repository-content section" - enhance: false - title: - selector: 'a[href*="/releases/tag/"]' - url: - selector: 'a[href*="/releases/tag/"]' - extractor: "href" diff --git a/app/registries/seed/official/configs/go.dev/release-history.yml b/app/registries/seed/official/configs/go.dev/release-history.yml deleted file mode 100644 index 77c58366b..000000000 --- a/app/registries/seed/official/configs/go.dev/release-history.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: go.dev/release-history -directory: - topics: - - tech - - product - title: "Go — Release history" - summary: "Official Go language release history and notes." -channel: - title: "Go — Release history" - url: https://go.dev/doc/devel/release - language: en - time_zone: UTC - ttl: 360 -selectors: - items: - selector: 'a[href^="/doc/go1."]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/grafana.com/whatsnew.yml b/app/registries/seed/official/configs/grafana.com/whatsnew.yml deleted file mode 100644 index d903cbc6b..000000000 --- a/app/registries/seed/official/configs/grafana.com/whatsnew.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: grafana.com/whatsnew -directory: - topics: - - tech - - product - title: "Grafana — What’s new" - summary: "What’s-new release notes for Grafana." -channel: - title: "Grafana — What’s new" - url: https://grafana.com/docs/grafana/latest/whatsnew/ - language: en - time_zone: UTC - ttl: 360 -selectors: - items: - selector: 'a.docs__menu-a[href^="/docs/grafana/latest/whatsnew/whats-new-in-v"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/haaretz.com/israel-news.yml b/app/registries/seed/official/configs/haaretz.com/israel-news.yml deleted file mode 100644 index 5c856dae1..000000000 --- a/app/registries/seed/official/configs/haaretz.com/israel-news.yml +++ /dev/null @@ -1,22 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: haaretz.com/israel-news -directory: - topics: - - news - title: "Haaretz — Israel news" - summary: "Israel news coverage from Haaretz in English." -channel: - title: "Haaretz — Israel news" - url: https://www.haaretz.com/israel-news - language: en - time_zone: Asia/Jerusalem - ttl: 360 -selectors: - items: - selector: 'h3 a[href*="/ty-article"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/hai.stanford.edu/news.yml b/app/registries/seed/official/configs/hai.stanford.edu/news.yml deleted file mode 100644 index 81a28b3b6..000000000 --- a/app/registries/seed/official/configs/hai.stanford.edu/news.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: hai.stanford.edu/news -directory: - topics: - - science - - research - title: "Stanford HAI — News" - summary: "News from the Stanford Institute for Human-Centered Artificial Intelligence." -channel: - title: "Stanford HAI — News" - url: https://hai.stanford.edu/news - language: en - time_zone: UTC - ttl: 360 -selectors: - items: - selector: 'a[class*="titleLink"][href*="/news/"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/iaapa.org/news.yml b/app/registries/seed/official/configs/iaapa.org/news.yml deleted file mode 100644 index d03cb0e30..000000000 --- a/app/registries/seed/official/configs/iaapa.org/news.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: iaapa.org/news -directory: - topics: - - entertainment - title: "IAAPA — News" - summary: "Industry news from the International Association of Amusement Parks." -channel: - language: en - title: "IAAPA — News" - url: https://iaapa.org/news-funworld - time_zone: UTC - ttl: 720 -enhance: false -selectors: - items: - selector: ".views-row > article" - title: - selector: h3 - url: - selector: "a" - extractor: "href" diff --git a/app/registries/seed/official/configs/icj-cij.org/press-releases.yml b/app/registries/seed/official/configs/icj-cij.org/press-releases.yml deleted file mode 100644 index 45068aaf2..000000000 --- a/app/registries/seed/official/configs/icj-cij.org/press-releases.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: icj-cij.org/press-releases -directory: - topics: - - civic - title: "ICJ — Press releases" - summary: "Press releases from the International Court of Justice." -channel: - title: "ICJ — Press releases" - url: https://www.icj-cij.org/press-releases - language: en - time_zone: UTC - ttl: 360 -selectors: - items: - selector: .views-row - enhance: false - title: - selector: .views-field-field-document-long-title - url: - selector: ".views-field-field-press-release-number a" - extractor: href diff --git a/app/registries/seed/official/configs/icrc.org/news.yml b/app/registries/seed/official/configs/icrc.org/news.yml deleted file mode 100644 index 2516bf9dc..000000000 --- a/app/registries/seed/official/configs/icrc.org/news.yml +++ /dev/null @@ -1,22 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: icrc.org/news -directory: - topics: - - civic - title: "ICRC — News" - summary: "Humanitarian news from the International Committee of the Red Cross." -channel: - title: "ICRC — News" - url: https://www.icrc.org/en/news - language: en - time_zone: Europe/Zurich - ttl: 360 -selectors: - items: - selector: 'a[href^="/en/article/"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/iea.org/news.yml b/app/registries/seed/official/configs/iea.org/news.yml deleted file mode 100644 index 3e854eca9..000000000 --- a/app/registries/seed/official/configs/iea.org/news.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: iea.org/news -directory: - topics: - - energy - - environment - title: "IEA — News" - summary: "Energy news and analysis from the International Energy Agency." -channel: - title: "IEA — News" - url: https://www.iea.org/news - language: en - time_zone: UTC - ttl: 360 -selectors: - items: - selector: 'a.m-news-detailed-listing__link[href^="/news/"]' - enhance: false - title: - selector: .m-news-detailed-listing__title - url: - extractor: href diff --git a/app/registries/seed/official/configs/iiss.org/online-analysis.yml b/app/registries/seed/official/configs/iiss.org/online-analysis.yml deleted file mode 100644 index 0e9b8651b..000000000 --- a/app/registries/seed/official/configs/iiss.org/online-analysis.yml +++ /dev/null @@ -1,29 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: iiss.org/online-analysis -directory: - topics: - - civic - - research - title: "IISS — Online analysis" - summary: "Strategic analysis from the International Institute for Strategic Studies." -strategy: botasaurus - -channel: - title: "IISS — Online analysis" - url: https://www.iiss.org/online-analysis - language: en - time_zone: Europe/London - ttl: 360 -request: - botasaurus: - wait_for_selector: 'a.feature[href*="/online-analysis/"][href*="/20"]' - wait_timeout_seconds: 20 -selectors: - items: - selector: 'a.feature[href*="/online-analysis/"][href*="/20"]' - enhance: false - title: - selector: h2 - url: - extractor: href diff --git a/app/registries/seed/official/configs/ikea.com/newsroom.yml b/app/registries/seed/official/configs/ikea.com/newsroom.yml deleted file mode 100644 index 3a4daa1af..000000000 --- a/app/registries/seed/official/configs/ikea.com/newsroom.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: ikea.com/newsroom -directory: - topics: - - consumer - - product - title: "IKEA — Newsroom" - summary: "Global newsroom announcements from IKEA." -channel: - title: "IKEA — Newsroom" - url: https://www.ikea.com/global/en/newsroom/ - language: en - time_zone: Europe/Stockholm - ttl: 360 -selectors: - items: - selector: 'a.card[href*="/global/en/newsroom/"]' - enhance: true - title: - selector: span.skapa-typography-heading-m - url: - extractor: href diff --git a/app/registries/seed/official/configs/imdb.com/ratings.yml b/app/registries/seed/official/configs/imdb.com/ratings.yml deleted file mode 100644 index 32936b7e3..000000000 --- a/app/registries/seed/official/configs/imdb.com/ratings.yml +++ /dev/null @@ -1,50 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: imdb.com/ratings -directory: - topics: - - entertainment - title: "IMDb — User ratings" - summary: "Film and TV ratings from a selectable IMDb user list." -strategy: botasaurus - -parameters: - user_id: - type: string - default: "ur7019649" - -channel: - language: en - title: "IMDb — User ratings" - url: https://www.imdb.com/user/%s/ratings - time_zone: UTC - ttl: 1440 -request: - botasaurus: - wait_timeout_seconds: 20 - wait_for_selector: "li.ipc-metadata-list-summary-item" -selectors: - items: - selector: "li.ipc-metadata-list-summary-item" - enhance: false - title: - selector: ".ipc-title__text" - post_process: - - name: gsub - pattern: "/^(\\d+.)\\s/" - replacement: "" - - name: template - string: 'Rated: "%{self}" %{user_rating} [IMDb: %{global_rating}]' - url: - selector: "a.ipc-title-link-wrapper" - extractor: "href" - description: - selector: ".ipc-html-content-inner-div" - global_rating: - selector: ".ipc-rating-star--imdb .ipc-rating-star--rating" - user_rating: - selector: ".dli-ratings-container .ipc-rating-star--otherUser span" - enclosure: - selector: ".ipc-image" - extractor: "attribute" - attribute: "src" diff --git a/app/registries/seed/official/configs/imo.org/press-briefings.yml b/app/registries/seed/official/configs/imo.org/press-briefings.yml deleted file mode 100644 index 0857264aa..000000000 --- a/app/registries/seed/official/configs/imo.org/press-briefings.yml +++ /dev/null @@ -1,28 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: imo.org/press-briefings -directory: - topics: - - civic - title: "IMO — Press briefings" - summary: "Press briefings from the International Maritime Organization." -strategy: botasaurus - -channel: - title: "IMO — Press briefings" - url: https://www.imo.org/en/MediaCentre/PressBriefings/Pages/default.aspx - language: en - time_zone: UTC - ttl: 360 -request: - botasaurus: - wait_for_selector: 'h3 a[href*="/PressBriefings/pages/"]' - wait_timeout_seconds: 20 -selectors: - items: - selector: 'h3 a[href*="/PressBriefings/pages/"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/imperial.ac.uk/news.yml b/app/registries/seed/official/configs/imperial.ac.uk/news.yml deleted file mode 100644 index fbeed688b..000000000 --- a/app/registries/seed/official/configs/imperial.ac.uk/news.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: imperial.ac.uk/news -directory: - topics: - - science - - research - title: "Imperial College London — News" - summary: "Research and university news from Imperial College London." -channel: - title: "Imperial College London — News" - url: https://www.imperial.ac.uk/news/articles/ - language: en - time_zone: Europe/London - ttl: 360 -selectors: - items: - selector: 'a.listing-item[href*="/news/articles/"]' - enhance: false - title: - selector: .listing-item__title - url: - extractor: href diff --git a/app/registries/seed/official/configs/ingenieur.de/karriere_arbeitsleben_heiko_mell.yml b/app/registries/seed/official/configs/ingenieur.de/karriere_arbeitsleben_heiko_mell.yml deleted file mode 100644 index 8df62e404..000000000 --- a/app/registries/seed/official/configs/ingenieur.de/karriere_arbeitsleben_heiko_mell.yml +++ /dev/null @@ -1,25 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json ---- -registry: - id: ingenieur.de/karriere_arbeitsleben_heiko_mell -directory: - topics: - - jobs - title: "ingenieur.de — Heiko Mell" - summary: "Karriere-Kolumnen von Heiko Mell auf ingenieur.de." -channel: - title: "ingenieur.de — Heiko Mell" - url: https://www.ingenieur.de/karriere/arbeitsleben/heiko-mell/ - language: de-DE - ttl: 360 - time_zone: UTC -selectors: - items: - selector: ".section > .ing-teaser" - title: - selector: ".ing-teaser__headline" - url: - selector: a:first - extractor: href - description: - selector: ".ing-teaser__excerpt" diff --git a/app/registries/seed/official/configs/isro.gov.in/press.yml b/app/registries/seed/official/configs/isro.gov.in/press.yml deleted file mode 100644 index ffd5162e1..000000000 --- a/app/registries/seed/official/configs/isro.gov.in/press.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: isro.gov.in/press -directory: - topics: - - science - - research - title: "ISRO — Press" - summary: "Press releases from the Indian Space Research Organisation." -channel: - title: "ISRO — Press" - url: https://www.isro.gov.in/Press.html - language: en - time_zone: Asia/Kolkata - ttl: 360 -selectors: - items: - selector: 'table tr a[href$=".html"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/issafrica.org/iss-today.yml b/app/registries/seed/official/configs/issafrica.org/iss-today.yml deleted file mode 100644 index ec530484d..000000000 --- a/app/registries/seed/official/configs/issafrica.org/iss-today.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: issafrica.org/iss-today -directory: - topics: - - civic - - research - title: "ISS Africa — ISS Today" - summary: "Daily analysis from the Institute for Security Studies Africa." -channel: - title: "ISS Africa — ISS Today" - url: https://issafrica.org/iss-today - language: en - time_zone: Africa/Johannesburg - ttl: 360 -selectors: - items: - selector: 'a.card[href^="/iss-today/"]' - enhance: false - title: - selector: h6.card-subtitle - url: - extractor: href diff --git a/app/registries/seed/official/configs/issafrica.org/press-releases.yml b/app/registries/seed/official/configs/issafrica.org/press-releases.yml deleted file mode 100644 index 79d212049..000000000 --- a/app/registries/seed/official/configs/issafrica.org/press-releases.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: issafrica.org/press-releases -directory: - topics: - - civic - - research - title: "ISS Africa — Press releases" - summary: "Press releases from the Institute for Security Studies Africa." -channel: - title: "ISS Africa — Press releases" - url: https://issafrica.org/about-us/press-releases - language: en - time_zone: Africa/Johannesburg - ttl: 720 -selectors: - items: - selector: 'a.card[href^="/about-us/press-releases/"]' - enhance: false - title: - selector: h6.card-subtitle - url: - extractor: href diff --git a/app/registries/seed/official/configs/kaist.ac.kr/news.yml b/app/registries/seed/official/configs/kaist.ac.kr/news.yml deleted file mode 100644 index 9805ed934..000000000 --- a/app/registries/seed/official/configs/kaist.ac.kr/news.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: kaist.ac.kr/news -directory: - topics: - - science - - research - title: "KAIST — News" - summary: "University and research news from KAIST." -channel: - title: "KAIST — News" - url: https://news.kaist.ac.kr/newsen/html/news/ - language: en - time_zone: Asia/Seoul - ttl: 360 -selectors: - items: - selector: 'a.lay[href*="mng_no="]' - enhance: false - title: - selector: strong.tis - url: - extractor: href diff --git a/app/registries/seed/official/configs/kinocheck.de/filmstarts.yml b/app/registries/seed/official/configs/kinocheck.de/filmstarts.yml deleted file mode 100644 index 558eecf22..000000000 --- a/app/registries/seed/official/configs/kinocheck.de/filmstarts.yml +++ /dev/null @@ -1,30 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: kinocheck.de/filmstarts -directory: - topics: - - entertainment - title: "KinoCheck — Filmstarts" - summary: "Kommende und aktuelle Filmstarts in Deutschland." -channel: - title: "KinoCheck — Filmstarts" - url: https://kinocheck.de/filmstarts - time_zone: Europe/Berlin - ttl: 1440 - language: de -selectors: - items: - selector: ".list-movies-wide-item .wrapper-content" - title: - selector: "h2" - post_process: - - name: "template" - string: "%{self} (%{start_date})" - url: - selector: "a" - extractor: "href" - description: - selector: ".information" - extractor: "html" - start_date: - selector: ".release-date .highlight" diff --git a/app/registries/seed/official/configs/kompas.com/nasional.yml b/app/registries/seed/official/configs/kompas.com/nasional.yml deleted file mode 100644 index e5655866b..000000000 --- a/app/registries/seed/official/configs/kompas.com/nasional.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: kompas.com/nasional -directory: - topics: - - news - - civic - title: "Kompas — Nasional" - summary: "Berita nasional dari Kompas.com." -channel: - title: "Kompas — Nasional" - url: https://nasional.kompas.com/ - language: id - time_zone: Asia/Jakarta - ttl: 360 -selectors: - items: - selector: '.articleList a[href*="/read/"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/lego.com/press-releases.yml b/app/registries/seed/official/configs/lego.com/press-releases.yml deleted file mode 100644 index f2cdc9b23..000000000 --- a/app/registries/seed/official/configs/lego.com/press-releases.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: lego.com/press-releases -directory: - topics: - - entertainment - - product - title: "LEGO — Press releases" - summary: "Official press releases from the LEGO Group." -channel: - title: "LEGO — Press releases" - url: https://www.lego.com/en-gb/aboutus/newsroom/press-releases - language: en-GB - time_zone: Europe/Copenhagen - ttl: 360 -selectors: - items: - selector: 'a.c-card--article[href*="/aboutus/news/20"]' - enhance: true - title: - selector: h2.c-card__title - url: - extractor: href diff --git a/app/registries/seed/official/configs/machinelearning.apple.com/research.yml b/app/registries/seed/official/configs/machinelearning.apple.com/research.yml deleted file mode 100644 index 785f7f8c6..000000000 --- a/app/registries/seed/official/configs/machinelearning.apple.com/research.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: machinelearning.apple.com/research -directory: - topics: - - tech - - research - title: "Apple Machine Learning — Research" - summary: "Machine learning research publications from Apple." -channel: - title: "Apple Machine Learning — Research" - url: https://machinelearning.apple.com/research/ - language: en - time_zone: UTC - ttl: 360 -selectors: - items: - selector: "ul.article-list > li.post-row > div > a.post-link" - enhance: false - title: - selector: h3.post-title - url: - extractor: href diff --git a/app/registries/seed/official/configs/maersk.com/press-releases.yml b/app/registries/seed/official/configs/maersk.com/press-releases.yml deleted file mode 100644 index 5a8f6ebea..000000000 --- a/app/registries/seed/official/configs/maersk.com/press-releases.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: maersk.com/press-releases -directory: - topics: - - finance - title: "Maersk — Press releases" - summary: "Corporate press releases from Maersk." -channel: - title: "Maersk — Press releases" - url: https://www.maersk.com/news/category/press-releases - language: en - time_zone: Europe/Copenhagen - ttl: 360 -selectors: - items: - selector: div.p-section__news__teaser - enhance: false - title: - selector: a.p-section__news__teaser__title - url: - selector: a.p-section__news__teaser__title - extractor: href diff --git a/app/registries/seed/official/configs/microsoft.com/azure-products.yml b/app/registries/seed/official/configs/microsoft.com/azure-products.yml deleted file mode 100644 index 4cfc4c0e4..000000000 --- a/app/registries/seed/official/configs/microsoft.com/azure-products.yml +++ /dev/null @@ -1,25 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: microsoft.com/azure-products -directory: - topics: - - tech - - product - title: "Microsoft — Azure Products" - summary: "Product updates and announcements from Microsoft Azure." -channel: - title: "Microsoft — Azure Products" - url: https://azure.microsoft.com/en-us/products - language: en - time_zone: US/Pacific - ttl: 3600 -selectors: - items: - selector: ".card-body" - title: - selector: ".h5" - url: - selector: ".link-group a:first-of-type" - extractor: "href" - description: - selector: "p" diff --git a/app/registries/seed/official/configs/mistral.ai/news.yml b/app/registries/seed/official/configs/mistral.ai/news.yml deleted file mode 100644 index 38a070656..000000000 --- a/app/registries/seed/official/configs/mistral.ai/news.yml +++ /dev/null @@ -1,24 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: mistral.ai/news -directory: - topics: - - tech - - research - title: "Mistral AI — News" - summary: "Product and research news from Mistral AI." -channel: - title: "Mistral AI — News" - url: https://mistral.ai/news/ - language: en - time_zone: UTC - ttl: 360 -selectors: - items: - selector: article.js-post - enhance: false - title: - selector: h2.js-post-title - url: - selector: a[href^="/news/"] - extractor: href diff --git a/app/registries/seed/official/configs/mozilla.org/security-advisories.yml b/app/registries/seed/official/configs/mozilla.org/security-advisories.yml deleted file mode 100644 index 2e0d5f3c3..000000000 --- a/app/registries/seed/official/configs/mozilla.org/security-advisories.yml +++ /dev/null @@ -1,24 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: mozilla.org/security-advisories -directory: - topics: - - security - - tech - title: "Mozilla — Security advisories" - summary: "Security advisories for Mozilla products." -channel: - title: "Mozilla — Security advisories" - url: https://www.mozilla.org/en-US/security/advisories/ - language: en - time_zone: UTC - ttl: 360 -selectors: - items: - selector: "main li" - enhance: false - title: - selector: 'a[href*="/security/advisories/mfsa"]' - url: - selector: 'a[href*="/security/advisories/mfsa"]' - extractor: href diff --git a/app/registries/seed/official/configs/msf.org/latest.yml b/app/registries/seed/official/configs/msf.org/latest.yml deleted file mode 100644 index f60f13455..000000000 --- a/app/registries/seed/official/configs/msf.org/latest.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: msf.org/latest -directory: - topics: - - news - - civic - title: "Médecins Sans Frontières — Latest" - summary: "Latest field and advocacy updates from Médecins Sans Frontières." -channel: - title: "Médecins Sans Frontières — Latest" - url: https://www.msf.org/latest-portal - language: en - time_zone: UTC - ttl: 360 -selectors: - items: - selector: a.card__title-link - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/nbim.no/press-releases.yml b/app/registries/seed/official/configs/nbim.no/press-releases.yml deleted file mode 100644 index f286fed76..000000000 --- a/app/registries/seed/official/configs/nbim.no/press-releases.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: nbim.no/press-releases -directory: - topics: - - finance - title: "Norges Bank Investment Management — Press releases" - summary: "Press releases from Norway’s sovereign wealth fund manager." -channel: - title: "Norges Bank Investment Management — Press releases" - url: https://www.nbim.no/en/news-and-insights/the-press/press-releases/ - language: en - time_zone: Europe/Oslo - ttl: 360 -selectors: - items: - selector: div.content-featured-content - enhance: true - title: - selector: h3 - url: - selector: a.chevron-link - extractor: href diff --git a/app/registries/seed/official/configs/news.cn/latest.yml b/app/registries/seed/official/configs/news.cn/latest.yml deleted file mode 100644 index 9312eb834..000000000 --- a/app/registries/seed/official/configs/news.cn/latest.yml +++ /dev/null @@ -1,22 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: news.cn/latest -directory: - topics: - - news - title: "Xinhua — Latest news" - summary: "Latest English news from Xinhua News Agency." -channel: - title: "Xinhua — Latest news" - url: https://english.news.cn/list/latestnews.htm - language: en - time_zone: Asia/Shanghai - ttl: 360 -selectors: - items: - selector: "#list .item .tit a" - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/news24.com/top-stories.yml b/app/registries/seed/official/configs/news24.com/top-stories.yml deleted file mode 100644 index e68f8f174..000000000 --- a/app/registries/seed/official/configs/news24.com/top-stories.yml +++ /dev/null @@ -1,22 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: news24.com/top-stories -directory: - topics: - - news - title: "News24 — Top stories" - summary: "Top stories from South African news site News24." -channel: - title: "News24 — Top stories" - url: https://www.news24.com/ - language: en - time_zone: Africa/Johannesburg - ttl: 180 -selectors: - items: - selector: "a.article-item__title" - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/newyorker.com/magazine.yml b/app/registries/seed/official/configs/newyorker.com/magazine.yml deleted file mode 100644 index 6cc171ddf..000000000 --- a/app/registries/seed/official/configs/newyorker.com/magazine.yml +++ /dev/null @@ -1,25 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json ---- -registry: - id: newyorker.com/magazine -directory: - topics: - - news - title: "The New Yorker — Magazine" - summary: "Current magazine stories from The New Yorker." -channel: - title: "The New Yorker — Magazine" - url: https://www.newyorker.com/magazine - language: en - ttl: 360 - time_zone: UTC -selectors: - items: - selector: .summary-item - title: - selector: h3 - url: - selector: a:first - extractor: href - description: - selector: .summary-item__dek diff --git a/app/registries/seed/official/configs/nigerianstat.gov.ng/news.yml b/app/registries/seed/official/configs/nigerianstat.gov.ng/news.yml deleted file mode 100644 index 7c635c88d..000000000 --- a/app/registries/seed/official/configs/nigerianstat.gov.ng/news.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: nigerianstat.gov.ng/news -directory: - topics: - - civic - - research - title: "Nigeria Bureau of Statistics — News" - summary: "Official statistical releases and news from Nigeria’s NBS." -channel: - title: "Nigeria Bureau of Statistics — News" - url: https://nigerianstat.gov.ng/ - language: en - time_zone: Africa/Lagos - ttl: 360 -selectors: - items: - selector: 'a[href*="/news/"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/nikkei.com/asia-technology.yml b/app/registries/seed/official/configs/nikkei.com/asia-technology.yml deleted file mode 100644 index 205026563..000000000 --- a/app/registries/seed/official/configs/nikkei.com/asia-technology.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: nikkei.com/asia-technology -directory: - topics: - - news - - tech - title: "Nikkei Asia — Technology" - summary: "Asia technology business coverage from Nikkei Asia." -channel: - title: "Nikkei Asia — Technology" - url: https://asia.nikkei.com/business/technology - language: en - time_zone: Asia/Tokyo - ttl: 360 -selectors: - items: - selector: 'a[data-trackable="headline"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/nokia.com/newsroom.yml b/app/registries/seed/official/configs/nokia.com/newsroom.yml deleted file mode 100644 index d9a25f177..000000000 --- a/app/registries/seed/official/configs/nokia.com/newsroom.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: nokia.com/newsroom -directory: - topics: - - tech - - product - title: "Nokia — Newsroom" - summary: "Corporate newsroom announcements from Nokia." -channel: - title: "Nokia — Newsroom" - url: https://www.nokia.com/newsroom/ - language: en - time_zone: Europe/Helsinki - ttl: 360 -selectors: - items: - selector: 'a.pp-block-item-container[href*="/newsroom/"]' - enhance: false - title: - selector: h3.pp-block-item-heading - url: - extractor: href diff --git a/app/registries/seed/official/configs/nomanssky.com/news.yml b/app/registries/seed/official/configs/nomanssky.com/news.yml deleted file mode 100644 index b3fc8257e..000000000 --- a/app/registries/seed/official/configs/nomanssky.com/news.yml +++ /dev/null @@ -1,29 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json ---- -registry: - id: nomanssky.com/news -directory: - topics: - - entertainment - title: "No Man’s Sky — News" - summary: "Game updates and news for No Man’s Sky." -channel: - title: "No Man’s Sky — News" - url: https://www.nomanssky.com/news/ - language: en-GB - ttl: 360 - time_zone: UTC -selectors: - items: - selector: article - title: - selector: ".post-title" - url: - selector: a.view-article - extractor: href - description: - selector: p:first - post_process: - - name: substring - start: 0 - end: -14 diff --git a/app/registries/seed/official/configs/notion.com/blog.yml b/app/registries/seed/official/configs/notion.com/blog.yml deleted file mode 100644 index 6e4efc349..000000000 --- a/app/registries/seed/official/configs/notion.com/blog.yml +++ /dev/null @@ -1,27 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: notion.com/blog -directory: - topics: - - tech - - product - title: "Notion — Blog" - summary: "Product updates and stories from the Notion blog." - -channel: - title: "Notion — Blog" - url: https://www.notion.com/blog - language: en - time_zone: UTC - ttl: 360 -selectors: - items: - selector: article.post-preview - enhance: false - title: - selector: h3 a[href*="/blog/"] - url: - selector: h3 a[href*="/blog/"] - extractor: href - description: - selector: '> a[href*="/blog/"]:not([title])' diff --git a/app/registries/seed/official/configs/novonordisk.com/latest-news.yml b/app/registries/seed/official/configs/novonordisk.com/latest-news.yml deleted file mode 100644 index e9cba5ebf..000000000 --- a/app/registries/seed/official/configs/novonordisk.com/latest-news.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: novonordisk.com/latest-news -directory: - topics: - - science - - product - title: "Novo Nordisk — Latest news" - summary: "Latest corporate and product news from Novo Nordisk." -channel: - title: "Novo Nordisk — Latest news" - url: https://www.novonordisk.com/news-and-media/latest-news.html - language: en - time_zone: Europe/Copenhagen - ttl: 360 -selectors: - items: - selector: 'a.ln-link-item[href*="news-details.html"]' - enhance: false - title: - selector: .location-info p.paragraph-s - url: - extractor: href diff --git a/app/registries/seed/official/configs/ntu.edu.sg/news.yml b/app/registries/seed/official/configs/ntu.edu.sg/news.yml deleted file mode 100644 index 130ad35ab..000000000 --- a/app/registries/seed/official/configs/ntu.edu.sg/news.yml +++ /dev/null @@ -1,29 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: ntu.edu.sg/news -directory: - topics: - - science - - research - title: "NTU Singapore — News" - summary: "University and research news from Nanyang Technological University." -strategy: botasaurus - -channel: - title: "NTU Singapore — News" - url: https://www.ntu.edu.sg/news - language: en - time_zone: Asia/Singapore - ttl: 360 -request: - botasaurus: - wait_for_selector: 'a.img-card[href*="/news/detail/"]' - wait_timeout_seconds: 20 -selectors: - items: - selector: 'a.img-card[href*="/news/detail/"]' - enhance: false - title: - selector: .img-card__title - url: - extractor: href diff --git a/app/registries/seed/official/configs/nupi.no/news.yml b/app/registries/seed/official/configs/nupi.no/news.yml deleted file mode 100644 index 223a53274..000000000 --- a/app/registries/seed/official/configs/nupi.no/news.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: nupi.no/news -directory: - topics: - - civic - - research - title: "NUPI — News" - summary: "News from the Norwegian Institute of International Affairs." -channel: - title: "NUPI — News" - url: https://www.nupi.no/en - language: en - time_zone: Europe/Oslo - ttl: 360 -selectors: - items: - selector: 'h2 a[href^="/en/news/"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/occrp.org/news.yml b/app/registries/seed/official/configs/occrp.org/news.yml deleted file mode 100644 index 6ec01fa82..000000000 --- a/app/registries/seed/official/configs/occrp.org/news.yml +++ /dev/null @@ -1,30 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: occrp.org/news -directory: - topics: - - news - - civic - title: "OCCRP — News" - summary: "Investigative journalism from the Organized Crime and Corruption Reporting Project." -strategy: botasaurus - -channel: - title: "OCCRP — News" - url: https://www.occrp.org/en/news - language: en - time_zone: UTC - ttl: 360 -request: - botasaurus: - wait_for_selector: article.news-item - wait_timeout_seconds: 20 -selectors: - items: - selector: article.news-item - enhance: false - title: - selector: h3.news-item__title a - url: - selector: h3.news-item__title a - extractor: href diff --git a/app/registries/seed/official/configs/oecd.org/newsroom.yml b/app/registries/seed/official/configs/oecd.org/newsroom.yml deleted file mode 100644 index 43194b044..000000000 --- a/app/registries/seed/official/configs/oecd.org/newsroom.yml +++ /dev/null @@ -1,29 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: oecd.org/newsroom -directory: - topics: - - finance - - civic - title: "OECD — Newsroom" - summary: "Newsroom releases from the Organisation for Economic Co-operation and Development." -strategy: botasaurus - -channel: - title: "OECD — Newsroom" - url: https://www.oecd.org/en/about/newsroom.html - language: en - time_zone: UTC - ttl: 360 -request: - botasaurus: - wait_for_selector: 'a[href*="/en/about/news/press-releases/"]' - wait_timeout_seconds: 20 -selectors: - items: - selector: 'a.card__title-link[href*="/en/about/news/press-releases/"], a.cmp-featured-card__link[href*="/en/about/news/press-releases/"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/opec.org/press-releases.yml b/app/registries/seed/official/configs/opec.org/press-releases.yml deleted file mode 100644 index d89fa2eba..000000000 --- a/app/registries/seed/official/configs/opec.org/press-releases.yml +++ /dev/null @@ -1,30 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: opec.org/press-releases -directory: - topics: - - energy - - civic - title: "OPEC — Press releases" - summary: "Official press releases from OPEC." -strategy: botasaurus - -channel: - title: "OPEC — Press releases" - url: https://www.opec.org/press-releases.html - language: en - time_zone: Europe/Vienna - ttl: 360 -request: - botasaurus: - wait_for_selector: 'div.pritem a[href*="pr-detail"]' - wait_timeout_seconds: 20 -selectors: - items: - selector: div.pritem - enhance: false - title: - selector: h1 - url: - selector: 'a[href*="pr-detail"]' - extractor: href diff --git a/app/registries/seed/official/configs/panapress.com/latest.yml b/app/registries/seed/official/configs/panapress.com/latest.yml deleted file mode 100644 index 4124a4f98..000000000 --- a/app/registries/seed/official/configs/panapress.com/latest.yml +++ /dev/null @@ -1,28 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: panapress.com/latest -directory: - topics: - - news - title: "PANA Press — Latest" - summary: "Latest African news from the Pan African News Agency." -strategy: botasaurus - -channel: - title: "PANA Press — Latest" - url: https://www.panapress.com/ - language: en - time_zone: UTC - ttl: 360 -request: - botasaurus: - wait_for_selector: 'h4 a[href*="lang2.html"]' - wait_timeout_seconds: 20 -selectors: - items: - selector: 'h4 a[href*="lang2.html"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/pankow.lebensmittel-kontrollergebnisse.de/search.yml b/app/registries/seed/official/configs/pankow.lebensmittel-kontrollergebnisse.de/search.yml deleted file mode 100644 index ca3a888c3..000000000 --- a/app/registries/seed/official/configs/pankow.lebensmittel-kontrollergebnisse.de/search.yml +++ /dev/null @@ -1,53 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json ---- -registry: - id: pankow.lebensmittel-kontrollergebnisse.de/search -directory: - topics: - - consumer - - civic - title: "Berlin-Pankow — Lebensmittelkontrollen" - summary: "Veröffentlichte Lebensmittel-Kontrollergebnisse für Berlin-Pankow." -channel: - title: "Berlin-Pankow — Lebensmittelkontrollen" - url: https://pankow.lebensmittel-kontrollergebnisse.de/Search - language: de - ttl: 360 - time_zone: Europe/Berlin -selectors: - items: - selector: main .card - title: - selector: h5 - post_process: - - name: template - string: | - %{self}: %{ergebnis} (%{punkte}/80 Punkte) - url: - selector: a - extractor: href - description: - selector: ".row" - extractor: html - post_process: - - name: html_to_markdown - - name: markdown_to_html - ergebnis: - selector: ".bewertung-panel .row:nth-child(3)" - post_process: - - name: substring - start: 9 - punkte: - selector: ".bewertung-panel .row:nth-child(2)" - post_process: - - name: substring - start: 10 - published_at: - selector: ".bewertung-panel .row:nth-child(1)" - post_process: - - name: substring - start: -10 - - name: parse_time - categories: - - ergebnis - - punkte diff --git a/app/registries/seed/official/configs/philstar.com/nation.yml b/app/registries/seed/official/configs/philstar.com/nation.yml deleted file mode 100644 index 87071a14b..000000000 --- a/app/registries/seed/official/configs/philstar.com/nation.yml +++ /dev/null @@ -1,22 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: philstar.com/nation -directory: - topics: - - news - title: "Philippine Star — Nation" - summary: "National news from The Philippine Star." -channel: - title: "Philippine Star — Nation" - url: https://www.philstar.com/nation - language: en - time_zone: Asia/Manila - ttl: 360 -selectors: - items: - selector: 'h2 a[href*="/nation/"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/phys.org/weekly.yml b/app/registries/seed/official/configs/phys.org/weekly.yml deleted file mode 100644 index a40b0486b..000000000 --- a/app/registries/seed/official/configs/phys.org/weekly.yml +++ /dev/null @@ -1,26 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: phys.org/weekly -directory: - topics: - - science - title: "Phys.org — Weekly" - summary: "Top science news of the week from Phys.org." -channel: - language: en - title: "Phys.org — Weekly" - url: https://phys.org/weekly-news/ - time_zone: Europe/London - ttl: 1440 -selectors: - items: - selector: ".sorted-news-list .sorted-article-content" - title: - selector: "h4" - category: - selector: ".text-info" - categories: - - category - url: - selector: ".news-link" - extractor: "href" diff --git a/app/registries/seed/official/configs/pib.gov.in/press-releases.yml b/app/registries/seed/official/configs/pib.gov.in/press-releases.yml deleted file mode 100644 index 21ae23d4c..000000000 --- a/app/registries/seed/official/configs/pib.gov.in/press-releases.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: pib.gov.in/press-releases -directory: - topics: - - civic - - news - title: "PIB India — Press releases" - summary: "Press releases from India’s Press Information Bureau." -channel: - title: "PIB India — Press releases" - url: https://www.pib.gov.in/AllRel.aspx?reg=3&lang=1 - language: en - time_zone: Asia/Kolkata - ttl: 360 -selectors: - items: - selector: 'a[href*="PressReleaseDetail.aspx"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/rbb24.de/meistgeklickt.yml b/app/registries/seed/official/configs/rbb24.de/meistgeklickt.yml deleted file mode 100644 index b28744a8a..000000000 --- a/app/registries/seed/official/configs/rbb24.de/meistgeklickt.yml +++ /dev/null @@ -1,22 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: rbb24.de/meistgeklickt -directory: - topics: - - news - title: "rbb24 — Meistgeklickt" - summary: "Most-clicked stories from rbb24." -channel: - title: "rbb24 — Meistgeklickt" - url: https://www.rbb24.de/das-beste/ - time_zone: Europe/Berlin - ttl: 30 - language: de -selectors: - items: - selector: "article.top-teaser a.top-teaser__text-container" - enhance: false - title: - extractor: "text" - url: - extractor: "href" diff --git a/app/registries/seed/official/configs/robinwood.de/aktuelles.yml b/app/registries/seed/official/configs/robinwood.de/aktuelles.yml deleted file mode 100644 index 58a03e08b..000000000 --- a/app/registries/seed/official/configs/robinwood.de/aktuelles.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: robinwood.de/aktuelles -directory: - topics: - - environment - title: "ROBIN WOOD — Aktuelles" - summary: "Aktuelle Meldungen der Umweltschutzorganisation ROBIN WOOD." -channel: - title: "ROBIN WOOD — Aktuelles" - url: https://www.robinwood.de/was-gibt-es-neues - time_zone: Europe/Berlin - ttl: 720 - language: de -selectors: - items: - selector: "article.node--view-mode-teaser" - enhance: false - title: - selector: "h2" - url: - selector: "a" - extractor: "href" diff --git a/app/registries/seed/official/configs/s3.amazonaws.com/popular_movies.yml b/app/registries/seed/official/configs/s3.amazonaws.com/popular_movies.yml deleted file mode 100644 index e109bf43b..000000000 --- a/app/registries/seed/official/configs/s3.amazonaws.com/popular_movies.yml +++ /dev/null @@ -1,35 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: s3.amazonaws.com/popular_movies -directory: - topics: - - entertainment - title: "Popular movies — JSON list" - summary: "Popular movies listing from a public JSON feed on Amazon S3." -# This generates a RSS of the daily updated -# https://github.com/sjlu/popular-movies -channel: - language: en - title: "Popular movies — JSON list" - url: https://s3.amazonaws.com/popular-movies/movies.json - time_zone: UTC - ttl: 1440 - json: true -selectors: - items: - selector: "array > object" - title: - selector: "title" - imdb_id: - selector: "imdb_id" - poster_url: - selector: "poster_url" - url: - post_process: - - name: template - string: "https://imdb.com/title/%{imdb_id}" - description: - post_process: - - name: template - string: "![](%{poster_url})" - - name: markdown_to_html diff --git a/app/registries/seed/official/configs/sacu.int/news.yml b/app/registries/seed/official/configs/sacu.int/news.yml deleted file mode 100644 index e1de83dce..000000000 --- a/app/registries/seed/official/configs/sacu.int/news.yml +++ /dev/null @@ -1,24 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: sacu.int/news -directory: - topics: - - civic - - news - title: "SACU — News" - summary: "News from the Southern African Customs Union." -channel: - title: "SACU — News" - url: https://www.sacu.int/news/ - language: en - time_zone: Africa/Windhoek - ttl: 360 -selectors: - items: - selector: ".card" - enhance: false - title: - selector: h3 - url: - selector: 'a[href*="/articles/latest-news/"]' - extractor: href diff --git a/app/registries/seed/official/configs/sadc.int/latest-news.yml b/app/registries/seed/official/configs/sadc.int/latest-news.yml deleted file mode 100644 index 2af67d5ff..000000000 --- a/app/registries/seed/official/configs/sadc.int/latest-news.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: sadc.int/latest-news -directory: - topics: - - civic - - news - title: "SADC — Latest news" - summary: "Latest news from the Southern African Development Community." -channel: - title: "SADC — Latest news" - url: https://www.sadc.int/latest-news - language: en - time_zone: UTC - ttl: 360 -selectors: - items: - selector: 'span.field-content a[href^="/latest-news/"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/sebastianvettel.de/news.yml b/app/registries/seed/official/configs/sebastianvettel.de/news.yml deleted file mode 100644 index 226563b24..000000000 --- a/app/registries/seed/official/configs/sebastianvettel.de/news.yml +++ /dev/null @@ -1,24 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json ---- -registry: - id: sebastianvettel.de/news -directory: - topics: - - sports - title: "Sebastian Vettel — News" - summary: "Neuigkeiten von der Website von Sebastian Vettel." -channel: - title: "Sebastian Vettel — News" - url: https://sebastianvettel.de/ - language: de-DE - ttl: 8640 - time_zone: Europe/Berlin -selectors: - items: - selector: ".gb-loop-item:has(a[href*='/news/'])" - enhance: false - title: - selector: ".gb-text:first-child" - url: - selector: "a[href*='/news/']" - extractor: href diff --git a/app/registries/seed/official/configs/shopify.com/blog.yml b/app/registries/seed/official/configs/shopify.com/blog.yml deleted file mode 100644 index 834e736f8..000000000 --- a/app/registries/seed/official/configs/shopify.com/blog.yml +++ /dev/null @@ -1,25 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: shopify.com/blog -directory: - topics: - - tech - - product - title: "Shopify — Blog" - summary: "Commerce tips and product updates from the Shopify blog." - -channel: - language: en - title: "Shopify — Blog" - url: https://www.shopify.com/blog/latest - time_zone: UTC - ttl: 360 -selectors: - items: - selector: article.article--index - enhance: false - title: - selector: '.blogPost a[href*="/blog/"]:not([href*="/topics/"])' - url: - selector: '.blogPost a[href*="/blog/"]:not([href*="/topics/"])' - extractor: href diff --git a/app/registries/seed/official/configs/sipri.org/news.yml b/app/registries/seed/official/configs/sipri.org/news.yml deleted file mode 100644 index 6778a9073..000000000 --- a/app/registries/seed/official/configs/sipri.org/news.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: sipri.org/news -directory: - topics: - - civic - - research - title: "SIPRI — News" - summary: "News from the Stockholm International Peace Research Institute." -channel: - title: "SIPRI — News" - url: https://www.sipri.org/news - language: en - time_zone: UTC - ttl: 360 -selectors: - items: - selector: 'h3.field-content a[href^="/news/20"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/softwareleadweekly.com/issues.yml b/app/registries/seed/official/configs/softwareleadweekly.com/issues.yml deleted file mode 100644 index d0a8c49af..000000000 --- a/app/registries/seed/official/configs/softwareleadweekly.com/issues.yml +++ /dev/null @@ -1,22 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: softwareleadweekly.com/issues -directory: - topics: - - tech - title: "Software Lead Weekly — Issues" - summary: "Weekly leadership newsletter issues from Software Lead Weekly." -channel: - language: en - title: "Software Lead Weekly — Issues" - url: https://softwareleadweekly.com/issues - time_zone: UTC - ttl: 720 -selectors: - items: - selector: ".table-issue" - title: - selector: "a" - url: - selector: "a" - extractor: "href" diff --git a/app/registries/seed/official/configs/solarthermalworld.org/news.yml b/app/registries/seed/official/configs/solarthermalworld.org/news.yml deleted file mode 100644 index c25d2fa07..000000000 --- a/app/registries/seed/official/configs/solarthermalworld.org/news.yml +++ /dev/null @@ -1,25 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: solarthermalworld.org/news -directory: - topics: - - energy - title: "Solarthermalworld — News" - summary: "Global news on solar thermal energy and heat." -channel: - language: en - title: "Solarthermalworld — News" - url: https://solarthermalworld.org/news - time_zone: UTC - ttl: 180 -selectors: - items: - selector: "article.post-wrapper" - title: - selector: "h2 a" - url: - selector: "h2 a" - extractor: "href" - description: - selector: '[property="content:encoded"]' - extractor: html diff --git a/app/registries/seed/official/configs/spektrum.de/meistgelesen.yml b/app/registries/seed/official/configs/spektrum.de/meistgelesen.yml deleted file mode 100644 index db7788f75..000000000 --- a/app/registries/seed/official/configs/spektrum.de/meistgelesen.yml +++ /dev/null @@ -1,22 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: spektrum.de/meistgelesen -directory: - topics: - - science - title: "Spektrum — Meistgelesen" - summary: "Most-read science articles from Spektrum." -channel: - title: "Spektrum — Meistgelesen" - url: https://www.spektrum.de/ - time_zone: Europe/Berlin - ttl: 60 - language: de -selectors: - items: - selector: ".teaser.teaser__list_box ol > li" - title: - selector: "a" - url: - selector: "a" - extractor: "href" diff --git a/app/registries/seed/official/configs/spiegel.de/impressum_autor.yml b/app/registries/seed/official/configs/spiegel.de/impressum_autor.yml deleted file mode 100644 index e6aa1346d..000000000 --- a/app/registries/seed/official/configs/spiegel.de/impressum_autor.yml +++ /dev/null @@ -1,39 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: spiegel.de/impressum_autor -directory: - topics: - - news - title: "DER SPIEGEL — Author articles" - summary: "Articles by a selectable DER SPIEGEL author profile." -parameters: - id: - type: string - default: "975b6ae0-0001-0003-0000-000000018282" - -channel: - title: "DER SPIEGEL — Author articles" - url: "https://www.spiegel.de/impressum/autor-%s" - time_zone: Europe/Berlin - ttl: 180 - language: de -selectors: - items: - selector: '[data-block-el="articleTeaser"] > article' - title: - selector: "h2" - prefixed_title: - selector: "h2" - url: - selector: "section > a" - extractor: "href" - description: - selector: ".leading-loose" - published_at: - selector: "footer" - extractor: text - post_process: - - name: substring - start: 0 - end: -5 - - name: parse_time diff --git a/app/registries/seed/official/configs/spotify.com/newsroom.yml b/app/registries/seed/official/configs/spotify.com/newsroom.yml deleted file mode 100644 index b9ef9fbc6..000000000 --- a/app/registries/seed/official/configs/spotify.com/newsroom.yml +++ /dev/null @@ -1,24 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: spotify.com/newsroom -directory: - topics: - - entertainment - title: "Spotify — Newsroom" - summary: "Corporate and product announcements from Spotify Newsroom." - -channel: - title: "Spotify — Newsroom" - url: https://newsroom.spotify.com/ - language: en - time_zone: UTC - ttl: 360 -selectors: - items: - selector: ".post-box.v2" - enhance: false - title: - selector: 'h3 a[href*="/20"]' - url: - selector: 'h3 a[href*="/20"]' - extractor: href diff --git a/app/registries/seed/official/configs/stackoverflow.com/hot_network_questions.yml b/app/registries/seed/official/configs/stackoverflow.com/hot_network_questions.yml deleted file mode 100644 index 2b031bb29..000000000 --- a/app/registries/seed/official/configs/stackoverflow.com/hot_network_questions.yml +++ /dev/null @@ -1,35 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: stackoverflow.com/hot_network_questions -directory: - topics: - - tech - title: "Stack Overflow — Hot Network Questions" - summary: "Trending developer questions from the Stack Overflow network." -strategy: botasaurus - -channel: - language: en - title: "Stack Overflow — Hot Network Questions" - url: https://stackoverflow.com/questions - time_zone: America/New_York - ttl: 30 -request: - botasaurus: - wait_timeout_seconds: 20 - wait_for_selector: "#hot-network-questions" -selectors: - items: - selector: "#hot-network-questions ul > li" - enhance: false - title: - selector: a - url: - selector: a - extractor: href - network: - selector: ".favicon" - extractor: attribute - attribute: title - categories: - - network diff --git a/app/registries/seed/official/configs/steuerzahler.de/news.yml b/app/registries/seed/official/configs/steuerzahler.de/news.yml deleted file mode 100644 index 07443d874..000000000 --- a/app/registries/seed/official/configs/steuerzahler.de/news.yml +++ /dev/null @@ -1,29 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: steuerzahler.de/news -directory: - topics: - - civic - title: "Bund der Steuerzahler — News" - summary: "Nachrichten vom Bund der Steuerzahler Deutschland." -channel: - title: "Bund der Steuerzahler — News" - url: https://www.steuerzahler.de/news/ - time_zone: Europe/Berlin - ttl: 720 - language: de -selectors: - items: - selector: ".bdst_presslist .bdst_presslist__block" - enhance: false - title: - selector: "a" - url: - selector: "a" - extractor: "href" - description: - selector: ".bdst_presslist__block__teaser" - published_at: - selector: ".bdst_presslist__block__date" - post_process: - - name: "parse_time" diff --git a/app/registries/seed/official/configs/stripes.com/index.yml b/app/registries/seed/official/configs/stripes.com/index.yml deleted file mode 100644 index fae6cef82..000000000 --- a/app/registries/seed/official/configs/stripes.com/index.yml +++ /dev/null @@ -1,27 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json ---- -registry: - id: stripes.com/index -directory: - topics: - - news - title: "Stars and Stripes — Front page" - summary: "Independent news for the U.S. military community from Stars and Stripes." -channel: - title: "Stars and Stripes — Front page" - url: https://www.stripes.com/ - language: en - ttl: 360 - time_zone: America/New_York -selectors: - items: - selector: '[class$="teaser"]' - title: - selector: a - url: - selector: a - extractor: href - description: - selector: img - extractor: attribute - attribute: alt diff --git a/app/registries/seed/official/configs/support.apple.com/en_gb_ht201222.yml b/app/registries/seed/official/configs/support.apple.com/en_gb_ht201222.yml deleted file mode 100644 index f97e60f60..000000000 --- a/app/registries/seed/official/configs/support.apple.com/en_gb_ht201222.yml +++ /dev/null @@ -1,37 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json ---- -registry: - id: support.apple.com/en_gb_ht201222 -directory: - topics: - - tech - - security - title: "Apple Support — Security releases" - summary: "Apple security update and release notes (HT201222 / related)." -strategy: botasaurus - -channel: - title: "Apple Support — Security releases" - url: https://support.apple.com/en-gb/100100 - language: en - ttl: 360 - time_zone: UTC -request: - botasaurus: - wait_for_selector: ".table-wrapper table tbody tr a" - wait_timeout_seconds: 20 -selectors: - items: - selector: ".table-wrapper table tbody > tr:not(:first-child)" - enhance: false - title: - selector: a - url: - selector: a - extractor: href - description: - selector: "td:nth-child(2)" - published_at: - selector: "td:nth-child(3)" - post_process: - - name: parse_time diff --git a/app/registries/seed/official/configs/support.apple.com/exchange_repair.yml b/app/registries/seed/official/configs/support.apple.com/exchange_repair.yml deleted file mode 100644 index 5321603f4..000000000 --- a/app/registries/seed/official/configs/support.apple.com/exchange_repair.yml +++ /dev/null @@ -1,34 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: support.apple.com/exchange_repair -directory: - topics: - - tech - - security - title: "Apple Support — Service programs" - summary: "Apple repair and exchange service program announcements." -channel: - language: en - title: "Apple Support — Service programs" - url: https://support.apple.com/service-programs - time_zone: America/Los_Angeles - ttl: 720 -selectors: - items: - selector: ".as-columns--2up-extended" - enhance: false - title: - selector: "a" - url: - selector: "a" - extractor: "href" - img: - selector: "img" - extractor: "attribute" - attribute: "src" - description: - post_process: - - name: template - string: | - # ![](%{img}) - - name: markdown_to_html diff --git a/app/registries/seed/official/configs/tap.info.tn/politics.yml b/app/registries/seed/official/configs/tap.info.tn/politics.yml deleted file mode 100644 index c460912c6..000000000 --- a/app/registries/seed/official/configs/tap.info.tn/politics.yml +++ /dev/null @@ -1,29 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: tap.info.tn/politics -directory: - topics: - - news - - civic - title: "TAP — Politics" - summary: "Political news from Tunisia’s official news agency (TAP)." -strategy: botasaurus - -channel: - title: "TAP — Politics" - url: https://www.tap.info.tn/en - language: en - time_zone: Africa/Tunis - ttl: 360 -request: - botasaurus: - wait_for_selector: "td.NewsItemHeadline a" - wait_timeout_seconds: 20 -selectors: - items: - selector: "td.NewsItemHeadline a" - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/telenor.com/newsroom.yml b/app/registries/seed/official/configs/telenor.com/newsroom.yml deleted file mode 100644 index 030b8ea30..000000000 --- a/app/registries/seed/official/configs/telenor.com/newsroom.yml +++ /dev/null @@ -1,22 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: telenor.com/newsroom -directory: - topics: - - tech - title: "Telenor — Newsroom" - summary: "Corporate newsroom updates from Telenor." -channel: - title: "Telenor — Newsroom" - url: https://www.telenor.com/media/newsroom/ - language: en - time_zone: Europe/Oslo - ttl: 360 -selectors: - items: - selector: 'a.link-wrap.teaser-wrap[href*="/media/newsroom/"]' - enhance: false - title: - selector: h2 - url: - extractor: href diff --git a/app/registries/seed/official/configs/teneriffa-news.com/news.yml b/app/registries/seed/official/configs/teneriffa-news.com/news.yml deleted file mode 100644 index 0ff7ec08a..000000000 --- a/app/registries/seed/official/configs/teneriffa-news.com/news.yml +++ /dev/null @@ -1,36 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: teneriffa-news.com/news -directory: - topics: - - news - - travel - title: "Teneriffa News — News" - summary: "Deutschsprachige Nachrichten von Teneriffa und den Kanaren." -channel: - title: "Teneriffa News — News" - url: https://www.teneriffa-news.com/news - time_zone: "Europe/Lisbon" - ttl: 720 - language: de -selectors: - items: - selector: ".main-content article" - title: - selector: "h2" - url: - selector: "a" - extractor: "href" - img_url: - selector: "img" - extractor: "attribute" - attribute: "data-lazy-src" - description: - selector: ".excerpt" - post_process: - - name: template - string: | - ![](%{img_url}) - - %{self} - - name: markdown_to_html diff --git a/app/registries/seed/official/configs/tennet.eu/news.yml b/app/registries/seed/official/configs/tennet.eu/news.yml deleted file mode 100644 index 99df43cc6..000000000 --- a/app/registries/seed/official/configs/tennet.eu/news.yml +++ /dev/null @@ -1,22 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: tennet.eu/news -directory: - topics: - - energy - title: "TenneT — News" - summary: "Grid and energy news from transmission operator TenneT." -channel: - title: "TenneT — News" - url: https://www.tennet.eu/news - language: en - time_zone: Europe/Amsterdam - ttl: 360 -selectors: - items: - selector: 'a[href^="/news/"]' - enhance: false - title: - selector: "h3, h4" - url: - extractor: href diff --git a/app/registries/seed/official/configs/test.de/archiv.yml b/app/registries/seed/official/configs/test.de/archiv.yml deleted file mode 100644 index e95ad41fe..000000000 --- a/app/registries/seed/official/configs/test.de/archiv.yml +++ /dev/null @@ -1,25 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json ---- -registry: - id: test.de/archiv -directory: - topics: - - consumer - title: "Stiftung Warentest — Archiv" - summary: "Archivierte Tests und Meldungen von Stiftung Warentest." -channel: - title: "Stiftung Warentest — Archiv" - url: https://www.test.de/archiv/ - language: de - ttl: 360 - time_zone: Europe/Berlin -selectors: - items: - selector: ".archive__list > li" - title: - selector: a - url: - selector: a:first - extractor: href - description: - selector: p diff --git a/app/registries/seed/official/configs/theeastafrican.co.ke/news.yml b/app/registries/seed/official/configs/theeastafrican.co.ke/news.yml deleted file mode 100644 index db9e51871..000000000 --- a/app/registries/seed/official/configs/theeastafrican.co.ke/news.yml +++ /dev/null @@ -1,29 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: theeastafrican.co.ke/news -directory: - topics: - - news - title: "The East African — News" - summary: "Regional news from The East African." -strategy: botasaurus - -channel: - title: "The East African — News" - url: https://www.theeastafrican.co.ke/tea/news - language: en - time_zone: Africa/Nairobi - ttl: 360 -request: - botasaurus: - wait_for_selector: 'a[href*="/tea/news/"][aria-label]' - wait_timeout_seconds: 20 -selectors: - items: - selector: 'a[href*="/tea/news/"][aria-label]' - enhance: false - title: - extractor: attribute - attribute: aria-label - url: - extractor: href diff --git a/app/registries/seed/official/configs/theguardian.com/international_mostpopular.yml b/app/registries/seed/official/configs/theguardian.com/international_mostpopular.yml deleted file mode 100644 index 333b3ea1b..000000000 --- a/app/registries/seed/official/configs/theguardian.com/international_mostpopular.yml +++ /dev/null @@ -1,22 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: theguardian.com/international_mostpopular -directory: - topics: - - news - title: "The Guardian — International most popular" - summary: "Most-read international stories from The Guardian." -channel: - language: en - title: "The Guardian — International most popular" - url: https://www.theguardian.com/international - time_zone: Europe/London - ttl: 60 -selectors: - items: - selector: '[data-link-name="most-viewed"] > ol > li' - title: - selector: "h4 span" - url: - selector: "a" - extractor: "href" diff --git a/app/registries/seed/official/configs/thenationalnews.com/news.yml b/app/registries/seed/official/configs/thenationalnews.com/news.yml deleted file mode 100644 index 911d873a6..000000000 --- a/app/registries/seed/official/configs/thenationalnews.com/news.yml +++ /dev/null @@ -1,22 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: thenationalnews.com/news -directory: - topics: - - news - title: "The National — News" - summary: "News from The National (UAE)." -channel: - title: "The National — News" - url: https://www.thenationalnews.com/news/ - language: en - time_zone: Asia/Dubai - ttl: 360 -selectors: - items: - selector: 'a[href*="/news/"][href*="/202"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/thoughtworks.com/insights.yml b/app/registries/seed/official/configs/thoughtworks.com/insights.yml deleted file mode 100644 index 85ac32315..000000000 --- a/app/registries/seed/official/configs/thoughtworks.com/insights.yml +++ /dev/null @@ -1,29 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: thoughtworks.com/insights -directory: - topics: - - tech - - research - title: "Thoughtworks — Insights" - summary: "Technology and business insights from the Thoughtworks blog." -strategy: botasaurus - -channel: - title: "Thoughtworks — Insights" - url: https://www.thoughtworks.com/insights/blog - language: en - ttl: 360 - time_zone: UTC -request: - botasaurus: - wait_timeout_seconds: 20 - wait_for_selector: 'a[href*="/insights/blog/"]' -selectors: - items: - selector: 'a[href*="/insights/blog/"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/tno.nl/news.yml b/app/registries/seed/official/configs/tno.nl/news.yml deleted file mode 100644 index 232b56ef4..000000000 --- a/app/registries/seed/official/configs/tno.nl/news.yml +++ /dev/null @@ -1,24 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: tno.nl/news -directory: - topics: - - research - - tech - title: "TNO — News" - summary: "Research and innovation news from TNO in the Netherlands." -channel: - title: "TNO — News" - url: https://www.tno.nl/en/newsroom/news/ - language: en - time_zone: Europe/Amsterdam - ttl: 360 -selectors: - items: - selector: div.collage-card-content - enhance: false - title: - selector: h3 - url: - selector: 'a[href*="/en/newsroom/20"]' - extractor: href diff --git a/app/registries/seed/official/configs/tourismusnetzwerk-brandenburg.de/aktuelle_nachrichten.yml b/app/registries/seed/official/configs/tourismusnetzwerk-brandenburg.de/aktuelle_nachrichten.yml deleted file mode 100644 index 0d383e873..000000000 --- a/app/registries/seed/official/configs/tourismusnetzwerk-brandenburg.de/aktuelle_nachrichten.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: tourismusnetzwerk-brandenburg.de/aktuelle_nachrichten -directory: - topics: - - travel - title: "Tourismusnetzwerk Brandenburg — Aktuelle Nachrichten" - summary: "Aktuelle Nachrichten für den Tourismus in Brandenburg." -channel: - title: "Tourismusnetzwerk Brandenburg — Aktuelle Nachrichten" - url: https://tourismusnetzwerk-brandenburg.de/ - time_zone: Europe/Berlin - ttl: 720 - language: de -enhance: false -selectors: - items: - selector: "article.node.article.wall-floating" - title: - selector: "h3.title a[rel='bookmark']" - url: - selector: "h3.title a[rel='bookmark']" - extractor: "href" diff --git a/app/registries/seed/official/configs/tralac.org/news.yml b/app/registries/seed/official/configs/tralac.org/news.yml deleted file mode 100644 index 0a1dbc09d..000000000 --- a/app/registries/seed/official/configs/tralac.org/news.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: tralac.org/news -directory: - topics: - - civic - - research - title: "tralac — News" - summary: "Trade law and policy news from the Trade Law Centre (tralac)." -channel: - title: "tralac — News" - url: https://www.tralac.org/news.html - language: en - time_zone: Africa/Johannesburg - ttl: 360 -selectors: - items: - selector: 'h3 a[href*="/news/article/"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/tudelft.nl/news.yml b/app/registries/seed/official/configs/tudelft.nl/news.yml deleted file mode 100644 index d0786ce32..000000000 --- a/app/registries/seed/official/configs/tudelft.nl/news.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: tudelft.nl/news -directory: - topics: - - science - - research - title: "TU Delft — News" - summary: "University and research news from Delft University of Technology." -channel: - title: "TU Delft — News" - url: https://www.tudelft.nl/en/about-tu-delft/current - language: en - time_zone: Europe/Amsterdam - ttl: 360 -selectors: - items: - selector: 'a.card[href*="/en/20"]' - enhance: true - title: - selector: h3 - url: - extractor: href diff --git a/app/registries/seed/official/configs/tue.nl/news.yml b/app/registries/seed/official/configs/tue.nl/news.yml deleted file mode 100644 index d5cecc03f..000000000 --- a/app/registries/seed/official/configs/tue.nl/news.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: tue.nl/news -directory: - topics: - - science - - research - title: "TU Eindhoven — News" - summary: "University and research news from Eindhoven University of Technology." -channel: - title: "TU Eindhoven — News" - url: https://www.tue.nl/en/news-and-events/news-overview - language: en - time_zone: Europe/Amsterdam - ttl: 360 -selectors: - items: - selector: 'a.entryBlock[href*="/en/news-and-events/news-overview/"]' - enhance: true - title: - selector: h3.entryBlock-title - url: - extractor: href diff --git a/app/registries/seed/official/configs/vattenfall.com/newsroom.yml b/app/registries/seed/official/configs/vattenfall.com/newsroom.yml deleted file mode 100644 index 9bcfbf559..000000000 --- a/app/registries/seed/official/configs/vattenfall.com/newsroom.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: vattenfall.com/newsroom -directory: - topics: - - energy - - environment - title: "Vattenfall — Newsroom" - summary: "Corporate and energy news from Vattenfall." -channel: - title: "Vattenfall — Newsroom" - url: https://group.vattenfall.com/press-and-media/newsroom - language: en - time_zone: Europe/Stockholm - ttl: 360 -selectors: - items: - selector: 'a.main-link[href*="/press-and-media/newsroom/20"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/visitfinland.com/press-releases.yml b/app/registries/seed/official/configs/visitfinland.com/press-releases.yml deleted file mode 100644 index 147788909..000000000 --- a/app/registries/seed/official/configs/visitfinland.com/press-releases.yml +++ /dev/null @@ -1,27 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: visitfinland.com/press-releases -directory: - topics: - - travel - title: "Visit Finland — Press releases" - summary: "Press releases from Visit Finland’s media centre." -strategy: botasaurus -channel: - title: "Visit Finland — Press releases" - url: https://media.visitfinland.com/en/media-press-releases/ - language: en - time_zone: Europe/Helsinki - ttl: 360 -request: - botasaurus: - wait_for_selector: 'a[href*="/en/media-press-releases/"]' - wait_timeout_seconds: 20 -selectors: - items: - selector: 'a.PageCard[href*="/en/media-press-releases/"]' - enhance: true - title: - selector: div.h5 - url: - extractor: href diff --git a/app/registries/seed/official/configs/webentwickler-jobs.de/in.yml b/app/registries/seed/official/configs/webentwickler-jobs.de/in.yml deleted file mode 100644 index cc35d7f46..000000000 --- a/app/registries/seed/official/configs/webentwickler-jobs.de/in.yml +++ /dev/null @@ -1,31 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json ---- -registry: - id: webentwickler-jobs.de/in -directory: - topics: - - jobs - title: "Webentwickler-Jobs — Region" - summary: "Webentwickler-Stellenangebote für eine wählbare Region in Deutschland." -parameters: - region: - type: string - default: "berlin" - -channel: - title: "Webentwickler-Jobs — Region" - url: https://www.webentwickler-jobs.de/in/%s - language: de - ttl: 360 - time_zone: Europe/Berlin -selectors: - items: - selector: ".job-item" - title: - selector: ".job-title" - url: - selector: a:first - extractor: href - description: - selector: "> div" - extractor: html diff --git a/app/registries/seed/official/configs/who.int/news.yml b/app/registries/seed/official/configs/who.int/news.yml deleted file mode 100644 index c100d5204..000000000 --- a/app/registries/seed/official/configs/who.int/news.yml +++ /dev/null @@ -1,30 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: who.int/news -directory: - topics: - - science - - civic - title: "WHO — News" - summary: "Official news and statements from the World Health Organization." -strategy: botasaurus - -channel: - title: "WHO — News" - url: https://www.who.int/news - language: en - time_zone: UTC - ttl: 360 -request: - botasaurus: - wait_for_selector: 'a.link-container[href*="/news/item/"]' - wait_timeout_seconds: 20 -selectors: - items: - selector: 'a.link-container[href*="/news/item/"]' - enhance: false - title: - extractor: attribute - attribute: aria-label - url: - extractor: href diff --git a/app/registries/seed/official/configs/worldbank.org/news.yml b/app/registries/seed/official/configs/worldbank.org/news.yml deleted file mode 100644 index b183bd718..000000000 --- a/app/registries/seed/official/configs/worldbank.org/news.yml +++ /dev/null @@ -1,29 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: worldbank.org/news -directory: - topics: - - finance - - civic - title: "World Bank — News" - summary: "Development and policy news from the World Bank." -strategy: botasaurus - -channel: - title: "World Bank — News" - url: https://www.worldbank.org/ext/en/news - language: en - time_zone: UTC - ttl: 360 -request: - botasaurus: - wait_for_selector: 'a.button[href*="/news/press-release/"]' - wait_timeout_seconds: 20 -selectors: - items: - selector: 'a.button[href*="/news/press-release/"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/www.consilium.europa.eu/press-releases.yml b/app/registries/seed/official/configs/www.consilium.europa.eu/press-releases.yml deleted file mode 100644 index b458bd2a8..000000000 --- a/app/registries/seed/official/configs/www.consilium.europa.eu/press-releases.yml +++ /dev/null @@ -1,28 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: consilium.europa.eu/press-releases -directory: - topics: - - civic - title: "Council of the EU — Press releases" - summary: "Press releases from the Council of the European Union." -strategy: botasaurus - -channel: - title: "Council of the EU — Press releases" - url: https://www.consilium.europa.eu/en/press/press-releases/ - language: en - time_zone: UTC - ttl: 360 -request: - botasaurus: - wait_for_selector: 'a.gsc-excerpt-item__link[href*="/press/press-releases/"]' - wait_timeout_seconds: 20 -selectors: - items: - selector: 'a.gsc-excerpt-item__link[href*="/en/press/press-releases/20"]' - enhance: false - title: - selector: span.gsc-excerpt-item__title - url: - extractor: href diff --git a/app/registries/seed/official/configs/www.ecdc.europa.eu/news-events.yml b/app/registries/seed/official/configs/www.ecdc.europa.eu/news-events.yml deleted file mode 100644 index 0835868f3..000000000 --- a/app/registries/seed/official/configs/www.ecdc.europa.eu/news-events.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: ecdc.europa.eu/news-events -directory: - topics: - - science - - civic - title: "ECDC — News and events" - summary: "News and events from the European Centre for Disease Prevention and Control." -channel: - title: "ECDC — News and events" - url: https://www.ecdc.europa.eu/en/news-events - language: en - time_zone: UTC - ttl: 360 -selectors: - items: - selector: 'a[href^="/en/news-events/"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/www.enisa.europa.eu/news.yml b/app/registries/seed/official/configs/www.enisa.europa.eu/news.yml deleted file mode 100644 index 228824c9e..000000000 --- a/app/registries/seed/official/configs/www.enisa.europa.eu/news.yml +++ /dev/null @@ -1,23 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: enisa.europa.eu/news -directory: - topics: - - security - - civic - title: "ENISA — News" - summary: "Cybersecurity news from the EU Agency for Cybersecurity (ENISA)." -channel: - title: "ENISA — News" - url: https://www.enisa.europa.eu/news - language: en - time_zone: UTC - ttl: 360 -selectors: - items: - selector: 'h3 a[href^="/news/"]' - enhance: false - title: - extractor: text - url: - extractor: href diff --git a/app/registries/seed/official/configs/www.europarl.europa.eu/press-room.yml b/app/registries/seed/official/configs/www.europarl.europa.eu/press-room.yml deleted file mode 100644 index 5da553bb9..000000000 --- a/app/registries/seed/official/configs/www.europarl.europa.eu/press-room.yml +++ /dev/null @@ -1,28 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: europarl.europa.eu/press-room -directory: - topics: - - civic - title: "European Parliament — Press room" - summary: "Press room releases from the European Parliament." -strategy: botasaurus - -channel: - title: "European Parliament — Press room" - url: https://www.europarl.europa.eu/news/en/press-room - language: en - time_zone: UTC - ttl: 360 -request: - botasaurus: - wait_for_selector: 'a[href*="/news/en/press-room/"]' - wait_timeout_seconds: 20 -selectors: - items: - selector: 'a[href*="/news/en/press-room/"][title="Read more"]' - enhance: false - title: - selector: span.ep_name - url: - extractor: href diff --git a/app/registries/seed/official/configs/www.gob.mx/profeco-prensa.yml b/app/registries/seed/official/configs/www.gob.mx/profeco-prensa.yml deleted file mode 100644 index 79fec1482..000000000 --- a/app/registries/seed/official/configs/www.gob.mx/profeco-prensa.yml +++ /dev/null @@ -1,24 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json -registry: - id: gob.mx/profeco-prensa -directory: - topics: - - consumer - - civic - title: "PROFECO — Prensa" - summary: "Comunicados de prensa de la Procuraduría Federal del Consumidor (México)." -channel: - title: "PROFECO — Prensa" - url: https://www.gob.mx/profeco/es/archivo/prensa - language: es - time_zone: America/Mexico_City - ttl: 360 -selectors: - items: - selector: article - enhance: false - title: - selector: h2 - url: - selector: 'a.small-link[href*="/profeco/prensa/"]' - extractor: href diff --git a/app/registries/seed/official/manifest.json b/app/registries/seed/official/manifest.json deleted file mode 100644 index 89a4d5ca3..000000000 --- a/app/registries/seed/official/manifest.json +++ /dev/null @@ -1,152 +0,0 @@ -{ - "format": "registry.v1", - "registry_id": "official", - "version": "seed", - "public_key_id": "html2rss:registry:2026", - "files": { - "configs/aalto.fi/news.yml": "be6418d7c29df5e6ec72d2768ba7fbb22bb97433fb293fd1a727eb4a5fd98236", - "configs/academictransfer.com/jobs.yml": "c17b3bfcb250604ffde4c7ad7c1cdf1fa1da01f862b20ec39354d1c6d902a954", - "configs/adfc.de/pressemitteilungen.yml": "ed07d0c8b2b57a18a66476aba923ac426a391da75c05eacb323b8f773cff5dde", - "configs/ahram.org.eg/egypt.yml": "73d5f85c86709804b73f375380382642debb45aaea7b43ea5f129fe00db5c6d1", - "configs/ai.meta.com/blog.yml": "1d872da4cb5ec904925bbbe2afc45207a72977292ff6d3dcb6e79e8f4f138137", - "configs/akorda.kz/events.yml": "87e61b20a5456635ec64c66c1aa0f3fc9f78c540a7aabd70738354ce2ea77285", - "configs/angop.ao/noticias.yml": "9304dc10fab78a5fe98067786dd8bbe3afe57c691d4930b88689628ed4bf44f4", - "configs/anthropic.com/news.yml": "679b3c28b6dd7e08bbfdaf8564e077e7c2ac4140eacbc9543756591aaf0621f5", - "configs/anu.edu.au/news.yml": "b9df2c362037a607c3c551503641411c042bed6151244dd8768db09e40289c0f", - "configs/apnews.com/hub.yml": "e891105a39e0be66211591e111ea1c20d31c6cd3bccd8d6304a535a6a28611fc", - "configs/apple.com/newsroom.yml": "87b705ce943aba697572e2ba3ca7ba847aa37688b09f8773af7ab6dfcfbbe8e6", - "configs/aps.dz/national-news.yml": "fd37cf647eeab34d579b3fa676fc4226d17d79421755592d33d625bac36cb972", - "configs/asml.com/press-releases.yml": "e08a7462c4a996c96300617ade224f0c89f6f95a6225005894c6ba5ee5702747", - "configs/aspi.org.au/strategist.yml": "d18a20ce40c71565ec8bca95f1bedd0e6993b6ee7376e4ca8817694d182c4752", - "configs/bbc.co.uk/available_episodes.yml": "447938f86901784ce8e822534fcd4578893aa52cd0796b872d58acb2c7b946e1", - "configs/bbc.com/mundo.yml": "9c240029d3764f879e0e287683a8fea055d94c7773f5bc3b8b79d1c93ac06cfd", - "configs/blog.mondediplo.net/feed.yml": "945b7d2018661eb5e107d4b108ecbbe0721cc6f6d196f1cb0fae669fea941310", - "configs/brookings.edu/articles.yml": "6ebaf4509a0f35ca7b4f33d0dcdea0217afc3535c5df50f446e9e3780446a18b", - "configs/bundesregierung.de/pressemitteilungen.yml": "855fc75ec8109525a064e3cb3941eca01ee4528bd8449949a11b1b8e98c65580", - "configs/canarianweekly.com/front.yml": "eb75b1c599bf30a2f8ac459bfc9b6e5a0467bdc60898c3edb287b8f913d06216", - "configs/caricom.org/news.yml": "b3a72ebf0f668295ce6866e8b27b6e0a72fea93c6f3634c770a6d95d8228a975", - "configs/carnegieendowment.org/research.yml": "4f353f70fa0a3cc025043495c3334364ebba41d541d1bd0e466eccff0ecd3ae8", - "configs/cepal.org/press-releases.yml": "47d60aafae23b97053a232feda8389cd35dd14b352a9ae43b3ee3da7437e6769", - "configs/chalmers.se/news.yml": "cfdc39fa8a2dcdaf29ba57c89847de8942b1de547a71501c4027b2af602721a7", - "configs/chathamhouse.org/research-publications.yml": "7e9268663fd1b3ed6e6b7484ba357fe091a192771c163f4af52bb91139e0d449", - "configs/cinemascore.com/index.yml": "abcfb21c9fff41106f5f7d7459393dd82662ea8f0f616897da4303f425ec1b9f", - "configs/cleanenergywire.org/news.yml": "df124817ca29345f4c03a39ba6c7af99cebc7ca0df3b43f66aa701960f53367f", - "configs/cnet.com/section_sub.yml": "f38ec616700966b921ee5adf77d584c0470fd04fdbf66ee20e379b31f2778feb", - "configs/cohere.com/blog.yml": "9b5be18602058c9b3e193c1d7c27a373b90d5771ad618aa249ac0d5fbb2d4bef", - "configs/computerbase.de/meistgelesen.yml": "549a8840eb5cc1998240f50fb5d00987b9bf34d708a1b5e9a59ceea8b78f5c13", - "configs/cplp.org/noticias.yml": "54fab81b4c1f1801baefa4b7af01af635b8bbca30767a4461467f022f1b34552", - "configs/csiro.au/news.yml": "bd16a57edc5fc0cd697b93a5c431ccec7dda0df845e58191b1cff2aaab84166a", - "configs/curia.europa.eu/press-releases.yml": "b23e6d984da5a3fd77151e7f647b260d226800b1315da630482ff936bf044dc6", - "configs/cutle.fish/index.yml": "04efa9acb4c9765d71df0411a623b9345f40649bedbca24b88ad3b441ac68cdd", - "configs/deepmind.google/blog.yml": "3dbc30f109bb169f1de06372236a2d352584808d11be8243ba93befcd94a7b41", - "configs/deraktionaer.de/meistgelesen.yml": "da9e238088745250f02604f98f5d3b85fc2e63b5c572546244dc386ebd512d44", - "configs/developer.apple.com/tutorials_data_documentation_technotes_json.yml": "60a974d2fab2e0c67bd9a948188a3972f15bec344dd8919349edcb05119ca012", - "configs/dfs.de/pressemitteilungen.yml": "6a3e8d83e0370812eb7b90fcbb659cc9356f284ecb66b9c35c684a2f4dd0ea36", - "configs/diis.dk/research.yml": "ce08a485afdcef6faa9a885e6fbf3e236fba89c64319c68c031a784bd1864fce", - "configs/dirco.gov.za/media-statements.yml": "42c6db2eed67544a27bf4e545ae9dc878a6b079373f12d2ff0d567a6a7939e34", - "configs/dsw-info.de/presse.yml": "eacac63e3d09a7f0748ef5e73ee5dfc26c18fa7af0a03d41483c20ea8328018b", - "configs/elastic.co/elasticsearch-release-notes.yml": "62c3f341563e9d44421639d7b6e54d0cb9eda7396995e3e14b4ea4efe762a810", - "configs/elysee.fr/actualites.yml": "e3958aa4d1db3214d4ce8a296d0f6d7ee3bd20c58953de37d3a3b91037532324", - "configs/equinor.com/news.yml": "7b9bbd0a2d0b8fcadf6ae311916d248a6f6590973725c57b8ffc3ce75c02faee", - "configs/espn.com/f1.yml": "3b98cc8ec8dcb164856a45490c0a99e0c206c8d6924c07dac1823d5630a19492", - "configs/ethz.ch/eth-news.yml": "a8a8f5e54203f4db8c11f64b990e13d7672079f89e5eb0984118610868834e6b", - "configs/eures.europa.eu/news.yml": "3e65bb8ee673cefb8476bb6bd68b79da835f6c3075a3c03e3f092e358e710bb2", - "configs/faa.gov/newsroom.yml": "6df127a4ac09a58dc03dad95769b34346bb55d194c124164291288e9d3d62f64", - "configs/fao.org/newsroom.yml": "2c481358fb0ee6515b4b7874b3b99f6599482f43475ae70915893ffd097e0a11", - "configs/fia.com/documents.yml": "42b6d296f1cdfcfef101e3edadcf1a579e11d38c3a9822a9fdfda55a8215125b", - "configs/figma.com/blog.yml": "f8e2013caa5ae449853652dc67ce177c1b9839fe469b27ca7d4740c50cc45c3e", - "configs/formula1.com/latest.yml": "9ac90ef1a0653e3dd8b09c721a4236dd7ec149a4a278a3151849909723f417be", - "configs/forumsec.org/publications.yml": "5cf0a07fa1fc6a3399a806076619aaf911ef0ad8e99081eef8832dd3f883f172", - "configs/ftc.gov/press-releases.yml": "fc0bef1c92a2d9309d058db4c1df907c6abc27999a1350812c1d9e5a2558d084", - "configs/github.com/releases.yml": "2071e5379acabd786750092b8d57f8ab87bc7c21627f77f5eeecc751b1e6f244", - "configs/go.dev/release-history.yml": "c104d8d68b5df8924b42bc1723656afc004cf808d6fe10301adc38d671483370", - "configs/grafana.com/whatsnew.yml": "6c9570538a07390c16dab99b2f1c848b4dd02125286df0964b8a85370a27a7f2", - "configs/haaretz.com/israel-news.yml": "9221cced7787014139b7f6a74315dc274294845179e86bf3d30761542bf5eaff", - "configs/hai.stanford.edu/news.yml": "e91bb67d9a948a4ae054006ba0b748cd46ec6389d46e9c138aad6224af1e5e27", - "configs/iaapa.org/news.yml": "b93b9cc1701f2a06c6d85a97315a1f0bdbd5c0a749c008f8923421730eae8041", - "configs/icj-cij.org/press-releases.yml": "40c4cd883dc6495fb56b2ccbceff0f8cf62fccd6b767ce466463e6b939709423", - "configs/icrc.org/news.yml": "cf2efef0b9b6082af1fea9d8b6fe1a24bebe92aa4070ddc3bc71aa02c60f94f5", - "configs/iea.org/news.yml": "364a230c750b26a2ad25d5e9e1d576516d6103f60ebf588813dadba7316edd6d", - "configs/iiss.org/online-analysis.yml": "e1c3a83a6d6d4e2e7cb3b2ad47329bd56a66dc31883d9f370c870932044e1e01", - "configs/ikea.com/newsroom.yml": "d80881b7ab88d7038da48e17d95a6268cc9a0d4af4189574cab09549d1ef3ecb", - "configs/imdb.com/ratings.yml": "152fe7418221ad0cc6cf38fd6f21a8dfc8575a80570c5eb9e3fe1b6c8959ddf8", - "configs/imo.org/press-briefings.yml": "4bd88ae16ee124ad285fc4cdaff4ee121c69b90215d77995a15cbc8b24f6085f", - "configs/imperial.ac.uk/news.yml": "e84228d9ef1d2909cd911c5bb3f377a9b049be0daf6621dd1fa0b35cdfcaf346", - "configs/ingenieur.de/karriere_arbeitsleben_heiko_mell.yml": "e2b5bcbe13c7fb651d66d4f4cbb107eedffaa1f9b8adf905ce17c53ea0ecc7d8", - "configs/isro.gov.in/press.yml": "ad43cebc15708b0881a730ec85cbd6622ca93581f7e78494701eb09701a3ec6f", - "configs/issafrica.org/iss-today.yml": "58764d7023fbfa050ab9737cf230ad2ef52ac34aa1a23b6e32cf222fd826427a", - "configs/issafrica.org/press-releases.yml": "1bc3e7746eea235b575bcc7d34e558675da8c8a9ca0dfdfe2c725241667741b9", - "configs/kaist.ac.kr/news.yml": "af091015cb2785e32e373314280d7b228cb5435d4a0367658175b954de9b033f", - "configs/kinocheck.de/filmstarts.yml": "7f1791684ef623fca5a4e69154ccd23a70e615b87ac1ff85aec2263ca274b895", - "configs/kompas.com/nasional.yml": "cbf4bec2d0174ca6c0e387795e5d87fd765cecf378e5a3937c469f559b3dad54", - "configs/lego.com/press-releases.yml": "563c6259b56b94c5056f6dae62cc886b6886267b751212111e6c7ad4896db5e9", - "configs/machinelearning.apple.com/research.yml": "2e79eeb0c7b82400305f5a914e0094635840e0973b25b69f16f882b730ced1ab", - "configs/maersk.com/press-releases.yml": "a8c4e71bd0636b0c672602521bea38002d629c2061431b205b1d0cf704f8e19a", - "configs/microsoft.com/azure-products.yml": "1231e44f9f8f75264f085a571123135a4232707c7d76c7d8f47fc4d849c2fe47", - "configs/mistral.ai/news.yml": "4558d7a403be7a5b10423ac49bd2a5827260412f744cd6fd10be246f0fdb578a", - "configs/mozilla.org/security-advisories.yml": "8a38a705a77b2bf62965b7beca737e6ade25dce312b0d1d91eef81caf881ed43", - "configs/msf.org/latest.yml": "18f6c66a34e568c1504f6523a7a1251f66a0567896406568ffef95b650e125a7", - "configs/nbim.no/press-releases.yml": "9d6b161885272f082a7353bcbcf28ddd6a63b1ed5fbd01569cccc78953e4478e", - "configs/news.cn/latest.yml": "f5e07692a521afe6dc1bf1488e72843eb44906d5b27fb466aa57de45ad51de43", - "configs/news24.com/top-stories.yml": "921a2e0b90454bf02748f32d6cdb9e7d12f2180ca2fbbec8034d64292fd03db8", - "configs/newyorker.com/magazine.yml": "6f24906f442636d43ae9d6160abe481a249f337d34f24932535b0732b1e41b5c", - "configs/nigerianstat.gov.ng/news.yml": "2f456a944228e744355ad8ea69cbc6a0f11e534965d0f3a7636c7d9e2fe48ecd", - "configs/nikkei.com/asia-technology.yml": "494aaa988a4e4b964b219106bea9756666bad24c84e77b60621d4877bcaae739", - "configs/nokia.com/newsroom.yml": "084f94d7eb1936a58e911365485f993a2a637fbe8d8679235ce18c44ef2e75ed", - "configs/nomanssky.com/news.yml": "a7b52c620147a3b742435d7a3ea236e9e523f4c7c29cf95bb988d7a6daaca8ef", - "configs/notion.com/blog.yml": "59e873eb557d6e09d369f4fd3a64d114b57f2393a727eb08e2c95231c8e3959b", - "configs/novonordisk.com/latest-news.yml": "9ea839fdda40ff6be93600f9b482c3301cd0034359059c6c2d7957a7497fb4ef", - "configs/ntu.edu.sg/news.yml": "9b92065f45ddd7d8dad77f8d87f99824fb2cbab883b1748b6d39b12a68e82b7d", - "configs/nupi.no/news.yml": "e133d35bd1ab66b5e1aa92bf0981201881358e265dc9e6e91ec85b19955cfda1", - "configs/occrp.org/news.yml": "15cd632800e600638e0660c756ca71a91101c75b367f9a9b297223b69cbff65d", - "configs/oecd.org/newsroom.yml": "fadb3b49c5bc93da074c28b9cf2879792f823178c161afc69ce1b17a327691ef", - "configs/opec.org/press-releases.yml": "3661947bcacc5e22875f8f798b4a9d5de76d1db356ca9a860bbdc84fd2af4a5f", - "configs/panapress.com/latest.yml": "0201c9673f242d307be7c931e23645206f665b6ece059d341855dcd7ec477697", - "configs/pankow.lebensmittel-kontrollergebnisse.de/search.yml": "762016dd3beb117bb4d1fcb4e2743eab102c89ad1cfcde7fb1d60c55303438ae", - "configs/philstar.com/nation.yml": "b42907100f34f9336c6454a8211ccc4b76072a298c1df632a49696c2e9448380", - "configs/phys.org/weekly.yml": "5074931172c944b27f1154c609ebe5500f95f793ebd721b5beb11ada00e57806", - "configs/pib.gov.in/press-releases.yml": "e1ae3053c37309361a1e0400a2e899788cdbe8a3eddc3f62d4dd9459498f7fc5", - "configs/rbb24.de/meistgeklickt.yml": "f927df2c4f12a44d48695a4789bfd5f62e918a4cc541fcf7f089ff0a3b3cc2ad", - "configs/robinwood.de/aktuelles.yml": "a200b6b155cb7b42c40bb77e450293c2c2cb09a71cc57498cbe1b88c3ad9f8ea", - "configs/s3.amazonaws.com/popular_movies.yml": "8b0088efc8f41d44f00ca266652518113be0a3922e715c78aab45411686bd761", - "configs/sacu.int/news.yml": "d61870d4f143feb7c283c47ce13b1a633ef7e7901eed7466f34217f06fdd7fda", - "configs/sadc.int/latest-news.yml": "c90c74d763f40af8d66aa0ff511e56aa7a59bb7ef314da1ac9c1fd5b5027daa4", - "configs/sebastianvettel.de/news.yml": "a3ee51e82483febc2ad9a756da78cd702b88c37a9279f0e9468dd92183101010", - "configs/shopify.com/blog.yml": "89bd2a11740d6df76b2a6ff4ca5f04c6c9db2d44fe060acda85d6fe3243eaac1", - "configs/sipri.org/news.yml": "75297f03e5e9d78287549d3e7ddfa370477b58d22c3add1adbef60c12e5ecc63", - "configs/softwareleadweekly.com/issues.yml": "b97a9715185c62c6ac3aca000877805017f67deb8ba040596a79859051d21051", - "configs/solarthermalworld.org/news.yml": "9b7ceb4378d1ac6f30b5cf6788570624456054d3fbfde8659bf53ded309799bc", - "configs/spektrum.de/meistgelesen.yml": "3d881280aa5525166fdf6076a0674ac3bf306d0b1924fd91a89ed0b43fe15ed6", - "configs/spiegel.de/impressum_autor.yml": "68f698cbad5a5b228540e7eda2c12c392832110d147c7eececd2266a8796a54a", - "configs/spotify.com/newsroom.yml": "e88395077225e4c0979479c1bf875718425f155fd07c564faae417cb2f116535", - "configs/stackoverflow.com/hot_network_questions.yml": "1245ba114d85e3cf3b41ffe858d4d8f968a051a5856a5adbe0bf2622afa920a2", - "configs/steuerzahler.de/news.yml": "9ab7977a9def32e9cd0ce8cb64dae1237195b86df2726dc3232eebfb6f8b1ecc", - "configs/stripes.com/index.yml": "4c1513ed009add5e4fb6c8fb9a1f867c3ddc43c1c090ff3a27cece6da3e3c417", - "configs/support.apple.com/en_gb_ht201222.yml": "ee8f911c64cd22aa0008e909df69cf37359ab8d1c8805d44109ddafdfef56491", - "configs/support.apple.com/exchange_repair.yml": "89a01591e2319e0dad9e21f150188788dbe7610510499918ba8dfe1454d41a67", - "configs/tap.info.tn/politics.yml": "52a03af97d79e091720caafe06b1ccf3b86f79d2fc4604ae245715ca1c6449d6", - "configs/telenor.com/newsroom.yml": "ac67b05f988ad88195ad9ed5475e97c312ccaecb218d8192bce4bcb2834a11e0", - "configs/teneriffa-news.com/news.yml": "1831fd2e924235d8afaaeab511b1974ada9f28aba7114a640d6f97c2e2bc4573", - "configs/tennet.eu/news.yml": "959c51e6ef260edc9abd8ab62ea53ec722174ce98490b946d2fa1c98a5e94119", - "configs/test.de/archiv.yml": "544823eff24c249f7d165b64c84abac24192cf161a574e1b082a9021ba2268a8", - "configs/theeastafrican.co.ke/news.yml": "a77f2883db750a87ec25de38eba4188968d7cb23e1ea00c9adf51ce22faa7c02", - "configs/theguardian.com/international_mostpopular.yml": "ca4847f874faa1ed3a4519244e2bb01d8a6cd2bb8ad04cb771421dedb689d9dd", - "configs/thenationalnews.com/news.yml": "e27e19ba4b520784a83263c5a2a9653b80f36bd887ab8943776b34225ae8f91c", - "configs/thoughtworks.com/insights.yml": "0347ed1dc185c9104920c4548fd8ac71f6ead355d84b812aad469113dbf65135", - "configs/tno.nl/news.yml": "27ab648acea80a41257a18aa03dec394093369de5eba50b403d93f546231187b", - "configs/tourismusnetzwerk-brandenburg.de/aktuelle_nachrichten.yml": "b26b0ca870ed70ffd4c9ab6ad50ab6c232e80c20206cfdc32f4c26f015b6e370", - "configs/tralac.org/news.yml": "c6c3a401c7620c306034e7fdc67a3bbe5cb416927b1533948e1b76ad515bc562", - "configs/tudelft.nl/news.yml": "370761e6b2941f894efc5db0fbe8fcec4db6c8eb6030b5daf8e91853b1d001e9", - "configs/tue.nl/news.yml": "8191ffed06ea44c9f6ae3670cacb8d56d751eb6e838370694580bc93a0062ddd", - "configs/vattenfall.com/newsroom.yml": "32f8c3ae3a75ce477556e1de454440a95438d1c7e5ecd4f31eeb0dc136cf280c", - "configs/visitfinland.com/press-releases.yml": "72cdd2b796710e08dd3a5b02851c4d2c72354a53b020553c299c4ec0832efaf5", - "configs/webentwickler-jobs.de/in.yml": "ddb1a06e38ef8e03629cf62679f448d332499f08736a22aed6d66238c6ac2dfc", - "configs/who.int/news.yml": "efa19b4fe3e83b31a8227d5c5bf5cee2db2aa7e3ec90607a8a47d344561a83be", - "configs/worldbank.org/news.yml": "45e9f7005f6c3cc74fe0b02a33529a68272188ad6b56aec09ef0505c567c44c2", - "configs/www.consilium.europa.eu/press-releases.yml": "4728634af76a4a7a2b3f059e961fad7c370254324b444634afe36fb1bf03afa6", - "configs/www.ecdc.europa.eu/news-events.yml": "8a265ae15e4fc05f09c75743f3d47fee996e1f575693610ab7fb519b6a653181", - "configs/www.enisa.europa.eu/news.yml": "964cba7952e4ed510ca840d291b5c81fb90c8303f19e36b49d6aced28efbdf09", - "configs/www.europarl.europa.eu/press-room.yml": "99ab566381c60e6cebb91ebbe9d434e9153591014bee1772ec5d62b607375079", - "configs/www.gob.mx/profeco-prensa.yml": "603177b81dbe88da04e7bdac5a5186b4af288ca29b8cfc19c096fd9b5b933bfc" - } -} diff --git a/app/web/registry/index.rb b/app/web/registry/index.rb index 5d6d19dc6..c442c802f 100644 --- a/app/web/registry/index.rb +++ b/app/web/registry/index.rb @@ -235,7 +235,7 @@ def domain_matches?(host, domain) end def bundle_directory(definition) - definition.mode == :path ? definition.path : Store.registry_dir(definition.id) + definition.mode == :path ? definition.path : Store.active_dir(definition.id) end def normalize_feed_id(feed_id) diff --git a/app/web/registry/store.rb b/app/web/registry/store.rb index c0d0b88e8..1a181a890 100644 --- a/app/web/registry/store.rb +++ b/app/web/registry/store.rb @@ -10,7 +10,7 @@ module Registry # Transactional storage engine for registry bundles and sync state. module Store # rubocop:disable Metrics/ModuleLength DEFAULT_DATA_ROOT = 'tmp/registry-data' - DEFAULT_SEED_ROOT = '/app/registries/seed' + DEFAULT_EMBEDDED_ROOT = '/app/registries' SYNC_STATE_FILE = '.sync-state.json' ## @@ -26,8 +26,8 @@ def data_root ## # @return [String] - def seed_root - File.expand_path(ENV.fetch('REGISTRY_SEED_ROOT', DEFAULT_SEED_ROOT)) + def embedded_root + File.expand_path(ENV.fetch('REGISTRY_EMBEDDED_ROOT', DEFAULT_EMBEDDED_ROOT)) end ## @@ -37,6 +37,29 @@ def registry_dir(registry_id) File.join(data_root, registry_id.to_s) end + ## + # @param registry_id [String, Symbol] + # @return [String] + def embedded_dir(registry_id) + File.join(embedded_root, registry_id.to_s) + end + + ## + # Resolves the active bundle directory, preferring synced runtime data + # and falling back to the baked image bundle. + # + # @param registry_id [String, Symbol] + # @return [String, nil] + def active_dir(registry_id) + synced = registry_dir(registry_id) + return synced if bundle_present_at?(synced) + + embedded = embedded_dir(registry_id) + return embedded if bundle_present_at?(embedded) + + nil + end + ## # @param registry_id [String, Symbol] # @return [String] @@ -48,13 +71,15 @@ def staging_dir(registry_id) # @param registry_id [String, Symbol] # @return [Boolean] def bundle_present?(registry_id) - bundle_present_at?(registry_dir(registry_id)) + bundle_present_at?(registry_dir(registry_id)) || bundle_present_at?(embedded_dir(registry_id)) end ## - # @param path [String] + # @param path [String, nil] # @return [Boolean] def bundle_present_at?(path) + return false unless path && File.directory?(path) + File.file?(File.join(path, Html2rss::Registry::Manifest::MANIFEST_FILE)) end @@ -123,22 +148,6 @@ def promote_staged!(registry_id) FileUtils.rm_rf(temp_root) if temp_root end - ## - # Copies seed bundle into data root if active bundle is missing. - # - # @param registry_id [String, Symbol] - # @param seed_path [String] - # @return [Boolean] - def seed_if_empty!(registry_id, seed_path:) # rubocop:disable Naming/PredicateMethod - return false if bundle_present?(registry_id) - raise Errors::LoadError, "Seed bundle missing: #{seed_path}" unless File.directory?(seed_path) - - target = registry_dir(registry_id) - FileUtils.mkdir_p(File.dirname(target)) - FileUtils.cp_r(seed_path, target) - true - end - ## # @param registry_id [String, Symbol] # @return [SyncState] diff --git a/app/web/registry/sync.rb b/app/web/registry/sync.rb index f7892d68b..af16fcb84 100644 --- a/app/web/registry/sync.rb +++ b/app/web/registry/sync.rb @@ -79,12 +79,13 @@ def boot! # rubocop:disable Metrics/MethodLength end return if ENV.fetch('RACK_ENV', 'development') == 'test' - Config.precedence.each do |id| - definition = Config.entry(id) - next unless definition.mode == :sync + if ENV.fetch('REGISTRY_SYNC_ON_BOOT', 'false') == 'true' + Config.precedence.each do |id| + definition = Config.entry(id) + next unless definition.mode == :sync - seed_registry!(id) - Thread.new { run(registry_id: id) } if boot_sync?(id) # rubocop:disable ThreadSafety/NewThread + Thread.new { run(registry_id: id) } # rubocop:disable ThreadSafety/NewThread + end end start_background_timer! @@ -197,22 +198,6 @@ def report_catalog_change!(registry_id, previous) # rubocop:disable Metrics/AbcS } ) end - - def seed_registry!(registry_id) - seed_path = Store.seed_path_for(registry_id) - Index.reload! if File.directory?(seed_path) && Store.seed_if_empty!(registry_id, seed_path:) - rescue StandardError - nil - end - - def boot_sync?(registry_id) - return true if ENV.fetch('REGISTRY_SYNC_ON_BOOT', 'false') == 'true' - - bundle = Index.current.bundle_for(registry_id) - return true unless bundle - - %w[seed test-fixture].include?(bundle.manifest.version) - end end end end diff --git a/spec/html2rss/web/registry/index_spec.rb b/spec/html2rss/web/registry/index_spec.rb index c70a10955..a4a600058 100644 --- a/spec/html2rss/web/registry/index_spec.rb +++ b/spec/html2rss/web/registry/index_spec.rb @@ -155,4 +155,61 @@ .to raise_error(Html2rss::Web::Registry::Errors::LoadError, /anthropic\.com/) end end + + describe 'embedded bundle resolution' do + let(:embedded_config_path) { File.join(Dir.pwd, 'tmp', 'embedded-registries.yml') } + let(:embedded_root) { File.join(Dir.pwd, 'tmp', 'embedded-root') } + + before do + FileUtils.mkdir_p(File.dirname(embedded_config_path)) + File.write( + embedded_config_path, + YAML.dump( + 'precedence' => ['official'], + 'registries' => { + 'official' => { + 'sync' => { 'channel' => 'html2rss-official' }, + 'catalog' => true, + 'public_key_id' => 'test-key', + 'public_key' => RegistrySyncTestHelpers::TEST_PUBLIC_KEY + } + } + ) + ) + ENV['REGISTRIES_CONFIG'] = embedded_config_path + ENV['REGISTRY_DATA_ROOT'] = File.join(Dir.pwd, 'tmp', 'empty-data-root') + ENV['REGISTRY_EMBEDDED_ROOT'] = embedded_root + FileUtils.rm_rf(ENV.fetch('REGISTRY_DATA_ROOT')) + FileUtils.rm_rf(embedded_root) + + embedded_official = File.join(embedded_root, 'official') + FileUtils.mkdir_p(embedded_official) + FileUtils.cp_r(File.join(RegistryTestHelpers::FIXTURES_ROOT, 'official', 'configs'), embedded_official) + FileUtils.cp( + File.join(RegistrySyncTestHelpers::SYNC_FIXTURES_ROOT, 'bundle', Html2rss::Registry::Manifest::MANIFEST_FILE), + File.join(embedded_official, Html2rss::Registry::Manifest::MANIFEST_FILE) + ) + manifest = Html2rss::Registry::Manifest.parse( + File.read(File.join(embedded_official, Html2rss::Registry::Manifest::MANIFEST_FILE)) + ) + Html2rss::Registry::Signer.sign!( + manifest, + key_pem: RegistrySyncTestHelpers::TEST_PRIVATE_KEY, + bundle_dir: embedded_official + ) + described_class.reload! + end + + after do + FileUtils.rm_f(embedded_config_path) + FileUtils.rm_rf(embedded_root) + FileUtils.rm_rf(ENV.fetch('REGISTRY_DATA_ROOT', nil)) + end + + it 'loads configs directly from the embedded root when no runtime sync data exists' do + expect(described_class.current.config_for('anthropic.com/news')).to include( + channel: hash_including(title: 'Anthropic — News') + ) + end + end end diff --git a/spec/html2rss/web/registry/store_spec.rb b/spec/html2rss/web/registry/store_spec.rb index 716d5a7b7..d6cd8a77f 100644 --- a/spec/html2rss/web/registry/store_spec.rb +++ b/spec/html2rss/web/registry/store_spec.rb @@ -11,9 +11,13 @@ let(:registry_id) { 'store-test' } let(:data_root) { File.join(Dir.pwd, 'tmp', 'store-spec-data') } + let(:embedded_root) { File.join(Dir.pwd, 'tmp', 'store-spec-embedded') } + before do ENV['REGISTRY_DATA_ROOT'] = data_root + ENV['REGISTRY_EMBEDDED_ROOT'] = embedded_root FileUtils.rm_rf(data_root) + FileUtils.rm_rf(embedded_root) end describe '.stage_bundle! and .promote_staged!' do @@ -44,7 +48,7 @@ FileUtils.rm_rf(path) FileUtils.mkdir_p(path) - File.write(File.join(path, 'placeholder.txt'), 'seed') + File.write(File.join(path, 'placeholder.txt'), 'content') expect(described_class.bundle_present?(registry_id)).to be(false) @@ -53,6 +57,39 @@ end end + describe '.active_dir' do + let(:embedded_root) { File.join(Dir.pwd, 'tmp', 'store-spec-embedded') } + + before do + ENV['REGISTRY_EMBEDDED_ROOT'] = embedded_root + FileUtils.rm_rf(embedded_root) + end + + it 'returns nil when neither synced nor embedded bundle exists' do + expect(described_class.active_dir(registry_id)).to be_nil + end + + it 'falls back to embedded bundle directory when synced bundle is absent' do + embedded_dir = described_class.embedded_dir(registry_id) + FileUtils.mkdir_p(embedded_dir) + File.write(File.join(embedded_dir, Html2rss::Registry::Manifest::MANIFEST_FILE), '{}') + + expect(described_class.active_dir(registry_id)).to eq(embedded_dir) + end + + it 'prefers synced runtime bundle over embedded bundle' do + embedded_dir = described_class.embedded_dir(registry_id) + FileUtils.mkdir_p(embedded_dir) + File.write(File.join(embedded_dir, Html2rss::Registry::Manifest::MANIFEST_FILE), '{}') + + synced_dir = described_class.registry_dir(registry_id) + FileUtils.mkdir_p(synced_dir) + File.write(File.join(synced_dir, Html2rss::Registry::Manifest::MANIFEST_FILE), '{}') + + expect(described_class.active_dir(registry_id)).to eq(synced_dir) + end + end + def build_bundle_dir(version) # rubocop:disable Metrics/MethodLength dir = Dir.mktmpdir("registry-store-#{version}") config_path = File.join(dir, 'configs', 'example.com', 'feed.yml') diff --git a/spec/html2rss/web/registry/sync_spec.rb b/spec/html2rss/web/registry/sync_spec.rb index 46d298ec0..d0ac20012 100644 --- a/spec/html2rss/web/registry/sync_spec.rb +++ b/spec/html2rss/web/registry/sync_spec.rb @@ -216,12 +216,11 @@ def write_policy_config(yaml) describe '.boot!' do it 'does not run in the test environment' do - allow(described_class).to receive(:seed_registry!) allow(described_class).to receive(:start_background_timer!) described_class.boot! - expect(described_class).not_to have_received(:seed_registry!) + expect(described_class).not_to have_received(:start_background_timer!) end end From 2a1e9e5cebe96fd92a5540fd80d44c7d3b07fd51 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Mon, 24 Aug 2026 15:06:06 +0200 Subject: [PATCH 29/33] feat(docker): bake official registry artifact at build time and drop seed files --- .gitignore | 4 ++++ Dockerfile | 17 +++++++++++++++-- bin/docker-build | 2 +- bin/prepare-registry-seed | 18 +++++------------- 4 files changed, 25 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index 380efe19e..622aebe01 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,7 @@ frontend/.astro # Bundler/vendored dependencies /vendor/ + +# Local bundle builds +/dist/ + diff --git a/Dockerfile b/Dockerfile index 15aefdfea..909549c64 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,7 +38,20 @@ RUN apk add --no-cache \ /usr/local/bundle/bundler/gems/*/.git \ /usr/local/bundle/cache/bundler/git -# Stage 3: Runtime +# Stage 3: Official Registry Artifact Builder +FROM ${RUBY_BASE_IMAGE} AS registry-builder + +ARG REGISTRY_BUNDLE_URL="https://github.com/html2rss/html2rss-configs/releases/latest/download/registry-bundle.tar.gz" + +WORKDIR /build + +# hadolint ignore=DL3018 +RUN apk add --no-cache curl tar ca-certificates \ + && curl -fsSL -o registry-bundle.tar.gz "${REGISTRY_BUNDLE_URL}" \ + && mkdir -p /build/official \ + && tar -xzf registry-bundle.tar.gz -C /build/official + +# Stage 4: Runtime FROM ${RUBY_BASE_IMAGE} LABEL maintainer="Gil Desmarais " @@ -91,7 +104,7 @@ COPY --chown=$USER:$USER bin ./bin COPY --chown=$USER:$USER Gemfile Gemfile.lock app.rb config.ru ./ COPY --chown=$USER:$USER app ./app COPY --chown=$USER:$USER config ./config -COPY --chown=$USER:$USER app/registries/seed ./app/registries/seed +COPY --from=registry-builder --chown=$USER:$USER /build/official ./registries/official COPY --chown=$USER:$USER public ./public COPY --from=frontend-builder --chown=$USER:$USER /app/frontend/dist ./frontend/dist diff --git a/bin/docker-build b/bin/docker-build index 7fd9be1c9..b88cbc487 100755 --- a/bin/docker-build +++ b/bin/docker-build @@ -1,5 +1,5 @@ #!/bin/bash set -eux -"$(dirname "$0")/prepare-registry-seed" docker build --no-cache -t html2rss/web -f Dockerfile . + diff --git a/bin/prepare-registry-seed b/bin/prepare-registry-seed index 59728eee9..acbd8b74b 100755 --- a/bin/prepare-registry-seed +++ b/bin/prepare-registry-seed @@ -5,27 +5,19 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" CONFIGS_REPO="${HTML2RSS_CONFIGS_ROOT:-$ROOT_DIR/../html2rss-configs}" -SEED_DIR="$ROOT_DIR/app/registries/seed/official" -OUTPUT_TAR="$(mktemp -t registry-seed.XXXXXX.tar.gz)" - -cleanup() { - rm -f "$OUTPUT_TAR" -} -trap cleanup EXIT +OUTPUT_TAR="${1:-$ROOT_DIR/dist/registry-bundle.tar.gz}" if [[ ! -d "$CONFIGS_REPO/configs" ]]; then echo "Missing configs tree at $CONFIGS_REPO/configs" >&2 exit 1 fi -echo "Building registry seed bundle from $CONFIGS_REPO" +mkdir -p "$(dirname "$OUTPUT_TAR")" +echo "Building registry bundle from $CONFIGS_REPO into $OUTPUT_TAR" ( cd "$CONFIGS_REPO" - REGISTRY_VERSION=seed BUNDLE_GEMFILE=tool/Gemfile bundle exec ruby tool/registry-build --output "$OUTPUT_TAR" + BUNDLE_GEMFILE=tool/Gemfile bundle exec ruby tool/registry-build --output "$OUTPUT_TAR" ) -rm -rf "$SEED_DIR" -mkdir -p "$SEED_DIR" -tar -xzf "$OUTPUT_TAR" -C "$SEED_DIR" +echo "Registry bundle ready at $OUTPUT_TAR" -echo "Seed bundle prepared at $SEED_DIR" From b36b10c0c3e004179c7a8759ffe44e772d5ddb25 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Mon, 24 Aug 2026 15:06:13 +0200 Subject: [PATCH 30/33] docs(registry): align terminology and document embedded registry delivery --- AGENTS.md | 2 +- CONTEXT.md | 2 +- config/registries.yml | 4 +-- docs/README.md | 8 ++--- docs/architecture.md | 20 +++++++++++ docs/registry-go-live.md | 73 +++++++++++++--------------------------- 6 files changed, 52 insertions(+), 57 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 5bb870392..5a053e053 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -111,6 +111,6 @@ Public feed-directory metadata from verified registry bundles and local `feeds.y | Root metadata | `GET /api/v1/` exposes `instance.catalog: { enabled, url }` and `instance.registries` sync status | | Contract SSOT | Request specs under `spec/html2rss/web/api/v1_spec.rb` and generated `public/openapi.yaml` | -Registry sync: `bin/html2rss-web registry status`; boot seed + optional sync via `Registry::Sync.boot!`. See [docs/README.md](docs/README.md#registry-sync-runbook). +Registry sync: `bin/html2rss-web registry status`; embedded bundle + optional runtime sync via `Registry::Sync.boot!`. See [docs/README.md](docs/README.md#registry-sync-runbook). After handler or envelope changes: `make openapi` and `make ci-ready`. diff --git a/CONTEXT.md b/CONTEXT.md index 9c0f8b1d6..bd65d1bc3 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -59,7 +59,7 @@ Shared emit plumbing for both channels (`RequestContext`, `LogSanitizer`, `AppLo Backend merge owner for registry bundles and local `feeds.yml` feeds. Builds catalog wire rows (`Registry::Index::CatalogRow`), enforces load-time trust and channel-domain allowlists, and serves `config_for` / `catalog_rows` / `status`. ### Registry Sync -Backend orchestration for fetch → verify → stage/promote of signed registry bundles. Owns boot seeding, background refresh, CLI exit codes, and catalog-change telemetry after promotion. +Backend orchestration for fetch → verify → stage/promote of signed registry bundles. Owns boot initialization, background refresh, CLI exit codes, and catalog-change telemetry after promotion. ### Sync Transport Backend HTTPS fetch, sync URL resolution (GitHub releases and channel defaults), and manifest version gating used by `Registry::Sync` and parse-time config resolution. diff --git a/config/registries.yml b/config/registries.yml index 9c3d4af85..2bbd9b557 100644 --- a/config/registries.yml +++ b/config/registries.yml @@ -11,8 +11,7 @@ registries: -----BEGIN PUBLIC KEY----- MCowBQYDK2VwAyEAiMbg/04MyC5azBdM/aeY0mNuA8JbP5/jOiNRwJ2KJHE= -----END PUBLIC KEY----- - # Pin the publisher Ed25519 public key for signed network sync. - # Seed bundles copied from the image use integrity-only verification. + # Pin the publisher Ed25519 public key for signed network sync and embedded bundle verification. # Production: keep auto_promote false and promote manually after review. # auto_promote: false # sync: @@ -20,3 +19,4 @@ registries: # max_version: v2026.08.22 # allowed_channel_domains: # - anthropic.com + diff --git a/docs/README.md b/docs/README.md index db0604b8d..3082556f7 100644 --- a/docs/README.md +++ b/docs/README.md @@ -321,11 +321,11 @@ Optional hardening: `allowed_channel_domains` suffix-matches every registry conf `Registry::Sync.boot!` runs during app boot (see `app/web/boot/setup.rb`): -1. **Seed** — copies the image-embedded official bundle (144 feeds) when no on-disk bundle exists. -2. **Sync on boot** — when `REGISTRY_SYNC_ON_BOOT=true`, or when the bundle is missing, fetches and verifies the latest release. -3. **Background refresh** — in standalone single-process mode, `REGISTRY_SYNC_INTERVAL_HOURS` (default `24`) re-syncs on a periodic timer. Set to `0` when using the `registry-sync` Compose service. +1. **Instant index** — loads the embedded official bundle (140+ feeds) directly from `/app/registries/official` (or `REGISTRY_DATA_ROOT/official` if an updated synced bundle exists). Zero copying, zero startup network traffic. +2. **Sync on boot** — when `REGISTRY_SYNC_ON_BOOT=true`, triggers an asynchronous background sync. +3. **Background refresh** — in standalone single-process mode, `REGISTRY_SYNC_INTERVAL_HOURS` (default `24`) re-syncs on a periodic timer. Set to `0` when using the `registry-sync` Compose service or in static/offline environments. -Network sync verifies Ed25519 signatures using the `public_key` pinned in `registries.yml`. Seed bundles from the Docker image use integrity-only verification. +Network sync and embedded bundle verification verify Ed25519 signatures using the `public_key` pinned in `registries.yml`. Local `path:` mounts use integrity-only verification. ### Add a corporate registry diff --git a/docs/architecture.md b/docs/architecture.md index 19530d7c9..f819e0582 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -58,3 +58,23 @@ Strategies are defined by the `html2rss` gem but can be configured here. - **Botasaurus**: Used for JavaScript-heavy websites or anti-bot protected pages (`BOTASAURUS_SCRAPER_URL`). To add or configure strategies, see `app/web/feeds/source_resolver.rb` and the `html2rss` gem documentation. + +## Feed Registries & Storage Architecture + +Feed configs are dynamically resolved from curated registry bundles and local feeds via `Registry::Index`: + +```mermaid +flowchart TD + req["Request (/{feed_id}.rss or /api/v1/configs)"] --> index["Registry::Index.current"] + index --> store["Registry::Store.active_dir(id)"] + store --> synced{"Synced Bundle in data_root?"} + synced -- Yes --> data["/app/data/registries/"] + synced -- No --> embedded["/app/registries/ (Baked Image Bundle)"] + data --> bundle["Html2rss::Registry::Bundle.load"] + embedded --> bundle + bundle --> config["Feed Config"] +``` + +- **Baked Embedded Bundles:** Immutable image layers at `/app/registries/official` provide offline, instant-boot feeds without startup copying. +- **Runtime Synced Bundles:** Periodic synchronization writes verified bundles to `/app/data/registries/`, dynamically taking precedence over the baked image bundle. + diff --git a/docs/registry-go-live.md b/docs/registry-go-live.md index 612138bf0..de1e6d585 100644 --- a/docs/registry-go-live.md +++ b/docs/registry-go-live.md @@ -48,11 +48,11 @@ Source of truth for feed YAML and signed release artifacts. ### 1.3 `html2rss-web` (runtime + Docker image) -Consumes verified bundles; ships an unsigned seed copy in the image. +Consumes verified bundles; bakes the official release bundle into the Docker image at build time. 1. Bump the `html2rss` gem dependency if the core contract changed. 2. Update `config/registries.yml` when the signing key or sync channel changes (see sections 2 and 5). -3. Build the image with a fresh seed (section 3). +3. Build the image with the official release artifact baked (section 3). 4. Merge to `main`, wait for CI (`ci` workflow) to pass. 5. Publish a GitHub Release on `html2rss-web` — [`.github/workflows/release.yml`](../../html2rss-web/.github/workflows/release.yml) builds and pushes `html2rss/web` tags (`latest`, semver, major, commit SHA). @@ -119,7 +119,7 @@ registries: ``` - `public_key_id` must match the value in signed `manifest.json` (default in `tool/registry-build`: `html2rss:registry:2026`). -- Network sync uses `:signed` trust and requires this pin. Seed bundles copied from the Docker image use `:integrity_only` trust (no signature check on load). +- Network sync and embedded bundle loading use `:signed` trust and require this pin. Local `path:` mounts use `:integrity_only` trust (no signature check on load). - **`auto_promote: false`** (default) writes verified bundles to `REGISTRY_DATA_ROOT//.staging/` without changing the active catalog. Promote after review with `bin/html2rss-web registry promote --registry official`. - **`pin_version`** resolves the GitHub tag release API instead of `/releases/latest`. - **`max_version`** rejects sync when the verified manifest version is newer than the cap (incident freeze). @@ -162,7 +162,7 @@ mise exec -- bundle exec ruby -rhtml2rss -ropenssl -e " " ``` -Unsigned local builds (no `--sign`) are valid for seed/integrity-only use only: +Unsigned local builds (no `--sign`) are valid for integrity-only use only: ```bash cd html2rss-configs @@ -171,44 +171,31 @@ make registry-build # writes dist/registry-bundle.tar.gz without manifest.sig --- -## 3. Web image build — seed preparation and Docker +## 3. Web image build — artifact baking and Docker -### 3.1 How seed gets into the image +### 3.1 How the official registry gets into the image | Step | What happens | | --- | --- | -| `bin/prepare-registry-seed` | Builds an **unsigned** bundle from `html2rss-configs` and extracts it to `app/registries/seed/official/` | -| `bin/docker-build` | Runs `prepare-registry-seed`, then `docker build --no-cache -t html2rss/web` | -| `Dockerfile` | `COPY app/registries/seed ./app/registries/seed` (image path `/app/registries/seed`) | -| Boot | `Registry::Sync.boot!` copies seed → `REGISTRY_DATA_ROOT/` when no on-disk bundle exists | - -`prepare-registry-seed` details: - -- Configs source: `HTML2RSS_CONFIGS_ROOT` (default: sibling `../html2rss-configs`) -- Runs `tool/registry-build` **without** `--sign` (integrity-only seed) -- Output directory: `app/registries/seed/official/` +| `Dockerfile` (`registry-builder`) | Downloads the official release `registry-bundle.tar.gz` and extracts it to `/build/official/` | +| `Dockerfile` (Runtime) | `COPY --from=registry-builder /build/official /app/registries/official` | +| Boot | `Registry::Index.current` loads `/app/registries/official` directly. Zero copying, zero network calls. | ### 3.2 Build locally -From a workspace with both repos checked out as siblings: - -```bash -cd html2rss-web -bin/docker-build -``` - -Or prepare seed only: +To build a production image fetching the latest official release: ```bash cd html2rss-web -HTML2RSS_CONFIGS_ROOT=/path/to/html2rss-configs bin/prepare-registry-seed -docker build --no-cache -t html2rss/web -f Dockerfile . +docker build -t html2rss/web -f Dockerfile . ``` -Set build metadata for production parity: +To build pinning a specific configs release tag: ```bash docker build \ + --build-arg REGISTRY_RELEASE_TAG=v2026.08.22 \ + --build-arg REGISTRY_BUNDLE_URL=https://github.com/html2rss/html2rss-configs/releases/download/v2026.08.22/registry-bundle.tar.gz \ --build-arg BUILD_TAG=1.2.3 \ --build-arg GIT_SHA="$(git rev-parse HEAD)" \ -t html2rss/web \ @@ -217,9 +204,9 @@ docker build \ ### 3.3 What operators get in the image -- **`/app/registries/seed/official/`** — unsigned bundle baked at build time (offline-first bootstrap). -- **`/app/config/registries.yml`** — default official registry with `sync.channel: html2rss-official`. -- **`/app/data/registries`** — empty at build; populated at runtime (seed copy + network sync). +- **`/app/registries/official/`** — Pre-verified official release bundle baked at build time (immutable image layer). +- **`/app/config/registries.yml`** — Default official registry configuration with pinned public key. +- **`/app/data/registries`** — Empty at build; used at runtime for network synchronization. --- @@ -237,7 +224,7 @@ docker compose pull html2rss-web docker compose up -d html2rss-web ``` -A new image updates the **seed** inside the container. Existing data in the volume is kept until sync replaces it. +A new image updates the **embedded bundle** inside the container layers. ### 4.2 Zero-config first boot @@ -245,9 +232,9 @@ With the stock `config/registries.yml`, no extra registry env vars are required. On boot (`Registry::Sync.boot!`): -1. **Seed** — if `REGISTRY_DATA_ROOT/official/` is empty, copy from `/app/registries/seed/official/`. -2. **Sync on boot** — runs when `REGISTRY_SYNC_ON_BOOT=true` **or** when no bundle is present on disk (after seed attempt). -3. **Background refresh** — when `REGISTRY_SYNC_INTERVAL_HOURS` > 0 (default **24**), re-sync on a jittered timer. Set to `0` to disable. +1. **Instant index** — loads directly from `/app/registries/official` (or `REGISTRY_DATA_ROOT/official` if an updated synced bundle exists). +2. **Sync on boot** — runs only when `REGISTRY_SYNC_ON_BOOT=true`. +3. **Background refresh** — when `REGISTRY_SYNC_INTERVAL_HOURS` > 0 (default **24**), re-syncs on a periodic timer. Set to `0` to disable. Official sync URL (from `config/registries.yml` + `Registry::Config`): @@ -257,7 +244,7 @@ Allowed outbound hosts (built-in): `api.github.com`, `github.com`, `objects.gith ### 4.3 Existing instance — volume retained -The named volume preserves synced bundles across container restarts and image upgrades. After pull + restart: +The named volume preserves runtime-synced bundles across container restarts. When present and valid, `Store.active_dir` prefers the volume copy over the embedded image copy. - Old bundle stays active until a successful sync swaps it. - Use `bin/html2rss-web registry sync` or wait for background refresh to pick up a new configs release without rebuilding the image (section 7). @@ -466,7 +453,7 @@ By design: - On failure, `last_error` is recorded; the previous bundle under `REGISTRY_DATA_ROOT//` remains served. - `Store.promote_bundle!` rolls back on swap failure. -If sync fails on first boot with no prior bundle, the instance serves the embedded seed bundle (144 feeds) until a signed sync succeeds. +If sync fails on first boot with no prior bundle in the volume, the instance serves the embedded official bundle (140+ feeds) directly from the container image. ### 8.3 Network / host errors @@ -509,22 +496,10 @@ environment: ### 8.5 CLI exit code non-zero with empty `last_error` -`bin/html2rss-web registry status` exits **1** when a sync-mode registry has **no on-disk bundle**. Run a sync or confirm seed copy succeeded: +`bin/html2rss-web registry status` exits **1** when a sync-mode registry has **no usable bundle** (neither in the volume nor embedded in the image). Run a sync or verify the embedded bundle is present: ```bash bin/html2rss-web registry sync --registry official bin/html2rss-web registry status ``` -### 8.6 Seed missing or stale in image - -If `app/registries/seed/official/` was not populated before `docker build`, first boot may have nothing to seed. - -Fix for maintainers: - -```bash -cd html2rss-web -bin/prepare-registry-seed # or bin/docker-build -``` - -Ensure `HTML2RSS_CONFIGS_ROOT` points at the configs checkout used for the release. From 25ff6a1960ccc88273c74e41798b6abfa13da032 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Mon, 24 Aug 2026 15:08:49 +0200 Subject: [PATCH 31/33] feat(docker): verify official registry signature and digests at build time --- Dockerfile | 15 +++++++++++++-- Gemfile.lock | 2 +- docs/registry-go-live.md | 2 +- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 909549c64..fb0c73e38 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,18 +38,29 @@ RUN apk add --no-cache \ /usr/local/bundle/bundler/gems/*/.git \ /usr/local/bundle/cache/bundler/git -# Stage 3: Official Registry Artifact Builder +# Stage 3: Official Registry Artifact Builder & Verifier FROM ${RUBY_BASE_IMAGE} AS registry-builder ARG REGISTRY_BUNDLE_URL="https://github.com/html2rss/html2rss-configs/releases/latest/download/registry-bundle.tar.gz" WORKDIR /build +COPY --from=builder /usr/local/bundle /usr/local/bundle +COPY config/registries.yml ./config/registries.yml + # hadolint ignore=DL3018 RUN apk add --no-cache curl tar ca-certificates \ && curl -fsSL -o registry-bundle.tar.gz "${REGISTRY_BUNDLE_URL}" \ && mkdir -p /build/official \ - && tar -xzf registry-bundle.tar.gz -C /build/official + && tar -xzf registry-bundle.tar.gz -C /build/official \ + && ruby -r html2rss -r yaml -e ' \ + config = YAML.safe_load_file("config/registries.yml"); \ + official = config.fetch("registries").fetch("official"); \ + key_id = official.fetch("public_key_id"); \ + pub_key = OpenSSL::PKey.read(official.fetch("public_key")); \ + manifest = Html2rss::Registry::Verifier.verify!("/build/official", trust: :signed, public_keys: { key_id => pub_key }); \ + puts "Official registry bundle successfully verified at build time: version=#{manifest.version}, key=#{key_id}" \ + ' # Stage 4: Runtime FROM ${RUBY_BASE_IMAGE} diff --git a/Gemfile.lock b/Gemfile.lock index c5b41a897..a0492acf3 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/html2rss/html2rss - revision: e1fbff732b143274a7a41b5fd150c32ebffc38bb + revision: b8659ce8c6f0aeccd251abb88e6e581b8abd29fb branch: feat/registry-v1 specs: html2rss (0.28.0) diff --git a/docs/registry-go-live.md b/docs/registry-go-live.md index de1e6d585..cfa7dd835 100644 --- a/docs/registry-go-live.md +++ b/docs/registry-go-live.md @@ -177,7 +177,7 @@ make registry-build # writes dist/registry-bundle.tar.gz without manifest.sig | Step | What happens | | --- | --- | -| `Dockerfile` (`registry-builder`) | Downloads the official release `registry-bundle.tar.gz` and extracts it to `/build/official/` | +| `Dockerfile` (`registry-builder`) | Downloads the official release `registry-bundle.tar.gz`, extracts it, and verifies its Ed25519 signature & file digests against `config/registries.yml` | | `Dockerfile` (Runtime) | `COPY --from=registry-builder /build/official /app/registries/official` | | Boot | `Registry::Index.current` loads `/app/registries/official` directly. Zero copying, zero network calls. | From 14d0315ddfa6ec20b15af6ef868d72bd00b9bfa2 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Mon, 24 Aug 2026 15:11:52 +0200 Subject: [PATCH 32/33] feat(cli): add registry verify subcommand and invoke in Docker build --- Dockerfile | 15 +++++--------- app/web/cli.rb | 37 +++++++++++++++++++++++++++++++++-- spec/html2rss/web/cli_spec.rb | 20 +++++++++++++++++++ 3 files changed, 60 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index fb0c73e38..0f16d5a53 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,24 +43,19 @@ FROM ${RUBY_BASE_IMAGE} AS registry-builder ARG REGISTRY_BUNDLE_URL="https://github.com/html2rss/html2rss-configs/releases/latest/download/registry-bundle.tar.gz" -WORKDIR /build +WORKDIR /app COPY --from=builder /usr/local/bundle /usr/local/bundle -COPY config/registries.yml ./config/registries.yml +COPY bin ./bin +COPY app ./app +COPY config ./config # hadolint ignore=DL3018 RUN apk add --no-cache curl tar ca-certificates \ && curl -fsSL -o registry-bundle.tar.gz "${REGISTRY_BUNDLE_URL}" \ && mkdir -p /build/official \ && tar -xzf registry-bundle.tar.gz -C /build/official \ - && ruby -r html2rss -r yaml -e ' \ - config = YAML.safe_load_file("config/registries.yml"); \ - official = config.fetch("registries").fetch("official"); \ - key_id = official.fetch("public_key_id"); \ - pub_key = OpenSSL::PKey.read(official.fetch("public_key")); \ - manifest = Html2rss::Registry::Verifier.verify!("/build/official", trust: :signed, public_keys: { key_id => pub_key }); \ - puts "Official registry bundle successfully verified at build time: version=#{manifest.version}, key=#{key_id}" \ - ' + && bin/html2rss-web registry verify --registry official --dir /build/official # Stage 4: Runtime FROM ${RUBY_BASE_IMAGE} diff --git a/app/web/cli.rb b/app/web/cli.rb index 9d7ae325d..bca687f21 100644 --- a/app/web/cli.rb +++ b/app/web/cli.rb @@ -44,6 +44,7 @@ def run_registry(args, out:, err:) # rubocop:disable Metrics/CyclomaticComplexit when 'status' then registry_status(args[1..] || [], out:) when 'sync' then registry_sync(args[1..] || []) when 'promote' then registry_promote(args[1..] || []) + when 'verify' then registry_verify(args[1..] || [], out:, err:) when '-h', '--help', 'help', nil print_registry_help(out:) 0 @@ -101,6 +102,36 @@ def registry_promote(args) Registry::Sync.cli_exit_code end + ## + # @param args [Array] + # @param out [IO] + # @param err [IO] + # @return [Integer] + def registry_verify(args, out:, err:) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity + options = { registry_id: 'official', dir: nil } + OptionParser.new do |opts| + opts.banner = 'Usage: html2rss-web registry verify [options]' + opts.on('--registry ID', 'Registry ID from config (default: official)') { options[:registry_id] = it } + opts.on('--dir PATH', 'Directory containing bundle to verify') { options[:dir] = it } + end.parse!(args) + + registry_id = options[:registry_id] + definition = Registry::Config.entry(registry_id) + raise Registry::Errors::ConfigError, "Unknown registry '#{registry_id}'" unless definition + + dir = options[:dir] || Registry::Store.active_dir(registry_id) || Registry::Store.embedded_dir(registry_id) + raise Registry::Errors::LoadError, "Bundle directory not found: #{dir}" unless dir && File.directory?(dir) + + trust = definition.mode == :path ? :integrity_only : :signed + public_keys = definition.mode == :path ? {} : { definition.public_key_id => definition.public_key }.compact + manifest = Html2rss::Registry::Verifier.verify!(dir, trust:, public_keys:) + out.puts "Verified registry bundle '#{registry_id}' (#{manifest.version}) at #{dir}" + 0 + rescue StandardError => error + err.puts "Registry verification failed: #{error.message}" + 1 + end + ## # @param out [IO] # @param err [IO] @@ -140,6 +171,7 @@ def print_root_help(out:) registry status [--registry ID] Show status of configured registries registry sync [--registry ID] [--dry-run] Fetch, verify, and sync registry bundles registry promote [--registry ID] Promote verified staged bundle to active + registry verify [--registry ID] [--dir DIR] Verify registry bundle signature and integrity healthcheck Verify container process liveness version Print version and runtime information HELP @@ -156,6 +188,7 @@ def print_registry_help(out:) status Show registry sync status table sync Fetch and verify remote registry bundle(s) promote Promote staged verified bundle to active + verify Verify bundle signature and file integrity against pinned configuration HELP end @@ -194,8 +227,8 @@ def print_status_table(registry_id, out:) # rubocop:disable Metrics/MethodLength def format_time(value) = value ? value.utc.iso8601 : '-' private_class_method :run_registry, :registry_status, :registry_sync, :registry_promote, - :run_healthcheck, :run_version, :print_root_help, :print_registry_help, - :target_registry_ids, :print_status_table, :format_time + :registry_verify, :run_healthcheck, :run_version, :print_root_help, + :print_registry_help, :target_registry_ids, :print_status_table, :format_time end end end diff --git a/spec/html2rss/web/cli_spec.rb b/spec/html2rss/web/cli_spec.rb index 3dafc2ca4..094e1fd45 100644 --- a/spec/html2rss/web/cli_spec.rb +++ b/spec/html2rss/web/cli_spec.rb @@ -95,6 +95,26 @@ expect(Html2rss::Web::Registry::Sync).to have_received(:promote_staged!).with(registry_id: 'official') end + it 'verifies valid registry bundles', :aggregate_failures do + exit_code = described_class.run( + ['registry', 'verify', '--registry', 'official', '--dir', 'spec/fixtures/registries/official'], + out: stdout, + err: stderr + ) + expect(exit_code).to eq(0) + expect(stdout.string).to include("Verified registry bundle 'official' (test-fixture)") + end + + it 'reports failure on invalid registry verification', :aggregate_failures do + exit_code = described_class.run( + ['registry', 'verify', '--registry', 'official', '--dir', 'tmp/non-existent'], + out: stdout, + err: stderr + ) + expect(exit_code).to eq(1) + expect(stderr.string).to include('Registry verification failed') + end + it 'handles unknown registry subcommands', :aggregate_failures do exit_code = described_class.run(%w[registry invalid-sub], out: stdout, err: stderr) expect(exit_code).to eq(1) From 7f2e3019569488e61af43ff7d32578cb0ab38d34 Mon Sep 17 00:00:00 2001 From: Gil Desmarais Date: Mon, 24 Aug 2026 18:26:15 +0200 Subject: [PATCH 33/33] chore(bundle): update html2rss git revision in Gemfile.lock --- Gemfile.lock | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index a0492acf3..9f4a6eb49 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/html2rss/html2rss - revision: b8659ce8c6f0aeccd251abb88e6e581b8abd29fb + revision: 5044272b9dcf404a1ef550e0496508cd6ce5687e branch: feat/registry-v1 specs: html2rss (0.28.0) @@ -375,6 +375,7 @@ CHECKSUMS faraday-net_http (3.4.4) sha256=0e78af151747ed1b00f33e25973b4bc220d7f16c00c39676817c8b12331eb588 hana (1.3.7) sha256=5425db42d651fea08859811c29d20446f16af196308162894db208cac5ce9b0d hashdiff (1.2.1) sha256=9c079dbc513dfc8833ab59c0c2d8f230fa28499cc5efb4b8dd276cf931457cd1 + html2rss (0.28.0) i18n (1.15.2) sha256=00f9eb62412fe593b2a65a97daa75300d37abb8f7202ec748e94b6d46a9dd1b5 io-console (0.9.2) sha256=efa74f891dd03c0939a931dfc6e74c2813d904763d456ea9762b0525e748db08 irb (1.18.0) sha256=de9454a0703a54704b9811a5ef31a60c86949fbf4013fcf244fabc7c775248e3