fix(registry): publish both Starlink plugins in the GitHub catalog

k8s/README.md

@@ -22,7 +22,8 @@ The application expects two credential sources at deploy time:
 Required keys for `developer-portal-secrets`:
 
 - `secret_key_base`
-- `github_token` (optional; raises GitHub API rate limits for the plugin registry)
+- `github_token` (authenticated GitHub access for regular catalog refreshes;
+  anonymous API limits are too low for the full plugin and add-on scan)
 
 A non-applied example is provided in [secret-template.yaml](/home/mfreeman/src/community/k8s/base/secret-template.yaml).
 A non-applied DB bootstrap example is provided in [db-credentials-template.yaml](/home/mfreeman/src/community/k8s/base/db-credentials-template.yaml).
@@ -50,5 +51,5 @@ Required non-secret runtime config comes from `developer-portal-config`:
 - `PGHOST`
 - `PGPORT`
 - `PGDATABASE`
-- `FORGEJO_API_BASE_URL` / `FORGEJO_CONTENT_BASE_URL` (in-cluster fetch)
-- `GITHUB_API_BASE_URL` (optional override; user-facing plugin links are rewritten to GitHub either way)
+- `GITHUB_API_BASE_URL` (`https://api.github.com`; manifests and release indexes
+  are fetched from GitHub)

k8s/base/configmap.yaml

@@ -10,5 +10,4 @@ data:
   PGHOST: developer-portal-pg-rw.serviceradar-developer.svc.cluster.local
   PGPORT: "5432"
   PGDATABASE: developer_portal
-  FORGEJO_API_BASE_URL: http://forgejo-http.forgejo.svc.cluster.local:3000/api/v1
-  FORGEJO_CONTENT_BASE_URL: http://forgejo-http.forgejo.svc.cluster.local:3000
+  GITHUB_API_BASE_URL: https://api.github.com

k8s/prod/kustomization.yaml

@@ -28,4 +28,4 @@ replicas:
 
 images:
   - name: registry.carverauto.dev/serviceradar/developer-portal
-    newTag: sha-ee09e056075f
+    newTag: sha-b4dbcf86c5a8

k8s/prod/secrets.yaml

@@ -5,6 +5,7 @@ metadata:
   namespace: serviceradar-developer
 spec:
   encryptedData:
+    github_token: AgDI7naMtX/sTNuZHkrxPdwa9IaM38EDO0FDztmgFyGnGC+350HpvhebVs8/64Fb69OrNscS0fMnoiyTEJAoQlR0/Bc3C25IceCUWmaeeX4jsLjWwR75z6ylry8pafP5tPugETmuX8Qcah8pgTgH4UyuRflxgReyYa0yLpO6J7d8ysboSmthpPbSpYfXelfNYoAavGqI/2eD+wGiopxTVwRzetT5fa6E0O/NAFRfqV0AGrLNizaKarz3zBMal05UcK6YcOkc7/0ztbRodChjsUFBcioT+cdmT36LSdRI0dIy6aqhMEYn6vBOvPtul7Bnttd92NJLVa4pxsizLfcr8Z9EOy+/I6nhAABaV+wfRPPlf5sj9u3e//z4hm5P0Jh1eF5zqdoQd2rq5aPf4I39a2hOseQhqgM/3/5lnAeWBAG5bsvLfPmdnHUp2q7h3QWpL16VFGfGev7lRFdhVVfeObKX5aj015VaSCz/ZBiz6pX4NH/vgHH0Yveq3ewoeE7VltPiaI7nfYY1mzEggn/2uQ4aDbYxGmAxFGWfWCf9Fu0cqtFsBZ8ZMQf/mM1k8MgmL85z6igCVKaZYZENcybiIbeP0O134537kkaeHM+IvOoYCEhaTEw2mLutmqGJA3a3GjWDYEV+iJZbpv4nAUsQVt7nAMIbtQGvek9IGSb6Ox2I/79N4LLdVvi03uYkDRL2RoGQwHfnrlbI9V4agee6KEXR7RWlaH3zUMYjFBApyzCg4wVoM8uSerbO
     secret_key_base: AgDGLBofWsa8VYBdeUuwzLi0zuXBjwbclxrCKhuptNqj812ahc/wP87b/toKS3TME8dfLIAaYu/p8OK5PU+697wmnsq0XZgtctMbApW1wnF5Z7sGVFfvQd5Iy4hedUnKPi/1lQh4sQ+iknD8eppGWeAOODgJYRAw74zM8x7r458u25KanSxLIl3deA6FmcZk9scZQyn9CAm3uetKQ7tRw4TY0m9CoDSepFgGHrX1rwur2W5nnmn2VznegBgyLIpIBgqEs48AQltNGbveuyDTD9GWtLJrOpKMGhpCg4/pkqn2gutiwXGl8HEEk9waP5vakb71u/0lDuHm9qJnUiOuKlM33lwmySUvbuU5aJ/UdRa3k1kiEU3uJExX8e2QaA8TzSnkytpkI7kYrvN+xovGG+XZQPpU1KXzGnmTsiDh1yf2qOfPe80MoWBE2MvNs693xwnlWJ/VmAKDmgkfqEYFGEafkbCFSR1+Leyfu2OrHXzYjmVfMQzxHeT76NoDSdwelGxP5MDQihNQGD2wmGjk9pXzXJg6V+gobhlNyxj6LhsN2fR7ADGMiH5aHZRVvygsDVIYTB+LeDa+BpNytX2MGYIiZkAYP42hR1W9KBnS6aOSV3wVui27RIE4XFWgknFbKsk9rzWVh08mzEGgGiaR+VvyIU7hVL8bjuS/MlHrhJJwVpDJIHRty+uV3mPqh+Txy8bcUa0l3J8Dm+wrWcBCy0sX1RI69lGFrGXIcRbhs582/ruFF3K3YXvEZI5MO0iJnkMQCi+YYWnQH7ECQ/kOBJCg9RfSNoa3T2zt1chJ1jBAf2tYhObqUEV+iOgVJflqYwHoQdDex9tuve75lEC+cKlSwiJ547jawX/FTQaSKPFgAQ==
   template:
     metadata:

k8s/staging/secrets.yaml

@@ -5,6 +5,7 @@ metadata:
   namespace: serviceradar-developer-staging
 spec:
   encryptedData:
+    github_token: AgCCNjSDW1pAjjocQsLbVBvRpm2jc1Jlj9q9BVlIriNhs3QGtwUIlIUTvNAA3Gjq5hUnIcM0F52Y2QWXu94r0191U7Qr7WgC/+BAzIZoEEXYPg07Jh9YlMoFVIxvN82K/P276nGMY+DQOsqF7Y6IhpUAjH6iztuREg/oVp/eBM1BImgP2CDF0HWhvvvmwow3mBqw5haOXKkDob0AOILCKOR1m3u6O6UHB7CdMAtVRHabcg6/6N7nKZljgJCDjgBhUvi5FjBXvAQ4C6JNo3rrzqt0FnDi9smZ6Q2CcQpJRYgmgbQuxZln8vhDwXPJhGNZT1KygCSXXrjvrbQqnaychR2fhU55qEIgAmCnDmyr4de6nVf0A7IHlIdcN6B3O2DOHCA/83dq78U1IvJQ6r87bgv1JBo61k8txEo6srOZ8LtPIRlYkFd523wbVLLt++ZsbhXzp4N+DuDTLwQYkJj3N8+bIWSPi0uDZvY4xmcSzB1EZIU2Qb3ID+DA4rfDJzVrCR8HwX4WEaipayuzdIGkH0TE1ZW3ZjX1IUg/pJYL5mdIQHLptHv14wGFtCtULQnNDqxUH2jzDoyOeCk5OEE9G8l0GxQpp/otmKjTpGX7E4nIvQOHvVIFA0QWibQ5SCMzdgh25CchI9mFukw56T6tPZb3HuxM+VZ5+lsGMI9jTbCWQYAqiAFdJM39XWT0Y/8bmL0SLVIdZVWSvdAiDARtxNSgAy6tnJA2ielmLLmCv7geTrFcVhIPFIGQ
     secret_key_base: AgBKdKBisc1+dFyZeqSzPzkHPTUs36AtOBSj6bp0W2h0vDl6TxChoZfa3aLH04Ma6V2WIQcpm0DQOdcCgQbq7llcjmZHvmnCeOCcyiTjhVI6xQ95EVRYQD81Yc5bCvXJQ/suAiv7yw+gff1h4VVgJUVpCSASqTKfiKch8IM4nuKNrrEagczqCq+BPGWtN3W/kF0T4q8kdDVliTDc7Ute8liUtRgkVIfch45mxpAKxBSoCdULWi5T2l5dG09msr8M8bMzF3FEMLuTVlQ0/pwge+fc2h8hesn04sWWzuZZVhsfSUyCw7YCoYOga1cJT3SSf6d00YfeN9pFFqJ2jf4mpkdXgVs4/Zr/m9VMs1BEbAHB2GjqSd4ui4VpU+TjhfAC501FQwGmKoe82krpRTM7ZGk1u7BhA5LLgdmbIuG+urRp8lClQo2tM7WW5U16miKYMBiOZmBYBuAFz8Kbodr9o9Vr1xh//XjQi9M1SjfV61UTI2C8JzUFv39pbG2soYZLJDkJ3UafMboo6a+OguUz6jCfQV8RXy+tKQtSxENOD2/xMIHXi/2lq7Q5kq20U3L5k67hDLHhmub17zY1Miwy90ItwZmX8IcJzQWKPxLxs9T4dRaimn3p1Kwu7BIT/Cne7oPtDHcZy90fvozDtBAeloxYwmwRvNt7ynZmdKcTiZAM7fLr35gPD1tTlrSLLwSPHRw6eunkA2rBiPaoJYLNNRA/pFT9bkd2G1pfjrtJHniaKfo+fEJlQjGZ4jTaCWU0ONK2dud9DtJuniStBz54UilgIGAwJOFXe/3W1UuyWZWeFEOI4MLmSOdbEbZh1upl1X3A/Y0Q0z7NV/EElZR/RFJLm8sa8Hzpp5e6piEJlMEJnw==
   template:
     metadata:

lib/developer_portal/registry/forgejo_source.ex

@@ -13,10 +13,12 @@ defmodule DeveloperPortal.Registry.ForgejoSource do
 
   @manifest_files [
     %{name: "plugin.yaml", kind: "check"},
+    %{name: "plugin.local.yaml", kind: "check", schema_variant: "local"},
     %{name: "plugin.console.yaml", kind: "console"},
     %{name: "plugin.inventory_sync.yaml", kind: "inventory_sync"},
     %{name: "plugin.stream.yaml", kind: "stream"},
     %{name: "manifest.json", kind: "check"},
+    %{name: "manifest.local.json", kind: "check", schema_variant: "local"},
     %{name: "manifest.console.json", kind: "console"},
     %{name: "manifest.inventory_sync.json", kind: "inventory_sync"},
     %{name: "manifest.stream.json", kind: "stream"}
@@ -64,7 +66,7 @@ defmodule DeveloperPortal.Registry.ForgejoSource do
 
             manifest_entry ->
               [
-                build_plugin(config, manifest_entry, manifest.kind, language, index, %{
+                build_plugin(config, manifest_entry, manifest, language, index, %{
                   source_url: source_url,
                   readme_url: readme_url,
                   directory: directory,
@@ -83,7 +85,7 @@ defmodule DeveloperPortal.Registry.ForgejoSource do
       []
   end
 
-  defp build_plugin(config, manifest_entry, kind, language, index, urls) do
+  defp build_plugin(config, manifest_entry, %{kind: kind} = variant, language, index, urls) do
     manifest = fetch_manifest!(config, manifest_entry)
     slug = fetch_id!(manifest)
     release = Map.get(index, slug)
@@ -94,6 +96,7 @@ defmodule DeveloperPortal.Registry.ForgejoSource do
     # can never crash the whole refresh.
     published = if signed, do: release, else: nil
     %{source_url: source_url, readme_url: readme_url, directory: directory, dist: dist} = urls
+    schema_variant = Map.get(variant, :schema_variant, kind)
 
     %Plugin{
       slug: slug,
@@ -110,7 +113,7 @@ defmodule DeveloperPortal.Registry.ForgejoSource do
       source_url: PublicURL.githubize(source_url),
       readme_url: PublicURL.githubize(readme_url),
       manifest_url: PublicURL.githubize(download_url(manifest_entry)),
-      config_schema_url: PublicURL.githubize(config_schema_url(kind, directory, dist)),
+      config_schema_url: PublicURL.githubize(config_schema_url(schema_variant, directory, dist)),
       wasm_url: PublicURL.githubize(find_download_url(dist, @wasm_candidates)),
       artifact_url: PublicURL.githubize(find_download_url(dist, @checksum_candidates)),
       signature_url: PublicURL.githubize(find_download_url(dist, @signature_candidates)),
@@ -338,6 +341,7 @@ defmodule DeveloperPortal.Registry.ForgejoSource do
   end
 
   defp schema_candidates("stream"), do: ["config.stream.schema.json"]
+  defp schema_candidates("local"), do: ["config.local.schema.json"]
   defp schema_candidates("console"), do: ["config.console.schema.json", "config.schema.json"]
 
   defp schema_candidates("inventory_sync"),

mix.lock

@@ -32,7 +32,6 @@
   "iterex": {:hex, :iterex, "0.1.2", "58f9b9b9a22a55cbfc7b5234a9c9c63eaac26d276b3db80936c0e1c60355a5a6", [:mix], [], "hexpm", "2e103b8bcc81757a9af121f6dc0df312c9a17220f302b1193ef720460d03029d"},
   "jason": {:hex, :jason, "1.4.5", "2e3a008590b0b8d7388c20293e9dcc9cf3e5d642fd2a114e4cbbb52e595d940a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "b0c823996102bcd0239b3c2444eb00409b72f6a140c1950bc8b457d836b30684"},
   "lazy_html": {:hex, :lazy_html, "0.1.12", "31a55ee622918fce988c94b06232227b42daa64e4eab14ac32081d0f3fd8db6f", [:make, :mix], [{:cc_precompiler, "~> 0.1", [hex: :cc_precompiler, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.9", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:fine, "~> 0.1.0", [hex: :fine, repo: "hexpm", optional: false]}], "hexpm", "8a0da594776caee58782c6f93b2abaa5bdb809daf8d43351a561f7de9dc2e2a8"},
-  "libgraph": {:hex, :libgraph, "0.16.0", "3936f3eca6ef826e08880230f806bfea13193e49bf153f93edcf0239d4fd1d07", [:mix], [], "hexpm", "41ca92240e8a4138c30a7e06466acc709b0cbb795c643e9e17174a178982d6bf"},
   "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"},
   "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"},
   "makeup_json": {:hex, :makeup_json, "1.0.0", "03886f244d9db4179a9ecfffde5e711480e04e16ef91ca484d2b51ca3251a18f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "5c8c559e658c7f7e91b96c4b8c40f5912ea0adff44b7afe73e4639d9c3f53b94"},
@@ -72,7 +71,6 @@
   "telemetry_metrics": {:hex, :telemetry_metrics, "1.1.0", "5bd5f3b5637e0abea0426b947e3ce5dd304f8b3bc6617039e2b5a008adc02f8f", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e7b79e8ddfde70adb6db8a6623d1778ec66401f366e9a8f5dd0955c56bc8ce67"},
   "telemetry_poller": {:hex, :telemetry_poller, "1.3.0", "d5c46420126b5ac2d72bc6580fb4f537d35e851cc0f8dbd571acf6d6e10f5ec7", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "51f18bed7128544a50f75897db9974436ea9bfba560420b646af27a9a9b35211"},
   "thousand_island": {:hex, :thousand_island, "1.5.0", "f50a213cac97262b6d5ebb85745aa2c00fec1413191e6e66834788d45425cecb", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "708923d40523e43cf99041ab37a0d4b0ec426ac6438fa3716ab23d919eaeb412"},
-  "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.1", "a48703a25c170eedadca83b11e88985af08d35f37c6f664d6dcfb106a97782fc", [:rebar3], [], "hexpm", "b3a917854ce3ae233619744ad1e0102e05673136776fb2fa76234f3e03b23642"},
   "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"},
   "websock_adapter": {:hex, :websock_adapter, "0.6.0", "73db5ab8aaefd1a876a97ce3e6afc96562625de69ef17a4e04426e034849d0b8", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "50021a85bce8f203b086705d9e0c5415e2c7eb05d319111b0428fe71f9934617"},
   "yamerl": {:hex, :yamerl, "0.10.0", "4ff81fee2f1f6a46f1700c0d880b24d193ddb74bd14ef42cb0bcf46e81ef2f8e", [:rebar3], [], "hexpm", "346adb2963f1051dc837a2364e4acf6eb7d80097c0f53cbdc3046ec8ec4b4e6e"},

test/developer_portal/registry/forgejo_source_test.exs

@@ -0,0 +1,83 @@
+defmodule DeveloperPortal.Registry.ForgejoSourceTest do
+  use ExUnit.Case, async: true
+
+  alias DeveloperPortal.Registry.ForgejoSource
+
+  for {cloud_file, local_file} <- [
+        {"plugin.yaml", "plugin.local.yaml"},
+        {"manifest.json", "manifest.local.json"}
+      ] do
+    test "discovers #{local_file} alongside #{cloud_file} with its own configuration schema" do
+      cloud_file = unquote(cloud_file)
+      local_file = unquote(local_file)
+
+      Req.Test.stub(__MODULE__, fn conn ->
+        case conn.request_path do
+          "/repos/example/catalog/contents/plugins" ->
+            Req.Test.json(conn, [entry("weather", "dir")])
+
+          "/repos/example/catalog/contents/plugins/weather" ->
+            files = [cloud_file, local_file, "config.schema.json", "config.local.schema.json"]
+            Req.Test.json(conn, Enum.map(files, &entry("weather/" <> &1, "file")))
+
+          "/repos/example/catalog/releases" ->
+            Req.Test.json(conn, [])
+
+          "/raw/plugins/weather/" <> file ->
+            id = if file == cloud_file, do: "weather-cloud", else: "weather-local"
+
+            conn
+            |> Plug.Conn.put_resp_content_type("text/plain")
+            |> Plug.Conn.send_resp(200, manifest(file, id))
+        end
+      end)
+
+      assert {:ok, plugins} =
+               ForgejoSource.fetch_plugins(
+                 api_base_url: "https://catalog.example.test",
+                 content_base_url: nil,
+                 owner: "example",
+                 repo: "catalog",
+                 ref: "main",
+                 plugin_root: "plugins",
+                 req_options: [plug: {Req.Test, __MODULE__}, retry: false]
+               )
+
+      assert [%{slug: "weather-cloud"} = cloud, %{slug: "weather-local"} = local] = plugins
+      assert cloud.sample_kind == "check"
+      assert local.sample_kind == "check"
+      assert String.ends_with?(cloud.manifest_url, "/" <> cloud_file)
+      assert String.ends_with?(local.manifest_url, "/" <> local_file)
+      assert cloud.config_schema_url == raw_url("weather/config.schema.json")
+      assert local.config_schema_url == raw_url("weather/config.local.schema.json")
+    end
+  end
+
+  defp entry(path, type) do
+    %{
+      "name" => Path.basename(path),
+      "path" => "plugins/" <> path,
+      "type" => type,
+      "html_url" => "https://catalog.example.test/source/plugins/" <> path,
+      "download_url" => raw_url(path)
+    }
+  end
+
+  defp raw_url(path), do: "https://catalog.example.test/raw/plugins/" <> path
+
+  defp manifest(file, id) do
+    attrs = %{
+      "id" => id,
+      "name" => id,
+      "version" => "0.0.1",
+      "description" => "Synthetic weather monitoring plugin",
+      "outputs" => "plugin_result"
+    }
+
+    if Path.extname(file) == ".json" do
+      Jason.encode!(attrs)
+    else
+      Enum.map_join(attrs, "\n", fn {key, value} -> "#{key}: #{value}" end)
+    end
+  end
+end