diff --git a/k8s/README.md b/k8s/README.md index 7ae4251..39b884c 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -22,7 +22,8 @@ The application expects two credential sources at deploy time: Required keys for `developer-portal-secrets`: - `secret_key_base` -- `github_token` (optional; raises GitHub API rate limits for the plugin registry) +- `github_token` (authenticated GitHub access for regular catalog refreshes; + anonymous API limits are too low for the full plugin and add-on scan) A non-applied example is provided in [secret-template.yaml](/home/mfreeman/src/community/k8s/base/secret-template.yaml). A non-applied DB bootstrap example is provided in [db-credentials-template.yaml](/home/mfreeman/src/community/k8s/base/db-credentials-template.yaml). @@ -50,5 +51,5 @@ Required non-secret runtime config comes from `developer-portal-config`: - `PGHOST` - `PGPORT` - `PGDATABASE` -- `FORGEJO_API_BASE_URL` / `FORGEJO_CONTENT_BASE_URL` (in-cluster fetch) -- `GITHUB_API_BASE_URL` (optional override; user-facing plugin links are rewritten to GitHub either way) +- `GITHUB_API_BASE_URL` (`https://api.github.com`; manifests and release indexes + are fetched from GitHub) diff --git a/k8s/base/configmap.yaml b/k8s/base/configmap.yaml index 8d0d803..4a684aa 100644 --- a/k8s/base/configmap.yaml +++ b/k8s/base/configmap.yaml @@ -10,5 +10,4 @@ data: PGHOST: developer-portal-pg-rw.serviceradar-developer.svc.cluster.local PGPORT: "5432" PGDATABASE: developer_portal - FORGEJO_API_BASE_URL: http://forgejo-http.forgejo.svc.cluster.local:3000/api/v1 - FORGEJO_CONTENT_BASE_URL: http://forgejo-http.forgejo.svc.cluster.local:3000 + GITHUB_API_BASE_URL: https://api.github.com diff --git a/k8s/prod/kustomization.yaml b/k8s/prod/kustomization.yaml index e071f2c..eccc4a0 100644 --- a/k8s/prod/kustomization.yaml +++ b/k8s/prod/kustomization.yaml @@ -28,4 +28,4 @@ replicas: images: - name: registry.carverauto.dev/serviceradar/developer-portal - newTag: sha-ee09e056075f + newTag: sha-b4dbcf86c5a8 diff --git a/k8s/prod/secrets.yaml b/k8s/prod/secrets.yaml index 25e14f7..9df1623 100644 --- a/k8s/prod/secrets.yaml +++ b/k8s/prod/secrets.yaml @@ -5,6 +5,7 @@ metadata: namespace: serviceradar-developer spec: encryptedData: + github_token: AgDI7naMtX/sTNuZHkrxPdwa9IaM38EDO0FDztmgFyGnGC+350HpvhebVs8/64Fb69OrNscS0fMnoiyTEJAoQlR0/Bc3C25IceCUWmaeeX4jsLjWwR75z6ylry8pafP5tPugETmuX8Qcah8pgTgH4UyuRflxgReyYa0yLpO6J7d8ysboSmthpPbSpYfXelfNYoAavGqI/2eD+wGiopxTVwRzetT5fa6E0O/NAFRfqV0AGrLNizaKarz3zBMal05UcK6YcOkc7/0ztbRodChjsUFBcioT+cdmT36LSdRI0dIy6aqhMEYn6vBOvPtul7Bnttd92NJLVa4pxsizLfcr8Z9EOy+/I6nhAABaV+wfRPPlf5sj9u3e//z4hm5P0Jh1eF5zqdoQd2rq5aPf4I39a2hOseQhqgM/3/5lnAeWBAG5bsvLfPmdnHUp2q7h3QWpL16VFGfGev7lRFdhVVfeObKX5aj015VaSCz/ZBiz6pX4NH/vgHH0Yveq3ewoeE7VltPiaI7nfYY1mzEggn/2uQ4aDbYxGmAxFGWfWCf9Fu0cqtFsBZ8ZMQf/mM1k8MgmL85z6igCVKaZYZENcybiIbeP0O134537kkaeHM+IvOoYCEhaTEw2mLutmqGJA3a3GjWDYEV+iJZbpv4nAUsQVt7nAMIbtQGvek9IGSb6Ox2I/79N4LLdVvi03uYkDRL2RoGQwHfnrlbI9V4agee6KEXR7RWlaH3zUMYjFBApyzCg4wVoM8uSerbO secret_key_base: AgDGLBofWsa8VYBdeUuwzLi0zuXBjwbclxrCKhuptNqj812ahc/wP87b/toKS3TME8dfLIAaYu/p8OK5PU+697wmnsq0XZgtctMbApW1wnF5Z7sGVFfvQd5Iy4hedUnKPi/1lQh4sQ+iknD8eppGWeAOODgJYRAw74zM8x7r458u25KanSxLIl3deA6FmcZk9scZQyn9CAm3uetKQ7tRw4TY0m9CoDSepFgGHrX1rwur2W5nnmn2VznegBgyLIpIBgqEs48AQltNGbveuyDTD9GWtLJrOpKMGhpCg4/pkqn2gutiwXGl8HEEk9waP5vakb71u/0lDuHm9qJnUiOuKlM33lwmySUvbuU5aJ/UdRa3k1kiEU3uJExX8e2QaA8TzSnkytpkI7kYrvN+xovGG+XZQPpU1KXzGnmTsiDh1yf2qOfPe80MoWBE2MvNs693xwnlWJ/VmAKDmgkfqEYFGEafkbCFSR1+Leyfu2OrHXzYjmVfMQzxHeT76NoDSdwelGxP5MDQihNQGD2wmGjk9pXzXJg6V+gobhlNyxj6LhsN2fR7ADGMiH5aHZRVvygsDVIYTB+LeDa+BpNytX2MGYIiZkAYP42hR1W9KBnS6aOSV3wVui27RIE4XFWgknFbKsk9rzWVh08mzEGgGiaR+VvyIU7hVL8bjuS/MlHrhJJwVpDJIHRty+uV3mPqh+Txy8bcUa0l3J8Dm+wrWcBCy0sX1RI69lGFrGXIcRbhs582/ruFF3K3YXvEZI5MO0iJnkMQCi+YYWnQH7ECQ/kOBJCg9RfSNoa3T2zt1chJ1jBAf2tYhObqUEV+iOgVJflqYwHoQdDex9tuve75lEC+cKlSwiJ547jawX/FTQaSKPFgAQ== template: metadata: diff --git a/k8s/staging/secrets.yaml b/k8s/staging/secrets.yaml index 078812e..e48f4ce 100644 --- a/k8s/staging/secrets.yaml +++ b/k8s/staging/secrets.yaml @@ -5,6 +5,7 @@ metadata: namespace: serviceradar-developer-staging spec: encryptedData: + github_token: AgCCNjSDW1pAjjocQsLbVBvRpm2jc1Jlj9q9BVlIriNhs3QGtwUIlIUTvNAA3Gjq5hUnIcM0F52Y2QWXu94r0191U7Qr7WgC/+BAzIZoEEXYPg07Jh9YlMoFVIxvN82K/P276nGMY+DQOsqF7Y6IhpUAjH6iztuREg/oVp/eBM1BImgP2CDF0HWhvvvmwow3mBqw5haOXKkDob0AOILCKOR1m3u6O6UHB7CdMAtVRHabcg6/6N7nKZljgJCDjgBhUvi5FjBXvAQ4C6JNo3rrzqt0FnDi9smZ6Q2CcQpJRYgmgbQuxZln8vhDwXPJhGNZT1KygCSXXrjvrbQqnaychR2fhU55qEIgAmCnDmyr4de6nVf0A7IHlIdcN6B3O2DOHCA/83dq78U1IvJQ6r87bgv1JBo61k8txEo6srOZ8LtPIRlYkFd523wbVLLt++ZsbhXzp4N+DuDTLwQYkJj3N8+bIWSPi0uDZvY4xmcSzB1EZIU2Qb3ID+DA4rfDJzVrCR8HwX4WEaipayuzdIGkH0TE1ZW3ZjX1IUg/pJYL5mdIQHLptHv14wGFtCtULQnNDqxUH2jzDoyOeCk5OEE9G8l0GxQpp/otmKjTpGX7E4nIvQOHvVIFA0QWibQ5SCMzdgh25CchI9mFukw56T6tPZb3HuxM+VZ5+lsGMI9jTbCWQYAqiAFdJM39XWT0Y/8bmL0SLVIdZVWSvdAiDARtxNSgAy6tnJA2ielmLLmCv7geTrFcVhIPFIGQ secret_key_base: AgBKdKBisc1+dFyZeqSzPzkHPTUs36AtOBSj6bp0W2h0vDl6TxChoZfa3aLH04Ma6V2WIQcpm0DQOdcCgQbq7llcjmZHvmnCeOCcyiTjhVI6xQ95EVRYQD81Yc5bCvXJQ/suAiv7yw+gff1h4VVgJUVpCSASqTKfiKch8IM4nuKNrrEagczqCq+BPGWtN3W/kF0T4q8kdDVliTDc7Ute8liUtRgkVIfch45mxpAKxBSoCdULWi5T2l5dG09msr8M8bMzF3FEMLuTVlQ0/pwge+fc2h8hesn04sWWzuZZVhsfSUyCw7YCoYOga1cJT3SSf6d00YfeN9pFFqJ2jf4mpkdXgVs4/Zr/m9VMs1BEbAHB2GjqSd4ui4VpU+TjhfAC501FQwGmKoe82krpRTM7ZGk1u7BhA5LLgdmbIuG+urRp8lClQo2tM7WW5U16miKYMBiOZmBYBuAFz8Kbodr9o9Vr1xh//XjQi9M1SjfV61UTI2C8JzUFv39pbG2soYZLJDkJ3UafMboo6a+OguUz6jCfQV8RXy+tKQtSxENOD2/xMIHXi/2lq7Q5kq20U3L5k67hDLHhmub17zY1Miwy90ItwZmX8IcJzQWKPxLxs9T4dRaimn3p1Kwu7BIT/Cne7oPtDHcZy90fvozDtBAeloxYwmwRvNt7ynZmdKcTiZAM7fLr35gPD1tTlrSLLwSPHRw6eunkA2rBiPaoJYLNNRA/pFT9bkd2G1pfjrtJHniaKfo+fEJlQjGZ4jTaCWU0ONK2dud9DtJuniStBz54UilgIGAwJOFXe/3W1UuyWZWeFEOI4MLmSOdbEbZh1upl1X3A/Y0Q0z7NV/EElZR/RFJLm8sa8Hzpp5e6piEJlMEJnw== template: metadata: diff --git a/lib/developer_portal/registry/forgejo_source.ex b/lib/developer_portal/registry/forgejo_source.ex index 6e87b2a..9d3f493 100644 --- a/lib/developer_portal/registry/forgejo_source.ex +++ b/lib/developer_portal/registry/forgejo_source.ex @@ -13,10 +13,12 @@ defmodule DeveloperPortal.Registry.ForgejoSource do @manifest_files [ %{name: "plugin.yaml", kind: "check"}, + %{name: "plugin.local.yaml", kind: "check", schema_variant: "local"}, %{name: "plugin.console.yaml", kind: "console"}, %{name: "plugin.inventory_sync.yaml", kind: "inventory_sync"}, %{name: "plugin.stream.yaml", kind: "stream"}, %{name: "manifest.json", kind: "check"}, + %{name: "manifest.local.json", kind: "check", schema_variant: "local"}, %{name: "manifest.console.json", kind: "console"}, %{name: "manifest.inventory_sync.json", kind: "inventory_sync"}, %{name: "manifest.stream.json", kind: "stream"} @@ -64,7 +66,7 @@ defmodule DeveloperPortal.Registry.ForgejoSource do manifest_entry -> [ - build_plugin(config, manifest_entry, manifest.kind, language, index, %{ + build_plugin(config, manifest_entry, manifest, language, index, %{ source_url: source_url, readme_url: readme_url, directory: directory, @@ -83,7 +85,7 @@ defmodule DeveloperPortal.Registry.ForgejoSource do [] end - defp build_plugin(config, manifest_entry, kind, language, index, urls) do + defp build_plugin(config, manifest_entry, %{kind: kind} = variant, language, index, urls) do manifest = fetch_manifest!(config, manifest_entry) slug = fetch_id!(manifest) release = Map.get(index, slug) @@ -94,6 +96,7 @@ defmodule DeveloperPortal.Registry.ForgejoSource do # can never crash the whole refresh. published = if signed, do: release, else: nil %{source_url: source_url, readme_url: readme_url, directory: directory, dist: dist} = urls + schema_variant = Map.get(variant, :schema_variant, kind) %Plugin{ slug: slug, @@ -110,7 +113,7 @@ defmodule DeveloperPortal.Registry.ForgejoSource do source_url: PublicURL.githubize(source_url), readme_url: PublicURL.githubize(readme_url), manifest_url: PublicURL.githubize(download_url(manifest_entry)), - config_schema_url: PublicURL.githubize(config_schema_url(kind, directory, dist)), + config_schema_url: PublicURL.githubize(config_schema_url(schema_variant, directory, dist)), wasm_url: PublicURL.githubize(find_download_url(dist, @wasm_candidates)), artifact_url: PublicURL.githubize(find_download_url(dist, @checksum_candidates)), signature_url: PublicURL.githubize(find_download_url(dist, @signature_candidates)), @@ -338,6 +341,7 @@ defmodule DeveloperPortal.Registry.ForgejoSource do end defp schema_candidates("stream"), do: ["config.stream.schema.json"] + defp schema_candidates("local"), do: ["config.local.schema.json"] defp schema_candidates("console"), do: ["config.console.schema.json", "config.schema.json"] defp schema_candidates("inventory_sync"), diff --git a/mix.lock b/mix.lock index b621d46..fff9a9d 100644 --- a/mix.lock +++ b/mix.lock @@ -32,7 +32,6 @@ "iterex": {:hex, :iterex, "0.1.2", "58f9b9b9a22a55cbfc7b5234a9c9c63eaac26d276b3db80936c0e1c60355a5a6", [:mix], [], "hexpm", "2e103b8bcc81757a9af121f6dc0df312c9a17220f302b1193ef720460d03029d"}, "jason": {:hex, :jason, "1.4.5", "2e3a008590b0b8d7388c20293e9dcc9cf3e5d642fd2a114e4cbbb52e595d940a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "b0c823996102bcd0239b3c2444eb00409b72f6a140c1950bc8b457d836b30684"}, "lazy_html": {:hex, :lazy_html, "0.1.12", "31a55ee622918fce988c94b06232227b42daa64e4eab14ac32081d0f3fd8db6f", [:make, :mix], [{:cc_precompiler, "~> 0.1", [hex: :cc_precompiler, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.9", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:fine, "~> 0.1.0", [hex: :fine, repo: "hexpm", optional: false]}], "hexpm", "8a0da594776caee58782c6f93b2abaa5bdb809daf8d43351a561f7de9dc2e2a8"}, - "libgraph": {:hex, :libgraph, "0.16.0", "3936f3eca6ef826e08880230f806bfea13193e49bf153f93edcf0239d4fd1d07", [:mix], [], "hexpm", "41ca92240e8a4138c30a7e06466acc709b0cbb795c643e9e17174a178982d6bf"}, "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, "makeup_json": {:hex, :makeup_json, "1.0.0", "03886f244d9db4179a9ecfffde5e711480e04e16ef91ca484d2b51ca3251a18f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "5c8c559e658c7f7e91b96c4b8c40f5912ea0adff44b7afe73e4639d9c3f53b94"}, @@ -72,7 +71,6 @@ "telemetry_metrics": {:hex, :telemetry_metrics, "1.1.0", "5bd5f3b5637e0abea0426b947e3ce5dd304f8b3bc6617039e2b5a008adc02f8f", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e7b79e8ddfde70adb6db8a6623d1778ec66401f366e9a8f5dd0955c56bc8ce67"}, "telemetry_poller": {:hex, :telemetry_poller, "1.3.0", "d5c46420126b5ac2d72bc6580fb4f537d35e851cc0f8dbd571acf6d6e10f5ec7", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "51f18bed7128544a50f75897db9974436ea9bfba560420b646af27a9a9b35211"}, "thousand_island": {:hex, :thousand_island, "1.5.0", "f50a213cac97262b6d5ebb85745aa2c00fec1413191e6e66834788d45425cecb", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "708923d40523e43cf99041ab37a0d4b0ec426ac6438fa3716ab23d919eaeb412"}, - "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.1", "a48703a25c170eedadca83b11e88985af08d35f37c6f664d6dcfb106a97782fc", [:rebar3], [], "hexpm", "b3a917854ce3ae233619744ad1e0102e05673136776fb2fa76234f3e03b23642"}, "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, "websock_adapter": {:hex, :websock_adapter, "0.6.0", "73db5ab8aaefd1a876a97ce3e6afc96562625de69ef17a4e04426e034849d0b8", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "50021a85bce8f203b086705d9e0c5415e2c7eb05d319111b0428fe71f9934617"}, "yamerl": {:hex, :yamerl, "0.10.0", "4ff81fee2f1f6a46f1700c0d880b24d193ddb74bd14ef42cb0bcf46e81ef2f8e", [:rebar3], [], "hexpm", "346adb2963f1051dc837a2364e4acf6eb7d80097c0f53cbdc3046ec8ec4b4e6e"}, diff --git a/test/developer_portal/registry/forgejo_source_test.exs b/test/developer_portal/registry/forgejo_source_test.exs new file mode 100644 index 0000000..7246692 --- /dev/null +++ b/test/developer_portal/registry/forgejo_source_test.exs @@ -0,0 +1,83 @@ +defmodule DeveloperPortal.Registry.ForgejoSourceTest do + use ExUnit.Case, async: true + + alias DeveloperPortal.Registry.ForgejoSource + + for {cloud_file, local_file} <- [ + {"plugin.yaml", "plugin.local.yaml"}, + {"manifest.json", "manifest.local.json"} + ] do + test "discovers #{local_file} alongside #{cloud_file} with its own configuration schema" do + cloud_file = unquote(cloud_file) + local_file = unquote(local_file) + + Req.Test.stub(__MODULE__, fn conn -> + case conn.request_path do + "/repos/example/catalog/contents/plugins" -> + Req.Test.json(conn, [entry("weather", "dir")]) + + "/repos/example/catalog/contents/plugins/weather" -> + files = [cloud_file, local_file, "config.schema.json", "config.local.schema.json"] + Req.Test.json(conn, Enum.map(files, &entry("weather/" <> &1, "file"))) + + "/repos/example/catalog/releases" -> + Req.Test.json(conn, []) + + "/raw/plugins/weather/" <> file -> + id = if file == cloud_file, do: "weather-cloud", else: "weather-local" + + conn + |> Plug.Conn.put_resp_content_type("text/plain") + |> Plug.Conn.send_resp(200, manifest(file, id)) + end + end) + + assert {:ok, plugins} = + ForgejoSource.fetch_plugins( + api_base_url: "https://catalog.example.test", + content_base_url: nil, + owner: "example", + repo: "catalog", + ref: "main", + plugin_root: "plugins", + req_options: [plug: {Req.Test, __MODULE__}, retry: false] + ) + + assert [%{slug: "weather-cloud"} = cloud, %{slug: "weather-local"} = local] = plugins + assert cloud.sample_kind == "check" + assert local.sample_kind == "check" + assert String.ends_with?(cloud.manifest_url, "/" <> cloud_file) + assert String.ends_with?(local.manifest_url, "/" <> local_file) + assert cloud.config_schema_url == raw_url("weather/config.schema.json") + assert local.config_schema_url == raw_url("weather/config.local.schema.json") + end + end + + defp entry(path, type) do + %{ + "name" => Path.basename(path), + "path" => "plugins/" <> path, + "type" => type, + "html_url" => "https://catalog.example.test/source/plugins/" <> path, + "download_url" => raw_url(path) + } + end + + defp raw_url(path), do: "https://catalog.example.test/raw/plugins/" <> path + + defp manifest(file, id) do + attrs = %{ + "id" => id, + "name" => id, + "version" => "0.0.1", + "description" => "Synthetic weather monitoring plugin", + "outputs" => "plugin_result" + } + + if Path.extname(file) == ".json" do + Jason.encode!(attrs) + else + Enum.map_join(attrs, "\n", fn {key, value} -> "#{key}: #{value}" end) + end + end +end