From b2b9f7daaf94bb31a9d5c7b7a2f94d7fb4126a15 Mon Sep 17 00:00:00 2001 From: SpecialBuilder Date: Fri, 4 Sep 2026 10:30:08 -0400 Subject: [PATCH 01/13] Create publish step for each distribution site --- .github/workflows/main.yml | 30 ++++++++++++++++++++++++++++++ beet-publish.yaml | 2 ++ test_plugin.py | 4 ++++ 3 files changed, 36 insertions(+) create mode 100644 beet-publish.yaml create mode 100644 test_plugin.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 698898197f..d0a5b4caf5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -135,3 +135,33 @@ jobs: with: name: 'Test world in ${{ matrix.version }} for ${{ github.sha }}' path: ${{ github.workspace }}/world/ + + publish: + runs-on: ubuntu-24.04 + if: github.event_name != 'pull_request' + needs: build + strategy: + matrix: + platform: [modrinth, smithed] + concurrency: + group: release + name: 'publish-${{ matrix.platform }}' + steps: + - uses: actions/checkout@v7 + + - name: Checkout release branch + uses: actions/checkout@v7 + with: + ref: release + path: release + + # TODO use cache from build? + - name: Set up uv + uses: astral-sh/setup-uv@v7 + + - name: Publish all modules + run: uv run beet -p beet-publish.yaml -l ${{ env.LOG_LEVEL }} build + env: + BEET_MODRINTH_TOKEN: ${{ secrets.MODRINTH_TOKEN }} + BEET_SMITHED_TOKEN: ${{ secrets.SMITHED_TOKEN }} + LOG_LEVEL: ${{ runner.debug == 1 && 'DEBUG' || 'INFO'}} diff --git a/beet-publish.yaml b/beet-publish.yaml new file mode 100644 index 0000000000..ebec2eb259 --- /dev/null +++ b/beet-publish.yaml @@ -0,0 +1,2 @@ +pipeline: + - test_plugin.say_hello diff --git a/test_plugin.py b/test_plugin.py new file mode 100644 index 0000000000..3f8567e3af --- /dev/null +++ b/test_plugin.py @@ -0,0 +1,4 @@ +from beet import Context + +def say_hello(ctx: Context): + print("Hello Action World") From f7702b4bf32ffc9870f6bae74a20fc239992d9e0 Mon Sep 17 00:00:00 2001 From: SpecialBuilder Date: Fri, 4 Sep 2026 11:47:10 -0400 Subject: [PATCH 02/13] Add final summarize step --- .github/workflows/main.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d0a5b4caf5..b1c7d9093f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -165,3 +165,10 @@ jobs: BEET_MODRINTH_TOKEN: ${{ secrets.MODRINTH_TOKEN }} BEET_SMITHED_TOKEN: ${{ secrets.SMITHED_TOKEN }} LOG_LEVEL: ${{ runner.debug == 1 && 'DEBUG' || 'INFO'}} + + summarize: + runs-on: ubuntu-24.04 + if: github.event_name != 'pull_request' + needs: publish + steps: + - uses: action/checkout@v7 From 936db098faafe0839b5cfb4d55ecf5749cfc3efb Mon Sep 17 00:00:00 2001 From: SpecialBuilder Date: Sun, 6 Sep 2026 13:26:24 -0400 Subject: [PATCH 03/13] Framework for loading partial beet.yaml data for the publish pipeline --- beet-publish.yaml | 10 ++++++++-- gm4/plugins/publish.py | 24 ++++++++++++++++++++++++ test_plugin.py | 3 +++ 3 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 gm4/plugins/publish.py diff --git a/beet-publish.yaml b/beet-publish.yaml index ebec2eb259..b623e16c1c 100644 --- a/beet-publish.yaml +++ b/beet-publish.yaml @@ -1,2 +1,8 @@ -pipeline: - - test_plugin.say_hello +broadcast: gm4_bat_grenades +pipeline: + - gm4.plugins.publish.dev_test_1 + - gm4.plugins.publish.my_load_config + +meta: + plugins: + - test_plugin.print_data diff --git a/gm4/plugins/publish.py b/gm4/plugins/publish.py new file mode 100644 index 0000000000..2eaf287169 --- /dev/null +++ b/gm4/plugins/publish.py @@ -0,0 +1,24 @@ +from beet import Context, Project, ProjectBuilder, PackConfig +from beet.toolchain.config import load_config + +def my_load_config(ctx: Context): + """Loads relavent fields from each module's beet.yaml, without inheriting the pipeline and running plugins""" + + config = load_config(ctx.directory/"beet.yaml") + + config.require = [] + config.pipeline = ctx.meta.get("plugins", []) + config.data_pack = PackConfig() + config.resource_pack = PackConfig() + + print(config) + + # run the new list of plugins from beet-publish.yaml + ctx.require( + ProjectBuilder( + Project( + config, + resolved_cache=ctx.cache + ) + ) + ) diff --git a/test_plugin.py b/test_plugin.py index 3f8567e3af..9351bc5453 100644 --- a/test_plugin.py +++ b/test_plugin.py @@ -2,3 +2,6 @@ def say_hello(ctx: Context): print("Hello Action World") + +def print_data(ctx: Context): + print(f"\nretrieved project {ctx.project_name}, {ctx.project_description}") From c8ea087ab285077ad9bc7ea68a8655bf4c543d4a Mon Sep 17 00:00:00 2001 From: SpecialBuilder Date: Sun, 6 Sep 2026 13:33:47 -0400 Subject: [PATCH 04/13] Send .beet_cache between build and publish jobs --- .github/workflows/main.yml | 14 ++++++++++++++ beet-publish.yaml | 1 + gm4/plugins/publish.py | 8 ++++++++ 3 files changed, 23 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b1c7d9093f..479b80417c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -78,6 +78,14 @@ jobs: directory: release branch: release + - name: Upload .beet_cache as artifact + if: github.event_name != 'pull_request' + uses: actions/upload-artifact@v7 + with: + name: .beet_cache + path: ${{ github.workspace }}/.beet_cache/ + include-hidden-files: true + test: strategy: fail-fast: false @@ -155,6 +163,12 @@ jobs: ref: release path: release + - name: Restore .beet_cache from artifact + uses: actions/download-artifact@v7 + with: + name: .beet_cache + path: ${{ github.workspace }}/.beet_cache + # TODO use cache from build? - name: Set up uv uses: astral-sh/setup-uv@v7 diff --git a/beet-publish.yaml b/beet-publish.yaml index b623e16c1c..cafd63603f 100644 --- a/beet-publish.yaml +++ b/beet-publish.yaml @@ -5,4 +5,5 @@ pipeline: meta: plugins: + - gm4.plugins.readme_generator - test_plugin.print_data diff --git a/gm4/plugins/publish.py b/gm4/plugins/publish.py index 2eaf287169..263b6f1fa9 100644 --- a/gm4/plugins/publish.py +++ b/gm4/plugins/publish.py @@ -1,6 +1,14 @@ from beet import Context, Project, ProjectBuilder, PackConfig from beet.toolchain.config import load_config +def beet_default(ctx: Context): + pass + +def dev_test_1(ctx: Context): + print(f"beet cache restored as: {list(ctx.cache.keys())}") + + print(f"gm4_manifest is :{ctx.cache["gm4_manifest"]}") + def my_load_config(ctx: Context): """Loads relavent fields from each module's beet.yaml, without inheriting the pipeline and running plugins""" From 0deb5ab7da566e3919201cbbe69c54eca2c3807d Mon Sep 17 00:00:00 2001 From: SpecialBuilder Date: Sun, 6 Sep 2026 14:15:50 -0400 Subject: [PATCH 05/13] Move publish plugins to new file --- beet-publish.yaml | 2 - gm4/plugins/output.py | 203 +--------------------------------- gm4/plugins/publish.py | 240 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 233 insertions(+), 212 deletions(-) diff --git a/beet-publish.yaml b/beet-publish.yaml index cafd63603f..86909beb1c 100644 --- a/beet-publish.yaml +++ b/beet-publish.yaml @@ -1,9 +1,7 @@ broadcast: gm4_bat_grenades pipeline: - - gm4.plugins.publish.dev_test_1 - gm4.plugins.publish.my_load_config meta: plugins: - gm4.plugins.readme_generator - - test_plugin.print_data diff --git a/gm4/plugins/output.py b/gm4/plugins/output.py index 4589c159a3..33de9311a3 100644 --- a/gm4/plugins/output.py +++ b/gm4/plugins/output.py @@ -1,21 +1,11 @@ from beet import Context from pathlib import Path import os -import json -import re -import requests import shutil import logging -from gm4.utils import run, Version, NoneAttribute -from gm4.plugins.manifest import ManifestConfig, ManifestCacheModel -parent_logger = logging.getLogger("gm4.output") -MODRINTH_API = "https://api.modrinth.com/v2" -MODRINTH_AUTH_KEY = "BEET_MODRINTH_TOKEN" -SMITHED_API = "https://api.smithed.dev/v2" -SMITHED_AUTH_KEY = "BEET_SMITHED_TOKEN" -USER_AGENT = "Gamemode4Dev/GM4_Datapacks/release-pipeline (gamemode4official@gmail.com)" +parent_logger = logging.getLogger("gm4.output") def beet_default(ctx: Context): @@ -73,14 +63,6 @@ def release(ctx: Context): """ Saves the zipped datapack and metadata to the ./release/{version} folder. Should be first in pipeline to properly wrap all other plugins cleanup phases - - If the module has the `version` and `meta.modrinth.project_id` fields, and - `BEET_MODRINTH_TOKEN` environment variable is set, will try to publish a - new version to Modrinth if it doesn't already exist. - - Similarly, if the module has the `version` and `meta.smithed.pack_id` fields, and - `BEET_SMITHED_TOKEN` environment variable is set, will try to publish a - new version to Smithed if it doesn't already exist. """ version_dir = os.getenv("VERSION", "26.2") release_dir = Path("release") / version_dir @@ -109,189 +91,6 @@ def release(ctx: Context): if "smithed_readme" in ctx.meta: ctx.meta['smithed_readme'].dump(smithed_readme_dir, f"{corrected_project_id}.md") - config = ctx.validate("gm4", ManifestConfig) - - # publish to download platforms - publish_modrinth(ctx, config, release_dir, file_name) - publish_smithed(ctx, config, file_name) - - -def publish_modrinth(ctx: Context, config: ManifestConfig, release_dir: Path, file_name: str): - '''Attempts to publish pack to modrinth''' - auth_token = os.getenv(MODRINTH_AUTH_KEY, None) - logger = parent_logger.getChild(f"modrinth.{ctx.project_id}") - if config.modrinth and auth_token: - # update page description - res = requests.get(f"{MODRINTH_API}/project/{config.modrinth.project_id}", headers={'Authorization': auth_token, 'User-Agent': USER_AGENT}) - if not (200 <= res.status_code < 300): - if res.status_code == 404: - logger.warning(f"Cannot edit description of modrinth project {config.modrinth.project_id} as it doesn't exist.") - else: - logger.warning(f"Failed to get project: {res.status_code} {res.text}") - return - existing_readme = res.json()["body"] - if existing_readme != (d:=ctx.meta['modrinth_readme'].text): - logger.debug("Readme and modrinth-page content differ. Updating webpage body") - res = requests.patch(f"{MODRINTH_API}/project/{config.modrinth.project_id}", headers={'Authorization': auth_token, 'User-Agent': USER_AGENT}, json={"body": d}) - if not (200 <= res.status_code < 300): - logger.warning(f"Failed to update description: {res.status_code} {res.text}") - logger.info(f"Successfully updated description of {ctx.project_name}", extra={"gh_annotate_skip": True}) - - # upload datapack zip - if ctx.project_version: - version = ctx.cache["gm4_manifest"].json["modules"].get(ctx.project_id, {}).get("version", None) - if version is None: - logger.warning("Full version number not available in ctx.meta. Skipping publishing") - return - - res = requests.get(f"{MODRINTH_API}/project/{config.modrinth.project_id}/version", headers={'Authorization': auth_token, 'User-Agent': USER_AGENT}) - if not (200 <= res.status_code < 300): - if res.status_code == 404: - logger.warning(f"Cannot publish to modrinth project {config.modrinth.project_id} as it doesn't exist.") - else: - logger.warning(f"Failed to get project versions: {res.status_code} {res.text}") - return - project_data = res.json() - - matching_version = next((v for v in project_data if v["version_number"] == str(version)), None) - if matching_version is not None: # patch version already exists - # update mc versions if necessary - if len(config.minecraft) > 0 and not set(matching_version["game_versions"]) == set(config.minecraft): - # supported versions has changed and is not empty - logger.debug("Additional MC version support has been added to an existing patch version. Updating existing modrinth version data") - res = requests.patch(f"{MODRINTH_API}/version/{matching_version['id']}", headers={'Authorization': auth_token, 'User-Agent': USER_AGENT}, json={ - "game_versions": config.minecraft - }) - if not (200 <= res.status_code < 300): - logger.warning(f"Failed to patch project versions: {res.status_code} {res.text}") - return - - if len(config.minecraft) > 0: - # supported versions is not empty, post new version - with open(release_dir / file_name, "rb") as f: - file_bytes = f.read() - - changelog = run(["git", "log", "-1", "--format=%s"]) - changelog = re.sub(r"\(#(\d+)\)", "([#\\1](https://github.com/Gamemode4Dev/GM4_Datapacks/pull/\\1))", changelog) - - res = requests.post(f"{MODRINTH_API}/version", headers={'Authorization': auth_token, 'User-Agent': USER_AGENT}, files={ - "data": json.dumps({ - "name": f"{ctx.project_name} v{version}", - "version_number": version, - "changelog": changelog, - "dependencies": [], - "game_versions": config.minecraft, - "version_type": "release", - "loaders": ["datapack"], - "featured": False, - "project_id": config.modrinth.project_id, - "file_parts": [file_name], - }), - file_name: file_bytes, - }) - if not (200 <= res.status_code < 300): - logger.warning(f"Failed to publish new version version: {res.status_code} {res.text}") - return - logger.info(f"Successfully published {res.json()['name']}", extra={"gh_annotate_skip": True}) - - -def publish_smithed(ctx: Context, config: ManifestConfig, file_name: str): - """Attempts to publish pack to smithed""" - auth_token = os.getenv(SMITHED_AUTH_KEY, None) - logger = parent_logger.getChild(f"smithed.{ctx.project_id}") - mc_version_dir = os.getenv("VERSION", "26.2") - manifest = ManifestCacheModel.model_validate(ctx.cache["gm4_manifest"].json) - project_id = stem if (stem:=ctx.directory.stem).startswith("lib") else ctx.project_id - - if config.smithed and auth_token: - version = (manifest.modules|manifest.libraries).get(project_id, NoneAttribute()).version or "" - - # get project data and existing versions - res = requests.get(f"{SMITHED_API}/packs/{config.smithed.pack_id}") - if not (200 <= res.status_code < 300): - if res.status_code == 404: - logger.warning(f"Cannot publish to smithed project {config.smithed.pack_id} as it doesn't exist.") - else: - logger.warning(f"Failed to get project: {res.status_code} {res.text}") - return - - project_data = res.json() - - # update description and pack image - # ensures they point to the most up-to-date mc version branch - project_versions = project_data["versions"] - newest_version = sorted([Version(v["name"]) for v in project_versions])[-1] - if Version(version) > newest_version: # only update the description if we're not patching an old version - project_display = project_data["display"] - current_icon = f"https://raw.githubusercontent.com/Gamemode4Dev/GM4_Datapacks/release/{mc_version_dir}/generated/pack_icons/{project_id}.png" - current_readme = f"https://raw.githubusercontent.com/Gamemode4Dev/GM4_Datapacks/release/{mc_version_dir}/generated/smithed_readmes/{project_id}.md" - - if project_display["icon"] != current_icon or project_display["webPage"] != current_readme: - logger.debug("Pack Icon or Readme hyperlink is incorrect. Updating project") - res = requests.patch(f"{SMITHED_API}/packs/{config.smithed.pack_id}", params={'token': auth_token}, - json={"data": { - "display": { - "icon": current_icon, - "webPage": current_readme, - }, - }}) - if not (200 <= res.status_code < 300): - logger.warning(f"Failed to update descripion: {res.status_code} {res.text}") - logger.info(f"{ctx.project_name} {res.text}", extra={"gh_annotate_skip": True}) - - matching_version = next((v for v in project_versions if v["name"] == str(version)), None) - if matching_version is not None: # patch version already exists - # update MC version if necessary - if len(config.minecraft) > 0 and not set(matching_version["supports"]) == set(config.minecraft): - # supported versions has changed and is not empty - logger.debug("Additional MC version support has been added to an existing patch version. Updating existing smithed version data") - res = requests.patch(f"{SMITHED_API}/packs/{config.smithed.pack_id}/versions/{matching_version['name']}", params={'token': auth_token}, json={ - "data": { - "supports": config.minecraft - } - }) - if not (200 <= res.status_code < 300): - logger.warning(f"Failed to patch project versions: {res.status_code} {res.text}") - return - - # permalink previous version (in that MC version) to the git history - commit_hash = run("cd release && git log -1 --format=%H") - matching_mc_versions = sorted((Version(v["name"]) for v in project_versions if set(v['supports']) & set(config.minecraft))) - prior_version_in_mc_version = matching_mc_versions[-1] if len(matching_mc_versions) > 0 else None # newest version number, with any MC overlap - prior_url: str = next((v["downloads"]["datapack"] for v in project_versions if Version(v["name"]) == prior_version_in_mc_version), "") - if "https://github.com/Gamemode4Dev/GM4_Datapacks/blob/" not in prior_url and prior_version_in_mc_version: - res = requests.patch(f"{SMITHED_API}/packs/{config.smithed.pack_id}/versions/{prior_version_in_mc_version}", params={'token': auth_token}, json={ - "data":{ - "downloads": { - "datapack": f"https://github.com/Gamemode4Dev/GM4_Datapacks/blob/{commit_hash}/{mc_version_dir}/{file_name}?raw=true", - "resourcepack": "" - } - } - }) - if not (200 <= res.status_code < 300): - logger.warning(f"Failed to permalink {project_id} version {prior_version_in_mc_version}: {res.status_code} {res.text}") - else: - logger.info(f"Permalinked {project_id} {prior_version_in_mc_version} to git history: {res.text}", extra={"gh_annotate_skip": True}) - - if len(config.minecraft) > 0: - # supported versions is not empty, post new version - res = requests.post(f"{SMITHED_API}/packs/{config.smithed.pack_id}/versions", - params={'token': auth_token, 'version': version}, - json={"data":{ - "downloads":{ - "datapack": f"https://raw.githubusercontent.com/Gamemode4Dev/GM4_Datapacks/release/{mc_version_dir}/{file_name}", - "resourcepack": "" - }, - "name": version, - "supports": config.minecraft, - "dependencies": [] - }} - ) - if not (200 <= res.status_code < 300): - logger.warning(f"Failed to publish new version of {ctx.project_name}: {res.status_code} {res.text}") - return - logger.info(f"{ctx.project_name} {res.text}", extra={"gh_annotate_skip": True}) - def clear_release(ctx: Context): """ diff --git a/gm4/plugins/publish.py b/gm4/plugins/publish.py index 263b6f1fa9..dc1c9771f2 100644 --- a/gm4/plugins/publish.py +++ b/gm4/plugins/publish.py @@ -1,25 +1,73 @@ from beet import Context, Project, ProjectBuilder, PackConfig -from beet.toolchain.config import load_config +from beet.toolchain.config import load_config as beet_load_config +import json +import re +import requests +import os +from gm4.utils import run, Version, NoneAttribute +from gm4.plugins.manifest import ManifestConfig, ManifestCacheModel +from pathlib import Path +import logging + +parent_logger = logging.getLogger("gm4.publish") + + +MODRINTH_API = "https://api.modrinth.com/v2" +MODRINTH_AUTH_KEY = "BEET_MODRINTH_TOKEN" +SMITHED_API = "https://api.smithed.dev/v2" +SMITHED_AUTH_KEY = "BEET_SMITHED_TOKEN" +USER_AGENT = "Gamemode4Dev/GM4_Datapacks/release-pipeline (gamemode4official@gmail.com)" def beet_default(ctx: Context): - pass + """Published all module release .zips from VERSION/release to download sites + + If the module has the `version` and `meta.modrinth.project_id` fields, and + `BEET_MODRINTH_TOKEN` environment variable is set, will try to publish a + new version to Modrinth if it doesn't already exist. + + Similarly, if the module has the `version` and `meta.smithed.pack_id` fields, and + `BEET_SMITHED_TOKEN` environment variable is set, will try to publish a + new version to Smithed if it doesn't already exist.""" + + version_dir = os.getenv("VERSION", "26.2") + release_dir = Path("release") / version_dir + + corrected_project_id = stem if (stem:=ctx.directory.stem).startswith("lib") else ctx.project_id + + file_name = f"{corrected_project_id}_{version_dir.replace('.', '_')}.zip" + + config = ctx.validate("gm4", ManifestConfig) + + publish_to = ctx.meta.get("gm4",{}).get("publish_to", None) + + # publish to download platforms + if publish_to == "smithed": + publish_smithed(ctx, config, file_name) + elif publish_to == "modrinth": + publish_modrinth(ctx, config, release_dir, file_name) def dev_test_1(ctx: Context): - print(f"beet cache restored as: {list(ctx.cache.keys())}") + # print(f"beet cache restored as: {list(ctx.cache.keys())}") + + # print(f"gm4_manifest is :{ctx.cache["gm4_manifest"]}") + print(ctx.meta.get("gm4")) - print(f"gm4_manifest is :{ctx.cache["gm4_manifest"]}") + if ctx.meta.get("gm4",{}).get("publish_to", None) == "modrinth": + print(f"publishing to modrinth!") + else: + print(f"publishing to smithed") + pass -def my_load_config(ctx: Context): +def load_config(ctx: Context): """Loads relavent fields from each module's beet.yaml, without inheriting the pipeline and running plugins""" - config = load_config(ctx.directory/"beet.yaml") + config = beet_load_config(ctx.directory/"beet.yaml") config.require = [] config.pipeline = ctx.meta.get("plugins", []) config.data_pack = PackConfig() config.resource_pack = PackConfig() - - print(config) + config.meta["gm4"] |= ctx.meta["gm4"] # pull in root gm4 meta # run the new list of plugins from beet-publish.yaml ctx.require( @@ -30,3 +78,179 @@ def my_load_config(ctx: Context): ) ) ) + +def publish_modrinth(ctx: Context, config: ManifestConfig, release_dir: Path, file_name: str): + '''Attempts to publish pack to modrinth''' + auth_token = os.getenv(MODRINTH_AUTH_KEY, None) + logger = parent_logger.getChild(f"modrinth.{ctx.project_id}") + if config.modrinth and auth_token: + # update page description + res = requests.get(f"{MODRINTH_API}/project/{config.modrinth.project_id}", headers={'Authorization': auth_token, 'User-Agent': USER_AGENT}) + if not (200 <= res.status_code < 300): + if res.status_code == 404: + logger.warning(f"Cannot edit description of modrinth project {config.modrinth.project_id} as it doesn't exist.") + else: + logger.warning(f"Failed to get project: {res.status_code} {res.text}") + return + existing_readme = res.json()["body"] + if existing_readme != (d:=ctx.meta['modrinth_readme'].text): + logger.debug("Readme and modrinth-page content differ. Updating webpage body") + res = requests.patch(f"{MODRINTH_API}/project/{config.modrinth.project_id}", headers={'Authorization': auth_token, 'User-Agent': USER_AGENT}, json={"body": d}) + if not (200 <= res.status_code < 300): + logger.warning(f"Failed to update description: {res.status_code} {res.text}") + logger.info(f"Successfully updated description of {ctx.project_name}", extra={"gh_annotate_skip": True}) + + # upload datapack zip + if ctx.project_version: + version = ctx.cache["gm4_manifest"].json["modules"].get(ctx.project_id, {}).get("version", None) + if version is None: + logger.warning("Full version number not available in ctx.meta. Skipping publishing") + return + + res = requests.get(f"{MODRINTH_API}/project/{config.modrinth.project_id}/version", headers={'Authorization': auth_token, 'User-Agent': USER_AGENT}) + if not (200 <= res.status_code < 300): + if res.status_code == 404: + logger.warning(f"Cannot publish to modrinth project {config.modrinth.project_id} as it doesn't exist.") + else: + logger.warning(f"Failed to get project versions: {res.status_code} {res.text}") + return + project_data = res.json() + + matching_version = next((v for v in project_data if v["version_number"] == str(version)), None) + if matching_version is not None: # patch version already exists + # update mc versions if necessary + if len(config.minecraft) > 0 and not set(matching_version["game_versions"]) == set(config.minecraft): + # supported versions has changed and is not empty + logger.debug("Additional MC version support has been added to an existing patch version. Updating existing modrinth version data") + res = requests.patch(f"{MODRINTH_API}/version/{matching_version['id']}", headers={'Authorization': auth_token, 'User-Agent': USER_AGENT}, json={ + "game_versions": config.minecraft + }) + if not (200 <= res.status_code < 300): + logger.warning(f"Failed to patch project versions: {res.status_code} {res.text}") + return + + if len(config.minecraft) > 0: + # supported versions is not empty, post new version + with open(release_dir / file_name, "rb") as f: + file_bytes = f.read() + + changelog = run(["git", "log", "-1", "--format=%s"]) + changelog = re.sub(r"\(#(\d+)\)", "([#\\1](https://github.com/Gamemode4Dev/GM4_Datapacks/pull/\\1))", changelog) + + res = requests.post(f"{MODRINTH_API}/version", headers={'Authorization': auth_token, 'User-Agent': USER_AGENT}, files={ + "data": json.dumps({ + "name": f"{ctx.project_name} v{version}", + "version_number": version, + "changelog": changelog, + "dependencies": [], + "game_versions": config.minecraft, + "version_type": "release", + "loaders": ["datapack"], + "featured": False, + "project_id": config.modrinth.project_id, + "file_parts": [file_name], + }), + file_name: file_bytes, + }) + if not (200 <= res.status_code < 300): + logger.warning(f"Failed to publish new version version: {res.status_code} {res.text}") + return + logger.info(f"Successfully published {res.json()['name']}", extra={"gh_annotate_skip": True}) + + +def publish_smithed(ctx: Context, config: ManifestConfig, file_name: str): + """Attempts to publish pack to smithed""" + auth_token = os.getenv(SMITHED_AUTH_KEY, None) + logger = parent_logger.getChild(f"smithed.{ctx.project_id}") + mc_version_dir = os.getenv("VERSION", "26.2") + manifest = ManifestCacheModel.model_validate(ctx.cache["gm4_manifest"].json) + project_id = stem if (stem:=ctx.directory.stem).startswith("lib") else ctx.project_id + + if config.smithed and auth_token: + version = (manifest.modules|manifest.libraries).get(project_id, NoneAttribute()).version or "" + + # get project data and existing versions + res = requests.get(f"{SMITHED_API}/packs/{config.smithed.pack_id}") + if not (200 <= res.status_code < 300): + if res.status_code == 404: + logger.warning(f"Cannot publish to smithed project {config.smithed.pack_id} as it doesn't exist.") + else: + logger.warning(f"Failed to get project: {res.status_code} {res.text}") + return + + project_data = res.json() + + # update description and pack image + # ensures they point to the most up-to-date mc version branch + project_versions = project_data["versions"] + newest_version = sorted([Version(v["name"]) for v in project_versions])[-1] + if Version(version) > newest_version: # only update the description if we're not patching an old version + project_display = project_data["display"] + current_icon = f"https://raw.githubusercontent.com/Gamemode4Dev/GM4_Datapacks/release/{mc_version_dir}/generated/pack_icons/{project_id}.png" + current_readme = f"https://raw.githubusercontent.com/Gamemode4Dev/GM4_Datapacks/release/{mc_version_dir}/generated/smithed_readmes/{project_id}.md" + + if project_display["icon"] != current_icon or project_display["webPage"] != current_readme: + logger.debug("Pack Icon or Readme hyperlink is incorrect. Updating project") + res = requests.patch(f"{SMITHED_API}/packs/{config.smithed.pack_id}", params={'token': auth_token}, + json={"data": { + "display": { + "icon": current_icon, + "webPage": current_readme, + }, + }}) + if not (200 <= res.status_code < 300): + logger.warning(f"Failed to update descripion: {res.status_code} {res.text}") + logger.info(f"{ctx.project_name} {res.text}", extra={"gh_annotate_skip": True}) + + matching_version = next((v for v in project_versions if v["name"] == str(version)), None) + if matching_version is not None: # patch version already exists + # update MC version if necessary + if len(config.minecraft) > 0 and not set(matching_version["supports"]) == set(config.minecraft): + # supported versions has changed and is not empty + logger.debug("Additional MC version support has been added to an existing patch version. Updating existing smithed version data") + res = requests.patch(f"{SMITHED_API}/packs/{config.smithed.pack_id}/versions/{matching_version['name']}", params={'token': auth_token}, json={ + "data": { + "supports": config.minecraft + } + }) + if not (200 <= res.status_code < 300): + logger.warning(f"Failed to patch project versions: {res.status_code} {res.text}") + return + + # permalink previous version (in that MC version) to the git history + commit_hash = run("cd release && git log -1 --format=%H") + matching_mc_versions = sorted((Version(v["name"]) for v in project_versions if set(v['supports']) & set(config.minecraft))) + prior_version_in_mc_version = matching_mc_versions[-1] if len(matching_mc_versions) > 0 else None # newest version number, with any MC overlap + prior_url: str = next((v["downloads"]["datapack"] for v in project_versions if Version(v["name"]) == prior_version_in_mc_version), "") + if "https://github.com/Gamemode4Dev/GM4_Datapacks/blob/" not in prior_url and prior_version_in_mc_version: + res = requests.patch(f"{SMITHED_API}/packs/{config.smithed.pack_id}/versions/{prior_version_in_mc_version}", params={'token': auth_token}, json={ + "data":{ + "downloads": { + "datapack": f"https://github.com/Gamemode4Dev/GM4_Datapacks/blob/{commit_hash}/{mc_version_dir}/{file_name}?raw=true", + "resourcepack": "" + } + } + }) + if not (200 <= res.status_code < 300): + logger.warning(f"Failed to permalink {project_id} version {prior_version_in_mc_version}: {res.status_code} {res.text}") + else: + logger.info(f"Permalinked {project_id} {prior_version_in_mc_version} to git history: {res.text}", extra={"gh_annotate_skip": True}) + + if len(config.minecraft) > 0: + # supported versions is not empty, post new version + res = requests.post(f"{SMITHED_API}/packs/{config.smithed.pack_id}/versions", + params={'token': auth_token, 'version': version}, + json={"data":{ + "downloads":{ + "datapack": f"https://raw.githubusercontent.com/Gamemode4Dev/GM4_Datapacks/release/{mc_version_dir}/{file_name}", + "resourcepack": "" + }, + "name": version, + "supports": config.minecraft, + "dependencies": [] + }} + ) + if not (200 <= res.status_code < 300): + logger.warning(f"Failed to publish new version of {ctx.project_name}: {res.status_code} {res.text}") + return + logger.info(f"{ctx.project_name} {res.text}", extra={"gh_annotate_skip": True}) From 195df2bd373625c3812bb316afae07579e9e5c71 Mon Sep 17 00:00:00 2001 From: SpecialBuilder Date: Sun, 6 Sep 2026 14:43:14 -0400 Subject: [PATCH 06/13] Only run the right publish pipeline for each workflow job --- .github/workflows/main.yml | 2 +- beet-publish.yaml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 479b80417c..51fe00b7cb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -174,7 +174,7 @@ jobs: uses: astral-sh/setup-uv@v7 - name: Publish all modules - run: uv run beet -p beet-publish.yaml -l ${{ env.LOG_LEVEL }} build + run: uv run beet -p beet-publish.yaml -l ${{ env.LOG_LEVEL }} -s meta.gm4.publish_to=${{ matrix.platform }} build env: BEET_MODRINTH_TOKEN: ${{ secrets.MODRINTH_TOKEN }} BEET_SMITHED_TOKEN: ${{ secrets.SMITHED_TOKEN }} diff --git a/beet-publish.yaml b/beet-publish.yaml index 86909beb1c..fc828489a8 100644 --- a/beet-publish.yaml +++ b/beet-publish.yaml @@ -1,7 +1,8 @@ broadcast: gm4_bat_grenades pipeline: - - gm4.plugins.publish.my_load_config + - gm4.plugins.publish.load_config meta: plugins: - gm4.plugins.readme_generator + - gm4.plugins.publish From 349dd373f9881fb2ca93b73e9408bc82b38e8f35 Mon Sep 17 00:00:00 2001 From: SpecialBuilder Date: Sun, 6 Sep 2026 19:47:39 -0400 Subject: [PATCH 07/13] Better logging for invalid tokens --- beet-publish.yaml | 15 +++++--- gm4/plugins/annotations.py | 2 +- gm4/plugins/publish.py | 78 ++++++++++++++++++++++++++------------ 3 files changed, 63 insertions(+), 32 deletions(-) diff --git a/beet-publish.yaml b/beet-publish.yaml index fc828489a8..e9c722ca51 100644 --- a/beet-publish.yaml +++ b/beet-publish.yaml @@ -1,8 +1,11 @@ -broadcast: gm4_bat_grenades pipeline: - - gm4.plugins.publish.load_config + - gm4.plugins.annotations + - gm4.plugins.publish.switch_platform -meta: - plugins: - - gm4.plugins.readme_generator - - gm4.plugins.publish + - broadcast: gm4_bat_grenades + pipeline: + - gm4.plugins.publish.load_config + meta: + plugins: + - gm4.plugins.readme_generator + - gm4.plugins.publish diff --git a/gm4/plugins/annotations.py b/gm4/plugins/annotations.py index 73bf568da4..9936f64457 100644 --- a/gm4/plugins/annotations.py +++ b/gm4/plugins/annotations.py @@ -31,7 +31,7 @@ def filter(record: logging.LogRecord): # summary handler holds onto certain records until the exit phase when it emits to a markdown summary sum_handler = SummaryHandler(1000, ctx.cache) - logging.getLogger("gm4.output").addHandler(sum_handler) + logging.getLogger("gm4.publish").addHandler(sum_handler) logging.getLogger("gm4.manifest.update_patch").addHandler(sum_handler) # after the whole build, flush the stored records and form the markdown summary diff --git a/gm4/plugins/publish.py b/gm4/plugins/publish.py index dc1c9771f2..b7a1321989 100644 --- a/gm4/plugins/publish.py +++ b/gm4/plugins/publish.py @@ -1,13 +1,16 @@ -from beet import Context, Project, ProjectBuilder, PackConfig -from beet.toolchain.config import load_config as beet_load_config import json -import re -import requests +import logging import os -from gm4.utils import run, Version, NoneAttribute -from gm4.plugins.manifest import ManifestConfig, ManifestCacheModel +import re +import sys from pathlib import Path -import logging + +import requests +from beet import Context, PackConfig, Project, ProjectBuilder +from beet.toolchain.config import load_config as beet_load_config + +from gm4.plugins.manifest import ManifestCacheModel, ManifestConfig +from gm4.utils import NoneAttribute, Version, run parent_logger = logging.getLogger("gm4.publish") @@ -28,6 +31,8 @@ def beet_default(ctx: Context): Similarly, if the module has the `version` and `meta.smithed.pack_id` fields, and `BEET_SMITHED_TOKEN` environment variable is set, will try to publish a new version to Smithed if it doesn't already exist.""" + + print(f"running publish on {ctx.project_id}") version_dir = os.getenv("VERSION", "26.2") release_dir = Path("release") / version_dir @@ -38,25 +43,31 @@ def beet_default(ctx: Context): config = ctx.validate("gm4", ManifestConfig) - publish_to = ctx.meta.get("gm4",{}).get("publish_to", None) + publish_to = ctx.cache["currently_publishing"].json.get("publish_to", None) + print(publish_to) - # publish to download platforms + # publish to download platforms, based on which gh job this is if publish_to == "smithed": publish_smithed(ctx, config, file_name) elif publish_to == "modrinth": publish_modrinth(ctx, config, release_dir, file_name) -def dev_test_1(ctx: Context): - # print(f"beet cache restored as: {list(ctx.cache.keys())}") +# def dev_test_1(ctx: Context): +# # print(f"beet cache restored as: {list(ctx.cache.keys())}") - # print(f"gm4_manifest is :{ctx.cache["gm4_manifest"]}") - print(ctx.meta.get("gm4")) +# # print(f"gm4_manifest is :{ctx.cache["gm4_manifest"]}") +# print(ctx.meta.get("gm4")) - if ctx.meta.get("gm4",{}).get("publish_to", None) == "modrinth": - print(f"publishing to modrinth!") - else: - print(f"publishing to smithed") - pass +# if ctx.meta.get("gm4",{}).get("publish_to", None) == "modrinth": +# print(f"publishing to modrinth!") +# else: +# print(f"publishing to smithed") +# pass + +def switch_platform(ctx: Context): + """Reads gm4.publish_to meta field to the cache, for subpipelines to check and run the correct publish plugin""" + publish_to: str = ctx.meta.get("gm4", {}).get("publish_to", None) + ctx.cache["currently_publishing"].json = {"publish_to": publish_to} def load_config(ctx: Context): """Loads relavent fields from each module's beet.yaml, without inheriting the pipeline and running plugins""" @@ -67,7 +78,6 @@ def load_config(ctx: Context): config.pipeline = ctx.meta.get("plugins", []) config.data_pack = PackConfig() config.resource_pack = PackConfig() - config.meta["gm4"] |= ctx.meta["gm4"] # pull in root gm4 meta # run the new list of plugins from beet-publish.yaml ctx.require( @@ -125,7 +135,10 @@ def publish_modrinth(ctx: Context, config: ManifestConfig, release_dir: Path, fi res = requests.patch(f"{MODRINTH_API}/version/{matching_version['id']}", headers={'Authorization': auth_token, 'User-Agent': USER_AGENT}, json={ "game_versions": config.minecraft }) - if not (200 <= res.status_code < 300): + if res.status_code == 401: + logger.critical(f"Authentication error, cancelling publish. Check token validity!: {res.status_code} {res.text}") + sys.exit(1) # quit the build and mark the github action as failed + elif not (200 <= res.status_code < 300): logger.warning(f"Failed to patch project versions: {res.status_code} {res.text}") return @@ -152,7 +165,10 @@ def publish_modrinth(ctx: Context, config: ManifestConfig, release_dir: Path, fi }), file_name: file_bytes, }) - if not (200 <= res.status_code < 300): + if res.status_code == 401: + logger.critical(f"Authentication error, cancelling publish. Check token validity!: {res.status_code} {res.text}") + sys.exit(1) # quit the build and mark the github action as failed + elif not (200 <= res.status_code < 300): logger.warning(f"Failed to publish new version version: {res.status_code} {res.text}") return logger.info(f"Successfully published {res.json()['name']}", extra={"gh_annotate_skip": True}) @@ -198,7 +214,10 @@ def publish_smithed(ctx: Context, config: ManifestConfig, file_name: str): "webPage": current_readme, }, }}) - if not (200 <= res.status_code < 300): + if res.status_code == 401: + logger.critical(f"Authentication error, cancelling publish. Check token validity!: {res.status_code} {res.text}") + sys.exit(1) # quit the build and mark the github action as failed + elif not (200 <= res.status_code < 300): logger.warning(f"Failed to update descripion: {res.status_code} {res.text}") logger.info(f"{ctx.project_name} {res.text}", extra={"gh_annotate_skip": True}) @@ -213,7 +232,10 @@ def publish_smithed(ctx: Context, config: ManifestConfig, file_name: str): "supports": config.minecraft } }) - if not (200 <= res.status_code < 300): + if res.status_code == 401: + logger.critical(f"Authentication error, cancelling publish. Check token validity!: {res.status_code} {res.text}") + sys.exit(1) # quit the build and mark the github action as failed + elif not (200 <= res.status_code < 300): logger.warning(f"Failed to patch project versions: {res.status_code} {res.text}") return @@ -231,7 +253,10 @@ def publish_smithed(ctx: Context, config: ManifestConfig, file_name: str): } } }) - if not (200 <= res.status_code < 300): + if res.status_code == 401: + logger.critical(f"Authentication error, cancelling publish. Check token validity!: {res.status_code} {res.text}") + sys.exit(1) # quit the build and mark the github action as failed + elif not (200 <= res.status_code < 300): logger.warning(f"Failed to permalink {project_id} version {prior_version_in_mc_version}: {res.status_code} {res.text}") else: logger.info(f"Permalinked {project_id} {prior_version_in_mc_version} to git history: {res.text}", extra={"gh_annotate_skip": True}) @@ -250,7 +275,10 @@ def publish_smithed(ctx: Context, config: ManifestConfig, file_name: str): "dependencies": [] }} ) - if not (200 <= res.status_code < 300): + if res.status_code == 401: + logger.critical(f"Authentication error, cancelling publish. Check token validity!: {res.status_code} {res.text}") + sys.exit(1) # quit the build and mark the github action as failed + elif not (200 <= res.status_code < 300): logger.warning(f"Failed to publish new version of {ctx.project_name}: {res.status_code} {res.text}") return logger.info(f"{ctx.project_name} {res.text}", extra={"gh_annotate_skip": True}) From f5b7f37c24e07136f5cb70ceab9667dc75dd517f Mon Sep 17 00:00:00 2001 From: SpecialBuilder Date: Sun, 6 Sep 2026 21:57:55 -0400 Subject: [PATCH 08/13] Aggregate logs from all jobs to form github summary --- .github/workflows/main.yml | 44 +++++++++++++++++++++++++++++++++++--- .gitignore | 1 + beet-summarize.yaml | 2 ++ gm4/plugins/annotations.py | 37 +++++++++++++++++++++++++++++--- gm4/plugins/publish.py | 12 +++++------ 5 files changed, 84 insertions(+), 12 deletions(-) create mode 100644 beet-summarize.yaml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 51fe00b7cb..2e3a41e29f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -86,6 +86,16 @@ jobs: path: ${{ github.workspace }}/.beet_cache/ include-hidden-files: true + - name: Rename log file + run: mv ${{ github.workspace }}/logs/summary_logs.pkl ${{ github.workspace }}/logs/build_log.pkl + + - name: Upload logs as artifact + if: github.event_name != 'pull_request' + uses: actions/upload-artifact@v7 + with: + path: ${{ github.workspace }}/logs/build_log.pkl + archive: false + test: strategy: fail-fast: false @@ -151,8 +161,8 @@ jobs: strategy: matrix: platform: [modrinth, smithed] - concurrency: - group: release + # concurrency: + # group: release name: 'publish-${{ matrix.platform }}' steps: - uses: actions/checkout@v7 @@ -180,9 +190,37 @@ jobs: BEET_SMITHED_TOKEN: ${{ secrets.SMITHED_TOKEN }} LOG_LEVEL: ${{ runner.debug == 1 && 'DEBUG' || 'INFO'}} + - name: Rename log file + run: mv ${{ github.workspace }}/logs/summary_logs.pkl ${{ github.workspace }}/logs/publish_${{ matrix.platform }}_log.pkl + + - name: Upload logs as artifact + uses: actions/upload-artifact@v7 + with: + path: ${{ github.workspace }}/logs/publish_${{ matrix.platform }}_log.pkl + archive: false + summarize: runs-on: ubuntu-24.04 if: github.event_name != 'pull_request' needs: publish steps: - - uses: action/checkout@v7 + - uses: actions/checkout@v7 + + - name: Restore .beet_cache from artifact + uses: actions/download-artifact@v7 + with: + name: .beet_cache + path: ${{ github.workspace }}/.beet_cache + + - name: Restore log files from artifacts + uses: actions/download-artifact@v8 + with: + pattern: '*_log.pkl' + path: ${{ github.workspace }}/logs + merge-multiple: true + + - name: Set up uv + uses: astral-sh/setup-uv@v7 + + - name: Summarize build logs + run: uv run beet -p beet-summarize.yaml build diff --git a/.gitignore b/.gitignore index f73c2cb7b3..443250364a 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,5 @@ __pycache__/ ### Beet ### out/ release/ +logs/ .beet_cache/ diff --git a/beet-summarize.yaml b/beet-summarize.yaml new file mode 100644 index 0000000000..6fa45f5106 --- /dev/null +++ b/beet-summarize.yaml @@ -0,0 +1,2 @@ +pipeline: + - gm4.plugins.annotations.load_and_summarize diff --git a/gm4/plugins/annotations.py b/gm4/plugins/annotations.py index 9936f64457..4fc6551da6 100644 --- a/gm4/plugins/annotations.py +++ b/gm4/plugins/annotations.py @@ -5,6 +5,7 @@ from functools import partial from pathlib import Path from typing import Any +import pickle from beet import Context, ProjectCache @@ -34,9 +35,28 @@ def filter(record: logging.LogRecord): logging.getLogger("gm4.publish").addHandler(sum_handler) logging.getLogger("gm4.manifest.update_patch").addHandler(sum_handler) - # after the whole build, flush the stored records and form the markdown summary + # after the whole build, flush the stored records to file for the later markdown summary action job yield - sum_handler.flush() + # FIXME sys.exit causes this process to cancel before being completed? Reconsider how to mark job as failed + sum_handler.flush_to_pickle() + +def load_and_summarize(ctx: Context): + """Loads log entries from previous gh step/job to aggregate""" + sum_handler = SummaryHandler(1000, ctx.cache) + + print(os.listdir("logs")) + + log_dir = Path("logs") + for log_file in log_dir.iterdir(): + print(f"loaded {log_file}") + with open(log_file, 'rb') as f: + log_buffer: list[logging.LogRecord] = pickle.load(f) + for log_entry in log_buffer: + print(f"{log_entry=}") + sum_handler.emit(log_entry) + + sum_handler.flush_to_summary() + LEVEL_CONVERSION = { logging.DEBUG: "debug", @@ -85,7 +105,7 @@ def __init__(self, capacity: int, beet_cache: ProjectCache): self.beet_cache = beet_cache self.summary_created = False - def flush(self): + def flush_to_summary(self): summary_entries: dict[str, Any] = {} this_manifest = ManifestCacheModel.model_validate(self.beet_cache["gm4_manifest"].json) @@ -132,6 +152,7 @@ def flush(self): table += f"\n {entry['name']} | {entry['ver_update']} | {nested_table}" summary = "# :rocket: Build Deployment Summary :rocket:\n"+table + print(summary) if not self.summary_created: env_file = os.getenv("GITHUB_STEP_SUMMARY") @@ -141,6 +162,16 @@ def flush(self): self.summary_created = True self.buffer.clear() + def flush_to_pickle(self): + """Writes buffer of log entries to file, for a later gh action step to aggregate into the summary file""" + print("flushing logs") + log_dir = Path("logs") + log_file = log_dir/"summary_logs.pkl" + os.makedirs(log_dir, exist_ok=True) + with open(log_file, "wb") as f: + pickle.dump(self.buffer, f) + self.buffer.clear() + def add_module_dir_to_diagnostics(ctx: Context): diff --git a/gm4/plugins/publish.py b/gm4/plugins/publish.py index b7a1321989..6d6d230a64 100644 --- a/gm4/plugins/publish.py +++ b/gm4/plugins/publish.py @@ -137,7 +137,7 @@ def publish_modrinth(ctx: Context, config: ManifestConfig, release_dir: Path, fi }) if res.status_code == 401: logger.critical(f"Authentication error, cancelling publish. Check token validity!: {res.status_code} {res.text}") - sys.exit(1) # quit the build and mark the github action as failed + # sys.exit(1) # quit the build and mark the github action as failed elif not (200 <= res.status_code < 300): logger.warning(f"Failed to patch project versions: {res.status_code} {res.text}") return @@ -167,7 +167,7 @@ def publish_modrinth(ctx: Context, config: ManifestConfig, release_dir: Path, fi }) if res.status_code == 401: logger.critical(f"Authentication error, cancelling publish. Check token validity!: {res.status_code} {res.text}") - sys.exit(1) # quit the build and mark the github action as failed + # sys.exit(1) # quit the build and mark the github action as failed elif not (200 <= res.status_code < 300): logger.warning(f"Failed to publish new version version: {res.status_code} {res.text}") return @@ -216,7 +216,7 @@ def publish_smithed(ctx: Context, config: ManifestConfig, file_name: str): }}) if res.status_code == 401: logger.critical(f"Authentication error, cancelling publish. Check token validity!: {res.status_code} {res.text}") - sys.exit(1) # quit the build and mark the github action as failed + # sys.exit(1) # quit the build and mark the github action as failed elif not (200 <= res.status_code < 300): logger.warning(f"Failed to update descripion: {res.status_code} {res.text}") logger.info(f"{ctx.project_name} {res.text}", extra={"gh_annotate_skip": True}) @@ -234,7 +234,7 @@ def publish_smithed(ctx: Context, config: ManifestConfig, file_name: str): }) if res.status_code == 401: logger.critical(f"Authentication error, cancelling publish. Check token validity!: {res.status_code} {res.text}") - sys.exit(1) # quit the build and mark the github action as failed + # sys.exit(1) # quit the build and mark the github action as failed elif not (200 <= res.status_code < 300): logger.warning(f"Failed to patch project versions: {res.status_code} {res.text}") return @@ -255,7 +255,7 @@ def publish_smithed(ctx: Context, config: ManifestConfig, file_name: str): }) if res.status_code == 401: logger.critical(f"Authentication error, cancelling publish. Check token validity!: {res.status_code} {res.text}") - sys.exit(1) # quit the build and mark the github action as failed + # sys.exit(1) # quit the build and mark the github action as failed elif not (200 <= res.status_code < 300): logger.warning(f"Failed to permalink {project_id} version {prior_version_in_mc_version}: {res.status_code} {res.text}") else: @@ -277,7 +277,7 @@ def publish_smithed(ctx: Context, config: ManifestConfig, file_name: str): ) if res.status_code == 401: logger.critical(f"Authentication error, cancelling publish. Check token validity!: {res.status_code} {res.text}") - sys.exit(1) # quit the build and mark the github action as failed + # sys.exit(1) # quit the build and mark the github action as failed elif not (200 <= res.status_code < 300): logger.warning(f"Failed to publish new version of {ctx.project_name}: {res.status_code} {res.text}") return From c151665e5a945b761ca84fa087368d5347e4ae68 Mon Sep 17 00:00:00 2001 From: SpecialBuilder Date: Mon, 7 Sep 2026 22:34:34 -0400 Subject: [PATCH 09/13] Cleanup action debug print statements --- gm4/plugins/annotations.py | 6 ------ gm4/plugins/publish.py | 3 --- test_plugin.py | 7 ------- 3 files changed, 16 deletions(-) delete mode 100644 test_plugin.py diff --git a/gm4/plugins/annotations.py b/gm4/plugins/annotations.py index 4fc6551da6..3655539a01 100644 --- a/gm4/plugins/annotations.py +++ b/gm4/plugins/annotations.py @@ -44,15 +44,11 @@ def load_and_summarize(ctx: Context): """Loads log entries from previous gh step/job to aggregate""" sum_handler = SummaryHandler(1000, ctx.cache) - print(os.listdir("logs")) - log_dir = Path("logs") for log_file in log_dir.iterdir(): - print(f"loaded {log_file}") with open(log_file, 'rb') as f: log_buffer: list[logging.LogRecord] = pickle.load(f) for log_entry in log_buffer: - print(f"{log_entry=}") sum_handler.emit(log_entry) sum_handler.flush_to_summary() @@ -152,7 +148,6 @@ def flush_to_summary(self): table += f"\n {entry['name']} | {entry['ver_update']} | {nested_table}" summary = "# :rocket: Build Deployment Summary :rocket:\n"+table - print(summary) if not self.summary_created: env_file = os.getenv("GITHUB_STEP_SUMMARY") @@ -164,7 +159,6 @@ def flush_to_summary(self): def flush_to_pickle(self): """Writes buffer of log entries to file, for a later gh action step to aggregate into the summary file""" - print("flushing logs") log_dir = Path("logs") log_file = log_dir/"summary_logs.pkl" os.makedirs(log_dir, exist_ok=True) diff --git a/gm4/plugins/publish.py b/gm4/plugins/publish.py index 6d6d230a64..74b696f1ed 100644 --- a/gm4/plugins/publish.py +++ b/gm4/plugins/publish.py @@ -32,8 +32,6 @@ def beet_default(ctx: Context): `BEET_SMITHED_TOKEN` environment variable is set, will try to publish a new version to Smithed if it doesn't already exist.""" - print(f"running publish on {ctx.project_id}") - version_dir = os.getenv("VERSION", "26.2") release_dir = Path("release") / version_dir @@ -44,7 +42,6 @@ def beet_default(ctx: Context): config = ctx.validate("gm4", ManifestConfig) publish_to = ctx.cache["currently_publishing"].json.get("publish_to", None) - print(publish_to) # publish to download platforms, based on which gh job this is if publish_to == "smithed": diff --git a/test_plugin.py b/test_plugin.py deleted file mode 100644 index 9351bc5453..0000000000 --- a/test_plugin.py +++ /dev/null @@ -1,7 +0,0 @@ -from beet import Context - -def say_hello(ctx: Context): - print("Hello Action World") - -def print_data(ctx: Context): - print(f"\nretrieved project {ctx.project_name}, {ctx.project_description}") From 8710727ba91ebe6d3c4503eab5377b0ff98173b0 Mon Sep 17 00:00:00 2001 From: SpecialBuilder Date: Mon, 7 Sep 2026 23:12:42 -0400 Subject: [PATCH 10/13] Include libraries in publish for smithed --- beet-publish.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beet-publish.yaml b/beet-publish.yaml index e9c722ca51..07483e6bdd 100644 --- a/beet-publish.yaml +++ b/beet-publish.yaml @@ -2,7 +2,7 @@ pipeline: - gm4.plugins.annotations - gm4.plugins.publish.switch_platform - - broadcast: gm4_bat_grenades + - broadcast: [gm4_*, lib_*] pipeline: - gm4.plugins.publish.load_config meta: From d333db54299dcf27c22417a8e2ac2f58bee51ee9 Mon Sep 17 00:00:00 2001 From: SpecialBuilder Date: Sun, 13 Sep 2026 14:10:41 -0400 Subject: [PATCH 11/13] Intra-job artifacts expire after 2 days --- .github/workflows/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2e3a41e29f..561f67b059 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -85,6 +85,7 @@ jobs: name: .beet_cache path: ${{ github.workspace }}/.beet_cache/ include-hidden-files: true + retention-days: 2 - name: Rename log file run: mv ${{ github.workspace }}/logs/summary_logs.pkl ${{ github.workspace }}/logs/build_log.pkl @@ -95,6 +96,7 @@ jobs: with: path: ${{ github.workspace }}/logs/build_log.pkl archive: false + retention-days: 2 test: strategy: @@ -198,6 +200,7 @@ jobs: with: path: ${{ github.workspace }}/logs/publish_${{ matrix.platform }}_log.pkl archive: false + retention-days: 2 summarize: runs-on: ubuntu-24.04 From 31f8ad9fa5232137770a631c22954d5aaac19715 Mon Sep 17 00:00:00 2001 From: SpecialBuilder Date: Sun, 13 Sep 2026 15:44:53 -0400 Subject: [PATCH 12/13] Use meta.json with PRs to preview version patch changes --- .github/workflows/main.yml | 9 +++------ gm4/plugins/annotations.py | 12 +++++++++--- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 561f67b059..44f96c197c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,7 +19,6 @@ jobs: - uses: actions/checkout@v7 - name: Checkout release branch - if: github.event_name != 'pull_request' uses: actions/checkout@v7 with: ref: release @@ -79,7 +78,6 @@ jobs: branch: release - name: Upload .beet_cache as artifact - if: github.event_name != 'pull_request' uses: actions/upload-artifact@v7 with: name: .beet_cache @@ -91,7 +89,6 @@ jobs: run: mv ${{ github.workspace }}/logs/summary_logs.pkl ${{ github.workspace }}/logs/build_log.pkl - name: Upload logs as artifact - if: github.event_name != 'pull_request' uses: actions/upload-artifact@v7 with: path: ${{ github.workspace }}/logs/build_log.pkl @@ -204,8 +201,8 @@ jobs: summarize: runs-on: ubuntu-24.04 - if: github.event_name != 'pull_request' - needs: publish + needs: [build, publish] + if: ${{ always() }} steps: - uses: actions/checkout@v7 @@ -226,4 +223,4 @@ jobs: uses: astral-sh/setup-uv@v7 - name: Summarize build logs - run: uv run beet -p beet-summarize.yaml build + run: uv run beet -p beet-summarize.yaml -s meta.gm4.summarize_type=${{ github.event_name != 'pull_request' && 'release' || 'pull_request' }} build diff --git a/gm4/plugins/annotations.py b/gm4/plugins/annotations.py index 3655539a01..50637fbc43 100644 --- a/gm4/plugins/annotations.py +++ b/gm4/plugins/annotations.py @@ -51,7 +51,13 @@ def load_and_summarize(ctx: Context): for log_entry in log_buffer: sum_handler.emit(log_entry) - sum_handler.flush_to_summary() + summary_title = "" + if (sum_type:=ctx.meta.get('gm4', {}).get('summarize_type')) == 'release': + summary_title = "Build Deployment Summary" + elif sum_type == 'pull_request': + summary_title = "Pull Request Deployment Preview" + + sum_handler.flush_to_summary(summary_title) LEVEL_CONVERSION = { @@ -101,7 +107,7 @@ def __init__(self, capacity: int, beet_cache: ProjectCache): self.beet_cache = beet_cache self.summary_created = False - def flush_to_summary(self): + def flush_to_summary(self, summary_title: str): summary_entries: dict[str, Any] = {} this_manifest = ManifestCacheModel.model_validate(self.beet_cache["gm4_manifest"].json) @@ -147,7 +153,7 @@ def flush_to_summary(self): table += f"\n {entry['name']} | {entry['ver_update']} | {nested_table}" - summary = "# :rocket: Build Deployment Summary :rocket:\n"+table + summary = f"# :rocket: {summary_title} :rocket:\n"+table if not self.summary_created: env_file = os.getenv("GITHUB_STEP_SUMMARY") From cb0a8df65d9019b534aa663953abc078b2ebf8c5 Mon Sep 17 00:00:00 2001 From: SpecialBuilder Date: Sun, 13 Sep 2026 16:55:57 -0400 Subject: [PATCH 13/13] Allow logging to mark action failed without sys.exit --- beet-publish.yaml | 1 + gm4/plugins/annotations.py | 1 - gm4/plugins/publish.py | 45 +++++++++++++++++++++++--------------- 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/beet-publish.yaml b/beet-publish.yaml index 07483e6bdd..21632fa28e 100644 --- a/beet-publish.yaml +++ b/beet-publish.yaml @@ -1,4 +1,5 @@ pipeline: + - gm4.plugins.publish.check_gh_action_failed - gm4.plugins.annotations - gm4.plugins.publish.switch_platform diff --git a/gm4/plugins/annotations.py b/gm4/plugins/annotations.py index 50637fbc43..d9ae07209d 100644 --- a/gm4/plugins/annotations.py +++ b/gm4/plugins/annotations.py @@ -37,7 +37,6 @@ def filter(record: logging.LogRecord): # after the whole build, flush the stored records to file for the later markdown summary action job yield - # FIXME sys.exit causes this process to cancel before being completed? Reconsider how to mark job as failed sum_handler.flush_to_pickle() def load_and_summarize(ctx: Context): diff --git a/gm4/plugins/publish.py b/gm4/plugins/publish.py index 74b696f1ed..734d71e293 100644 --- a/gm4/plugins/publish.py +++ b/gm4/plugins/publish.py @@ -42,6 +42,10 @@ def beet_default(ctx: Context): config = ctx.validate("gm4", ManifestConfig) publish_to = ctx.cache["currently_publishing"].json.get("publish_to", None) + + # check if build already failed, and do not continue attempting to publish + if ctx.cache['gh_action_failed'].json.get('gh_action_failed', False): + return # publish to download platforms, based on which gh job this is if publish_to == "smithed": @@ -49,18 +53,6 @@ def beet_default(ctx: Context): elif publish_to == "modrinth": publish_modrinth(ctx, config, release_dir, file_name) -# def dev_test_1(ctx: Context): -# # print(f"beet cache restored as: {list(ctx.cache.keys())}") - -# # print(f"gm4_manifest is :{ctx.cache["gm4_manifest"]}") -# print(ctx.meta.get("gm4")) - -# if ctx.meta.get("gm4",{}).get("publish_to", None) == "modrinth": -# print(f"publishing to modrinth!") -# else: -# print(f"publishing to smithed") -# pass - def switch_platform(ctx: Context): """Reads gm4.publish_to meta field to the cache, for subpipelines to check and run the correct publish plugin""" publish_to: str = ctx.meta.get("gm4", {}).get("publish_to", None) @@ -86,6 +78,17 @@ def load_config(ctx: Context): ) ) +def mark_gh_action_failed(ctx: Context): + """Marks beet cache that pipeline failed, allowing for cleanup steps (like annotations)""" + ctx.cache['gh_action_failed'].json = {'gh_action_failed': True} + +def check_gh_action_failed(ctx: Context): + """Checks beet cache and marks github action runner as 'failed'""" + yield + is_failed = ctx.cache['gh_action_failed'].json.get('gh_action_failed', False) + if is_failed: + sys.exit(1) + def publish_modrinth(ctx: Context, config: ManifestConfig, release_dir: Path, file_name: str): '''Attempts to publish pack to modrinth''' auth_token = os.getenv(MODRINTH_AUTH_KEY, None) @@ -134,7 +137,8 @@ def publish_modrinth(ctx: Context, config: ManifestConfig, release_dir: Path, fi }) if res.status_code == 401: logger.critical(f"Authentication error, cancelling publish. Check token validity!: {res.status_code} {res.text}") - # sys.exit(1) # quit the build and mark the github action as failed + ctx.require(mark_gh_action_failed) + return elif not (200 <= res.status_code < 300): logger.warning(f"Failed to patch project versions: {res.status_code} {res.text}") return @@ -164,7 +168,8 @@ def publish_modrinth(ctx: Context, config: ManifestConfig, release_dir: Path, fi }) if res.status_code == 401: logger.critical(f"Authentication error, cancelling publish. Check token validity!: {res.status_code} {res.text}") - # sys.exit(1) # quit the build and mark the github action as failed + ctx.require(mark_gh_action_failed) + return elif not (200 <= res.status_code < 300): logger.warning(f"Failed to publish new version version: {res.status_code} {res.text}") return @@ -213,7 +218,8 @@ def publish_smithed(ctx: Context, config: ManifestConfig, file_name: str): }}) if res.status_code == 401: logger.critical(f"Authentication error, cancelling publish. Check token validity!: {res.status_code} {res.text}") - # sys.exit(1) # quit the build and mark the github action as failed + ctx.require(mark_gh_action_failed) + return elif not (200 <= res.status_code < 300): logger.warning(f"Failed to update descripion: {res.status_code} {res.text}") logger.info(f"{ctx.project_name} {res.text}", extra={"gh_annotate_skip": True}) @@ -231,7 +237,8 @@ def publish_smithed(ctx: Context, config: ManifestConfig, file_name: str): }) if res.status_code == 401: logger.critical(f"Authentication error, cancelling publish. Check token validity!: {res.status_code} {res.text}") - # sys.exit(1) # quit the build and mark the github action as failed + ctx.require(mark_gh_action_failed) + return elif not (200 <= res.status_code < 300): logger.warning(f"Failed to patch project versions: {res.status_code} {res.text}") return @@ -252,7 +259,8 @@ def publish_smithed(ctx: Context, config: ManifestConfig, file_name: str): }) if res.status_code == 401: logger.critical(f"Authentication error, cancelling publish. Check token validity!: {res.status_code} {res.text}") - # sys.exit(1) # quit the build and mark the github action as failed + ctx.require(mark_gh_action_failed) + return elif not (200 <= res.status_code < 300): logger.warning(f"Failed to permalink {project_id} version {prior_version_in_mc_version}: {res.status_code} {res.text}") else: @@ -274,7 +282,8 @@ def publish_smithed(ctx: Context, config: ManifestConfig, file_name: str): ) if res.status_code == 401: logger.critical(f"Authentication error, cancelling publish. Check token validity!: {res.status_code} {res.text}") - # sys.exit(1) # quit the build and mark the github action as failed + ctx.require(mark_gh_action_failed) + return elif not (200 <= res.status_code < 300): logger.warning(f"Failed to publish new version of {ctx.project_name}: {res.status_code} {res.text}") return