Split publish action into separate github job for better reporting

.github/workflows/main.yml

@@ -19,7 +19,6 @@ jobs:
       - uses: actions/checkout@v7
 
       - name: Checkout release branch
-        if: github.event_name != 'pull_request'
         uses: actions/checkout@v7
         with:
           ref: release
@@ -78,6 +77,24 @@ jobs:
           directory: release
           branch: release
 
+      - name: Upload .beet_cache as artifact
+        uses: actions/upload-artifact@v7
+        with:
+          name: .beet_cache
+          path: ${{ github.workspace }}/.beet_cache/
+          include-hidden-files: true
+          retention-days: 2
+
+      - name: Rename log file
+        run: mv ${{ github.workspace }}/logs/summary_logs.pkl ${{ github.workspace }}/logs/build_log.pkl
+
+      - name: Upload logs as artifact
+        uses: actions/upload-artifact@v7
+        with:
+          path: ${{ github.workspace }}/logs/build_log.pkl
+          archive: false
+          retention-days: 2
+
   test:
     strategy:
       fail-fast: false
@@ -135,3 +152,75 @@ jobs:
         with:
           name: 'Test world in ${{ matrix.version }} for ${{ github.sha }}'
           path: ${{ github.workspace }}/world/
+
+  publish:
+    runs-on: ubuntu-24.04
+    if: github.event_name != 'pull_request'
+    needs: build
+    strategy:
+      matrix:
+        platform: [modrinth, smithed]
+    # concurrency:
+    #   group: release
+    name: 'publish-${{ matrix.platform }}'
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Checkout release branch
+        uses: actions/checkout@v7
+        with:
+          ref: release
+          path: release
+
+      - name: Restore .beet_cache from artifact
+        uses: actions/download-artifact@v7
+        with:
+          name: .beet_cache
+          path: ${{ github.workspace }}/.beet_cache
+
+      # TODO use cache from build? 
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v7
+
+      - name: Publish all modules
+        run: uv run beet -p beet-publish.yaml -l ${{ env.LOG_LEVEL }} -s meta.gm4.publish_to=${{ matrix.platform }} build
+        env:
+          BEET_MODRINTH_TOKEN: ${{ secrets.MODRINTH_TOKEN }}
+          BEET_SMITHED_TOKEN: ${{ secrets.SMITHED_TOKEN }}
+          LOG_LEVEL: ${{ runner.debug == 1 && 'DEBUG' || 'INFO'}}
+
+      - name: Rename log file
+        run: mv ${{ github.workspace }}/logs/summary_logs.pkl ${{ github.workspace }}/logs/publish_${{ matrix.platform }}_log.pkl
+
+      - name: Upload logs as artifact
+        uses: actions/upload-artifact@v7
+        with:
+          path: ${{ github.workspace }}/logs/publish_${{ matrix.platform }}_log.pkl
+          archive: false
+          retention-days: 2
+
+  summarize:
+    runs-on: ubuntu-24.04
+    needs: [build, publish]
+    if: ${{ always() }}
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Restore .beet_cache from artifact
+        uses: actions/download-artifact@v7
+        with:
+          name: .beet_cache
+          path: ${{ github.workspace }}/.beet_cache
+
+      - name: Restore log files from artifacts
+        uses: actions/download-artifact@v8
+        with:
+          pattern: '*_log.pkl'
+          path: ${{ github.workspace }}/logs
+          merge-multiple: true
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v7
+
+      - name: Summarize build logs
+        run: uv run beet -p beet-summarize.yaml -s meta.gm4.summarize_type=${{ github.event_name != 'pull_request' && 'release' || 'pull_request' }} build

.gitignore

@@ -15,4 +15,5 @@ __pycache__/
 ### Beet ###
 out/
 release/
+logs/
 .beet_cache/

beet-publish.yaml

@@ -0,0 +1,12 @@
+pipeline:
+  - gm4.plugins.publish.check_gh_action_failed
+  - gm4.plugins.annotations
+  - gm4.plugins.publish.switch_platform
+
+  - broadcast: [gm4_*, lib_*]
+    pipeline:
+      - gm4.plugins.publish.load_config
+    meta:
+      plugins:
+        - gm4.plugins.readme_generator
+        - gm4.plugins.publish

beet-summarize.yaml

@@ -0,0 +1,2 @@
+pipeline: 
+  - gm4.plugins.annotations.load_and_summarize

gm4/plugins/annotations.py

@@ -5,6 +5,7 @@
 from functools import partial
 from pathlib import Path
 from typing import Any
+import pickle
 
 from beet import Context, ProjectCache
 
@@ -31,12 +32,32 @@ def filter(record: logging.LogRecord):
 
     # summary handler holds onto certain records until the exit phase when it emits to a markdown summary
     sum_handler = SummaryHandler(1000, ctx.cache)
-    logging.getLogger("gm4.output").addHandler(sum_handler)
+    logging.getLogger("gm4.publish").addHandler(sum_handler)
     logging.getLogger("gm4.manifest.update_patch").addHandler(sum_handler)
 
-    # after the whole build, flush the stored records and form the markdown summary
+    # after the whole build, flush the stored records to file for the later markdown summary action job
     yield
-    sum_handler.flush()
+    sum_handler.flush_to_pickle()
+
+def load_and_summarize(ctx: Context):
+    """Loads log entries from previous gh step/job to aggregate"""
+    sum_handler = SummaryHandler(1000, ctx.cache)
+
+    log_dir = Path("logs")
+    for log_file in log_dir.iterdir():
+        with open(log_file, 'rb') as f:
+            log_buffer: list[logging.LogRecord] = pickle.load(f)
+        for log_entry in log_buffer:
+            sum_handler.emit(log_entry)
+
+    summary_title = ""
+    if (sum_type:=ctx.meta.get('gm4', {}).get('summarize_type')) == 'release':
+        summary_title = "Build Deployment Summary"
+    elif sum_type == 'pull_request':
+        summary_title = "Pull Request Deployment Preview"
+
+    sum_handler.flush_to_summary(summary_title)
+
 
 LEVEL_CONVERSION = {
     logging.DEBUG: "debug",
@@ -85,7 +106,7 @@ def __init__(self, capacity: int, beet_cache: ProjectCache):
         self.beet_cache = beet_cache
         self.summary_created = False
 
-    def flush(self):
+    def flush_to_summary(self, summary_title: str):
         summary_entries: dict[str, Any] = {}
 
         this_manifest = ManifestCacheModel.model_validate(self.beet_cache["gm4_manifest"].json)
@@ -131,7 +152,7 @@ def flush(self):
 
             table += f"\n {entry['name']} | {entry['ver_update']} | {nested_table}"
 
-        summary = "# :rocket: Build Deployment Summary :rocket:\n"+table
+        summary = f"# :rocket: {summary_title} :rocket:\n"+table
 
         if not self.summary_created:
             env_file = os.getenv("GITHUB_STEP_SUMMARY")
@@ -141,6 +162,15 @@ def flush(self):
                 self.summary_created = True
         self.buffer.clear()
 
+    def flush_to_pickle(self):
+        """Writes buffer of log entries to file, for a later gh action step to aggregate into the summary file"""
+        log_dir = Path("logs")
+        log_file = log_dir/"summary_logs.pkl"
+        os.makedirs(log_dir, exist_ok=True)
+        with open(log_file, "wb") as f:
+            pickle.dump(self.buffer, f)
+        self.buffer.clear()
+
 
 
 def add_module_dir_to_diagnostics(ctx: Context):