[WIP] Metsr Sim CoSimulator Implementation

.coveragerc

@@ -26,4 +26,4 @@ exclude_lines =
 
 ignore_errors = True
 show_missing = True
-precision = 2
+precision = 2

.git-blame-ignore-revs

@@ -5,3 +5,5 @@
 c6c83f95ff370b75c3ee7130dbd8071bfe8b285a
 # Cleaned up test quote spacing
 995cd182924dc9e3dbbc941c5b75454ea0cdaaca
+# Ran black on entire codebase 
+cb51d08fda00df5588a418df42d9d652472f505f

.github/scripts/run-ssm-command.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+INSTANCE_ID="${1:?Missing instance ID}"
+EXECUTION_TIMEOUT="${2:-3600}"
+
+# Build the remote shell script as a JSON array for AWS-RunShellScript.
+commands=$(jq -Rs -c 'split("\n") | if .[-1] == "" then .[:-1] else . end')
+
+# Run the script on the EC2 instance through SSM.
+command_id=$(aws ssm send-command \
+  --instance-ids "$INSTANCE_ID" \
+  --document-name AWS-RunShellScript \
+  --parameters "commands=$commands,executionTimeout=$EXECUTION_TIMEOUT" \
+  --query 'Command.CommandId' \
+  --output text)
+
+# Poll SSM until the remote command finishes, then print its output.
+while true; do
+  status=$(aws ssm get-command-invocation \
+    --command-id "$command_id" \
+    --instance-id "$INSTANCE_ID" \
+    --query 'Status' \
+    --output text)
+
+  case "$status" in
+    Success)
+      break
+      ;;
+    Failed|Cancelled|TimedOut|Cancelling)
+      aws ssm get-command-invocation --command-id "$command_id" --instance-id "$INSTANCE_ID" --query 'StandardOutputContent' --output text
+      aws ssm get-command-invocation --command-id "$command_id" --instance-id "$INSTANCE_ID" --query 'StandardErrorContent' --output text
+      exit 1
+      ;;
+    Pending|InProgress|Delayed)
+      sleep 10
+      ;;
+  esac
+done
+
+aws ssm get-command-invocation \
+  --command-id "$command_id" \
+  --instance-id "$INSTANCE_ID" \
+  --query 'StandardOutputContent' \
+  --output text

.github/workflows/check-formatting.yml

@@ -5,15 +5,24 @@ on:
   pull_request:
   workflow_call:
 
+permissions:
+  contents: read
+
 jobs:
   check:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
+        with:
+          persist-credentials: false
 
       - name: Run black to check formatting
-        uses: psf/black@stable
+        uses: psf/black@87928e6d6761a4a6d22250e1fee5601b3998086e
+        with:
+          version: "25.1.0"
 
       - name: Run isort to check import order
-        uses: isort/isort-action@v1
+        uses: isort/isort-action@24d8a7a51d33ca7f36c3f23598dafa33f7071326
+        with:
+          isort-version: "5.12.0"

.github/workflows/run-benchmarks.yml

@@ -0,0 +1,130 @@
+name: Run Benchmarks
+
+on:
+  issue_comment:
+    types: [created]
+
+jobs:
+  check-permission:
+    runs-on: ubuntu-latest
+    permissions:
+        contents: none
+    if: >
+      github.event.issue.pull_request &&
+      startsWith(github.event.comment.body, '!benchmark')
+    steps:
+      - name: Check user's permission level
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea
+        with:
+          script: |
+            const username = context.payload.comment.user.login;
+            const { data } = await github.rest.repos.getCollaboratorPermissionLevel({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              username: username,
+            });
+            const userRole = data.role_name;
+            const allowedRoles = ["admin", "write", "maintain", "triage"];
+            console.log(`User ${username} has role: ${userRole}`);
+            // Fail the job if user doesn't have allowed role
+            if (!allowedRoles.includes(userRole)) {
+              throw new Error(`User does not have sufficient permissions to run the benchmarks.`);
+            }
+
+  run-benchmarks:
+    needs: check-permission
+    runs-on: ubuntu-latest
+    permissions:
+        pull-requests: write
+    steps:
+      - name: Benchmarks Started Comment
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea
+        with:
+          script: |
+            const commentId = context.payload.comment.id;
+            const originalBody = context.payload.comment.body;
+            // Append a message to indicate benchmarks have started running
+            const newBody = `${originalBody}\n\n🚀 Benchmarks are running...`;
+            // Update the comment with the new body text
+            await github.rest.issues.updateComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              comment_id: commentId,
+              body: newBody,
+            });
+
+      - name: Fetch PR Details
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea
+        with:
+            script: |
+              const pr = await github.rest.pulls.get({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                pull_number: context.issue.number
+              });
+
+              const body = context.payload.comment.body.trim();
+              const requestedSha = body.split(/\s+/)[1];
+
+              core.exportVariable('BASE_SHA', pr.data.base.sha);
+              core.exportVariable('COMPARE_SHA', requestedSha);
+
+      - name: Print PR Details
+        run: |
+          echo "COMPARE_SHA: $COMPARE_SHA"
+          echo "BASE_SHA: $BASE_SHA"
+
+      - name: Checkout base branch
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
+        with:
+          ref: ${{ env.BASE_SHA }}
+          path: base
+          persist-credentials: false
+
+      - name: Checkout compare branch
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
+        with:
+          ref: ${{ env.COMPARE_SHA }}
+          path: compare
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38
+        with:
+          python-version: '3.x'
+
+      - name: Install pyperf
+        run: |
+          python -m pip install --upgrade pip
+          pip install pyperf
+
+      - name: Run benchmark on base branch
+        working-directory: base
+        run: python tools/benchmarking/ci/run_benchmarks.py -o ../base.json
+
+      - name: Run benchmark on compare branch
+        working-directory: compare
+        run: python tools/benchmarking/ci/run_benchmarks.py -o ../head.json
+
+      - name: Compare benchmarks results
+        id: compare
+        run: |
+          python -m pyperf compare_to --table --table-format=md -G --min-speed=10 base.json head.json > comparison.txt
+          echo "Comparison results:"
+          cat comparison.txt
+
+      - name: Post benchmark results
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea
+        with:
+          script: |
+            const fs = require('fs');
+            const result = fs.readFileSync('comparison.txt', 'utf8');
+            const commentId = context.payload.comment.id;
+            const originalBody = context.payload.comment.body;
+            const newBody = `${originalBody}\n\n🚀 Benchmarks completed.\n\n${result}`;
+            await github.rest.issues.updateComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              comment_id: commentId,
+              body: newBody,
+            });

.github/workflows/run-coverage.yml

@@ -17,31 +17,36 @@ on:
         description: Git ref on which to run the tests.
         type: string
 
+permissions:
+  contents: read
+
 jobs:
   coverage:
     strategy:
       fail-fast: true
       matrix:
-        python-version: ["3.11"]
+        python-version: ["3.12"]
         os: [ubuntu-latest]
         extras: ["test-full"]
     runs-on: ${{ matrix.os }}
 
     steps:
     - name: Checkout given ref
-      uses: actions/checkout@v3
+      uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
       if: inputs.ref != ''
       with:
         ref: ${{ inputs.ref }}
+        persist-credentials: false
 
     - name: Checkout current branch
-      uses: actions/checkout@v3
+      uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
       if: inputs.ref == ''
       with:
         ref: ${{ github.ref }}
+        persist-credentials: false
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38
       with:
         python-version: ${{ matrix.python-version }}
         cache: 'pip'
@@ -57,9 +62,9 @@ jobs:
     - name: Run and report code coverage
       run: |
         pytest --cov --cov-report json
-    
+
     - name: Upload coverage reports to Codecov
-      uses: codecov/codecov-action@v4.0.1
+      uses: codecov/codecov-action@0565863a31f2c772f9f0395002a31e3f06189574
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
-        slug: BerkeleyLearnVerify/Scenic
+        slug: BerkeleyLearnVerify/Scenic