ci: retry Project item visibility after add (#2068)

.github/workflows/project-intake.yml

@@ -176,6 +176,77 @@ jobs:
             return "$status"
           }
 
+          project_intake_graphql_item_after_add() {
+            local project_number="$1"
+            local item_id="$2"
+            local item_list_json
+            local item_json
+            local command_status
+            local attempt
+
+            for attempt in 1 2 3; do
+              if item_list_json="$(PROJECT_INTAKE_ALLOW_PROJECT_FALLBACK=1 project_intake_gh \
+                "list Project items" gh project item-list "$project_number" --owner "$BASE_PROJECT_OWNER" --format json --limit 1000)"; then
+                item_json="$(jq --arg id "$item_id" '.items[] | select(.id == $id)' <<<"$item_list_json")"
+                if [[ -n "$item_json" ]]; then
+                  printf '%s\n' "$item_json"
+                  return 0
+                fi
+              else
+                command_status=$?
+                return "$command_status"
+              fi
+
+              if (( attempt < 3 )); then
+                echo "::warning::Project item '$item_id' is not visible after it was added (attempt $attempt/3); retrying in ${attempt}s." >&2
+                sleep "$attempt"
+              fi
+            done
+
+            echo "::error::Project item '$item_id' was not returned after the bounded visibility retry." >&2
+            return 1
+          }
+
+          project_intake_rest_item_after_add() {
+            local owner_api_prefix="$1"
+            local project_number="$2"
+            local item_id="$3"
+            local field_ids_csv="$4"
+            local escaped_issue_title
+            local item_search_json
+            local item_json
+            local command_status
+            local attempt
+
+            escaped_issue_title="${issue_title//\"/\\\"}"
+            for attempt in 1 2 3; do
+              if item_search_json="$(project_intake_gh "find REST Project item after add" gh api --method GET \
+                "$owner_api_prefix/projectsV2/$project_number/items" \
+                -f per_page=100 \
+                -f q="repo:$GITHUB_REPOSITORY is:issue title:\"$escaped_issue_title\"" \
+                -f fields="$field_ids_csv")"; then
+                item_json="$(jq --argjson issue_id "$issue_id" --arg item_id "$item_id" \
+                  '.[] | select((.id | tostring) == $item_id and .content.id == $issue_id)' \
+                  <<<"$item_search_json")"
+                if [[ -n "$item_json" ]]; then
+                  printf '%s\n' "$item_json"
+                  return 0
+                fi
+              else
+                command_status=$?
+                return "$command_status"
+              fi
+
+              if (( attempt < 3 )); then
+                echo "::warning::REST Project item '$item_id' is not visible after it was added (attempt $attempt/3); retrying in ${attempt}s." >&2
+                sleep "$attempt"
+              fi
+            done
+
+            echo "::error::REST Project item '$item_id' was not returned after the bounded visibility retry." >&2
+            return 1
+          }
+
           issue_json="$(project_intake_gh "view issue" gh api "repos/$GITHUB_REPOSITORY/issues/$issue_number")"
           issue_state="$(jq -r '.state | ascii_upcase' <<<"$issue_json")"
           issue_url="$(jq -r '.html_url' <<<"$issue_json")"
@@ -233,17 +304,12 @@ jobs:
               command_status=$?
               return "$command_status"
             fi
-            if item_json="$(PROJECT_INTAKE_ALLOW_PROJECT_FALLBACK=1 project_intake_gh \
-              "list Project items" gh project item-list "$project_number" --owner "$BASE_PROJECT_OWNER" --format json --limit 1000)"; then
-              item_json="$(jq --arg id "$item_id" '.items[] | select(.id == $id)' <<<"$item_json")"
+            if item_json="$(project_intake_graphql_item_after_add "$project_number" "$item_id")"; then
+              :
             else
               command_status=$?
               return "$command_status"
             fi
-            if [[ -z "$item_json" ]]; then
-              echo "::error::Project item '$item_id' was not returned after it was added." >&2
-              return 1
-            fi
             if fields_json="$(PROJECT_INTAKE_ALLOW_PROJECT_FALLBACK=1 project_intake_gh \
               "list Project fields" gh project field-list "$project_number" --owner "$BASE_PROJECT_OWNER" --format json)"; then
               :
@@ -415,11 +481,18 @@ jobs:
               added_item_json="$(project_intake_gh "add REST Project item" gh api --method POST \
                 "$owner_api_prefix/projectsV2/$project_number/items" \
                 -f type=Issue -F id="$issue_id")"
-              item_id="$(jq -r '.value.id // ""' <<<"$added_item_json")"
+              item_id="$(jq -r '.id // .value.id // ""' <<<"$added_item_json")"
               if [[ -z "$item_id" ]]; then
                 echo "::error::REST Project item add did not return an item id." >&2
                 return 1
               fi
+              if item_json="$(project_intake_rest_item_after_add \
+                "$owner_api_prefix" "$project_number" "$item_id" "$field_ids_csv")"; then
+                :
+              else
+                command_status=$?
+                return "$command_status"
+              fi
             fi
 
             current_item_json="$(project_intake_gh "read REST Project fields" gh api --method GET \

templates/project-intake.yml

@@ -176,6 +176,77 @@ jobs:
             return "$status"
           }
 
+          project_intake_graphql_item_after_add() {
+            local project_number="$1"
+            local item_id="$2"
+            local item_list_json
+            local item_json
+            local command_status
+            local attempt
+
+            for attempt in 1 2 3; do
+              if item_list_json="$(PROJECT_INTAKE_ALLOW_PROJECT_FALLBACK=1 project_intake_gh \
+                "list Project items" gh project item-list "$project_number" --owner "$BASE_PROJECT_OWNER" --format json --limit 1000)"; then
+                item_json="$(jq --arg id "$item_id" '.items[] | select(.id == $id)' <<<"$item_list_json")"
+                if [[ -n "$item_json" ]]; then
+                  printf '%s\n' "$item_json"
+                  return 0
+                fi
+              else
+                command_status=$?
+                return "$command_status"
+              fi
+
+              if (( attempt < 3 )); then
+                echo "::warning::Project item '$item_id' is not visible after it was added (attempt $attempt/3); retrying in ${attempt}s." >&2
+                sleep "$attempt"
+              fi
+            done
+
+            echo "::error::Project item '$item_id' was not returned after the bounded visibility retry." >&2
+            return 1
+          }
+
+          project_intake_rest_item_after_add() {
+            local owner_api_prefix="$1"
+            local project_number="$2"
+            local item_id="$3"
+            local field_ids_csv="$4"
+            local escaped_issue_title
+            local item_search_json
+            local item_json
+            local command_status
+            local attempt
+
+            escaped_issue_title="${issue_title//\"/\\\"}"
+            for attempt in 1 2 3; do
+              if item_search_json="$(project_intake_gh "find REST Project item after add" gh api --method GET \
+                "$owner_api_prefix/projectsV2/$project_number/items" \
+                -f per_page=100 \
+                -f q="repo:$GITHUB_REPOSITORY is:issue title:\"$escaped_issue_title\"" \
+                -f fields="$field_ids_csv")"; then
+                item_json="$(jq --argjson issue_id "$issue_id" --arg item_id "$item_id" \
+                  '.[] | select((.id | tostring) == $item_id and .content.id == $issue_id)' \
+                  <<<"$item_search_json")"
+                if [[ -n "$item_json" ]]; then
+                  printf '%s\n' "$item_json"
+                  return 0
+                fi
+              else
+                command_status=$?
+                return "$command_status"
+              fi
+
+              if (( attempt < 3 )); then
+                echo "::warning::REST Project item '$item_id' is not visible after it was added (attempt $attempt/3); retrying in ${attempt}s." >&2
+                sleep "$attempt"
+              fi
+            done
+
+            echo "::error::REST Project item '$item_id' was not returned after the bounded visibility retry." >&2
+            return 1
+          }
+
           issue_json="$(project_intake_gh "view issue" gh api "repos/$GITHUB_REPOSITORY/issues/$issue_number")"
           issue_state="$(jq -r '.state | ascii_upcase' <<<"$issue_json")"
           issue_url="$(jq -r '.html_url' <<<"$issue_json")"
@@ -233,17 +304,12 @@ jobs:
               command_status=$?
               return "$command_status"
             fi
-            if item_json="$(PROJECT_INTAKE_ALLOW_PROJECT_FALLBACK=1 project_intake_gh \
-              "list Project items" gh project item-list "$project_number" --owner "$BASE_PROJECT_OWNER" --format json --limit 1000)"; then
-              item_json="$(jq --arg id "$item_id" '.items[] | select(.id == $id)' <<<"$item_json")"
+            if item_json="$(project_intake_graphql_item_after_add "$project_number" "$item_id")"; then
+              :
             else
               command_status=$?
               return "$command_status"
             fi
-            if [[ -z "$item_json" ]]; then
-              echo "::error::Project item '$item_id' was not returned after it was added." >&2
-              return 1
-            fi
             if fields_json="$(PROJECT_INTAKE_ALLOW_PROJECT_FALLBACK=1 project_intake_gh \
               "list Project fields" gh project field-list "$project_number" --owner "$BASE_PROJECT_OWNER" --format json)"; then
               :
@@ -415,11 +481,18 @@ jobs:
               added_item_json="$(project_intake_gh "add REST Project item" gh api --method POST \
                 "$owner_api_prefix/projectsV2/$project_number/items" \
                 -f type=Issue -F id="$issue_id")"
-              item_id="$(jq -r '.value.id // ""' <<<"$added_item_json")"
+              item_id="$(jq -r '.id // .value.id // ""' <<<"$added_item_json")"
               if [[ -z "$item_id" ]]; then
                 echo "::error::REST Project item add did not return an item id." >&2
                 return 1
               fi
+              if item_json="$(project_intake_rest_item_after_add \
+                "$owner_api_prefix" "$project_number" "$item_id" "$field_ids_csv")"; then
+                :
+              else
+                command_status=$?
+                return "$command_status"
+              fi
             fi
 
             current_item_json="$(project_intake_gh "read REST Project fields" gh api --method GET \

tests/github_workflow_test_support.py

@@ -329,7 +329,17 @@ def _write_project_intake_mocks(tmp_path: Path) -> Path:
         exit 1
         ;;
     esac
-    project_intake_mock_graphql_item
+    count_file="${PROJECT_INTAKE_STATE:?}/graphql-item-list-count"
+    count=0
+    [[ ! -f "$count_file" ]] || count="$(cat "$count_file")"
+    count=$((count + 1))
+    printf '%s\\n' "$count" > "$count_file"
+    if [[ "${PROJECT_INTAKE_ITEM_NEVER_VISIBLE:-0}" == "1" ||
+          ( "${PROJECT_INTAKE_DELAYED_ITEM_VISIBILITY:-0}" == "1" && "$count" == "1" ) ]]; then
+      printf '{"items":[]}\\n'
+    else
+      project_intake_mock_graphql_item
+    fi
     ;;
   project\\ field-list*)
     cat <<'JSON'
@@ -393,14 +403,27 @@ def _write_project_intake_mocks(tmp_path: Path) -> Path:
       printf '403 Forbidden: REST item lookup failed\\n' >&2
       exit 1
     fi
-    if [[ "${PROJECT_INTAKE_ITEM_EXISTS:-1}" == "1" ]]; then
+    count_file="${PROJECT_INTAKE_STATE:?}/rest-item-search-count"
+    count=0
+    [[ ! -f "$count_file" ]] || count="$(cat "$count_file")"
+    count=$((count + 1))
+    printf '%s\\n' "$count" > "$count_file"
+    if [[ "${PROJECT_INTAKE_ITEM_EXISTS:-1}" == "1" ||
+          ( -f "${PROJECT_INTAKE_STATE:?}/rest-item-added" &&
+            "${PROJECT_INTAKE_ITEM_NEVER_VISIBLE:-0}" != "1" &&
+            ( "${PROJECT_INTAKE_DELAYED_ITEM_VISIBILITY:-0}" != "1" || "$count" != "2" ) ) ]]; then
       printf '[{"id":101,"content":{"id":1311,"number":1311,"title":"Project Intake test issue"}}]\\n'
     else
       printf '[]\\n'
     fi
     ;;
   api\\ --method\\ POST\\ *projectsV2/1/items*)
-    printf '{"value":{"id":101}}\\n'
+    : > "${PROJECT_INTAKE_STATE:?}/rest-item-added"
+    if [[ "${PROJECT_INTAKE_REST_ADD_RESPONSE:-nested}" == "top-level" ]]; then
+      printf '{"id":101}\\n'
+    else
+      printf '{"value":{"id":101}}\\n'
+    fi
     ;;
   api\\ --method\\ PATCH\\ *projectsV2/1/items/101*)
     payload="$(cat)"

tests/test_project_intake_workflow.py

@@ -178,6 +178,34 @@ def test_project_intake_primary_path_is_idempotent_for_complete_fields(tmp_path:
     assert not (tmp_path / "edits.log").exists()
 
 
+def test_project_intake_retries_graphql_item_visibility_after_add(tmp_path: Path) -> None:
+    result = run_project_intake_script(
+        tmp_path,
+        PROJECT_INTAKE_DELAYED_ITEM_VISIBILITY="1",
+    )
+
+    assert result.returncode == 0, result.stderr
+    assert "not visible after it was added" in result.stderr
+    assert (tmp_path / "sleep.log").read_text(encoding="utf-8") == "1\n"
+    gh_log = (tmp_path / "gh.log").read_text(encoding="utf-8")
+    assert gh_log.count("project item-add") == 1
+    assert gh_log.count("project item-list") == 3
+
+
+def test_project_intake_fails_closed_when_graphql_item_never_becomes_visible(
+    tmp_path: Path,
+) -> None:
+    result = run_project_intake_script(
+        tmp_path,
+        PROJECT_INTAKE_ITEM_NEVER_VISIBLE="1",
+    )
+
+    assert result.returncode != 0
+    assert "bounded visibility retry" in result.stderr
+    assert (tmp_path / "sleep.log").read_text(encoding="utf-8") == "1\n2\n"
+    assert "Synced issue" not in result.stdout
+
+
 def test_project_intake_rest_fallback_applies_closed_status(tmp_path: Path) -> None:
     result = run_project_intake_script(
         tmp_path,
@@ -206,6 +234,26 @@ def test_project_intake_rest_fallback_adds_a_missing_exact_item(tmp_path: Path)
     )
 
 
+def test_project_intake_rest_fallback_accepts_live_add_response_after_delay(
+    tmp_path: Path,
+) -> None:
+    result = run_project_intake_script(
+        tmp_path,
+        PROJECT_INTAKE_GRAPHQL_FAILURE="quota",
+        PROJECT_INTAKE_ITEM_EXISTS="0",
+        PROJECT_INTAKE_REST_ADD_RESPONSE="top-level",
+        PROJECT_INTAKE_DELAYED_ITEM_VISIBILITY="1",
+    )
+
+    assert result.returncode == 0, result.stderr
+    assert "via REST fallback" in result.stdout
+    assert "not visible after it was added" in result.stderr
+    assert (tmp_path / "sleep.log").read_text(encoding="utf-8") == "1\n"
+    gh_log = (tmp_path / "gh.log").read_text(encoding="utf-8")
+    assert gh_log.count("api --method POST orgs/basefoundry/projectsV2/1/items") == 1
+    assert gh_log.count("api --method GET orgs/basefoundry/projectsV2/1/items -f") == 3
+
+
 def test_project_intake_rest_failures_remain_fail_closed(tmp_path: Path) -> None:
     result = run_project_intake_script(
         tmp_path,