diff --git a/.github/actions/nf-test-action/action.yml b/.github/actions/nf-test-action/action.yml index 799e5db0..998dcd40 100644 --- a/.github/actions/nf-test-action/action.yml +++ b/.github/actions/nf-test-action/action.yml @@ -13,6 +13,14 @@ inputs: paths: description: "Test paths" required: true + jfrog_username: + description: "JFrog registry username" + required: false + default: "" + jfrog_password: + description: "JFrog registry password or token" + required: false + default: "" runs: using: "composite" @@ -32,6 +40,14 @@ runs: with: python-version: "3.11" + - name: Install cross-repo component dependencies + shell: bash + run: | + mapfile -t metas < <(find subworkflows -name meta.yml) + if (( ${#metas[@]} > 0 )); then + bash .github/scripts/install-components.sh "${metas[@]}" + fi + - name: Install nf-test uses: nf-core/setup-nf-test@v1 with: @@ -72,6 +88,34 @@ runs: nextflow secrets set ONCOKB_TOKEN $ONCOKB_TOKEN + - name: Login to JFrog Container Registry (docker) + if: ${{ inputs.profile == 'docker' && inputs.jfrog_username != '' }} + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3 + with: + registry: mskcc.jfrog.io + username: ${{ inputs.jfrog_username }} + password: ${{ inputs.jfrog_password }} + + - name: Configure JFrog auth for Apptainer/Singularity + if: ${{ contains(inputs.profile, 'singularity') && inputs.jfrog_username != '' }} + shell: bash + env: + JFROG_USERNAME: ${{ inputs.jfrog_username }} + JFROG_PASSWORD: ${{ inputs.jfrog_password }} + run: | + # Scope JFrog creds to mskcc.jfrog.io only via a Docker-format auth + # file. Setting APPTAINER_DOCKER_USERNAME/PASSWORD globally would send + # JFrog basic-auth to every docker:// pull (incl. ghcr.io, quay.io), + # which the other registries reject with 403. + AUTH=$(printf '%s:%s' "$JFROG_USERNAME" "$JFROG_PASSWORD" | base64 -w0) + for d in "$HOME/.apptainer" "$HOME/.singularity"; do + mkdir -p "$d" + cat > "$d/docker-config.json" < also accepted +# org_path: nf-core # optional, MSK-only extension — nf-core's +# # CLI infers this from git_remote instead and +# # ignores this key, so it's safe to include +# are sparse-checked-out into `modules///`. +# +# Notes: +# - Only leaf components are installed; transitive `components:` in the +# fetched modules are not recursively resolved. +# - An existing `modules///` directory is treated as a cache hit +# and not refreshed even if the declared ref has changed. CI always +# starts from a fresh checkout, so this is fine in practice; for local +# re-runs, delete the directory to force a refetch. +# - Default ref is `master`. Set `git_sha:` or `branch:` per component to +# override. +# - Keep this schema in sync with what `nf-core subworkflows install` +# expects (see docs/ci.md) — a mismatch breaks installing the +# subworkflow into a downstream pipeline via the standard nf-core CLI, +# even though this script (and CI) would still work. + +set -euo pipefail + +DEFAULT_REMOTE="https://github.com/nf-core/modules.git" +DEFAULT_REF="master" + +# Allowlist for path components built from meta.yml input. +NAME_RE='^[a-z0-9_]+(/[a-z0-9_]+)?$' +ORG_RE='^[a-z0-9_-]+$' + +TMPROOT=$(mktemp -d) +trap 'rm -rf "$TMPROOT"' EXIT + +declare -A seen + +for meta in "$@"; do + [[ -f "$meta" ]] || { echo "skip: $meta not found"; continue; } + n=$(yq '.components | length // 0' "$meta") + for i in $(seq 0 $((n - 1))); do + kind=$(yq ".components[$i] | tag" "$meta") + [[ "$kind" == "!!str" ]] && continue # bare = local, skip + + name=$(yq -r ".components[$i] | keys | .[0]" "$meta") + export COMP_NAME="$name" + + org=$(yq -r ".components[$i][env(COMP_NAME)].org_path // \"nf-core\"" "$meta") + remote=$(yq -r ".components[$i][env(COMP_NAME)].git_remote // \"$DEFAULT_REMOTE\"" "$meta") + ref=$(yq -r ".components[$i][env(COMP_NAME)].git_sha // .components[$i][env(COMP_NAME)].branch // \"$DEFAULT_REF\"" "$meta") + + [[ "$name" =~ $NAME_RE ]] || { echo "ERROR: invalid component name: $name (in $meta)" >&2; exit 1; } + [[ "$org" =~ $ORG_RE ]] || { echo "ERROR: invalid org_path: $org (in $meta)" >&2; exit 1; } + + key="$remote|$org|$name|$ref" + [[ -n "${seen[$key]:-}" ]] && continue + seen[$key]=1 + + dest="modules/$org/$name" + if [[ -d "$dest" ]]; then + echo "✓ $dest already present" + continue + fi + + echo "→ fetching $org/$name from $remote@$ref" + tmp="$TMPROOT/$org-${name//\//_}-$ref" + mkdir -p "$tmp" + git -C "$tmp" init -q + git -C "$tmp" remote add origin "$remote" + git -C "$tmp" config core.sparseCheckout true + echo "modules/$org/$name/" > "$tmp/.git/info/sparse-checkout" + git -C "$tmp" fetch --depth 1 origin "$ref" -q + git -C "$tmp" checkout -q FETCH_HEAD + mkdir -p "$(dirname "$dest")" + mv "$tmp/modules/$org/$name" "$dest" + done +done diff --git a/.github/skip_nf_test.json b/.github/skip_nf_test.json index 633dc3e7..2bcecc70 100644 --- a/.github/skip_nf_test.json +++ b/.github/skip_nf_test.json @@ -38,7 +38,9 @@ "subworkflows/msk/phylowgs", "subworkflows/msk/generate_mutated_peptides", "subworkflows/msk/neoantigen_editing", - "subworkflows/msk/traceback" + "subworkflows/msk/traceback", + "modules/msk/hlahd", + "subworkflows/msk/hlahd_from_bam" ], "docker": [], "singularity": [] diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 30223b81..e41e3532 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -135,6 +135,8 @@ jobs: shard: ${{ matrix.shard }} total_shards: ${{ env.TOTAL_SHARDS }} paths: "${{ join(fromJson(steps.filter.outputs.filtered_paths), ' ') }}" + jfrog_username: ${{ secrets.JFROG_USERNAME }} + jfrog_password: ${{ secrets.JFROG_PASSWORD }} confirm-pass: runs-on: ubuntu-latest diff --git a/README.md b/README.md index c82b6bac..8d89a301 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ A repository for hosting [Nextflow DSL2](https://www.nextflow.io/docs/latest/dsl - [Table of contents](#table-of-contents) - [Using existing modules](#using-existing-modules) - [Adding new modules](#adding-new-modules) + - [CI](#ci) - [Help](#help) - [Citation](#citation) @@ -141,6 +142,11 @@ If you wish to contribute a new module, please see the documentation on the [MSK > Please be kind to our code reviewers and submit one pull request per module :) +## CI + +See [docs/ci.md](docs/ci.md) for how `nf-test` CI works, including how +subworkflows declare and fetch cross-repo (nf-core) component dependencies. + ## Help For further information or help, don't hesitate to get in touch on [Slack `#mskcc-omics-workflows` channel](https://mskcc.enterprise.slack.com/archives/C040T4MFCHJ). diff --git a/docs/ci.md b/docs/ci.md new file mode 100644 index 00000000..1197da52 --- /dev/null +++ b/docs/ci.md @@ -0,0 +1,214 @@ +# CI: how `nf-test` runs in this repo + +This documents the moving parts behind `.github/workflows/nf-test.yml` for +contributors adding or debugging modules/subworkflows. It focuses on one +piece that's easy to miss: **cross-repo component installation** — how a +subworkflow declares a dependency on an nf-core module and gets it fetched +automatically in CI. + +## Contents +- [Pipeline overview](#pipeline-overview) +- [Change detection & sharding](#change-detection--sharding) +- [`nf-test-action`: the composite action](#nf-test-action-the-composite-action) +- [Cross-repo component installation](#cross-repo-component-installation) +- [Declaring a component dependency](#declaring-a-component-dependency) +- [Installing a subworkflow into a pipeline](#installing-a-subworkflow-into-a-pipeline) +- [`confirm-pass`](#confirm-pass) + +## Pipeline overview + +`nf-test.yml` has three jobs: + +1. **`nf-test-changes`** — diffs the PR against its base and produces a list + of changed module/subworkflow test paths (tagged via nf-test `tags`). +2. **`nf-test`** — a matrix over `profile: [conda, docker, singularity]` × + `shard: [1..5]`. Each cell runs a slice of the changed tests via the + `nf-test-action` composite action. +3. **`confirm-pass`** — a gate job that fails the overall check if any + matrix cell failed or was cancelled, and passes only once every cell has + reported a result. This is the single required status check on PRs. + +Only modules/subworkflows whose test files changed (or whose declared +dependencies changed) are run — `nf-test-changes` uses +[`detect-nf-test-changes`](https://github.com/adamrtalbot/detect-nf-test-changes) +for this, so an unrelated PR doesn't re-run the whole repo's test suite. +`.github/skip_nf_test.json` can additionally skip specific paths per-profile +(e.g. a tool that's flaky under conda). + +## Change detection & sharding + +`nf-test-changes` emits a JSON array of changed test file paths as an +output. Each `nf-test` matrix cell filters that array against +`skip_nf_test.json` for its profile, then calls nf-test with +`--shard /` so the (filtered) test set is spread across 5 +shards per profile. Sharding is by *individual test*, not by file — a +subworkflow test file with 3 tests can have its tests land on 3 different +shards. This is why, when triaging a CI failure, you should check sibling +shards for the *other* tests in the same file rather than assuming the +whole file failed together. + +## `nf-test-action`: the composite action + +`.github/actions/nf-test-action/action.yml` is the shared setup + run +sequence used by every matrix cell. In order: + +1. Set up Java, Nextflow, Python. +2. **Install cross-repo component dependencies** (see below) — runs before + nf-test is even installed, since it just populates directories nf-test + will need on disk. +3. Install `nf-test` itself. +4. Set up Apptainer (singularity profile only) / Conda (conda profile only). +5. Configure Nextflow secrets (Sentieon, OncoKB) if present. +6. Log in to the JFrog registry for Docker, and write a scoped + `docker-config.json` for Apptainer/Singularity (scoped to + `mskcc.jfrog.io` only — setting JFrog creds globally would leak + basic-auth to `ghcr.io`/`quay.io` pulls and get rejected with 403). +7. Run `nf-test test --profile= --shard / ... `. +8. On failure, append a results table to the job summary from the TAP + output. + +## Cross-repo component installation + +This repo only version-controls `modules/msk/` and `subworkflows/msk/` — +`modules/nf-core/` and `subworkflows/nf-core/` are **gitignored** +(see `.gitignore`). They're not vendored; they're fetched fresh into the +workspace before each CI run (and must be fetched manually for local +testing — see below). + +The fetch is driven by `.github/scripts/install-components.sh`, called once +per matrix cell with every subworkflow `meta.yml` file as an argument: + +```bash +mapfile -t metas < <(find subworkflows -name meta.yml) +bash .github/scripts/install-components.sh "${metas[@]}" +``` + +For each `meta.yml`, the script reads its `components:` list (via `yq`) and, +for each entry that is a **dict** (not a bare string — see next section), does +a shallow sparse checkout: + +```bash +git init +git remote add origin "$remote" # default: https://github.com/nf-core/modules.git +git config core.sparseCheckout true +echo "modules/$org/$name/" > .git/info/sparse-checkout +git fetch --depth 1 origin "$ref" # default ref: master +git checkout FETCH_HEAD +mv modules/$org/$name # or subworkflows/$org/$name +``` + +i.e. it's a shallow, path-scoped clone of just that one module/subworkflow +directory — not a full checkout of `nf-core/modules`. This keeps CI fast: +only the components actually declared as dependencies get pulled, from +whatever ref is specified (defaulting to `nf-core/modules@master`). + +Two caching behaviors worth knowing: +- **Dedup within a run**: if two subworkflows declare the same + `(remote, org_path, name, ref)`, it's only fetched once (`seen[$key]`). +- **Skip if already present**: if `modules/$org/$name` already exists on + disk, the script leaves it alone rather than re-fetching. CI always + starts from a clean checkout, so this only matters for local reruns — if + you change a component's declared `git_sha`/`branch` and rerun locally, + delete the stale directory first or you'll silently keep testing the old + ref. + +## Declaring a component dependency + +In a subworkflow's `meta.yml`, `components:` mixes two entry shapes: + +```yaml +components: + - hlahd # bare string: local msk module, already + # under version control here — skipped by + # install-components.sh + - samtools/collate: # dict, nested under the component name: + git_remote: https://github.com/nf-core/modules.git # external, fetched at CI time + # optional: org_path: nf-core # MSK-only extension, defaults to nf-core + # optional: git_sha: # pins to a commit + # optional: branch: # pins to a branch (git_sha wins if both set) +``` + +**The nesting matters.** This has to be `- : {...}`, not a flat +`- {name: , ...}` — the nested-under-name shape is what +`nf-core subworkflows install` itself expects +(`nf_core/components/components_utils.py::get_components_to_install`), and +`install-components.sh` mirrors it on purpose so the two consumers agree +on one schema. Getting this wrong doesn't break CI (which only exercises +`install-components.sh`) — it breaks installing the subworkflow into a +downstream pipeline with the standard nf-core CLI, which is silent until +someone actually tries it. See +[Installing a subworkflow into a pipeline](#installing-a-subworkflow-into-a-pipeline). + +`org_path` defaults to `nf-core` and determines the destination directory +(`modules//`) as well as being validated against +`^[a-z0-9_-]+$`; `name` is validated against `^[a-z0-9_]+(/[a-z0-9_]+)?$`. +Both are used to build a filesystem path, so the script rejects anything +that doesn't match before shelling out to git. `org_path` isn't part of +nf-core's own schema (it infers the org from `git_remote` instead) — nf-core +silently ignores the extra key, so it's safe to keep as an MSK-only +extension for the rare case a remote's inferred org doesn't match what you +want on disk. + +**When adding a new nf-core module dependency to a subworkflow** (e.g. we +added `samtools/collate` while fixing the `hlahd_from_bam` read-pairing bug +in PR #241): + +1. Add the dict entry to the subworkflow's `meta.yml` `components:` list. +2. Fetch it locally the same way CI does (or just run any nf-test command — + `install-components.sh` isn't wired into a local nf-test invocation + automatically, so do it once by hand): + ```bash + TMP=$(mktemp -d) + git -C "$TMP" init -q + git -C "$TMP" remote add origin https://github.com/nf-core/modules.git + git -C "$TMP" config core.sparseCheckout true + echo "modules/nf-core//" > "$TMP/.git/info/sparse-checkout" + git -C "$TMP" fetch --depth 1 origin master -q + git -C "$TMP" checkout -q FETCH_HEAD + mv "$TMP/modules/nf-core/" modules/nf-core/ + ``` +3. `include { ... } from '../../../modules/nf-core//main'` in the + subworkflow's `main.nf`. +4. Run tests locally with `--profile docker` before pushing — CI will fetch + the same component fresh from the declared ref, so a local pass here is + a reliable predictor of CI behavior for this step. + +## Installing a subworkflow into a pipeline + +A downstream pipeline consumes modules/subworkflows from this repo with the +standard nf-core CLI, pointed at the MSK remote: + +```bash +nf-core subworkflows \ + --git-remote https://github.com/mskcc-omics-workflows/modules.git \ + --branch feature/hlahd \ + install hlahd_from_bam +``` + +This is a real command doing real work, not a wrapper around +`install-components.sh` — the CLI parses the subworkflow's `meta.yml` +`components:` list itself to resolve transitive dependencies, then clones +each one (local MSK components from this repo, external ones from their own +`git_remote`) and records everything in the pipeline's `modules.json`. For +`hlahd_from_bam` that means: the `hlahd` module (local), plus +`samtools/view`, `gatk4/revertsam`, `samtools/collate`, and +`samtools/fastq` (all from `nf-core/modules@master`) — five components from +two different remotes, installed with one command. + +Verified end-to-end against this branch in a scratch pipeline: all five +land under `modules/{msk,nf-core}/...` and `subworkflows/msk/hlahd_from_bam/`, +and `modules.json` records each with its own `git_sha`/`branch` and an +`installed_by: [hlahd_from_bam]` back-reference. This is exactly why the +`components:` schema in the previous section has to match nf-core's own +parser — this install path doesn't go through `install-components.sh` at +all, so a schema drift between the two would only surface here, not in CI. + +## `confirm-pass` + +`confirm-pass` is `needs: [nf-test]` with `if: always()`, so it evaluates +after every matrix cell has finished regardless of outcome. It fails if +`needs.*.result` contains `failure` or `cancelled`, and only passes if it +contains `success` — meaning a single failing shard fails the whole check, +even though the other 29 matrix cells are green. This is intentional: it's +the one branch-protection-required status, so PR authors have a single +check to watch rather than 16 individual ones. diff --git a/modules/msk/hlahd/environment.yml b/modules/msk/hlahd/environment.yml new file mode 100644 index 00000000..4c59b932 --- /dev/null +++ b/modules/msk/hlahd/environment.yml @@ -0,0 +1,7 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json +channels: + - conda-forge + - bioconda +dependencies: + - "YOUR-TOOL=HERE" diff --git a/modules/msk/hlahd/main.nf b/modules/msk/hlahd/main.nf new file mode 100644 index 00000000..a68840ae --- /dev/null +++ b/modules/msk/hlahd/main.nf @@ -0,0 +1,64 @@ +process HLAHD { + tag "$meta.id" + label 'process_high' + + conda "${moduleDir}/environment.yml" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'docker://mskcc.jfrog.io/omicswf-docker-prod-local/mskcc-omics-workflows/hlahd:1.7.1': + 'mskcc.jfrog.io/omicswf-docker-prod-local/mskcc-omics-workflows/hlahd:1.7.1' }" + + input: + tuple val(meta), path(fastq_1), path(fastq_2) + + output: + tuple val(meta), path("${prefix}/result/${prefix}_final.result.txt"), emit: result + tuple val(meta), path("${prefix}/result/${prefix}_*.est.txt"), emit: result_per_locus + path "versions.yml", emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def min_read = task.ext.args2 ?: '100' + prefix = task.ext.prefix ?: "${meta.id}" + def install_dir = '/opt/hlahd/current' + """ + if [[ \$(ulimit -n) -lt 1024 ]]; then ulimit -n 1024; fi + + ln -sf /usr/bin/python3 ./python + export PATH=\$PWD:${install_dir}/bin:\$PATH + + mkdir -p ${prefix} + + bash ${install_dir}/bin/hlahd.sh \\ + -t ${task.cpus} \\ + -m ${min_read} \\ + -f ${install_dir}/freq_data \\ + ${args} \\ + ${fastq_1} \\ + ${fastq_2} \\ + ${install_dir}/HLA_gene.split.3.50.0.txt \\ + ${install_dir}/dictionary \\ + ${prefix} \\ + . + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + hlahd: \$(bash ${install_dir}/bin/hlahd.sh 2>&1 | grep -oP 'HLA-HD version \\K[0-9.]+' | head -1) + END_VERSIONS + """ + + stub: + prefix = task.ext.prefix ?: "${meta.id}" + """ + mkdir -p ${prefix}/result + touch ${prefix}/result/${prefix}_final.result.txt + touch ${prefix}/result/${prefix}_A.est.txt + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + hlahd: \$(bash /opt/hlahd/current/bin/hlahd.sh 2>&1 | grep -oP 'HLA-HD version \\K[0-9.]+' | head -1) + END_VERSIONS + """ +} diff --git a/modules/msk/hlahd/meta.yml b/modules/msk/hlahd/meta.yml new file mode 100644 index 00000000..b268a910 --- /dev/null +++ b/modules/msk/hlahd/meta.yml @@ -0,0 +1,67 @@ +name: "hlahd" +description: HLA typing from paired-end FASTQ reads using HLA-HD +keywords: + - HLA + - immunology + - typing + - genomics +tools: + - "hlahd": + description: + "HLA-HD (HLA typing from High-quality Dictionary) performs HLA typing + from paired-end FASTQ reads using bowtie2 alignment against HLA allele dictionaries." + homepage: "https://w3.genome.med.kyoto-u.ac.jp/HLA-HD/" + documentation: "https://w3.genome.med.kyoto-u.ac.jp/HLA-HD/" + licence: + - "ACADEMIC SOFTWARE LICENSE" + identifier: "" +input: + - - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1', single_end:false ]` + - fastq_1: + type: file + description: First read of paired-end FASTQ input + pattern: "*.{fastq,fastq.gz,fq,fq.gz}" + ontologies: + - edam: http://edamontology.org/format_1930 + - fastq_2: + type: file + description: Second read of paired-end FASTQ input + pattern: "*.{fastq,fastq.gz,fq,fq.gz}" + ontologies: + - edam: http://edamontology.org/format_1930 +output: + result: + - - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1', single_end:false ]` + - ${prefix}/result/${prefix}_final.result.txt: + type: file + description: Final HLA typing result file containing best-call alleles for all loci + pattern: "**/result/*_final.result.txt" + result_per_locus: + - - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1', single_end:false ]` + - ${prefix}/result/${prefix}_*.est.txt: + type: file + description: Per-locus HLA estimation files (one file per HLA gene) + pattern: "**/result/*_*.est.txt" + versions: + - versions.yml: + type: file + description: File containing software versions + pattern: "versions.yml" + ontologies: + - edam: http://edamontology.org/format_3750 +authors: + - "@johnoooh" +maintainers: + - "@johnoooh" diff --git a/modules/msk/hlahd/tests/main.nf.test b/modules/msk/hlahd/tests/main.nf.test new file mode 100644 index 00000000..d8e73a86 --- /dev/null +++ b/modules/msk/hlahd/tests/main.nf.test @@ -0,0 +1,66 @@ +nextflow_process { + + name "Test Process HLAHD" + script "../main.nf" + process "HLAHD" + + tag "modules" + tag "modules_msk" + tag "hlahd" + + test("hlahd - fastq pair - result txt") { + + when { + process { + """ + input[0] = [ + [ id:'test_sample', single_end:false ], // meta map + file(params.test_data_mskcc['hlahd']['fastq_1'], checkIfExists: true), + file(params.test_data_mskcc['hlahd']['fastq_2'], checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot( + process.out.result.collect { meta, f -> + [meta, path(f).readLines().findAll { it.startsWith("A\t") }] + }, + process.out.versions + ).match() + } + ) + } + + } + + test("hlahd - fastq pair - stub") { + + options "-stub" + + when { + process { + """ + input[0] = [ + [ id:'test_sample', single_end:false ], // meta map + file(params.test_data_mskcc['hlahd']['fastq_1'], checkIfExists: true), + file(params.test_data_mskcc['hlahd']['fastq_2'], checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert path(process.out.result.get(0).get(1)).exists() }, + { assert snapshot(process.out.versions).match() } + ) + } + + } + +} diff --git a/modules/msk/hlahd/tests/main.nf.test.snap b/modules/msk/hlahd/tests/main.nf.test.snap new file mode 100644 index 00000000..9480566d --- /dev/null +++ b/modules/msk/hlahd/tests/main.nf.test.snap @@ -0,0 +1,37 @@ +{ + "hlahd - fastq pair - stub": { + "content": [ + [ + "versions.yml:md5,f196d451477cda61837f7cfb2ed3c9b4" + ] + ], + "timestamp": "2026-03-05T16:10:44.004384", + "meta": { + "nf-test": "0.9.2", + "nextflow": "25.10.4" + } + }, + "hlahd - fastq pair - result txt": { + "content": [ + [ + [ + { + "id": "test_sample", + "single_end": false + }, + [ + "A\tHLA-A*01:01:01\tHLA-A*29:02:01" + ] + ] + ], + [ + "versions.yml:md5,f196d451477cda61837f7cfb2ed3c9b4" + ] + ], + "timestamp": "2026-05-14T11:55:00.000000", + "meta": { + "nf-test": "0.9.4", + "nextflow": "25.10.4" + } + } +} diff --git a/modules/msk/hlahd/tests/tags.yml b/modules/msk/hlahd/tests/tags.yml new file mode 100644 index 00000000..51631aae --- /dev/null +++ b/modules/msk/hlahd/tests/tags.yml @@ -0,0 +1,2 @@ +hlahd: + - modules/msk/hlahd/** diff --git a/subworkflows/msk/hlahd_from_bam/main.nf b/subworkflows/msk/hlahd_from_bam/main.nf new file mode 100644 index 00000000..3a1fd45f --- /dev/null +++ b/subworkflows/msk/hlahd_from_bam/main.nf @@ -0,0 +1,88 @@ +include { SAMTOOLS_VIEW } from '../../../modules/nf-core/samtools/view/main' +include { GATK4_REVERTSAM } from '../../../modules/nf-core/gatk4/revertsam/main' +include { SAMTOOLS_COLLATE } from '../../../modules/nf-core/samtools/collate/main' +include { SAMTOOLS_FASTQ } from '../../../modules/nf-core/samtools/fastq/main' +include { HLAHD } from '../../../modules/msk/hlahd/main' + +workflow HLAHD_FROM_BAM { + + take: + ch_bam // channel: [ val(meta), path(bam), path(bai) ] + skip_revert_sam // val: Boolean + + main: + + ch_versions = Channel.empty() + + // + // MODULE: Extract HLA region from BAM using samtools view. + // The caller configures the region to extract via ext.args in modules.config, + // e.g. ext.args = '-b chr6:28000000-34000000' + // + SAMTOOLS_VIEW( + ch_bam, + [[],[],[]], + [[],[]], + [[],[]], + [] + ) + + // + // Optional: Revert base quality score recalibration with GATK4 RevertSam. + // Set skip_revert_sam = true when the BAM has no BQSR applied (e.g. already + // in OQ-restored state, or produced by a tool that does not perform BQSR). + // + if (!skip_revert_sam) { + + GATK4_REVERTSAM( + SAMTOOLS_VIEW.out.bam + ) + ch_for_fastq = GATK4_REVERTSAM.out.bam + + } else { + + ch_for_fastq = SAMTOOLS_VIEW.out.bam + + } + + // + // MODULE: Collate reads by name before FASTQ conversion. + // samtools fastq requires name-grouped input to pair mates correctly; + // SAMTOOLS_VIEW's output is coordinate-sorted, and GATK4_REVERTSAM's + // queryname sort is not guaranteed (e.g. when skip_revert_sam is true). + // Collating unconditionally makes pairing correct regardless of branch. + // + SAMTOOLS_COLLATE( + ch_for_fastq, + [[],[],[]] + ) + + // + // MODULE: Convert BAM to paired FASTQ files. + // SAMTOOLS_FASTQ emits .out.fastq as [ meta, [fq1, fq2] ]; unpack into + // separate paths so HLAHD receives the three-element tuple it expects. + // + SAMTOOLS_FASTQ( + SAMTOOLS_COLLATE.out.bam, + false + ) + + ch_fastq_for_hlahd = SAMTOOLS_FASTQ.out.fastq + .map { meta, fastqs -> + def (fq1, fq2) = fastqs + [meta, fq1, fq2] + } + + // + // MODULE: Run HLA-HD to call HLA alleles from paired FASTQ files. + // + HLAHD( + ch_fastq_for_hlahd + ) + ch_versions = ch_versions.mix(HLAHD.out.versions.first()) + + emit: + result = HLAHD.out.result // channel: [ val(meta), path(result/*_final.result.txt) ] + result_per_locus = HLAHD.out.result_per_locus // channel: [ val(meta), path(result/*_*.est.txt) ] + versions = ch_versions // channel: [ path(versions.yml) ] +} diff --git a/subworkflows/msk/hlahd_from_bam/meta.yml b/subworkflows/msk/hlahd_from_bam/meta.yml new file mode 100644 index 00000000..103e37b9 --- /dev/null +++ b/subworkflows/msk/hlahd_from_bam/meta.yml @@ -0,0 +1,73 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json +name: "hlahd_from_bam" +description: | + Extract the HLA region from a coordinate-sorted BAM file, optionally revert + base quality score recalibration with GATK4 RevertSam, convert to paired + FASTQ files with samtools fastq, and run HLA-HD for high-resolution HLA + typing. +keywords: + - HLA + - typing + - BAM + - immunology + - samtools + - gatk4 +components: + - samtools/view: + git_remote: https://github.com/nf-core/modules.git + - gatk4/revertsam: + git_remote: https://github.com/nf-core/modules.git + - samtools/collate: + git_remote: https://github.com/nf-core/modules.git + - samtools/fastq: + git_remote: https://github.com/nf-core/modules.git + - hlahd +input: + - - meta: + type: map + description: | + Groovy Map containing sample information. + e.g. [ id:'sample_01', single_end:false ] + - bam: + type: file + description: | + Input BAM file. Must be coordinate-sorted and indexed. + pattern: "*.bam" + - bai: + type: file + description: BAM index file. + pattern: "*.bai" + - skip_revert_sam: + type: boolean + description: | + When true, skip the GATK4 RevertSam step. Set to true if the BAM was + not processed with base quality score recalibration (BQSR), or if + original base qualities have already been restored. +output: + - result: + - meta: + type: map + description: Groovy Map containing sample information. + - "**/result/*_final.result.txt": + type: file + description: | + Tab-separated HLA allele calls for all typed loci produced by + HLA-HD. One file per sample. + pattern: "**/result/*_final.result.txt" + - result_per_locus: + - meta: + type: map + description: Groovy Map containing sample information. + - "**/result/*_*.est.txt": + type: file + description: Per-locus HLA estimation files produced by HLA-HD. + pattern: "**/result/*_*.est.txt" + - versions: + - "versions.yml": + type: file + description: File containing software versions for all tools used. + pattern: "versions.yml" +authors: + - "@johnoooh" +maintainers: + - "@johnoooh" diff --git a/subworkflows/msk/hlahd_from_bam/tests/main.nf.test b/subworkflows/msk/hlahd_from_bam/tests/main.nf.test new file mode 100644 index 00000000..c078b4cc --- /dev/null +++ b/subworkflows/msk/hlahd_from_bam/tests/main.nf.test @@ -0,0 +1,110 @@ +nextflow_workflow { + + name "Test Subworkflow HLAHD_FROM_BAM" + script "../main.nf" + workflow "HLAHD_FROM_BAM" + + tag "subworkflows" + tag "subworkflows_msk" + tag "subworkflows/hlahd_from_bam" + tag "hlahd_from_bam" + tag "hlahd" + tag "samtools" + tag "gatk4" + + test("hlahd_from_bam - bam - with revert sam - result") { + + config "./nextflow.config" + + when { + workflow { + """ + input[0] = Channel.value([ + [ id:'test_sample', single_end:false ], // meta map + file(params.test_data_mskcc['hlahd']['bam'], checkIfExists: true), + file(params.test_data_mskcc['hlahd']['bai'], checkIfExists: true) + ]) + + input[1] = false + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out.result.collect { meta, f -> + [meta, path(f).readLines().findAll { it.startsWith("A\t") }] + }, + workflow.out.versions + ).match() + } + ) + } + } + + test("hlahd_from_bam - bam - skip revert sam - result") { + + config "./nextflow.config" + + when { + workflow { + """ + input[0] = Channel.value([ + [ id:'test_sample', single_end:false ], // meta map + file(params.test_data_mskcc['hlahd']['bam'], checkIfExists: true), + file(params.test_data_mskcc['hlahd']['bai'], checkIfExists: true) + ]) + + input[1] = true + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out.result.collect { meta, f -> + [meta, path(f).readLines().findAll { it.startsWith("A\t") }] + }, + workflow.out.versions + ).match() + } + ) + } + } + + test("hlahd_from_bam - bam - stub") { + + config "./nextflow.config" + options "-stub" + + when { + workflow { + """ + input[0] = Channel.value([ + [ id:'test_sample', single_end:false ], // meta map + file('test_hla_region.bam', checkIfExists: false), + file('test_hla_region.bam.bai', checkIfExists: false) + ]) + + input[1] = false + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out.result[0][0], + file(workflow.out.result[0][1]).name, + workflow.out.versions + ).match() + } + ) + } + } +} diff --git a/subworkflows/msk/hlahd_from_bam/tests/main.nf.test.snap b/subworkflows/msk/hlahd_from_bam/tests/main.nf.test.snap new file mode 100644 index 00000000..f5e9bc58 --- /dev/null +++ b/subworkflows/msk/hlahd_from_bam/tests/main.nf.test.snap @@ -0,0 +1,65 @@ +{ + "hlahd_from_bam - bam - stub": { + "content": [ + { + "id": "test_sample", + "single_end": false + }, + "test_sample_final.result.txt", + [ + "versions.yml:md5,e0264ab44efbd0fd97853a90160075a1" + ] + ], + "timestamp": "2026-04-27T18:18:22.295121", + "meta": { + "nf-test": "0.9.4", + "nextflow": "25.10.4" + } + }, + "hlahd_from_bam - bam - skip revert sam - result": { + "content": [ + [ + [ + { + "id": "test_sample", + "single_end": false + }, + [ + "A\tHLA-A*01:01:01\tHLA-A*29:02:01" + ] + ] + ], + [ + "versions.yml:md5,e0264ab44efbd0fd97853a90160075a1" + ] + ], + "timestamp": "2026-05-14T11:55:00.000000", + "meta": { + "nf-test": "0.9.4", + "nextflow": "25.10.4" + } + }, + "hlahd_from_bam - bam - with revert sam - result": { + "content": [ + [ + [ + { + "id": "test_sample", + "single_end": false + }, + [ + "A\tHLA-A*01:01:01\tHLA-A*29:02:01" + ] + ] + ], + [ + "versions.yml:md5,e0264ab44efbd0fd97853a90160075a1" + ] + ], + "timestamp": "2026-05-14T11:55:00.000000", + "meta": { + "nf-test": "0.9.4", + "nextflow": "25.10.4" + } + } +} diff --git a/subworkflows/msk/hlahd_from_bam/tests/nextflow.config b/subworkflows/msk/hlahd_from_bam/tests/nextflow.config new file mode 100644 index 00000000..634a7ebc --- /dev/null +++ b/subworkflows/msk/hlahd_from_bam/tests/nextflow.config @@ -0,0 +1,14 @@ +process { + withName: 'HLAHD_FROM_BAM:SAMTOOLS_VIEW' { + ext.prefix = { "${meta.id}.hla_region" } + } + withName: 'HLAHD_FROM_BAM:GATK4_REVERTSAM' { + ext.prefix = { "${meta.id}.reverted" } + } + withName: 'HLAHD_FROM_BAM:SAMTOOLS_COLLATE' { + ext.prefix = { "${meta.id}.collated" } + } + withName: 'HLAHD_FROM_BAM:SAMTOOLS_FASTQ' { + ext.prefix = { "${meta.id}" } + } +} diff --git a/subworkflows/msk/hlahd_from_bam/tests/tags.yml b/subworkflows/msk/hlahd_from_bam/tests/tags.yml new file mode 100644 index 00000000..4da44ddc --- /dev/null +++ b/subworkflows/msk/hlahd_from_bam/tests/tags.yml @@ -0,0 +1,3 @@ +subworkflows/hlahd_from_bam: + - subworkflows/msk/hlahd_from_bam/** + - modules/msk/hlahd/** diff --git a/tests/config/test_data.config b/tests/config/test_data.config index 963b084c..11d768a4 100644 --- a/tests/config/test_data.config +++ b/tests/config/test_data.config @@ -787,6 +787,12 @@ params { iedb_alignments = "${params.test_data_base_msk}/neoantigen/neoantigen/neoantigenEditing/data/IEDB_alignments/iedb_alignments_3-OLTS.txt" test_annotated = "${params.test_data_base_msk}/neoantigen/neoantigen/test_patient_test_annotated.json" } + 'hlahd' { + fastq_1 = "${params.test_data_base_msk}/hlahd/hlahd/test_R1.fastq.gz" + fastq_2 = "${params.test_data_base_msk}/hlahd/hlahd/test_R2.fastq.gz" + bam = "${params.test_data_base_msk}/hlahd/hlahd/test_hla_region.bam" + bai = "${params.test_data_base_msk}/hlahd/hlahd/test_hla_region.bam.bai" + } 'genome_nexus' { test_maf = "${params.test_data_base_msk}/feature/genome_nexus_subworkflow/mafs/test.maf" sample2_sample1_annotated_maf = "${params.test_data_base_msk}/feature/genome_nexus_subworkflow/mafs/sample2_sample1_annotated.maf"