add shellcheck

.github/workflows/shellcheck.yml

@@ -0,0 +1,20 @@
+# Lint all shell scripts in the repository with ShellCheck.
+# Picks up any *.sh file in the repo excluding .git
+name: ShellCheck
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  shellcheck:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v7
+      - name: Run ShellCheck
+        run: |
+          find . -type f -name '*.sh' -not -path './.git/*' -print0 \
+            | xargs -0 shellcheck

mimic-iii/buildmimic/duckdb/import_duckdb.sh

@@ -58,7 +58,8 @@ elif [ -n "$3" ]; then
     die "Usage: ./import_duckdb.sh mimic_data_dir [output_db]"
 elif [ -s "$OUTFILE" ]; then
 	yell "File \"$OUTFILE\" already exists."
-	read -p "Continue? (y/d/n) 'y' continues, 'd' deletes original file, 'n' stops: " yn
+	printf "Continue? (y/d/n) 'y' continues, 'd' deletes original file, 'n' stops: "
+	read -r yn
 	case $yn in
 		[Yy]* ) ;; # OK
 		[Nn]* ) exit;;

mimic-iii/buildmimic/postgres/create_mimic_user.sh

@@ -41,7 +41,7 @@ else
 fi
 
 # check if SUDO is needed by checking if we can login with postgres without it
-err2=`psql postgres postgres -c "select 1;" 2>&1 >/dev/null`
+err2=$(psql postgres postgres -c "select 1;" 2>&1 >/dev/null)
 
 if [[ $err2 == *"Peer authentication failed for user"* ]]; then
   # we need to call sudo every time for postgres

mimic-iii/tests/md5.sh

@@ -1,20 +1,21 @@
+#!/bin/bash
 # run in the folder:
 #  ./md5.sh checksum_md5_zipped.txt
 # OR
 #  ./md5.sh checksum_md5_unzipped.txt
-while read p; do
-  A="$(echo $p | cut -d' ' -f1)"
-  B="$(echo $p | cut -d' ' -f2)"
-  if [ ! -f $B ]
+while read -r p; do
+  A="$(echo "$p" | cut -d' ' -f1)"
+  B="$(echo "$p" | cut -d' ' -f2)"
+  if [ ! -f "$B" ]
   then
     echo "ERROR: $B does not exist"
     continue
   fi
-  read C< <(md5 -q $B)
-  echo $B $A = $C
-  if [ $C != $A ]
+  read -r C < <(md5 -q "$B")
+  echo "$B" "$A" = "$C"
+  if [ "$C" != "$A" ]
   then
     echo "ERROR: $B wrong md5"
     exit
   fi
-done <$1
+done < "$1"

mimic-iv-cxr/txt/chexpert/run_chexpert_on_files.sh

@@ -22,11 +22,12 @@ fi
 sleep 2
 
 # loop through each .csv file in the section folder
-for fn in `ls $REPORT_PATH`; do
-    echo `date`: $fn
-    fn_stem=`echo $fn | cut -d. -f 1`
+for fn in "$REPORT_PATH"/*; do
+    fn=${fn##*/}
+    echo "$(date): $fn"
+    fn_stem=$(echo "$fn" | cut -d. -f 1)
     # run chexpert - must be run from chexpert folder
-    python $CHEXPERT_PATH/label.py  --verbose --reports_path $REPORT_PATH/$fn --output_path ${fn_stem}_labeled.csv --mention_phrases_dir $CHEXPERT_PATH/phrases/mention --unmention_phrases_dir $CHEXPERT_PATH/phrases/unmention --pre_negation_uncertainty_path $CHEXPERT_PATH/patterns/pre_negation_uncertainty.txt --negation_path $CHEXPERT_PATH/patterns/negation.txt --post_negation_uncertainty_path $CHEXPERT_PATH/patterns/post_negation_uncertainty.txt
-    echo `date`: done!
+    python "$CHEXPERT_PATH/label.py"  --verbose --reports_path "$REPORT_PATH/$fn" --output_path "${fn_stem}_labeled.csv" --mention_phrases_dir "$CHEXPERT_PATH/phrases/mention" --unmention_phrases_dir "$CHEXPERT_PATH/phrases/unmention" --pre_negation_uncertainty_path "$CHEXPERT_PATH/patterns/pre_negation_uncertainty.txt" --negation_path "$CHEXPERT_PATH/patterns/negation.txt" --post_negation_uncertainty_path "$CHEXPERT_PATH/patterns/post_negation_uncertainty.txt"
+    echo "$(date): done!"
     echo ''
 done

mimic-iv-cxr/txt/negbio/run_negbio.sh

@@ -32,8 +32,9 @@ sleep 2
 #  mimic_cxr_001.csv
 #  mimic_cxr_002.csv
 # .. etc
-for fn in `ls $BASE_FOLDER`;
+for fn in "$BASE_FOLDER"/*;
 do
+  fn=${fn##*/}
   echo "Looping through files with mimic_cxr_###.csv pattern."
   # validate it's a mimic_cxr sections file
   if [[ $fn =~ ^mimic_cxr_[0-9]+.csv$ ]];
@@ -44,18 +45,18 @@ do
     # all intermediate files will be saved in this folder
     export OUTPUT_DIR=${INPUT_FILE::-4}
 
-    echo $OUTPUT_DIR - running NegBio..
-    python $NEGBIO_PATH/negbio/negbio_csv2bioc.py --output $OUTPUT_DIR/report $INPUT_FILE
-    python $NEGBIO_PATH/negbio/negbio_pipeline.py section_split --pattern $NEGBIO_PATH/patterns/section_titles_cxr8.txt --output $OUTPUT_DIR/sections $OUTPUT_DIR/report/* --workers=6
-    python $NEGBIO_PATH/negbio/negbio_pipeline.py ssplit --output $OUTPUT_DIR/ssplit $OUTPUT_DIR/sections/* --workers=6
-    python $NEGBIO_PATH/negbio/negbio_pipeline.py parse --output $OUTPUT_DIR/parse $OUTPUT_DIR/ssplit/* --workers=6
-    python $NEGBIO_PATH/negbio/negbio_pipeline.py ptb2ud --output $OUTPUT_DIR/ud $OUTPUT_DIR/parse/* --workers=6
-    python $NEGBIO_PATH/negbio/negbio_pipeline.py dner_regex --phrases_file $NEGBIO_PATH/patterns/chexpert_phrases.yml --output $OUTPUT_DIR/dner $OUTPUT_DIR/ud/* --suffix=.chexpert-regex.xml --workers=6 --overwrite
-    python $NEGBIO_PATH/negbio/negbio_pipeline.py neg2 --output $OUTPUT_DIR/neg --pre-negation-uncertainty-patterns $NEGBIO_PATH/patterns/chexpert_pre_negation_uncertainty.yml --neg-patterns $NEGBIO_PATH/patterns/neg_patterns2.yml --post-negation-uncertainty-patterns $NEGBIO_PATH/patterns/post_negation_uncertainty.yml --neg-regex-patterns $NEGBIO_PATH/patterns/neg_regex_patterns.yml --uncertainty-regex-patterns $NEGBIO_PATH/patterns/uncertainty_regex_patterns.yml $OUTPUT_DIR/dner/* --workers=6
+    echo "$OUTPUT_DIR" - running NegBio..
+    python "$NEGBIO_PATH/negbio/negbio_csv2bioc.py" --output "$OUTPUT_DIR/report" "$INPUT_FILE"
+    python "$NEGBIO_PATH/negbio/negbio_pipeline.py" section_split --pattern "$NEGBIO_PATH/patterns/section_titles_cxr8.txt" --output "$OUTPUT_DIR/sections" "$OUTPUT_DIR"/report/* --workers=6
+    python "$NEGBIO_PATH/negbio/negbio_pipeline.py" ssplit --output "$OUTPUT_DIR/ssplit" "$OUTPUT_DIR"/sections/* --workers=6
+    python "$NEGBIO_PATH/negbio/negbio_pipeline.py" parse --output "$OUTPUT_DIR/parse" "$OUTPUT_DIR"/ssplit/* --workers=6
+    python "$NEGBIO_PATH/negbio/negbio_pipeline.py" ptb2ud --output "$OUTPUT_DIR/ud" "$OUTPUT_DIR"/parse/* --workers=6
+    python "$NEGBIO_PATH/negbio/negbio_pipeline.py" dner_regex --phrases_file "$NEGBIO_PATH/patterns/chexpert_phrases.yml" --output "$OUTPUT_DIR/dner" "$OUTPUT_DIR"/ud/* --suffix=.chexpert-regex.xml --workers=6 --overwrite
+    python "$NEGBIO_PATH/negbio/negbio_pipeline.py" neg2 --output "$OUTPUT_DIR/neg" --pre-negation-uncertainty-patterns "$NEGBIO_PATH/patterns/chexpert_pre_negation_uncertainty.yml" --neg-patterns "$NEGBIO_PATH/patterns/neg_patterns2.yml" --post-negation-uncertainty-patterns "$NEGBIO_PATH/patterns/post_negation_uncertainty.yml" --neg-regex-patterns "$NEGBIO_PATH/patterns/neg_regex_patterns.yml" --uncertainty-regex-patterns "$NEGBIO_PATH/patterns/uncertainty_regex_patterns.yml" "$OUTPUT_DIR"/dner/* --workers=6
 
     # ultimate filename we save the labels to
     export OUTPUT_LABELS=$OUTPUT_DIR/${fn::-4}_labels.csv
-    python $NEGBIO_PATH/negbio/ext/chexpert_collect_labels.py --phrases_file $NEGBIO_PATH/patterns/chexpert_phrases.yml --output $OUTPUT_LABELS $OUTPUT_DIR/neg/*
+    python "$NEGBIO_PATH/negbio/ext/chexpert_collect_labels.py" --phrases_file "$NEGBIO_PATH/patterns/chexpert_phrases.yml" --output "$OUTPUT_LABELS" "$OUTPUT_DIR"/neg/*
   fi
 done
 echo "Done looping through files."

mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh

@@ -58,7 +58,8 @@ elif [ -n "$3" ]; then
     die "Usage: ./import_duckdb.sh mimic_data_dir [output_db]"
 elif [ -s "$OUTFILE" ]; then
   yell "File \"$OUTFILE\" already exists."
-  read -p "Continue? (y/d/n) 'y' continues, 'd' deletes original file, 'n' stops: " yn
+  printf "Continue? (y/d/n) 'y' continues, 'd' deletes original file, 'n' stops: "
+  read -r yn
   case $yn in
       [Yy]* ) ;; # OK
       [Nn]* ) exit;;

mimic-iv-note/buildmimic/duckdb/import_duckdb.sh

@@ -58,7 +58,8 @@ elif [ -n "$3" ]; then
     die "Usage: ./import_duckdb.sh mimic_data_dir [output_db]"
 elif [ -s "$OUTFILE" ]; then
   yell "File \"$OUTFILE\" already exists."
-  read -p "Continue? (y/d/n) 'y' continues, 'd' deletes original file, 'n' stops: " yn
+  printf "Continue? (y/d/n) 'y' continues, 'd' deletes original file, 'n' stops: "
+  read -r yn
   case $yn in
       [Yy]* ) ;; # OK
       [Nn]* ) exit;;

mimic-iv/buildmimic/duckdb/import_duckdb.sh

@@ -58,7 +58,8 @@ elif [ -n "$3" ]; then
     die "Usage: ./import_duckdb.sh mimic_data_dir [output_db]"
 elif [ -s "$OUTFILE" ]; then
   yell "File \"$OUTFILE\" already exists."
-  read -p "Continue? (y/d/n) 'y' continues, 'd' deletes original file, 'n' stops: " yn
+  printf "Continue? (y/d/n) 'y' continues, 'd' deletes original file, 'n' stops: "
+  read -r yn
   case $yn in
       [Yy]* ) ;; # OK
       [Nn]* ) exit;;

mimic-iv/concepts/copy_concepts_to_versioned_schema.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 # This script copies the concepts in the BigQuery table mimiciv_derived to mimiciv_${VERSION}_derived.
-if [ -z "$$1" ]; then
+if [ -z "$1" ]; then
   echo "Usage: $0 <version>"
   exit 1
 fi
@@ -9,29 +9,29 @@ export TARGET_DATASET=mimiciv_$1_derived
 export PROJECT_ID=physionet-data
 
 # check if the target dataset exists
-if bq ls --datasets --project_id ${PROJECT_ID} | grep -q ${TARGET_DATASET}; then
+if bq ls --datasets --project_id "${PROJECT_ID}" | grep -q "${TARGET_DATASET}"; then
     echo "Using existing dataset ${TARGET_DATASET}."
     # drop the existing tables in the target dataset
     # this includes ones which may not be in the source dataset
-    for TABLE in `bq ls ${PROJECT_ID}:${TARGET_DATASET} | cut -d' ' -f3`;
+    for TABLE in $(bq ls "${PROJECT_ID}:${TARGET_DATASET}" | cut -d' ' -f3);
     do
         # skip the first line of dashes
         if [[ "${TABLE:0:2}" == '--' ]]; then
             continue
         fi
-        bq rm -f -q ${PROJECT_ID}:${TARGET_DATASET}.${TABLE}
+        bq rm -f -q "${PROJECT_ID}:${TARGET_DATASET}.${TABLE}"
     done
 else
     echo "Creating dataset ${PROJECT_ID}:${TARGET_DATASET}"
-    bq mk --dataset ${PROJECT_ID}:${TARGET_DATASET}
+    bq mk --dataset "${PROJECT_ID}:${TARGET_DATASET}"
 fi
 
 echo "Copying tables from ${SOURCE_DATASET} to ${TARGET_DATASET}."
-for TABLE in `bq ls -n 500 ${PROJECT_ID}:${SOURCE_DATASET} | cut -d' ' -f3`;
+for TABLE in $(bq ls -n 500 "${PROJECT_ID}:${SOURCE_DATASET}" | cut -d' ' -f3);
 do
     # skip the first line of dashes
     if [[ "${TABLE:0:2}" == '--' ]]; then
       continue
     fi
-    bq cp -f -q ${PROJECT_ID}:${SOURCE_DATASET}.${TABLE} ${PROJECT_ID}:${TARGET_DATASET}.${TABLE}
+    bq cp -f -q "${PROJECT_ID}:${SOURCE_DATASET}.${TABLE}" "${PROJECT_ID}:${TARGET_DATASET}.${TABLE}"
 done

mimic-iv/concepts/make_concepts.sh

@@ -6,17 +6,17 @@ export MIMIC_VERSION="3.1"
 
 # specify bigquery query command options
 # note: max_rows=1 *displays* only one row, but all rows are inserted into the destination table
-BQ_OPTIONS='--quiet --headless --max_rows=0 --use_legacy_sql=False --replace'
+BQ_OPTIONS=(--quiet --headless --max_rows=0 --use_legacy_sql=False --replace)
 
 # drop the existing tables in the target dataset
-for TABLE in `bq ls physionet-data:${TARGET_DATASET} | cut -d' ' -f3`;
+for TABLE in $(bq ls "physionet-data:${TARGET_DATASET}" | cut -d' ' -f3);
 do
     # skip the first line of dashes
     if [[ "${TABLE:0:2}" == '--' ]]; then
       continue
     fi
   echo "Dropping table ${TARGET_DATASET}.${TABLE}"
-  bq rm -f -q ${TARGET_DATASET}.${TABLE}
+  bq rm -f -q "${TARGET_DATASET}.${TABLE}"
 done
 
 # create a _version table to store the mimic-iv version, git commit hash, and latest git tag
@@ -40,11 +40,12 @@ VALUES
 EOF
 
 # generate a few tables first as the desired order isn't alphabetical
+# shellcheck disable=SC2043  # single-entry loop kept so more tables can be added
 for table_path in demographics/icustay_times;
 do
-  table=`echo $table_path | rev | cut -d/ -f1 | rev`
+  table=$(echo "${table_path}" | rev | cut -d/ -f1 | rev)
   echo "Generating ${TARGET_DATASET}.${table}"
-  bq query ${BQ_OPTIONS} --destination_table=${TARGET_DATASET}.${table} < ${table_path}.sql
+  bq query "${BQ_OPTIONS[@]}" --destination_table="${TARGET_DATASET}.${table}" < "${table_path}.sql"
 done
 
 # generate tables in subfolders
@@ -54,12 +55,13 @@ done
 # * organfailure depends on measurement
 for d in demographics comorbidity measurement medication organfailure treatment firstday score sepsis;
 do
-    for fn in `ls $d`;
+    for fn_path in "${d}"/*;
     do
+        fn=${fn_path##*/}
         # only run SQL queries
         if [[ "${fn: -4}" == ".sql" ]]; then
             # table name is file name minus extension
-            tbl=`echo $fn | rev | cut -d. -f2- | rev`
+            tbl=$(echo "${fn}" | rev | cut -d. -f2- | rev)
 
             # skip certain tables where order matters
             skip=0
@@ -76,7 +78,7 @@ do
 
             # not skipping - so generate the table on bigquery
             echo "Generating ${TARGET_DATASET}.${tbl}"
-            bq query ${BQ_OPTIONS} --destination_table=${TARGET_DATASET}.${tbl} < ${d}/${fn}
+            bq query "${BQ_OPTIONS[@]}" --destination_table="${TARGET_DATASET}.${tbl}" < "${fn_path}"
         fi
     done
 done
@@ -85,8 +87,8 @@ echo "Now generating tables which were skipped due to depending on other tables.
 # generate tables after the above, and in a specific order to ensure dependencies are met
 for table_path in firstday/first_day_sofa organfailure/kdigo_stages organfailure/meld medication/vasoactive_agent medication/norepinephrine_equivalent_dose sepsis/sepsis3;
 do
-  table=`echo $table_path | rev | cut -d/ -f1 | rev`
+  table=$(echo "${table_path}" | rev | cut -d/ -f1 | rev)
 
   echo "Generating ${TARGET_DATASET}.${table}"
-  bq query ${BQ_OPTIONS} --destination_table=${TARGET_DATASET}.${table} < ${table_path}.sql
+  bq query "${BQ_OPTIONS[@]}" --destination_table="${TARGET_DATASET}.${table}" < "${table_path}.sql"
 done