From cfb181889814e69f83551ae6e44a69e57aa3dd29 Mon Sep 17 00:00:00 2001 From: Alistair Johnson Date: Sat, 25 Jul 2026 20:43:20 -0400 Subject: [PATCH] add shellcheck --- .github/workflows/shellcheck.yml | 20 +++ mimic-iii/buildmimic/duckdb/import_duckdb.sh | 3 +- .../buildmimic/postgres/create_mimic_user.sh | 2 +- mimic-iii/concepts/make-concepts.sh | 143 +++++++++--------- mimic-iii/tests/md5.sh | 17 ++- .../txt/chexpert/run_chexpert_on_files.sh | 11 +- mimic-iv-cxr/txt/negbio/run_negbio.sh | 21 +-- .../buildmimic/duckdb/import_duckdb.sh | 3 +- .../buildmimic/duckdb/import_duckdb.sh | 3 +- mimic-iv/buildmimic/duckdb/import_duckdb.sh | 3 +- .../copy_concepts_to_versioned_schema.sh | 14 +- mimic-iv/concepts/make_concepts.sh | 22 +-- 12 files changed, 146 insertions(+), 116 deletions(-) create mode 100644 .github/workflows/shellcheck.yml diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml new file mode 100644 index 000000000..d85a517c8 --- /dev/null +++ b/.github/workflows/shellcheck.yml @@ -0,0 +1,20 @@ +# Lint all shell scripts in the repository with ShellCheck. +# Picks up any *.sh file in the repo excluding .git +name: ShellCheck + +on: + push: + branches: + - main + pull_request: + +jobs: + shellcheck: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v7 + - name: Run ShellCheck + run: | + find . -type f -name '*.sh' -not -path './.git/*' -print0 \ + | xargs -0 shellcheck diff --git a/mimic-iii/buildmimic/duckdb/import_duckdb.sh b/mimic-iii/buildmimic/duckdb/import_duckdb.sh index 77b76c7d1..3eb1e1296 100755 --- a/mimic-iii/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iii/buildmimic/duckdb/import_duckdb.sh @@ -58,7 +58,8 @@ elif [ -n "$3" ]; then die "Usage: ./import_duckdb.sh mimic_data_dir [output_db]" elif [ -s "$OUTFILE" ]; then yell "File \"$OUTFILE\" already exists." - read -p "Continue? (y/d/n) 'y' continues, 'd' deletes original file, 'n' stops: " yn + printf "Continue? (y/d/n) 'y' continues, 'd' deletes original file, 'n' stops: " + read -r yn case $yn in [Yy]* ) ;; # OK [Nn]* ) exit;; diff --git a/mimic-iii/buildmimic/postgres/create_mimic_user.sh b/mimic-iii/buildmimic/postgres/create_mimic_user.sh index 772764ce7..9b333dcfa 100755 --- a/mimic-iii/buildmimic/postgres/create_mimic_user.sh +++ b/mimic-iii/buildmimic/postgres/create_mimic_user.sh @@ -41,7 +41,7 @@ else fi # check if SUDO is needed by checking if we can login with postgres without it -err2=`psql postgres postgres -c "select 1;" 2>&1 >/dev/null` +err2=$(psql postgres postgres -c "select 1;" 2>&1 >/dev/null) if [[ $err2 == *"Peer authentication failed for user"* ]]; then # we need to call sudo every time for postgres diff --git a/mimic-iii/concepts/make-concepts.sh b/mimic-iii/concepts/make-concepts.sh index a7d1152c5..8b0707a14 100644 --- a/mimic-iii/concepts/make-concepts.sh +++ b/mimic-iii/concepts/make-concepts.sh @@ -1,8 +1,9 @@ +#!/bin/bash # set the project # gcloud config set project physionet-data export TARGET_DATASET='mimiciii_derived' -export BQ_FLAGS='--use_legacy_sql=False --replace' +BQ_FLAGS=(--use_legacy_sql=False --replace) echo '' echo '===' @@ -14,108 +15,108 @@ echo '' set -x echo 'Top level files..' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.code_status < code_status.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.echo_data < echo_data.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".code_status < code_status.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".echo_data < echo_data.sql echo 'Running queries in 10 directories.' echo 'Directory 1: demographics' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.heightweight < demographics/heightweight.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.icustay_detail < demographics/icustay_detail.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".heightweight < demographics/heightweight.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".icustay_detail < demographics/icustay_detail.sql # Durations (usually of treatments) echo 'Directory 2: durations' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.ventilation_classification < durations/ventilation_classification.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.ventilation_durations < durations/ventilation_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.crrt_durations < durations/crrt_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.adenosine_durations < durations/adenosine_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.dobutamine_durations < durations/dobutamine_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.dopamine_durations < durations/dopamine_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.epinephrine_durations < durations/epinephrine_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.isuprel_durations < durations/isuprel_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.milrinone_durations < durations/milrinone_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.norepinephrine_durations < durations/norepinephrine_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.phenylephrine_durations < durations/phenylephrine_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.vasopressin_durations < durations/vasopressin_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.vasopressor_durations < durations/vasopressor_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.weight_durations < durations/weight_durations.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".ventilation_classification < durations/ventilation_classification.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".ventilation_durations < durations/ventilation_durations.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".crrt_durations < durations/crrt_durations.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".adenosine_durations < durations/adenosine_durations.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".dobutamine_durations < durations/dobutamine_durations.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".dopamine_durations < durations/dopamine_durations.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".epinephrine_durations < durations/epinephrine_durations.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".isuprel_durations < durations/isuprel_durations.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".milrinone_durations < durations/milrinone_durations.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".norepinephrine_durations < durations/norepinephrine_durations.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".phenylephrine_durations < durations/phenylephrine_durations.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".vasopressin_durations < durations/vasopressin_durations.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".vasopressor_durations < durations/vasopressor_durations.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".weight_durations < durations/weight_durations.sql # dose queries for vasopressors -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.dobutamine_dose < durations/dobutamine_dose.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.dopamine_dose < durations/dopamine_dose.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.epinephrine_dose < durations/epinephrine_dose.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.norepinephrine_dose < durations/norepinephrine_dose.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.phenylephrine_dose < durations/phenylephrine_dose.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.vasopressin_dose < durations/vasopressin_dose.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".dobutamine_dose < durations/dobutamine_dose.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".dopamine_dose < durations/dopamine_dose.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".epinephrine_dose < durations/epinephrine_dose.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".norepinephrine_dose < durations/norepinephrine_dose.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".phenylephrine_dose < durations/phenylephrine_dose.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".vasopressin_dose < durations/vasopressin_dose.sql # "pivoted" tables which have icustay_id / timestamp as the primary key echo 'Directory 3: pivoted tables' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_vital < pivot/pivoted_vital.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_uo < pivot/pivoted_uo.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_rrt < pivot/pivoted_rrt.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_lab < pivot/pivoted_lab.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_invasive_lines < pivot/pivoted_invasive_lines.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_icp < pivot/pivoted_icp.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_height < pivot/pivoted_height.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_gcs < pivot/pivoted_gcs.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_fio2 < pivot/pivoted_fio2.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_bg < pivot/pivoted_bg.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_vital < pivot/pivoted_vital.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_uo < pivot/pivoted_uo.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_rrt < pivot/pivoted_rrt.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_lab < pivot/pivoted_lab.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_invasive_lines < pivot/pivoted_invasive_lines.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_icp < pivot/pivoted_icp.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_height < pivot/pivoted_height.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_gcs < pivot/pivoted_gcs.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_fio2 < pivot/pivoted_fio2.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_bg < pivot/pivoted_bg.sql # pivoted_bg_art must be run after pivoted_bg -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_bg_art < pivot/pivoted_bg_art.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_bg_art < pivot/pivoted_bg_art.sql # pivoted oasis depends on icustay_hours in demographics -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_oasis < pivot/pivoted_oasis.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_oasis < pivot/pivoted_oasis.sql # pivoted sofa depends on many above pivoted views, ventilation_durations, and dose queries -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_sofa < pivot/pivoted_sofa.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".pivoted_sofa < pivot/pivoted_sofa.sql echo 'Directory 4: comorbidity' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.elixhauser_ahrq_v37 < comorbidity/elixhauser_ahrq_v37.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.elixhauser_ahrq_v37_no_drg < comorbidity/elixhauser_ahrq_v37_no_drg.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.elixhauser_quan < comorbidity/elixhauser_quan.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.elixhauser_score_ahrq < comorbidity/elixhauser_score_ahrq.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.elixhauser_score_quan < comorbidity/elixhauser_score_quan.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".elixhauser_ahrq_v37 < comorbidity/elixhauser_ahrq_v37.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".elixhauser_ahrq_v37_no_drg < comorbidity/elixhauser_ahrq_v37_no_drg.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".elixhauser_quan < comorbidity/elixhauser_quan.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".elixhauser_score_ahrq < comorbidity/elixhauser_score_ahrq.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".elixhauser_score_quan < comorbidity/elixhauser_score_quan.sql echo 'Directory 5: firstday' # data which is extracted from a patient's first ICU stay -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.blood_gas_first_day < firstday/blood_gas_first_day.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.blood_gas_first_day_arterial < firstday/blood_gas_first_day_arterial.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.gcs_first_day < firstday/gcs_first_day.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.labs_first_day < firstday/labs_first_day.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.rrt_first_day < firstday/rrt_first_day.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.urine_output_first_day < firstday/urine_output_first_day.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.ventilation_first_day < firstday/ventilation_first_day.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.vitals_first_day < firstday/vitals_first_day.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.weight_first_day < firstday/weight_first_day.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".blood_gas_first_day < firstday/blood_gas_first_day.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".blood_gas_first_day_arterial < firstday/blood_gas_first_day_arterial.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".gcs_first_day < firstday/gcs_first_day.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".labs_first_day < firstday/labs_first_day.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".rrt_first_day < firstday/rrt_first_day.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".urine_output_first_day < firstday/urine_output_first_day.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".ventilation_first_day < firstday/ventilation_first_day.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".vitals_first_day < firstday/vitals_first_day.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".weight_first_day < firstday/weight_first_day.sql echo 'Directory 6: fluid_balance' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.urine_output < fluid_balance/urine_output.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".urine_output < fluid_balance/urine_output.sql echo 'Directory 7: sepsis' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.angus < sepsis/angus.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.martin < sepsis/martin.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.explicit < sepsis/explicit.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".angus < sepsis/angus.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".martin < sepsis/martin.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".explicit < sepsis/explicit.sql # diagnosis mapping using CCS echo 'Directory 8: diagnosis' # load the ccs_multi_dx.csv.gz file into bq -bq load --source_format=CSV ${TARGET_DATASET}.ccs_multi_dx diagnosis/ccs_multi_dx.csv.gz diagnosis/ccs_multi_dx.json -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.ccs_dx < diagnosis/ccs_dx.sql +bq load --source_format=CSV "${TARGET_DATASET}".ccs_multi_dx diagnosis/ccs_multi_dx.csv.gz diagnosis/ccs_multi_dx.json +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".ccs_dx < diagnosis/ccs_dx.sql # Organ failure scores echo 'Directory 9: organfailure' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.kdigo_creatinine < organfailure/kdigo_creatinine.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.kdigo_uo < organfailure/kdigo_uo.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.kdigo_stages < organfailure/kdigo_stages.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.kdigo_stages_7day < organfailure/kdigo_stages_7day.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.kdigo_stages_48hr < organfailure/kdigo_stages_48hr.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.meld < organfailure/meld.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".kdigo_creatinine < organfailure/kdigo_creatinine.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".kdigo_uo < organfailure/kdigo_uo.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".kdigo_stages < organfailure/kdigo_stages.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".kdigo_stages_7day < organfailure/kdigo_stages_7day.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".kdigo_stages_48hr < organfailure/kdigo_stages_48hr.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".meld < organfailure/meld.sql # Severity of illness scores (requires many views from above) echo 'Directory 10: severityscores' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.oasis < severityscores/oasis.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.sofa < severityscores/sofa.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.saps < severityscores/saps.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.sapsii < severityscores/sapsii.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.apsiii < severityscores/apsiii.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.lods < severityscores/lods.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.sirs < severityscores/sirs.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".oasis < severityscores/oasis.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".sofa < severityscores/sofa.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".saps < severityscores/saps.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".sapsii < severityscores/sapsii.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".apsiii < severityscores/apsiii.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".lods < severityscores/lods.sql +bq query "${BQ_FLAGS[@]}" --destination_table="${TARGET_DATASET}".sirs < severityscores/sirs.sql echo 'Finished creating concepts.' \ No newline at end of file diff --git a/mimic-iii/tests/md5.sh b/mimic-iii/tests/md5.sh index e78de75a0..578983bb4 100644 --- a/mimic-iii/tests/md5.sh +++ b/mimic-iii/tests/md5.sh @@ -1,20 +1,21 @@ +#!/bin/bash # run in the folder: # ./md5.sh checksum_md5_zipped.txt # OR # ./md5.sh checksum_md5_unzipped.txt -while read p; do - A="$(echo $p | cut -d' ' -f1)" - B="$(echo $p | cut -d' ' -f2)" - if [ ! -f $B ] +while read -r p; do + A="$(echo "$p" | cut -d' ' -f1)" + B="$(echo "$p" | cut -d' ' -f2)" + if [ ! -f "$B" ] then echo "ERROR: $B does not exist" continue fi - read C< <(md5 -q $B) - echo $B $A = $C - if [ $C != $A ] + read -r C < <(md5 -q "$B") + echo "$B" "$A" = "$C" + if [ "$C" != "$A" ] then echo "ERROR: $B wrong md5" exit fi -done <$1 +done < "$1" diff --git a/mimic-iv-cxr/txt/chexpert/run_chexpert_on_files.sh b/mimic-iv-cxr/txt/chexpert/run_chexpert_on_files.sh index 52e34e0ff..2c2ef14fb 100644 --- a/mimic-iv-cxr/txt/chexpert/run_chexpert_on_files.sh +++ b/mimic-iv-cxr/txt/chexpert/run_chexpert_on_files.sh @@ -22,11 +22,12 @@ fi sleep 2 # loop through each .csv file in the section folder -for fn in `ls $REPORT_PATH`; do - echo `date`: $fn - fn_stem=`echo $fn | cut -d. -f 1` +for fn in "$REPORT_PATH"/*; do + fn=${fn##*/} + echo "$(date): $fn" + fn_stem=$(echo "$fn" | cut -d. -f 1) # run chexpert - must be run from chexpert folder - python $CHEXPERT_PATH/label.py --verbose --reports_path $REPORT_PATH/$fn --output_path ${fn_stem}_labeled.csv --mention_phrases_dir $CHEXPERT_PATH/phrases/mention --unmention_phrases_dir $CHEXPERT_PATH/phrases/unmention --pre_negation_uncertainty_path $CHEXPERT_PATH/patterns/pre_negation_uncertainty.txt --negation_path $CHEXPERT_PATH/patterns/negation.txt --post_negation_uncertainty_path $CHEXPERT_PATH/patterns/post_negation_uncertainty.txt - echo `date`: done! + python "$CHEXPERT_PATH/label.py" --verbose --reports_path "$REPORT_PATH/$fn" --output_path "${fn_stem}_labeled.csv" --mention_phrases_dir "$CHEXPERT_PATH/phrases/mention" --unmention_phrases_dir "$CHEXPERT_PATH/phrases/unmention" --pre_negation_uncertainty_path "$CHEXPERT_PATH/patterns/pre_negation_uncertainty.txt" --negation_path "$CHEXPERT_PATH/patterns/negation.txt" --post_negation_uncertainty_path "$CHEXPERT_PATH/patterns/post_negation_uncertainty.txt" + echo "$(date): done!" echo '' done \ No newline at end of file diff --git a/mimic-iv-cxr/txt/negbio/run_negbio.sh b/mimic-iv-cxr/txt/negbio/run_negbio.sh index 13cba4795..19d0f7f43 100755 --- a/mimic-iv-cxr/txt/negbio/run_negbio.sh +++ b/mimic-iv-cxr/txt/negbio/run_negbio.sh @@ -32,8 +32,9 @@ sleep 2 # mimic_cxr_001.csv # mimic_cxr_002.csv # .. etc -for fn in `ls $BASE_FOLDER`; +for fn in "$BASE_FOLDER"/*; do + fn=${fn##*/} echo "Looping through files with mimic_cxr_###.csv pattern." # validate it's a mimic_cxr sections file if [[ $fn =~ ^mimic_cxr_[0-9]+.csv$ ]]; @@ -44,18 +45,18 @@ do # all intermediate files will be saved in this folder export OUTPUT_DIR=${INPUT_FILE::-4} - echo $OUTPUT_DIR - running NegBio.. - python $NEGBIO_PATH/negbio/negbio_csv2bioc.py --output $OUTPUT_DIR/report $INPUT_FILE - python $NEGBIO_PATH/negbio/negbio_pipeline.py section_split --pattern $NEGBIO_PATH/patterns/section_titles_cxr8.txt --output $OUTPUT_DIR/sections $OUTPUT_DIR/report/* --workers=6 - python $NEGBIO_PATH/negbio/negbio_pipeline.py ssplit --output $OUTPUT_DIR/ssplit $OUTPUT_DIR/sections/* --workers=6 - python $NEGBIO_PATH/negbio/negbio_pipeline.py parse --output $OUTPUT_DIR/parse $OUTPUT_DIR/ssplit/* --workers=6 - python $NEGBIO_PATH/negbio/negbio_pipeline.py ptb2ud --output $OUTPUT_DIR/ud $OUTPUT_DIR/parse/* --workers=6 - python $NEGBIO_PATH/negbio/negbio_pipeline.py dner_regex --phrases_file $NEGBIO_PATH/patterns/chexpert_phrases.yml --output $OUTPUT_DIR/dner $OUTPUT_DIR/ud/* --suffix=.chexpert-regex.xml --workers=6 --overwrite - python $NEGBIO_PATH/negbio/negbio_pipeline.py neg2 --output $OUTPUT_DIR/neg --pre-negation-uncertainty-patterns $NEGBIO_PATH/patterns/chexpert_pre_negation_uncertainty.yml --neg-patterns $NEGBIO_PATH/patterns/neg_patterns2.yml --post-negation-uncertainty-patterns $NEGBIO_PATH/patterns/post_negation_uncertainty.yml --neg-regex-patterns $NEGBIO_PATH/patterns/neg_regex_patterns.yml --uncertainty-regex-patterns $NEGBIO_PATH/patterns/uncertainty_regex_patterns.yml $OUTPUT_DIR/dner/* --workers=6 + echo "$OUTPUT_DIR" - running NegBio.. + python "$NEGBIO_PATH/negbio/negbio_csv2bioc.py" --output "$OUTPUT_DIR/report" "$INPUT_FILE" + python "$NEGBIO_PATH/negbio/negbio_pipeline.py" section_split --pattern "$NEGBIO_PATH/patterns/section_titles_cxr8.txt" --output "$OUTPUT_DIR/sections" "$OUTPUT_DIR"/report/* --workers=6 + python "$NEGBIO_PATH/negbio/negbio_pipeline.py" ssplit --output "$OUTPUT_DIR/ssplit" "$OUTPUT_DIR"/sections/* --workers=6 + python "$NEGBIO_PATH/negbio/negbio_pipeline.py" parse --output "$OUTPUT_DIR/parse" "$OUTPUT_DIR"/ssplit/* --workers=6 + python "$NEGBIO_PATH/negbio/negbio_pipeline.py" ptb2ud --output "$OUTPUT_DIR/ud" "$OUTPUT_DIR"/parse/* --workers=6 + python "$NEGBIO_PATH/negbio/negbio_pipeline.py" dner_regex --phrases_file "$NEGBIO_PATH/patterns/chexpert_phrases.yml" --output "$OUTPUT_DIR/dner" "$OUTPUT_DIR"/ud/* --suffix=.chexpert-regex.xml --workers=6 --overwrite + python "$NEGBIO_PATH/negbio/negbio_pipeline.py" neg2 --output "$OUTPUT_DIR/neg" --pre-negation-uncertainty-patterns "$NEGBIO_PATH/patterns/chexpert_pre_negation_uncertainty.yml" --neg-patterns "$NEGBIO_PATH/patterns/neg_patterns2.yml" --post-negation-uncertainty-patterns "$NEGBIO_PATH/patterns/post_negation_uncertainty.yml" --neg-regex-patterns "$NEGBIO_PATH/patterns/neg_regex_patterns.yml" --uncertainty-regex-patterns "$NEGBIO_PATH/patterns/uncertainty_regex_patterns.yml" "$OUTPUT_DIR"/dner/* --workers=6 # ultimate filename we save the labels to export OUTPUT_LABELS=$OUTPUT_DIR/${fn::-4}_labels.csv - python $NEGBIO_PATH/negbio/ext/chexpert_collect_labels.py --phrases_file $NEGBIO_PATH/patterns/chexpert_phrases.yml --output $OUTPUT_LABELS $OUTPUT_DIR/neg/* + python "$NEGBIO_PATH/negbio/ext/chexpert_collect_labels.py" --phrases_file "$NEGBIO_PATH/patterns/chexpert_phrases.yml" --output "$OUTPUT_LABELS" "$OUTPUT_DIR"/neg/* fi done echo "Done looping through files." diff --git a/mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh b/mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh index 88cf287b5..102e92d8f 100644 --- a/mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh @@ -58,7 +58,8 @@ elif [ -n "$3" ]; then die "Usage: ./import_duckdb.sh mimic_data_dir [output_db]" elif [ -s "$OUTFILE" ]; then yell "File \"$OUTFILE\" already exists." - read -p "Continue? (y/d/n) 'y' continues, 'd' deletes original file, 'n' stops: " yn + printf "Continue? (y/d/n) 'y' continues, 'd' deletes original file, 'n' stops: " + read -r yn case $yn in [Yy]* ) ;; # OK [Nn]* ) exit;; diff --git a/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh b/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh index b194212fb..20e283139 100644 --- a/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh @@ -58,7 +58,8 @@ elif [ -n "$3" ]; then die "Usage: ./import_duckdb.sh mimic_data_dir [output_db]" elif [ -s "$OUTFILE" ]; then yell "File \"$OUTFILE\" already exists." - read -p "Continue? (y/d/n) 'y' continues, 'd' deletes original file, 'n' stops: " yn + printf "Continue? (y/d/n) 'y' continues, 'd' deletes original file, 'n' stops: " + read -r yn case $yn in [Yy]* ) ;; # OK [Nn]* ) exit;; diff --git a/mimic-iv/buildmimic/duckdb/import_duckdb.sh b/mimic-iv/buildmimic/duckdb/import_duckdb.sh index 8e72733c3..4ff9fc219 100755 --- a/mimic-iv/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iv/buildmimic/duckdb/import_duckdb.sh @@ -58,7 +58,8 @@ elif [ -n "$3" ]; then die "Usage: ./import_duckdb.sh mimic_data_dir [output_db]" elif [ -s "$OUTFILE" ]; then yell "File \"$OUTFILE\" already exists." - read -p "Continue? (y/d/n) 'y' continues, 'd' deletes original file, 'n' stops: " yn + printf "Continue? (y/d/n) 'y' continues, 'd' deletes original file, 'n' stops: " + read -r yn case $yn in [Yy]* ) ;; # OK [Nn]* ) exit;; diff --git a/mimic-iv/concepts/copy_concepts_to_versioned_schema.sh b/mimic-iv/concepts/copy_concepts_to_versioned_schema.sh index df0afa8fa..aca68b661 100644 --- a/mimic-iv/concepts/copy_concepts_to_versioned_schema.sh +++ b/mimic-iv/concepts/copy_concepts_to_versioned_schema.sh @@ -1,6 +1,6 @@ #!/bin/bash # This script copies the concepts in the BigQuery table mimiciv_derived to mimiciv_${VERSION}_derived. -if [ -z "$$1" ]; then +if [ -z "$1" ]; then echo "Usage: $0 " exit 1 fi @@ -9,29 +9,29 @@ export TARGET_DATASET=mimiciv_$1_derived export PROJECT_ID=physionet-data # check if the target dataset exists -if bq ls --datasets --project_id ${PROJECT_ID} | grep -q ${TARGET_DATASET}; then +if bq ls --datasets --project_id "${PROJECT_ID}" | grep -q "${TARGET_DATASET}"; then echo "Using existing dataset ${TARGET_DATASET}." # drop the existing tables in the target dataset # this includes ones which may not be in the source dataset - for TABLE in `bq ls ${PROJECT_ID}:${TARGET_DATASET} | cut -d' ' -f3`; + for TABLE in $(bq ls "${PROJECT_ID}:${TARGET_DATASET}" | cut -d' ' -f3); do # skip the first line of dashes if [[ "${TABLE:0:2}" == '--' ]]; then continue fi - bq rm -f -q ${PROJECT_ID}:${TARGET_DATASET}.${TABLE} + bq rm -f -q "${PROJECT_ID}:${TARGET_DATASET}.${TABLE}" done else echo "Creating dataset ${PROJECT_ID}:${TARGET_DATASET}" - bq mk --dataset ${PROJECT_ID}:${TARGET_DATASET} + bq mk --dataset "${PROJECT_ID}:${TARGET_DATASET}" fi echo "Copying tables from ${SOURCE_DATASET} to ${TARGET_DATASET}." -for TABLE in `bq ls -n 500 ${PROJECT_ID}:${SOURCE_DATASET} | cut -d' ' -f3`; +for TABLE in $(bq ls -n 500 "${PROJECT_ID}:${SOURCE_DATASET}" | cut -d' ' -f3); do # skip the first line of dashes if [[ "${TABLE:0:2}" == '--' ]]; then continue fi - bq cp -f -q ${PROJECT_ID}:${SOURCE_DATASET}.${TABLE} ${PROJECT_ID}:${TARGET_DATASET}.${TABLE} + bq cp -f -q "${PROJECT_ID}:${SOURCE_DATASET}.${TABLE}" "${PROJECT_ID}:${TARGET_DATASET}.${TABLE}" done diff --git a/mimic-iv/concepts/make_concepts.sh b/mimic-iv/concepts/make_concepts.sh index 112bcdc22..78c64ac15 100644 --- a/mimic-iv/concepts/make_concepts.sh +++ b/mimic-iv/concepts/make_concepts.sh @@ -6,17 +6,17 @@ export MIMIC_VERSION="3.1" # specify bigquery query command options # note: max_rows=1 *displays* only one row, but all rows are inserted into the destination table -BQ_OPTIONS='--quiet --headless --max_rows=0 --use_legacy_sql=False --replace' +BQ_OPTIONS=(--quiet --headless --max_rows=0 --use_legacy_sql=False --replace) # drop the existing tables in the target dataset -for TABLE in `bq ls physionet-data:${TARGET_DATASET} | cut -d' ' -f3`; +for TABLE in $(bq ls "physionet-data:${TARGET_DATASET}" | cut -d' ' -f3); do # skip the first line of dashes if [[ "${TABLE:0:2}" == '--' ]]; then continue fi echo "Dropping table ${TARGET_DATASET}.${TABLE}" - bq rm -f -q ${TARGET_DATASET}.${TABLE} + bq rm -f -q "${TARGET_DATASET}.${TABLE}" done # create a _version table to store the mimic-iv version, git commit hash, and latest git tag @@ -40,11 +40,12 @@ VALUES EOF # generate a few tables first as the desired order isn't alphabetical +# shellcheck disable=SC2043 # single-entry loop kept so more tables can be added for table_path in demographics/icustay_times; do - table=`echo $table_path | rev | cut -d/ -f1 | rev` + table=$(echo "${table_path}" | rev | cut -d/ -f1 | rev) echo "Generating ${TARGET_DATASET}.${table}" - bq query ${BQ_OPTIONS} --destination_table=${TARGET_DATASET}.${table} < ${table_path}.sql + bq query "${BQ_OPTIONS[@]}" --destination_table="${TARGET_DATASET}.${table}" < "${table_path}.sql" done # generate tables in subfolders @@ -54,12 +55,13 @@ done # * organfailure depends on measurement for d in demographics comorbidity measurement medication organfailure treatment firstday score sepsis; do - for fn in `ls $d`; + for fn_path in "${d}"/*; do + fn=${fn_path##*/} # only run SQL queries if [[ "${fn: -4}" == ".sql" ]]; then # table name is file name minus extension - tbl=`echo $fn | rev | cut -d. -f2- | rev` + tbl=$(echo "${fn}" | rev | cut -d. -f2- | rev) # skip certain tables where order matters skip=0 @@ -76,7 +78,7 @@ do # not skipping - so generate the table on bigquery echo "Generating ${TARGET_DATASET}.${tbl}" - bq query ${BQ_OPTIONS} --destination_table=${TARGET_DATASET}.${tbl} < ${d}/${fn} + bq query "${BQ_OPTIONS[@]}" --destination_table="${TARGET_DATASET}.${tbl}" < "${fn_path}" fi done done @@ -85,8 +87,8 @@ echo "Now generating tables which were skipped due to depending on other tables. # generate tables after the above, and in a specific order to ensure dependencies are met for table_path in firstday/first_day_sofa organfailure/kdigo_stages organfailure/meld medication/vasoactive_agent medication/norepinephrine_equivalent_dose sepsis/sepsis3; do - table=`echo $table_path | rev | cut -d/ -f1 | rev` + table=$(echo "${table_path}" | rev | cut -d/ -f1 | rev) echo "Generating ${TARGET_DATASET}.${table}" - bq query ${BQ_OPTIONS} --destination_table=${TARGET_DATASET}.${table} < ${table_path}.sql + bq query "${BQ_OPTIONS[@]}" --destination_table="${TARGET_DATASET}.${table}" < "${table_path}.sql" done