From 0d36c15505433f9e816042a6c4b59440bdf0437c Mon Sep 17 00:00:00 2001 From: Taksh Date: Sat, 25 Jul 2026 08:58:17 +0300 Subject: [PATCH 1/3] fix: load paths, imports, transpile, and CI permissions Quote/escape datadirs and passwords in loaders, accept plain CSV in duckdb/sqlite imports, utf-8 transpile writes, and don't fail sqlfluff annotate on fork PRs. Co-authored-by: Cursor --- .github/workflows/lint_sqlfluff.yml | 6 +- mimic-iii/buildmimic/duckdb/import_duckdb.sh | 8 +- mimic-iii/buildmimic/postgres/Makefile | 10 +- .../buildmimic/postgres/create_mimic_user.sh | 15 +- mimic-iii/buildmimic/sqlite/import.py | 53 ++++++- mimic-iii/buildmimic/sqlite/import.sh | 6 +- mimic-iii/concepts/make-concepts.sh | 138 +++++++++--------- .../demographics/icustay_hours.sql | 2 +- mimic-iii/concepts_duckdb/duckdb.sql | 4 +- .../postgres-make-concepts.sql | 2 +- .../txt/chexpert/run_chexpert_on_files.sh | 18 ++- mimic-iv-cxr/txt/negbio/run_negbio.sh | 21 +-- .../buildmimic/duckdb/import_duckdb.sh | 21 ++- .../buildmimic/duckdb/import_duckdb.sh | 21 ++- mimic-iv/buildmimic/duckdb/import_duckdb.sh | 26 +++- mimic-iv/buildmimic/mysql/validate_demo.sql | 6 + .../buildmimic/postgres/validate_demo.sql | 6 + mimic-iv/buildmimic/sqlite/import.py | 4 +- mimic-iv/buildmimic/sqlite/import.sh | 6 +- mimic-iv/concepts/validate_concepts.sh | 5 +- src/mimic_utils/sqlglot_dialects/duckdb.py | 12 ++ src/mimic_utils/sqlglot_dialects/postgres.py | 16 ++ src/mimic_utils/transpile.py | 2 +- tests/test_transpile.py | 7 + 24 files changed, 284 insertions(+), 131 deletions(-) diff --git a/.github/workflows/lint_sqlfluff.yml b/.github/workflows/lint_sqlfluff.yml index 0986785bf..139834acc 100644 --- a/.github/workflows/lint_sqlfluff.yml +++ b/.github/workflows/lint_sqlfluff.yml @@ -35,8 +35,12 @@ jobs: shell: bash run: sqlfluff lint --format github-annotation --annotation-level failure --nofail ${{ steps.get_files_to_lint.outputs.lintees }} > annotations.json - name: Annotate + # Fork PRs cannot create check runs with GITHUB_TOKEN; lint already ran. + continue-on-error: true + if: steps.get_files_to_lint.outputs.lintees != '' uses: yuzutech/annotations-action@v0.6.0 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" title: "SQLFluff Lint" - input: "./annotations.json" \ No newline at end of file + input: "./annotations.json" + ignore-missing-file: true \ No newline at end of file diff --git a/mimic-iii/buildmimic/duckdb/import_duckdb.sh b/mimic-iii/buildmimic/duckdb/import_duckdb.sh index 3eb1e1296..36d332208 100755 --- a/mimic-iii/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iii/buildmimic/duckdb/import_duckdb.sh @@ -29,7 +29,7 @@ usage () { die " USAGE: ./import_duckdb.sh mimic_data_dir [output_db] WHERE: - mimic_data_dir directory that contains csv.gz or csv files + mimic_data_dir directory that contains csv.tar.gz or csv files output_db: optional filename for duckdb file (default: mimic3.db)\ " } @@ -85,9 +85,11 @@ make_table_name () { # load data into database find "$MIMIC_DIR" -type f -regex '.*\.csv\(.gz\)*' | while IFS= read -r FILE; do make_table_name "$FILE" - echo "Loading $FILE .. \c" + # Escape single quotes for SQL string literal + FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g") + printf "Loading %s .. " "$FILE" try duckdb "$OUTFILE" <<-EOSQL - COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); + COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); EOSQL echo "done!" done && echo "Successfully finished loading data into $OUTFILE." diff --git a/mimic-iii/buildmimic/postgres/Makefile b/mimic-iii/buildmimic/postgres/Makefile index 15b4c5213..29953adbd 100644 --- a/mimic-iii/buildmimic/postgres/Makefile +++ b/mimic-iii/buildmimic/postgres/Makefile @@ -104,7 +104,7 @@ mimic-build-gz: @echo '------------------' @echo '' @sleep 2 - psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data_gz.sql -v mimic_data_dir=${datadir} + psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data_gz.sql -v "mimic_data_dir=$(DATADIR)" @echo '' @echo '--------------------' @echo '-- Adding indexes --' @@ -151,7 +151,7 @@ mimic-build: @echo '------------------' @echo '' @sleep 2 - psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data.sql -v mimic_data_dir=${DATADIR} + psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data.sql -v "mimic_data_dir=$(DATADIR)" @echo '' @echo '--------------------' @echo '-- Adding indexes --' @@ -184,7 +184,7 @@ ifeq ("$(physionetuser)","") @echo 'Call the makefile again with physionetuser=' @echo ' e.g. make eicu-download datadir=/path/to/data physionetuser=hello@physionet.org' else - wget --user $(physionetuser) --ask-password -P $(DATADIR) -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETURL)" + wget --user $(physionetuser) --ask-password -P "$(DATADIR)" -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETURL)" endif mimic-demo-download: @@ -192,7 +192,7 @@ mimic-demo-download: @echo '-- Downloading MIMIC-III from PhysioNet --' @echo '------------------------------------------' @echo '' - wget --user $(physionetuser) --ask-password -P $(DATADIR) -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETDEMOURL)" + wget --user $(physionetuser) --ask-password -P "$(DATADIR)" -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETDEMOURL)" #This is fairly inelegant and could be tidied with a for loop and an if to check for gzip, #but need to maintain compatibility with Windows, which baffling lacks these things @@ -281,7 +281,7 @@ concepts: @echo '---------------------' @echo '' @sleep 2 - cd ../../concepts_postgres/ && psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -c "CREATE SCHEMA IF NOT EXISTS mimiciii_derived;" -f postgres-make-concepts.sql + cd ../../concepts/ && psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f make-concepts.sql .PHONY: help mimic clean diff --git a/mimic-iii/buildmimic/postgres/create_mimic_user.sh b/mimic-iii/buildmimic/postgres/create_mimic_user.sh index 9b333dcfa..838ddc9ed 100755 --- a/mimic-iii/buildmimic/postgres/create_mimic_user.sh +++ b/mimic-iii/buildmimic/postgres/create_mimic_user.sh @@ -20,6 +20,13 @@ else echo "User is set to '$MIMIC_USER'"; fi +# escape ' in password for SQL string literal +MIMIC_PASSWORD_SQL=$(printf '%s' "$MIMIC_PASSWORD" | sed "s/'/''/g") +# quote SQL identifiers (double any embedded ") +sql_ident () { printf '%s' "$1" | sed 's/"/""/g; s/^/"/; s/$/"/'; } +MIMIC_USER_SQL=$(sql_ident "$MIMIC_USER") +MIMIC_DB_SQL=$(sql_ident "$MIMIC_DB") + PSQL='psql' # add in the host/port, if they were specified (not null, -n) @@ -58,11 +65,11 @@ fi if [ "$MIMIC_USER" != "postgres" ]; then # we need to create this user via postgres # use SUDO to login as postgres - $PSQL -U postgres -d postgres -c "DROP USER IF EXISTS $MIMIC_USER; CREATE USER $MIMIC_USER WITH PASSWORD '$MIMIC_PASSWORD';" + $PSQL -U postgres -d postgres -c "DROP USER IF EXISTS $MIMIC_USER_SQL; CREATE USER $MIMIC_USER_SQL WITH PASSWORD '$MIMIC_PASSWORD_SQL';" fi if [ "$MIMIC_DB" != "postgres" ]; then # drop and recreate the database - $PSQL -U postgres -d postgres -c "DROP DATABASE IF EXISTS $MIMIC_DB;" - $PSQL -U postgres -d postgres -c "CREATE DATABASE $MIMIC_DB OWNER $MIMIC_USER;" -fi \ No newline at end of file + $PSQL -U postgres -d postgres -c "DROP DATABASE IF EXISTS $MIMIC_DB_SQL;" + $PSQL -U postgres -d postgres -c "CREATE DATABASE $MIMIC_DB_SQL OWNER $MIMIC_USER_SQL;" +fi diff --git a/mimic-iii/buildmimic/sqlite/import.py b/mimic-iii/buildmimic/sqlite/import.py index 801651c7a..4e0113d35 100644 --- a/mimic-iii/buildmimic/sqlite/import.py +++ b/mimic-iii/buildmimic/sqlite/import.py @@ -10,6 +10,34 @@ CHUNKSIZE = 10 ** 6 CONNECTION_STRING = "sqlite:///{}".format(DATABASE_NAME) +# Column dtypes for tables that trigger pandas mixed-type warnings when +# loaded with the default low_memory chunked inference (see #1237). +# Types mirror mimic-iii/buildmimic/postgres/postgres_create_tables.sql. +TABLE_DTYPES = { + "chartevents": { + "WARNING": "Int64", + "ERROR": "Int64", + "RESULTSTATUS": "string", + "STOPPED": "string", + }, + "datetimeevents": { + "WARNING": "Int64", + "ERROR": "Int64", + "RESULTSTATUS": "string", + "STOPPED": "string", + }, + "inputevents_cv": { + "ORIGINALRATE": "float64", + "ORIGINALRATEUOM": "string", + "ORIGINALSITE": "string", + }, + "noteevents": { + "CHARTTIME": "string", + "STORETIME": "string", + "ISERROR": "string", + }, +} + def _table_name_from_csv(filename: str) -> str: """Derive SQL table name from a CSV path (literal suffix, not str.strip).""" @@ -21,20 +49,39 @@ def _table_name_from_csv(filename: str) -> str: return name.lower() +def _read_csv(path, table, **kwargs): + # low_memory=False avoids dtype re-inference across chunks; table-specific + # dtypes pin the columns that otherwise flip between numeric/object. + return pd.read_csv( + path, + index_col="ROW_ID", + low_memory=False, + dtype=TABLE_DTYPES.get(table), + **kwargs, + ) + + if os.path.exists(DATABASE_NAME): msg = "File {} already exists.".format(DATABASE_NAME) print(msg) sys.exit() +# Prefer .csv.gz when both exist for the same table; also load plain .csv +# (import.sh already accepts both). +files_by_table = {} +for f in glob("*.csv"): + files_by_table[_table_name_from_csv(f)] = f for f in glob("*.csv.gz"): + files_by_table[_table_name_from_csv(f)] = f + +for table, f in sorted(files_by_table.items()): print("Starting processing {}".format(f)) - table = _table_name_from_csv(f) if os.path.getsize(f) < THRESHOLD_SIZE: - df = pd.read_csv(f, index_col="ROW_ID") + df = _read_csv(f, table) df.to_sql(table, CONNECTION_STRING) else: # If the file is too large, let's do the work in chunks - for chunk in pd.read_csv(f, index_col="ROW_ID", chunksize=CHUNKSIZE): + for chunk in _read_csv(f, table, chunksize=CHUNKSIZE): chunk.to_sql(table, CONNECTION_STRING, if_exists="append") print("Finished processing {}".format(f)) diff --git a/mimic-iii/buildmimic/sqlite/import.sh b/mimic-iii/buildmimic/sqlite/import.sh index dbf6b9ce4..58b85abb4 100755 --- a/mimic-iii/buildmimic/sqlite/import.sh +++ b/mimic-iii/buildmimic/sqlite/import.sh @@ -28,11 +28,11 @@ for FILE in *; do TABLE_NAME=$(echo "${FILE%%.*}" | tr "[:upper:]" "[:lower:]") case "$FILE" in *csv) - IMPORT_CMD=".import $FILE $TABLE_NAME" + IMPORT_CMD=".import \"$FILE\" $TABLE_NAME" ;; # need to decompress csv before load *csv.gz) - IMPORT_CMD=".import \"|gzip -dc $FILE\" $TABLE_NAME" + IMPORT_CMD=".import \"|gzip -dc \\\"$FILE\\\"\" $TABLE_NAME" ;; # not a data file so skip *) @@ -40,7 +40,7 @@ for FILE in *; do ;; esac echo "Loading $FILE." - sqlite3 $OUTFILE < "$LOAD_COUNT_FILE" +find "$MIMIC_DIR" -type f \( -name '*.csv' -o -name '*.csv.gz' \) | sort | while IFS= read -r FILE; do make_table_name "$FILE" # skip directories which we do not expect in mimic-iv-ed # avoids syntax errors if mimic-iv in the same dir - case $DIRNAME in + case "$DIRNAME" in (ed) ;; # OK (*) continue; esac + FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g") echo "Loading $FILE .. " try duckdb "$OUTFILE" <<-EOSQL - COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); + COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); EOSQL + echo x >> "$LOAD_COUNT_FILE" echo "done!" -done && echo "Successfully finished loading data into $OUTFILE." +done || exit $? + +if [ ! -s "$LOAD_COUNT_FILE" ]; then + die "No .csv / .csv.gz files loaded from $MIMIC_DIR (expected ed/)." +fi +echo "Successfully finished loading data into $OUTFILE." diff --git a/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh b/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh index 20e283139..4db098c14 100644 --- a/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh @@ -29,7 +29,7 @@ usage () { die " USAGE: ./import_duckdb.sh mimic_data_dir [output_db] WHERE: - mimic_data_dir directory that contains csv.gz or csv files + mimic_data_dir directory that contains csv.tar.gz or csv files output_db: optional filename for duckdb file (default: mimic4_note.db)\ " } @@ -97,18 +97,23 @@ make_table_name () { # load data into database -find "$MIMIC_DIR" -type f -name '*.csv???' | sort | while IFS= read -r FILE; do +# Match both .csv and .csv.gz ( '*.csv???' only matched .csv.gz ). +LOAD_COUNT_FILE=$(mktemp) || die "mktemp failed" +trap 'rm -f "$LOAD_COUNT_FILE"' EXIT +: > "$LOAD_COUNT_FILE" +find "$MIMIC_DIR" -type f \( -name '*.csv' -o -name '*.csv.gz' \) | sort | while IFS= read -r FILE; do make_table_name "$FILE" # skip directories which we do not expect in mimic-iv-note # avoids syntax errors if mimic-iv in the same dir - case $DIRNAME in + case "$DIRNAME" in (note) ;; # OK (*) continue; esac + FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g") echo "Loading $FILE .." OUTPUT=$(duckdb "$OUTFILE" 2>&1 <<-EOSQL - COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); + COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); EOSQL ) # If the table is missing in the DB, we emit a warning and continue. @@ -123,5 +128,11 @@ EOSQL yell "$OUTPUT" die "Exiting due to load error." fi + echo x >> "$LOAD_COUNT_FILE" echo "done!" -done && echo "Successfully finished loading data into $OUTFILE." +done || exit $? + +if [ ! -s "$LOAD_COUNT_FILE" ]; then + die "No .csv / .csv.gz files loaded from $MIMIC_DIR (expected note/)." +fi +echo "Successfully finished loading data into $OUTFILE." diff --git a/mimic-iv/buildmimic/duckdb/import_duckdb.sh b/mimic-iv/buildmimic/duckdb/import_duckdb.sh index 4ff9fc219..3213a8fd8 100755 --- a/mimic-iv/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iv/buildmimic/duckdb/import_duckdb.sh @@ -29,7 +29,7 @@ usage () { die " USAGE: ./import_duckdb.sh mimic_data_dir [output_db] WHERE: - mimic_data_dir directory that contains csv.gz or csv files + mimic_data_dir directory that contains csv.tar.gz or csv files output_db: optional filename for duckdb file (default: mimic4.db)\ " } @@ -102,18 +102,26 @@ make_table_name () { # load data into database -find "$MIMIC_DIR" -type f -name '*.csv???' | sort | while IFS= read -r FILE; do +# Match both uncompressed (.csv) and gzip (.csv.gz). The old '*.csv???' +# pattern only matched names with exactly three chars after ".csv" (i.e. .gz), +# so plain .csv files were silently skipped while the script still reported success. +LOAD_COUNT_FILE=$(mktemp) || die "mktemp failed" +trap 'rm -f "$LOAD_COUNT_FILE"' EXIT +: > "$LOAD_COUNT_FILE" +find "$MIMIC_DIR" -type f \( -name '*.csv' -o -name '*.csv.gz' \) | sort | while IFS= read -r FILE; do make_table_name "$FILE" # skip directories which we do not expect in mimic-iv # avoids syntax errors if mimic-iv-ed in the same dir - case $DIRNAME in + case "$DIRNAME" in (hosp|icu) ;; # OK (*) continue; esac - echo "Loading $FILE .. \c" + # Escape single quotes for SQL string literal + FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g") + printf "Loading %s .. " "$FILE" OUTPUT=$(duckdb "$OUTFILE" 2>&1 <<-EOSQL - COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); + COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); EOSQL ) # If the table is missing in the DB, we emit a warning and continue. @@ -128,5 +136,11 @@ EOSQL yell "$OUTPUT" die "Exiting due to load error." fi + echo x >> "$LOAD_COUNT_FILE" echo "done!" -done && echo "Successfully finished loading data into $OUTFILE." +done || exit $? + +if [ ! -s "$LOAD_COUNT_FILE" ]; then + die "No .csv / .csv.gz files loaded from $MIMIC_DIR (expected hosp/ and icu/)." +fi +echo "Successfully finished loading data into $OUTFILE." diff --git a/mimic-iv/buildmimic/mysql/validate_demo.sql b/mimic-iv/buildmimic/mysql/validate_demo.sql index 575e84133..ebf4f02dc 100644 --- a/mimic-iv/buildmimic/mysql/validate_demo.sql +++ b/mimic-iv/buildmimic/mysql/validate_demo.sql @@ -31,14 +31,17 @@ FROM ( SELECT 'poe_detail' AS tbl, 3795 AS row_count UNION ALL SELECT 'prescriptions' AS tbl, 18087 AS row_count UNION ALL SELECT 'procedures_icd' AS tbl, 722 AS row_count UNION ALL + SELECT 'provider' AS tbl, 40508 AS row_count UNION ALL SELECT 'services' AS tbl, 319 AS row_count UNION ALL SELECT 'transfers' AS tbl, 1190 AS row_count UNION ALL -- icu data SELECT 'icustays' AS tbl, 140 AS row_count UNION ALL + SELECT 'caregiver' AS tbl, 15468 AS row_count UNION ALL SELECT 'd_items' AS tbl, 4014 AS row_count UNION ALL SELECT 'chartevents' AS tbl, 668862 AS row_count UNION ALL SELECT 'datetimeevents' AS tbl, 15280 AS row_count UNION ALL SELECT 'inputevents' AS tbl, 20404 AS row_count UNION ALL + SELECT 'ingredientevents' AS tbl, 25728 AS row_count UNION ALL SELECT 'outputevents' AS tbl, 9362 AS row_count UNION ALL SELECT 'procedureevents' AS tbl, 1468 AS row_count ) exp @@ -64,14 +67,17 @@ INNER JOIN SELECT 'poe_detail' AS tbl, count(*) AS row_count FROM poe_detail UNION ALL SELECT 'prescriptions' AS tbl, count(*) AS row_count FROM prescriptions UNION ALL SELECT 'procedures_icd' AS tbl, count(*) AS row_count FROM procedures_icd UNION ALL + SELECT 'provider' AS tbl, count(*) AS row_count FROM provider UNION ALL SELECT 'services' AS tbl, count(*) AS row_count FROM services UNION ALL SELECT 'transfers' AS tbl, count(*) AS row_count FROM transfers UNION ALL -- icu data SELECT 'icustays' AS tbl, count(*) AS row_count FROM icustays UNION ALL + SELECT 'caregiver' AS tbl, count(*) AS row_count FROM caregiver UNION ALL SELECT 'chartevents' AS tbl, count(*) AS row_count FROM chartevents UNION ALL SELECT 'd_items' AS tbl, count(*) AS row_count FROM d_items UNION ALL SELECT 'datetimeevents' AS tbl, count(*) AS row_count FROM datetimeevents UNION ALL SELECT 'inputevents' AS tbl, count(*) AS row_count FROM inputevents UNION ALL + SELECT 'ingredientevents' AS tbl, count(*) AS row_count FROM ingredientevents UNION ALL SELECT 'outputevents' AS tbl, count(*) AS row_count FROM outputevents UNION ALL SELECT 'procedureevents' AS tbl, count(*) AS row_count FROM procedureevents ) obs diff --git a/mimic-iv/buildmimic/postgres/validate_demo.sql b/mimic-iv/buildmimic/postgres/validate_demo.sql index 303dd98e4..049497e56 100644 --- a/mimic-iv/buildmimic/postgres/validate_demo.sql +++ b/mimic-iv/buildmimic/postgres/validate_demo.sql @@ -22,14 +22,17 @@ WITH expected AS SELECT 'poe_detail' AS tbl, 3795 AS row_count UNION ALL SELECT 'prescriptions' AS tbl, 18087 AS row_count UNION ALL SELECT 'procedures_icd' AS tbl, 722 AS row_count UNION ALL + SELECT 'provider' AS tbl, 40508 AS row_count UNION ALL SELECT 'services' AS tbl, 319 AS row_count UNION ALL SELECT 'transfers' AS tbl, 1190 AS row_count UNION ALL -- icu data SELECT 'icustays' AS tbl, 140 AS row_count UNION ALL + SELECT 'caregiver' AS tbl, 15468 AS row_count UNION ALL SELECT 'd_items' AS tbl, 4014 AS row_count UNION ALL SELECT 'chartevents' AS tbl, 668862 AS row_count UNION ALL SELECT 'datetimeevents' AS tbl, 15280 AS row_count UNION ALL SELECT 'inputevents' AS tbl, 20404 AS row_count UNION ALL + SELECT 'ingredientevents' AS tbl, 25728 AS row_count UNION ALL SELECT 'outputevents' AS tbl, 9362 AS row_count UNION ALL SELECT 'procedureevents' AS tbl, 1468 AS row_count ) @@ -54,14 +57,17 @@ WITH expected AS SELECT 'poe_detail' AS tbl, count(*) AS row_count FROM mimiciv_hosp.poe_detail UNION ALL SELECT 'prescriptions' AS tbl, count(*) AS row_count FROM mimiciv_hosp.prescriptions UNION ALL SELECT 'procedures_icd' AS tbl, count(*) AS row_count FROM mimiciv_hosp.procedures_icd UNION ALL + SELECT 'provider' AS tbl, count(*) AS row_count FROM mimiciv_hosp.provider UNION ALL SELECT 'services' AS tbl, count(*) AS row_count FROM mimiciv_hosp.services UNION ALL SELECT 'transfers' AS tbl, count(*) AS row_count FROM mimiciv_hosp.transfers UNION ALL -- icu data SELECT 'icustays' AS tbl, count(*) AS row_count FROM mimiciv_icu.icustays UNION ALL + SELECT 'caregiver' AS tbl, count(*) AS row_count FROM mimiciv_icu.caregiver UNION ALL SELECT 'chartevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.chartevents UNION ALL SELECT 'd_items' AS tbl, count(*) AS row_count FROM mimiciv_icu.d_items UNION ALL SELECT 'datetimeevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.datetimeevents UNION ALL SELECT 'inputevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.inputevents UNION ALL + SELECT 'ingredientevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.ingredientevents UNION ALL SELECT 'outputevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.outputevents UNION ALL SELECT 'procedureevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.procedureevents ) diff --git a/mimic-iv/buildmimic/sqlite/import.py b/mimic-iv/buildmimic/sqlite/import.py index 68d436479..31b880c44 100644 --- a/mimic-iv/buildmimic/sqlite/import.py +++ b/mimic-iv/buildmimic/sqlite/import.py @@ -110,7 +110,7 @@ def main(): if args.limit > 0: for f in data_files: if 'patients' in f.name: - pt = pd.read_csv(f) + pt = pd.read_csv(f, low_memory=False) break if pt is None: raise FileNotFoundError('Unable to find a patients file in current folder.') @@ -158,7 +158,7 @@ def main(): tablename = tablenames[i] print("Starting processing {}".format(tablename), end='.. ') if os.path.getsize(f) < THRESHOLD_SIZE: - df = pd.read_csv(f, dtype=mimic_dtypes) + df = pd.read_csv(f, dtype=mimic_dtypes, low_memory=False) df = process_dataframe(df, subjects=subjects) df.to_sql(tablename, connection, index=False) row_counts[tablename] += len(df) diff --git a/mimic-iv/buildmimic/sqlite/import.sh b/mimic-iv/buildmimic/sqlite/import.sh index 08ded2881..fb642d1ce 100755 --- a/mimic-iv/buildmimic/sqlite/import.sh +++ b/mimic-iv/buildmimic/sqlite/import.sh @@ -29,11 +29,11 @@ for FILE in */**.csv*; do TABLE_NAME=$(echo "${BASENAME%%.*}" | tr "[:upper:]" "[:lower:]") case "$FILE" in *csv) - IMPORT_CMD=".import $FILE $TABLE_NAME" + IMPORT_CMD=".import \"$FILE\" $TABLE_NAME" ;; # need to decompress csv before load *csv.gz) - IMPORT_CMD=".import \"|gzip -dc $FILE\" $TABLE_NAME" + IMPORT_CMD=".import \"|gzip -dc \\\"$FILE\\\"\" $TABLE_NAME" ;; # not a data file so skip *) @@ -41,7 +41,7 @@ for FILE in */**.csv*; do ;; esac echo "Loading $FILE." - sqlite3 $OUTFILE < list via generate_series (inclusive), for UNNEST. +def _generate_array_sql(self: DuckDB.Generator, expression: exp.Expression) -> str: + start = self.sql(expression, "start") + end = self.sql(expression, "end") + return f"(SELECT list(g) FROM generate_series({start}, {end}) AS t(g))" + + class MimicDuckDB(DuckDB): class Generator(DuckDB.Generator): def datatype_sql(self, expression: exp.DataType) -> str: @@ -54,4 +65,5 @@ def datatype_sql(self, expression: exp.DataType) -> str: **DuckDB.Generator.TRANSFORMS, exp.RegexpExtract: _regexp_extract_sql, exp.ParseDatetime: _parse_datetime_sql, + exp.GenerateSeries: _generate_array_sql, } diff --git a/src/mimic_utils/sqlglot_dialects/postgres.py b/src/mimic_utils/sqlglot_dialects/postgres.py index d33698558..509b369b2 100644 --- a/src/mimic_utils/sqlglot_dialects/postgres.py +++ b/src/mimic_utils/sqlglot_dialects/postgres.py @@ -23,8 +23,11 @@ def _unit(expression: exp.Expression, default: str = "DAY") -> str: # The logic is as follows: # * DAY -> difference of the two calendar dates (date subtraction = whole days) # * YEAR -> difference of the two calendar years +# * MONTH -> year*12 + month difference (calendar month boundaries) # * sub-day units -> truncate both operands to the unit (which makes the # elapsed seconds an exact multiple of the unit) then divide. +# WEEK is intentionally unsupported: BigQuery week starts (SUNDAY/ISO) do not +# match PostgreSQL DATE_TRUNC('week') Monday semantics. # https://cloud.google.com/bigquery/docs/reference/standard-sql/datetime_functions#datetime_diff _SECONDS_PER_UNIT = {"SECOND": 1, "MINUTE": 60, "HOUR": 3600} @@ -39,6 +42,19 @@ def _datetime_diff_sql(self: Postgres.Generator, expression: exp.Expression) -> return f"(CAST({end} AS DATE) - CAST({start} AS DATE))" if unit == "YEAR": return f"CAST(EXTRACT(YEAR FROM {end}) - EXTRACT(YEAR FROM {start}) AS BIGINT)" + if unit == "MONTH": + # Calendar month boundaries (matches BigQuery DATETIME_DIFF MONTH). + return ( + "CAST((EXTRACT(YEAR FROM {end}) - EXTRACT(YEAR FROM {start})) * 12 " + "+ (EXTRACT(MONTH FROM {end}) - EXTRACT(MONTH FROM {start})) AS BIGINT)" + ).format(end=end, start=start) + if unit not in _SECONDS_PER_UNIT: + # WEEK (and WEEK(SUNDAY)/ISO) need BigQuery's week-start semantics; + # refuse rather than emit a silently wrong days/7 division. + raise ValueError( + f"Unsupported DATETIME_DIFF unit {unit!r}; " + "expected SECOND, MINUTE, HOUR, DAY, MONTH, or YEAR" + ) lo = unit.lower() factor = _SECONDS_PER_UNIT[unit] diff --git a/src/mimic_utils/transpile.py b/src/mimic_utils/transpile.py index d4a65541e..29924c187 100644 --- a/src/mimic_utils/transpile.py +++ b/src/mimic_utils/transpile.py @@ -113,7 +113,7 @@ def transpile_file( f"CREATE TABLE {derived_schema}{Path(source_file).stem} AS\n" ) + transpiled_query - with open(destination_file, "w") as write_file: + with open(destination_file, "w", encoding="utf-8") as write_file: write_file.write(transpiled_query) diff --git a/tests/test_transpile.py b/tests/test_transpile.py index 952240bca..151d075f4 100644 --- a/tests/test_transpile.py +++ b/tests/test_transpile.py @@ -52,6 +52,9 @@ def t(bq: str, dialect: str) -> str: "SELECT (CAST(a.dischtime AS DATE) - CAST(a.admittime AS DATE)) FROM t AS a"), ("diff_year_pg", "SELECT DATETIME_DIFF(a.b, a.c, YEAR) FROM t a", "postgres", "SELECT CAST(EXTRACT(YEAR FROM a.b) - EXTRACT(YEAR FROM a.c) AS BIGINT) FROM t AS a"), + ("diff_month_pg", "SELECT DATETIME_DIFF(a.b, a.c, MONTH) FROM t a", "postgres", + "SELECT CAST((EXTRACT(YEAR FROM a.b) - EXTRACT(YEAR FROM a.c)) * 12 " + "+ (EXTRACT(MONTH FROM a.b) - EXTRACT(MONTH FROM a.c)) AS BIGINT) FROM t AS a"), # DuckDB handles DATETIME_DIFF natively (boundary count), operands swapped ("diff_hour_duckdb", "SELECT DATETIME_DIFF(a.outtime, a.intime, HOUR) FROM t a", "duckdb", "SELECT DATE_DIFF('HOUR', a.intime, a.outtime) FROM t AS a"), @@ -71,6 +74,9 @@ def t(bq: str, dialect: str) -> str: # GENERATE_ARRAY -> ARRAY(SELECT ... GENERATE_SERIES) (postgres) ("generate_array_pg", "SELECT GENERATE_ARRAY(-24, 5) AS hrs FROM t", "postgres", "SELECT ARRAY(SELECT * FROM GENERATE_SERIES(-24, 5)) AS hrs FROM t"), + # GENERATE_ARRAY -> list via generate_series (duckdb); must stay list for UNNEST + ("generate_array_duckdb", "SELECT GENERATE_ARRAY(-24, 5) AS hrs FROM t", "duckdb", + "SELECT (SELECT list(g) FROM generate_series(-24, 5) AS t(g)) AS hrs FROM t"), # handled natively by sqlglot 30.x (regression guards) ("datetime_date_pg", "SELECT DATETIME(me.chartdate) FROM t me", "postgres", @@ -216,6 +222,7 @@ def test_mimic_iii_concept_transpiles_and_reparses(sql_file, dialect): ("2150-01-02 01:00:00", "2150-01-01 23:00:00", "DAY", 1), # crosses midnight ("2150-01-01 23:00:00", "2150-01-01 01:00:00", "DAY", 0), # same day ("2155-06-15 00:00:00", "2150-01-01 00:00:00", "YEAR", 5), + ("2150-04-01 00:00:00", "2150-01-15 00:00:00", "MONTH", 3), ("2150-01-01 00:01:00", "2150-01-01 00:00:59", "MINUTE", 1), ("2150-01-01 00:00:30", "2150-01-01 00:00:10", "SECOND", 20), ("2150-01-01 01:00:00", "2150-01-01 03:00:00", "HOUR", -2), # negative direction From 2a9312787037ba1910df9f22347fec6771cfdb2c Mon Sep 17 00:00:00 2001 From: Taksh Date: Wed, 29 Jul 2026 18:33:16 +0300 Subject: [PATCH 2/3] fix: quote NegBio pipeline paths for ShellCheck SC2086 Co-authored-by: Cursor --- mimic-iv-cxr/txt/negbio/run_negbio.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/mimic-iv-cxr/txt/negbio/run_negbio.sh b/mimic-iv-cxr/txt/negbio/run_negbio.sh index dab200cd4..93a98a149 100755 --- a/mimic-iv-cxr/txt/negbio/run_negbio.sh +++ b/mimic-iv-cxr/txt/negbio/run_negbio.sh @@ -38,25 +38,25 @@ do fn=$(basename "$fn_path") echo "Looping through files with mimic_cxr_###.csv pattern." # validate it's a mimic_cxr sections file - if [[ $fn =~ ^mimic_cxr_[0-9]+.csv$ ]]; + if [[ "$fn" =~ ^mimic_cxr_[0-9]+\.csv$ ]]; then - export INPUT_FILE=$fn_path + export INPUT_FILE="$fn_path" # remove extension from filename # sets the folder location to stem of original filename # all intermediate files will be saved in this folder - export OUTPUT_DIR=${INPUT_FILE::-4} + export OUTPUT_DIR="${INPUT_FILE%.csv}" echo "$OUTPUT_DIR - running NegBio.." python "$NEGBIO_PATH/negbio/negbio_csv2bioc.py" --output "$OUTPUT_DIR/report" "$INPUT_FILE" - python $NEGBIO_PATH/negbio/negbio_pipeline.py section_split --pattern $NEGBIO_PATH/patterns/section_titles_cxr8.txt --output $OUTPUT_DIR/sections $OUTPUT_DIR/report/* --workers=6 - python $NEGBIO_PATH/negbio/negbio_pipeline.py ssplit --output $OUTPUT_DIR/ssplit $OUTPUT_DIR/sections/* --workers=6 - python $NEGBIO_PATH/negbio/negbio_pipeline.py parse --output $OUTPUT_DIR/parse $OUTPUT_DIR/ssplit/* --workers=6 - python $NEGBIO_PATH/negbio/negbio_pipeline.py ptb2ud --output $OUTPUT_DIR/ud $OUTPUT_DIR/parse/* --workers=6 - python $NEGBIO_PATH/negbio/negbio_pipeline.py dner_regex --phrases_file $NEGBIO_PATH/patterns/chexpert_phrases.yml --output $OUTPUT_DIR/dner $OUTPUT_DIR/ud/* --suffix=.chexpert-regex.xml --workers=6 --overwrite - python $NEGBIO_PATH/negbio/negbio_pipeline.py neg2 --output $OUTPUT_DIR/neg --pre-negation-uncertainty-patterns $NEGBIO_PATH/patterns/chexpert_pre_negation_uncertainty.yml --neg-patterns $NEGBIO_PATH/patterns/neg_patterns2.yml --post-negation-uncertainty-patterns $NEGBIO_PATH/patterns/post_negation_uncertainty.yml --neg-regex-patterns $NEGBIO_PATH/patterns/neg_regex_patterns.yml --uncertainty-regex-patterns $NEGBIO_PATH/patterns/uncertainty_regex_patterns.yml $OUTPUT_DIR/dner/* --workers=6 + python "$NEGBIO_PATH/negbio/negbio_pipeline.py" section_split --pattern "$NEGBIO_PATH/patterns/section_titles_cxr8.txt" --output "$OUTPUT_DIR/sections" "$OUTPUT_DIR"/report/* --workers=6 + python "$NEGBIO_PATH/negbio/negbio_pipeline.py" ssplit --output "$OUTPUT_DIR/ssplit" "$OUTPUT_DIR"/sections/* --workers=6 + python "$NEGBIO_PATH/negbio/negbio_pipeline.py" parse --output "$OUTPUT_DIR/parse" "$OUTPUT_DIR"/ssplit/* --workers=6 + python "$NEGBIO_PATH/negbio/negbio_pipeline.py" ptb2ud --output "$OUTPUT_DIR/ud" "$OUTPUT_DIR"/parse/* --workers=6 + python "$NEGBIO_PATH/negbio/negbio_pipeline.py" dner_regex --phrases_file "$NEGBIO_PATH/patterns/chexpert_phrases.yml" --output "$OUTPUT_DIR/dner" "$OUTPUT_DIR"/ud/* --suffix=.chexpert-regex.xml --workers=6 --overwrite + python "$NEGBIO_PATH/negbio/negbio_pipeline.py" neg2 --output "$OUTPUT_DIR/neg" --pre-negation-uncertainty-patterns "$NEGBIO_PATH/patterns/chexpert_pre_negation_uncertainty.yml" --neg-patterns "$NEGBIO_PATH/patterns/neg_patterns2.yml" --post-negation-uncertainty-patterns "$NEGBIO_PATH/patterns/post_negation_uncertainty.yml" --neg-regex-patterns "$NEGBIO_PATH/patterns/neg_regex_patterns.yml" --uncertainty-regex-patterns "$NEGBIO_PATH/patterns/uncertainty_regex_patterns.yml" "$OUTPUT_DIR"/dner/* --workers=6 # ultimate filename we save the labels to - export OUTPUT_LABELS=$OUTPUT_DIR/${fn::-4}_labels.csv + export OUTPUT_LABELS="$OUTPUT_DIR/${fn%.csv}_labels.csv" python "$NEGBIO_PATH/negbio/ext/chexpert_collect_labels.py" --phrases_file "$NEGBIO_PATH/patterns/chexpert_phrases.yml" --output "$OUTPUT_LABELS" "$OUTPUT_DIR"/neg/* fi done From 270b6d3a38515fae1f1db178d4804fd2b21d3d73 Mon Sep 17 00:00:00 2001 From: Taksh Date: Wed, 29 Jul 2026 22:08:38 +0300 Subject: [PATCH 3/3] Address review: drop defensive extras, fix SQLFluff annotate perms Co-authored-by: Cursor --- .github/workflows/lint_sqlfluff.yml | 7 +-- mimic-iii/buildmimic/duckdb/import_duckdb.sh | 2 +- mimic-iii/buildmimic/postgres/Makefile | 2 +- mimic-iii/buildmimic/sqlite/import.py | 44 +------------------ .../demographics/icustay_hours.sql | 2 +- .../txt/chexpert/run_chexpert_on_files.sh | 18 +++----- mimic-iv-cxr/txt/negbio/run_negbio.sh | 15 +++---- mimic-iv/concepts/validate_concepts.sh | 5 +-- src/mimic_utils/sqlglot_dialects/duckdb.py | 12 ----- tests/test_transpile.py | 3 -- 10 files changed, 23 insertions(+), 87 deletions(-) diff --git a/.github/workflows/lint_sqlfluff.yml b/.github/workflows/lint_sqlfluff.yml index 139834acc..5b6031623 100644 --- a/.github/workflows/lint_sqlfluff.yml +++ b/.github/workflows/lint_sqlfluff.yml @@ -6,6 +6,9 @@ on: jobs: lint-mimic-iv: runs-on: ubuntu-latest + permissions: + contents: read + checks: write steps: - name: checkout uses: actions/checkout@v7 @@ -35,12 +38,10 @@ jobs: shell: bash run: sqlfluff lint --format github-annotation --annotation-level failure --nofail ${{ steps.get_files_to_lint.outputs.lintees }} > annotations.json - name: Annotate - # Fork PRs cannot create check runs with GITHUB_TOKEN; lint already ran. - continue-on-error: true if: steps.get_files_to_lint.outputs.lintees != '' uses: yuzutech/annotations-action@v0.6.0 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" title: "SQLFluff Lint" input: "./annotations.json" - ignore-missing-file: true \ No newline at end of file + ignore-unauthorized-error: true diff --git a/mimic-iii/buildmimic/duckdb/import_duckdb.sh b/mimic-iii/buildmimic/duckdb/import_duckdb.sh index 36d332208..1069f790e 100755 --- a/mimic-iii/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iii/buildmimic/duckdb/import_duckdb.sh @@ -29,7 +29,7 @@ usage () { die " USAGE: ./import_duckdb.sh mimic_data_dir [output_db] WHERE: - mimic_data_dir directory that contains csv.tar.gz or csv files + mimic_data_dir directory that contains csv.gz or csv files output_db: optional filename for duckdb file (default: mimic3.db)\ " } diff --git a/mimic-iii/buildmimic/postgres/Makefile b/mimic-iii/buildmimic/postgres/Makefile index 29953adbd..83a61ee5a 100644 --- a/mimic-iii/buildmimic/postgres/Makefile +++ b/mimic-iii/buildmimic/postgres/Makefile @@ -281,7 +281,7 @@ concepts: @echo '---------------------' @echo '' @sleep 2 - cd ../../concepts/ && psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f make-concepts.sql + cd ../../concepts_postgres/ && psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -c "CREATE SCHEMA IF NOT EXISTS mimiciii_derived;" -f postgres-make-concepts.sql .PHONY: help mimic clean diff --git a/mimic-iii/buildmimic/sqlite/import.py b/mimic-iii/buildmimic/sqlite/import.py index 4e0113d35..66052b2a8 100644 --- a/mimic-iii/buildmimic/sqlite/import.py +++ b/mimic-iii/buildmimic/sqlite/import.py @@ -10,34 +10,6 @@ CHUNKSIZE = 10 ** 6 CONNECTION_STRING = "sqlite:///{}".format(DATABASE_NAME) -# Column dtypes for tables that trigger pandas mixed-type warnings when -# loaded with the default low_memory chunked inference (see #1237). -# Types mirror mimic-iii/buildmimic/postgres/postgres_create_tables.sql. -TABLE_DTYPES = { - "chartevents": { - "WARNING": "Int64", - "ERROR": "Int64", - "RESULTSTATUS": "string", - "STOPPED": "string", - }, - "datetimeevents": { - "WARNING": "Int64", - "ERROR": "Int64", - "RESULTSTATUS": "string", - "STOPPED": "string", - }, - "inputevents_cv": { - "ORIGINALRATE": "float64", - "ORIGINALRATEUOM": "string", - "ORIGINALSITE": "string", - }, - "noteevents": { - "CHARTTIME": "string", - "STORETIME": "string", - "ISERROR": "string", - }, -} - def _table_name_from_csv(filename: str) -> str: """Derive SQL table name from a CSV path (literal suffix, not str.strip).""" @@ -49,18 +21,6 @@ def _table_name_from_csv(filename: str) -> str: return name.lower() -def _read_csv(path, table, **kwargs): - # low_memory=False avoids dtype re-inference across chunks; table-specific - # dtypes pin the columns that otherwise flip between numeric/object. - return pd.read_csv( - path, - index_col="ROW_ID", - low_memory=False, - dtype=TABLE_DTYPES.get(table), - **kwargs, - ) - - if os.path.exists(DATABASE_NAME): msg = "File {} already exists.".format(DATABASE_NAME) print(msg) @@ -77,11 +37,11 @@ def _read_csv(path, table, **kwargs): for table, f in sorted(files_by_table.items()): print("Starting processing {}".format(f)) if os.path.getsize(f) < THRESHOLD_SIZE: - df = _read_csv(f, table) + df = pd.read_csv(f, index_col="ROW_ID") df.to_sql(table, CONNECTION_STRING) else: # If the file is too large, let's do the work in chunks - for chunk in _read_csv(f, table, chunksize=CHUNKSIZE): + for chunk in pd.read_csv(f, index_col="ROW_ID", chunksize=CHUNKSIZE): chunk.to_sql(table, CONNECTION_STRING, if_exists="append") print("Finished processing {}".format(f)) diff --git a/mimic-iii/concepts_duckdb/demographics/icustay_hours.sql b/mimic-iii/concepts_duckdb/demographics/icustay_hours.sql index 12624abff..7b459f3e8 100644 --- a/mimic-iii/concepts_duckdb/demographics/icustay_hours.sql +++ b/mimic-iii/concepts_duckdb/demographics/icustay_hours.sql @@ -4,7 +4,7 @@ WITH all_hours AS ( SELECT it.icustay_id, STRPTIME(STRFTIME(CAST(it.intime_hr + INTERVAL '59' MINUTE AS TIMESTAMP), '%Y-%m-%d %H:00:00'), '%Y-%m-%d %H:00:00') AS endtime, - (SELECT list(g) FROM generate_series(-24, CAST(CEIL(DATE_DIFF('HOUR', it.intime_hr, it.outtime_hr)) AS BIGINT)) AS t(g)) AS hrs + GENERATE_SERIES(-24, CAST(CEIL(DATE_DIFF('HOUR', it.intime_hr, it.outtime_hr)) AS BIGINT)) AS hrs FROM mimiciii_derived.icustay_times AS it ) SELECT diff --git a/mimic-iv-cxr/txt/chexpert/run_chexpert_on_files.sh b/mimic-iv-cxr/txt/chexpert/run_chexpert_on_files.sh index f996541a2..2c2ef14fb 100644 --- a/mimic-iv-cxr/txt/chexpert/run_chexpert_on_files.sh +++ b/mimic-iv-cxr/txt/chexpert/run_chexpert_on_files.sh @@ -22,20 +22,12 @@ fi sleep 2 # loop through each .csv file in the section folder -for fn_path in "$REPORT_PATH"/*.csv; do - [ -f "$fn_path" ] || continue - fn=$(basename "$fn_path") +for fn in "$REPORT_PATH"/*; do + fn=${fn##*/} echo "$(date): $fn" - fn_stem=${fn%.csv} + fn_stem=$(echo "$fn" | cut -d. -f 1) # run chexpert - must be run from chexpert folder - python "$CHEXPERT_PATH/label.py" --verbose \ - --reports_path "$fn_path" \ - --output_path "${fn_stem}_labeled.csv" \ - --mention_phrases_dir "$CHEXPERT_PATH/phrases/mention" \ - --unmention_phrases_dir "$CHEXPERT_PATH/phrases/unmention" \ - --pre_negation_uncertainty_path "$CHEXPERT_PATH/patterns/pre_negation_uncertainty.txt" \ - --negation_path "$CHEXPERT_PATH/patterns/negation.txt" \ - --post_negation_uncertainty_path "$CHEXPERT_PATH/patterns/post_negation_uncertainty.txt" + python "$CHEXPERT_PATH/label.py" --verbose --reports_path "$REPORT_PATH/$fn" --output_path "${fn_stem}_labeled.csv" --mention_phrases_dir "$CHEXPERT_PATH/phrases/mention" --unmention_phrases_dir "$CHEXPERT_PATH/phrases/unmention" --pre_negation_uncertainty_path "$CHEXPERT_PATH/patterns/pre_negation_uncertainty.txt" --negation_path "$CHEXPERT_PATH/patterns/negation.txt" --post_negation_uncertainty_path "$CHEXPERT_PATH/patterns/post_negation_uncertainty.txt" echo "$(date): done!" echo '' -done +done \ No newline at end of file diff --git a/mimic-iv-cxr/txt/negbio/run_negbio.sh b/mimic-iv-cxr/txt/negbio/run_negbio.sh index 93a98a149..19d0f7f43 100755 --- a/mimic-iv-cxr/txt/negbio/run_negbio.sh +++ b/mimic-iv-cxr/txt/negbio/run_negbio.sh @@ -32,21 +32,20 @@ sleep 2 # mimic_cxr_001.csv # mimic_cxr_002.csv # .. etc -for fn_path in "$BASE_FOLDER"/mimic_cxr_*.csv; +for fn in "$BASE_FOLDER"/*; do - [ -f "$fn_path" ] || continue - fn=$(basename "$fn_path") + fn=${fn##*/} echo "Looping through files with mimic_cxr_###.csv pattern." # validate it's a mimic_cxr sections file - if [[ "$fn" =~ ^mimic_cxr_[0-9]+\.csv$ ]]; + if [[ $fn =~ ^mimic_cxr_[0-9]+.csv$ ]]; then - export INPUT_FILE="$fn_path" + export INPUT_FILE=${BASE_FOLDER}/$fn # remove extension from filename # sets the folder location to stem of original filename # all intermediate files will be saved in this folder - export OUTPUT_DIR="${INPUT_FILE%.csv}" + export OUTPUT_DIR=${INPUT_FILE::-4} - echo "$OUTPUT_DIR - running NegBio.." + echo "$OUTPUT_DIR" - running NegBio.. python "$NEGBIO_PATH/negbio/negbio_csv2bioc.py" --output "$OUTPUT_DIR/report" "$INPUT_FILE" python "$NEGBIO_PATH/negbio/negbio_pipeline.py" section_split --pattern "$NEGBIO_PATH/patterns/section_titles_cxr8.txt" --output "$OUTPUT_DIR/sections" "$OUTPUT_DIR"/report/* --workers=6 python "$NEGBIO_PATH/negbio/negbio_pipeline.py" ssplit --output "$OUTPUT_DIR/ssplit" "$OUTPUT_DIR"/sections/* --workers=6 @@ -56,7 +55,7 @@ do python "$NEGBIO_PATH/negbio/negbio_pipeline.py" neg2 --output "$OUTPUT_DIR/neg" --pre-negation-uncertainty-patterns "$NEGBIO_PATH/patterns/chexpert_pre_negation_uncertainty.yml" --neg-patterns "$NEGBIO_PATH/patterns/neg_patterns2.yml" --post-negation-uncertainty-patterns "$NEGBIO_PATH/patterns/post_negation_uncertainty.yml" --neg-regex-patterns "$NEGBIO_PATH/patterns/neg_regex_patterns.yml" --uncertainty-regex-patterns "$NEGBIO_PATH/patterns/uncertainty_regex_patterns.yml" "$OUTPUT_DIR"/dner/* --workers=6 # ultimate filename we save the labels to - export OUTPUT_LABELS="$OUTPUT_DIR/${fn%.csv}_labels.csv" + export OUTPUT_LABELS=$OUTPUT_DIR/${fn::-4}_labels.csv python "$NEGBIO_PATH/negbio/ext/chexpert_collect_labels.py" --phrases_file "$NEGBIO_PATH/patterns/chexpert_phrases.yml" --output "$OUTPUT_LABELS" "$OUTPUT_DIR"/neg/* fi done diff --git a/mimic-iv/concepts/validate_concepts.sh b/mimic-iv/concepts/validate_concepts.sh index 645a82a3e..eeaddb826 100755 --- a/mimic-iv/concepts/validate_concepts.sh +++ b/mimic-iv/concepts/validate_concepts.sh @@ -32,8 +32,7 @@ fail=0 missing=0 while read -r tbl; do [ -z "${tbl}" ] && continue - # exact field match (regex grep can false-hit prefixes) - if ! awk -F, -v t="${tbl}" '$1 == t { found=1 } END { exit !found }' <<< "${ACTUAL}"; then + if ! grep -q "^${tbl}," <<< "${ACTUAL}"; then echo "MISSING: expected table ${DATASET}.${tbl} was not built" missing=1 fail=1 @@ -48,7 +47,7 @@ empty=0 while IFS=, read -r tbl rows; do [ -z "${tbl}" ] && continue # only validate the tables we actually build (ignore _metadata and any strays) - if grep -Fxq "${tbl}" <<< "${EXPECTED}" && [ "${rows:-}" -eq 0 ]; then + if grep -qx "${tbl}" <<< "${EXPECTED}" && [ "${rows}" -eq 0 ]; then echo "EMPTY: table ${DATASET}.${tbl} has 0 rows" empty=1 fail=1 diff --git a/src/mimic_utils/sqlglot_dialects/duckdb.py b/src/mimic_utils/sqlglot_dialects/duckdb.py index 2c3262b94..12e83bfd0 100644 --- a/src/mimic_utils/sqlglot_dialects/duckdb.py +++ b/src/mimic_utils/sqlglot_dialects/duckdb.py @@ -3,10 +3,6 @@ - ``NUMERIC`` is ``DECIMAL(38, 9)``, but DuckDB's bare ``DECIMAL`` defaults to ``DECIMAL(18, 3)``, which silently rounds values to three decimal places (e.g. ``CAST(0.0255 AS NUMERIC)`` becomes ``0.026`` before any explicit ``ROUND``). -- ``GENERATE_ARRAY`` must remain a LIST (for later ``UNNEST``). Native sqlglot -emits ``GENERATE_SERIES``, which is a set-returning function; wrapping as a -list matches Postgres ``ARRAY(GENERATE_SERIES)`` and keeps ``UNNEST(hrs)`` -reliable across DuckDB versions (#1736). """ import re @@ -45,13 +41,6 @@ def _parse_datetime_sql(self: DuckDB.Generator, expression: exp.ParseDatetime) - return f"STRPTIME({this}, {fmt})" -# GENERATE_ARRAY(a, b) -> list via generate_series (inclusive), for UNNEST. -def _generate_array_sql(self: DuckDB.Generator, expression: exp.Expression) -> str: - start = self.sql(expression, "start") - end = self.sql(expression, "end") - return f"(SELECT list(g) FROM generate_series({start}, {end}) AS t(g))" - - class MimicDuckDB(DuckDB): class Generator(DuckDB.Generator): def datatype_sql(self, expression: exp.DataType) -> str: @@ -65,5 +54,4 @@ def datatype_sql(self, expression: exp.DataType) -> str: **DuckDB.Generator.TRANSFORMS, exp.RegexpExtract: _regexp_extract_sql, exp.ParseDatetime: _parse_datetime_sql, - exp.GenerateSeries: _generate_array_sql, } diff --git a/tests/test_transpile.py b/tests/test_transpile.py index 151d075f4..8907c7ed4 100644 --- a/tests/test_transpile.py +++ b/tests/test_transpile.py @@ -74,9 +74,6 @@ def t(bq: str, dialect: str) -> str: # GENERATE_ARRAY -> ARRAY(SELECT ... GENERATE_SERIES) (postgres) ("generate_array_pg", "SELECT GENERATE_ARRAY(-24, 5) AS hrs FROM t", "postgres", "SELECT ARRAY(SELECT * FROM GENERATE_SERIES(-24, 5)) AS hrs FROM t"), - # GENERATE_ARRAY -> list via generate_series (duckdb); must stay list for UNNEST - ("generate_array_duckdb", "SELECT GENERATE_ARRAY(-24, 5) AS hrs FROM t", "duckdb", - "SELECT (SELECT list(g) FROM generate_series(-24, 5) AS t(g)) AS hrs FROM t"), # handled natively by sqlglot 30.x (regression guards) ("datetime_date_pg", "SELECT DATETIME(me.chartdate) FROM t me", "postgres",