diff --git a/processing_engine/destination_writer/destination_writer.py b/processing_engine/destination_writer/destination_writer.py index a38bbae..58f9a86 100644 --- a/processing_engine/destination_writer/destination_writer.py +++ b/processing_engine/destination_writer/destination_writer.py @@ -24,6 +24,7 @@ def write_file(config,logger,dataframe): destination_file = get_destination_location(row,destination).split('!')[0] if destination_file is not None: shutil.copyfile(RelativeFileArchiveURI.split('!')[0],destination_file) + written_archives.append(RelativeFileArchiveURI.split('!')[0]) else: destination_file = get_destination_location(row,destination) if destination_file is not None: diff --git a/processing_engine/source_extractor/source_extractor.py b/processing_engine/source_extractor/source_extractor.py index 63bc368..fecaa3f 100644 --- a/processing_engine/source_extractor/source_extractor.py +++ b/processing_engine/source_extractor/source_extractor.py @@ -92,6 +92,15 @@ def handle_store(event): }) return received +def list_files_recursive(path): + entries = [] + for entry in os.listdir(path): + full_path = os.path.join(path, entry) + if os.path.isdir(full_path): + entries.extend(list_files_recursive(full_path)) + else: + entries.append(full_path) + return entries def source_extractor(config, logger): for name, source in config['sources'].items(): @@ -110,19 +119,23 @@ def source_extractor(config, logger): included_extensions.append('zip') archive_extraction_dir = pathlib.Path("./received_dicoms") / name archive_extraction_dir.mkdir(parents=True, exist_ok=True) - #TODO this should maybe do nested directories - file_names = [fn for fn in os.listdir(source['directory']) + file_names = [fn for fn in list_files_recursive(source['directory']) if any(fn.endswith(ext) for ext in included_extensions)] file_regex = source.get("file_regex",None) if(file_regex is not None): - file_names = [fn for fn in file_names if re.match(f'{file_regex}',fn)] + file_names = [fn for fn in file_names if re.match(f'{file_regex}', os.path.basename(fn))] if archive_extraction_dir is not None: for file in file_names: if file.endswith('.7z'): - with py7zr.SevenZipFile(os.path.join(source['directory'],file), mode='r') as archive: - archive.extractall(path=os.path.join(archive_extraction_dir,file))#the count isn't right now + with py7zr.SevenZipFile(file, mode='r') as archive: + archive.extractall(path=os.path.join(archive_extraction_dir, os.path.basename(file))) if(file.endswith('.zip')): - print('todo') + zip_extract_dir = os.path.join(archive_extraction_dir, os.path.basename(file)) + os.makedirs(zip_extract_dir, exist_ok=True) + with zipfile.ZipFile(file, 'r') as zf: + for member in zf.namelist(): + if member.lower().endswith('.dcm'): + zf.extract(member, zip_extract_dir) logger.info({ "message": f"found {len(file_names)} files in {name} directory", "timestamp": datetime.datetime.now(datetime.UTC).timestamp(), diff --git a/tests/test_source_extractor.py b/tests/test_source_extractor.py index 2f52613..a128acd 100644 --- a/tests/test_source_extractor.py +++ b/tests/test_source_extractor.py @@ -3,6 +3,7 @@ import pydicom from pydicom.dataset import Dataset, FileMetaDataset import py7zr +import zipfile from source_extractor.source_extractor import _build_identifier, _cmove, source_extractor @@ -312,12 +313,13 @@ def test_pacs_source_passes_pdu(mock_cmove): # ── source_extractor: filesystem – include_archives ─────────────────────── -def test_filesystem_source_with_include_archives_counts_archive_files(tmp_path): +def test_filesystem_source_with_include_archives_counts_archive_files(tmp_path, monkeypatch): """With include_archives, .zip and .7z files are included in the reported count.""" + monkeypatch.chdir(tmp_path) (tmp_path / "scan.dcm").touch() - (tmp_path / "backup.zip").touch() - # (tmp_path / "compressed.7z").touch() (tmp_path / "readme.txt").touch() + with zipfile.ZipFile(tmp_path / "backup.zip", "w") as zf: + zf.writestr("placeholder.dcm", b"DICM") with py7zr.SevenZipFile( (tmp_path / "compressed.7z"), 'w') as z: z.write(tmp_path / "scan.dcm") logger = Mock() @@ -379,3 +381,27 @@ def test_filesystem_source_with_file_regex_no_match_reports_zero(tmp_path): source_extractor(config, logger) assert logger.info.call_args.args[0]["message"] == "found 0 files in images directory" + + +def test_filesystem_source_extracts_dcm_files_from_zip_into_named_subfolder(tmp_path, monkeypatch): + """Zip extraction places .dcm members into archive_extraction_dir//.""" + monkeypatch.chdir(tmp_path) + + # Build a zip containing one .dcm and one unrelated file + zip_path = tmp_path / "scans.zip" + with zipfile.ZipFile(zip_path, "w") as zf: + zf.writestr("patient1.dcm", b"DICM") + zf.writestr("notes.txt", b"ignore me") + + logger = Mock() + config = {"sources": {"images": { + "type": "filesystem", + "directory": str(tmp_path), + "include_archives": True, + }}} + + source_extractor(config, logger) + + extract_dir = tmp_path / "received_dicoms" / "images" / "scans.zip" + assert (extract_dir / "patient1.dcm").exists() + assert not (extract_dir / "notes.txt").exists()