handle archives in source extractor

processing_engine/destination_writer/destination_writer.py

@@ -24,6 +24,7 @@ def write_file(config,logger,dataframe):
                     destination_file = get_destination_location(row,destination).split('!')[0]
                     if destination_file is not None:
                         shutil.copyfile(RelativeFileArchiveURI.split('!')[0],destination_file)
+                        written_archives.append(RelativeFileArchiveURI.split('!')[0])
             else:
                 destination_file = get_destination_location(row,destination)
                 if destination_file is not None:

processing_engine/source_extractor/source_extractor.py

@@ -92,6 +92,15 @@ def handle_store(event):
     })
     return received
 
+def list_files_recursive(path):
+    entries = []
+    for entry in os.listdir(path):
+        full_path = os.path.join(path, entry)
+        if os.path.isdir(full_path):
+            entries.extend(list_files_recursive(full_path))
+        else:
+            entries.append(full_path)
+    return entries
 
 def source_extractor(config, logger):
     for name, source in config['sources'].items():
@@ -110,19 +119,23 @@ def source_extractor(config, logger):
                 included_extensions.append('zip')
                 archive_extraction_dir = pathlib.Path("./received_dicoms") / name
                 archive_extraction_dir.mkdir(parents=True, exist_ok=True)
-            #TODO this should maybe do nested directories
-            file_names = [fn for fn in os.listdir(source['directory'])
+            file_names = [fn for fn in list_files_recursive(source['directory'])
               if any(fn.endswith(ext) for ext in included_extensions)]
             file_regex = source.get("file_regex",None)
             if(file_regex is not None):
-                file_names = [fn for fn in file_names if re.match(f'{file_regex}',fn)]
+                file_names = [fn for fn in file_names if re.match(f'{file_regex}', os.path.basename(fn))]
             if archive_extraction_dir is not None:
                 for file in file_names:
                     if file.endswith('.7z'):
-                        with py7zr.SevenZipFile(os.path.join(source['directory'],file), mode='r') as archive:
-                            archive.extractall(path=os.path.join(archive_extraction_dir,file))#the count isn't right now
+                        with py7zr.SevenZipFile(file, mode='r') as archive:
+                            archive.extractall(path=os.path.join(archive_extraction_dir, os.path.basename(file)))
                     if(file.endswith('.zip')):
-                        print('todo')
+                        zip_extract_dir = os.path.join(archive_extraction_dir, os.path.basename(file))
+                        os.makedirs(zip_extract_dir, exist_ok=True)
+                        with zipfile.ZipFile(file, 'r') as zf:
+                            for member in zf.namelist():
+                                if member.lower().endswith('.dcm'):
+                                    zf.extract(member, zip_extract_dir)
             logger.info({
                 "message": f"found {len(file_names)} files in {name} directory",
                 "timestamp": datetime.datetime.now(datetime.UTC).timestamp(),

tests/test_source_extractor.py

@@ -3,6 +3,7 @@
 import pydicom
 from pydicom.dataset import Dataset, FileMetaDataset
 import py7zr
+import zipfile
 
 from source_extractor.source_extractor import _build_identifier, _cmove, source_extractor
 
@@ -312,12 +313,13 @@ def test_pacs_source_passes_pdu(mock_cmove):
 
 # ── source_extractor: filesystem – include_archives ───────────────────────
 
-def test_filesystem_source_with_include_archives_counts_archive_files(tmp_path):
+def test_filesystem_source_with_include_archives_counts_archive_files(tmp_path, monkeypatch):
     """With include_archives, .zip and .7z files are included in the reported count."""
+    monkeypatch.chdir(tmp_path)
     (tmp_path / "scan.dcm").touch()
-    (tmp_path / "backup.zip").touch()
-    # (tmp_path / "compressed.7z").touch()
     (tmp_path / "readme.txt").touch()
+    with zipfile.ZipFile(tmp_path / "backup.zip", "w") as zf:
+        zf.writestr("placeholder.dcm", b"DICM")
     with py7zr.SevenZipFile( (tmp_path / "compressed.7z"), 'w') as z:
         z.write(tmp_path / "scan.dcm")
     logger = Mock()
@@ -379,3 +381,27 @@ def test_filesystem_source_with_file_regex_no_match_reports_zero(tmp_path):
     source_extractor(config, logger)
 
     assert logger.info.call_args.args[0]["message"] == "found 0 files in images directory"
+
+
+def test_filesystem_source_extracts_dcm_files_from_zip_into_named_subfolder(tmp_path, monkeypatch):
+    """Zip extraction places .dcm members into archive_extraction_dir/<zipname>/."""
+    monkeypatch.chdir(tmp_path)
+
+    # Build a zip containing one .dcm and one unrelated file
+    zip_path = tmp_path / "scans.zip"
+    with zipfile.ZipFile(zip_path, "w") as zf:
+        zf.writestr("patient1.dcm", b"DICM")
+        zf.writestr("notes.txt", b"ignore me")
+
+    logger = Mock()
+    config = {"sources": {"images": {
+        "type": "filesystem",
+        "directory": str(tmp_path),
+        "include_archives": True,
+    }}}
+
+    source_extractor(config, logger)
+
+    extract_dir = tmp_path / "received_dicoms" / "images" / "scans.zip"
+    assert (extract_dir / "patient1.dcm").exists()
+    assert not (extract_dir / "notes.txt").exists()