From 39b86e4a8bdcf971f976aaec68bcc306b56a4312 Mon Sep 17 00:00:00 2001 From: Logic <38597904+Logiquo@users.noreply.github.com> Date: Sun, 12 Apr 2026 13:19:38 -0400 Subject: [PATCH 01/61] Fix Interpretability Methods `target_class_idx` (#926) * rename arg name for chefer * Initial attempts to fix the interpretability target_class_idx * Support negative prediction for interpretability metric. * Fix tests * Fix more tests * Revert "Support negative prediction for interpretability metric." This reverts commit fe8c8ad07c54a9ed315c7d8c7a2001afa45b5fd8. * Reapply "Support all samples for interpretability metric" * Initial attempt for the filter * Fixup * Fix sample_class handling * fixup * fix test * Fix arg name * Add example * fix docs --- examples/cxr/covid19cxr_tutorial.ipynb | 2 +- examples/cxr/covid19cxr_tutorial.py | 2 +- examples/cxr/covid19cxr_tutorial_display.py | 2 +- .../interpretability/custom_sample_filter.py | 189 ++++++++++++++++++ .../interpret/methods/base_interpreter.py | 46 ++++- pyhealth/interpret/methods/chefer.py | 19 +- pyhealth/interpret/methods/deeplift.py | 79 ++------ pyhealth/interpret/methods/gim.py | 69 ++----- pyhealth/interpret/methods/ig_gim.py | 66 +----- .../interpret/methods/integrated_gradients.py | 81 ++------ pyhealth/interpret/methods/lime.py | 40 ++-- pyhealth/interpret/methods/shap.py | 97 +-------- pyhealth/metrics/interpretability/__init__.py | 12 +- pyhealth/metrics/interpretability/base.py | 160 ++++++++------- .../metrics/interpretability/evaluator.py | 152 ++++++++++---- pyhealth/metrics/interpretability/utils.py | 155 +++++++++----- tests/core/test_deeplift.py | 6 +- tests/core/test_gim.py | 4 +- tests/core/test_ig_gim.py | 7 +- tests/core/test_integrated_gradients.py | 12 +- tests/core/test_interp_metrics.py | 28 ++- tests/core/test_lime.py | 12 +- tests/core/test_shap.py | 32 +-- tests/core/test_transformer.py | 8 +- 24 files changed, 691 insertions(+), 589 deletions(-) create mode 100644 examples/interpretability/custom_sample_filter.py diff --git a/examples/cxr/covid19cxr_tutorial.ipynb b/examples/cxr/covid19cxr_tutorial.ipynb index 2a04844c5..ec10756a1 100644 --- a/examples/cxr/covid19cxr_tutorial.ipynb +++ b/examples/cxr/covid19cxr_tutorial.ipynb @@ -1339,7 +1339,7 @@ " # Input size is inferred automatically from image dimensions\n", " result = chefer_gen.attribute(\n", " interpolate=True,\n", - " class_index=pred_class,\n", + " target_class_idx=pred_class,\n", " **batch\n", " )\n", " attr_map = result[\"image\"] # Keyed by task schema's feature key\n", diff --git a/examples/cxr/covid19cxr_tutorial.py b/examples/cxr/covid19cxr_tutorial.py index 0f24f4b58..06b134f93 100644 --- a/examples/cxr/covid19cxr_tutorial.py +++ b/examples/cxr/covid19cxr_tutorial.py @@ -131,7 +131,7 @@ # Compute attribution for each class in the prediction set overlays = [] for class_idx in predset_class_indices: - attr_map = chefer.attribute(class_index=class_idx, **batch)["image"] + attr_map = chefer.attribute(target_class_idx=class_idx, **batch)["image"] _, _, overlay = visualize_image_attr( image=batch["image"][0], attribution=attr_map[0, 0], diff --git a/examples/cxr/covid19cxr_tutorial_display.py b/examples/cxr/covid19cxr_tutorial_display.py index 3f6a33b82..f3a4acddb 100644 --- a/examples/cxr/covid19cxr_tutorial_display.py +++ b/examples/cxr/covid19cxr_tutorial_display.py @@ -128,7 +128,7 @@ # Compute attribution for each class in the prediction set overlays = [] for class_idx in predset_class_indices: - attr_map = chefer.attribute(class_index=class_idx, **batch)["image"] + attr_map = chefer.attribute(target_class_idx=class_idx, **batch)["image"] _, _, overlay = visualize_image_attr( image=batch["image"][0], attribution=attr_map[0, 0], diff --git a/examples/interpretability/custom_sample_filter.py b/examples/interpretability/custom_sample_filter.py new file mode 100644 index 000000000..da59546c5 --- /dev/null +++ b/examples/interpretability/custom_sample_filter.py @@ -0,0 +1,189 @@ +"""Evaluate all interpretability methods on StageNet + MIMIC-IV dataset using comprehensiveness +and sufficiency metrics. + +This example demonstrates: +1. Loading a pre-trained StageNet model with processors and MIMIC-IV dataset +2. Computing attributions with various interpretability methods +3. Evaluating attribution faithfulness with Comprehensiveness & Sufficiency for each method +4. Presenting results in a summary table +""" + +import datetime +import argparse + +import torch +from pyhealth.datasets import MIMIC4Dataset, get_dataloader, split_by_patient +from pyhealth.interpret.methods import * +from pyhealth.metrics.interpretability import evaluate_attribution +from pyhealth.metrics.interpretability.utils import SampleClass +from pyhealth.models import Transformer +from pyhealth.tasks import MortalityPredictionStageNetMIMIC4 +from pyhealth.trainer import Trainer +from pyhealth.datasets.utils import load_processors +from pathlib import Path +import pandas as pd + +# python -u examples/interpretability/custom_sample_filter.py --pos_threshold 0.5 --neg_threshold 0.1 --device cuda:2 +def main(): + parser = argparse.ArgumentParser( + description="Comma separated list of interpretability methods to evaluate" + ) + parser.add_argument( + "--pos_threshold", + type=float, + default=None, + help="Positive threshold for interpretability evaluation (default: 0.5).", + ) + parser.add_argument( + "--neg_threshold", + type=float, + default=None, + help="Negative threshold for interpretability evaluation (default: 0.5).", + ) + parser.add_argument( + "--device", + type=str, + default="cuda:0", + help="Device to use for evaluation (default: cuda:0)", + ) + args = parser.parse_args() + """Main execution function.""" + print("=" * 70) + print("Interpretability Metrics Example: Transformer + MIMIC-IV") + print("=" * 70) + + now = datetime.datetime.now() + print(f"Start Time: {now.strftime('%Y-%m-%d %H:%M:%S')}") + + # Set path + CACHE_DIR = Path("/home/yongdaf2/interpret/cache/mp_mimic4") + CKPTS_DIR = Path("/shared/eng/pyhealth_dka/ckpts/mp_transformer_mimic4") + OUTPUT_DIR = Path("/home/yongdaf2/interpret/output/mp_transformer_mimic4") + CACHE_DIR.mkdir(parents=True, exist_ok=True) + CKPTS_DIR.mkdir(parents=True, exist_ok=True) + OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + print(f"\nUsing cache dir: {CACHE_DIR}") + print(f"Using checkpoints dir: {CKPTS_DIR}") + print(f"Using output dir: {OUTPUT_DIR}") + + # Set device + device = args.device + print(f"\nUsing device: {device}") + + # Load MIMIC-IV dataset + print("\n Loading MIMIC-IV dataset...") + base_dataset = MIMIC4Dataset( + ehr_root="/srv/local/data/physionet.org/files/mimiciv/2.2/", + ehr_tables=[ + "patients", + "admissions", + "diagnoses_icd", + "procedures_icd", + "labevents", + ], + cache_dir=str(CACHE_DIR), + num_workers=16, + ) + + # Apply mortality prediction task + if not (CKPTS_DIR / "input_processors.pkl").exists(): + raise FileNotFoundError(f"Input processors not found in {CKPTS_DIR}. ") + if not (CKPTS_DIR / "output_processors.pkl").exists(): + raise FileNotFoundError(f"Output processors not found in {CKPTS_DIR}. ") + input_processors, output_processors = load_processors(str(CKPTS_DIR)) + print("✓ Loaded input and output processors from checkpoint directory.") + + sample_dataset = base_dataset.set_task( + MortalityPredictionStageNetMIMIC4(), + num_workers=16, + input_processors=input_processors, + output_processors=output_processors, + ) + print(f"✓ Loaded {len(sample_dataset)} samples") + + # Split dataset and get test loader + _, _, test_dataset = split_by_patient(sample_dataset, [0.9, 0.09, 0.01], seed=233) + test_loader = get_dataloader(test_dataset, batch_size=16, shuffle=False) + print(f"✓ Test set: {len(test_dataset)} samples") + + # Initialize and load pre-trained model + print("\n Loading pre-trained Transformer model...") + model = Transformer( + dataset=sample_dataset, + embedding_dim=128, + heads=4, + dropout=0.3, + num_layers=3, + ) + + trainer = Trainer(model=model, device=device) + trainer.load_ckpt(str(CKPTS_DIR / "best.ckpt")) + model = model.to(device) + model.eval() + print(f"✓ Loaded checkpoint: {CKPTS_DIR / 'best.ckpt'}") + print(f"✓ Model moved to {device}") + + pos_threshold = args.pos_threshold + neg_threshold = args.neg_threshold + def sample_filter_fn( + y_probs: torch.Tensor, + classifier_type: str, + ) -> torch.Tensor: + """ + Custom sample filter function that classifies samples based on + positive and negative probability thresholds. + + negative samples: 0 < y_probs < neg_threshold + ignored samples: neg_threshold <= y_probs < pos_threshold + positive samples: y_probs >= pos_threshold + """ + nonlocal pos_threshold, neg_threshold + batch_size = y_probs.shape[0] + result = torch.full( + (batch_size,), + SampleClass.POSITIVE, + dtype=torch.long, + device=y_probs.device, + ) + if classifier_type in ("binary", "multilabel"): + if pos_threshold is not None: + result[y_probs < pos_threshold] = SampleClass.IGNORE + if neg_threshold is not None: + result[y_probs < neg_threshold] = SampleClass.NEGATIVE + return result + + interpreter = IntegratedGradients(model, use_embeddings=True) + print(f"\nEvaluating using Integrated Gradients...") + + # Option 1: Functional API (simple one-off evaluation) + print("\nEvaluating with Functional API on full dataset...") + print("Using: evaluate_attribution(model, dataloader, method, ...)") + + results_functional = evaluate_attribution( + model, + test_loader, + interpreter, + metrics=["comprehensiveness", "sufficiency"], + percentages=[25, 50, 99], + sample_filter=sample_filter_fn, + ) + + print("\n" + "=" * 70) + print("Dataset-Wide Results (Functional API)") + print("=" * 70) + comp = results_functional["comprehensiveness"] + suff = results_functional["sufficiency"] + print(f"\nComprehensiveness: {comp:.4f}") + print(f"Sufficiency: {suff:.4f}") + + print("") + print("=" * 70) + print("Summary of Results for All Methods") + print({"Method": "Integrated Gradients", "Comprehensiveness": comp, "Sufficiency": suff}) + + end = datetime.datetime.now() + print(f"End Time: {end.strftime('%Y-%m-%d %H:%M:%S')}") + print(f"Total Duration: {end - now}") + +if __name__ == "__main__": + main() diff --git a/pyhealth/interpret/methods/base_interpreter.py b/pyhealth/interpret/methods/base_interpreter.py index de75c897a..fb17690ae 100644 --- a/pyhealth/interpret/methods/base_interpreter.py +++ b/pyhealth/interpret/methods/base_interpreter.py @@ -10,7 +10,7 @@ """ from abc import ABC, abstractmethod -from typing import Dict, cast +from typing import Dict, Optional, cast import torch import torch.nn as nn @@ -138,8 +138,12 @@ def attribute( by the task's ``input_schema``. - Label key (optional): Ground truth labels, may be needed by some methods for loss computation. - - ``class_index`` (optional): Target class for attribution. - If not provided, uses the predicted class. + - ``target_class_idx`` (Optional[int]): Target class for + attribution. For binary classification (single logit + output), this is a no-op because there is only one + output. For multi-class or multi-label classification, + specifies which class index to explain. If not provided, + uses the argmax of logits. - Additional method-specific parameters (e.g., ``baseline``, ``steps``, ``interpolate``). @@ -207,6 +211,42 @@ def attribute( """ pass + def _resolve_target_indices( + self, + logits: torch.Tensor, + target_class_idx: Optional[int], + ) -> torch.Tensor: + """Resolve target class indices for attribution. + + Returns a ``[batch]`` tensor of class indices identifying which + logit to explain. All prediction modes share this single code + path: + + * **Binary** (single logit): ``target_class_idx`` is a no-op + because there is only one output. Always returns zeros + (index 0). + * **Multi-class / multi-label**: uses ``target_class_idx`` if + given, otherwise the argmax of logits. + + Args: + logits: Model output logits, shape ``[batch, num_classes]``. + target_class_idx: Optional user-specified class index. + + Returns: + ``torch.LongTensor`` of shape ``[batch]``. + """ + if logits.shape[-1] == 1: + # Single logit output — nothing to select. + return torch.zeros( + logits.shape[0], device=logits.device, dtype=torch.long, + ) + if target_class_idx is not None: + return torch.full( + (logits.shape[0],), target_class_idx, + device=logits.device, dtype=torch.long, + ) + return logits.argmax(dim=-1) + def _prediction_mode(self) -> str: """Resolve the prediction mode from the model. diff --git a/pyhealth/interpret/methods/chefer.py b/pyhealth/interpret/methods/chefer.py index 5efca68eb..26ce6ffd2 100644 --- a/pyhealth/interpret/methods/chefer.py +++ b/pyhealth/interpret/methods/chefer.py @@ -128,7 +128,7 @@ class CheferRelevance(BaseInterpreter): >>> print(attributions["conditions"].shape) # [batch, num_tokens] >>> >>> # Optional: attribute to a specific class (e.g., class 1) - >>> attributions = interpreter.attribute(class_index=1, **batch) + >>> attributions = interpreter.attribute(target_class_idx=1, **batch) """ def __init__(self, model: BaseModel): @@ -139,14 +139,16 @@ def __init__(self, model: BaseModel): def attribute( self, - class_index: Optional[int] = None, + target_class_idx: Optional[int] = None, **data, ) -> Dict[str, torch.Tensor]: """Compute relevance scores for each input token. Args: - class_index: Target class index to compute attribution for. - If None (default), uses the model's predicted class. + target_class_idx: Target class index to compute attribution for. + If None (default), uses the argmax of model output. + For binary classification (single logit output), this is + a no-op because there is only one output. **data: Input data from dataloader batch containing feature keys and label key. @@ -163,15 +165,10 @@ def attribute( self.model.set_attention_hooks(False) # --- 2. Backward from target class --- - if class_index is None: - class_index_t = torch.argmax(logits, dim=-1) - elif isinstance(class_index, int): - class_index_t = torch.tensor(class_index) - else: - class_index_t = class_index + target_indices = self._resolve_target_indices(logits, target_class_idx) one_hot = F.one_hot( - class_index_t.detach().clone(), logits.size(1) + target_indices.detach().clone(), logits.size(1) ).float() one_hot = one_hot.requires_grad_(True) scalar = torch.sum(one_hot.to(logits.device) * logits) diff --git a/pyhealth/interpret/methods/deeplift.py b/pyhealth/interpret/methods/deeplift.py index 29f99d795..8f00f8a8b 100644 --- a/pyhealth/interpret/methods/deeplift.py +++ b/pyhealth/interpret/methods/deeplift.py @@ -411,35 +411,7 @@ def attribute( with torch.no_grad(): base_logits = self.model.forward(**inputs)["logit"] - mode = self._prediction_mode() - if mode == "binary": - if target_class_idx is not None: - target = torch.tensor([target_class_idx], device=device) - else: - target = (torch.sigmoid(base_logits) > 0.5).long() - elif mode == "multiclass": - if target_class_idx is not None: - target = F.one_hot( - torch.tensor(target_class_idx, device=device), - num_classes=base_logits.shape[-1], - ).float() - else: - target = torch.argmax(base_logits, dim=-1) - target = F.one_hot( - target, num_classes=base_logits.shape[-1] - ).float() - elif mode == "multilabel": - if target_class_idx is not None: - target = F.one_hot( - torch.tensor(target_class_idx, device=device), - num_classes=base_logits.shape[-1], - ).float() - else: - target = (torch.sigmoid(base_logits) > 0.5).float() - else: - raise ValueError( - "Unsupported prediction mode for DeepLIFT attribution." - ) + target_indices = self._resolve_target_indices(base_logits, target_class_idx) # Generate baselines if baseline is None: @@ -491,7 +463,7 @@ def attribute( inputs=inputs, xs=values, bs=baselines, - target=target, + target_indices=target_indices, token_keys=token_keys, ) @@ -505,7 +477,7 @@ def _deeplift( inputs: Dict[str, tuple[torch.Tensor, ...]], xs: Dict[str, torch.Tensor], bs: Dict[str, torch.Tensor], - target: torch.Tensor, + target_indices: torch.Tensor, token_keys: set[str], ) -> Dict[str, torch.Tensor]: """Core DeepLIFT computation using the Rescale rule. @@ -517,8 +489,7 @@ def _deeplift( inputs: Full input tuples keyed by feature name. xs: Input values (embedded if token features with use_embeddings). bs: Baseline values (embedded if token features with use_embeddings). - target: Target tensor for computing the scalar output to - differentiate (one-hot for multiclass, class idx for binary). + target_indices: [batch] tensor of target class indices. token_keys: Set of feature keys that are token (already embedded). Returns: @@ -590,9 +561,9 @@ def _maybe_embed_continuous(value_dict: dict[str, torch.Tensor]) -> dict[str, to baseline_logits = baseline_output["logit"] # type: ignore[index] # Compute per-sample target outputs - target_output = self._compute_target_output(logits, target) + target_output = self._compute_target_output(logits, target_indices) baseline_target_output = self._compute_target_output( - baseline_logits, target + baseline_logits, target_indices ) self.model.zero_grad(set_to_none=True) @@ -626,46 +597,22 @@ def _maybe_embed_continuous(value_dict: dict[str, torch.Tensor]) -> dict[str, to def _compute_target_output( self, logits: torch.Tensor, - target: torch.Tensor, + target_indices: torch.Tensor, ) -> torch.Tensor: """Compute per-sample target output. - Creates a differentiable per-sample scalar from the model logits - that, when summed and differentiated, gives the gradient of the - target class logit w.r.t. the input. + Selects the target-class logit for each sample. Args: - logits: Model output logits, shape [batch, num_classes] or - [batch, 1]. - target: Target tensor. For binary: [batch] or [1] with 0/1 - class indices. For multiclass/multilabel: [batch, num_classes] - one-hot or multi-hot tensor. + logits: Model output logits, shape [batch, num_classes]. + target_indices: [batch] tensor of target class indices. Returns: Per-sample target output tensor, shape [batch]. """ - target_f = target.to(logits.device).float() - mode = self._prediction_mode() - - if mode == "binary": - while target_f.dim() < logits.dim(): - target_f = target_f.unsqueeze(-1) - target_f = target_f.expand_as(logits) - signs = 2.0 * target_f - 1.0 - # Sum over all dims except batch to get per-sample scalar - per_sample = (signs * logits) - if per_sample.dim() > 1: - per_sample = per_sample.sum(dim=tuple(range(1, per_sample.dim()))) - return per_sample - else: - # multiclass or multilabel: target is one-hot/multi-hot - while target_f.dim() < logits.dim(): - target_f = target_f.unsqueeze(0) - target_f = target_f.expand_as(logits) - per_sample = (target_f * logits) - if per_sample.dim() > 1: - per_sample = per_sample.sum(dim=tuple(range(1, per_sample.dim()))) - return per_sample + return logits.gather( + 1, target_indices.unsqueeze(1) + ).squeeze(1) # ------------------------------------------------------------------ # Completeness enforcement diff --git a/pyhealth/interpret/methods/gim.py b/pyhealth/interpret/methods/gim.py index d1b74b573..abbb9388a 100644 --- a/pyhealth/interpret/methods/gim.py +++ b/pyhealth/interpret/methods/gim.py @@ -365,8 +365,9 @@ def attribute( """Compute GIM attributions for a batch. Args: - target_class_idx: Target class index for attribution. If None, - uses the model's predicted class. + target_class_idx: Target class index for attribution. For + binary classification (single logit output), this is a + no-op. If None, uses the argmax of model output. **kwargs: Input data dictionary from a dataloader batch containing feature tensors or tuples of tensors for each modality, plus optional label tensors. @@ -419,35 +420,7 @@ def attribute( with torch.no_grad(): base_logits = self.model.forward(**inputs)["logit"] - mode = self._prediction_mode() - if mode == "binary": - if target_class_idx is not None: - target = torch.tensor([target_class_idx], device=device) - else: - target = (torch.sigmoid(base_logits) > 0.5).long() - elif mode == "multiclass": - if target_class_idx is not None: - target = F.one_hot( - torch.tensor(target_class_idx, device=device), - num_classes=base_logits.shape[-1], - ).float() - else: - target = torch.argmax(base_logits, dim=-1) - target = F.one_hot( - target, num_classes=base_logits.shape[-1] - ).float() - elif mode == "multilabel": - if target_class_idx is not None: - target = F.one_hot( - torch.tensor(target_class_idx, device=device), - num_classes=base_logits.shape[-1], - ).float() - else: - target = (torch.sigmoid(base_logits) > 0.5).float() - else: - raise ValueError( - "Unsupported prediction mode for GIM attribution." - ) + target_indices = self._resolve_target_indices(base_logits, target_class_idx) # Embed values and detach for gradient attribution. # Split features by type using is_token(): @@ -521,7 +494,7 @@ def attribute( output = self.model.forward_from_embedding(**forward_inputs) logits = output["logit"] # type: ignore[assignment] - target_output = self._compute_target_output(logits, target) + target_output = self._compute_target_output(logits, target_indices) # Clear stale gradients, then backpropagate through the # GIM-modified computational graph. @@ -552,39 +525,23 @@ def attribute( def _compute_target_output( self, logits: torch.Tensor, - target: torch.Tensor, + target_indices: torch.Tensor, ) -> torch.Tensor: """Compute scalar target output for backpropagation. - Creates a differentiable scalar from the model logits that, - when differentiated, gives the gradient of the target class - logit w.r.t. the input. + Selects the target-class logit for each sample and sums over + the batch to produce a single differentiable scalar. Args: - logits: Model output logits, shape [batch, num_classes] or - [batch, 1]. - target: Target tensor. For binary: [batch] or [1] with 0/1 - class indices. For multiclass/multilabel: [batch, num_classes] - one-hot or multi-hot tensor. + logits: Model output logits, shape [batch, num_classes]. + target_indices: [batch] tensor of target class indices. Returns: Scalar tensor for backpropagation. """ - target_f = target.to(logits.device).float() - mode = self._prediction_mode() - - if mode == "binary": - while target_f.dim() < logits.dim(): - target_f = target_f.unsqueeze(-1) - target_f = target_f.expand_as(logits) - signs = 2.0 * target_f - 1.0 - return (signs * logits).sum() - else: - # multiclass or multilabel: target is one-hot/multi-hot - while target_f.dim() < logits.dim(): - target_f = target_f.unsqueeze(0) - target_f = target_f.expand_as(logits) - return (target_f * logits).sum() + return logits.gather( + 1, target_indices.unsqueeze(1) + ).squeeze(1).sum() # ------------------------------------------------------------------ # Utility helpers diff --git a/pyhealth/interpret/methods/ig_gim.py b/pyhealth/interpret/methods/ig_gim.py index a33f5529b..49c2fa6c0 100644 --- a/pyhealth/interpret/methods/ig_gim.py +++ b/pyhealth/interpret/methods/ig_gim.py @@ -105,8 +105,9 @@ def attribute( near-zero for continuous features). steps: Number of interpolation steps. Overrides the instance default when given. - target_class_idx: Target class for attribution. ``None`` - uses the model's predicted class. + target_class_idx: Target class for attribution. For binary + classification (single logit output), this is a no-op. + ``None`` uses the argmax of model output. **kwargs: Dataloader batch (feature tensors + optional labels). Returns: @@ -155,10 +156,7 @@ def attribute( with torch.no_grad(): base_logits = self.model.forward(**inputs)["logit"] - mode = self._prediction_mode() - target = self._resolve_target( - base_logits, mode, target_class_idx, device - ) + target_indices = self._resolve_target_indices(base_logits, target_class_idx) # ----- baselines ----- if baseline is None: @@ -201,7 +199,7 @@ def attribute( xs=values, bs=baselines, steps=steps, - target=target, + target_indices=target_indices, token_keys=token_keys, continuous_keys=continuous_keys, ) @@ -217,7 +215,7 @@ def _integrated_gradients_gim( xs: Dict[str, torch.Tensor], bs: Dict[str, torch.Tensor], steps: int, - target: torch.Tensor, + target_indices: torch.Tensor, token_keys: set[str], continuous_keys: set[str], ) -> Dict[str, torch.Tensor]: @@ -280,7 +278,7 @@ def _integrated_gradients_gim( with _GIMHookContext(self.model, self.temperature): output = self.model.forward_from_embedding(**forward_inputs) logits = output["logit"] - target_output = self._compute_target_output(logits, target) + target_output = self._compute_target_output(logits, target_indices) self.model.zero_grad(set_to_none=True) target_output.backward(retain_graph=True) @@ -308,58 +306,16 @@ def _integrated_gradients_gim( # ------------------------------------------------------------------ # Target helpers (shared logic with IG / GIM) # ------------------------------------------------------------------ - @staticmethod - def _resolve_target( - logits: torch.Tensor, - mode: str, - target_class_idx: Optional[int], - device: torch.device, - ) -> torch.Tensor: - """Convert logits and optional class index into a target tensor.""" - if mode == "binary": - if target_class_idx is not None: - return torch.tensor([target_class_idx], device=device) - return (torch.sigmoid(logits) > 0.5).long() - - if mode == "multiclass": - if target_class_idx is not None: - return F.one_hot( - torch.tensor(target_class_idx, device=device), - num_classes=logits.shape[-1], - ).float() - target = torch.argmax(logits, dim=-1) - return F.one_hot(target, num_classes=logits.shape[-1]).float() - - if mode == "multilabel": - if target_class_idx is not None: - return F.one_hot( - torch.tensor(target_class_idx, device=device), - num_classes=logits.shape[-1], - ).float() - return (torch.sigmoid(logits) > 0.5).float() - - raise ValueError(f"Unsupported prediction mode: {mode}") def _compute_target_output( self, logits: torch.Tensor, - target: torch.Tensor, + target_indices: torch.Tensor, ) -> torch.Tensor: """Scalar target output for backpropagation.""" - target_f = target.to(logits.device).float() - mode = self._prediction_mode() - - if mode == "binary": - while target_f.dim() < logits.dim(): - target_f = target_f.unsqueeze(-1) - target_f = target_f.expand_as(logits) - signs = 2.0 * target_f - 1.0 - return (signs * logits).sum() - else: - while target_f.dim() < logits.dim(): - target_f = target_f.unsqueeze(0) - target_f = target_f.expand_as(logits) - return (target_f * logits).sum() + return logits.gather( + 1, target_indices.unsqueeze(1) + ).squeeze(1).sum() # ------------------------------------------------------------------ # Baseline generation diff --git a/pyhealth/interpret/methods/integrated_gradients.py b/pyhealth/interpret/methods/integrated_gradients.py index a529a6f3f..249f5a5e9 100644 --- a/pyhealth/interpret/methods/integrated_gradients.py +++ b/pyhealth/interpret/methods/integrated_gradients.py @@ -217,9 +217,11 @@ def attribute( the integral. If None, uses self.steps (set during initialization). More steps lead to better approximation but slower computation. - target_class_idx: Target class index for attribution - computation. If None, uses the predicted class (argmax of - model output). + target_class_idx: Target class index for attribution. + For binary classification (single logit output), this is + a no-op because there is only one output. For multi-class + or multi-label, specifies which class to explain. If None, + uses the argmax of model output. **kwargs: Input data dictionary from a dataloader batch containing: - Feature keys (e.g., 'conditions', 'procedures'): @@ -324,35 +326,7 @@ def attribute( with torch.no_grad(): base_logits = self.model.forward(**inputs)["logit"] - mode = self._prediction_mode() - if mode == "binary": - if target_class_idx is not None: - target = torch.tensor([target_class_idx], device=device) - else: - target = (torch.sigmoid(base_logits) > 0.5).long() - elif mode == "multiclass": - if target_class_idx is not None: - target = F.one_hot( - torch.tensor(target_class_idx, device=device), - num_classes=base_logits.shape[-1], - ).float() - else: - target = torch.argmax(base_logits, dim=-1) - target = F.one_hot( - target, num_classes=base_logits.shape[-1] - ).float() - elif mode == "multilabel": - if target_class_idx is not None: - target = F.one_hot( - torch.tensor(target_class_idx, device=device), - num_classes=base_logits.shape[-1], - ).float() - else: - target = (torch.sigmoid(base_logits) > 0.5).float() - else: - raise ValueError( - "Unsupported prediction mode for Integrated Gradients attribution." - ) + target_indices = self._resolve_target_indices(base_logits, target_class_idx) # Generate baselines if baseline is None: @@ -405,7 +379,7 @@ def attribute( xs=values, bs=baselines, steps=steps, - target=target, + target_indices=target_indices, ) return self._map_to_input_shapes(attributions, shapes) @@ -419,7 +393,7 @@ def _integrated_gradients( xs: Dict[str, torch.Tensor], bs: Dict[str, torch.Tensor], steps: int, - target: torch.Tensor, + target_indices: torch.Tensor, ) -> Dict[str, torch.Tensor]: """Compute integrated gradients via Riemann sum approximation. @@ -438,8 +412,7 @@ def _integrated_gradients( xs: Input values (embedded if use_embeddings=True). bs: Baseline values (embedded if use_embeddings=True). steps: Number of interpolation steps. - target: Target tensor for computing the scalar output to - differentiate (one-hot for multiclass, class idx for binary). + target_indices: [batch] tensor of target class indices. Returns: Dictionary mapping feature keys to attribution tensors. @@ -513,7 +486,7 @@ def _integrated_gradients( logits = output["logit"] # Compute target output and backward pass - target_output = self._compute_target_output(logits, target) + target_output = self._compute_target_output(logits, target_indices) self.model.zero_grad() target_output.backward(retain_graph=True) @@ -550,41 +523,23 @@ def _integrated_gradients( def _compute_target_output( self, logits: torch.Tensor, - target: torch.Tensor, + target_indices: torch.Tensor, ) -> torch.Tensor: """Compute scalar target output for backpropagation. - Creates a differentiable scalar from the model logits that, - when differentiated, gives the gradient of the target class - logit w.r.t. the input. + Selects the target-class logit for each sample and sums over + the batch to produce a single differentiable scalar. Args: - logits: Model output logits, shape [batch, num_classes] or - [batch, 1]. - target: Target tensor. For binary: [batch] or [1] with 0/1 - class indices. For multiclass/multilabel: [batch, num_classes] - one-hot or multi-hot tensor. + logits: Model output logits, shape [batch, num_classes]. + target_indices: [batch] tensor of target class indices. Returns: Scalar tensor for backpropagation. """ - target_f = target.to(logits.device).float() - mode = self._prediction_mode() - - if mode == "binary": - # target shape: [1] or [batch, 1] with 0/1 values - # Convert to signs: 0 -> -1, 1 -> 1 - while target_f.dim() < logits.dim(): - target_f = target_f.unsqueeze(-1) - target_f = target_f.expand_as(logits) - signs = 2.0 * target_f - 1.0 - return (signs * logits).sum() - else: - # multiclass or multilabel: target is one-hot/multi-hot - while target_f.dim() < logits.dim(): - target_f = target_f.unsqueeze(0) - target_f = target_f.expand_as(logits) - return (target_f * logits).sum() + return logits.gather( + 1, target_indices.unsqueeze(1) + ).squeeze(1).sum() # ------------------------------------------------------------------ # Baseline generation diff --git a/pyhealth/interpret/methods/lime.py b/pyhealth/interpret/methods/lime.py index 5176bfeaf..4e407fccf 100644 --- a/pyhealth/interpret/methods/lime.py +++ b/pyhealth/interpret/methods/lime.py @@ -250,25 +250,7 @@ def attribute( # Extract and prepare inputs base_logits = self.model.forward(**inputs)["logit"] - # Enforce target class selection for multi-class models to avoid class flipping - if self._prediction_mode() == "binary": - if target_class_idx is not None: - target = torch.tensor([target_class_idx], device=device) - else: - target = (torch.sigmoid(base_logits) > 0.5).long() - elif self._prediction_mode() == "multiclass": - if target_class_idx is not None: - target = torch.nn.functional.one_hot(torch.tensor(target_class_idx, device=device), num_classes=base_logits.shape[-1]) - else: - target = torch.argmax(base_logits, dim=-1) - target = torch.nn.functional.one_hot(target, num_classes=base_logits.shape[-1]) - elif self._prediction_mode() == "multilabel": - if target_class_idx is not None: - target = torch.nn.functional.one_hot(torch.tensor(target_class_idx, device=device), num_classes=base_logits.shape[-1]) - else: - target = torch.sigmoid(base_logits) > 0.5 - else: - raise ValueError("Unsupported prediction mode for LIME attribution.") + target_indices = self._resolve_target_indices(base_logits, target_class_idx) if baseline is None: baselines = self._generate_baseline(values, use_embeddings=self.use_embeddings) @@ -309,7 +291,7 @@ def attribute( xs=values, bs=baselines, n_features=n_features, - target=target, + target_indices=target_indices, ) return self._map_to_input_shapes(out, shapes) @@ -323,7 +305,7 @@ def _compute_lime( xs: Dict[str, torch.Tensor], bs: Dict[str, torch.Tensor], n_features: dict[str, int], - target: torch.Tensor, + target_indices: torch.Tensor, ) -> Dict[str, torch.Tensor]: """Compute LIME coefficients using interpretable linear model. @@ -376,7 +358,7 @@ def _compute_lime( pred = self._evaluate_sample( inputs, perturb, - target, + target_indices, ) # Create perturbed sample for each batch item @@ -475,15 +457,19 @@ def _evaluate_sample( self, inputs: dict[str, tuple[torch.Tensor, ...]], perturb: dict[str, torch.Tensor], - target: torch.Tensor, + target_indices: torch.Tensor, ) -> torch.Tensor: """Evaluate model prediction for a perturbed sample. + Returns the model's prediction for the target class, so the + weighted linear regression approximates the model's actual + output (not a distance to a label). + Args: inputs: Original input tuples (used for non-value fields like time/mask). perturb: Perturbed sample tensors. Token features are already embedded; continuous features are still in raw space. - target: Target class tensor. + target_indices: [batch] tensor of target class indices. Returns: Model prediction for the perturbed sample, shape (batch_size, ). @@ -523,8 +509,10 @@ def _evaluate_sample( # model's regular forward pass handle embedding internally. logits = self.model.forward(**inputs)["logit"] - # Reduce to [batch_size, ] by taking absolute difference from target class logit - return (target - logits).abs().mean(dim=tuple(range(1, logits.ndim))) + # Extract the target class prediction (logits, not label distances) + return logits.gather( + 1, target_indices.unsqueeze(1) + ).squeeze(1) def _compute_similarity( self, diff --git a/pyhealth/interpret/methods/shap.py b/pyhealth/interpret/methods/shap.py index 46d40a977..df52ea732 100644 --- a/pyhealth/interpret/methods/shap.py +++ b/pyhealth/interpret/methods/shap.py @@ -220,33 +220,7 @@ def attribute( # Extract and prepare inputs base_logits = self.model.forward(**inputs)["logit"] - # Enforce target class selection for multi-class models to avoid class flipping - if self._prediction_mode() == "binary": - if target_class_idx is not None: - target = torch.tensor([target_class_idx], device=device) - else: - target = (torch.sigmoid(base_logits) > 0.5).long() - elif self._prediction_mode() == "multiclass": - if target_class_idx is not None: - target = torch.nn.functional.one_hot( - torch.tensor(target_class_idx, device=device), - num_classes=base_logits.shape[-1], - ) - else: - target = torch.argmax(base_logits, dim=-1) - target = torch.nn.functional.one_hot( - target, num_classes=base_logits.shape[-1] - ) - elif self._prediction_mode() == "multilabel": - if target_class_idx is not None: - target = torch.nn.functional.one_hot( - torch.tensor(target_class_idx, device=device), - num_classes=base_logits.shape[-1], - ) - else: - target = torch.sigmoid(base_logits) > 0.5 - else: - raise ValueError("Unsupported prediction mode for SHAP attribution.") + target_indices = self._resolve_target_indices(base_logits, target_class_idx) if baseline is None: baselines = self._generate_background_samples( @@ -295,7 +269,7 @@ def attribute( xs=values, bs=baselines, n_features=n_features, - target=target, + target_indices=target_indices, ) return self._map_to_input_shapes(out, shapes) @@ -309,7 +283,7 @@ def _compute_kernel_shap( xs: Dict[str, torch.Tensor], bs: Dict[str, torch.Tensor], n_features: dict[str, int], - target: torch.Tensor, + target_indices: torch.Tensor, ) -> Dict[str, torch.Tensor]: """Compute SHAP values using the Kernel SHAP approximation method. @@ -325,7 +299,7 @@ def _compute_kernel_shap( xs: Dictionary of input values (or embeddings). bs: Dictionary of baseline values (or embeddings). n_features: Dictionary mapping feature keys to feature counts. - target: Target tensor for prediction comparison. + target_indices: [batch] tensor of target class indices. Returns: Dictionary mapping feature keys to SHAP value tensors. @@ -353,7 +327,7 @@ def _compute_kernel_shap( coalition, keys, n_features, batch_size ) perturb = self._create_perturbed_sample(xs, bs, gates) - pred = self._evaluate_sample(inputs, perturb, target) + pred = self._evaluate_sample(inputs, perturb, target_indices) coalition_vectors.append(coalition.float()) coalition_preds.append(pred.detach()) @@ -374,7 +348,7 @@ def _compute_kernel_shap( coalition, keys, n_features, batch_size ) perturb = self._create_perturbed_sample(xs, bs, gates) - pred = self._evaluate_sample(inputs, perturb, target) + pred = self._evaluate_sample(inputs, perturb, target_indices) coalition_vectors.append(coalition.float()) coalition_preds.append(pred.detach()) @@ -480,7 +454,7 @@ def _evaluate_sample( self, inputs: dict[str, tuple[torch.Tensor, ...]], perturb: dict[str, torch.Tensor], - target: torch.Tensor, + target_indices: torch.Tensor, ) -> torch.Tensor: """Evaluate model prediction for a perturbed sample. @@ -492,9 +466,7 @@ def _evaluate_sample( Args: inputs: Original input tuples from the dataloader. perturb: Dictionary of perturbed value tensors. - target: Target tensor used to select which class prediction to - return. For binary this is a 0/1 scalar or (batch,1) tensor; - for multiclass/multilabel it is a one-hot vector. + target_indices: [batch] tensor of target class indices. Returns: Target-class prediction scalar per batch item, shape (batch_size,). @@ -537,56 +509,9 @@ def _evaluate_sample( # model's regular forward pass handle embedding internally. logits = self.model.forward(**inputs)["logit"] - return self._extract_target_prediction(logits, target) - - def _extract_target_prediction( - self, - logits: torch.Tensor, - target: torch.Tensor, - ) -> torch.Tensor: - """Extract the model's prediction for the target class. - - Kernel SHAP decomposes f(x) ≈ φ₀ + Σ φᵢ zᵢ via weighted least squares. - Using **raw logits** (unbounded) rather than probabilities (bounded - [0, 1]) is critical: sigmoid compression squashes coalition differences - in the saturated regions, producing uniformly small SHAP values and - degraded feature rankings. - - Args: - logits: Raw model logits, shape (batch_size, n_classes) or - (batch_size, 1). - target: Target indicator. Binary: scalar/tensor with 0 or 1. - Multiclass: one-hot tensor. Multilabel: multi-hot tensor. - - Returns: - Scalar prediction per batch item, shape (batch_size,). - """ - mode = self._prediction_mode() - - if mode == "binary": - # Use raw logit — not sigmoid probability — to preserve the - # dynamic range that Kernel SHAP's linear decomposition needs. - logit = logits.squeeze(-1) # (batch,) - t = target.float() - if t.dim() > 1: - t = t.squeeze(-1) - # target=1 → logit (higher logit ⇒ more positive class) - # target=0 → −logit (higher value ⇒ more negative class) - return t * logit + (1 - t) * (-logit) - - elif mode == "multiclass": - # target is one-hot; dot-product extracts the target-class logit - return (target.float() * logits).sum(dim=-1) # (batch,) - - elif mode == "multilabel": - # target is multi-hot; average logits over active labels - t = target.float() - n_active = t.sum(dim=-1).clamp(min=1) # avoid div-by-zero - return (t * logits).sum(dim=-1) / n_active # (batch,) - - else: - # regression or unknown — just return the logit - return logits.squeeze(-1) + return logits.gather( + 1, target_indices.unsqueeze(1) + ).squeeze(1) # ------------------------------------------------------------------ # Weighted least squares solver diff --git a/pyhealth/metrics/interpretability/__init__.py b/pyhealth/metrics/interpretability/__init__.py index d0de26057..13bb0831a 100644 --- a/pyhealth/metrics/interpretability/__init__.py +++ b/pyhealth/metrics/interpretability/__init__.py @@ -4,7 +4,12 @@ from .comprehensiveness import ComprehensivenessMetric from .evaluator import Evaluator, evaluate_attribution from .sufficiency import SufficiencyMetric -from .utils import create_validity_mask, get_model_predictions +from .utils import ( + SampleClass, + SampleFilterFn, + get_model_predictions, + threshold_sample_filter, +) __all__ = [ "ComprehensivenessMetric", @@ -12,7 +17,10 @@ "RemovalBasedMetric", "Evaluator", "evaluate_attribution", + # Sample classification + "SampleClass", + "SampleFilterFn", + "threshold_sample_filter", # Utility functions "get_model_predictions", - "create_validity_mask", ] diff --git a/pyhealth/metrics/interpretability/base.py b/pyhealth/metrics/interpretability/base.py index d3dc120f9..ef388402b 100644 --- a/pyhealth/metrics/interpretability/base.py +++ b/pyhealth/metrics/interpretability/base.py @@ -11,7 +11,11 @@ from pyhealth.models import BaseModel -from .utils import create_validity_mask, get_model_predictions +from .utils import ( + SampleClass, + SampleFilterFn, + get_model_predictions, +) class RemovalBasedMetric(ABC): @@ -30,8 +34,15 @@ class RemovalBasedMetric(ABC): - 'mean': Set ablated features to feature mean across batch - 'noise': Add Gaussian noise to ablated features Default: 'zero'. - positive_threshold: Threshold for positive class in binary - classification. Default: 0.5. + sample_filter: A callable that classifies each sample for evaluation. + Signature: (class_probs, classifier_type) -> sample_classes + where class_probs has shape (batch_size,) and contains the + probability for the predicted class (sigmoid/softmax output + with target class already applied), and sample_classes is a + tensor of SampleClass values. + - SampleClass.POSITIVE: evaluate with attributions as-is + - SampleClass.NEGATIVE: evaluate with negated attributions + - SampleClass.IGNORE: exclude from evaluation """ def __init__( @@ -39,12 +50,13 @@ def __init__( model: BaseModel, percentages: List[float] = [1, 5, 10, 20, 50], ablation_strategy: str = "zero", - positive_threshold: float = 0.5, + *, + sample_filter: SampleFilterFn, ): self.model = model self.percentages = percentages self.ablation_strategy = ablation_strategy - self._positive_threshold = positive_threshold + self._sample_filter = sample_filter self.model.eval() # Detect classifier type from model @@ -111,7 +123,7 @@ def _detect_classifier_type(self): self.num_classes = 2 print("[RemovalBasedMetric] Detected BINARY classifier") print(" - Output shape: [batch, 1] with P(class=1)") - print(" - Only evaluates positive predictions (>=threshold)") + print(" - Evaluates both positive and negative predictions") elif mode == "multiclass": self.classifier_type = "multiclass" # Get num_classes from processor @@ -365,42 +377,38 @@ def compute( samples have value 0. Note: - For binary classifiers, the valid_mask indicates samples with - P(class=1) >= threshold (default 0.5). Use this mask to filter - scores during averaging or analysis. + For binary classifiers, all samples are evaluated + (both positive and negative predictions). For class 0 + predictions, attributions are negated internally so that + feature importance is measured relative to the predicted + class. """ # Get original predictions (returns 3 values) - original_probs, pred_classes, original_class_probs = get_model_predictions( + y_probs, target_class_idx, sample_class = get_model_predictions( model=self.model, inputs=inputs, classifier_type=self.classifier_type, - pred_classes=predicted_class, - positive_threshold=self._positive_threshold, + sample_filter=self._sample_filter, ) - - if predicted_class is not None: - pred_classes = predicted_class - - batch_size = original_probs.shape[0] - - # Create validity mask using helper - valid_mask = create_validity_mask( - original_probs, - self.classifier_type, - self._positive_threshold, - ) - - # For binary: determine which samples to evaluate - if self.classifier_type == "binary": - positive_mask = pred_classes == 1 - num_positive = positive_mask.sum().item() - num_negative = (~positive_mask).sum().item() - else: - positive_mask = torch.ones( - batch_size, dtype=torch.bool, device=original_probs.device - ) - num_positive = batch_size - num_negative = 0 + + batch_size = y_probs.shape[0] + + # Validity mask: IGNORE samples excluded + val_mask = sample_class != SampleClass.IGNORE + + # For NEGATIVE samples, negate attributions so that + # "top features" become those most important for the predicted + # class (features with low class-1 attribution support class 0). + neg_mask = sample_class == SampleClass.NEGATIVE + if neg_mask.any(): + attributions = { + key: torch.where( + neg_mask.view(-1, *([1] * (attr.dim() - 1))), + -attr, + attr, + ) + for key, attr in attributions.items() + } # Debug output (if requested and returning per percentage) if debug and return_per_percentage: @@ -411,37 +419,21 @@ def compute( print(f"Classifier type: {self.classifier_type}") if self.classifier_type == "binary": - print(f"Positive class samples: {num_positive}") - print(f"Negative class samples: {num_negative}") - print("NOTE: Only computing metrics for POSITIVE class") - - print(f"Original probs shape: {original_probs.shape}") - print(f"Predicted classes: {pred_classes.tolist()}") - - if self.classifier_type == "binary": - print("\nOriginal probabilities P(class=1):") - for i, prob in enumerate(original_probs): - status = "EVAL" if positive_mask[i] else "SKIP" - print(f" Sample {i} [{status}]: {prob.item():.6f}") - else: - print("\nOriginal probabilities (all classes):") - for i, probs in enumerate(original_probs): - print(f" Sample {i}: {probs.tolist()}") + print(f"Positive class samples: {(sample_class == SampleClass.POSITIVE).sum().item()}") + print(f"Negative class samples: {(sample_class == SampleClass.NEGATIVE).sum().item()}") + print("NOTE: Evaluating BOTH positive and negative predictions") print("\nOriginal probs for predicted class:") - for i, prob in enumerate(original_class_probs): - if self.classifier_type == "binary": - status = "EVAL" if positive_mask[i] else "SKIP" - print(f" Sample {i} [{status}]: {prob.item():.6f}") - else: - print(f" Sample {i}: {prob.item():.6f}") + for i, prob in enumerate(y_probs): + cls = target_class_idx[i].item() + print(f" Sample {i} [class={cls}]: {prob.item():.6f}") # Store results per percentage if return_per_percentage: results = {} else: # Accumulator for averaging - metric_scores = torch.zeros(batch_size, device=original_probs.device) + metric_scores = torch.zeros(batch_size, device=y_probs.device) # Compute metrics across all percentages for percentage in self.percentages: @@ -452,18 +444,24 @@ def compute( ablated_inputs = self._create_ablated_inputs(inputs, masks) # Get predictions on ablated inputs - ablated_probs, _, ablated_class_probs = get_model_predictions( + ablated_probs, _, _ = get_model_predictions( model=self.model, inputs=ablated_inputs, - pred_classes=pred_classes, # Use same predicted classes from original to avoid shifts + target_class_idx=target_class_idx, # Use same predicted classes from original to avoid shifts + sample_class=sample_class, # Use same sample classes to ensure consistency classifier_type=self.classifier_type, - positive_threshold=self._positive_threshold, ) # Compute probability drop - prob_drop = torch.zeros(batch_size, device=original_probs.device) - prob_drop[positive_mask] = ( - original_class_probs[positive_mask] - ablated_class_probs[positive_mask] + original_class_probs = y_probs + original_class_probs[neg_mask] = -original_class_probs[neg_mask] + + ablated_class_probs = ablated_probs + ablated_class_probs[neg_mask] = -ablated_class_probs[neg_mask] + + prob_drop = torch.zeros(batch_size, device=y_probs.device) + prob_drop[val_mask] = ( + original_class_probs[val_mask] - ablated_class_probs[val_mask] ) # Debug output for this percentage @@ -476,8 +474,8 @@ def compute( if self.classifier_type == "binary": print("\nAblated probabilities P(class=1):") for i, prob in enumerate(ablated_probs): - status = "EVAL" if positive_mask[i] else "SKIP" - print(f" Sample {i} [{status}]: {prob.item():.6f}") + cls = target_class_idx[i].item() + print(f" Sample {i} [class={cls}]: {prob.item():.6f}") else: print("\nAblated probabilities (all classes):") for i, probs in enumerate(ablated_probs): @@ -485,18 +483,16 @@ def compute( print("\nProbability drops (original - ablated):") for i, drop in enumerate(prob_drop): - if drop == 0 and not positive_mask[i]: - print(f" Sample {i} [SKIP]: " f"0.000000 (negative class)") - else: - orig = original_class_probs[i].item() - abl = ablated_class_probs[i].item() - print( - f" Sample {i} [EVAL]: {drop.item():.6f} " - f"({orig:.6f} - {abl:.6f})" - ) + orig = original_class_probs[i].item() + abl = ablated_class_probs[i].item() + cls = target_class_idx[i].item() + print( + f" Sample {i} [class={cls}]: {drop.item():.6f} " + f"({orig:.6f} - {abl:.6f})" + ) # Check for unexpected negative values - evaluated_drops = prob_drop[positive_mask] + evaluated_drops = prob_drop[val_mask] neg_mask = evaluated_drops < 0 if neg_mask.any(): neg_count = neg_mask.sum().item() @@ -511,15 +507,15 @@ def compute( print(" - Attribution quality may be poor") if return_per_percentage: - results[percentage] = prob_drop + results[percentage] = prob_drop # type: ignore else: # Accumulate for averaging - metric_scores = metric_scores + prob_drop + metric_scores = metric_scores + prob_drop # type: ignore # Return appropriate format if return_per_percentage: - return results + return results # type: ignore else: # Average across percentages - metric_scores = metric_scores / len(self.percentages) - return metric_scores, valid_mask + metric_scores = metric_scores / len(self.percentages) # type: ignore + return metric_scores, val_mask diff --git a/pyhealth/metrics/interpretability/evaluator.py b/pyhealth/metrics/interpretability/evaluator.py index fe6f99f61..1344358eb 100644 --- a/pyhealth/metrics/interpretability/evaluator.py +++ b/pyhealth/metrics/interpretability/evaluator.py @@ -4,7 +4,8 @@ using removal-based metrics like Comprehensiveness and Sufficiency. """ -from typing import Dict, List +from typing import Dict, List, Optional +import warnings import torch @@ -12,6 +13,7 @@ from .comprehensiveness import ComprehensivenessMetric from .sufficiency import SufficiencyMetric +from .utils import SampleClass, SampleFilterFn, threshold_sample_filter class Evaluator: @@ -30,17 +32,46 @@ class Evaluator: - 'mean': Set ablated features to feature mean across batch - 'noise': Add Gaussian noise to ablated features Default: 'zero'. - positive_threshold: Threshold for positive class in binary - classification. Samples with P(class=1) >= threshold are - considered valid for evaluation. Default: 0.5. + sample_filter: A callable that classifies each sample for evaluation. + Signature: (class_probs, classifier_type) -> sample_classes + where class_probs has shape (batch_size,) and contains the + class probability used for filtering. For binary single-logit + models, this is ``P(class=1)``. For multiclass/multilabel + models, this is the gathered target-class probability. + ``sample_classes`` is a tensor of SampleClass values: + - SampleClass.POSITIVE: evaluate with attributions as-is + - SampleClass.NEGATIVE: evaluate with negated attributions + - SampleClass.IGNORE: exclude from evaluation + If None, uses default_sample_filter. + positive_threshold: .. deprecated:: + This parameter is deprecated and will be removed in a future + release. Use ``sample_filter`` with + :func:`threshold_sample_filter` instead. + Threshold for positive class in binary classification. + Default: None. Examples: >>> from pyhealth.models import StageNet >>> from pyhealth.metrics.interpretability import Evaluator + >>> from pyhealth.metrics.interpretability.utils import ( + ... SampleClass, + ... threshold_sample_filter, + ... ) >>> - >>> # Initialize evaluator + >>> # Initialize evaluator with default filter >>> evaluator = Evaluator(model) >>> + >>> # Initialize with custom filter that ignores low-confidence + >>> def confident_filter(class_probs, classifier_type): + ... batch_size = class_probs.shape[0] + ... result = torch.full( + ... (batch_size,), SampleClass.POSITIVE, + ... dtype=torch.long, device=class_probs.device, + ... ) + ... result[class_probs < 0.6] = SampleClass.IGNORE + ... return result + >>> evaluator = Evaluator(model, sample_filter=confident_filter) + >>> >>> # Evaluate on a single batch >>> inputs = {'conditions': torch.randn(32, 50)} >>> attributions = {'conditions': torch.randn(32, 50)} @@ -61,24 +92,53 @@ def __init__( model: BaseModel, percentages: List[float] = [1, 5, 10, 20, 50], ablation_strategy: str = "zero", - positive_threshold: float = 0.5, + sample_filter: Optional[SampleFilterFn] = None, + positive_threshold: Optional[float] = None, ): self.model = model self.percentages = percentages self.ablation_strategy = ablation_strategy self.positive_threshold = positive_threshold + + # Resolve the effective sample filter: + # 1. explicit sample_filter wins + # 2. positive_threshold → threshold_sample_filter(positive_threshold) + # 3. fallback → default (threshold_sample_filter(0.5)) + if sample_filter is not None: + if positive_threshold is not None: + warnings.warn( + "Both sample_filter and positive_threshold were given. " + "sample_filter takes precedence; positive_threshold is " + "ignored.", + UserWarning, + stacklevel=2, + ) + resolved_filter = sample_filter + elif positive_threshold is not None: + warnings.warn( + "positive_threshold is deprecated and will be removed in a " + "future release. Use sample_filter with " + "threshold_sample_filter() instead.", + DeprecationWarning, + stacklevel=2, + ) + resolved_filter = threshold_sample_filter(positive_threshold) + else: + resolved_filter = threshold_sample_filter(0.5) + + self.sample_filter = resolved_filter self.metrics = { "comprehensiveness": ComprehensivenessMetric( model, percentages=percentages, ablation_strategy=ablation_strategy, - positive_threshold=positive_threshold, + sample_filter=resolved_filter, ), "sufficiency": SufficiencyMetric( model, percentages=percentages, ablation_strategy=ablation_strategy, - positive_threshold=positive_threshold, + sample_filter=resolved_filter, ), } @@ -113,8 +173,9 @@ def evaluate( Example: {'comprehensiveness': {10: tensor(...), 20: ...}} Note: - For binary classifiers, valid_mask indicates samples with - P(class=1) >= threshold. Use: scores[valid_mask].mean() + For binary classifiers, all samples are evaluated + (both positive and negative predictions). + Use: scores[valid_mask].mean() Examples: >>> # Default: averaged scores @@ -166,16 +227,18 @@ def evaluate_attribution( Returns: Dictionary mapping metric names to their average scores - across the entire dataset. For binary classifiers, only - positive class (predicted class=1) samples are included - in the average. + across the entire dataset. Samples marked ``IGNORE`` by the + configured ``sample_filter`` are excluded from the average. Example: {'comprehensiveness': 0.345, 'sufficiency': 0.123} Note: - For binary classifiers, negative class (predicted class=0) - samples are excluded from the average, as ablation metrics - are not meaningful for the default/null class. + For binary classifiers, both positive and negative samples can + be evaluated. Negative samples are handled by negating the + attribution scores before top-feature selection, which makes + the probability drop equivalent to the drop in confidence for + class 0. Use ``sample_filter`` to include or exclude whichever + subsets you want in the dataset average. Examples: >>> from pyhealth.interpret.methods import IntegratedGradients @@ -245,13 +308,15 @@ def evaluate_attribution( ) # Accumulate statistics incrementally (no tensor storage) + first_metric = metrics[0] + batch_size = len(batch_results[first_metric][0]) + total_samples += batch_size + for metric_name in metrics: scores, valid_mask = batch_results[metric_name] # Track statistics efficiently - batch_size = len(scores) num_valid = valid_mask.sum().item() - total_samples += batch_size total_valid[metric_name] += num_valid # Update running sum (valid scores only) @@ -325,20 +390,19 @@ def evaluate_attribution( print(" * Important features not correctly identified") print(" * Consider checking attribution method") - valid_ratio = sum(total_valid.values()) / (len(metrics) * total_samples) - if valid_ratio < 0.1: + valid_ratio = sum(total_valid.values()) / (len(metrics) * total_samples) if total_samples > 0 else 0 + if valid_ratio < 0.1 and total_samples > 0: print(f"\n⚠ WARNING: Only {valid_ratio*100:.1f}% valid samples") print(" - Most predictions are negative class") print(" - Consider:") print(" * Checking model predictions distribution") - print(" * Adjusting positive_threshold parameter") + print(" * Adjusting sample_filter to include more samples") print(" * Using balanced test set") print(f"{'='*70}\n") return results - # Functional API (wraps Evaluator for convenience) def evaluate_attribution( model: BaseModel, @@ -347,7 +411,8 @@ def evaluate_attribution( metrics: List[str] = ["comprehensiveness", "sufficiency"], percentages: List[float] = [1, 5, 10, 20, 50], ablation_strategy: str = "zero", - positive_threshold: float = 0.5, + sample_filter: Optional[SampleFilterFn] = None, + positive_threshold: Optional[float] = None, ) -> Dict[str, float]: """Evaluate an attribution method across a dataset (functional API). @@ -371,27 +436,37 @@ def evaluate_attribution( - 'mean': Set ablated features to feature mean across batch - 'noise': Add Gaussian noise to ablated features Default: 'zero'. - positive_threshold: Threshold for positive class in binary - classification. Samples with P(class=1) >= threshold are - considered valid for evaluation. Default: 0.5. + sample_filter: A callable that classifies each sample for + evaluation. Signature: + (class_probs, classifier_type) -> sample_classes + where class_probs has shape (batch_size,) and contains the + probability for the predicted class (sigmoid/softmax output + with target class already applied), and sample_classes is a + tensor of SampleClass values: + - SampleClass.POSITIVE: evaluate with attributions as-is + - SampleClass.NEGATIVE: evaluate with negated attributions + - SampleClass.IGNORE: exclude from evaluation + If None, uses default_sample_filter. + positive_threshold: .. deprecated:: + This parameter is deprecated and will be removed in a future + release. Use ``sample_filter`` with + :func:`threshold_sample_filter` instead. + Threshold for positive class in binary classification. + Default: None. Returns: Dictionary mapping metric names to their average scores across the entire dataset. Averaging uses mask-based filtering - to include only valid samples (positive predictions for binary). + to exclude IGNORE samples. Example: {'comprehensiveness': 0.345, 'sufficiency': 0.123} - Note: - For binary classifiers, only samples with P(class=1) >= threshold - are included in the average, as ablation metrics are not - meaningful for negative predictions. - Examples: >>> from pyhealth.interpret.methods import IntegratedGradients >>> from pyhealth.metrics.interpretability import ( ... evaluate_attribution ... ) + >>> from pyhealth.metrics.interpretability.utils import SampleClass >>> >>> # Simple one-off evaluation >>> ig = IntegratedGradients(model, use_embeddings=True) @@ -402,10 +477,18 @@ def evaluate_attribution( ... ) >>> print(f"Comprehensiveness: {results['comprehensiveness']:.4f}") >>> - >>> # Custom threshold for binary classification + >>> # Custom filter to ignore uncertain predictions + >>> def ignore_uncertain(class_probs, classifier_type): + ... batch_size = class_probs.shape[0] + ... result = torch.full( + ... (batch_size,), SampleClass.POSITIVE, + ... dtype=torch.long, device=class_probs.device, + ... ) + ... result[class_probs < 0.7] = SampleClass.IGNORE + ... return result >>> results = evaluate_attribution( ... model, test_loader, ig, - ... positive_threshold=0.7 # Only evaluate high-confidence + ... sample_filter=ignore_uncertain, ... ) >>> >>> # For comparing multiple methods efficiently, use Evaluator: @@ -420,6 +503,7 @@ def evaluate_attribution( model, percentages=percentages, ablation_strategy=ablation_strategy, + sample_filter=sample_filter, positive_threshold=positive_threshold, ) return evaluator.evaluate_attribution(dataloader, method, metrics=metrics) diff --git a/pyhealth/metrics/interpretability/utils.py b/pyhealth/metrics/interpretability/utils.py index 5206bbf1d..3f10814cb 100644 --- a/pyhealth/metrics/interpretability/utils.py +++ b/pyhealth/metrics/interpretability/utils.py @@ -4,7 +4,8 @@ metrics to avoid code duplication and improve maintainability. """ -from typing import Dict, Optional, Tuple +from enum import IntEnum +from typing import Callable, Dict, Optional, Tuple import torch import torch.nn.functional as F @@ -12,12 +13,83 @@ from pyhealth.models import BaseModel +class SampleClass(IntEnum): + """Classification of how a sample should be treated during evaluation. + + Attributes: + POSITIVE: Evaluate sample with attributions as-is. + Used for predicted positive class in binary, or all + samples in multiclass/multilabel. + NEGATIVE: Evaluate sample with negated attributions. + Used for predicted negative class in binary classification, + where feature importance is measured relative to the + predicted class (class 0). + IGNORE: Exclude sample from evaluation entirely. + Useful for filtering out low-confidence predictions or + samples that should not contribute to the metric. + """ + + POSITIVE = 1 + NEGATIVE = -1 + IGNORE = 0 + + +# Type alias for sample filter functions. +# Signature: (y_probs, classifier_type) -> sample_classes +# y_probs has shape (batch_size,). For binary single-logit models this is +# P(class=1); for multiclass/multilabel models this is the gathered +# target-class probability. +SampleFilterFn = Callable[[torch.Tensor, str], torch.Tensor] + + +def threshold_sample_filter(threshold: float = 0.5) -> SampleFilterFn: + """Create a filter based on a probability threshold. + + For binary and multilabel classifiers, samples whose predicted-class + probability is at or above ``threshold`` are marked POSITIVE; all + others are marked IGNORE. + + For multiclass classifiers, all samples are marked POSITIVE + (the argmax class always has a well-defined probability). + + Args: + threshold: Minimum predicted-class probability to include + the sample. Default: 0.5. + + Returns: + A sample filter function. + + Examples: + >>> # Create a filter that ignores uncertain predictions + >>> my_filter = threshold_sample_filter(0.7) + >>> evaluator = Evaluator(model, sample_filter=my_filter) + """ + + def filter_fn( + y_probs: torch.Tensor, + classifier_type: str, + ) -> torch.Tensor: + batch_size = y_probs.shape[0] + result = torch.full( + (batch_size,), + SampleClass.POSITIVE, + dtype=torch.long, + device=y_probs.device, + ) + if classifier_type in ("binary", "multilabel"): + result[y_probs < threshold] = SampleClass.IGNORE + return result + + return filter_fn + + def get_model_predictions( model: BaseModel, inputs: Dict[str, torch.Tensor], classifier_type: str, - pred_classes: Optional[torch.Tensor] = None, - positive_threshold: float = 0.5, + sample_filter: Optional[SampleFilterFn] = None, + sample_class: Optional[torch.Tensor] = None, + target_class_idx: Optional[torch.Tensor] = None, ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """Get model predictions, probabilities, and class-specific probabilities. @@ -25,18 +97,22 @@ def get_model_predictions( model: PyHealth BaseModel that returns dict with 'y_prob' or 'logit' inputs: Model inputs dict classifier_type: One of 'binary', 'multiclass', 'multilabel', 'unknown' - pred_classes: (Optional) Pre-computed predicted classes, this would ensure ablated runs + target_class_idx: (Optional) Pre-computed target class indices, this would ensure ablated runs are consistent with original predictions. If None, will compute from model outputs. - positive_threshold: Threshold for binary classification (default: 0.5) + sample_filter: A callable that classifies each sample for evaluation. + Signature: (class_probs, classifier_type) -> sample_classes + where class_probs has shape (batch_size,). For binary + single-logit models this is ``P(class=1)``; otherwise it is + the gathered target-class probability. ``sample_classes`` is + a tensor of SampleClass values. Returns: - Tuple of (y_prob, pred_classes, class_probs): + Tuple of (y_prob, target_class_idx, sample_classes): - y_prob: All class probabilities - Binary: shape (batch_size, 1), values are P(class=1) - Multiclass: shape (batch_size, num_classes) - - pred_classes: Predicted class indices, shape (batch_size,) - - class_probs: Probability for each sample's predicted class, - shape (batch_size,) + - target_class_idx: Target class indices, shape (batch_size,) + - sample_classes: SampleClass values for each sample, shape (batch_size,) """ with torch.no_grad(): outputs = model(**inputs) @@ -46,7 +122,7 @@ def get_model_predictions( y_prob = outputs["y_prob"] elif "logit" in outputs: logits = outputs["logit"] - if classifier_type == "binary": + if classifier_type in ["binary", "multilabel"]: y_prob = torch.sigmoid(logits) else: y_prob = F.softmax(logits, dim=-1) @@ -57,45 +133,22 @@ def get_model_predictions( if y_prob.dim() == 1: y_prob = y_prob.unsqueeze(-1) - # Get predicted classes based on classifier type - if classifier_type == "binary": - # For binary: class 1 if P(class=1) >= threshold, else 0 - pred_classes = (y_prob.squeeze(-1) >= positive_threshold).long() if pred_classes is None else pred_classes - # For binary, class_probs is P(class=1) - class_probs = y_prob.squeeze(-1) - else: - # For multiclass/multilabel: argmax - pred_classes = torch.argmax(y_prob, dim=-1) if pred_classes is None else pred_classes - # Gather probabilities for predicted classes - class_probs = y_prob.gather(1, pred_classes.unsqueeze(1)).squeeze(1) - assert pred_classes is not None, "pred_classes should have been set either by input or computation." - - return y_prob, pred_classes, class_probs - + if target_class_idx is None: + target_class_idx = torch.argmax(y_prob, dim=-1) + + y_prob = y_prob.gather( + dim=-1, + index=target_class_idx.unsqueeze(-1), + ).squeeze(-1) + + # Apply sample filter + if sample_class is None: + if sample_filter is None: + raise ValueError("sample_filter must be provided if sample_class is None") + sample_class = sample_filter(y_prob, classifier_type) + + y_prob[sample_class == SampleClass.IGNORE] = 0.0 # Set ignored samples' probs to 0 + target_class_idx[sample_class == SampleClass.IGNORE] = 0 # Mark ignored samples with invalid class index + + return y_prob, target_class_idx, sample_class -def create_validity_mask( - y_prob: torch.Tensor, - classifier_type: str, - positive_threshold: float = 0.5, -) -> torch.Tensor: - """Create a mask indicating which samples are valid for metric computation. - - For binary classifiers, only positive predictions (P(class=1) >= threshold) - are considered valid. For multiclass/multilabel, all samples are valid. - - Args: - y_prob: Model probability outputs - classifier_type: One of 'binary', 'multiclass', 'multilabel' - positive_threshold: Threshold for binary classification (default: 0.5) - - Returns: - Boolean tensor of shape (batch_size,) where True indicates valid samples - """ - batch_size = y_prob.shape[0] - - if classifier_type == "binary": - # For binary: valid = P(class=1) >= threshold - return y_prob.squeeze(-1) >= positive_threshold - else: - # For multiclass/multilabel: all samples are valid - return torch.ones(batch_size, dtype=torch.bool, device=y_prob.device) diff --git a/tests/core/test_deeplift.py b/tests/core/test_deeplift.py index 8fe832d82..29dbf8328 100644 --- a/tests/core/test_deeplift.py +++ b/tests/core/test_deeplift.py @@ -88,15 +88,15 @@ def test_basic_attribution(self): self.assertIsInstance(attributions["procedures"], torch.Tensor) def test_attribution_with_target_class(self): - """Test attribution computation with specific target class.""" + """For binary (single logit), target_class_idx is a no-op.""" dl = DeepLift(self.model) data_batch = next(iter(self.test_loader)) attr_class_0 = dl.attribute(**data_batch, target_class_idx=0) attr_class_1 = dl.attribute(**data_batch, target_class_idx=1) - # Attributions should differ for different classes - self.assertFalse( + # Single-logit binary: target_class_idx is a no-op, both should match + self.assertTrue( torch.allclose(attr_class_0["conditions"], attr_class_1["conditions"]) ) diff --git a/tests/core/test_gim.py b/tests/core/test_gim.py index 284931883..4e7cfb753 100644 --- a/tests/core/test_gim.py +++ b/tests/core/test_gim.py @@ -282,8 +282,8 @@ def _manual_token_attribution( output = model.forward_from_embedding(codes=tuple(parts), label=labels) logits = output["logit"] - # Binary mode: target class 0 → sign = -1 (2*0 - 1) - target = (-1.0 * logits).sum() + # Binary (single logit): _resolve_target_indices always selects index 0. + target = logits.sum() model.zero_grad(set_to_none=True) if embeddings.grad is not None: diff --git a/tests/core/test_ig_gim.py b/tests/core/test_ig_gim.py index 38565227d..fed1b436f 100644 --- a/tests/core/test_ig_gim.py +++ b/tests/core/test_ig_gim.py @@ -567,7 +567,7 @@ def test_auto_target_class(self): self.assertEqual(attrs["codes"].shape, self.tokens.shape) def test_different_target_classes(self): - """Attributions for different target classes should differ.""" + """For binary (single logit), target_class_idx is a no-op.""" model = _ToyModel() ig_gim = IntegratedGradientGIM(model, temperature=1.0, steps=10) @@ -578,9 +578,10 @@ def test_different_target_classes(self): codes=self.tokens, label=self.labels, target_class_idx=1, )["codes"] - self.assertFalse( + # Single-logit binary: target_class_idx is a no-op, both should match + self.assertTrue( torch.allclose(attrs_0, attrs_1), - "Different target classes should give different attributions", + "Single-logit binary: target_class_idx is a no-op", ) # ----- Temporal tuple inputs ----- diff --git a/tests/core/test_integrated_gradients.py b/tests/core/test_integrated_gradients.py index 7c9212280..78ec4fab5 100644 --- a/tests/core/test_integrated_gradients.py +++ b/tests/core/test_integrated_gradients.py @@ -93,7 +93,7 @@ def test_basic_attribution(self): self.assertIsInstance(attributions["procedures"], torch.Tensor) def test_attribution_with_target_class(self): - """Test attribution computation with specific target class.""" + """For binary (single logit), target_class_idx is a no-op.""" ig = IntegratedGradients(self.model) data_batch = next(iter(self.test_loader)) @@ -103,8 +103,8 @@ def test_attribution_with_target_class(self): # Compute attributions for class 1 attr_class_1 = ig.attribute(**data_batch, target_class_idx=1, steps=10) - # Check that attributions are different for different classes - self.assertFalse( + # Single-logit binary: target_class_idx is a no-op, both should match + self.assertTrue( torch.allclose(attr_class_0["conditions"], attr_class_1["conditions"]) ) @@ -307,7 +307,7 @@ def test_attribution_shapes_stagenet(self): self.assertEqual(attributions[key].shape, value_tensor.shape) def test_attribution_with_target_class_stagenet(self): - """Test attribution with specific target class for StageNet.""" + """For binary (single logit), target_class_idx is a no-op.""" ig = IntegratedGradients(self.model) data_batch = next(iter(self.test_loader)) @@ -315,8 +315,8 @@ def test_attribution_with_target_class_stagenet(self): attr_0 = ig.attribute(**data_batch, target_class_idx=0, steps=10) attr_1 = ig.attribute(**data_batch, target_class_idx=1, steps=10) - # Check that attributions differ for different classes - self.assertFalse(torch.allclose(attr_0["codes"], attr_1["codes"])) + # Single-logit binary: target_class_idx is a no-op, both should match + self.assertTrue(torch.allclose(attr_0["codes"], attr_1["codes"])) def test_attribution_values_finite_stagenet(self): """Test that StageNet attributions are finite.""" diff --git a/tests/core/test_interp_metrics.py b/tests/core/test_interp_metrics.py index 9c46cf4ee..c415f1a9c 100644 --- a/tests/core/test_interp_metrics.py +++ b/tests/core/test_interp_metrics.py @@ -17,6 +17,7 @@ ComprehensivenessMetric, Evaluator, SufficiencyMetric, + threshold_sample_filter, ) from pyhealth.models import StageNet @@ -122,6 +123,7 @@ def setUp(self): # Initialize Integrated Gradients for attribution computation self.ig = IntegratedGradients(self.model, use_embeddings=True) + self.sample_filter = threshold_sample_filter() # Helper method to create attributions for a batch using IG def _create_attributions(self, batch, target_class_idx=1): @@ -227,7 +229,10 @@ def test_comprehensiveness_metric_basic(self): # Initialize metric comp = ComprehensivenessMetric( - self.model, percentages=[10, 20, 50], ablation_strategy="zero" + self.model, + percentages=[10, 20, 50], + ablation_strategy="zero", + sample_filter=self.sample_filter, ) # Compute scores - now returns (scores, valid_mask) tuple @@ -257,7 +262,10 @@ def test_sufficiency_metric_basic(self): # Initialize metric suff = SufficiencyMetric( - self.model, percentages=[10, 20, 50], ablation_strategy="zero" + self.model, + percentages=[10, 20, 50], + ablation_strategy="zero", + sample_filter=self.sample_filter, ) # Compute scores - now returns (scores, valid_mask) tuple @@ -282,7 +290,11 @@ def test_detailed_scores(self): """Test that detailed scores return per-percentage results.""" attributions = self._create_attributions(self.batch) - comp = ComprehensivenessMetric(self.model, percentages=[10, 20, 50]) + comp = ComprehensivenessMetric( + self.model, + percentages=[10, 20, 50], + sample_filter=self.sample_filter, + ) # Get detailed scores using return_per_percentage=True detailed = comp.compute(self.batch, attributions, return_per_percentage=True) @@ -305,7 +317,10 @@ def test_ablation_strategies(self): for strategy in strategies: comp = ComprehensivenessMetric( - self.model, percentages=[10, 20], ablation_strategy=strategy + self.model, + percentages=[10, 20], + ablation_strategy=strategy, + sample_filter=self.sample_filter, ) # Compute returns (scores, valid_mask) tuple scores, valid_mask = comp.compute(self.batch, attributions) @@ -414,7 +429,10 @@ def test_percentage_sensitivity(self): attributions = self._create_attributions(self.batch) comp = ComprehensivenessMetric( - self.model, percentages=[1, 10, 50], ablation_strategy="zero" + self.model, + percentages=[1, 10, 50], + ablation_strategy="zero", + sample_filter=self.sample_filter, ) detailed = comp.compute(self.batch, attributions, return_per_percentage=True) diff --git a/tests/core/test_lime.py b/tests/core/test_lime.py index ab061cb4a..55dcf0ff0 100644 --- a/tests/core/test_lime.py +++ b/tests/core/test_lime.py @@ -308,7 +308,7 @@ def test_target_class_idx_none(self): self.assertEqual(attributions["x"].shape, inputs.shape) def test_target_class_idx_specified(self): - """Should handle specific target class index.""" + """For binary (single logit), target_class_idx is a no-op.""" inputs = torch.tensor([[1.0, 0.5, -0.3]]) attr_class_0 = self.explainer.attribute( @@ -323,8 +323,8 @@ def test_target_class_idx_specified(self): target_class_idx=1, ) - # Attributions should differ for different classes - self.assertFalse(torch.allclose(attr_class_0["x"], attr_class_1["x"], atol=0.01)) + # Single-logit binary: target_class_idx is a no-op, both should match + self.assertTrue(torch.allclose(attr_class_0["x"], attr_class_1["x"], atol=0.01)) def test_attribution_values_are_finite(self): """Test that attribution values are finite (no NaN or Inf).""" @@ -777,7 +777,7 @@ def test_lime_mlp_basic_attribution(self): self.assertIsInstance(attributions["procedures"], torch.Tensor) def test_lime_mlp_with_target_class(self): - """Test LIME attribution with specific target class.""" + """For binary (single logit), target_class_idx is a no-op.""" explainer = LimeExplainer( self.model, use_embeddings=True, @@ -792,8 +792,8 @@ def test_lime_mlp_with_target_class(self): # Compute attributions for class 1 attr_class_1 = explainer.attribute(**data_batch, target_class_idx=1) - # Check that attributions are different for different classes - self.assertFalse( + # Single-logit binary: target_class_idx is a no-op, both should match + self.assertTrue( torch.allclose(attr_class_0["conditions"], attr_class_1["conditions"], atol=0.01) ) diff --git a/tests/core/test_shap.py b/tests/core/test_shap.py index 8c03a1c1f..25d703a0d 100644 --- a/tests/core/test_shap.py +++ b/tests/core/test_shap.py @@ -318,7 +318,7 @@ def test_target_class_idx_none(self): self.assertEqual(attributions["x"].shape, inputs.shape) def test_target_class_idx_specified(self): - """Should handle specific target class index.""" + """For binary (single logit), target_class_idx is a no-op.""" inputs = torch.tensor([[1.0, 0.5, -0.3]]) attr_class_0 = self.explainer.attribute( @@ -333,8 +333,8 @@ def test_target_class_idx_specified(self): target_class_idx=1, ) - # Attributions should differ for different classes - self.assertFalse(torch.allclose(attr_class_0["x"], attr_class_1["x"], atol=0.01)) + # Single-logit binary: target_class_idx is a no-op, both should match + self.assertTrue(torch.allclose(attr_class_0["x"], attr_class_1["x"], atol=0.01)) def test_attribution_values_are_finite(self): """Test that attribution values are finite (no NaN or Inf).""" @@ -696,7 +696,7 @@ def test_shap_mlp_basic_attribution(self): self.assertIsInstance(attributions["procedures"], torch.Tensor) def test_shap_mlp_with_target_class(self): - """Test SHAP attribution with specific target class.""" + """For binary (single logit), target_class_idx is a no-op.""" explainer = ShapExplainer(self.model) data_batch = next(iter(self.test_loader)) @@ -706,8 +706,8 @@ def test_shap_mlp_with_target_class(self): # Compute attributions for class 1 attr_class_1 = explainer.attribute(**data_batch, target_class_idx=1) - # Check that attributions are different for different classes - self.assertFalse( + # Single-logit binary: target_class_idx is a no-op, both should match + self.assertTrue( torch.allclose(attr_class_0["conditions"], attr_class_1["conditions"], atol=0.01) ) @@ -1023,23 +1023,13 @@ def test_kernel_weight_computation_edge_cases(self): self.assertTrue(torch.isfinite(weight_partial)) def test_target_prediction_extraction_binary(self): - """Test target prediction extraction for binary classification.""" - explainer = ShapExplainer( - self.model, - use_embeddings=False, - ) + """Test target prediction for binary classification via gather.""" # Single logit (binary classification) logits_binary = torch.tensor([[0.5], [1.0], [-0.3]]) - - # Class 1 target tensor - target_1 = torch.tensor([1, 1, 1]) - pred_1 = explainer._extract_target_prediction(logits_binary, target_1) - self.assertEqual(pred_1.shape, (3,)) - - # Class 0 target tensor - target_0 = torch.tensor([0, 0, 0]) - pred_0 = explainer._extract_target_prediction(logits_binary, target_0) - self.assertEqual(pred_0.shape, (3,)) + target_indices = torch.zeros(3, dtype=torch.long) + pred = logits_binary.gather(1, target_indices.unsqueeze(1)).squeeze(1) + self.assertEqual(pred.shape, (3,)) + torch.testing.assert_close(pred, torch.tensor([0.5, 1.0, -0.3])) def test_shape_mapping_simple(self): """Test mapping SHAP values back to input shapes.""" diff --git a/tests/core/test_transformer.py b/tests/core/test_transformer.py index a5fa6cc6b..e74468fc5 100644 --- a/tests/core/test_transformer.py +++ b/tests/core/test_transformer.py @@ -154,8 +154,7 @@ def test_chefer_relevance(self): relevance = CheferRelevance(model) # Test with explicitly specified class index - data_batch["class_index"] = 0 - scores = relevance.get_relevance_matrix(**data_batch) + scores = relevance.get_relevance_matrix(target_class_idx=0, **data_batch) # Verify that scores are returned for all feature keys self.assertIsInstance(scores, dict) @@ -167,9 +166,8 @@ def test_chefer_relevance(self): # Verify scores are non-negative (due to clamping in relevance computation) self.assertTrue(torch.all(scores[feature_key] >= 0)) - # Test without specifying class_index (should use predicted class) - data_batch_no_idx = {k: v for k, v in data_batch.items() if k != "class_index"} - scores_auto = relevance.get_relevance_matrix(**data_batch_no_idx) + # Test without specifying target_class_idx (should use predicted class) + scores_auto = relevance.get_relevance_matrix(**data_batch) # Verify that scores are returned self.assertIsInstance(scores_auto, dict) From f7dd84858f34283440f8beb799805b97fcb6fcae Mon Sep 17 00:00:00 2001 From: John Wu <54558896+jhnwu3@users.noreply.github.com> Date: Mon, 13 Apr 2026 16:17:07 -0500 Subject: [PATCH 02/61] small fix + bump to pyproject.toml ver. for bug fixed release on pypi (#927) * small fix + bump to pyproject.toml ver. for bug fixed release on pypi * We don't really have someone qualified for a second review, and this broken CI is leading to a lot of issues here. Will revert if it doesn't resolve here. --- pixi.lock | 70 ++++++++++++++++++- .../predictionset/base_conformal/__init__.py | 63 ++++++++++------- .../predictionset/cluster/cluster_label.py | 21 +++--- pyhealth/calib/predictionset/label.py | 21 +++--- pyproject.toml | 4 +- 5 files changed, 128 insertions(+), 51 deletions(-) diff --git a/pixi.lock b/pixi.lock index 0f11d28d7..42762b62b 100644 --- a/pixi.lock +++ b/pixi.lock @@ -2224,6 +2224,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8c095d6_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_hd72426e_102.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda + - pypi: https://files.pythonhosted.org/packages/18/a6/907a406bb7d359e6a63f99c313846d9eec4f7e6f7437809e03aa00fa3074/absl_py-2.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/30/dd/0107f0aa179869ee9f47ef5a2686abd5e022fdc82af901d535e52fe91ce1/accelerate-1.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/f9/25753b9de3029d3eb2487755520b98eb72b0cb562d8974329c6e19831063/axial_positional_embedding-0.3.12-py3-none-any.whl @@ -2240,6 +2241,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/1d/54/a46920229d12c3a6e9f0081d1bdaeffad23c1826353ace95714faee926e5/dask-2025.11.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/46/ec/da78855318971c2be94d0283a41de6941a6b9f16146fb00babc74903ae01/distributed-2025.11.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d5/18/9f4f975ca87a390832b1c22478f3702fcdf739f83211e24d054b7551270d/editdistance-0.8.1.tar.gz - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4d/36/2a115987e2d8c300a974597416d9de88f2444426de9571f4b59b2cca3acc/filelock-3.18.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/75/b4/b96bb66f6f8cc4669de44a158099b249c8159231d254ab6b092909388be5/fonttools-4.59.0-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl @@ -2269,6 +2271,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/5d/ba/459f18c16f2b3fc1a1ca871f72f07d70c07bf768ad0a507a698b8052ac58/msgpack-1.1.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/87/0d/1861d1599571974b15b025e12b142d8e6b42ad66c8a07a89cb0fc21f1e03/narwhals-2.13.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/8d/776adee7bbf76365fdd7f2552710282c79a4ead5d2a46408c9043a2b70ba/networkx-3.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/9d/91/04e965f8e717ba0ab4bdca5c112deeab11c9e750d94c4d4602f050295d39/nltk-3.9.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/19/49/4df9123aafa7b539317bf6d342cb6d227e49f7a35b99c287a6109b13dd93/numpy-2.2.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/af/eb/ff4b8c503fa1f1796679dce648854d58751982426e4e4b37d6fce49d259c/nvidia_cublas_cu12-12.6.4.1-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/49/60/7b6497946d74bcf1de852a21824d63baad12cd417db4195fc1bfe59db953/nvidia_cuda_cupti_cu12-12.6.80-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl @@ -2308,6 +2311,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/34/43/3f250ec28edff1c06ffaa25faddbe13ae85c11a9724894cbdcf89427de78/rdkit-2025.3.3-cp313-cp313-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/db/60/1eeca2074f5b87df394fccaa432ae3fc06c9c9bfa97c5051aed70e6e00c2/regex-2024.11.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/7c/e4/56027c4a6b4ae70ca9de302488c5ca95ad4a39e190093d6c1a8ace08341b/requests-2.32.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e2/c5/9136736c37022a6ad27fea38f3111eb8f02fe75d067f9a985cc358653102/rouge_score-0.1.2.tar.gz - pypi: https://files.pythonhosted.org/packages/fc/51/727abb13f44c1fcf6d145979e1535a35794db0f6e450a0cb46aa24732fe2/s3transfer-0.16.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a6/f8/dae3421624fcc87a89d42e1898a798bc7ff72c61f38973a65d60df8f124c/safetensors-0.5.3-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/99/72/c86a4cd867816350fe8dee13f30222340b9cd6b96173955819a5561810c5/scikit_learn-1.7.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl @@ -2360,6 +2364,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/readline-8.2-h8382b9d_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/tk-8.6.13-noxft_h5688188_102.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda + - pypi: https://files.pythonhosted.org/packages/18/a6/907a406bb7d359e6a63f99c313846d9eec4f7e6f7437809e03aa00fa3074/absl_py-2.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/30/dd/0107f0aa179869ee9f47ef5a2686abd5e022fdc82af901d535e52fe91ce1/accelerate-1.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/f9/25753b9de3029d3eb2487755520b98eb72b0cb562d8974329c6e19831063/axial_positional_embedding-0.3.12-py3-none-any.whl @@ -2376,6 +2381,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/1d/54/a46920229d12c3a6e9f0081d1bdaeffad23c1826353ace95714faee926e5/dask-2025.11.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/46/ec/da78855318971c2be94d0283a41de6941a6b9f16146fb00babc74903ae01/distributed-2025.11.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d5/18/9f4f975ca87a390832b1c22478f3702fcdf739f83211e24d054b7551270d/editdistance-0.8.1.tar.gz - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4d/36/2a115987e2d8c300a974597416d9de88f2444426de9571f4b59b2cca3acc/filelock-3.18.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b5/57/7969af50b26408be12baa317c6147588db5b38af2759e6df94554dbc5fdb/fonttools-4.59.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl @@ -2405,6 +2411,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/d3/68/93180dce57f684a61a88a45ed13047558ded2be46f03acb8dec6d7c513af/msgpack-1.1.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl - pypi: https://files.pythonhosted.org/packages/87/0d/1861d1599571974b15b025e12b142d8e6b42ad66c8a07a89cb0fc21f1e03/narwhals-2.13.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/8d/776adee7bbf76365fdd7f2552710282c79a4ead5d2a46408c9043a2b70ba/networkx-3.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/9d/91/04e965f8e717ba0ab4bdca5c112deeab11c9e750d94c4d4602f050295d39/nltk-3.9.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/85/c5/e19c8f99d83fd377ec8c7e0cf627a8049746da54afc24ef0a0cb73d5dfb5/numpy-2.2.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl - pypi: https://files.pythonhosted.org/packages/77/20/77907765e29b2eba6bd8821872284d91170d7084f670855b2dfcb249ea14/obstore-0.8.2-cp313-cp313-manylinux_2_24_aarch64.whl - pypi: https://files.pythonhosted.org/packages/7e/95/e0770cf1ad9667492f56b732f44398ef2756d61df914e10d121a3cad013a/ogb-1.3.6-py3-none-any.whl @@ -2430,6 +2437,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/ff/5f/907a48c5f9b83302b4530605df1325963977fdf06753d3d8610d16c40197/rdkit-2025.3.3-cp313-cp313-manylinux_2_28_aarch64.whl - pypi: https://files.pythonhosted.org/packages/fc/fd/37868b75eaf63843165f1d2122ca6cb94bfc0271e4428cf58c0616786dce/regex-2024.11.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl - pypi: https://files.pythonhosted.org/packages/7c/e4/56027c4a6b4ae70ca9de302488c5ca95ad4a39e190093d6c1a8ace08341b/requests-2.32.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e2/c5/9136736c37022a6ad27fea38f3111eb8f02fe75d067f9a985cc358653102/rouge_score-0.1.2.tar.gz - pypi: https://files.pythonhosted.org/packages/fc/51/727abb13f44c1fcf6d145979e1535a35794db0f6e450a0cb46aa24732fe2/s3transfer-0.16.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5d/9a/add3e6fef267658075c5a41573c26d42d80c935cdc992384dfae435feaef/safetensors-0.5.3-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl - pypi: https://files.pythonhosted.org/packages/e8/66/277967b29bd297538dc7a6ecfb1a7dce751beabd0d7f7a2233be7a4f7832/scikit_learn-1.7.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl @@ -2472,6 +2480,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.2-h1d1bf99_2.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h892fb3f_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda + - pypi: https://files.pythonhosted.org/packages/18/a6/907a406bb7d359e6a63f99c313846d9eec4f7e6f7437809e03aa00fa3074/absl_py-2.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/30/dd/0107f0aa179869ee9f47ef5a2686abd5e022fdc82af901d535e52fe91ce1/accelerate-1.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/f9/25753b9de3029d3eb2487755520b98eb72b0cb562d8974329c6e19831063/axial_positional_embedding-0.3.12-py3-none-any.whl @@ -2488,6 +2497,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/1d/54/a46920229d12c3a6e9f0081d1bdaeffad23c1826353ace95714faee926e5/dask-2025.11.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/46/ec/da78855318971c2be94d0283a41de6941a6b9f16146fb00babc74903ae01/distributed-2025.11.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d5/18/9f4f975ca87a390832b1c22478f3702fcdf739f83211e24d054b7551270d/editdistance-0.8.1.tar.gz - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4d/36/2a115987e2d8c300a974597416d9de88f2444426de9571f4b59b2cca3acc/filelock-3.18.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f3/bb/390990e7c457d377b00890d9f96a3ca13ae2517efafb6609c1756e213ba4/fonttools-4.59.0-cp313-cp313-macosx_10_13_universal2.whl @@ -2517,6 +2527,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/92/dc/c385f38f2c2433333345a82926c6bfa5ecfff3ef787201614317b58dd8be/msgpack-1.1.2-cp313-cp313-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/87/0d/1861d1599571974b15b025e12b142d8e6b42ad66c8a07a89cb0fc21f1e03/narwhals-2.13.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/8d/776adee7bbf76365fdd7f2552710282c79a4ead5d2a46408c9043a2b70ba/networkx-3.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/9d/91/04e965f8e717ba0ab4bdca5c112deeab11c9e750d94c4d4602f050295d39/nltk-3.9.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/9e/14520dc3dadf3c803473bd07e9b2bd1b69bc583cb2497b47000fed2fa92f/numpy-2.2.6-cp313-cp313-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/ea/4d/699359774ce6330130536d008bfc32827fab0c25a00238d015a5974a3d1d/obstore-0.8.2-cp313-cp313-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/7e/95/e0770cf1ad9667492f56b732f44398ef2756d61df914e10d121a3cad013a/ogb-1.3.6-py3-none-any.whl @@ -2542,6 +2553,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/3b/0b/6ab0cc692b2890f4f7c74f6ffd4bba748dcb9312d5a7bd2328cb82204da1/rdkit-2025.3.3-cp313-cp313-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/09/c9/4e68181a4a652fb3ef5099e077faf4fd2a694ea6e0f806a7737aff9e758a/regex-2024.11.6-cp313-cp313-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/7c/e4/56027c4a6b4ae70ca9de302488c5ca95ad4a39e190093d6c1a8ace08341b/requests-2.32.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e2/c5/9136736c37022a6ad27fea38f3111eb8f02fe75d067f9a985cc358653102/rouge_score-0.1.2.tar.gz - pypi: https://files.pythonhosted.org/packages/fc/51/727abb13f44c1fcf6d145979e1535a35794db0f6e450a0cb46aa24732fe2/s3transfer-0.16.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b8/3b/11f1b4a2f5d2ab7da34ecc062b0bc301f2be024d110a6466726bec8c055c/safetensors-0.5.3-cp38-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/71/f3/f1df377d1bdfc3e3e2adc9c119c238b182293e6740df4cbeac6de2cc3e23/scikit_learn-1.7.1-cp313-cp313-macosx_12_0_arm64.whl @@ -2585,6 +2597,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/ucrt-10.0.22621.0-h57928b3_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/vc-14.3-h41ae7f8_26.conda - conda: https://conda.anaconda.org/conda-forge/win-64/vc14_runtime-14.44.35208-h818238b_26.conda + - pypi: https://files.pythonhosted.org/packages/18/a6/907a406bb7d359e6a63f99c313846d9eec4f7e6f7437809e03aa00fa3074/absl_py-2.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/30/dd/0107f0aa179869ee9f47ef5a2686abd5e022fdc82af901d535e52fe91ce1/accelerate-1.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/f9/25753b9de3029d3eb2487755520b98eb72b0cb562d8974329c6e19831063/axial_positional_embedding-0.3.12-py3-none-any.whl @@ -2602,6 +2615,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/1d/54/a46920229d12c3a6e9f0081d1bdaeffad23c1826353ace95714faee926e5/dask-2025.11.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/46/ec/da78855318971c2be94d0283a41de6941a6b9f16146fb00babc74903ae01/distributed-2025.11.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d5/18/9f4f975ca87a390832b1c22478f3702fcdf739f83211e24d054b7551270d/editdistance-0.8.1.tar.gz - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4d/36/2a115987e2d8c300a974597416d9de88f2444426de9571f4b59b2cca3acc/filelock-3.18.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a0/ee/f626cd372932d828508137a79b85167fdcf3adab2e3bed433f295c596c6a/fonttools-4.59.0-cp313-cp313-win_amd64.whl @@ -2630,6 +2644,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/74/07/1ed8277f8653c40ebc65985180b007879f6a836c525b3885dcc6448ae6cb/msgpack-1.1.2-cp313-cp313-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/87/0d/1861d1599571974b15b025e12b142d8e6b42ad66c8a07a89cb0fc21f1e03/narwhals-2.13.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/8d/776adee7bbf76365fdd7f2552710282c79a4ead5d2a46408c9043a2b70ba/networkx-3.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/9d/91/04e965f8e717ba0ab4bdca5c112deeab11c9e750d94c4d4602f050295d39/nltk-3.9.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cb/3b/d58c12eafcb298d4e6d0d40216866ab15f59e55d148a5658bb3132311fcf/numpy-2.2.6-cp313-cp313-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/14/dd/916c6777222db3271e9fb3cf9a97ed92b3a9b3e465bdeec96de9ab809d53/obstore-0.8.2-cp313-cp313-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/7e/95/e0770cf1ad9667492f56b732f44398ef2756d61df914e10d121a3cad013a/ogb-1.3.6-py3-none-any.whl @@ -2655,6 +2670,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/98/da/164e31b607c0cf22f1179cd15fa058780f940b21ec42ba3c9026c21897e3/rdkit-2025.3.3-cp313-cp313-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/45/94/bc295babb3062a731f52621cdc992d123111282e291abaf23faa413443ea/regex-2024.11.6-cp313-cp313-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/7c/e4/56027c4a6b4ae70ca9de302488c5ca95ad4a39e190093d6c1a8ace08341b/requests-2.32.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e2/c5/9136736c37022a6ad27fea38f3111eb8f02fe75d067f9a985cc358653102/rouge_score-0.1.2.tar.gz - pypi: https://files.pythonhosted.org/packages/fc/51/727abb13f44c1fcf6d145979e1535a35794db0f6e450a0cb46aa24732fe2/s3transfer-0.16.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/69/e2/b011c38e5394c4c18fb5500778a55ec43ad6106126e74723ffaee246f56e/safetensors-0.5.3-cp38-abi3-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/e2/47/9291cfa1db1dae9880420d1e07dbc7e8dd4a7cdbc42eaba22512e6bde958/scikit_learn-1.7.1-cp313-cp313-win_amd64.whl @@ -3213,6 +3229,11 @@ packages: purls: [] size: 8191 timestamp: 1744137672556 +- pypi: https://files.pythonhosted.org/packages/18/a6/907a406bb7d359e6a63f99c313846d9eec4f7e6f7437809e03aa00fa3074/absl_py-2.4.0-py3-none-any.whl + name: absl-py + version: 2.4.0 + sha256: 88476fd881ca8aab94ffa78b7b6c632a782ab3ba1cd19c9bd423abc4fb4cd28d + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/30/dd/0107f0aa179869ee9f47ef5a2686abd5e022fdc82af901d535e52fe91ce1/accelerate-1.10.0-py3-none-any.whl name: accelerate version: 1.10.0 @@ -3958,6 +3979,11 @@ packages: - pkg:pypi/editables?source=hash-mapping size: 10828 timestamp: 1733208220327 +- pypi: https://files.pythonhosted.org/packages/d5/18/9f4f975ca87a390832b1c22478f3702fcdf739f83211e24d054b7551270d/editdistance-0.8.1.tar.gz + name: editdistance + version: 0.8.1 + sha256: d1cdf80a5d5014b0c9126a69a42ce55a457b457f6986ff69ca98e4fe4d2d8fed + requires_python: '>=3.8' - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl name: einops version: 0.8.2 @@ -5913,6 +5939,32 @@ packages: - pkg:pypi/nh3?source=hash-mapping size: 584955 timestamp: 1756737407424 +- pypi: https://files.pythonhosted.org/packages/9d/91/04e965f8e717ba0ab4bdca5c112deeab11c9e750d94c4d4602f050295d39/nltk-3.9.4-py3-none-any.whl + name: nltk + version: 3.9.4 + sha256: f2fa301c3a12718ce4a0e9305c5675299da5ad9e26068218b69d692fda84828f + requires_dist: + - click + - joblib + - regex>=2021.8.3 + - tqdm + - numpy ; extra == 'machine-learning' + - python-crfsuite ; extra == 'machine-learning' + - scikit-learn ; extra == 'machine-learning' + - scipy ; extra == 'machine-learning' + - matplotlib ; extra == 'plot' + - pyparsing ; extra == 'tgrep' + - twython ; extra == 'twitter' + - requests ; extra == 'corenlp' + - scipy ; extra == 'all' + - python-crfsuite ; extra == 'all' + - pyparsing ; extra == 'all' + - requests ; extra == 'all' + - numpy ; extra == 'all' + - scikit-learn ; extra == 'all' + - twython ; extra == 'all' + - matplotlib ; extra == 'all' + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/19/49/4df9123aafa7b539317bf6d342cb6d227e49f7a35b99c287a6109b13dd93/numpy-2.2.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl name: numpy version: 2.2.6 @@ -7029,8 +7081,8 @@ packages: timestamp: 1750615908735 - pypi: ./ name: pyhealth - version: 2.0.0 - sha256: f07719f9dceb759c35507216c8033d2f915d241418d4fad2ab51b37c0e73260f + version: 2.0.1 + sha256: bf368461a8e66f93ad43f5880295045cbabe6688064af9a650a92bdaf1665332 requires_dist: - torch~=2.7.1 - torchvision @@ -7055,6 +7107,10 @@ packages: - more-itertools~=10.8.0 - einops>=0.8.0 - linear-attention-transformer>=0.19.1 + - torch-geometric>=2.6.0 ; extra == 'graph' + - editdistance~=0.8.1 ; extra == 'nlp' + - rouge-score~=0.1.2 ; extra == 'nlp' + - nltk~=3.9.1 ; extra == 'nlp' requires_python: '>=3.12,<3.14' - pypi: https://files.pythonhosted.org/packages/05/e7/df2285f3d08fee213f2d041540fa4fc9ca6c2d44cf36d3a035bf2a8d2bcc/pyparsing-3.2.3-py3-none-any.whl name: pyparsing @@ -7416,6 +7472,16 @@ packages: - pkg:pypi/rich?source=compressed-mapping size: 201098 timestamp: 1753436991345 +- pypi: https://files.pythonhosted.org/packages/e2/c5/9136736c37022a6ad27fea38f3111eb8f02fe75d067f9a985cc358653102/rouge_score-0.1.2.tar.gz + name: rouge-score + version: 0.1.2 + sha256: c7d4da2683e68c9abf0135ef915d63a46643666f848e558a1b9f7ead17ff0f04 + requires_dist: + - absl-py + - nltk + - numpy + - six>=1.14.0 + requires_python: '>=3.7' - pypi: https://files.pythonhosted.org/packages/fc/51/727abb13f44c1fcf6d145979e1535a35794db0f6e450a0cb46aa24732fe2/s3transfer-0.16.0-py3-none-any.whl name: s3transfer version: 0.16.0 diff --git a/pyhealth/calib/predictionset/base_conformal/__init__.py b/pyhealth/calib/predictionset/base_conformal/__init__.py index 8e37c0d6d..3451f9062 100644 --- a/pyhealth/calib/predictionset/base_conformal/__init__.py +++ b/pyhealth/calib/predictionset/base_conformal/__init__.py @@ -28,21 +28,32 @@ __all__ = ["BaseConformal"] -def _query_quantile(scores: np.ndarray, alpha: float) -> float: - """Compute the alpha-quantile of scores for conformal prediction. +def _query_quantile(nc_scores: np.ndarray, alpha: float) -> float: + """Compute the conformal quantile threshold on non-conformity scores. + + Implements the standard split conformal quantile: + q = ceil((1-alpha)*(N+1))-th smallest non-conformity score. + + The (N+1) term accounts for the test sample being conceptually added to + the calibration set: the threshold from N calibration NC scores using this + formula is equivalent to augmenting with the test NC score (N+1 total) and + checking whether it falls within the top (1-alpha) fraction. Args: - scores: Array of conformity scores - alpha: Quantile level (between 0 and 1), typically the miscoverage rate + nc_scores: Non-conformity scores (higher = less conforming) + alpha: Miscoverage rate (between 0 and 1) Returns: - The alpha-quantile of scores + NC threshold q. Include class y if nc_score(X, y) <= q. + Returns +inf when calibration set is too small (N < 1/alpha - 1), + meaning all classes are included to preserve coverage. """ - scores = np.sort(scores) - N = len(scores) - # Use ceiling to get conservative coverage - loc = int(np.ceil(alpha * (N + 1))) - 1 - return -np.inf if loc == -1 else scores[loc] + nc_scores = np.sort(nc_scores) + N = len(nc_scores) + loc = int(np.ceil((1 - alpha) * (N + 1))) - 1 # 0-indexed + if loc >= N: + return np.inf # calibration set too small: include everything + return float(nc_scores[loc]) def _query_weighted_quantile( @@ -177,23 +188,22 @@ def __init__( # Will be set during calibration self.t = None - def _compute_conformity_scores( + def _compute_nc_scores( self, y_prob: np.ndarray, y_true: np.ndarray ) -> np.ndarray: - """Compute conformity scores from predictions and true labels. + """Compute non-conformity scores from predictions and true labels. Args: y_prob: Predicted probabilities of shape (N, K) y_true: True class labels of shape (N,) Returns: - Conformity scores of shape (N,) + Non-conformity scores of shape (N,) — higher means less conforming. """ N = len(y_true) if self.score_type == "aps" or self.score_type == "threshold": - # Use probability of true class as conformity score - # Higher score = more conforming (better prediction) - scores = y_prob[np.arange(N), y_true] + # NC score = 1 - p(true class); higher = less conforming + scores = 1.0 - y_prob[np.arange(N), y_true] else: raise ValueError(f"Unknown score_type: {self.score_type}") @@ -217,13 +227,13 @@ def calibrate(self, cal_dataset: IterableDataset): y_true = cal_dataset_dict["y_true"] N, K = y_prob.shape - # Compute conformity scores - conformity_scores = self._compute_conformity_scores(y_prob, y_true) + # Compute non-conformity scores (higher = less conforming) + nc_scores = self._compute_nc_scores(y_prob, y_true) - # Compute quantile thresholds + # Compute quantile thresholds (NC threshold: include y if nc <= t) if isinstance(self.alpha, float): # Marginal coverage: single threshold - t = _query_quantile(conformity_scores, self.alpha) + t = _query_quantile(nc_scores, self.alpha) else: # Class-conditional coverage: one threshold per class if len(self.alpha) != K: @@ -235,15 +245,15 @@ def calibrate(self, cal_dataset: IterableDataset): for k in range(K): mask = y_true == k if np.sum(mask) > 0: - class_scores = conformity_scores[mask] + class_scores = nc_scores[mask] t_k = _query_quantile(class_scores, self.alpha[k]) else: - # If no calibration examples, use -inf (include all) + # No calibration examples for this class: include always print( f"Warning: No calibration examples for class {k}, " - "using -inf threshold" + "using +inf threshold" ) - t_k = -np.inf + t_k = np.inf t.append(t_k) self.t = torch.tensor(t, device=self.device) @@ -267,9 +277,8 @@ def forward(self, **kwargs) -> Dict[str, torch.Tensor]: pred = self.model(**kwargs) - # Construct prediction set by thresholding probabilities - # Include classes with probability >= threshold - pred["y_predset"] = pred["y_prob"] >= self.t + # Include class y if its NC score (1 - p(y)) <= NC threshold self.t + pred["y_predset"] = (1.0 - pred["y_prob"]) <= self.t return pred diff --git a/pyhealth/calib/predictionset/cluster/cluster_label.py b/pyhealth/calib/predictionset/cluster/cluster_label.py index a29bdb854..f56a325fa 100644 --- a/pyhealth/calib/predictionset/cluster/cluster_label.py +++ b/pyhealth/calib/predictionset/cluster/cluster_label.py @@ -214,8 +214,8 @@ def calibrate( print(f"Cluster assignments: {np.bincount(cal_cluster_labels)}") - # Compute conformity scores (probabilities of true class) - conformity_scores = y_prob[np.arange(N), y_true] + # Compute non-conformity scores (higher = less conforming) + conformity_scores = 1.0 - y_prob[np.arange(N), y_true] # Compute cluster-specific thresholds self.cluster_thresholds = {} @@ -226,13 +226,13 @@ def calibrate( if len(cluster_scores) == 0: print( f"Warning: No calibration samples in cluster {cluster_id}, " - "using -inf threshold (include all classes)" + "using +inf NC threshold (include all classes)" ) if isinstance(self.alpha, float): - self.cluster_thresholds[cluster_id] = -np.inf + self.cluster_thresholds[cluster_id] = np.inf else: self.cluster_thresholds[cluster_id] = np.array( - [-np.inf] * K + [np.inf] * K ) else: if isinstance(self.alpha, float): @@ -240,7 +240,7 @@ def calibrate( t = _query_quantile(cluster_scores, self.alpha) self.cluster_thresholds[cluster_id] = t else: - # Class-conditional coverage: one threshold per class per cluster + # Class-conditional: one threshold per class per cluster if len(self.alpha) != K: raise ValueError( f"alpha must have length {K} for class-conditional " @@ -253,12 +253,12 @@ def calibrate( class_scores = cluster_scores[class_mask] t_k = _query_quantile(class_scores, self.alpha[k]) else: - # If no calibration examples for this class in this cluster + # No examples for this class in cluster: include always print( f"Warning: No calibration examples for class {k} " - f"in cluster {cluster_id}, using -inf threshold" + f"in cluster {cluster_id}, using +inf threshold" ) - t_k = -np.inf + t_k = np.inf t.append(t_k) self.cluster_thresholds[cluster_id] = np.array(t) @@ -313,7 +313,8 @@ def forward(self, **kwargs) -> Dict[str, torch.Tensor]: ) cluster_thresholds = cluster_thresholds.view(view_shape) - pred["y_predset"] = pred["y_prob"] >= cluster_thresholds + # Include class y if its NC score (1 - p(y)) <= NC threshold + pred["y_predset"] = (1.0 - pred["y_prob"]) <= cluster_thresholds pred.pop("embed", None) # do not expose internal embedding to caller return pred diff --git a/pyhealth/calib/predictionset/label.py b/pyhealth/calib/predictionset/label.py index 435a0e9d1..8ff87d070 100644 --- a/pyhealth/calib/predictionset/label.py +++ b/pyhealth/calib/predictionset/label.py @@ -16,19 +16,13 @@ from torch.utils.data import Subset from pyhealth.calib.base_classes import SetPredictor +from pyhealth.calib.predictionset.base_conformal import _query_quantile from pyhealth.calib.utils import prepare_numpy_dataset from pyhealth.models import BaseModel __all__ = ["LABEL"] -def _query_quantile(scores, alpha): - scores = np.sort(scores) - N = len(scores) - loc = int(np.floor(alpha * (N + 1))) - 1 - return -np.inf if loc == -1 else scores[loc] - - class LABEL(SetPredictor): """LABEL: Least ambiguous set-valued classifiers with bounded error levels. @@ -110,11 +104,17 @@ def calibrate(self, cal_dataset: Subset): y_true = cal_dataset["y_true"] N, K = cal_dataset["y_prob"].shape + # NC scores: 1 - p(true class); higher = less conforming if isinstance(self.alpha, float): - t = _query_quantile(y_prob[np.arange(N), y_true], self.alpha) + t = _query_quantile( + 1.0 - y_prob[np.arange(N), y_true], self.alpha + ) else: t = [ - _query_quantile(y_prob[y_true == k, k], self.alpha[k]) for k in range(K) + _query_quantile( + 1.0 - y_prob[y_true == k, k], self.alpha[k] + ) + for k in range(K) ] self.t = torch.tensor(t, device=self.device) @@ -127,7 +127,8 @@ def forward(self, **kwargs) -> Dict[str, torch.Tensor]: :rtype: Dict[str, torch.Tensor] """ pred = self.model(**kwargs) - pred["y_predset"] = pred["y_prob"] > self.t + # Include class y if its NC score (1 - p(y)) <= NC threshold + pred["y_predset"] = (1.0 - pred["y_prob"]) <= self.t return pred if __name__ == "__main__": diff --git a/pyproject.toml b/pyproject.toml index 934d4f1bb..98f88d47b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ [project] name = "pyhealth" # must be kept in sync with git tags; is not updated by any automation tools -version = "2.0.0" +version = "2.0.1" authors = [ {name = "John Wu", email = "johnwu3@illinois.edu"}, {name = "Chaoqi Yang"}, @@ -107,7 +107,7 @@ build-env = { features = ["pyverbase", "build-env"], solve-group = "default" } # specify where to download the build backend that builds conda files [tool.pixi.package.build.backend] name = "pixi-build-python" -version = "==2.0.0" +version = "==2.0.1" channels = [ "https://prefix.dev/pixi-build-backends", "https://prefix.dev/conda-forge", From 8004398be8f9b3d4e3a3bdc7a5f4079cc71c2ced Mon Sep 17 00:00:00 2001 From: haoyu-haoyu <85037553+haoyu-haoyu@users.noreply.github.com> Date: Mon, 13 Apr 2026 22:19:02 +0100 Subject: [PATCH 03/61] test: add unit tests for RNN and MultimodalRNN models (#936) Add test_rnn.py with 12 test cases covering: TestRNN (8 tests): - Model initialization with correct attributes - Forward pass output structure and shapes - Backward pass gradient propagation - Embedding extraction via embed=True - Custom hyperparameters (embedding_dim, hidden_dim) - LSTM cell type variant - Vanilla RNN cell type variant - Bidirectional RNN variant TestMultimodalRNN (4 tests): - Initialization with correct sequential/non-sequential classification - Forward pass with mixed modalities (sequence + multi_hot + tensor) - Backward pass gradient propagation - Embedding extraction with correct mixed-modality dimensions Follows the established test pattern from test_mlp.py and test_tcn.py using create_sample_dataset with synthetic data. Ref #425 --- tests/core/test_rnn.py | 275 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 275 insertions(+) create mode 100644 tests/core/test_rnn.py diff --git a/tests/core/test_rnn.py b/tests/core/test_rnn.py new file mode 100644 index 000000000..a1bd52905 --- /dev/null +++ b/tests/core/test_rnn.py @@ -0,0 +1,275 @@ +import unittest + +import torch + +from pyhealth.datasets import create_sample_dataset, get_dataloader +from pyhealth.models import RNN +from pyhealth.models.rnn import MultimodalRNN + + +class TestRNN(unittest.TestCase): + """Test cases for the RNN model.""" + + def setUp(self): + """Set up test data and model.""" + self.samples = [ + { + "patient_id": "patient-0", + "visit_id": "visit-0", + "conditions": ["cond-33", "cond-86", "cond-80", "cond-12"], + "procedures": ["proc-12", "proc-45", "proc-23"], + "label": 0, + }, + { + "patient_id": "patient-1", + "visit_id": "visit-1", + "conditions": ["cond-33", "cond-86", "cond-80"], + "procedures": ["proc-12"], + "label": 1, + }, + ] + + self.input_schema = { + "conditions": "sequence", + "procedures": "sequence", + } + self.output_schema = {"label": "binary"} + + self.dataset = create_sample_dataset( + samples=self.samples, + input_schema=self.input_schema, + output_schema=self.output_schema, + dataset_name="test", + ) + + self.model = RNN(dataset=self.dataset) + + def test_model_initialization(self): + """Test that the RNN model initializes correctly.""" + self.assertIsInstance(self.model, RNN) + self.assertEqual(self.model.embedding_dim, 128) + self.assertEqual(self.model.hidden_dim, 128) + self.assertEqual(len(self.model.feature_keys), 2) + self.assertIn("conditions", self.model.feature_keys) + self.assertIn("procedures", self.model.feature_keys) + self.assertEqual(self.model.label_key, "label") + + def test_model_forward(self): + """Test that the RNN model forward pass works correctly.""" + train_loader = get_dataloader(self.dataset, batch_size=2, shuffle=True) + data_batch = next(iter(train_loader)) + + with torch.no_grad(): + ret = self.model(**data_batch) + + self.assertIn("loss", ret) + self.assertIn("y_prob", ret) + self.assertIn("y_true", ret) + self.assertIn("logit", ret) + + self.assertEqual(ret["y_prob"].shape[0], 2) + self.assertEqual(ret["y_true"].shape[0], 2) + self.assertEqual(ret["logit"].shape[0], 2) + self.assertEqual(ret["loss"].dim(), 0) + + def test_model_backward(self): + """Test that the RNN model backward pass works correctly.""" + train_loader = get_dataloader(self.dataset, batch_size=2, shuffle=True) + data_batch = next(iter(train_loader)) + + ret = self.model(**data_batch) + ret["loss"].backward() + + has_gradient = False + for param in self.model.parameters(): + if param.requires_grad and param.grad is not None: + has_gradient = True + break + self.assertTrue( + has_gradient, "No parameters have gradients after backward pass" + ) + + def test_model_with_embedding(self): + """Test that the RNN model returns embeddings when requested.""" + train_loader = get_dataloader(self.dataset, batch_size=2, shuffle=True) + data_batch = next(iter(train_loader)) + data_batch["embed"] = True + + with torch.no_grad(): + ret = self.model(**data_batch) + + self.assertIn("embed", ret) + self.assertEqual(ret["embed"].shape[0], 2) + expected_embed_dim = len(self.model.feature_keys) * self.model.hidden_dim + self.assertEqual(ret["embed"].shape[1], expected_embed_dim) + + def test_custom_hyperparameters(self): + """Test RNN model with custom hyperparameters.""" + model = RNN( + dataset=self.dataset, + embedding_dim=64, + hidden_dim=32, + ) + + self.assertEqual(model.embedding_dim, 64) + self.assertEqual(model.hidden_dim, 32) + + train_loader = get_dataloader(self.dataset, batch_size=2, shuffle=True) + data_batch = next(iter(train_loader)) + + with torch.no_grad(): + ret = model(**data_batch) + + self.assertIn("loss", ret) + self.assertIn("y_prob", ret) + + def test_rnn_type_lstm(self): + """Test RNN model with LSTM cell type.""" + model = RNN( + dataset=self.dataset, + rnn_type="LSTM", + ) + + train_loader = get_dataloader(self.dataset, batch_size=2, shuffle=True) + data_batch = next(iter(train_loader)) + + with torch.no_grad(): + ret = model(**data_batch) + + self.assertIn("loss", ret) + self.assertEqual(ret["y_prob"].shape[0], 2) + + def test_rnn_type_vanilla(self): + """Test RNN model with vanilla RNN cell type.""" + model = RNN( + dataset=self.dataset, + rnn_type="RNN", + ) + + train_loader = get_dataloader(self.dataset, batch_size=2, shuffle=True) + data_batch = next(iter(train_loader)) + + with torch.no_grad(): + ret = model(**data_batch) + + self.assertIn("loss", ret) + self.assertEqual(ret["y_prob"].shape[0], 2) + + def test_bidirectional(self): + """Test RNN model with bidirectional layers.""" + model = RNN( + dataset=self.dataset, + bidirectional=True, + ) + + train_loader = get_dataloader(self.dataset, batch_size=2, shuffle=True) + data_batch = next(iter(train_loader)) + + with torch.no_grad(): + ret = model(**data_batch) + + self.assertIn("loss", ret) + self.assertEqual(ret["y_prob"].shape[0], 2) + + +class TestMultimodalRNN(unittest.TestCase): + """Test cases for the MultimodalRNN model with mixed input modalities.""" + + def setUp(self): + """Set up test data with both sequential and non-sequential features.""" + self.samples = [ + { + "patient_id": "patient-0", + "visit_id": "visit-0", + "conditions": ["cond-33", "cond-86", "cond-80"], + "demographics": ["asian", "male"], + "vitals": [120.0, 80.0, 98.6], + "label": 1, + }, + { + "patient_id": "patient-1", + "visit_id": "visit-1", + "conditions": ["cond-12", "cond-52"], + "demographics": ["white", "female"], + "vitals": [110.0, 75.0, 98.2], + "label": 0, + }, + ] + + self.input_schema = { + "conditions": "sequence", + "demographics": "multi_hot", + "vitals": "tensor", + } + self.output_schema = {"label": "binary"} + + self.dataset = create_sample_dataset( + samples=self.samples, + input_schema=self.input_schema, + output_schema=self.output_schema, + dataset_name="test", + ) + + self.model = MultimodalRNN(dataset=self.dataset) + + def test_model_initialization(self): + """Test that the MultimodalRNN model initializes correctly.""" + self.assertIsInstance(self.model, MultimodalRNN) + self.assertEqual(len(self.model.feature_keys), 3) + self.assertIn("conditions", self.model.sequential_features) + self.assertIn("demographics", self.model.non_sequential_features) + self.assertIn("vitals", self.model.non_sequential_features) + + def test_model_forward(self): + """Test that the MultimodalRNN forward pass works correctly.""" + train_loader = get_dataloader(self.dataset, batch_size=2, shuffle=True) + data_batch = next(iter(train_loader)) + + with torch.no_grad(): + ret = self.model(**data_batch) + + self.assertIn("loss", ret) + self.assertIn("y_prob", ret) + self.assertIn("y_true", ret) + self.assertIn("logit", ret) + + self.assertEqual(ret["y_prob"].shape[0], 2) + self.assertEqual(ret["loss"].dim(), 0) + + def test_model_backward(self): + """Test that the MultimodalRNN backward pass works correctly.""" + train_loader = get_dataloader(self.dataset, batch_size=2, shuffle=True) + data_batch = next(iter(train_loader)) + + ret = self.model(**data_batch) + ret["loss"].backward() + + has_gradient = False + for param in self.model.parameters(): + if param.requires_grad and param.grad is not None: + has_gradient = True + break + self.assertTrue( + has_gradient, "No parameters have gradients after backward pass" + ) + + def test_model_with_embedding(self): + """Test that the MultimodalRNN returns embeddings when requested.""" + train_loader = get_dataloader(self.dataset, batch_size=2, shuffle=True) + data_batch = next(iter(train_loader)) + data_batch["embed"] = True + + with torch.no_grad(): + ret = self.model(**data_batch) + + self.assertIn("embed", ret) + self.assertEqual(ret["embed"].shape[0], 2) + expected_embed_dim = ( + len(self.model.sequential_features) * self.model.hidden_dim + + len(self.model.non_sequential_features) * self.model.embedding_dim + ) + self.assertEqual(ret["embed"].shape[1], expected_embed_dim) + + +if __name__ == "__main__": + unittest.main() From 8c0f157f73518fd68d00866b666f86693029b51c Mon Sep 17 00:00:00 2001 From: Arjun Chatterjee Date: Mon, 13 Apr 2026 16:23:47 -0500 Subject: [PATCH 04/61] Conformal Methods and Scripts (#942) * Fixed repo to be able to run TUEV/TUAB + updated example scripts * Args need to be passed correctly * Minor fixes and precomputed STFT logic * Fix the test files to reflect codebase changes * Args update * test script fixes * dataset path update * fix contrawr - small change * divide by 0 error * Incorporate tfm logic * Fix label stuff * tuab fixes * fix metrics * aggregate alphas * Fix splitting and add tfm weights * fix tfm+tuab * updates scripts and haoyu splitter * fix conflict * Remove weightfiles from tracking and add to .gitignore Weight files are large binaries distributed separately; untrack all existing .pth files under weightfiles/ and add weightfiles/ to .gitignore so they are excluded from future commits and the PR. Made-with: Cursor * normalization = 95% * temporarily re-add weight files * 16 workers * tuab sanity check * consistent log outputs * test tuab * change back to multiclass * update conformal scripts * remove weightfiles * oops * fix tests --- .../conformal_eeg/test_tfm_tuab_inference.py | 197 ++++++++++++++++++ .../conformal_eeg/test_tfm_tuev_inference.py | 112 ++++++++++ .../tuab_conventional_conformal.py | 32 ++- .../tuab_covariate_shift_conformal.py | 32 ++- .../conformal_eeg/tuab_kmeans_conformal.py | 32 ++- examples/conformal_eeg/tuab_ncp_conformal.py | 49 +++-- .../tuev_conventional_conformal.py | 7 +- .../tuev_covariate_shift_conformal.py | 7 +- .../conformal_eeg/tuev_kmeans_conformal.py | 7 +- examples/conformal_eeg/tuev_ncp_conformal.py | 22 +- pyhealth/tasks/temple_university_EEG_tasks.py | 11 +- tests/core/test_tuab.py | 2 +- 12 files changed, 447 insertions(+), 63 deletions(-) create mode 100644 examples/conformal_eeg/test_tfm_tuab_inference.py create mode 100644 examples/conformal_eeg/test_tfm_tuev_inference.py diff --git a/examples/conformal_eeg/test_tfm_tuab_inference.py b/examples/conformal_eeg/test_tfm_tuab_inference.py new file mode 100644 index 000000000..400518469 --- /dev/null +++ b/examples/conformal_eeg/test_tfm_tuab_inference.py @@ -0,0 +1,197 @@ +""" +Quick inference test: TFMTokenizer on TUAB using local weightfiles/. + +Two weight setups (ask your PI which matches their training): + + 1) Default (matches conformal example scripts): + - tokenizer: weightfiles/tfm_tokenizer_last.pth (multi-dataset tokenizer) + - classifier: weightfiles/TFM_Tokenizer_multiple_finetuned_on_TUAB/.../best_model.pth + + 2) PI benchmark TUAB-specific files (place in weightfiles/): + - tokenizer: tfm_tokenizer_tuab.pth + - classifier: tfm_encoder_best_model_tuab.pth + Use: --pi-tuab-weights + +Split modes: + - conformal (default): same test set as conformal runs (TUH eval via patient conformal split). + - pi_benchmark: train/val ratio [0.875, 0.125] on train partition; test = TUH eval (same patients as official eval). + +Usage: + python examples/conformal_eeg/test_tfm_tuab_inference.py + python examples/conformal_eeg/test_tfm_tuab_inference.py --pi-tuab-weights + python examples/conformal_eeg/test_tfm_tuab_inference.py \\ + --tuab-pi-weights-dir /shared/eng/conformal_eeg --split pi_benchmark + python examples/conformal_eeg/test_tfm_tuab_inference.py --tokenizer-weights PATH --classifier-weights PATH +""" + +import argparse +import os +import time + +import torch + +from pyhealth.datasets import ( + TUABDataset, + get_dataloader, + split_by_patient_conformal_tuh, + split_by_patient_tuh, +) +from pyhealth.models import TFMTokenizer +from pyhealth.tasks import EEGAbnormalTUAB +from pyhealth.trainer import Trainer + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +WEIGHTFILES = os.path.join(REPO_ROOT, "weightfiles") +DEFAULT_TOKENIZER = os.path.join(WEIGHTFILES, "tfm_tokenizer_last.pth") +CLASSIFIER_WEIGHTS_DIR = os.path.join( + WEIGHTFILES, "TFM_Tokenizer_multiple_finetuned_on_TUAB" +) +PI_TOKENIZER = os.path.join(WEIGHTFILES, "tfm_tokenizer_tuab.pth") +PI_CLASSIFIER = os.path.join(WEIGHTFILES, "tfm_encoder_best_model_tuab.pth") + + +def main(): + parser = argparse.ArgumentParser(description="TFM TUAB inference sanity check") + parser.add_argument( + "--root", + type=str, + default="/srv/local/data/TUH/tuh_eeg_abnormal/v3.0.0/edf", + help="Path to TUAB edf/ directory.", + ) + parser.add_argument("--gpu_id", type=int, default=0) + parser.add_argument( + "--seed", + type=int, + default=1, + choices=[1, 2, 3, 4, 5], + help="Which fine-tuned classifier folder _1.._5 (only if not using --classifier-weights).", + ) + parser.add_argument( + "--pi-tuab-weights", + action="store_true", + help="Use PI TUAB-specific files under weightfiles/: tfm_tokenizer_tuab.pth, " + "tfm_encoder_best_model_tuab.pth", + ) + parser.add_argument( + "--tuab-pi-weights-dir", + type=str, + default=None, + metavar="DIR", + help="Directory containing PI's TUAB TFM files (e.g. /shared/eng/conformal_eeg). " + "Loads tfm_tokenizer_tuab.pth + tfm_encoder_best_model_tuab.pth from there. " + "Overrides --pi-tuab-weights and default weightfiles paths unless " + "--tokenizer-weights / --classifier-weights are set explicitly.", + ) + parser.add_argument( + "--tokenizer-weights", + type=str, + default=None, + help="Override tokenizer checkpoint path.", + ) + parser.add_argument( + "--classifier-weights", + type=str, + default=None, + help="Override classifier checkpoint path (single .pth file).", + ) + parser.add_argument( + "--split", + type=str, + choices=["conformal", "pi_benchmark"], + default="conformal", + help="conformal: same as EEG conformal scripts; pi_benchmark: 0.875/0.125 train/val on train partition.", + ) + parser.add_argument( + "--split-seed", + type=int, + default=42, + help="RNG seed for patient shuffle (pi_benchmark and conformal).", + ) + args = parser.parse_args() + device = f"cuda:{args.gpu_id}" if torch.cuda.is_available() else "cpu" + + if args.tuab_pi_weights_dir is not None: + d = os.path.expanduser(args.tuab_pi_weights_dir) + tok = os.path.join(d, "tfm_tokenizer_tuab.pth") + cls_path = os.path.join(d, "tfm_encoder_best_model_tuab.pth") + elif args.pi_tuab_weights: + tok = PI_TOKENIZER + cls_path = PI_CLASSIFIER + else: + tok = DEFAULT_TOKENIZER + cls_path = os.path.join( + CLASSIFIER_WEIGHTS_DIR, + f"TFM_Tokenizer_multiple_finetuned_on_TUAB_{args.seed}", + "best_model.pth", + ) + + if args.tokenizer_weights is not None: + tok = args.tokenizer_weights + if args.classifier_weights is not None: + cls_path = args.classifier_weights + + print(f"Device: {device}") + print(f"TUAB root: {args.root}") + print(f"Split mode: {args.split}") + print(f"Tokenizer weights: {tok}") + print(f"Classifier weights: {cls_path}") + + t0 = time.time() + base_dataset = TUABDataset(root=args.root, subset="both") + print(f"Dataset loaded in {time.time() - t0:.1f}s") + + t0 = time.time() + sample_dataset = base_dataset.set_task( + EEGAbnormalTUAB( + resample_rate=200, + normalization="95th_percentile", + compute_stft=True, + ), + num_workers=16, + ) + print(f"Task set in {time.time() - t0:.1f}s | total samples: {len(sample_dataset)}") + + if args.split == "conformal": + _, _, _, test_ds = split_by_patient_conformal_tuh( + dataset=sample_dataset, + ratios=[0.6, 0.2, 0.2], + seed=args.split_seed, + ) + else: + _, _, test_ds = split_by_patient_tuh( + sample_dataset, + [0.875, 0.125], + seed=args.split_seed, + ) + + test_loader = get_dataloader(test_ds, batch_size=32, shuffle=False) + print(f"Test set size: {len(test_ds)}") + + model = TFMTokenizer(dataset=sample_dataset).to(device) + model.load_pretrained_weights( + tokenizer_checkpoint_path=tok, + classifier_checkpoint_path=cls_path, + ) + + trainer = Trainer( + model=model, + device=device, + metrics=[ + "accuracy", + "balanced_accuracy", + "f1_weighted", + "f1_macro", + "roc_auc_weighted_ovr", + ], + enable_logging=False, + ) + t0 = time.time() + results = trainer.evaluate(test_loader) + print(f"\nEval time: {time.time() - t0:.1f}s") + print("\n=== Test Results ===") + for metric, value in results.items(): + print(f" {metric}: {value:.4f}") + + +if __name__ == "__main__": + main() diff --git a/examples/conformal_eeg/test_tfm_tuev_inference.py b/examples/conformal_eeg/test_tfm_tuev_inference.py new file mode 100644 index 000000000..fede48d4d --- /dev/null +++ b/examples/conformal_eeg/test_tfm_tuev_inference.py @@ -0,0 +1,112 @@ +""" +Quick inference test: TFMTokenizer on TUEV using local weightfiles/. + +Mirrors the PI's benchmark script but uses the weightfiles/ paths already +present in this repo. No training — pure inference to verify weights and +normalization are correct. + +Usage: + python examples/conformal_eeg/test_tfm_tuev_inference.py + python examples/conformal_eeg/test_tfm_tuev_inference.py --gpu_id 1 + python examples/conformal_eeg/test_tfm_tuev_inference.py --seed 2 # use _2/best_model.pth +""" + +import argparse +import os +import time + +import torch + +from pyhealth.datasets import TUEVDataset, get_dataloader, split_by_patient_conformal_tuh +from pyhealth.models import TFMTokenizer +from pyhealth.tasks import EEGEventsTUEV +from pyhealth.trainer import Trainer + +TUEV_ROOT = "/srv/local/data/TUH/tuh_eeg_events/v2.0.0/edf/" + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +TOKENIZER_WEIGHTS = os.path.join(REPO_ROOT, "weightfiles", "tfm_tokenizer_last.pth") +CLASSIFIER_WEIGHTS_DIR = os.path.join( + REPO_ROOT, "weightfiles", "TFM_Tokenizer_multiple_finetuned_on_TUEV" +) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--gpu_id", type=int, default=0) + parser.add_argument( + "--seed", type=int, default=1, choices=[1, 2, 3, 4, 5], + help="Which fine-tuned classifier to use (1-5)." + ) + args = parser.parse_args() + device = f"cuda:{args.gpu_id}" if torch.cuda.is_available() else "cpu" + + classifier_weights = os.path.join( + CLASSIFIER_WEIGHTS_DIR, + f"TFM_Tokenizer_multiple_finetuned_on_TUEV_{args.seed}", + "best_model.pth", + ) + + print(f"Device: {device}") + print(f"Tokenizer weights: {TOKENIZER_WEIGHTS}") + print(f"Classifier weights: {classifier_weights}") + + # ------------------------------------------------------------------ # + # STEP 1: Load dataset + # ------------------------------------------------------------------ # + t0 = time.time() + base_dataset = TUEVDataset(root=TUEV_ROOT, subset="both") + print(f"Dataset loaded in {time.time() - t0:.1f}s") + + # ------------------------------------------------------------------ # + # STEP 2: Set task — normalization="95th_percentile" matches training + # ------------------------------------------------------------------ # + t0 = time.time() + sample_dataset = base_dataset.set_task( + EEGEventsTUEV( + resample_rate=200, + normalization="95th_percentile", + compute_stft=True, + ) + ) + print(f"Task set in {time.time() - t0:.1f}s | total samples: {len(sample_dataset)}") + + # ------------------------------------------------------------------ # + # STEP 3: Extract fixed test set (TUH eval partition) + # ------------------------------------------------------------------ # + _, _, _, test_ds = split_by_patient_conformal_tuh( + dataset=sample_dataset, + ratios=[0.6, 0.2, 0.2], + seed=42, + ) + test_loader = get_dataloader(test_ds, batch_size=32, shuffle=False) + print(f"Test set size: {len(test_ds)}") + + # ------------------------------------------------------------------ # + # STEP 4: Load TFMTokenizer with pre-trained weights (no training) + # ------------------------------------------------------------------ # + model = TFMTokenizer(dataset=sample_dataset).to(device) + model.load_pretrained_weights( + tokenizer_checkpoint_path=TOKENIZER_WEIGHTS, + classifier_checkpoint_path=classifier_weights, + ) + + # ------------------------------------------------------------------ # + # STEP 5: Evaluate + # ------------------------------------------------------------------ # + trainer = Trainer( + model=model, + device=device, + metrics=["accuracy", "f1_weighted", "f1_macro"], + enable_logging=False, + ) + t0 = time.time() + results = trainer.evaluate(test_loader) + print(f"\nEval time: {time.time() - t0:.1f}s") + print("\n=== Test Results ===") + for metric, value in results.items(): + print(f" {metric}: {value:.4f}") + + +if __name__ == "__main__": + main() diff --git a/examples/conformal_eeg/tuab_conventional_conformal.py b/examples/conformal_eeg/tuab_conventional_conformal.py index 0f6b32401..7789b83e7 100644 --- a/examples/conformal_eeg/tuab_conventional_conformal.py +++ b/examples/conformal_eeg/tuab_conventional_conformal.py @@ -123,13 +123,16 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--weights-dir", type=str, - default="weightfiles/TFM_Tokenizer_multiple_finetuned_on_TUAB", - help="Root folder of fine-tuned TFM classifier checkpoints (only with --model tfm).", + default="/shared/eng/conformal_eeg", + help="Root folder of TFM classifier checkpoints (only with --model tfm). " + "If the directory contains tfm_encoder_best_model_tuab.pth directly, " + "that single checkpoint is used for all seeds (PI TUAB setup). " + "Otherwise expects per-seed subdirs {base}_1..N/best_model.pth.", ) parser.add_argument( "--tokenizer-weights", type=str, - default="weightfiles/tfm_tokenizer_last.pth", + default="/shared/eng/conformal_eeg/tfm_tokenizer_tuab.pth", help="Path to the pre-trained TFM tokenizer weights (only with --model tfm).", ) parser.add_argument( @@ -152,9 +155,19 @@ def _do_split(dataset, ratios, seed, split_type): def _load_tfm_weights(model, args, run_idx: int) -> None: - """Load pre-trained tokenizer + fine-tuned classifier for run_idx (0-based).""" - base = os.path.basename(args.weights_dir) - classifier_path = os.path.join(args.weights_dir, f"{base}_{run_idx + 1}", "best_model.pth") + """Load pre-trained tokenizer + fine-tuned classifier for run_idx (0-based). + + Supports two layouts: + - Single classifier (PI TUAB setup): weights_dir/tfm_encoder_best_model_tuab.pth + Used for all seeds — only the data split varies across runs. + - Per-seed subdirs: weights_dir/{base}_{run_idx+1}/best_model.pth + """ + single = os.path.join(args.weights_dir, "tfm_encoder_best_model_tuab.pth") + if os.path.isfile(single): + classifier_path = single + else: + base = os.path.basename(args.weights_dir) + classifier_path = os.path.join(args.weights_dir, f"{base}_{run_idx + 1}", "best_model.pth") print(f" Loading TFM weights (run {run_idx + 1}): {classifier_path}") model.load_pretrained_weights( tokenizer_checkpoint_path=args.tokenizer_weights, @@ -283,7 +296,7 @@ def _print_multi_seed_summary( n_runs = len(all_metrics) print("\n" + "=" * 80) - print("Per-run LABEL results (fixed test set = TUH eval partition)") + print(f"Per-run results — alpha={alpha} (LABEL, fixed test set = TUH eval partition)") print("=" * 80) print(f" {'Run':<4} {'Seed':<6} {'Accuracy':<10} {'ROC-AUC':<10} {'F1':<8} " f"{'Coverage':<10} {'Miscoverage':<12} {'Avg set size':<12}") @@ -295,7 +308,8 @@ def _print_multi_seed_summary( f"{m['miscoverage']:<12.4f} {m['avg_set_size']:<12.2f}") print("\n" + "=" * 80) - print(f"LABEL summary (mean \u00b1 std over {n_runs} runs, fixed test set)") + print(f"Summary — alpha={alpha} (mean \u00b1 std over {n_runs} runs, fixed test set)") + print(" Method: LABEL") print("=" * 80) print(f" Accuracy: {accs.mean():.4f} \u00b1 {accs.std():.4f}") print(f" ROC-AUC: {roc_aucs.mean():.4f} \u00b1 {roc_aucs.std():.4f}") @@ -334,7 +348,7 @@ def _main(args: argparse.Namespace) -> None: print("STEP 1: Load TUAB + build task dataset (shared across all seeds)") print("=" * 80) dataset = TUABDataset(root=str(root), subset=args.subset, dev=args.quick_test) - sample_dataset = dataset.set_task(EEGAbnormalTUAB()) + sample_dataset = dataset.set_task(EEGAbnormalTUAB(normalization="95th_percentile"), num_workers=16) if args.quick_test and len(sample_dataset) > quick_test_max_samples: sample_dataset = sample_dataset.subset(range(quick_test_max_samples)) print(f"Capped to {quick_test_max_samples} samples for quick-test.") diff --git a/examples/conformal_eeg/tuab_covariate_shift_conformal.py b/examples/conformal_eeg/tuab_covariate_shift_conformal.py index 33a810ab1..11460d139 100644 --- a/examples/conformal_eeg/tuab_covariate_shift_conformal.py +++ b/examples/conformal_eeg/tuab_covariate_shift_conformal.py @@ -129,13 +129,16 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--weights-dir", type=str, - default="weightfiles/TFM_Tokenizer_multiple_finetuned_on_TUAB", - help="Root folder of fine-tuned TFM classifier checkpoints (only with --model tfm).", + default="/shared/eng/conformal_eeg", + help="Root folder of TFM classifier checkpoints (only with --model tfm). " + "If the directory contains tfm_encoder_best_model_tuab.pth directly, " + "that single checkpoint is used for all seeds (PI TUAB setup). " + "Otherwise expects per-seed subdirs {base}_1..N/best_model.pth.", ) parser.add_argument( "--tokenizer-weights", type=str, - default="weightfiles/tfm_tokenizer_last.pth", + default="/shared/eng/conformal_eeg/tfm_tokenizer_tuab.pth", help="Path to the pre-trained TFM tokenizer weights (only with --model tfm).", ) parser.add_argument( @@ -158,9 +161,19 @@ def _do_split(dataset, ratios, seed, split_type): def _load_tfm_weights(model, args, run_idx: int) -> None: - """Load pre-trained tokenizer + fine-tuned classifier for run_idx (0-based).""" - base = os.path.basename(args.weights_dir) - classifier_path = os.path.join(args.weights_dir, f"{base}_{run_idx + 1}", "best_model.pth") + """Load pre-trained tokenizer + fine-tuned classifier for run_idx (0-based). + + Supports two layouts: + - Single classifier (PI TUAB setup): weights_dir/tfm_encoder_best_model_tuab.pth + Used for all seeds — only the data split varies across runs. + - Per-seed subdirs: weights_dir/{base}_{run_idx+1}/best_model.pth + """ + single = os.path.join(args.weights_dir, "tfm_encoder_best_model_tuab.pth") + if os.path.isfile(single): + classifier_path = single + else: + base = os.path.basename(args.weights_dir) + classifier_path = os.path.join(args.weights_dir, f"{base}_{run_idx + 1}", "best_model.pth") print(f" Loading TFM weights (run {run_idx + 1}): {classifier_path}") model.load_pretrained_weights( tokenizer_checkpoint_path=args.tokenizer_weights, @@ -299,7 +312,7 @@ def _print_multi_seed_summary( n_runs = len(all_metrics) print("\n" + "=" * 80) - print("Per-run CovariateLabel results (fixed test set = TUH eval partition)") + print(f"Per-run results — alpha={alpha} (CovariateLabel, fixed test set = TUH eval partition)") print("=" * 80) print(f" {'Run':<4} {'Seed':<6} {'Accuracy':<10} {'ROC-AUC':<10} {'F1':<8} " f"{'Coverage':<10} {'Miscoverage':<12} {'Avg set size':<12}") @@ -311,7 +324,8 @@ def _print_multi_seed_summary( f"{m['miscoverage']:<12.4f} {m['avg_set_size']:<12.2f}") print("\n" + "=" * 80) - print(f"CovariateLabel summary (mean \u00b1 std over {n_runs} runs, fixed test set)") + print(f"Summary — alpha={alpha} (mean \u00b1 std over {n_runs} runs, fixed test set)") + print(" Method: CovariateLabel") print("=" * 80) print(f" Accuracy: {accs.mean():.4f} \u00b1 {accs.std():.4f}") print(f" ROC-AUC: {roc_aucs.mean():.4f} \u00b1 {roc_aucs.std():.4f}") @@ -350,7 +364,7 @@ def _main(args: argparse.Namespace) -> None: print("STEP 1: Load TUAB + build task dataset (shared across all seeds)") print("=" * 80) dataset = TUABDataset(root=str(root), subset=args.subset, dev=args.quick_test) - sample_dataset = dataset.set_task(EEGAbnormalTUAB()) + sample_dataset = dataset.set_task(EEGAbnormalTUAB(normalization="95th_percentile"), num_workers=16) if args.quick_test and len(sample_dataset) > quick_test_max_samples: sample_dataset = sample_dataset.subset(range(quick_test_max_samples)) print(f"Capped to {quick_test_max_samples} samples for quick-test.") diff --git a/examples/conformal_eeg/tuab_kmeans_conformal.py b/examples/conformal_eeg/tuab_kmeans_conformal.py index 67152bfff..99b9742f4 100644 --- a/examples/conformal_eeg/tuab_kmeans_conformal.py +++ b/examples/conformal_eeg/tuab_kmeans_conformal.py @@ -132,13 +132,16 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--weights-dir", type=str, - default="weightfiles/TFM_Tokenizer_multiple_finetuned_on_TUAB", - help="Root folder of fine-tuned TFM classifier checkpoints (only with --model tfm).", + default="/shared/eng/conformal_eeg", + help="Root folder of TFM classifier checkpoints (only with --model tfm). " + "If the directory contains tfm_encoder_best_model_tuab.pth directly, " + "that single checkpoint is used for all seeds (PI TUAB setup). " + "Otherwise expects per-seed subdirs {base}_1..N/best_model.pth.", ) parser.add_argument( "--tokenizer-weights", type=str, - default="weightfiles/tfm_tokenizer_last.pth", + default="/shared/eng/conformal_eeg/tfm_tokenizer_tuab.pth", help="Path to the pre-trained TFM tokenizer weights (only with --model tfm).", ) parser.add_argument( @@ -161,9 +164,19 @@ def _do_split(dataset, ratios, seed, split_type): def _load_tfm_weights(model, args, run_idx: int) -> None: - """Load pre-trained tokenizer + fine-tuned classifier for run_idx (0-based).""" - base = os.path.basename(args.weights_dir) - classifier_path = os.path.join(args.weights_dir, f"{base}_{run_idx + 1}", "best_model.pth") + """Load pre-trained tokenizer + fine-tuned classifier for run_idx (0-based). + + Supports two layouts: + - Single classifier (PI TUAB setup): weights_dir/tfm_encoder_best_model_tuab.pth + Used for all seeds — only the data split varies across runs. + - Per-seed subdirs: weights_dir/{base}_{run_idx+1}/best_model.pth + """ + single = os.path.join(args.weights_dir, "tfm_encoder_best_model_tuab.pth") + if os.path.isfile(single): + classifier_path = single + else: + base = os.path.basename(args.weights_dir) + classifier_path = os.path.join(args.weights_dir, f"{base}_{run_idx + 1}", "best_model.pth") print(f" Loading TFM weights (run {run_idx + 1}): {classifier_path}") model.load_pretrained_weights( tokenizer_checkpoint_path=args.tokenizer_weights, @@ -308,7 +321,7 @@ def _print_multi_seed_summary( n_runs = len(all_metrics) print("\n" + "=" * 80) - print("Per-run ClusterLabel results (fixed test set = TUH eval partition)") + print(f"Per-run results — alpha={alpha} (ClusterLabel, fixed test set = TUH eval partition)") print("=" * 80) print(f" {'Run':<4} {'Seed':<6} {'Accuracy':<10} {'ROC-AUC':<10} {'F1':<8} " f"{'Coverage':<10} {'Miscoverage':<12} {'Avg set size':<12}") @@ -320,7 +333,8 @@ def _print_multi_seed_summary( f"{m['miscoverage']:<12.4f} {m['avg_set_size']:<12.2f}") print("\n" + "=" * 80) - print(f"ClusterLabel summary (mean \u00b1 std over {n_runs} runs, fixed test set)") + print(f"Summary — alpha={alpha} (mean \u00b1 std over {n_runs} runs, fixed test set)") + print(" Method: ClusterLabel") print("=" * 80) print(f" Accuracy: {accs.mean():.4f} \u00b1 {accs.std():.4f}") print(f" ROC-AUC: {roc_aucs.mean():.4f} \u00b1 {roc_aucs.std():.4f}") @@ -360,7 +374,7 @@ def _main(args: argparse.Namespace) -> None: print("STEP 1: Load TUAB + build task dataset (shared across all seeds)") print("=" * 80) dataset = TUABDataset(root=str(root), subset=args.subset, dev=args.quick_test) - sample_dataset = dataset.set_task(EEGAbnormalTUAB()) + sample_dataset = dataset.set_task(EEGAbnormalTUAB(normalization="95th_percentile"), num_workers=16) if args.quick_test and len(sample_dataset) > quick_test_max_samples: sample_dataset = sample_dataset.subset(range(quick_test_max_samples)) print(f"Capped to {quick_test_max_samples} samples for quick-test.") diff --git a/examples/conformal_eeg/tuab_ncp_conformal.py b/examples/conformal_eeg/tuab_ncp_conformal.py index be658d794..90166686a 100644 --- a/examples/conformal_eeg/tuab_ncp_conformal.py +++ b/examples/conformal_eeg/tuab_ncp_conformal.py @@ -136,13 +136,16 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--weights-dir", type=str, - default="weightfiles/TFM_Tokenizer_multiple_finetuned_on_TUAB", - help="Root folder of fine-tuned TFM classifier checkpoints (only with --model tfm).", + default="/shared/eng/conformal_eeg", + help="Root folder of TFM classifier checkpoints (only with --model tfm). " + "If the directory contains tfm_encoder_best_model_tuab.pth directly, " + "that single checkpoint is used for all seeds (PI TUAB setup). " + "Otherwise expects per-seed subdirs {base}_1..N/best_model.pth.", ) parser.add_argument( "--tokenizer-weights", type=str, - default="weightfiles/tfm_tokenizer_last.pth", + default="/shared/eng/conformal_eeg/tfm_tokenizer_tuab.pth", help="Path to the pre-trained TFM tokenizer weights (only with --model tfm).", ) parser.add_argument( @@ -165,9 +168,19 @@ def _do_split(dataset, ratios, seed, split_type): def _load_tfm_weights(model, args, run_idx: int) -> None: - """Load pre-trained tokenizer + fine-tuned classifier for run_idx (0-based).""" - base = os.path.basename(args.weights_dir) - classifier_path = os.path.join(args.weights_dir, f"{base}_{run_idx + 1}", "best_model.pth") + """Load pre-trained tokenizer + fine-tuned classifier for run_idx (0-based). + + Supports two layouts: + - Single classifier (PI TUAB setup): weights_dir/tfm_encoder_best_model_tuab.pth + Used for all seeds — only the data split varies across runs. + - Per-seed subdirs: weights_dir/{base}_{run_idx+1}/best_model.pth + """ + single = os.path.join(args.weights_dir, "tfm_encoder_best_model_tuab.pth") + if os.path.isfile(single): + classifier_path = single + else: + base = os.path.basename(args.weights_dir) + classifier_path = os.path.join(args.weights_dir, f"{base}_{run_idx + 1}", "best_model.pth") print(f" Loading TFM weights (run {run_idx + 1}): {classifier_path}") model.load_pretrained_weights( tokenizer_checkpoint_path=args.tokenizer_weights, @@ -319,7 +332,7 @@ def _run(args: argparse.Namespace) -> None: print("STEP 1: Load TUAB + build task dataset") print("=" * 80) dataset = TUABDataset(root=str(root), subset=args.subset, dev=args.quick_test) - sample_dataset = dataset.set_task(EEGAbnormalTUAB()) + sample_dataset = dataset.set_task(EEGAbnormalTUAB(normalization="95th_percentile"), num_workers=16) if args.quick_test and len(sample_dataset) > quick_test_max_samples: sample_dataset = sample_dataset.subset(range(quick_test_max_samples)) print(f"Capped to {quick_test_max_samples} samples for quick-test.") @@ -410,7 +423,10 @@ def _run(args: argparse.Namespace) -> None: set_sizes = np.array([m["avg_set_size"] for m in mlist]) if not use_multi_seed: - print(f"\nNCP Results (alpha={alpha}):") + print("\n" + "=" * 80) + print(f"Summary — alpha={alpha} (single run, fixed test set)") + print(" Method: NeighborhoodLabel") + print("=" * 80) print(f" Accuracy: {accs[0]:.4f}") print(f" ROC-AUC: {roc_aucs[0]:.4f}") print(f" F1: {f1s[0]:.4f}") @@ -421,7 +437,7 @@ def _run(args: argparse.Namespace) -> None: print(f" k_neighbors: {args.k_neighbors}, lambda_L: {args.lambda_L}") else: print("\n" + "=" * 80) - print(f"Per-run NCP results — alpha={alpha} (target coverage={1-alpha:.0%})") + print(f"Per-run results — alpha={alpha} (NeighborhoodLabel, fixed test set = TUH eval partition)") print("=" * 80) print(f" {'Run':<4} {'Seed':<6} {'Accuracy':<10} {'ROC-AUC':<10} {'F1':<8} " f"{'Coverage':<10} {'Miscoverage':<12} {'Avg set size':<12}") @@ -431,14 +447,15 @@ def _run(args: argparse.Namespace) -> None: f"{f1s[i]:<8.4f} {coverages[i]:<10.4f} {miscovs[i]:<12.4f} {set_sizes[i]:<12.2f}") print("\n" + "=" * 80) - print(f"NCP summary — alpha={alpha} (mean ± std over {n_runs} runs, fixed test set)") + print(f"Summary — alpha={alpha} (mean \u00b1 std over {n_runs} runs, fixed test set)") + print(" Method: NeighborhoodLabel") print("=" * 80) - print(f" Accuracy: {accs.mean():.4f} ± {accs.std():.4f}") - print(f" ROC-AUC: {roc_aucs.mean():.4f} ± {roc_aucs.std():.4f}") - print(f" F1: {f1s.mean():.4f} ± {f1s.std():.4f}") - print(f" Empirical coverage: {coverages.mean():.4f} ± {coverages.std():.4f}") - print(f" Empirical miscoverage: {miscovs.mean():.4f} ± {miscovs.std():.4f}") - print(f" Average set size: {set_sizes.mean():.2f} ± {set_sizes.std():.2f}") + print(f" Accuracy: {accs.mean():.4f} \u00b1 {accs.std():.4f}") + print(f" ROC-AUC: {roc_aucs.mean():.4f} \u00b1 {roc_aucs.std():.4f}") + print(f" F1: {f1s.mean():.4f} \u00b1 {f1s.std():.4f}") + print(f" Empirical coverage: {coverages.mean():.4f} \u00b1 {coverages.std():.4f}") + print(f" Empirical miscoverage: {miscovs.mean():.4f} \u00b1 {miscovs.std():.4f}") + print(f" Average set size: {set_sizes.mean():.2f} \u00b1 {set_sizes.std():.2f}") print(f" Target coverage: {1 - alpha:.0%} (alpha={alpha})") print(f" k_neighbors: {args.k_neighbors}, lambda_L: {args.lambda_L}") print(f" Test set size: {n_test} (fixed across runs)") diff --git a/examples/conformal_eeg/tuev_conventional_conformal.py b/examples/conformal_eeg/tuev_conventional_conformal.py index fe41aaff4..e5542c7d4 100644 --- a/examples/conformal_eeg/tuev_conventional_conformal.py +++ b/examples/conformal_eeg/tuev_conventional_conformal.py @@ -283,7 +283,7 @@ def _print_multi_seed_summary( n_runs = len(all_metrics) print("\n" + "=" * 80) - print("Per-run LABEL results (fixed test set = TUH eval partition)") + print(f"Per-run results — alpha={alpha} (LABEL, fixed test set = TUH eval partition)") print("=" * 80) print(f" {'Run':<4} {'Seed':<6} {'Accuracy':<10} {'F1-Wt':<10} " f"{'Coverage':<10} {'Miscoverage':<12} {'Avg set size':<12}") @@ -295,7 +295,8 @@ def _print_multi_seed_summary( f"{m['miscoverage']:<12.4f} {m['avg_set_size']:<12.2f}") print("\n" + "=" * 80) - print(f"LABEL summary (mean \u00b1 std over {n_runs} runs, fixed test set)") + print(f"Summary — alpha={alpha} (mean \u00b1 std over {n_runs} runs, fixed test set)") + print(" Method: LABEL") print("=" * 80) print(f" Accuracy: {accs.mean():.4f} \u00b1 {accs.std():.4f}") print(f" F1 (weighted): {f1s.mean():.4f} \u00b1 {f1s.std():.4f}") @@ -332,7 +333,7 @@ def _main(args: argparse.Namespace) -> None: print("STEP 1: Load TUEV + build task dataset (shared across all seeds)") print("=" * 80) dataset = TUEVDataset(root=str(root), subset=args.subset, dev=args.quick_test) - sample_dataset = dataset.set_task(EEGEventsTUEV()) + sample_dataset = dataset.set_task(EEGEventsTUEV(normalization="95th_percentile"), num_workers=16) if args.quick_test and len(sample_dataset) > quick_test_max_samples: sample_dataset = sample_dataset.subset(range(quick_test_max_samples)) print(f"Capped to {quick_test_max_samples} samples for quick-test.") diff --git a/examples/conformal_eeg/tuev_covariate_shift_conformal.py b/examples/conformal_eeg/tuev_covariate_shift_conformal.py index 97169a0d7..d1bcba20a 100644 --- a/examples/conformal_eeg/tuev_covariate_shift_conformal.py +++ b/examples/conformal_eeg/tuev_covariate_shift_conformal.py @@ -296,7 +296,7 @@ def _print_multi_seed_summary( n_runs = len(all_metrics) print("\n" + "=" * 80) - print("Per-run CovariateLabel results (fixed test set = TUH eval partition)") + print(f"Per-run results — alpha={alpha} (CovariateLabel, fixed test set = TUH eval partition)") print("=" * 80) print(f" {'Run':<4} {'Seed':<6} {'Accuracy':<10} {'F1-Wt':<10} " f"{'Coverage':<10} {'Miscoverage':<12} {'Avg set size':<12}") @@ -308,7 +308,8 @@ def _print_multi_seed_summary( f"{m['miscoverage']:<12.4f} {m['avg_set_size']:<12.2f}") print("\n" + "=" * 80) - print(f"CovariateLabel summary (mean \u00b1 std over {n_runs} runs, fixed test set)") + print(f"Summary — alpha={alpha} (mean \u00b1 std over {n_runs} runs, fixed test set)") + print(" Method: CovariateLabel") print("=" * 80) print(f" Accuracy: {accs.mean():.4f} \u00b1 {accs.std():.4f}") print(f" F1 (weighted): {f1s.mean():.4f} \u00b1 {f1s.std():.4f}") @@ -345,7 +346,7 @@ def _main(args: argparse.Namespace) -> None: print("STEP 1: Load TUEV + build task dataset (shared across all seeds)") print("=" * 80) dataset = TUEVDataset(root=str(root), subset=args.subset, dev=args.quick_test) - sample_dataset = dataset.set_task(EEGEventsTUEV()) + sample_dataset = dataset.set_task(EEGEventsTUEV(normalization="95th_percentile"), num_workers=16) if args.quick_test and len(sample_dataset) > quick_test_max_samples: sample_dataset = sample_dataset.subset(range(quick_test_max_samples)) print(f"Capped to {quick_test_max_samples} samples for quick-test.") diff --git a/examples/conformal_eeg/tuev_kmeans_conformal.py b/examples/conformal_eeg/tuev_kmeans_conformal.py index 598ad43b1..faad50eaa 100644 --- a/examples/conformal_eeg/tuev_kmeans_conformal.py +++ b/examples/conformal_eeg/tuev_kmeans_conformal.py @@ -305,7 +305,7 @@ def _print_multi_seed_summary( n_runs = len(all_metrics) print("\n" + "=" * 80) - print("Per-run ClusterLabel results (fixed test set = TUH eval partition)") + print(f"Per-run results — alpha={alpha} (ClusterLabel, fixed test set = TUH eval partition)") print("=" * 80) print(f" {'Run':<4} {'Seed':<6} {'Accuracy':<10} {'F1-Wt':<10} " f"{'Coverage':<10} {'Miscoverage':<12} {'Avg set size':<12}") @@ -317,7 +317,8 @@ def _print_multi_seed_summary( f"{m['miscoverage']:<12.4f} {m['avg_set_size']:<12.2f}") print("\n" + "=" * 80) - print(f"ClusterLabel summary (mean \u00b1 std over {n_runs} runs, fixed test set)") + print(f"Summary — alpha={alpha} (mean \u00b1 std over {n_runs} runs, fixed test set)") + print(" Method: ClusterLabel") print("=" * 80) print(f" Accuracy: {accs.mean():.4f} \u00b1 {accs.std():.4f}") print(f" F1 (weighted): {f1s.mean():.4f} \u00b1 {f1s.std():.4f}") @@ -355,7 +356,7 @@ def _main(args: argparse.Namespace) -> None: print("STEP 1: Load TUEV + build task dataset (shared across all seeds)") print("=" * 80) dataset = TUEVDataset(root=str(root), subset=args.subset, dev=args.quick_test) - sample_dataset = dataset.set_task(EEGEventsTUEV()) + sample_dataset = dataset.set_task(EEGEventsTUEV(normalization="95th_percentile"), num_workers=16) if args.quick_test and len(sample_dataset) > quick_test_max_samples: sample_dataset = sample_dataset.subset(range(quick_test_max_samples)) print(f"Capped to {quick_test_max_samples} samples for quick-test.") diff --git a/examples/conformal_eeg/tuev_ncp_conformal.py b/examples/conformal_eeg/tuev_ncp_conformal.py index 2721656b0..77cc98475 100644 --- a/examples/conformal_eeg/tuev_ncp_conformal.py +++ b/examples/conformal_eeg/tuev_ncp_conformal.py @@ -319,7 +319,7 @@ def _run(args: argparse.Namespace) -> None: print("STEP 1: Load TUEV + build task dataset") print("=" * 80) dataset = TUEVDataset(root=str(root), subset=args.subset, dev=args.quick_test) - sample_dataset = dataset.set_task(EEGEventsTUEV()) + sample_dataset = dataset.set_task(EEGEventsTUEV(normalization="95th_percentile"), num_workers=16) if args.quick_test and len(sample_dataset) > quick_test_max_samples: sample_dataset = sample_dataset.subset(range(quick_test_max_samples)) print(f"Capped to {quick_test_max_samples} samples for quick-test.") @@ -409,7 +409,10 @@ def _run(args: argparse.Namespace) -> None: set_sizes = np.array([m["avg_set_size"] for m in mlist]) if not use_multi_seed: - print(f"\nNCP Results (alpha={alpha}):") + print("\n" + "=" * 80) + print(f"Summary — alpha={alpha} (single run, fixed test set)") + print(" Method: NeighborhoodLabel") + print("=" * 80) print(f" Accuracy: {accs[0]:.4f}") print(f" F1 (weighted): {f1s[0]:.4f}") print(f" Empirical coverage: {coverages[0]:.4f}") @@ -419,7 +422,7 @@ def _run(args: argparse.Namespace) -> None: print(f" k_neighbors: {args.k_neighbors}, lambda_L: {args.lambda_L}") else: print("\n" + "=" * 80) - print(f"Per-run NCP results — alpha={alpha} (target coverage={1-alpha:.0%})") + print(f"Per-run results — alpha={alpha} (NeighborhoodLabel, fixed test set = TUH eval partition)") print("=" * 80) print(f" {'Run':<4} {'Seed':<6} {'Accuracy':<10} {'F1-Wt':<10} " f"{'Coverage':<10} {'Miscoverage':<12} {'Avg set size':<12}") @@ -429,13 +432,14 @@ def _run(args: argparse.Namespace) -> None: f"{coverages[i]:<10.4f} {miscovs[i]:<12.4f} {set_sizes[i]:<12.2f}") print("\n" + "=" * 80) - print(f"NCP summary — alpha={alpha} (mean ± std over {n_runs} runs, fixed test set)") + print(f"Summary — alpha={alpha} (mean \u00b1 std over {n_runs} runs, fixed test set)") + print(" Method: NeighborhoodLabel") print("=" * 80) - print(f" Accuracy: {accs.mean():.4f} ± {accs.std():.4f}") - print(f" F1 (weighted): {f1s.mean():.4f} ± {f1s.std():.4f}") - print(f" Empirical coverage: {coverages.mean():.4f} ± {coverages.std():.4f}") - print(f" Empirical miscoverage: {miscovs.mean():.4f} ± {miscovs.std():.4f}") - print(f" Average set size: {set_sizes.mean():.2f} ± {set_sizes.std():.2f}") + print(f" Accuracy: {accs.mean():.4f} \u00b1 {accs.std():.4f}") + print(f" F1 (weighted): {f1s.mean():.4f} \u00b1 {f1s.std():.4f}") + print(f" Empirical coverage: {coverages.mean():.4f} \u00b1 {coverages.std():.4f}") + print(f" Empirical miscoverage: {miscovs.mean():.4f} \u00b1 {miscovs.std():.4f}") + print(f" Average set size: {set_sizes.mean():.2f} \u00b1 {set_sizes.std():.2f}") print(f" Target coverage: {1 - alpha:.0%} (alpha={alpha})") print(f" k_neighbors: {args.k_neighbors}, lambda_L: {args.lambda_L}") print(f" Test set size: {n_test} (fixed across runs)") diff --git a/pyhealth/tasks/temple_university_EEG_tasks.py b/pyhealth/tasks/temple_university_EEG_tasks.py index fc2ba702a..13e8c7206 100644 --- a/pyhealth/tasks/temple_university_EEG_tasks.py +++ b/pyhealth/tasks/temple_university_EEG_tasks.py @@ -229,7 +229,16 @@ class EEGAbnormalTUAB(BaseTask): task_name: str = "EEG_abnormal" input_schema: Dict[str, str] = {"signal": "tensor", "stft": "tensor"} - output_schema: Dict[str, str] = {"label": "binary"} + # NOTE: TUAB is a binary classification task (normal=0 vs abnormal=1), but the + # output schema is intentionally set to "multiclass" rather than "binary". + # Reason: PyHealth's conformal prediction methods (LABEL, ClusterLabel, + # NeighborhoodLabel, CovariateLabel) require multiclass mode — they calibrate + # prediction sets by thresholding a full (n, K) probability matrix, which is + # only produced by a softmax output (multiclass). Binary mode uses sigmoid and + # outputs (n, 1), which is incompatible with the CP calibration math. + # For a 2-class problem, 2-class softmax is mathematically equivalent to + # sigmoid, so there is no loss of correctness, just a different representation. + output_schema: Dict[str, str] = {"label": "multiclass"} def __init__(self, resample_rate: float = 200, diff --git a/tests/core/test_tuab.py b/tests/core/test_tuab.py index bf6e25d45..2559b13a8 100644 --- a/tests/core/test_tuab.py +++ b/tests/core/test_tuab.py @@ -531,7 +531,7 @@ def test_task_schema_attributes(self): task = EEGAbnormalTUAB() self.assertEqual(task.task_name, "EEG_abnormal") self.assertEqual(task.input_schema, {"signal": "tensor", "stft": "tensor"}) - self.assertEqual(task.output_schema, {"label": "binary"}) + self.assertEqual(task.output_schema, {"label": "multiclass"}) def test_task_schema_no_stft(self): task = EEGAbnormalTUAB(compute_stft=False) From 7d95dea733a209e514c9cbb0aeae7036f8c76370 Mon Sep 17 00:00:00 2001 From: Colton <110850842+capccode@users.noreply.github.com> Date: Tue, 14 Apr 2026 15:24:54 -0500 Subject: [PATCH 05/61] Dev/grasp full pipeline (#905) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: migrate GRASP model from PyHealth 1.0 to 2.0 API Co-Authored-By: lookman-olowo Co-Authored-By: christiana-beard Co-Authored-By: ddhangdd * feat: add GRASP mortality prediction notebook and fix cluster_num Co-Authored-By: Colton Loew Co-Authored-By: lookman-olowo Co-Authored-By: christiana-beard Co-Authored-By: ddhangdd * Restore code_mapping support in SequenceProcessor for PyHealth 2.0 Adds optional code_mapping parameter to SequenceProcessor that maps granular medical codes to grouped vocabularies (e.g. ICD9CM→CCSCM) before building the embedding table. Resolves the functional gap from the 1.x→2.0 rewrite where code_mapping was removed. Ref #535 Co-Authored-By: lookman-olowo * Add RNN baseline and code_mapping comparison notebooks for MIMIC-III Two identical notebooks for A/B testing code_mapping impact on mortality prediction. Only difference is the schema override in Step 2. Both use seed=42 for reproducible splits. Co-Authored-By: lookman-olowo * fix(tasks): extract NDC codes instead of drug names for prescription mapping event.drug returns drug names (e.g. "Aspirin") which produce zero matches in CrossMap NDC→ATC; event.ndc returns actual NDC codes enabling 3/3 feature mapping for mortality and readmission tasks. Co-Authored-By: Colton Loew Co-Authored-By: lookman-olowo Co-Authored-By: christiana-beard Co-Authored-By: ddhangdd * test(tasks): add tests verifying NDC extraction in drug tasks Checks that mortality and readmission task processors build vocabulary from NDC codes (numeric strings) rather than drug names (e.g. "Aspirin"), confirming the event.drug -> event.ndc fix works correctly. Co-Authored-By: Colton Loew Co-Authored-By: lookman-olowo Co-Authored-By: christiana-beard Co-Authored-By: ddhangdd * fix(tasks): fix missed MortalityPredictionMIMIC4 event.drug and update docs - Fix event.drug -> event.ndc in MortalityPredictionMIMIC4 (line 282) - Update readmission task docstrings to reflect NDC extraction Co-Authored-By: Colton Loew Co-Authored-By: lookman-olowo Co-Authored-By: christiana-beard Co-Authored-By: ddhangdd * fix(tasks): fix DrugRecommendationMIMIC3 to extract NDC codes DrugRecommendationMIMIC3 used prescriptions/drug (drug names) via Polars column select; changed to prescriptions/ndc to match MIMIC-4 variant and enable NDC->ATC code mapping. Co-Authored-By: Colton Loew Co-Authored-By: lookman-olowo Co-Authored-By: christiana-beard Co-Authored-By: ddhangdd * fix(models): guard RNNLayer and ConCare against zero-length sequences RNNLayer: clamp sequence lengths to min 1 so pack_padded_sequence does not crash on all-zero masks, matching TCNLayer (tcn.py:186). ConCare: guard covariance divisor with max(n-1, 1) to prevent ZeroDivisionError when attention produces single-element features. Both edge cases are triggered when code_mapping collapses vocabularies and some patients have all codes map to , producing all-zero embeddings and all-zero masks. Co-Authored-By: Colton Loew Co-Authored-By: lookman-olowo Co-Authored-By: christiana-beard Co-Authored-By: ddhangdd * docs: add docstrings to SequenceProcessor class and fit method Co-Authored-By: Colton Loew Co-Authored-By: lookman-olowo Co-Authored-By: christiana-beard Co-Authored-By: ddhangdd * docs: add docstrings, type hints, and fix test dims for GRASP module Co-Authored-By: Colton Loew Co-Authored-By: lookman-olowo Co-Authored-By: christiana-beard Co-Authored-By: ddhangdd * feat: add GRASP mortality prediction notebooks for baseline and code_mapping Baseline notebook runs GRASP with raw ICD-9/NDC codes. Code_mapping notebook collapses vocab via ICD9CM→CCSCM, ICD9PROC→CCSPROC, NDC→ATC for trainable embeddings on full MIMIC-III. Co-Authored-By: Colton Loew Co-Authored-By: lookman-olowo Co-Authored-By: christiana-beard Co-Authored-By: ddhangdd * fix(models): guard ConCare and GRASP against batch_size=1 crashes - ConCare FinalAttentionQKV: bare .squeeze() removed batch dim when batch_size=1, causing IndexError in softmax. Use .squeeze(-1) and .squeeze(1) to target only the intended dimensions. - ConCare cov(): division by zero when x.size(1)==1. Guard with max(). - GRASP grasp_encoder: remove stale torch.squeeze(hidden_t, 0) that collapsed [1, hidden] to [hidden] with batch_size=1. Both RNNLayer and ConCareLayer already return [batch, hidden]. - GRASP random_init: clamp num_centers to num_points to prevent ValueError when cluster_num > batch_size. Co-Authored-By: Colton Loew Co-Authored-By: lookman-olowo Co-Authored-By: christiana-beard Co-Authored-By: ddhangdd * feat: add GRASP mortality prediction notebooks for baseline and code_mapping Baseline notebook runs GRASP with raw ICD-9/NDC codes. Code_mapping notebook collapses vocab via ICD9CM→CCSCM, ICD9PROC→CCSPROC, NDC→ATC for trainable embeddings on full MIMIC-III. Co-Authored-By: Colton Loew Co-Authored-By: lookman-olowo Co-Authored-By: christiana-beard Co-Authored-By: ddhangdd * Add code_mapping as task __init__ argument Allow tasks to accept a code_mapping dict that upgrades input_schema entries so SequenceProcessor maps raw codes (e.g. ICD9CM) to grouped vocabularies (e.g. CCSCM) at fit/process time. This avoids manual schema manipulation after task construction. - Add code_mapping parameter to BaseTask.__init__() - Thread **kwargs + super().__init__() through all task subclasses with existing __init__ methods (4 readmission tasks, 1 multimodal mortality task) - Add 17 tests covering SequenceProcessor mapping and task-level code_mapping initialization Co-Authored-By: Colton Loew Co-Authored-By: lookman-olowo Co-Authored-By: christiana-beard Co-Authored-By: ddhangdd * Update code_mapping notebook to use task init argument Replace manual task.input_schema override with the new code_mapping parameter on MortalityPredictionMIMIC3(). Co-Authored-By: Colton Loew Co-Authored-By: lookman-olowo Co-Authored-By: christiana-beard Co-Authored-By: ddhangdd * feat(examples): add ConCare hyperparameter grid sweep script Mirrors the GRASP+ConCare mortality notebook pipeline exactly (same tables, split, seed, metrics) but sweeps 72 configurations of embedding_dim, hidden_dim, cluster_num, lr, and weight_decay. Results are logged to sweep_results.csv. Supports --root for pointing at local MIMIC-III, --code-mapping, --dev, and --monitor. Co-Authored-By: Colton Loew Co-Authored-By: lookman-olowo Co-Authored-By: christiana-beard Co-Authored-By: ddhangdd * chore(sweep): increase early stopping patience from 10 to 15 epochs Smaller ConCare configs (embedding_dim=8/16) may learn slower and need more epochs before plateauing. Co-Authored-By: Colton Loew Co-Authored-By: lookman-olowo Co-Authored-By: christiana-beard Co-Authored-By: ddhangdd * Initial plan * fix: filter falsy NDCs, guard None tokens in process(), fix NDC regex Co-Authored-By: Colton Loew Co-Authored-By: lookman-olowo Co-Authored-By: christiana-beard Co-authored-by: ddhangdd <43976109+ddhangdd@users.noreply.github.com> * refactor(sweep): rename and generalize sweep script for all backbones Rename sweep_concare_grasp.py → sweep_grasp.py. Now supports --block GRU|ConCare|LSTM with per-backbone default grids, --resume for crash recovery, --grid JSON override, auto-dated output dirs (sweep/{BLOCK}_{YYYYMMDD}_{HHMMSS}_{mapping}/), and config.json saved alongside results for reproducibility. Co-Authored-By: Colton Loew Co-Authored-By: lookman-olowo Co-Authored-By: christiana-beard Co-Authored-By: ddhangdd * test(sweep): add unit and integration tests for sweep_grasp utilities Covers grid building, combo hashing, CSV resume parsing, output directory naming, and end-to-end single-config runs for GRU and ConCare on synthetic data (13 tests, all passing). Co-Authored-By: Colton Loew Co-Authored-By: lookman-olowo Co-Authored-By: christiana-beard Co-Authored-By: ddhangdd * docs(sweep): add tmux copy-paste instructions for each paper run Co-Authored-By: Colton Loew Co-Authored-By: lookman-olowo Co-Authored-By: christiana-beard Co-Authored-By: ddhangdd * chore(examples): adds cleans examples, removes util script * Delete tests/core/test_grasp.py we removed grasp script from examples, dropped test * Revert "Delete tests/core/test_grasp.py" This reverts commit 0d957581bcbea454411ea91750f62f8696d43b85. * fix: remove orphaned sweep test, restore grasp tests * feat(grasp): add static_key support for demographic features with tests * fix(test): add valid NDC to test prescriptions so readmit test produces both labels --------- Co-authored-by: lookman-olowo Co-authored-by: christiana-beard Co-authored-by: ddhangdd Co-authored-by: Lookman Olowo <42081779+lookman-olowo@users.noreply.github.com> Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: ddhangdd <43976109+ddhangdd@users.noreply.github.com> Co-authored-by: ddhangdd Co-authored-by: Colton Loew Co-authored-by: lookman-olowo Co-authored-by: christiana-beard Co-authored-by: ddhangdd Co-authored-by: lookman-olowo --- .../mortality_mimic3_grasp.py | 29 +- ...mimic3_grasp_gru_code_mapping_cached.ipynb | 3258 ++++++++++++++ ...mortality_mimic3_rnn_baseline_cached.ipynb | 3039 +++++++++++++ ..._mimic3_rnn_with_code_mapping_cached.ipynb | 3843 +++++++++++++++++ pyhealth/models/grasp.py | 300 +- pyhealth/models/rnn.py | 9 +- pyhealth/processors/sequence_processor.py | 68 +- pyhealth/tasks/base_task.py | 37 +- pyhealth/tasks/drug_recommendation.py | 2 +- pyhealth/tasks/mortality_prediction.py | 16 +- pyhealth/tasks/readmission_prediction.py | 62 +- .../core/mimic4demo/hosp/prescriptions.csv | 6 +- tests/core/test_code_mapping.py | 313 ++ tests/core/test_concare.py | 18 + tests/core/test_drug_ndc_extraction.py | 100 + tests/core/test_grasp.py | 174 +- tests/core/test_zero_length_sequence_guard.py | 121 + 17 files changed, 11161 insertions(+), 234 deletions(-) create mode 100644 examples/mortality_prediction/mortality_mimic3_grasp_gru_code_mapping_cached.ipynb create mode 100644 examples/mortality_prediction/mortality_mimic3_rnn_baseline_cached.ipynb create mode 100644 examples/mortality_prediction/mortality_mimic3_rnn_with_code_mapping_cached.ipynb create mode 100644 tests/core/test_code_mapping.py create mode 100644 tests/core/test_drug_ndc_extraction.py create mode 100644 tests/core/test_zero_length_sequence_guard.py diff --git a/examples/mortality_prediction/mortality_mimic3_grasp.py b/examples/mortality_prediction/mortality_mimic3_grasp.py index e335373d6..c2c5c0369 100644 --- a/examples/mortality_prediction/mortality_mimic3_grasp.py +++ b/examples/mortality_prediction/mortality_mimic3_grasp.py @@ -1,37 +1,36 @@ +import tempfile + from pyhealth.datasets import MIMIC3Dataset from pyhealth.datasets import split_by_patient, get_dataloader from pyhealth.models import GRASP -from pyhealth.tasks import mortality_prediction_mimic3_fn +from pyhealth.tasks import MortalityPredictionMIMIC3 from pyhealth.trainer import Trainer if __name__ == "__main__": # STEP 1: load data base_dataset = MIMIC3Dataset( - root="/srv/local/data/physionet.org/files/mimiciii/1.4", + root="https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III", tables=["DIAGNOSES_ICD", "PROCEDURES_ICD", "PRESCRIPTIONS"], + cache_dir=tempfile.TemporaryDirectory().name, + dev=True, ) - base_dataset.stat() + base_dataset.stats() # STEP 2: set task - sample_dataset = base_dataset.set_task(mortality_prediction_mimic3_fn) - sample_dataset.stat() + task = MortalityPredictionMIMIC3() + sample_dataset = base_dataset.set_task(task) train_dataset, val_dataset, test_dataset = split_by_patient( sample_dataset, [0.8, 0.1, 0.1] ) - train_dataloader = get_dataloader(train_dataset, batch_size=256, shuffle=True) - val_dataloader = get_dataloader(val_dataset, batch_size=256, shuffle=False) - test_dataloader = get_dataloader(test_dataset, batch_size=256, shuffle=False) + train_dataloader = get_dataloader(train_dataset, batch_size=32, shuffle=True) + val_dataloader = get_dataloader(val_dataset, batch_size=32, shuffle=False) + test_dataloader = get_dataloader(test_dataset, batch_size=32, shuffle=False) # STEP 3: define model model = GRASP( dataset=sample_dataset, - feature_keys=["conditions", "procedures"], - label_key="label", - mode="binary", - use_embedding=[True, True, True], - embedding_dim=32, - hidden_dim=32, + cluster_num=2, ) # STEP 4: define trainer @@ -39,7 +38,7 @@ trainer.train( train_dataloader=train_dataloader, val_dataloader=val_dataloader, - epochs=5, + epochs=1, monitor="roc_auc", ) diff --git a/examples/mortality_prediction/mortality_mimic3_grasp_gru_code_mapping_cached.ipynb b/examples/mortality_prediction/mortality_mimic3_grasp_gru_code_mapping_cached.ipynb new file mode 100644 index 000000000..700cb8ff0 --- /dev/null +++ b/examples/mortality_prediction/mortality_mimic3_grasp_gru_code_mapping_cached.ipynb @@ -0,0 +1,3258 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# GRASP: Mortality Prediction on MIMIC-III (With code_mapping)\n", + "\n", + "This notebook runs the GRASP model for mortality prediction **with** `code_mapping` enabled.\n", + "Raw codes are mapped to grouped vocabularies before building the embedding table:\n", + "- ICD9CM → CCSCM (diagnosis codes → CCS categories)\n", + "- ICD9PROC → CCSPROC (procedure codes → CCS categories)\n", + "- NDC → ATC (drug codes → ATC categories)\n", + "\n", + "**Paper**: Liantao Ma et al. \"GRASP: Generic Framework for Health Status Representation Learning Based on Incorporating Knowledge from Similar Patients.\" AAAI 2021.\n", + "\n", + "GRASP encodes patient sequences with a backbone (ConCare, GRU, or LSTM), clusters patients via k-means, refines cluster representations with a 2-layer GCN, and blends cluster-level knowledge back into individual patient representations via a learned gating mechanism.\n", + "\n", + "**Model:** GRASP (GRU backbone + GCN cluster refinement) \n", + "**Task:** In-hospital mortality prediction \n", + "**Dataset:** Synthetic MIMIC-III (`dev=False`)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Load the MIMIC-III Dataset\n", + "\n", + "We load the MIMIC-III dataset using PyHealth's `MIMIC3Dataset` class. We use the synthetic dataset hosted on GCS, which requires no credentials.\n", + "\n", + "- `root`: URL to the synthetic MIMIC-III data\n", + "- `tables`: Clinical tables to load (diagnoses, procedures, prescriptions)\n", + "- `dev`: Set to `False` for the full dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mWARNING: Ignoring invalid distribution ~orch (/home/lolowo2/.local/lib/python3.13/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution ~umpy (/home/lolowo2/.local/lib/python3.13/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0mProcessing /home/lolowo2/git/PyHealth_full_pipeline\n", + " Installing build dependencies ... \u001b[?25ldone\n", + "\u001b[?25h Getting requirements to build wheel ... \u001b[?25ldone\n", + "\u001b[?25h Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n", + "\u001b[?25hBuilding wheels for collected packages: pyhealth\n", + " Building wheel for pyhealth (pyproject.toml) ... \u001b[?25ldone\n", + "\u001b[?25h Created wheel for pyhealth: filename=pyhealth-2.0.0-py3-none-any.whl size=602972 sha256=b80f6a6f8692914913c8d955aaf3e37bc5b227b1478bd9002a9faf130aa92dac\n", + " Stored in directory: /tmp/pip-ephem-wheel-cache-9z0d50r7/wheels/a8/55/b7/a62685e2c4f1fab8fd3203610776bd74fee9d3b37834ede4f0\n", + "Successfully built pyhealth\n", + "Installing collected packages: pyhealth\n", + " Attempting uninstall: pyhealth\n", + " Found existing installation: pyhealth 2.0.0\n", + " Uninstalling pyhealth-2.0.0:\n", + " Successfully uninstalled pyhealth-2.0.0\n", + "\u001b[33mWARNING: Ignoring invalid distribution ~orch (/home/lolowo2/.local/lib/python3.13/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution ~umpy (/home/lolowo2/.local/lib/python3.13/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0mSuccessfully installed pyhealth-2.0.0\n", + "\u001b[33mWARNING: Ignoring invalid distribution ~orch (/home/lolowo2/.local/lib/python3.13/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution ~umpy (/home/lolowo2/.local/lib/python3.13/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution ~orch (/home/lolowo2/.local/lib/python3.13/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution ~umpy (/home/lolowo2/.local/lib/python3.13/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution ~orch (/home/lolowo2/.local/lib/python3.13/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution ~umpy (/home/lolowo2/.local/lib/python3.13/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0mRequirement already satisfied: ipywidgets in /home/lolowo2/.local/lib/python3.13/site-packages (8.1.8)\n", + "Requirement already satisfied: comm>=0.1.3 in /home/lolowo2/.local/lib/python3.13/site-packages (from ipywidgets) (0.2.3)\n", + "Requirement already satisfied: ipython>=6.1.0 in /home/lolowo2/.local/lib/python3.13/site-packages (from ipywidgets) (9.10.0)\n", + "Requirement already satisfied: traitlets>=4.3.1 in /home/lolowo2/.local/lib/python3.13/site-packages (from ipywidgets) (5.14.3)\n", + "Requirement already satisfied: widgetsnbextension~=4.0.14 in /home/lolowo2/.local/lib/python3.13/site-packages (from ipywidgets) (4.0.15)\n", + "Requirement already satisfied: jupyterlab_widgets~=3.0.15 in /home/lolowo2/.local/lib/python3.13/site-packages (from ipywidgets) (3.0.16)\n", + "Requirement already satisfied: decorator>=4.3.2 in /home/lolowo2/.local/lib/python3.13/site-packages (from ipython>=6.1.0->ipywidgets) (5.2.1)\n", + "Requirement already satisfied: ipython-pygments-lexers>=1.0.0 in /home/lolowo2/.local/lib/python3.13/site-packages (from ipython>=6.1.0->ipywidgets) (1.1.1)\n", + "Requirement already satisfied: jedi>=0.18.1 in /home/lolowo2/.local/lib/python3.13/site-packages (from ipython>=6.1.0->ipywidgets) (0.19.2)\n", + "Requirement already satisfied: matplotlib-inline>=0.1.5 in /home/lolowo2/.local/lib/python3.13/site-packages (from ipython>=6.1.0->ipywidgets) (0.2.1)\n", + "Requirement already satisfied: pexpect>4.3 in /home/lolowo2/.local/lib/python3.13/site-packages (from ipython>=6.1.0->ipywidgets) (4.9.0)\n", + "Requirement already satisfied: prompt_toolkit<3.1.0,>=3.0.41 in /home/lolowo2/.local/lib/python3.13/site-packages (from ipython>=6.1.0->ipywidgets) (3.0.52)\n", + "Requirement already satisfied: pygments>=2.11.0 in /home/lolowo2/.local/lib/python3.13/site-packages (from ipython>=6.1.0->ipywidgets) (2.19.2)\n", + "Requirement already satisfied: stack_data>=0.6.0 in /home/lolowo2/.local/lib/python3.13/site-packages (from ipython>=6.1.0->ipywidgets) (0.6.3)\n", + "Requirement already satisfied: wcwidth in /home/lolowo2/.local/lib/python3.13/site-packages (from prompt_toolkit<3.1.0,>=3.0.41->ipython>=6.1.0->ipywidgets) (0.6.0)\n", + "Requirement already satisfied: parso<0.9.0,>=0.8.4 in /home/lolowo2/.local/lib/python3.13/site-packages (from jedi>=0.18.1->ipython>=6.1.0->ipywidgets) (0.8.6)\n", + "Requirement already satisfied: ptyprocess>=0.5 in /home/lolowo2/.local/lib/python3.13/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets) (0.7.0)\n", + "Requirement already satisfied: executing>=1.2.0 in /home/lolowo2/.local/lib/python3.13/site-packages (from stack_data>=0.6.0->ipython>=6.1.0->ipywidgets) (2.2.1)\n", + "Requirement already satisfied: asttokens>=2.1.0 in /home/lolowo2/.local/lib/python3.13/site-packages (from stack_data>=0.6.0->ipython>=6.1.0->ipywidgets) (3.0.1)\n", + "Requirement already satisfied: pure-eval in /home/lolowo2/.local/lib/python3.13/site-packages (from stack_data>=0.6.0->ipython>=6.1.0->ipywidgets) (0.2.3)\n", + "\u001b[33mWARNING: Ignoring invalid distribution ~orch (/home/lolowo2/.local/lib/python3.13/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution ~umpy (/home/lolowo2/.local/lib/python3.13/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution ~orch (/home/lolowo2/.local/lib/python3.13/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution ~umpy (/home/lolowo2/.local/lib/python3.13/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution ~orch (/home/lolowo2/.local/lib/python3.13/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution ~umpy (/home/lolowo2/.local/lib/python3.13/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution ~orch (/home/lolowo2/.local/lib/python3.13/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution ~umpy (/home/lolowo2/.local/lib/python3.13/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m" + ] + } + ], + "source": [ + "!pip install --user --force-reinstall --no-deps /home/lolowo2/git/PyHealth_full_pipeline\n", + "!pip install --user ipywidgets" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "No config path provided, using default config\n", + "Initializing mimic3 dataset from /home/lolowo2 (dev mode: False)\n", + "Using provided cache_dir: /tmp/tmpcxipj_37/4f338cfd-b388-50e8-9d9c-fa4872e51b6c\n", + "No cached event dataframe found. Creating: /tmp/tmpcxipj_37/4f338cfd-b388-50e8-9d9c-fa4872e51b6c/global_event_df.parquet\n", + "Scanning table: patients from /home/lolowo2/PATIENTS.csv.gz\n", + "Scanning table: admissions from /home/lolowo2/ADMISSIONS.csv.gz\n", + "Scanning table: icustays from /home/lolowo2/ICUSTAYS.csv.gz\n", + "Scanning table: diagnoses_icd from /home/lolowo2/DIAGNOSES_ICD.csv.gz\n", + "Joining with table: /home/lolowo2/ADMISSIONS.csv.gz\n", + "Scanning table: procedures_icd from /home/lolowo2/PROCEDURES_ICD.csv.gz\n", + "Joining with table: /home/lolowo2/ADMISSIONS.csv.gz\n", + "Scanning table: prescriptions from /home/lolowo2/PRESCRIPTIONS.csv.gz\n", + "Joining with table: /home/lolowo2/ADMISSIONS.csv.gz\n", + "Caching event dataframe to /tmp/tmpcxipj_37/4f338cfd-b388-50e8-9d9c-fa4872e51b6c/global_event_df.parquet...\n", + "Dataset: mimic3\n", + "Dev mode: False\n", + "Number of patients: 46520\n", + "Number of events: 5214620\n" + ] + } + ], + "source": [ + "import tempfile\n", + "\n", + "from pyhealth.datasets import MIMIC3Dataset\n", + "\n", + "base_dataset = MIMIC3Dataset(\n", + " root=\"/home/lolowo2\",\n", + " tables=[\"DIAGNOSES_ICD\", \"PROCEDURES_ICD\", \"PRESCRIPTIONS\"],\n", + " cache_dir=tempfile.TemporaryDirectory().name,\n", + " dev=False,\n", + ")\n", + "\n", + "base_dataset.stats()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Define the Mortality Prediction Task\n", + "\n", + "The `MortalityPredictionMIMIC3` task extracts samples from the raw EHR data:\n", + "- Extracts diagnosis codes (ICD-9), procedure codes, and drug information from each visit\n", + "- Creates binary labels based on in-hospital mortality\n", + "- Filters out visits without sufficient clinical codes\n", + "\n", + "We override the task's `input_schema` to enable `code_mapping` on each sequence feature.\n", + "This is the **only difference** from the baseline notebook." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Setting task MortalityPredictionMIMIC3 for mimic3 base dataset...\n", + "Task cache paths: task_df=/tmp/tmpcxipj_37/4f338cfd-b388-50e8-9d9c-fa4872e51b6c/tasks/MortalityPredictionMIMIC3_c67969dc-13b3-5ab7-977f-60956867cc5d/task_df.ld, samples=/tmp/tmpcxipj_37/4f338cfd-b388-50e8-9d9c-fa4872e51b6c/tasks/MortalityPredictionMIMIC3_c67969dc-13b3-5ab7-977f-60956867cc5d/samples_cdbbc602-34e2-5a41-8643-4c76b08829f6.ld\n", + "Applying task transformations on data with 1 workers...\n", + "Detected Jupyter notebook environment, setting num_workers to 1\n", + "Single worker mode, processing sequentially\n", + "Worker 0 started processing 46520 patients. (Polars threads: 16)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 0%| | 0/46520 [00:00, )\n", + "procedures: 204 codes (including , )\n", + "drugs: 1295 codes (including , )\n", + "\n", + "Total samples: 9583\n", + "Mortality rate: 12.05%\n", + "Positive samples: 1155\n", + "Negative samples: 8428\n" + ] + } + ], + "source": [ + "print(\"Sample structure:\")\n", + "print(samples[0])\n", + "\n", + "print(\"\\n\" + \"=\" * 50)\n", + "print(\"Processor Vocabulary Sizes:\")\n", + "print(\"=\" * 50)\n", + "for key, proc in samples.input_processors.items():\n", + " if hasattr(proc, 'code_vocab'):\n", + " print(f\"{key}: {len(proc.code_vocab)} codes (including , )\")\n", + "\n", + "mortality_count = sum(float(s.get(\"mortality\", 0)) for s in samples)\n", + "print(f\"\\nTotal samples: {len(samples)}\")\n", + "print(f\"Mortality rate: {mortality_count / len(samples) * 100:.2f}%\")\n", + "print(f\"Positive samples: {int(mortality_count)}\")\n", + "print(f\"Negative samples: {len(samples) - int(mortality_count)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Split the Dataset\n", + "\n", + "We split the data by patient to avoid data leakage — all visits from a given patient go into the same split." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Training samples: 7712\n", + "Validation samples: 909\n", + "Test samples: 962\n" + ] + } + ], + "source": [ + "from pyhealth.datasets import split_by_patient\n", + "\n", + "train_dataset, val_dataset, test_dataset = split_by_patient(\n", + " samples, [0.8, 0.1, 0.1], seed=42\n", + ")\n", + "\n", + "print(f\"Training samples: {len(train_dataset)}\")\n", + "print(f\"Validation samples: {len(val_dataset)}\")\n", + "print(f\"Test samples: {len(test_dataset)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Create Data Loaders\n", + "\n", + "Data loaders batch the samples and handle data feeding during training and evaluation." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Training batches: 241\n", + "Validation batches: 29\n", + "Test batches: 31\n" + ] + } + ], + "source": [ + "from pyhealth.datasets import get_dataloader\n", + "\n", + "train_dataloader = get_dataloader(train_dataset, batch_size=32, shuffle=True)\n", + "val_dataloader = get_dataloader(val_dataset, batch_size=32, shuffle=False)\n", + "test_dataloader = get_dataloader(test_dataset, batch_size=32, shuffle=False)\n", + "\n", + "print(f\"Training batches: {len(train_dataloader)}\")\n", + "print(f\"Validation batches: {len(val_dataloader)}\")\n", + "print(f\"Test batches: {len(test_dataloader)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Initialize the GRASP Model\n", + "\n", + "The GRASP model automatically handles different feature types via `EmbeddingModel`.\n", + "Sequence features (diagnosis/procedure/drug codes) are embedded using learned embeddings,\n", + "and each feature gets its own `GRASPLayer`.\n", + "\n", + "### Key Parameters:\n", + "- `embedding_dim`: Dimension of code embeddings (default: 128)\n", + "- `hidden_dim`: Hidden dimension of the backbone (default: 128)\n", + "- `cluster_num`: Number of patient clusters for knowledge sharing (default: 2)\n", + "- `block`: Backbone encoder — `\"ConCare\"`, `\"GRU\"`, or `\"LSTM\"` (default: `\"ConCare\"`)\n", + "- `dropout`: Dropout rate for regularization (default: 0.5)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model initialized with 213,895 parameters\n", + "\n", + "Model architecture:\n", + "GRASP(\n", + " (embedding_model): EmbeddingModel(embedding_layers=ModuleDict(\n", + " (conditions): Embedding(268, 64, padding_idx=0)\n", + " (procedures): Embedding(204, 64, padding_idx=0)\n", + " (drugs): Embedding(1295, 64, padding_idx=0)\n", + " ))\n", + " (grasp): ModuleDict(\n", + " (conditions): GRASPLayer(\n", + " (backbone): RNNLayer(\n", + " (dropout_layer): Dropout(p=0, inplace=False)\n", + " (rnn): GRU(64, 64, batch_first=True)\n", + " )\n", + " (relu): ReLU()\n", + " (tanh): Tanh()\n", + " (sigmoid): Sigmoid()\n", + " (dropout): Dropout(p=0.5, inplace=False)\n", + " (weight1): Linear(in_features=64, out_features=1, bias=True)\n", + " (weight2): Linear(in_features=64, out_features=1, bias=True)\n", + " (GCN): GraphConvolution()\n", + " (GCN_2): GraphConvolution()\n", + " (bn): BatchNorm1d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (procedures): GRASPLayer(\n", + " (backbone): RNNLayer(\n", + " (dropout_layer): Dropout(p=0, inplace=False)\n", + " (rnn): GRU(64, 64, batch_first=True)\n", + " )\n", + " (relu): ReLU()\n", + " (tanh): Tanh()\n", + " (sigmoid): Sigmoid()\n", + " (dropout): Dropout(p=0.5, inplace=False)\n", + " (weight1): Linear(in_features=64, out_features=1, bias=True)\n", + " (weight2): Linear(in_features=64, out_features=1, bias=True)\n", + " (GCN): GraphConvolution()\n", + " (GCN_2): GraphConvolution()\n", + " (bn): BatchNorm1d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (drugs): GRASPLayer(\n", + " (backbone): RNNLayer(\n", + " (dropout_layer): Dropout(p=0, inplace=False)\n", + " (rnn): GRU(64, 64, batch_first=True)\n", + " )\n", + " (relu): ReLU()\n", + " (tanh): Tanh()\n", + " (sigmoid): Sigmoid()\n", + " (dropout): Dropout(p=0.5, inplace=False)\n", + " (weight1): Linear(in_features=64, out_features=1, bias=True)\n", + " (weight2): Linear(in_features=64, out_features=1, bias=True)\n", + " (GCN): GraphConvolution()\n", + " (GCN_2): GraphConvolution()\n", + " (bn): BatchNorm1d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " )\n", + " (fc): Linear(in_features=192, out_features=1, bias=True)\n", + ")\n" + ] + } + ], + "source": [ + "from pyhealth.models import GRASP\n", + "\n", + "model = GRASP(\n", + " dataset=samples,\n", + " embedding_dim=64,\n", + " hidden_dim=64,\n", + " cluster_num=4,\n", + " block=\"GRU\",\n", + " dropout=0.5,\n", + ")\n", + "\n", + "print(f\"Model initialized with {sum(p.numel() for p in model.parameters()):,} parameters\")\n", + "print(f\"\\nModel architecture:\")\n", + "print(model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 7: Train the Model\n", + "\n", + "We use PyHealth's `Trainer` class which handles:\n", + "- Training loop with automatic batching\n", + "- Validation during training\n", + "- Model checkpointing based on validation metrics\n", + "\n", + "We monitor the **ROC-AUC** score on the validation set." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "GRASP(\n", + " (embedding_model): EmbeddingModel(embedding_layers=ModuleDict(\n", + " (conditions): Embedding(268, 64, padding_idx=0)\n", + " (procedures): Embedding(204, 64, padding_idx=0)\n", + " (drugs): Embedding(1295, 64, padding_idx=0)\n", + " ))\n", + " (grasp): ModuleDict(\n", + " (conditions): GRASPLayer(\n", + " (backbone): RNNLayer(\n", + " (dropout_layer): Dropout(p=0, inplace=False)\n", + " (rnn): GRU(64, 64, batch_first=True)\n", + " )\n", + " (relu): ReLU()\n", + " (tanh): Tanh()\n", + " (sigmoid): Sigmoid()\n", + " (dropout): Dropout(p=0.5, inplace=False)\n", + " (weight1): Linear(in_features=64, out_features=1, bias=True)\n", + " (weight2): Linear(in_features=64, out_features=1, bias=True)\n", + " (GCN): GraphConvolution()\n", + " (GCN_2): GraphConvolution()\n", + " (bn): BatchNorm1d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (procedures): GRASPLayer(\n", + " (backbone): RNNLayer(\n", + " (dropout_layer): Dropout(p=0, inplace=False)\n", + " (rnn): GRU(64, 64, batch_first=True)\n", + " )\n", + " (relu): ReLU()\n", + " (tanh): Tanh()\n", + " (sigmoid): Sigmoid()\n", + " (dropout): Dropout(p=0.5, inplace=False)\n", + " (weight1): Linear(in_features=64, out_features=1, bias=True)\n", + " (weight2): Linear(in_features=64, out_features=1, bias=True)\n", + " (GCN): GraphConvolution()\n", + " (GCN_2): GraphConvolution()\n", + " (bn): BatchNorm1d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (drugs): GRASPLayer(\n", + " (backbone): RNNLayer(\n", + " (dropout_layer): Dropout(p=0, inplace=False)\n", + " (rnn): GRU(64, 64, batch_first=True)\n", + " )\n", + " (relu): ReLU()\n", + " (tanh): Tanh()\n", + " (sigmoid): Sigmoid()\n", + " (dropout): Dropout(p=0.5, inplace=False)\n", + " (weight1): Linear(in_features=64, out_features=1, bias=True)\n", + " (weight2): Linear(in_features=64, out_features=1, bias=True)\n", + " (GCN): GraphConvolution()\n", + " (GCN_2): GraphConvolution()\n", + " (bn): BatchNorm1d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " )\n", + " (fc): Linear(in_features=192, out_features=1, bias=True)\n", + ")\n", + "Metrics: ['roc_auc', 'pr_auc', 'accuracy', 'f1']\n", + "Device: cuda\n", + "\n", + "Training:\n", + "Batch size: 32\n", + "Optimizer: \n", + "Optimizer params: {'lr': 0.0005}\n", + "Weight decay: 0.0\n", + "Max grad norm: None\n", + "Val dataloader: \n", + "Monitor: pr_auc\n", + "Monitor criterion: max\n", + "Epochs: 50\n", + "Patience: None\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c6f58fef76a347788b9477bb99bbb7a9", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Epoch 0 / 50: 0%| | 0/241 [00:00 0.5 else 'Survival'}\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.11" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/mortality_prediction/mortality_mimic3_rnn_baseline_cached.ipynb b/examples/mortality_prediction/mortality_mimic3_rnn_baseline_cached.ipynb new file mode 100644 index 000000000..686cfaeb0 --- /dev/null +++ b/examples/mortality_prediction/mortality_mimic3_rnn_baseline_cached.ipynb @@ -0,0 +1,3039 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# RNN: Mortality Prediction on MIMIC-III (Baseline — No code_mapping)\n", + "\n", + "This notebook runs the RNN model for mortality prediction **without** `code_mapping`.\n", + "Raw ICD-9 and NDC codes are used as-is for the embedding vocabulary.\n", + "\n", + "**Model:** RNN (GRU) \n", + "**Task:** In-hospital mortality prediction \n", + "**Dataset:** Synthetic MIMIC-III (`dev=False`)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Load the MIMIC-III Dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting ipywidgets\n", + " Using cached ipywidgets-8.1.8-py3-none-any.whl.metadata (2.4 kB)\n", + "Requirement already satisfied: comm>=0.1.3 in /opt/conda/lib/python3.13/site-packages (from ipywidgets) (0.2.3)\n", + "Requirement already satisfied: ipython>=6.1.0 in /opt/conda/lib/python3.13/site-packages (from ipywidgets) (9.8.0)\n", + "Requirement already satisfied: traitlets>=4.3.1 in /opt/conda/lib/python3.13/site-packages (from ipywidgets) (5.14.3)\n", + "Collecting widgetsnbextension~=4.0.14 (from ipywidgets)\n", + " Using cached widgetsnbextension-4.0.15-py3-none-any.whl.metadata (1.6 kB)\n", + "Collecting jupyterlab_widgets~=3.0.15 (from ipywidgets)\n", + " Using cached jupyterlab_widgets-3.0.16-py3-none-any.whl.metadata (20 kB)\n", + "Requirement already satisfied: decorator>=4.3.2 in /opt/conda/lib/python3.13/site-packages (from ipython>=6.1.0->ipywidgets) (5.2.1)\n", + "Requirement already satisfied: ipython-pygments-lexers>=1.0.0 in /opt/conda/lib/python3.13/site-packages (from ipython>=6.1.0->ipywidgets) (1.1.1)\n", + "Requirement already satisfied: jedi>=0.18.1 in /opt/conda/lib/python3.13/site-packages (from ipython>=6.1.0->ipywidgets) (0.19.2)\n", + "Requirement already satisfied: matplotlib-inline>=0.1.5 in /opt/conda/lib/python3.13/site-packages (from ipython>=6.1.0->ipywidgets) (0.2.1)\n", + "Requirement already satisfied: pexpect>4.3 in /opt/conda/lib/python3.13/site-packages (from ipython>=6.1.0->ipywidgets) (4.9.0)\n", + "Requirement already satisfied: prompt_toolkit<3.1.0,>=3.0.41 in /opt/conda/lib/python3.13/site-packages (from ipython>=6.1.0->ipywidgets) (3.0.52)\n", + "Requirement already satisfied: pygments>=2.11.0 in /opt/conda/lib/python3.13/site-packages (from ipython>=6.1.0->ipywidgets) (2.19.2)\n", + "Requirement already satisfied: stack_data>=0.6.0 in /opt/conda/lib/python3.13/site-packages (from ipython>=6.1.0->ipywidgets) (0.6.3)\n", + "Requirement already satisfied: wcwidth in /opt/conda/lib/python3.13/site-packages (from prompt_toolkit<3.1.0,>=3.0.41->ipython>=6.1.0->ipywidgets) (0.2.14)\n", + "Requirement already satisfied: parso<0.9.0,>=0.8.4 in /opt/conda/lib/python3.13/site-packages (from jedi>=0.18.1->ipython>=6.1.0->ipywidgets) (0.8.5)\n", + "Requirement already satisfied: ptyprocess>=0.5 in /opt/conda/lib/python3.13/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets) (0.7.0)\n", + "Requirement already satisfied: executing>=1.2.0 in /opt/conda/lib/python3.13/site-packages (from stack_data>=0.6.0->ipython>=6.1.0->ipywidgets) (2.2.1)\n", + "Requirement already satisfied: asttokens>=2.1.0 in /opt/conda/lib/python3.13/site-packages (from stack_data>=0.6.0->ipython>=6.1.0->ipywidgets) (3.0.1)\n", + "Requirement already satisfied: pure_eval in /opt/conda/lib/python3.13/site-packages (from stack_data>=0.6.0->ipython>=6.1.0->ipywidgets) (0.2.3)\n", + "Using cached ipywidgets-8.1.8-py3-none-any.whl (139 kB)\n", + "Using cached jupyterlab_widgets-3.0.16-py3-none-any.whl (914 kB)\n", + "Using cached widgetsnbextension-4.0.15-py3-none-any.whl (2.2 MB)\n", + "Installing collected packages: widgetsnbextension, jupyterlab_widgets, ipywidgets\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3/3\u001b[0m [ipywidgets]\n", + "\u001b[1A\u001b[2KSuccessfully installed ipywidgets-8.1.8 jupyterlab_widgets-3.0.16 widgetsnbextension-4.0.15\n" + ] + } + ], + "source": [ + "# !pip install git+https://github.com/lookman-olowo/PyHealth.git@refactor/grasp-model\n", + "!pip install ipywidgets\n", + "\n", + "# ! pip uninstall pyhealth" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "No config path provided, using default config\n", + "Initializing mimic3 dataset from /home/lolowo2 (dev mode: False)\n", + "Using provided cache_dir: /tmp/tmphrbg7b9w/4f338cfd-b388-50e8-9d9c-fa4872e51b6c\n", + "No cached event dataframe found. Creating: /tmp/tmphrbg7b9w/4f338cfd-b388-50e8-9d9c-fa4872e51b6c/global_event_df.parquet\n", + "Scanning table: patients from /home/lolowo2/PATIENTS.csv.gz\n", + "Scanning table: admissions from /home/lolowo2/ADMISSIONS.csv.gz\n", + "Scanning table: icustays from /home/lolowo2/ICUSTAYS.csv.gz\n", + "Scanning table: diagnoses_icd from /home/lolowo2/DIAGNOSES_ICD.csv.gz\n", + "Joining with table: /home/lolowo2/ADMISSIONS.csv.gz\n", + "Scanning table: procedures_icd from /home/lolowo2/PROCEDURES_ICD.csv.gz\n", + "Joining with table: /home/lolowo2/ADMISSIONS.csv.gz\n", + "Scanning table: prescriptions from /home/lolowo2/PRESCRIPTIONS.csv.gz\n", + "Joining with table: /home/lolowo2/ADMISSIONS.csv.gz\n", + "Caching event dataframe to /tmp/tmphrbg7b9w/4f338cfd-b388-50e8-9d9c-fa4872e51b6c/global_event_df.parquet...\n", + "Dataset: mimic3\n", + "Dev mode: False\n", + "Number of patients: 46520\n", + "Number of events: 5214620\n" + ] + } + ], + "source": [ + "import tempfile\n", + "\n", + "from pyhealth.datasets import MIMIC3Dataset\n", + "\n", + "base_dataset = MIMIC3Dataset(\n", + " root=\"/home/lolowo2\",\n", + " tables=[\"DIAGNOSES_ICD\", \"PROCEDURES_ICD\", \"PRESCRIPTIONS\"],\n", + " cache_dir=tempfile.TemporaryDirectory().name,\n", + " dev=False,\n", + ")\n", + "\n", + "base_dataset.stats()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Define the Mortality Prediction Task" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Setting task MortalityPredictionMIMIC3 for mimic3 base dataset...\n", + "Task cache paths: task_df=/tmp/tmphrbg7b9w/4f338cfd-b388-50e8-9d9c-fa4872e51b6c/tasks/MortalityPredictionMIMIC3_187a839c-4f67-585a-bbb3-355429e27594/task_df.ld, samples=/tmp/tmphrbg7b9w/4f338cfd-b388-50e8-9d9c-fa4872e51b6c/tasks/MortalityPredictionMIMIC3_187a839c-4f67-585a-bbb3-355429e27594/samples_cdbbc602-34e2-5a41-8643-4c76b08829f6.ld\n", + "Applying task transformations on data with 1 workers...\n", + "Detected Jupyter notebook environment, setting num_workers to 1\n", + "Single worker mode, processing sequentially\n", + "Worker 0 started processing 46520 patients. (Polars threads: 16)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 0%| | 0/46520 [00:00, )\n", + "procedures: 1304 codes (including , )\n", + "drugs: 2624 codes (including , )\n", + "\n", + "Total samples: 9583\n", + "Mortality rate: 12.05%\n", + "Positive samples: 1155\n", + "Negative samples: 8428\n" + ] + } + ], + "source": [ + "print(\"Sample structure:\")\n", + "print(samples[0])\n", + "\n", + "print(\"\\n\" + \"=\" * 50)\n", + "print(\"Processor Vocabulary Sizes:\")\n", + "print(\"=\" * 50)\n", + "for key, proc in samples.input_processors.items():\n", + " if hasattr(proc, 'code_vocab'):\n", + " print(f\"{key}: {len(proc.code_vocab)} codes (including , )\")\n", + "\n", + "mortality_count = sum(float(s.get(\"mortality\", 0)) for s in samples)\n", + "print(f\"\\nTotal samples: {len(samples)}\")\n", + "print(f\"Mortality rate: {mortality_count / len(samples) * 100:.2f}%\")\n", + "print(f\"Positive samples: {int(mortality_count)}\")\n", + "print(f\"Negative samples: {len(samples) - int(mortality_count)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Split the Dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Training samples: 7711\n", + "Validation samples: 909\n", + "Test samples: 963\n" + ] + } + ], + "source": [ + "from pyhealth.datasets import split_by_patient\n", + "\n", + "train_dataset, val_dataset, test_dataset = split_by_patient(\n", + " samples, [0.8, 0.1, 0.1], seed=42\n", + ")\n", + "\n", + "print(f\"Training samples: {len(train_dataset)}\")\n", + "print(f\"Validation samples: {len(val_dataset)}\")\n", + "print(f\"Test samples: {len(test_dataset)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Create Data Loaders" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Training batches: 241\n", + "Validation batches: 29\n", + "Test batches: 31\n" + ] + } + ], + "source": [ + "from pyhealth.datasets import get_dataloader\n", + "\n", + "train_dataloader = get_dataloader(train_dataset, batch_size=32, shuffle=True)\n", + "val_dataloader = get_dataloader(val_dataset, batch_size=32, shuffle=False)\n", + "test_dataloader = get_dataloader(test_dataset, batch_size=32, shuffle=False)\n", + "\n", + "print(f\"Training batches: {len(train_dataloader)}\")\n", + "print(f\"Validation batches: {len(val_dataloader)}\")\n", + "print(f\"Test batches: {len(test_dataloader)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Initialize the RNN Model" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model initialized with 1,325,441 parameters\n", + "\n", + "Model architecture:\n", + "RNN(\n", + " (embedding_model): EmbeddingModel(embedding_layers=ModuleDict(\n", + " (conditions): Embedding(4102, 128, padding_idx=0)\n", + " (procedures): Embedding(1304, 128, padding_idx=0)\n", + " (drugs): Embedding(2624, 128, padding_idx=0)\n", + " ))\n", + " (rnn): ModuleDict(\n", + " (conditions): RNNLayer(\n", + " (dropout_layer): Dropout(p=0.5, inplace=False)\n", + " (rnn): GRU(128, 128, batch_first=True)\n", + " )\n", + " (procedures): RNNLayer(\n", + " (dropout_layer): Dropout(p=0.5, inplace=False)\n", + " (rnn): GRU(128, 128, batch_first=True)\n", + " )\n", + " (drugs): RNNLayer(\n", + " (dropout_layer): Dropout(p=0.5, inplace=False)\n", + " (rnn): GRU(128, 128, batch_first=True)\n", + " )\n", + " )\n", + " (fc): Linear(in_features=384, out_features=1, bias=True)\n", + ")\n" + ] + } + ], + "source": [ + "from pyhealth.models import RNN\n", + "\n", + "model = RNN(\n", + " dataset=samples,\n", + " embedding_dim=128,\n", + " hidden_dim=128,\n", + ")\n", + "\n", + "print(f\"Model initialized with {sum(p.numel() for p in model.parameters()):,} parameters\")\n", + "print(f\"\\nModel architecture:\")\n", + "print(model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 7: Train the Model" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "RNN(\n", + " (embedding_model): EmbeddingModel(embedding_layers=ModuleDict(\n", + " (conditions): Embedding(4102, 128, padding_idx=0)\n", + " (procedures): Embedding(1304, 128, padding_idx=0)\n", + " (drugs): Embedding(2624, 128, padding_idx=0)\n", + " ))\n", + " (rnn): ModuleDict(\n", + " (conditions): RNNLayer(\n", + " (dropout_layer): Dropout(p=0.5, inplace=False)\n", + " (rnn): GRU(128, 128, batch_first=True)\n", + " )\n", + " (procedures): RNNLayer(\n", + " (dropout_layer): Dropout(p=0.5, inplace=False)\n", + " (rnn): GRU(128, 128, batch_first=True)\n", + " )\n", + " (drugs): RNNLayer(\n", + " (dropout_layer): Dropout(p=0.5, inplace=False)\n", + " (rnn): GRU(128, 128, batch_first=True)\n", + " )\n", + " )\n", + " (fc): Linear(in_features=384, out_features=1, bias=True)\n", + ")\n", + "Metrics: ['roc_auc', 'pr_auc', 'accuracy', 'f1']\n", + "Device: cuda\n", + "\n", + "Training:\n", + "Batch size: 32\n", + "Optimizer: \n", + "Optimizer params: {'lr': 0.001}\n", + "Weight decay: 0.0\n", + "Max grad norm: None\n", + "Val dataloader: \n", + "Monitor: roc_auc\n", + "Monitor criterion: max\n", + "Epochs: 50\n", + "Patience: None\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "00c0ee325e7a4e9e89e7c901e69d37c9", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Epoch 0 / 50: 0%| | 0/241 [00:00pyhealth==2.0.0)\n", + " Using cached dask-2025.11.0-py3-none-any.whl.metadata (3.8 kB)\n", + "Collecting einops>=0.8.0 (from pyhealth==2.0.0)\n", + " Using cached einops-0.8.2-py3-none-any.whl.metadata (13 kB)\n", + "Collecting linear-attention-transformer>=0.19.1 (from pyhealth==2.0.0)\n", + " Using cached linear_attention_transformer-0.19.1-py3-none-any.whl.metadata (787 bytes)\n", + "Collecting litdata~=0.2.59 (from pyhealth==2.0.0)\n", + " Using cached litdata-0.2.61-py3-none-any.whl.metadata (69 kB)\n", + "Collecting mne~=1.10.0 (from pyhealth==2.0.0)\n", + " Using cached mne-1.10.2-py3-none-any.whl.metadata (21 kB)\n", + "Collecting more-itertools~=10.8.0 (from pyhealth==2.0.0)\n", + " Using cached more_itertools-10.8.0-py3-none-any.whl.metadata (39 kB)\n", + "Collecting narwhals~=2.13.0 (from pyhealth==2.0.0)\n", + " Using cached narwhals-2.13.0-py3-none-any.whl.metadata (12 kB)\n", + "Collecting networkx (from pyhealth==2.0.0)\n", + " Using cached networkx-3.6.1-py3-none-any.whl.metadata (6.8 kB)\n", + "Collecting numpy~=2.2.0 (from pyhealth==2.0.0)\n", + " Using cached numpy-2.2.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (62 kB)\n", + "Collecting ogb>=1.3.5 (from pyhealth==2.0.0)\n", + " Using cached ogb-1.3.6-py3-none-any.whl.metadata (6.2 kB)\n", + "Collecting pandas~=2.3.1 (from pyhealth==2.0.0)\n", + " Using cached pandas-2.3.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl.metadata (91 kB)\n", + "Collecting peft (from pyhealth==2.0.0)\n", + " Using cached peft-0.18.1-py3-none-any.whl.metadata (14 kB)\n", + "Collecting polars~=1.35.2 (from pyhealth==2.0.0)\n", + " Using cached polars-1.35.2-py3-none-any.whl.metadata (10 kB)\n", + "Collecting pyarrow~=22.0.0 (from pyhealth==2.0.0)\n", + " Using cached pyarrow-22.0.0-cp313-cp313-manylinux_2_28_x86_64.whl.metadata (3.2 kB)\n", + "Collecting pydantic~=2.11.7 (from pyhealth==2.0.0)\n", + " Using cached pydantic-2.11.10-py3-none-any.whl.metadata (68 kB)\n", + "Collecting rdkit (from pyhealth==2.0.0)\n", + " Using cached rdkit-2025.9.5-cp313-cp313-manylinux_2_28_x86_64.whl.metadata (3.8 kB)\n", + "Collecting scikit-learn~=1.7.0 (from pyhealth==2.0.0)\n", + " Using cached scikit_learn-1.7.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (11 kB)\n", + "Collecting torchvision (from pyhealth==2.0.0)\n", + " Using cached torchvision-0.25.0-cp313-cp313-manylinux_2_28_x86_64.whl.metadata (5.4 kB)\n", + "Collecting torch~=2.7.1 (from pyhealth==2.0.0)\n", + " Using cached torch-2.7.1-cp313-cp313-manylinux_2_28_x86_64.whl.metadata (29 kB)\n", + "Collecting tqdm (from pyhealth==2.0.0)\n", + " Using cached tqdm-4.67.3-py3-none-any.whl.metadata (57 kB)\n", + "Collecting transformers~=4.53.2 (from pyhealth==2.0.0)\n", + " Using cached transformers-4.53.3-py3-none-any.whl.metadata (40 kB)\n", + "Collecting urllib3~=2.5.0 (from pyhealth==2.0.0)\n", + " Using cached urllib3-2.5.0-py3-none-any.whl.metadata (6.5 kB)\n", + "Collecting click>=8.1 (from dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth==2.0.0)\n", + " Using cached click-8.3.1-py3-none-any.whl.metadata (2.6 kB)\n", + "Collecting cloudpickle>=3.0.0 (from dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth==2.0.0)\n", + " Using cached cloudpickle-3.1.2-py3-none-any.whl.metadata (7.1 kB)\n", + "Collecting fsspec>=2021.09.0 (from dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth==2.0.0)\n", + " Using cached fsspec-2026.2.0-py3-none-any.whl.metadata (10 kB)\n", + "Collecting packaging>=20.0 (from dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth==2.0.0)\n", + " Using cached packaging-26.0-py3-none-any.whl.metadata (3.3 kB)\n", + "Collecting partd>=1.4.0 (from dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth==2.0.0)\n", + " Using cached partd-1.4.2-py3-none-any.whl.metadata (4.6 kB)\n", + "Collecting pyyaml>=5.3.1 (from dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth==2.0.0)\n", + " Using cached pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (2.4 kB)\n", + "Collecting toolz>=0.10.0 (from dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth==2.0.0)\n", + " Using cached toolz-1.1.0-py3-none-any.whl.metadata (5.1 kB)\n", + "Collecting lz4>=4.3.2 (from dask[complete]~=2025.11.0->pyhealth==2.0.0)\n", + " Using cached lz4-4.4.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (3.8 kB)\n", + "Collecting lightning-utilities (from litdata~=0.2.59->pyhealth==2.0.0)\n", + " Using cached lightning_utilities-0.15.2-py3-none-any.whl.metadata (5.7 kB)\n", + "Collecting filelock (from litdata~=0.2.59->pyhealth==2.0.0)\n", + " Using cached filelock-3.24.3-py3-none-any.whl.metadata (2.0 kB)\n", + "Collecting boto3 (from litdata~=0.2.59->pyhealth==2.0.0)\n", + " Using cached boto3-1.42.53-py3-none-any.whl.metadata (6.7 kB)\n", + "Collecting requests (from litdata~=0.2.59->pyhealth==2.0.0)\n", + " Using cached requests-2.32.5-py3-none-any.whl.metadata (4.9 kB)\n", + "Collecting tifffile (from litdata~=0.2.59->pyhealth==2.0.0)\n", + " Downloading tifffile-2026.2.20-py3-none-any.whl.metadata (30 kB)\n", + "Collecting obstore (from litdata~=0.2.59->pyhealth==2.0.0)\n", + " Using cached obstore-0.8.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (840 bytes)\n", + "Collecting decorator (from mne~=1.10.0->pyhealth==2.0.0)\n", + " Using cached decorator-5.2.1-py3-none-any.whl.metadata (3.9 kB)\n", + "Collecting jinja2 (from mne~=1.10.0->pyhealth==2.0.0)\n", + " Using cached jinja2-3.1.6-py3-none-any.whl.metadata (2.9 kB)\n", + "Collecting lazy-loader>=0.3 (from mne~=1.10.0->pyhealth==2.0.0)\n", + " Using cached lazy_loader-0.4-py3-none-any.whl.metadata (7.6 kB)\n", + "Collecting matplotlib>=3.7 (from mne~=1.10.0->pyhealth==2.0.0)\n", + " Using cached matplotlib-3.10.8-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (52 kB)\n", + "Collecting pooch>=1.5 (from mne~=1.10.0->pyhealth==2.0.0)\n", + " Using cached pooch-1.9.0-py3-none-any.whl.metadata (10 kB)\n", + "Collecting scipy>=1.11 (from mne~=1.10.0->pyhealth==2.0.0)\n", + " Using cached scipy-1.17.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl.metadata (62 kB)\n", + "Collecting python-dateutil>=2.8.2 (from pandas~=2.3.1->pyhealth==2.0.0)\n", + " Using cached python_dateutil-2.9.0.post0-py2.py3-none-any.whl.metadata (8.4 kB)\n", + "Collecting pytz>=2020.1 (from pandas~=2.3.1->pyhealth==2.0.0)\n", + " Using cached pytz-2025.2-py2.py3-none-any.whl.metadata (22 kB)\n", + "Collecting tzdata>=2022.7 (from pandas~=2.3.1->pyhealth==2.0.0)\n", + " Using cached tzdata-2025.3-py2.py3-none-any.whl.metadata (1.4 kB)\n", + "Collecting polars-runtime-32==1.35.2 (from polars~=1.35.2->pyhealth==2.0.0)\n", + " Using cached polars_runtime_32-1.35.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (1.5 kB)\n", + "Collecting annotated-types>=0.6.0 (from pydantic~=2.11.7->pyhealth==2.0.0)\n", + " Using cached annotated_types-0.7.0-py3-none-any.whl.metadata (15 kB)\n", + "Collecting pydantic-core==2.33.2 (from pydantic~=2.11.7->pyhealth==2.0.0)\n", + " Using cached pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.8 kB)\n", + "Collecting typing-extensions>=4.12.2 (from pydantic~=2.11.7->pyhealth==2.0.0)\n", + " Using cached typing_extensions-4.15.0-py3-none-any.whl.metadata (3.3 kB)\n", + "Collecting typing-inspection>=0.4.0 (from pydantic~=2.11.7->pyhealth==2.0.0)\n", + " Using cached typing_inspection-0.4.2-py3-none-any.whl.metadata (2.6 kB)\n", + "Collecting joblib>=1.2.0 (from scikit-learn~=1.7.0->pyhealth==2.0.0)\n", + " Using cached joblib-1.5.3-py3-none-any.whl.metadata (5.5 kB)\n", + "Collecting threadpoolctl>=3.1.0 (from scikit-learn~=1.7.0->pyhealth==2.0.0)\n", + " Using cached threadpoolctl-3.6.0-py3-none-any.whl.metadata (13 kB)\n", + "Collecting setuptools (from torch~=2.7.1->pyhealth==2.0.0)\n", + " Using cached setuptools-82.0.0-py3-none-any.whl.metadata (6.6 kB)\n", + "Collecting sympy>=1.13.3 (from torch~=2.7.1->pyhealth==2.0.0)\n", + " Using cached sympy-1.14.0-py3-none-any.whl.metadata (12 kB)\n", + "Collecting nvidia-cuda-nvrtc-cu12==12.6.77 (from torch~=2.7.1->pyhealth==2.0.0)\n", + " Using cached nvidia_cuda_nvrtc_cu12-12.6.77-py3-none-manylinux2014_x86_64.whl.metadata (1.5 kB)\n", + "Collecting nvidia-cuda-runtime-cu12==12.6.77 (from torch~=2.7.1->pyhealth==2.0.0)\n", + " Using cached nvidia_cuda_runtime_cu12-12.6.77-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (1.5 kB)\n", + "Collecting nvidia-cuda-cupti-cu12==12.6.80 (from torch~=2.7.1->pyhealth==2.0.0)\n", + " Using cached nvidia_cuda_cupti_cu12-12.6.80-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (1.6 kB)\n", + "Collecting nvidia-cudnn-cu12==9.5.1.17 (from torch~=2.7.1->pyhealth==2.0.0)\n", + " Using cached nvidia_cudnn_cu12-9.5.1.17-py3-none-manylinux_2_28_x86_64.whl.metadata (1.6 kB)\n", + "Collecting nvidia-cublas-cu12==12.6.4.1 (from torch~=2.7.1->pyhealth==2.0.0)\n", + " Using cached nvidia_cublas_cu12-12.6.4.1-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (1.5 kB)\n", + "Collecting nvidia-cufft-cu12==11.3.0.4 (from torch~=2.7.1->pyhealth==2.0.0)\n", + " Using cached nvidia_cufft_cu12-11.3.0.4-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (1.5 kB)\n", + "Collecting nvidia-curand-cu12==10.3.7.77 (from torch~=2.7.1->pyhealth==2.0.0)\n", + " Using cached nvidia_curand_cu12-10.3.7.77-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (1.5 kB)\n", + "Collecting nvidia-cusolver-cu12==11.7.1.2 (from torch~=2.7.1->pyhealth==2.0.0)\n", + " Using cached nvidia_cusolver_cu12-11.7.1.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (1.6 kB)\n", + "Collecting nvidia-cusparse-cu12==12.5.4.2 (from torch~=2.7.1->pyhealth==2.0.0)\n", + " Using cached nvidia_cusparse_cu12-12.5.4.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (1.6 kB)\n", + "Collecting nvidia-cusparselt-cu12==0.6.3 (from torch~=2.7.1->pyhealth==2.0.0)\n", + " Using cached nvidia_cusparselt_cu12-0.6.3-py3-none-manylinux2014_x86_64.whl.metadata (6.8 kB)\n", + "Collecting nvidia-nccl-cu12==2.26.2 (from torch~=2.7.1->pyhealth==2.0.0)\n", + " Using cached nvidia_nccl_cu12-2.26.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (2.0 kB)\n", + "Collecting nvidia-nvtx-cu12==12.6.77 (from torch~=2.7.1->pyhealth==2.0.0)\n", + " Using cached nvidia_nvtx_cu12-12.6.77-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (1.6 kB)\n", + "Collecting nvidia-nvjitlink-cu12==12.6.85 (from torch~=2.7.1->pyhealth==2.0.0)\n", + " Using cached nvidia_nvjitlink_cu12-12.6.85-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl.metadata (1.5 kB)\n", + "Collecting nvidia-cufile-cu12==1.11.1.6 (from torch~=2.7.1->pyhealth==2.0.0)\n", + " Using cached nvidia_cufile_cu12-1.11.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (1.5 kB)\n", + "Collecting triton==3.3.1 (from torch~=2.7.1->pyhealth==2.0.0)\n", + " Using cached triton-3.3.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl.metadata (1.5 kB)\n", + "Collecting huggingface-hub<1.0,>=0.30.0 (from transformers~=4.53.2->pyhealth==2.0.0)\n", + " Using cached huggingface_hub-0.36.2-py3-none-any.whl.metadata (15 kB)\n", + "Collecting regex!=2019.12.17 (from transformers~=4.53.2->pyhealth==2.0.0)\n", + " Using cached regex-2026.2.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (40 kB)\n", + "Collecting tokenizers<0.22,>=0.21 (from transformers~=4.53.2->pyhealth==2.0.0)\n", + " Using cached tokenizers-0.21.4-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.7 kB)\n", + "Collecting safetensors>=0.4.3 (from transformers~=4.53.2->pyhealth==2.0.0)\n", + " Using cached safetensors-0.7.0-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.1 kB)\n", + "Collecting hf-xet<2.0.0,>=1.1.3 (from huggingface-hub<1.0,>=0.30.0->transformers~=4.53.2->pyhealth==2.0.0)\n", + " Using cached hf_xet-1.2.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.9 kB)\n", + "Collecting axial-positional-embedding (from linear-attention-transformer>=0.19.1->pyhealth==2.0.0)\n", + " Using cached axial_positional_embedding-0.3.12-py3-none-any.whl.metadata (4.3 kB)\n", + "Collecting linformer>=0.1.0 (from linear-attention-transformer>=0.19.1->pyhealth==2.0.0)\n", + " Using cached linformer-0.2.3-py3-none-any.whl.metadata (602 bytes)\n", + "Collecting local-attention (from linear-attention-transformer>=0.19.1->pyhealth==2.0.0)\n", + " Using cached local_attention-1.11.2-py3-none-any.whl.metadata (929 bytes)\n", + "Collecting product-key-memory>=0.1.5 (from linear-attention-transformer>=0.19.1->pyhealth==2.0.0)\n", + " Using cached product_key_memory-0.3.0-py3-none-any.whl.metadata (4.9 kB)\n", + "Collecting contourpy>=1.0.1 (from matplotlib>=3.7->mne~=1.10.0->pyhealth==2.0.0)\n", + " Using cached contourpy-1.3.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl.metadata (5.5 kB)\n", + "Collecting cycler>=0.10 (from matplotlib>=3.7->mne~=1.10.0->pyhealth==2.0.0)\n", + " Using cached cycler-0.12.1-py3-none-any.whl.metadata (3.8 kB)\n", + "Collecting fonttools>=4.22.0 (from matplotlib>=3.7->mne~=1.10.0->pyhealth==2.0.0)\n", + " Using cached fonttools-4.61.1-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl.metadata (114 kB)\n", + "Collecting kiwisolver>=1.3.1 (from matplotlib>=3.7->mne~=1.10.0->pyhealth==2.0.0)\n", + " Using cached kiwisolver-1.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (6.3 kB)\n", + "Collecting pillow>=8 (from matplotlib>=3.7->mne~=1.10.0->pyhealth==2.0.0)\n", + " Using cached pillow-12.1.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl.metadata (8.8 kB)\n", + "Collecting pyparsing>=3 (from matplotlib>=3.7->mne~=1.10.0->pyhealth==2.0.0)\n", + " Using cached pyparsing-3.3.2-py3-none-any.whl.metadata (5.8 kB)\n", + "Collecting six>=1.12.0 (from ogb>=1.3.5->pyhealth==2.0.0)\n", + " Using cached six-1.17.0-py2.py3-none-any.whl.metadata (1.7 kB)\n", + "Collecting outdated>=0.2.0 (from ogb>=1.3.5->pyhealth==2.0.0)\n", + " Using cached outdated-0.2.2-py2.py3-none-any.whl.metadata (4.7 kB)\n", + "Collecting littleutils (from outdated>=0.2.0->ogb>=1.3.5->pyhealth==2.0.0)\n", + " Using cached littleutils-0.2.4-py3-none-any.whl.metadata (679 bytes)\n", + "Collecting locket (from partd>=1.4.0->dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth==2.0.0)\n", + " Using cached locket-1.0.0-py2.py3-none-any.whl.metadata (2.8 kB)\n", + "Collecting platformdirs>=2.5.0 (from pooch>=1.5->mne~=1.10.0->pyhealth==2.0.0)\n", + " Using cached platformdirs-4.9.2-py3-none-any.whl.metadata (4.7 kB)\n", + "Collecting colt5-attention>=0.10.14 (from product-key-memory>=0.1.5->linear-attention-transformer>=0.19.1->pyhealth==2.0.0)\n", + " Using cached CoLT5_attention-0.11.1-py3-none-any.whl.metadata (737 bytes)\n", + "Collecting hyper-connections>=0.1.8 (from local-attention->linear-attention-transformer>=0.19.1->pyhealth==2.0.0)\n", + " Using cached hyper_connections-0.4.9-py3-none-any.whl.metadata (6.7 kB)\n", + "Collecting torch-einops-utils>=0.0.20 (from hyper-connections>=0.1.8->local-attention->linear-attention-transformer>=0.19.1->pyhealth==2.0.0)\n", + " Using cached torch_einops_utils-0.0.30-py3-none-any.whl.metadata (2.1 kB)\n", + "Collecting charset_normalizer<4,>=2 (from requests->litdata~=0.2.59->pyhealth==2.0.0)\n", + " Using cached charset_normalizer-3.4.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (37 kB)\n", + "Collecting idna<4,>=2.5 (from requests->litdata~=0.2.59->pyhealth==2.0.0)\n", + " Using cached idna-3.11-py3-none-any.whl.metadata (8.4 kB)\n", + "Collecting certifi>=2017.4.17 (from requests->litdata~=0.2.59->pyhealth==2.0.0)\n", + " Using cached certifi-2026.1.4-py3-none-any.whl.metadata (2.5 kB)\n", + "Collecting mpmath<1.4,>=1.1.0 (from sympy>=1.13.3->torch~=2.7.1->pyhealth==2.0.0)\n", + " Using cached mpmath-1.3.0-py3-none-any.whl.metadata (8.6 kB)\n", + "Collecting psutil (from accelerate->pyhealth==2.0.0)\n", + " Using cached psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl.metadata (22 kB)\n", + "Collecting botocore<1.43.0,>=1.42.53 (from boto3->litdata~=0.2.59->pyhealth==2.0.0)\n", + " Using cached botocore-1.42.53-py3-none-any.whl.metadata (5.9 kB)\n", + "Collecting jmespath<2.0.0,>=0.7.1 (from boto3->litdata~=0.2.59->pyhealth==2.0.0)\n", + " Using cached jmespath-1.1.0-py3-none-any.whl.metadata (7.6 kB)\n", + "Collecting s3transfer<0.17.0,>=0.16.0 (from boto3->litdata~=0.2.59->pyhealth==2.0.0)\n", + " Using cached s3transfer-0.16.0-py3-none-any.whl.metadata (1.7 kB)\n", + "Collecting distributed==2025.11.0 (from dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth==2.0.0)\n", + " Using cached distributed-2025.11.0-py3-none-any.whl.metadata (3.4 kB)\n", + "Collecting bokeh>=3.1.0 (from dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth==2.0.0)\n", + " Using cached bokeh-3.8.2-py3-none-any.whl.metadata (10 kB)\n", + "Collecting msgpack>=1.0.2 (from distributed==2025.11.0->dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth==2.0.0)\n", + " Using cached msgpack-1.1.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (8.1 kB)\n", + "Collecting sortedcontainers>=2.0.5 (from distributed==2025.11.0->dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth==2.0.0)\n", + " Using cached sortedcontainers-2.4.0-py2.py3-none-any.whl.metadata (10 kB)\n", + "Collecting tblib>=1.6.0 (from distributed==2025.11.0->dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth==2.0.0)\n", + " Using cached tblib-3.2.2-py3-none-any.whl.metadata (27 kB)\n", + "Collecting tornado>=6.2.0 (from distributed==2025.11.0->dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth==2.0.0)\n", + " Using cached tornado-6.5.4-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (2.8 kB)\n", + "Collecting zict>=3.0.0 (from distributed==2025.11.0->dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth==2.0.0)\n", + " Using cached zict-3.0.0-py2.py3-none-any.whl.metadata (899 bytes)\n", + "Collecting xyzservices>=2021.09.1 (from bokeh>=3.1.0->dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth==2.0.0)\n", + " Using cached xyzservices-2025.11.0-py3-none-any.whl.metadata (4.3 kB)\n", + "Collecting MarkupSafe>=2.0 (from jinja2->mne~=1.10.0->pyhealth==2.0.0)\n", + " Using cached markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (2.7 kB)\n", + "INFO: pip is looking at multiple versions of torchvision to determine which version is compatible with other requirements. This could take a while.\n", + "Collecting torchvision (from pyhealth==2.0.0)\n", + " Using cached torchvision-0.24.1-cp313-cp313-manylinux_2_28_x86_64.whl.metadata (5.9 kB)\n", + " Using cached torchvision-0.24.0-cp313-cp313-manylinux_2_28_x86_64.whl.metadata (5.9 kB)\n", + " Using cached torchvision-0.23.0-cp313-cp313-manylinux_2_28_x86_64.whl.metadata (6.1 kB)\n", + " Using cached torchvision-0.22.1-cp313-cp313-manylinux_2_28_x86_64.whl.metadata (6.1 kB)\n", + "Using cached dask-2025.11.0-py3-none-any.whl (1.5 MB)\n", + "Using cached litdata-0.2.61-py3-none-any.whl (205 kB)\n", + "Using cached mne-1.10.2-py3-none-any.whl (7.4 MB)\n", + "Using cached more_itertools-10.8.0-py3-none-any.whl (69 kB)\n", + "Using cached narwhals-2.13.0-py3-none-any.whl (426 kB)\n", + "Using cached numpy-2.2.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (16.5 MB)\n", + "Using cached pandas-2.3.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl (12.3 MB)\n", + "Using cached polars-1.35.2-py3-none-any.whl (783 kB)\n", + "Using cached polars_runtime_32-1.35.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (41.3 MB)\n", + "Using cached pyarrow-22.0.0-cp313-cp313-manylinux_2_28_x86_64.whl (47.7 MB)\n", + "Using cached pydantic-2.11.10-py3-none-any.whl (444 kB)\n", + "Using cached pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.0 MB)\n", + "Using cached scikit_learn-1.7.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (9.4 MB)\n", + "Using cached torch-2.7.1-cp313-cp313-manylinux_2_28_x86_64.whl (821.0 MB)\n", + "Using cached nvidia_cublas_cu12-12.6.4.1-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (393.1 MB)\n", + "Using cached nvidia_cuda_cupti_cu12-12.6.80-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (8.9 MB)\n", + "Using cached nvidia_cuda_nvrtc_cu12-12.6.77-py3-none-manylinux2014_x86_64.whl (23.7 MB)\n", + "Using cached nvidia_cuda_runtime_cu12-12.6.77-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (897 kB)\n", + "Using cached nvidia_cudnn_cu12-9.5.1.17-py3-none-manylinux_2_28_x86_64.whl (571.0 MB)\n", + "Using cached nvidia_cufft_cu12-11.3.0.4-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (200.2 MB)\n", + "Using cached nvidia_cufile_cu12-1.11.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (1.1 MB)\n", + "Using cached nvidia_curand_cu12-10.3.7.77-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (56.3 MB)\n", + "Using cached nvidia_cusolver_cu12-11.7.1.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (158.2 MB)\n", + "Using cached nvidia_cusparse_cu12-12.5.4.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (216.6 MB)\n", + "Using cached nvidia_cusparselt_cu12-0.6.3-py3-none-manylinux2014_x86_64.whl (156.8 MB)\n", + "Using cached nvidia_nccl_cu12-2.26.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (201.3 MB)\n", + "Using cached nvidia_nvjitlink_cu12-12.6.85-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl (19.7 MB)\n", + "Using cached nvidia_nvtx_cu12-12.6.77-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (89 kB)\n", + "Using cached triton-3.3.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl (155.7 MB)\n", + "Using cached transformers-4.53.3-py3-none-any.whl (10.8 MB)\n", + "Using cached huggingface_hub-0.36.2-py3-none-any.whl (566 kB)\n", + "Using cached hf_xet-1.2.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.3 MB)\n", + "Using cached tokenizers-0.21.4-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.1 MB)\n", + "Using cached urllib3-2.5.0-py3-none-any.whl (129 kB)\n", + "Using cached annotated_types-0.7.0-py3-none-any.whl (13 kB)\n", + "Using cached click-8.3.1-py3-none-any.whl (108 kB)\n", + "Using cached cloudpickle-3.1.2-py3-none-any.whl (22 kB)\n", + "Using cached einops-0.8.2-py3-none-any.whl (65 kB)\n", + "Using cached fsspec-2026.2.0-py3-none-any.whl (202 kB)\n", + "Using cached joblib-1.5.3-py3-none-any.whl (309 kB)\n", + "Using cached lazy_loader-0.4-py3-none-any.whl (12 kB)\n", + "Using cached linear_attention_transformer-0.19.1-py3-none-any.whl (12 kB)\n", + "Using cached linformer-0.2.3-py3-none-any.whl (6.2 kB)\n", + "Using cached lz4-4.4.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (1.4 MB)\n", + "Using cached matplotlib-3.10.8-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (8.7 MB)\n", + "Using cached contourpy-1.3.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl (362 kB)\n", + "Using cached cycler-0.12.1-py3-none-any.whl (8.3 kB)\n", + "Using cached fonttools-4.61.1-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl (4.9 MB)\n", + "Using cached kiwisolver-1.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (1.5 MB)\n", + "Using cached ogb-1.3.6-py3-none-any.whl (78 kB)\n", + "Using cached outdated-0.2.2-py2.py3-none-any.whl (7.5 kB)\n", + "Using cached packaging-26.0-py3-none-any.whl (74 kB)\n", + "Using cached partd-1.4.2-py3-none-any.whl (18 kB)\n", + "Using cached pillow-12.1.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl (7.0 MB)\n", + "Using cached pooch-1.9.0-py3-none-any.whl (67 kB)\n", + "Using cached platformdirs-4.9.2-py3-none-any.whl (21 kB)\n", + "Using cached product_key_memory-0.3.0-py3-none-any.whl (8.3 kB)\n", + "Using cached CoLT5_attention-0.11.1-py3-none-any.whl (18 kB)\n", + "Using cached local_attention-1.11.2-py3-none-any.whl (9.5 kB)\n", + "Using cached hyper_connections-0.4.9-py3-none-any.whl (28 kB)\n", + "Using cached pyparsing-3.3.2-py3-none-any.whl (122 kB)\n", + "Using cached python_dateutil-2.9.0.post0-py2.py3-none-any.whl (229 kB)\n", + "Using cached pytz-2025.2-py2.py3-none-any.whl (509 kB)\n", + "Using cached pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (801 kB)\n", + "Using cached regex-2026.2.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (803 kB)\n", + "Using cached requests-2.32.5-py3-none-any.whl (64 kB)\n", + "Using cached charset_normalizer-3.4.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (153 kB)\n", + "Using cached idna-3.11-py3-none-any.whl (71 kB)\n", + "Using cached certifi-2026.1.4-py3-none-any.whl (152 kB)\n", + "Using cached safetensors-0.7.0-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (507 kB)\n", + "Using cached scipy-1.17.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl (35.0 MB)\n", + "Using cached setuptools-82.0.0-py3-none-any.whl (1.0 MB)\n", + "Using cached six-1.17.0-py2.py3-none-any.whl (11 kB)\n", + "Using cached sympy-1.14.0-py3-none-any.whl (6.3 MB)\n", + "Using cached mpmath-1.3.0-py3-none-any.whl (536 kB)\n", + "Using cached threadpoolctl-3.6.0-py3-none-any.whl (18 kB)\n", + "Using cached toolz-1.1.0-py3-none-any.whl (58 kB)\n", + "Using cached torch_einops_utils-0.0.30-py3-none-any.whl (7.2 kB)\n", + "Using cached tqdm-4.67.3-py3-none-any.whl (78 kB)\n", + "Using cached typing_extensions-4.15.0-py3-none-any.whl (44 kB)\n", + "Using cached typing_inspection-0.4.2-py3-none-any.whl (14 kB)\n", + "Using cached tzdata-2025.3-py2.py3-none-any.whl (348 kB)\n", + "Using cached accelerate-1.12.0-py3-none-any.whl (380 kB)\n", + "Using cached axial_positional_embedding-0.3.12-py3-none-any.whl (6.7 kB)\n", + "Using cached boto3-1.42.53-py3-none-any.whl (140 kB)\n", + "Using cached botocore-1.42.53-py3-none-any.whl (14.6 MB)\n", + "Using cached jmespath-1.1.0-py3-none-any.whl (20 kB)\n", + "Using cached s3transfer-0.16.0-py3-none-any.whl (86 kB)\n", + "Using cached distributed-2025.11.0-py3-none-any.whl (1.0 MB)\n", + "Using cached bokeh-3.8.2-py3-none-any.whl (7.2 MB)\n", + "Using cached jinja2-3.1.6-py3-none-any.whl (134 kB)\n", + "Using cached locket-1.0.0-py2.py3-none-any.whl (4.4 kB)\n", + "Using cached markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (22 kB)\n", + "Using cached msgpack-1.1.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (424 kB)\n", + "Using cached psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl (155 kB)\n", + "Using cached sortedcontainers-2.4.0-py2.py3-none-any.whl (29 kB)\n", + "Using cached tblib-3.2.2-py3-none-any.whl (12 kB)\n", + "Using cached tornado-6.5.4-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (445 kB)\n", + "Using cached xyzservices-2025.11.0-py3-none-any.whl (93 kB)\n", + "Using cached zict-3.0.0-py2.py3-none-any.whl (43 kB)\n", + "Using cached decorator-5.2.1-py3-none-any.whl (9.2 kB)\n", + "Using cached filelock-3.24.3-py3-none-any.whl (24 kB)\n", + "Using cached lightning_utilities-0.15.2-py3-none-any.whl (29 kB)\n", + "Using cached littleutils-0.2.4-py3-none-any.whl (8.1 kB)\n", + "Using cached networkx-3.6.1-py3-none-any.whl (2.1 MB)\n", + "Using cached obstore-0.8.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.8 MB)\n", + "Using cached peft-0.18.1-py3-none-any.whl (556 kB)\n", + "Using cached rdkit-2025.9.5-cp313-cp313-manylinux_2_28_x86_64.whl (36.7 MB)\n", + "Downloading tifffile-2026.2.20-py3-none-any.whl (234 kB)\n", + "Using cached torchvision-0.22.1-cp313-cp313-manylinux_2_28_x86_64.whl (7.5 MB)\n", + "Building wheels for collected packages: pyhealth\n", + " Building wheel for pyhealth (pyproject.toml) ... \u001b[?25ldone\n", + "\u001b[?25h Created wheel for pyhealth: filename=pyhealth-2.0.0-py3-none-any.whl size=604150 sha256=5bfe03e29f88226c7d31fd6c4711c6c6571f3aad2eaac8ad262c25090fa91dc8\n", + " Stored in directory: /tmp/pip-ephem-wheel-cache-im1ziqna/wheels/0b/64/bf/707e0b63217d7bdf418a843b530d1900f1d86594caf43fac94\n", + "Successfully built pyhealth\n", + "Installing collected packages: sortedcontainers, pytz, nvidia-cusparselt-cu12, mpmath, zict, xyzservices, urllib3, tzdata, typing-extensions, tqdm, tornado, toolz, threadpoolctl, tblib, sympy, six, setuptools, safetensors, regex, pyyaml, pyparsing, pyarrow, psutil, polars-runtime-32, platformdirs, pillow, packaging, obstore, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufile-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, numpy, networkx, narwhals, msgpack, more-itertools, MarkupSafe, lz4, locket, littleutils, kiwisolver, joblib, jmespath, idna, hf-xet, fsspec, fonttools, filelock, einops, decorator, cycler, cloudpickle, click, charset_normalizer, certifi, annotated-types, typing-inspection, triton, tifffile, scipy, requests, rdkit, python-dateutil, pydantic-core, polars, partd, nvidia-cusparse-cu12, nvidia-cufft-cu12, nvidia-cudnn-cu12, lightning-utilities, lazy-loader, jinja2, contourpy, scikit-learn, pydantic, pooch, pandas, outdated, nvidia-cusolver-cu12, matplotlib, huggingface-hub, dask, botocore, torch, tokenizers, s3transfer, mne, distributed, bokeh, transformers, torchvision, torch-einops-utils, ogb, linformer, boto3, axial-positional-embedding, accelerate, peft, litdata, hyper-connections, local-attention, colt5-attention, product-key-memory, linear-attention-transformer, pyhealth\n", + "\u001b[2K Attempting uninstall: sortedcontainers\n", + "\u001b[2K Found existing installation: sortedcontainers 2.4.0\n", + "\u001b[2K Uninstalling sortedcontainers-2.4.0:\n", + "\u001b[2K Successfully uninstalled sortedcontainers-2.4.0\n", + "\u001b[2K Attempting uninstall: pytz\n", + "\u001b[2K Found existing installation: pytz 2025.2\n", + "\u001b[2K Uninstalling pytz-2025.2:\n", + "\u001b[2K Successfully uninstalled pytz-2025.2\n", + "\u001b[2K Attempting uninstall: nvidia-cusparselt-cu12\n", + "\u001b[2K Found existing installation: nvidia-cusparselt-cu12 0.6.3\n", + "\u001b[2K Uninstalling nvidia-cusparselt-cu12-0.6.3:\n", + "\u001b[2K Successfully uninstalled nvidia-cusparselt-cu12-0.6.3\n", + "\u001b[2K Attempting uninstall: mpmath━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 2/111\u001b[0m [nvidia-cusparselt-cu12]\n", + "\u001b[2K Found existing installation: mpmath 1.3.0━━━━━━━━\u001b[0m \u001b[32m 2/111\u001b[0m [nvidia-cusparselt-cu12]\n", + "\u001b[2K Uninstalling mpmath-1.3.0:━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 2/111\u001b[0m [nvidia-cusparselt-cu12]\n", + "\u001b[2K Successfully uninstalled mpmath-1.3.0━━━━━━━━━━\u001b[0m \u001b[32m 2/111\u001b[0m [nvidia-cusparselt-cu12]\n", + "\u001b[2K Attempting uninstall: zict━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 3/111\u001b[0m [mpmath]12]\n", + "\u001b[2K Found existing installation: zict 3.0.0━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 3/111\u001b[0m [mpmath]\n", + "\u001b[2K Uninstalling zict-3.0.0:━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 3/111\u001b[0m [mpmath]\n", + "\u001b[2K Successfully uninstalled zict-3.0.0━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 3/111\u001b[0m [mpmath]\n", + "\u001b[2K Attempting uninstall: xyzservices━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 3/111\u001b[0m [mpmath]\n", + "\u001b[2K Found existing installation: xyzservices 2025.11.0━━━━━━━━\u001b[0m \u001b[32m 3/111\u001b[0m [mpmath]\n", + "\u001b[2K Uninstalling xyzservices-2025.11.0:━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 3/111\u001b[0m [mpmath]\n", + "\u001b[2K Successfully uninstalled xyzservices-2025.11.0━━━━━━━━━━\u001b[0m \u001b[32m 3/111\u001b[0m [mpmath]\n", + "\u001b[2K Attempting uninstall: urllib3━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 3/111\u001b[0m [mpmath]\n", + "\u001b[2K Found existing installation: urllib3 2.5.0━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 3/111\u001b[0m [mpmath]\n", + "\u001b[2K Uninstalling urllib3-2.5.0:━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 3/111\u001b[0m [mpmath]\n", + "\u001b[2K Successfully uninstalled urllib3-2.5.0━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 3/111\u001b[0m [mpmath]\n", + "\u001b[2K Attempting uninstall: tzdata━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 3/111\u001b[0m [mpmath]\n", + "\u001b[2K Found existing installation: tzdata 2025.3━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 7/111\u001b[0m [tzdata]\n", + "\u001b[2K Uninstalling tzdata-2025.3:━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 7/111\u001b[0m [tzdata]\n", + "\u001b[2K Successfully uninstalled tzdata-2025.3━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 7/111\u001b[0m [tzdata]\n", + "\u001b[2K Attempting uninstall: typing-extensions━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 7/111\u001b[0m [tzdata]\n", + "\u001b[2K Found existing installation: typing_extensions 4.15.0━━━━━\u001b[0m \u001b[32m 7/111\u001b[0m [tzdata]\n", + "\u001b[2K Uninstalling typing_extensions-4.15.0:━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 7/111\u001b[0m [tzdata]\n", + "\u001b[2K Successfully uninstalled typing_extensions-4.15.0━━━━━━━━━━━\u001b[0m \u001b[32m 8/111\u001b[0m [typing-extensions]\n", + "\u001b[2K Attempting uninstall: tqdm━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 8/111\u001b[0m [typing-extensions]\n", + "\u001b[2K Found existing installation: tqdm 4.67.3━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 8/111\u001b[0m [typing-extensions]\n", + "\u001b[2K Uninstalling tqdm-4.67.3:━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 8/111\u001b[0m [typing-extensions]\n", + "\u001b[2K Successfully uninstalled tqdm-4.67.3━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 8/111\u001b[0m [typing-extensions]\n", + "\u001b[2K Attempting uninstall: tornado━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 8/111\u001b[0m [typing-extensions]\n", + "\u001b[2K Found existing installation: tornado 6.5.4━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 8/111\u001b[0m [typing-extensions]\n", + "\u001b[2K Uninstalling tornado-6.5.4:━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 8/111\u001b[0m [typing-extensions]\n", + "\u001b[2K Successfully uninstalled tornado-6.5.4━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 8/111\u001b[0m [typing-extensions]\n", + "\u001b[2K Attempting uninstall: toolzm━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 10/111\u001b[0m [tornado]sions]\n", + "\u001b[2K Found existing installation: toolz 1.1.0━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 10/111\u001b[0m [tornado]\n", + "\u001b[2K Uninstalling toolz-1.1.0:━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 10/111\u001b[0m [tornado]\n", + "\u001b[2K Successfully uninstalled toolz-1.1.0━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 10/111\u001b[0m [tornado]\n", + "\u001b[2K Attempting uninstall: threadpoolctl━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 10/111\u001b[0m [tornado]\n", + "\u001b[2K Found existing installation: threadpoolctl 3.6.0━━━━━━━━━━\u001b[0m \u001b[32m 10/111\u001b[0m [tornado]\n", + "\u001b[2K Uninstalling threadpoolctl-3.6.0:━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 10/111\u001b[0m [tornado]\n", + "\u001b[2K Successfully uninstalled threadpoolctl-3.6.0━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 12/111\u001b[0m [threadpoolctl]\n", + "\u001b[2K Attempting uninstall: tblib━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 12/111\u001b[0m [threadpoolctl]\n", + "\u001b[2K Found existing installation: tblib 3.2.2━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 12/111\u001b[0m [threadpoolctl]\n", + "\u001b[2K Uninstalling tblib-3.2.2:━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 12/111\u001b[0m [threadpoolctl]\n", + "\u001b[2K Successfully uninstalled tblib-3.2.2━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 12/111\u001b[0m [threadpoolctl]\n", + "\u001b[2K Attempting uninstall: sympy━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 12/111\u001b[0m [threadpoolctl]\n", + "\u001b[2K Found existing installation: sympy 1.14.0━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 12/111\u001b[0m [threadpoolctl]\n", + "\u001b[2K Uninstalling sympy-1.14.0:0m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 14/111\u001b[0m [sympy]ctl]\n", + "\u001b[2K Successfully uninstalled sympy-1.14.0━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 14/111\u001b[0m [sympy]\n", + "\u001b[2K Attempting uninstall: six\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 14/111\u001b[0m [sympy]\n", + "\u001b[2K Found existing installation: six 1.17.0━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 15/111\u001b[0m [six]\n", + "\u001b[2K Uninstalling six-1.17.0:━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 15/111\u001b[0m [six]\n", + "\u001b[2K Successfully uninstalled six-1.17.0━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 15/111\u001b[0m [six]\n", + "\u001b[2K Attempting uninstall: setuptools━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 15/111\u001b[0m [six]\n", + "\u001b[2K Found existing installation: setuptools 82.0.0━━━━━━━━━━━━\u001b[0m \u001b[32m 15/111\u001b[0m [six]\n", + "\u001b[2K Uninstalling setuptools-82.0.0:━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 15/111\u001b[0m [six]\n", + "\u001b[2K Successfully uninstalled setuptools-82.0.0━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 16/111\u001b[0m [setuptools]\n", + "\u001b[2K Attempting uninstall: safetensors━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 16/111\u001b[0m [setuptools]\n", + "\u001b[2K Found existing installation: safetensors 0.7.0━━━━━━━━━━━━\u001b[0m \u001b[32m 16/111\u001b[0m [setuptools]\n", + "\u001b[2K Uninstalling safetensors-0.7.0:━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 16/111\u001b[0m [setuptools]\n", + "\u001b[2K Successfully uninstalled safetensors-0.7.0━━━━━━━━━━━━━━\u001b[0m \u001b[32m 16/111\u001b[0m [setuptools]\n", + "\u001b[2K Attempting uninstall: regex━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 16/111\u001b[0m [setuptools]\n", + "\u001b[2K Found existing installation: regex 2026.2.19━━━━━━━━━━━━━━\u001b[0m \u001b[32m 16/111\u001b[0m [setuptools]\n", + "\u001b[2K Uninstalling regex-2026.2.19:━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 16/111\u001b[0m [setuptools]\n", + "\u001b[2K Successfully uninstalled regex-2026.2.19━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 16/111\u001b[0m [setuptools]\n", + "\u001b[2K Attempting uninstall: pyyaml90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 18/111\u001b[0m [regex]]\n", + "\u001b[2K Found existing installation: PyYAML 6.0.3━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 18/111\u001b[0m [regex]\n", + "\u001b[2K Uninstalling PyYAML-6.0.3:━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 18/111\u001b[0m [regex]\n", + "\u001b[2K Successfully uninstalled PyYAML-6.0.3━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 18/111\u001b[0m [regex]\n", + "\u001b[2K Attempting uninstall: pyparsing━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 18/111\u001b[0m [regex]\n", + "\u001b[2K Found existing installation: pyparsing 3.3.2━━━━━━━━━━━━━━\u001b[0m \u001b[32m 18/111\u001b[0m [regex]\n", + "\u001b[2K Uninstalling pyparsing-3.3.2:━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 18/111\u001b[0m [regex]\n", + "\u001b[2K Successfully uninstalled pyparsing-3.3.2━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 18/111\u001b[0m [regex]\n", + "\u001b[2K Attempting uninstall: pyarrow━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 18/111\u001b[0m [regex]\n", + "\u001b[2K Found existing installation: pyarrow 22.0.0━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 18/111\u001b[0m [regex]\n", + "\u001b[2K Uninstalling pyarrow-22.0.0:━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 18/111\u001b[0m [regex]\n", + "\u001b[2K Successfully uninstalled pyarrow-22.0.0━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 18/111\u001b[0m [regex]\n", + "\u001b[2K Attempting uninstall: psutil[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 21/111\u001b[0m [pyarrow]\n", + "\u001b[2K Found existing installation: psutil 7.2.2━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 21/111\u001b[0m [pyarrow]\n", + "\u001b[2K Uninstalling psutil-7.2.2:━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 21/111\u001b[0m [pyarrow]\n", + "\u001b[2K Successfully uninstalled psutil-7.2.2━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 21/111\u001b[0m [pyarrow]\n", + "\u001b[2K Attempting uninstall: polars-runtime-32━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 21/111\u001b[0m [pyarrow]\n", + "\u001b[2K Found existing installation: polars-runtime-32 1.35.2━━━━━\u001b[0m \u001b[32m 21/111\u001b[0m [pyarrow]\n", + "\u001b[2K Uninstalling polars-runtime-32-1.35.2:━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 21/111\u001b[0m [pyarrow]\n", + "\u001b[2K Successfully uninstalled polars-runtime-32-1.35.2━━━━━━━\u001b[0m \u001b[32m 21/111\u001b[0m [pyarrow]\n", + "\u001b[2K Attempting uninstall: platformdirs━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 23/111\u001b[0m [polars-runtime-32]\n", + "\u001b[2K Found existing installation: platformdirs 4.9.2━━━━━━━━━━━\u001b[0m \u001b[32m 23/111\u001b[0m [polars-runtime-32]\n", + "\u001b[2K Uninstalling platformdirs-4.9.2:━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 23/111\u001b[0m [polars-runtime-32]\n", + "\u001b[2K Successfully uninstalled platformdirs-4.9.2━━━━━━━━━━━━━\u001b[0m \u001b[32m 23/111\u001b[0m [polars-runtime-32]\n", + "\u001b[2K Attempting uninstall: pillowm━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 23/111\u001b[0m [polars-runtime-32]\n", + "\u001b[2K Found existing installation: pillow 12.1.1━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 23/111\u001b[0m [polars-runtime-32]\n", + "\u001b[2K Uninstalling pillow-12.1.1:━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 23/111\u001b[0m [polars-runtime-32]\n", + "\u001b[2K Successfully uninstalled pillow-12.1.1━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 23/111\u001b[0m [polars-runtime-32]\n", + "\u001b[2K Attempting uninstall: packaging90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 25/111\u001b[0m [pillow]ime-32]\n", + "\u001b[2K Found existing installation: packaging 26.0━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 25/111\u001b[0m [pillow]\n", + "\u001b[2K Uninstalling packaging-26.0:━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 25/111\u001b[0m [pillow]\n", + "\u001b[2K Successfully uninstalled packaging-26.0━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 25/111\u001b[0m [pillow]\n", + "\u001b[2K Attempting uninstall: obstorem━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 25/111\u001b[0m [pillow]\n", + "\u001b[2K Found existing installation: obstore 0.8.2━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 27/111\u001b[0m [obstore]\n", + "\u001b[2K Uninstalling obstore-0.8.2:m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 27/111\u001b[0m [obstore]\n", + "\u001b[2K Successfully uninstalled obstore-0.8.2━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 27/111\u001b[0m [obstore]\n", + "\u001b[2K Attempting uninstall: nvidia-nvtx-cu12━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 27/111\u001b[0m [obstore]\n", + "\u001b[2K Found existing installation: nvidia-nvtx-cu12 12.6.77━━━━━\u001b[0m \u001b[32m 27/111\u001b[0m [obstore]\n", + "\u001b[2K Uninstalling nvidia-nvtx-cu12-12.6.77:━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 27/111\u001b[0m [obstore]\n", + "\u001b[2K Successfully uninstalled nvidia-nvtx-cu12-12.6.77━━━━━━━\u001b[0m \u001b[32m 27/111\u001b[0m [obstore]\n", + "\u001b[2K Attempting uninstall: nvidia-nvjitlink-cu12━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 27/111\u001b[0m [obstore]\n", + "\u001b[2K Found existing installation: nvidia-nvjitlink-cu12 12.6.85\u001b[0m \u001b[32m 27/111\u001b[0m [obstore]\n", + "\u001b[2K Uninstalling nvidia-nvjitlink-cu12-12.6.85:━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 27/111\u001b[0m [obstore]\n", + "\u001b[2K Successfully uninstalled nvidia-nvjitlink-cu12-12.6.85━━\u001b[0m \u001b[32m 27/111\u001b[0m [obstore]\n", + "\u001b[2K Attempting uninstall: nvidia-nccl-cu12━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 29/111\u001b[0m [nvidia-nvjitlink-cu12]\n", + "\u001b[2K Found existing installation: nvidia-nccl-cu12 2.26.2━━━━━━\u001b[0m \u001b[32m 29/111\u001b[0m [nvidia-nvjitlink-cu12]\n", + "\u001b[2K Uninstalling nvidia-nccl-cu12-2.26.2:━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 29/111\u001b[0m [nvidia-nvjitlink-cu12]\n", + "\u001b[2K Successfully uninstalled nvidia-nccl-cu12-2.26.2━━━━━━━━\u001b[0m \u001b[32m 29/111\u001b[0m [nvidia-nvjitlink-cu12]\n", + "\u001b[2K Attempting uninstall: nvidia-curand-cu12━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 30/111\u001b[0m [nvidia-nccl-cu12]]\n", + "\u001b[2K Found existing installation: nvidia-curand-cu12 10.3.7.77━\u001b[0m \u001b[32m 30/111\u001b[0m [nvidia-nccl-cu12]\n", + "\u001b[2K Uninstalling nvidia-curand-cu12-10.3.7.77:━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 30/111\u001b[0m [nvidia-nccl-cu12]\n", + "\u001b[2K Successfully uninstalled nvidia-curand-cu12-10.3.7.77━━━\u001b[0m \u001b[32m 30/111\u001b[0m [nvidia-nccl-cu12]\n", + "\u001b[2K Attempting uninstall: nvidia-cufile-cu12━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 31/111\u001b[0m [nvidia-curand-cu12]\n", + "\u001b[2K Found existing installation: nvidia-cufile-cu12 1.11.1.6━━\u001b[0m \u001b[32m 31/111\u001b[0m [nvidia-curand-cu12]\n", + "\u001b[2K Uninstalling nvidia-cufile-cu12-1.11.1.6:━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 31/111\u001b[0m [nvidia-curand-cu12]\n", + "\u001b[2K Successfully uninstalled nvidia-cufile-cu12-1.11.1.6━━━━\u001b[0m \u001b[32m 31/111\u001b[0m [nvidia-curand-cu12]\n", + "\u001b[2K Attempting uninstall: nvidia-cuda-runtime-cu12━━━━━━━━━━━━━━\u001b[0m \u001b[32m 31/111\u001b[0m [nvidia-curand-cu12]\n", + "\u001b[2K Found existing installation: nvidia-cuda-runtime-cu12 12.6.77m \u001b[32m 31/111\u001b[0m [nvidia-curand-cu12]\n", + "\u001b[2K Uninstalling nvidia-cuda-runtime-cu12-12.6.77:━━━━━━━━━━━━\u001b[0m \u001b[32m 31/111\u001b[0m [nvidia-curand-cu12]\n", + "\u001b[2K Successfully uninstalled nvidia-cuda-runtime-cu12-12.6.77[0m \u001b[32m 31/111\u001b[0m [nvidia-curand-cu12]\n", + "\u001b[2K Attempting uninstall: nvidia-cuda-nvrtc-cu12━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 31/111\u001b[0m [nvidia-curand-cu12]\n", + "\u001b[2K Found existing installation: nvidia-cuda-nvrtc-cu12 12.6.77[0m \u001b[32m 31/111\u001b[0m [nvidia-curand-cu12]\n", + "\u001b[2K Uninstalling nvidia-cuda-nvrtc-cu12-12.6.77:━━━━━━━━━━━━━━\u001b[0m \u001b[32m 31/111\u001b[0m [nvidia-curand-cu12]\n", + "\u001b[2K Successfully uninstalled nvidia-cuda-nvrtc-cu12-12.6.77━\u001b[0m \u001b[32m 31/111\u001b[0m [nvidia-curand-cu12]\n", + "\u001b[2K Attempting uninstall: nvidia-cuda-cupti-cu12━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 34/111\u001b[0m [nvidia-cuda-nvrtc-cu12]\n", + "\u001b[2K Found existing installation: nvidia-cuda-cupti-cu12 12.6.80[0m \u001b[32m 34/111\u001b[0m [nvidia-cuda-nvrtc-cu12]\n", + "\u001b[2K Uninstalling nvidia-cuda-cupti-cu12-12.6.80:━━━━━━━━━━━━━━\u001b[0m \u001b[32m 34/111\u001b[0m [nvidia-cuda-nvrtc-cu12]\n", + "\u001b[2K Successfully uninstalled nvidia-cuda-cupti-cu12-12.6.80━\u001b[0m \u001b[32m 34/111\u001b[0m [nvidia-cuda-nvrtc-cu12]\n", + "\u001b[2K Attempting uninstall: nvidia-cublas-cu12━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 34/111\u001b[0m [nvidia-cuda-nvrtc-cu12]\n", + "\u001b[2K Found existing installation: nvidia-cublas-cu12 12.6.4.1━━\u001b[0m \u001b[32m 34/111\u001b[0m [nvidia-cuda-nvrtc-cu12]\n", + "\u001b[2K Uninstalling nvidia-cublas-cu12-12.6.4.1:━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 34/111\u001b[0m [nvidia-cuda-nvrtc-cu12]\n", + "\u001b[2K Successfully uninstalled nvidia-cublas-cu12-12.6.4.1━━━━\u001b[0m \u001b[32m 34/111\u001b[0m [nvidia-cuda-nvrtc-cu12]\n", + "\u001b[2K Attempting uninstall: numpy╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 36/111\u001b[0m [nvidia-cublas-cu12]\n", + "\u001b[2K Found existing installation: numpy 2.2.6━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 36/111\u001b[0m [nvidia-cublas-cu12]\n", + "\u001b[2K Uninstalling numpy-2.2.6:m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 36/111\u001b[0m [nvidia-cublas-cu12]\n", + "\u001b[2K Successfully uninstalled numpy-2.2.6━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 36/111\u001b[0m [nvidia-cublas-cu12]\n", + "\u001b[2K Attempting uninstall: networkx[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 37/111\u001b[0m [numpy]las-cu12]\n", + "\u001b[2K Found existing installation: networkx 3.6.1━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 37/111\u001b[0m [numpy]\n", + "\u001b[2K Uninstalling networkx-3.6.1:[90m━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 37/111\u001b[0m [numpy]\n", + "\u001b[2K Successfully uninstalled networkx-3.6.1━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 37/111\u001b[0m [numpy]\n", + "\u001b[2K Attempting uninstall: narwhals[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 38/111\u001b[0m [networkx]\n", + "\u001b[2K Found existing installation: narwhals 2.13.0━━━━━━━━━━━━━━\u001b[0m \u001b[32m 38/111\u001b[0m [networkx]\n", + "\u001b[2K Uninstalling narwhals-2.13.0:90m━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 38/111\u001b[0m [networkx]\n", + "\u001b[2K Successfully uninstalled narwhals-2.13.0━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 38/111\u001b[0m [networkx]\n", + "\u001b[2K Attempting uninstall: msgpack╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 39/111\u001b[0m [narwhals]\n", + "\u001b[2K Found existing installation: msgpack 1.1.2━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 39/111\u001b[0m [narwhals]\n", + "\u001b[2K Uninstalling msgpack-1.1.2:m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 39/111\u001b[0m [narwhals]\n", + "\u001b[2K Successfully uninstalled msgpack-1.1.2━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 39/111\u001b[0m [narwhals]\n", + "\u001b[2K Attempting uninstall: more-itertools━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 39/111\u001b[0m [narwhals]\n", + "\u001b[2K Found existing installation: more-itertools 10.8.0━━━━━━━━\u001b[0m \u001b[32m 39/111\u001b[0m [narwhals]\n", + "\u001b[2K Uninstalling more-itertools-10.8.0:━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 39/111\u001b[0m [narwhals]\n", + "\u001b[2K Successfully uninstalled more-itertools-10.8.0━━━━━━━━━━\u001b[0m \u001b[32m 39/111\u001b[0m [narwhals]\n", + "\u001b[2K Attempting uninstall: MarkupSafe0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 41/111\u001b[0m [more-itertools]\n", + "\u001b[2K Found existing installation: MarkupSafe 3.0.3━━━━━━━━━━━━━\u001b[0m \u001b[32m 41/111\u001b[0m [more-itertools]\n", + "\u001b[2K Uninstalling MarkupSafe-3.0.3:90m━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 41/111\u001b[0m [more-itertools]\n", + "\u001b[2K Successfully uninstalled MarkupSafe-3.0.3━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 41/111\u001b[0m [more-itertools]\n", + "\u001b[2K Attempting uninstall: lz4╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 41/111\u001b[0m [more-itertools]\n", + "\u001b[2K Found existing installation: lz4 4.4.5━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 41/111\u001b[0m [more-itertools]\n", + "\u001b[2K Uninstalling lz4-4.4.5:╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 41/111\u001b[0m [more-itertools]\n", + "\u001b[2K Successfully uninstalled lz4-4.4.5━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 41/111\u001b[0m [more-itertools]\n", + "\u001b[2K Attempting uninstall: locket0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 41/111\u001b[0m [more-itertools]\n", + "\u001b[2K Found existing installation: locket 1.0.0━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 41/111\u001b[0m [more-itertools]\n", + "\u001b[2K Uninstalling locket-1.0.0:0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 41/111\u001b[0m [more-itertools]\n", + "\u001b[2K Successfully uninstalled locket-1.0.0━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 41/111\u001b[0m [more-itertools]\n", + "\u001b[2K Attempting uninstall: littleutils0m━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 41/111\u001b[0m [more-itertools]\n", + "\u001b[2K Found existing installation: littleutils 0.2.4━━━━━━━━━━━━\u001b[0m \u001b[32m 41/111\u001b[0m [more-itertools]\n", + "\u001b[2K Uninstalling littleutils-0.2.4:0m━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 41/111\u001b[0m [more-itertools]\n", + "\u001b[2K Successfully uninstalled littleutils-0.2.4━━━━━━━━━━━━━━\u001b[0m \u001b[32m 41/111\u001b[0m [more-itertools]\n", + "\u001b[2K Attempting uninstall: kiwisolver90m━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 41/111\u001b[0m [more-itertools]\n", + "\u001b[2K Found existing installation: kiwisolver 1.4.9━━━━━━━━━━━━━\u001b[0m \u001b[32m 41/111\u001b[0m [more-itertools]\n", + "\u001b[2K Uninstalling kiwisolver-1.4.9:90m━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 41/111\u001b[0m [more-itertools]\n", + "\u001b[2K Successfully uninstalled kiwisolver-1.4.9━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 41/111\u001b[0m [more-itertools]\n", + "\u001b[2K Attempting uninstall: joblib0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 41/111\u001b[0m [more-itertools]\n", + "\u001b[2K Found existing installation: joblib 1.5.3━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 41/111\u001b[0m [more-itertools]\n", + "\u001b[2K Uninstalling joblib-1.5.3:0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 41/111\u001b[0m [more-itertools]\n", + "\u001b[2K Successfully uninstalled joblib-1.5.3━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 41/111\u001b[0m [more-itertools]\n", + "\u001b[2K Attempting uninstall: jmespathm╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 47/111\u001b[0m [joblib]ols]\n", + "\u001b[2K Found existing installation: jmespath 1.1.0━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 47/111\u001b[0m [joblib]\n", + "\u001b[2K Uninstalling jmespath-1.1.0:0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 47/111\u001b[0m [joblib]\n", + "\u001b[2K Successfully uninstalled jmespath-1.1.0━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 47/111\u001b[0m [joblib]\n", + "\u001b[2K Attempting uninstall: idnam╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 47/111\u001b[0m [joblib]\n", + "\u001b[2K Found existing installation: idna 3.11━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 47/111\u001b[0m [joblib]\n", + "\u001b[2K Uninstalling idna-3.11:1m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 47/111\u001b[0m [joblib]\n", + "\u001b[2K Successfully uninstalled idna-3.11━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 47/111\u001b[0m [joblib]\n", + "\u001b[2K Attempting uninstall: hf-xet\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 47/111\u001b[0m [joblib]\n", + "\u001b[2K Found existing installation: hf-xet 1.2.0━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 50/111\u001b[0m [hf-xet]\n", + "\u001b[2K Uninstalling hf-xet-1.2.0:m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 50/111\u001b[0m [hf-xet]\n", + "\u001b[2K Successfully uninstalled hf-xet-1.2.0━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 50/111\u001b[0m [hf-xet]\n", + "\u001b[2K Attempting uninstall: fsspecm╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 50/111\u001b[0m [hf-xet]\n", + "\u001b[2K Found existing installation: fsspec 2026.2.0━━━━━━━━━━━━━━\u001b[0m \u001b[32m 50/111\u001b[0m [hf-xet]\n", + "\u001b[2K Uninstalling fsspec-2026.2.0:[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 50/111\u001b[0m [hf-xet]\n", + "\u001b[2K Successfully uninstalled fsspec-2026.2.0━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 50/111\u001b[0m [hf-xet]\n", + "\u001b[2K Attempting uninstall: fonttools[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 50/111\u001b[0m [hf-xet]\n", + "\u001b[2K Found existing installation: fonttools 4.61.1━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 52/111\u001b[0m [fonttools]\n", + "\u001b[2K Uninstalling fonttools-4.61.1:0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 52/111\u001b[0m [fonttools]\n", + "\u001b[2K Successfully uninstalled fonttools-4.61.1━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 52/111\u001b[0m [fonttools]\n", + "\u001b[2K Attempting uninstall: filelock91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 52/111\u001b[0m [fonttools]\n", + "\u001b[2K Found existing installation: filelock 3.24.3━━━━━━━━━━━━━━\u001b[0m \u001b[32m 52/111\u001b[0m [fonttools]\n", + "\u001b[2K Uninstalling filelock-3.24.3:[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 52/111\u001b[0m [fonttools]\n", + "\u001b[2K Successfully uninstalled filelock-3.24.3━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 52/111\u001b[0m [fonttools]\n", + "\u001b[2K Attempting uninstall: einopsm╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 52/111\u001b[0m [fonttools]\n", + "\u001b[2K Found existing installation: einops 0.8.2━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 52/111\u001b[0m [fonttools]\n", + "\u001b[2K Uninstalling einops-0.8.2:m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 52/111\u001b[0m [fonttools]\n", + "\u001b[2K Successfully uninstalled einops-0.8.2━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 52/111\u001b[0m [fonttools]\n", + "\u001b[2K Attempting uninstall: decorator90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 54/111\u001b[0m [einops]\n", + "\u001b[2K Found existing installation: decorator 5.2.1━━━━━━━━━━━━━━\u001b[0m \u001b[32m 54/111\u001b[0m [einops]\n", + "\u001b[2K Uninstalling decorator-5.2.1:\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 54/111\u001b[0m [einops]\n", + "\u001b[2K Successfully uninstalled decorator-5.2.1━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 55/111\u001b[0m [decorator]\n", + "\u001b[2K Attempting uninstall: cycler1m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 55/111\u001b[0m [decorator]\n", + "\u001b[2K Found existing installation: cycler 0.12.1━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 55/111\u001b[0m [decorator]\n", + "\u001b[2K Uninstalling cycler-0.12.1:m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 55/111\u001b[0m [decorator]\n", + "\u001b[2K Successfully uninstalled cycler-0.12.1━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 55/111\u001b[0m [decorator]\n", + "\u001b[2K Attempting uninstall: cloudpickle0m\u001b[90m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 55/111\u001b[0m [decorator]\n", + "\u001b[2K Found existing installation: cloudpickle 3.1.2━━━━━━━━━━━━\u001b[0m \u001b[32m 55/111\u001b[0m [decorator]\n", + "\u001b[2K Uninstalling cloudpickle-3.1.2:0m\u001b[90m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 55/111\u001b[0m [decorator]\n", + "\u001b[2K Successfully uninstalled cloudpickle-3.1.2━━━━━━━━━━━━━━\u001b[0m \u001b[32m 55/111\u001b[0m [decorator]\n", + "\u001b[2K Attempting uninstall: click91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 55/111\u001b[0m [decorator]\n", + "\u001b[2K Found existing installation: click 8.3.1━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 55/111\u001b[0m [decorator]\n", + "\u001b[2K Uninstalling click-8.3.1:[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 58/111\u001b[0m [click]\n", + "\u001b[2K Successfully uninstalled click-8.3.1m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 58/111\u001b[0m [click]\n", + "\u001b[2K Attempting uninstall: charset_normalizerm━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 58/111\u001b[0m [click]\n", + "\u001b[2K Found existing installation: charset-normalizer 3.4.4━━━━━\u001b[0m \u001b[32m 58/111\u001b[0m [click]\n", + "\u001b[2K Uninstalling charset-normalizer-3.4.4:m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 58/111\u001b[0m [click]\n", + "\u001b[2K Successfully uninstalled charset-normalizer-3.4.4━━━━━━━\u001b[0m \u001b[32m 58/111\u001b[0m [click]\n", + "\u001b[2K Attempting uninstall: certifi1m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 58/111\u001b[0m [click]\n", + "\u001b[2K Found existing installation: certifi 2026.1.4━━━━━━━━━━━━━\u001b[0m \u001b[32m 58/111\u001b[0m [click]\n", + "\u001b[2K Uninstalling certifi-2026.1.4:\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 58/111\u001b[0m [click]\n", + "\u001b[2K Successfully uninstalled certifi-2026.1.4━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 58/111\u001b[0m [click]\n", + "\u001b[2K Attempting uninstall: annotated-types[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 58/111\u001b[0m [click]\n", + "\u001b[2K Found existing installation: annotated-types 0.7.0━━━━━━━━\u001b[0m \u001b[32m 58/111\u001b[0m [click]\n", + "\u001b[2K Uninstalling annotated-types-0.7.0:[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 58/111\u001b[0m [click]\n", + "\u001b[2K Successfully uninstalled annotated-types-0.7.0━━━━━━━━━━\u001b[0m \u001b[32m 58/111\u001b[0m [click]\n", + "\u001b[2K Attempting uninstall: typing-inspection0m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 58/111\u001b[0m [click]\n", + "\u001b[2K Found existing installation: typing-inspection 0.4.2━━━━━━\u001b[0m \u001b[32m 58/111\u001b[0m [click]\n", + "\u001b[2K Uninstalling typing-inspection-0.4.2:0m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 58/111\u001b[0m [click]\n", + "\u001b[2K Successfully uninstalled typing-inspection-0.4.2━━━━━━━━\u001b[0m \u001b[32m 58/111\u001b[0m [click]\n", + "\u001b[2K Attempting uninstall: triton91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 58/111\u001b[0m [click]\n", + "\u001b[2K Found existing installation: triton 3.3.1━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 58/111\u001b[0m [click]\n", + "\u001b[2K Uninstalling triton-3.3.1:91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 58/111\u001b[0m [click]\n", + "\u001b[2K Successfully uninstalled triton-3.3.1━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 58/111\u001b[0m [click]\n", + "\u001b[2K Attempting uninstall: tifffile0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 63/111\u001b[0m [triton]\n", + "\u001b[2K Found existing installation: tifffile 2026.2.16━━━━━━━━━━━\u001b[0m \u001b[32m 63/111\u001b[0m [triton]\n", + "\u001b[2K Uninstalling tifffile-2026.2.16:\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 63/111\u001b[0m [triton]\n", + "\u001b[2K Successfully uninstalled tifffile-2026.2.16━━━━━━━━━━━━━\u001b[0m \u001b[32m 63/111\u001b[0m [triton]\n", + "\u001b[2K Attempting uninstall: scipy━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 64/111\u001b[0m [tifffile]\n", + "\u001b[2K Found existing installation: scipy 1.17.0m━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 64/111\u001b[0m [tifffile]\n", + "\u001b[2K Uninstalling scipy-1.17.0:m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 64/111\u001b[0m [tifffile]\n", + "\u001b[2K Successfully uninstalled scipy-1.17.090m━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 64/111\u001b[0m [tifffile]\n", + "\u001b[2K Attempting uninstall: requests[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 65/111\u001b[0m [scipy]\n", + "\u001b[2K Found existing installation: requests 2.32.5━━━━━━━━━━━━━━\u001b[0m \u001b[32m 65/111\u001b[0m [scipy]\n", + "\u001b[2K Uninstalling requests-2.32.5:90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 65/111\u001b[0m [scipy]\n", + "\u001b[2K Successfully uninstalled requests-2.32.5━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 65/111\u001b[0m [scipy]\n", + "\u001b[2K Attempting uninstall: rdkit0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 65/111\u001b[0m [scipy]\n", + "\u001b[2K Found existing installation: rdkit 2025.9.5━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 65/111\u001b[0m [scipy]\n", + "\u001b[2K Uninstalling rdkit-2025.9.5:[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 65/111\u001b[0m [scipy]\n", + "\u001b[2K Successfully uninstalled rdkit-2025.9.5m━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 65/111\u001b[0m [scipy]\n", + "\u001b[2K Attempting uninstall: python-dateutil0m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 67/111\u001b[0m [rdkit]\n", + "\u001b[2K Found existing installation: python-dateutil 2.9.0.post0━━\u001b[0m \u001b[32m 67/111\u001b[0m [rdkit]\n", + "\u001b[2K Uninstalling python-dateutil-2.9.0.post0:0m━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 67/111\u001b[0m [rdkit]\n", + "\u001b[2K Successfully uninstalled python-dateutil-2.9.0.post0━━━━\u001b[0m \u001b[32m 67/111\u001b[0m [rdkit]\n", + "\u001b[2K Attempting uninstall: pydantic-core[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 68/111\u001b[0m [python-dateutil]\n", + "\u001b[2K Found existing installation: pydantic_core 2.33.2━━━━━━━━━\u001b[0m \u001b[32m 68/111\u001b[0m [python-dateutil]\n", + "\u001b[2K Uninstalling pydantic_core-2.33.2:\u001b[0m\u001b[90m━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 68/111\u001b[0m [python-dateutil]\n", + "\u001b[2K Successfully uninstalled pydantic_core-2.33.2━━━━━━━━━━━\u001b[0m \u001b[32m 68/111\u001b[0m [python-dateutil]\n", + "\u001b[2K Attempting uninstall: polars0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 68/111\u001b[0m [python-dateutil]\n", + "\u001b[2K Found existing installation: polars 1.35.2m━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 68/111\u001b[0m [python-dateutil]\n", + "\u001b[2K Uninstalling polars-1.35.2:m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 68/111\u001b[0m [python-dateutil]\n", + "\u001b[2K Successfully uninstalled polars-1.35.290m━━━━━━━━━━━━━━━\u001b[0m \u001b[32m 68/111\u001b[0m [python-dateutil]\n", + "\u001b[2K Attempting uninstall: partd━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━\u001b[0m \u001b[32m 70/111\u001b[0m [polars]util]\n", + "\u001b[2K Found existing installation: partd 1.4.2[90m━━━━━━━━━━━━━━\u001b[0m \u001b[32m 70/111\u001b[0m [polars]\n", + "\u001b[2K Uninstalling partd-1.4.2:\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━\u001b[0m \u001b[32m 70/111\u001b[0m [polars]\n", + "\u001b[2K Successfully uninstalled partd-1.4.2m\u001b[90m━━━━━━━━━━━━━━\u001b[0m \u001b[32m 70/111\u001b[0m [polars]\n", + "\u001b[2K Attempting uninstall: nvidia-cusparse-cu12[90m━━━━━━━━━━━━━━\u001b[0m \u001b[32m 70/111\u001b[0m [polars]\n", + "\u001b[2K Found existing installation: nvidia-cusparse-cu12 12.5.4.2\u001b[0m \u001b[32m 70/111\u001b[0m [polars]\n", + "\u001b[2K Uninstalling nvidia-cusparse-cu12-12.5.4.2:m━━━━━━━━━━━━━━\u001b[0m \u001b[32m 70/111\u001b[0m [polars]\n", + "\u001b[2K Successfully uninstalled nvidia-cusparse-cu12-12.5.4.2━━\u001b[0m \u001b[32m 70/111\u001b[0m [polars]\n", + "\u001b[2K Attempting uninstall: nvidia-cufft-cu12m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━\u001b[0m \u001b[32m 72/111\u001b[0m [nvidia-cusparse-cu12]\n", + "\u001b[2K Found existing installation: nvidia-cufft-cu12 11.3.0.4━━━━━━━\u001b[0m \u001b[32m 73/111\u001b[0m [nvidia-cufft-cu12]\n", + "\u001b[2K Uninstalling nvidia-cufft-cu12-11.3.0.4:\u001b[90m━━━━━━━━━━━━━\u001b[0m \u001b[32m 73/111\u001b[0m [nvidia-cufft-cu12]\n", + "\u001b[2K Successfully uninstalled nvidia-cufft-cu12-11.3.0.4━━━━━\u001b[0m \u001b[32m 73/111\u001b[0m [nvidia-cufft-cu12]\n", + "\u001b[2K Attempting uninstall: nvidia-cudnn-cu120m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━\u001b[0m \u001b[32m 73/111\u001b[0m [nvidia-cufft-cu12]\n", + "\u001b[2K Found existing installation: nvidia-cudnn-cu12 9.5.1.17━━━\u001b[0m \u001b[32m 73/111\u001b[0m [nvidia-cufft-cu12]\n", + "\u001b[2K Uninstalling nvidia-cudnn-cu12-9.5.1.17:\u001b[90m━━━━━━━━━━━━━\u001b[0m \u001b[32m 73/111\u001b[0m [nvidia-cufft-cu12]\n", + "\u001b[2K Successfully uninstalled nvidia-cudnn-cu12-9.5.1.17━━━━━\u001b[0m \u001b[32m 73/111\u001b[0m [nvidia-cufft-cu12]\n", + "\u001b[2K Attempting uninstall: lightning-utilities╸\u001b[0m\u001b[90m━━━━━━━━━━━━━\u001b[0m \u001b[32m 74/111\u001b[0m [nvidia-cudnn-cu12]\n", + "\u001b[2K Found existing installation: lightning-utilities 0.15.2━━━\u001b[0m \u001b[32m 74/111\u001b[0m [nvidia-cudnn-cu12]\n", + "\u001b[2K Uninstalling lightning-utilities-0.15.2:\u001b[90m━━━━━━━━━━━━━\u001b[0m \u001b[32m 74/111\u001b[0m [nvidia-cudnn-cu12]\n", + "\u001b[2K Successfully uninstalled lightning-utilities-0.15.2━━━━━━━━━\u001b[0m \u001b[32m 75/111\u001b[0m [lightning-utilities]\n", + "\u001b[2K Attempting uninstall: lazy-loader\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━\u001b[0m \u001b[32m 75/111\u001b[0m [lightning-utilities]\n", + "\u001b[2K Found existing installation: lazy_loader 0.40m━━━━━━━━━━━━\u001b[0m \u001b[32m 75/111\u001b[0m [lightning-utilities]\n", + "\u001b[2K Uninstalling lazy_loader-0.4:0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━\u001b[0m \u001b[32m 75/111\u001b[0m [lightning-utilities]\n", + "\u001b[2K Successfully uninstalled lazy_loader-0.4[90m━━━━━━━━━━━━\u001b[0m \u001b[32m 75/111\u001b[0m [lightning-utilities]\n", + "\u001b[2K Attempting uninstall: jinja2━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━\u001b[0m \u001b[32m 75/111\u001b[0m [lightning-utilities]\n", + "\u001b[2K Found existing installation: Jinja2 3.1.6\u001b[90m━━━━━━━━━━━━\u001b[0m \u001b[32m 75/111\u001b[0m [lightning-utilities]\n", + "\u001b[2K Uninstalling Jinja2-3.1.6:━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━\u001b[0m \u001b[32m 75/111\u001b[0m [lightning-utilities]\n", + "\u001b[2K Successfully uninstalled Jinja2-3.1.60m\u001b[90m━━━━━━━━━━━━\u001b[0m \u001b[32m 75/111\u001b[0m [lightning-utilities]\n", + "\u001b[2K Attempting uninstall: contourpy0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━\u001b[0m \u001b[32m 75/111\u001b[0m [lightning-utilities]\n", + "\u001b[2K Found existing installation: contourpy 1.3.30m━━━━━━━━━━━━\u001b[0m \u001b[32m 75/111\u001b[0m [lightning-utilities]\n", + "\u001b[2K Uninstalling contourpy-1.3.3:0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━\u001b[0m \u001b[32m 75/111\u001b[0m [lightning-utilities]\n", + "\u001b[2K Successfully uninstalled contourpy-1.3.3[90m━━━━━━━━━━━━\u001b[0m \u001b[32m 75/111\u001b[0m [lightning-utilities]\n", + "\u001b[2K Attempting uninstall: scikit-learn[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━\u001b[0m \u001b[32m 75/111\u001b[0m [lightning-utilities]\n", + "\u001b[2K Found existing installation: scikit-learn 1.7.2[90m━━━━━━━━━━━\u001b[0m \u001b[32m 79/111\u001b[0m [scikit-learn]es]\n", + "\u001b[2K Uninstalling scikit-learn-1.7.2:\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━\u001b[0m \u001b[32m 79/111\u001b[0m [scikit-learn]\n", + "\u001b[2K Successfully uninstalled scikit-learn-1.7.20m━━━━━━━━━━━\u001b[0m \u001b[32m 79/111\u001b[0m [scikit-learn]\n", + "\u001b[2K Attempting uninstall: pydantic━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━\u001b[0m \u001b[32m 79/111\u001b[0m [scikit-learn]\n", + "\u001b[2K Found existing installation: pydantic 2.11.100m━━━━━━━━━━━\u001b[0m \u001b[32m 79/111\u001b[0m [scikit-learn]\n", + "\u001b[2K Uninstalling pydantic-2.11.10:━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━\u001b[0m \u001b[32m 80/111\u001b[0m [pydantic]\n", + "\u001b[2K Successfully uninstalled pydantic-2.11.10[90m━━━━━━━━━━━\u001b[0m \u001b[32m 80/111\u001b[0m [pydantic]\n", + "\u001b[2K Attempting uninstall: pooch━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━\u001b[0m \u001b[32m 80/111\u001b[0m [pydantic]\n", + "\u001b[2K Found existing installation: pooch 1.9.00m\u001b[90m━━━━━━━━━━━\u001b[0m \u001b[32m 80/111\u001b[0m [pydantic]\n", + "\u001b[2K Uninstalling pooch-1.9.0:━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━\u001b[0m \u001b[32m 80/111\u001b[0m [pydantic]\n", + "\u001b[2K Successfully uninstalled pooch-1.9.0\u001b[0m\u001b[90m━━━━━━━━━━━\u001b[0m \u001b[32m 80/111\u001b[0m [pydantic]\n", + "\u001b[2K Attempting uninstall: pandas━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━\u001b[0m \u001b[32m 80/111\u001b[0m [pydantic]\n", + "\u001b[2K Found existing installation: pandas 2.3.3m\u001b[90m━━━━━━━━━━━\u001b[0m \u001b[32m 80/111\u001b[0m [pydantic]\n", + "\u001b[2K Uninstalling pandas-2.3.3:━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━\u001b[0m \u001b[32m 80/111\u001b[0m [pydantic]\n", + "\u001b[2K Successfully uninstalled pandas-2.3.391m╸\u001b[0m\u001b[90m━━━━━━━━━━\u001b[0m \u001b[32m 82/111\u001b[0m [pandas]\n", + "\u001b[2K Attempting uninstall: outdated━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━\u001b[0m \u001b[32m 82/111\u001b[0m [pandas]\n", + "\u001b[2K Found existing installation: outdated 0.2.2\u001b[90m━━━━━━━━━━\u001b[0m \u001b[32m 82/111\u001b[0m [pandas]\n", + "\u001b[2K Uninstalling outdated-0.2.2:━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━\u001b[0m \u001b[32m 82/111\u001b[0m [pandas]\n", + "\u001b[2K Successfully uninstalled outdated-0.2.20m\u001b[90m━━━━━━━━━━\u001b[0m \u001b[32m 82/111\u001b[0m [pandas]\n", + "\u001b[2K Attempting uninstall: nvidia-cusolver-cu121m╸\u001b[0m\u001b[90m━━━━━━━━━━\u001b[0m \u001b[32m 83/111\u001b[0m [outdated]\n", + "\u001b[2K Found existing installation: nvidia-cusolver-cu12 11.7.1.2\u001b[0m \u001b[32m 83/111\u001b[0m [outdated]\n", + "\u001b[2K Uninstalling nvidia-cusolver-cu12-11.7.1.2:\u001b[90m━━━━━━━━━━\u001b[0m \u001b[32m 83/111\u001b[0m [outdated]\n", + "\u001b[2K Successfully uninstalled nvidia-cusolver-cu12-11.7.1.2━━\u001b[0m \u001b[32m 83/111\u001b[0m [outdated]\n", + "\u001b[2K Attempting uninstall: matplotlib━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━\u001b[0m \u001b[32m 84/111\u001b[0m [nvidia-cusolver-cu12]\n", + "\u001b[2K Found existing installation: matplotlib 3.10.890m━━━━━━━━━\u001b[0m \u001b[32m 84/111\u001b[0m [nvidia-cusolver-cu12]\n", + "\u001b[2K Uninstalling matplotlib-3.10.8:[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━\u001b[0m \u001b[32m 84/111\u001b[0m [nvidia-cusolver-cu12]\n", + "\u001b[2K Successfully uninstalled matplotlib-3.10.8\u001b[0m\u001b[90m━━━━━━━━━\u001b[0m \u001b[32m 85/111\u001b[0m [matplotlib]-cu12]\n", + "\u001b[2K Attempting uninstall: huggingface-hub[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━\u001b[0m \u001b[32m 85/111\u001b[0m [matplotlib]\n", + "\u001b[2K Found existing installation: huggingface_hub 0.36.2━━━━━━━\u001b[0m \u001b[32m 85/111\u001b[0m [matplotlib]\n", + "\u001b[2K Uninstalling huggingface_hub-0.36.2:91m╸\u001b[0m\u001b[90m━━━━━━━━━\u001b[0m \u001b[32m 85/111\u001b[0m [matplotlib]\n", + "\u001b[2K Successfully uninstalled huggingface_hub-0.36.2━━━━━━━━━\u001b[0m \u001b[32m 85/111\u001b[0m [matplotlib]\n", + "\u001b[2K Attempting uninstall: dask━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━\u001b[0m \u001b[32m 86/111\u001b[0m [huggingface-hub]\n", + "\u001b[2K Found existing installation: dask 2025.11.0m\u001b[90m━━━━━━━━━\u001b[0m \u001b[32m 86/111\u001b[0m [huggingface-hub]\n", + "\u001b[2K Uninstalling dask-2025.11.0:━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━\u001b[0m \u001b[32m 86/111\u001b[0m [huggingface-hub]\n", + "\u001b[2K Successfully uninstalled dask-2025.11.0[0m\u001b[90m━━━━━━━━━\u001b[0m \u001b[32m 86/111\u001b[0m [huggingface-hub]\n", + "\u001b[2K Attempting uninstall: botocore━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━\u001b[0m \u001b[32m 87/111\u001b[0m [dask]ce-hub]\n", + "\u001b[2K Found existing installation: botocore 1.42.53\u001b[90m━━━━━━━━\u001b[0m \u001b[32m 87/111\u001b[0m [dask]\n", + "\u001b[2K Uninstalling botocore-1.42.53:━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━\u001b[0m \u001b[32m 88/111\u001b[0m [botocore]\n", + "\u001b[2K Successfully uninstalled botocore-1.42.530m\u001b[90m━━━━━━━━\u001b[0m \u001b[32m 88/111\u001b[0m [botocore]\n", + "\u001b[2K Attempting uninstall: torch━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━\u001b[0m \u001b[32m 88/111\u001b[0m [botocore]\n", + "\u001b[2K Found existing installation: torch 2.7.1╸\u001b[0m\u001b[90m━━━━━━━━\u001b[0m \u001b[32m 88/111\u001b[0m [botocore]\n", + "\u001b[2K Uninstalling torch-2.7.1:━━━━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━\u001b[0m \u001b[32m 89/111\u001b[0m [torch]\n", + "\u001b[2K Successfully uninstalled torch-2.7.190m╺\u001b[0m\u001b[90m━━━━━━━\u001b[0m \u001b[32m 89/111\u001b[0m [torch]\n", + "\u001b[2K Attempting uninstall: tokenizers━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━\u001b[0m \u001b[32m 89/111\u001b[0m [torch]\n", + "\u001b[2K Found existing installation: tokenizers 0.21.4\u001b[90m━━━━━━━\u001b[0m \u001b[32m 89/111\u001b[0m [torch]\n", + "\u001b[2K Uninstalling tokenizers-0.21.4:━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━\u001b[0m \u001b[32m 89/111\u001b[0m [torch]\n", + "\u001b[2K Successfully uninstalled tokenizers-0.21.40m\u001b[90m━━━━━━━\u001b[0m \u001b[32m 89/111\u001b[0m [torch]\n", + "\u001b[2K Attempting uninstall: s3transfer━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━\u001b[0m \u001b[32m 90/111\u001b[0m [tokenizers]\n", + "\u001b[2K Found existing installation: s3transfer 0.16.0\u001b[90m━━━━━━━\u001b[0m \u001b[32m 90/111\u001b[0m [tokenizers]\n", + "\u001b[2K Uninstalling s3transfer-0.16.0:━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━\u001b[0m \u001b[32m 90/111\u001b[0m [tokenizers]\n", + "\u001b[2K Successfully uninstalled s3transfer-0.16.00m\u001b[90m━━━━━━━\u001b[0m \u001b[32m 90/111\u001b[0m [tokenizers]\n", + "\u001b[2K Attempting uninstall: mne━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━\u001b[0m \u001b[32m 91/111\u001b[0m [s3transfer]\n", + "\u001b[2K Found existing installation: mne 1.10.21m╸\u001b[0m\u001b[90m━━━━━━━\u001b[0m \u001b[32m 91/111\u001b[0m [s3transfer]\n", + "\u001b[2K Uninstalling mne-1.10.2:━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━\u001b[0m \u001b[32m 91/111\u001b[0m [s3transfer]\n", + "\u001b[2K Successfully uninstalled mne-1.10.2[91m╸\u001b[0m\u001b[90m━━━━━━━\u001b[0m \u001b[32m 91/111\u001b[0m [s3transfer]\n", + "\u001b[2K Attempting uninstall: distributed━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━\u001b[0m \u001b[32m 92/111\u001b[0m [mne]er]\n", + "\u001b[2K Found existing installation: distributed 2025.11.00m━━━━━━\u001b[0m \u001b[32m 92/111\u001b[0m [mne]\n", + "\u001b[2K Uninstalling distributed-2025.11.0:0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━\u001b[0m \u001b[32m 92/111\u001b[0m [mne]\n", + "\u001b[2K Successfully uninstalled distributed-2025.11.0[90m━━━━━━\u001b[0m \u001b[32m 92/111\u001b[0m [mne]\n", + "\u001b[2K Attempting uninstall: bokeh━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━\u001b[0m \u001b[32m 93/111\u001b[0m [distributed]\n", + "\u001b[2K Found existing installation: bokeh 3.8.21m╸\u001b[0m\u001b[90m━━━━━━\u001b[0m \u001b[32m 93/111\u001b[0m [distributed]\n", + "\u001b[2K Uninstalling bokeh-3.8.2:━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━\u001b[0m \u001b[32m 93/111\u001b[0m [distributed]\n", + "\u001b[2K Successfully uninstalled bokeh-3.8.2[91m╸\u001b[0m\u001b[90m━━━━━━\u001b[0m \u001b[32m 93/111\u001b[0m [distributed]\n", + "\u001b[2K Attempting uninstall: transformers━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━\u001b[0m \u001b[32m 94/111\u001b[0m [bokeh]d]\n", + "\u001b[2K Found existing installation: transformers 4.53.3[90m━━━━━━\u001b[0m \u001b[32m 94/111\u001b[0m [bokeh]\n", + "\u001b[2K Uninstalling transformers-4.53.3:\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━\u001b[0m \u001b[32m 94/111\u001b[0m [bokeh]\n", + "\u001b[2K Successfully uninstalled transformers-4.53.3m\u001b[90m━━━━━━\u001b[0m \u001b[32m 94/111\u001b[0m [bokeh]\n", + "\u001b[2K Attempting uninstall: torchvision━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━\u001b[0m \u001b[32m 95/111\u001b[0m [transformers]\n", + "\u001b[2K Found existing installation: torchvision 0.22.1m\u001b[90m━━━━━\u001b[0m \u001b[32m 95/111\u001b[0m [transformers]\n", + "\u001b[2K Uninstalling torchvision-0.22.1:━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━\u001b[0m \u001b[32m 95/111\u001b[0m [transformers]\n", + "\u001b[2K Successfully uninstalled torchvision-0.22.1[0m\u001b[90m━━━━━\u001b[0m \u001b[32m 95/111\u001b[0m [transformers]\n", + "\u001b[2K Attempting uninstall: torch-einops-utils\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━\u001b[0m \u001b[32m 96/111\u001b[0m [torchvision]\n", + "\u001b[2K Found existing installation: torch-einops-utils 0.0.30━━━━\u001b[0m \u001b[32m 96/111\u001b[0m [torchvision]\n", + "\u001b[2K Uninstalling torch-einops-utils-0.0.30:[91m╸\u001b[0m\u001b[90m━━━━━\u001b[0m \u001b[32m 96/111\u001b[0m [torchvision]\n", + "\u001b[2K Successfully uninstalled torch-einops-utils-0.0.30m━━━━━\u001b[0m \u001b[32m 96/111\u001b[0m [torchvision]\n", + "\u001b[2K Attempting uninstall: ogb━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━\u001b[0m \u001b[32m 96/111\u001b[0m [torchvision]\n", + "\u001b[2K Found existing installation: ogb 1.3.6\u001b[91m╸\u001b[0m\u001b[90m━━━━━\u001b[0m \u001b[32m 96/111\u001b[0m [torchvision]\n", + "\u001b[2K Uninstalling ogb-1.3.6:━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━\u001b[0m \u001b[32m 96/111\u001b[0m [torchvision]\n", + "\u001b[2K Successfully uninstalled ogb-1.3.60m\u001b[91m╸\u001b[0m\u001b[90m━━━━━\u001b[0m \u001b[32m 96/111\u001b[0m [torchvision]\n", + "\u001b[2K Attempting uninstall: linformer━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━\u001b[0m \u001b[32m 96/111\u001b[0m [torchvision]\n", + "\u001b[2K Found existing installation: linformer 0.2.3[91m╸\u001b[0m\u001b[90m━━━━\u001b[0m \u001b[32m 99/111\u001b[0m [linformer]\n", + "\u001b[2K Uninstalling linformer-0.2.3:━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━\u001b[0m \u001b[32m 99/111\u001b[0m [linformer]\n", + "\u001b[2K Successfully uninstalled linformer-0.2.31m╸\u001b[0m\u001b[90m━━━━\u001b[0m \u001b[32m 99/111\u001b[0m [linformer]\n", + "\u001b[2K Attempting uninstall: boto3━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━\u001b[0m \u001b[32m 99/111\u001b[0m [linformer]\n", + "\u001b[2K Found existing installation: boto3 1.42.531m╸\u001b[0m\u001b[90m━━━━\u001b[0m \u001b[32m 99/111\u001b[0m [linformer]\n", + "\u001b[2K Uninstalling boto3-1.42.53:━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━\u001b[0m \u001b[32m 99/111\u001b[0m [linformer]\n", + "\u001b[2K Successfully uninstalled boto3-1.42.53[91m╸\u001b[0m\u001b[90m━━━━\u001b[0m \u001b[32m 99/111\u001b[0m [linformer]\n", + "\u001b[2K Attempting uninstall: axial-positional-embedding[0m\u001b[90m━━━━\u001b[0m \u001b[32m 99/111\u001b[0m [linformer]\n", + "\u001b[2K Found existing installation: axial_positional_embedding 0.3.12 \u001b[32m 99/111\u001b[0m [linformer]\n", + "\u001b[2K Uninstalling axial_positional_embedding-0.3.12:0m\u001b[90m━━━━\u001b[0m \u001b[32m 99/111\u001b[0m [linformer]\n", + "\u001b[2K Successfully uninstalled axial_positional_embedding-0.3.120m \u001b[32m 99/111\u001b[0m [linformer]\n", + "\u001b[2K Attempting uninstall: accelerate━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━\u001b[0m \u001b[32m 99/111\u001b[0m [linformer]\n", + "\u001b[2K Found existing installation: accelerate 1.12.0[0m\u001b[90m━━━━\u001b[0m \u001b[32m 99/111\u001b[0m [linformer]\n", + "\u001b[2K Uninstalling accelerate-1.12.0:━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━\u001b[0m \u001b[32m 99/111\u001b[0m [linformer]\n", + "\u001b[2K Successfully uninstalled accelerate-1.12.0╸\u001b[0m\u001b[90m━━━━\u001b[0m \u001b[32m 99/111\u001b[0m [linformer]\n", + "\u001b[2K Attempting uninstall: peft━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━\u001b[0m \u001b[32m102/111\u001b[0m [accelerate]\n", + "\u001b[2K Found existing installation: peft 0.18.1\u001b[91m╸\u001b[0m\u001b[90m━━━\u001b[0m \u001b[32m102/111\u001b[0m [accelerate]\n", + "\u001b[2K Uninstalling peft-0.18.1:━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━\u001b[0m \u001b[32m102/111\u001b[0m [accelerate]\n", + "\u001b[2K Successfully uninstalled peft-0.18.10m\u001b[91m╸\u001b[0m\u001b[90m━━━\u001b[0m \u001b[32m102/111\u001b[0m [accelerate]\n", + "\u001b[2K Attempting uninstall: litdata━━━━━━━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━\u001b[0m \u001b[32m103/111\u001b[0m [peft]e]\n", + "\u001b[2K Found existing installation: litdata 0.2.6190m╺\u001b[0m\u001b[90m━━\u001b[0m \u001b[32m103/111\u001b[0m [peft]\n", + "\u001b[2K Uninstalling litdata-0.2.61:━━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━\u001b[0m \u001b[32m103/111\u001b[0m [peft]\n", + "\u001b[2K Successfully uninstalled litdata-0.2.61\u001b[90m╺\u001b[0m\u001b[90m━━\u001b[0m \u001b[32m103/111\u001b[0m [peft]\n", + "\u001b[2K Attempting uninstall: hyper-connections━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━\u001b[0m \u001b[32m104/111\u001b[0m [litdata]\n", + "\u001b[2K Found existing installation: hyper-connections 0.4.9[90m━━\u001b[0m \u001b[32m104/111\u001b[0m [litdata]\n", + "\u001b[2K Uninstalling hyper-connections-0.4.9:\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━\u001b[0m \u001b[32m104/111\u001b[0m [litdata]\n", + "\u001b[2K Successfully uninstalled hyper-connections-0.4.9m\u001b[90m━━\u001b[0m \u001b[32m104/111\u001b[0m [litdata]\n", + "\u001b[2K Attempting uninstall: local-attention━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━\u001b[0m \u001b[32m104/111\u001b[0m [litdata]\n", + "\u001b[2K Found existing installation: local-attention 1.11.2\u001b[90m━━\u001b[0m \u001b[32m104/111\u001b[0m [litdata]\n", + "\u001b[2K Uninstalling local-attention-1.11.2:━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━\u001b[0m \u001b[32m104/111\u001b[0m [litdata]\n", + "\u001b[2K Successfully uninstalled local-attention-1.11.20m\u001b[90m━━\u001b[0m \u001b[32m104/111\u001b[0m [litdata]\n", + "\u001b[2K Attempting uninstall: colt5-attention━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━\u001b[0m \u001b[32m104/111\u001b[0m [litdata]\n", + "\u001b[2K Found existing installation: CoLT5-attention 0.11.1\u001b[90m━━\u001b[0m \u001b[32m104/111\u001b[0m [litdata]\n", + "\u001b[2K Uninstalling CoLT5-attention-0.11.1:━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━\u001b[0m \u001b[32m104/111\u001b[0m [litdata]\n", + "\u001b[2K Successfully uninstalled CoLT5-attention-0.11.10m\u001b[90m━━\u001b[0m \u001b[32m104/111\u001b[0m [litdata]\n", + "\u001b[2K Attempting uninstall: product-key-memory[0m\u001b[90m╺\u001b[0m\u001b[90m━━\u001b[0m \u001b[32m104/111\u001b[0m [litdata]\n", + "\u001b[2K Found existing installation: product_key_memory 0.3.090m━━\u001b[0m \u001b[32m104/111\u001b[0m [litdata]\n", + "\u001b[2K Uninstalling product_key_memory-0.3.0:[0m\u001b[90m╺\u001b[0m\u001b[90m━━\u001b[0m \u001b[32m104/111\u001b[0m [litdata]\n", + "\u001b[2K Successfully uninstalled product_key_memory-0.3.0\u001b[90m━━\u001b[0m \u001b[32m104/111\u001b[0m [litdata]\n", + "\u001b[2K Attempting uninstall: linear-attention-transformer91m╸\u001b[0m\u001b[90m━\u001b[0m \u001b[32m108/111\u001b[0m [product-key-memory]\n", + "\u001b[2K Found existing installation: linear-attention-transformer 0.19.1[32m108/111\u001b[0m [product-key-memory]\n", + "\u001b[2K Uninstalling linear-attention-transformer-0.19.1:[0m\u001b[90m━\u001b[0m \u001b[32m108/111\u001b[0m [product-key-memory]\n", + "\u001b[2K Successfully uninstalled linear-attention-transformer-0.19.1 \u001b[32m108/111\u001b[0m [product-key-memory]\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m111/111\u001b[0m [pyhealth]pyhealth]uct-key-memory]\n", + "\u001b[1A\u001b[2KSuccessfully installed MarkupSafe-3.0.3 accelerate-1.12.0 annotated-types-0.7.0 axial-positional-embedding-0.3.12 bokeh-3.8.2 boto3-1.42.53 botocore-1.42.53 certifi-2026.1.4 charset_normalizer-3.4.4 click-8.3.1 cloudpickle-3.1.2 colt5-attention-0.11.1 contourpy-1.3.3 cycler-0.12.1 dask-2025.11.0 decorator-5.2.1 distributed-2025.11.0 einops-0.8.2 filelock-3.24.3 fonttools-4.61.1 fsspec-2026.2.0 hf-xet-1.2.0 huggingface-hub-0.36.2 hyper-connections-0.4.9 idna-3.11 jinja2-3.1.6 jmespath-1.1.0 joblib-1.5.3 kiwisolver-1.4.9 lazy-loader-0.4 lightning-utilities-0.15.2 linear-attention-transformer-0.19.1 linformer-0.2.3 litdata-0.2.61 littleutils-0.2.4 local-attention-1.11.2 locket-1.0.0 lz4-4.4.5 matplotlib-3.10.8 mne-1.10.2 more-itertools-10.8.0 mpmath-1.3.0 msgpack-1.1.2 narwhals-2.13.0 networkx-3.6.1 numpy-2.2.6 nvidia-cublas-cu12-12.6.4.1 nvidia-cuda-cupti-cu12-12.6.80 nvidia-cuda-nvrtc-cu12-12.6.77 nvidia-cuda-runtime-cu12-12.6.77 nvidia-cudnn-cu12-9.5.1.17 nvidia-cufft-cu12-11.3.0.4 nvidia-cufile-cu12-1.11.1.6 nvidia-curand-cu12-10.3.7.77 nvidia-cusolver-cu12-11.7.1.2 nvidia-cusparse-cu12-12.5.4.2 nvidia-cusparselt-cu12-0.6.3 nvidia-nccl-cu12-2.26.2 nvidia-nvjitlink-cu12-12.6.85 nvidia-nvtx-cu12-12.6.77 obstore-0.8.2 ogb-1.3.6 outdated-0.2.2 packaging-26.0 pandas-2.3.3 partd-1.4.2 peft-0.18.1 pillow-12.1.1 platformdirs-4.9.2 polars-1.35.2 polars-runtime-32-1.35.2 pooch-1.9.0 product-key-memory-0.3.0 psutil-7.2.2 pyarrow-22.0.0 pydantic-2.11.10 pydantic-core-2.33.2 pyhealth-2.0.0 pyparsing-3.3.2 python-dateutil-2.9.0.post0 pytz-2025.2 pyyaml-6.0.3 rdkit-2025.9.5 regex-2026.2.19 requests-2.32.5 s3transfer-0.16.0 safetensors-0.7.0 scikit-learn-1.7.2 scipy-1.17.0 setuptools-82.0.0 six-1.17.0 sortedcontainers-2.4.0 sympy-1.14.0 tblib-3.2.2 threadpoolctl-3.6.0 tifffile-2026.2.20 tokenizers-0.21.4 toolz-1.1.0 torch-2.7.1 torch-einops-utils-0.0.30 torchvision-0.22.1 tornado-6.5.4 tqdm-4.67.3 transformers-4.53.3 triton-3.3.1 typing-extensions-4.15.0 typing-inspection-0.4.2 tzdata-2025.3 urllib3-2.5.0 xyzservices-2025.11.0 zict-3.0.0\n", + "Requirement already satisfied: ipywidgets in /opt/conda/lib/python3.13/site-packages (8.1.8)\n", + "Requirement already satisfied: comm>=0.1.3 in /opt/conda/lib/python3.13/site-packages (from ipywidgets) (0.2.3)\n", + "Requirement already satisfied: ipython>=6.1.0 in /opt/conda/lib/python3.13/site-packages (from ipywidgets) (9.8.0)\n", + "Requirement already satisfied: traitlets>=4.3.1 in /opt/conda/lib/python3.13/site-packages (from ipywidgets) (5.14.3)\n", + "Requirement already satisfied: widgetsnbextension~=4.0.14 in /opt/conda/lib/python3.13/site-packages (from ipywidgets) (4.0.15)\n", + "Requirement already satisfied: jupyterlab_widgets~=3.0.15 in /opt/conda/lib/python3.13/site-packages (from ipywidgets) (3.0.16)\n", + "Requirement already satisfied: decorator>=4.3.2 in /opt/conda/lib/python3.13/site-packages (from ipython>=6.1.0->ipywidgets) (5.2.1)\n", + "Requirement already satisfied: ipython-pygments-lexers>=1.0.0 in /opt/conda/lib/python3.13/site-packages (from ipython>=6.1.0->ipywidgets) (1.1.1)\n", + "Requirement already satisfied: jedi>=0.18.1 in /opt/conda/lib/python3.13/site-packages (from ipython>=6.1.0->ipywidgets) (0.19.2)\n", + "Requirement already satisfied: matplotlib-inline>=0.1.5 in /opt/conda/lib/python3.13/site-packages (from ipython>=6.1.0->ipywidgets) (0.2.1)\n", + "Requirement already satisfied: pexpect>4.3 in /opt/conda/lib/python3.13/site-packages (from ipython>=6.1.0->ipywidgets) (4.9.0)\n", + "Requirement already satisfied: prompt_toolkit<3.1.0,>=3.0.41 in /opt/conda/lib/python3.13/site-packages (from ipython>=6.1.0->ipywidgets) (3.0.52)\n", + "Requirement already satisfied: pygments>=2.11.0 in /opt/conda/lib/python3.13/site-packages (from ipython>=6.1.0->ipywidgets) (2.19.2)\n", + "Requirement already satisfied: stack_data>=0.6.0 in /opt/conda/lib/python3.13/site-packages (from ipython>=6.1.0->ipywidgets) (0.6.3)\n", + "Requirement already satisfied: wcwidth in /opt/conda/lib/python3.13/site-packages (from prompt_toolkit<3.1.0,>=3.0.41->ipython>=6.1.0->ipywidgets) (0.2.14)\n", + "Requirement already satisfied: parso<0.9.0,>=0.8.4 in /opt/conda/lib/python3.13/site-packages (from jedi>=0.18.1->ipython>=6.1.0->ipywidgets) (0.8.5)\n", + "Requirement already satisfied: ptyprocess>=0.5 in /opt/conda/lib/python3.13/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets) (0.7.0)\n", + "Requirement already satisfied: executing>=1.2.0 in /opt/conda/lib/python3.13/site-packages (from stack_data>=0.6.0->ipython>=6.1.0->ipywidgets) (2.2.1)\n", + "Requirement already satisfied: asttokens>=2.1.0 in /opt/conda/lib/python3.13/site-packages (from stack_data>=0.6.0->ipython>=6.1.0->ipywidgets) (3.0.1)\n", + "Requirement already satisfied: pure_eval in /opt/conda/lib/python3.13/site-packages (from stack_data>=0.6.0->ipython>=6.1.0->ipywidgets) (0.2.3)\n" + ] + } + ], + "source": [ + "!pip install --force-reinstall git+https://github.com/lookman-olowo/PyHealth.git@feature/code-mapping\n", + "!pip install ipywidgets\n", + "\n", + "# ! pip uninstall pyhealth -y" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "No config path provided, using default config\n", + "Initializing mimic3 dataset from /home/lolowo2 (dev mode: False)\n", + "Using provided cache_dir: /tmp/tmp5gf_ocv4/4f338cfd-b388-50e8-9d9c-fa4872e51b6c\n", + "No cached event dataframe found. Creating: /tmp/tmp5gf_ocv4/4f338cfd-b388-50e8-9d9c-fa4872e51b6c/global_event_df.parquet\n", + "Scanning table: patients from /home/lolowo2/PATIENTS.csv.gz\n", + "Scanning table: admissions from /home/lolowo2/ADMISSIONS.csv.gz\n", + "Scanning table: icustays from /home/lolowo2/ICUSTAYS.csv.gz\n", + "Scanning table: diagnoses_icd from /home/lolowo2/DIAGNOSES_ICD.csv.gz\n", + "Joining with table: /home/lolowo2/ADMISSIONS.csv.gz\n", + "Scanning table: procedures_icd from /home/lolowo2/PROCEDURES_ICD.csv.gz\n", + "Joining with table: /home/lolowo2/ADMISSIONS.csv.gz\n", + "Scanning table: prescriptions from /home/lolowo2/PRESCRIPTIONS.csv.gz\n", + "Joining with table: /home/lolowo2/ADMISSIONS.csv.gz\n", + "Caching event dataframe to /tmp/tmp5gf_ocv4/4f338cfd-b388-50e8-9d9c-fa4872e51b6c/global_event_df.parquet...\n", + "Dataset: mimic3\n", + "Dev mode: False\n", + "Number of patients: 46520\n", + "Number of events: 5214620\n" + ] + } + ], + "source": [ + "import tempfile\n", + "\n", + "from pyhealth.datasets import MIMIC3Dataset\n", + "\n", + "base_dataset = MIMIC3Dataset(\n", + " root=\"/home/lolowo2\",\n", + " tables=[\"DIAGNOSES_ICD\", \"PROCEDURES_ICD\", \"PRESCRIPTIONS\"],\n", + " cache_dir=tempfile.TemporaryDirectory().name,\n", + " dev=False,\n", + ")\n", + "\n", + "base_dataset.stats()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Define the Mortality Prediction Task\n", + "\n", + "We override the task's `input_schema` to enable `code_mapping` on each sequence feature.\n", + "This is the **only difference** from the baseline notebook." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Setting task MortalityPredictionMIMIC3 for mimic3 base dataset...\n", + "Task cache paths: task_df=/tmp/tmp5gf_ocv4/4f338cfd-b388-50e8-9d9c-fa4872e51b6c/tasks/MortalityPredictionMIMIC3_c67969dc-13b3-5ab7-977f-60956867cc5d/task_df.ld, samples=/tmp/tmp5gf_ocv4/4f338cfd-b388-50e8-9d9c-fa4872e51b6c/tasks/MortalityPredictionMIMIC3_c67969dc-13b3-5ab7-977f-60956867cc5d/samples_cdbbc602-34e2-5a41-8643-4c76b08829f6.ld\n", + "Applying task transformations on data with 1 workers...\n", + "Detected Jupyter notebook environment, setting num_workers to 1\n", + "Single worker mode, processing sequentially\n", + "Worker 0 started processing 46520 patients. (Polars threads: 16)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 0%| | 0/46520 [00:00, )\n", + "procedures: 204 codes (including , )\n", + "drugs: 2624 codes (including , )\n", + "\n", + "Total samples: 9583\n", + "Mortality rate: 12.05%\n", + "Positive samples: 1155\n", + "Negative samples: 8428\n" + ] + } + ], + "source": [ + "print(\"Sample structure:\")\n", + "print(samples[0])\n", + "\n", + "print(\"\\n\" + \"=\" * 50)\n", + "print(\"Processor Vocabulary Sizes:\")\n", + "print(\"=\" * 50)\n", + "for key, proc in samples.input_processors.items():\n", + " if hasattr(proc, 'code_vocab'):\n", + " print(f\"{key}: {len(proc.code_vocab)} codes (including , )\")\n", + "\n", + "mortality_count = sum(float(s.get(\"mortality\", 0)) for s in samples)\n", + "print(f\"\\nTotal samples: {len(samples)}\")\n", + "print(f\"Mortality rate: {mortality_count / len(samples) * 100:.2f}%\")\n", + "print(f\"Positive samples: {int(mortality_count)}\")\n", + "print(f\"Negative samples: {len(samples) - int(mortality_count)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Split the Dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Training samples: 7713\n", + "Validation samples: 909\n", + "Test samples: 961\n" + ] + } + ], + "source": [ + "from pyhealth.datasets import split_by_patient\n", + "\n", + "train_dataset, val_dataset, test_dataset = split_by_patient(\n", + " samples, [0.8, 0.1, 0.1], seed=42\n", + ")\n", + "\n", + "print(f\"Training samples: {len(train_dataset)}\")\n", + "print(f\"Validation samples: {len(val_dataset)}\")\n", + "print(f\"Test samples: {len(test_dataset)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Create Data Loaders" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Training batches: 242\n", + "Validation batches: 29\n", + "Test batches: 31\n" + ] + } + ], + "source": [ + "from pyhealth.datasets import get_dataloader\n", + "\n", + "train_dataloader = get_dataloader(train_dataset, batch_size=32, shuffle=True)\n", + "val_dataloader = get_dataloader(val_dataset, batch_size=32, shuffle=False)\n", + "test_dataloader = get_dataloader(test_dataset, batch_size=32, shuffle=False)\n", + "\n", + "print(f\"Training batches: {len(train_dataloader)}\")\n", + "print(f\"Validation batches: {len(val_dataloader)}\")\n", + "print(f\"Test batches: {len(test_dataloader)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Initialize the RNN Model" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model initialized with 693,889 parameters\n", + "\n", + "Model architecture:\n", + "RNN(\n", + " (embedding_model): EmbeddingModel(embedding_layers=ModuleDict(\n", + " (conditions): Embedding(268, 128, padding_idx=0)\n", + " (procedures): Embedding(204, 128, padding_idx=0)\n", + " (drugs): Embedding(2624, 128, padding_idx=0)\n", + " ))\n", + " (rnn): ModuleDict(\n", + " (conditions): RNNLayer(\n", + " (dropout_layer): Dropout(p=0.5, inplace=False)\n", + " (rnn): GRU(128, 128, batch_first=True)\n", + " )\n", + " (procedures): RNNLayer(\n", + " (dropout_layer): Dropout(p=0.5, inplace=False)\n", + " (rnn): GRU(128, 128, batch_first=True)\n", + " )\n", + " (drugs): RNNLayer(\n", + " (dropout_layer): Dropout(p=0.5, inplace=False)\n", + " (rnn): GRU(128, 128, batch_first=True)\n", + " )\n", + " )\n", + " (fc): Linear(in_features=384, out_features=1, bias=True)\n", + ")\n" + ] + } + ], + "source": [ + "from pyhealth.models import RNN\n", + "\n", + "model = RNN(\n", + " dataset=samples,\n", + " embedding_dim=128,\n", + " hidden_dim=128,\n", + ")\n", + "\n", + "print(f\"Model initialized with {sum(p.numel() for p in model.parameters()):,} parameters\")\n", + "print(f\"\\nModel architecture:\")\n", + "print(model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 7: Train the Model" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "RNN(\n", + " (embedding_model): EmbeddingModel(embedding_layers=ModuleDict(\n", + " (conditions): Embedding(268, 128, padding_idx=0)\n", + " (procedures): Embedding(204, 128, padding_idx=0)\n", + " (drugs): Embedding(2624, 128, padding_idx=0)\n", + " ))\n", + " (rnn): ModuleDict(\n", + " (conditions): RNNLayer(\n", + " (dropout_layer): Dropout(p=0.5, inplace=False)\n", + " (rnn): GRU(128, 128, batch_first=True)\n", + " )\n", + " (procedures): RNNLayer(\n", + " (dropout_layer): Dropout(p=0.5, inplace=False)\n", + " (rnn): GRU(128, 128, batch_first=True)\n", + " )\n", + " (drugs): RNNLayer(\n", + " (dropout_layer): Dropout(p=0.5, inplace=False)\n", + " (rnn): GRU(128, 128, batch_first=True)\n", + " )\n", + " )\n", + " (fc): Linear(in_features=384, out_features=1, bias=True)\n", + ")\n", + "Metrics: ['roc_auc', 'pr_auc', 'accuracy', 'f1']\n", + "Device: cuda\n", + "\n", + "Training:\n", + "Batch size: 32\n", + "Optimizer: \n", + "Optimizer params: {'lr': 0.001}\n", + "Weight decay: 0.0\n", + "Max grad norm: None\n", + "Val dataloader: \n", + "Monitor: roc_auc\n", + "Monitor criterion: max\n", + "Epochs: 50\n", + "Patience: None\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ce78ac0065654d1cbaa86f6c80d302ba", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Epoch 0 / 50: 0%| | 0/242 [00:00 torch.Tensor: + """Randomly select initial cluster centers from the dataset. + + Args: + dataset: tensor of shape [num_points, dimension]. + num_centers: number of cluster centers to select. + device: target device for the output tensor. + + Returns: + Tensor of shape [num_centers, dimension] with selected centers. + """ num_points = dataset.size(0) dimension = dataset.size(1) - # print("random size", dataset.size()) - # print("numcenter", num_centers) + num_centers = min(num_centers, num_points) indices = torch.tensor( np.array(random.sample(range(num_points), k=num_centers)), dtype=torch.long @@ -33,13 +53,22 @@ def random_init(dataset, num_centers, device): return centers -# Compute for each data point the closest center -def compute_codes(dataset, centers): +def compute_codes( + dataset: torch.Tensor, centers: torch.Tensor +) -> torch.Tensor: + """Assign each data point to its closest cluster center. + + Args: + dataset: tensor of shape [num_points, dimension]. + centers: tensor of shape [num_centers, dimension]. + + Returns: + Long tensor of shape [num_points] with cluster assignments. + """ num_points = dataset.size(0) dimension = dataset.size(1) num_centers = centers.size(0) - # print("size:", dataset.size(), centers.size()) # 5e8 should vary depending on the free memory on the GPU # Ideally, automatically ;) chunk_size = int(5e8 / num_centers) @@ -60,8 +89,23 @@ def compute_codes(dataset, centers): return codes -# Compute new centers as means of the data points forming the clusters -def update_centers(dataset, codes, num_centers, device): +def update_centers( + dataset: torch.Tensor, + codes: torch.Tensor, + num_centers: int, + device: torch.device, +) -> torch.Tensor: + """Recompute cluster centers as the mean of assigned data points. + + Args: + dataset: tensor of shape [num_points, dimension]. + codes: long tensor of shape [num_points] with cluster assignments. + num_centers: number of clusters. + device: target device for the output tensor. + + Returns: + Tensor of shape [num_centers, dimension] with updated centers. + """ num_points = dataset.size(0) dimension = dataset.size(1) centers = torch.zeros(num_centers, dimension, dtype=torch.float).to(device=device) @@ -77,7 +121,20 @@ def update_centers(dataset, codes, num_centers, device): return centers -def cluster(dataset, num_centers, device): +def cluster( + dataset: torch.Tensor, num_centers: int, device: torch.device +) -> Tuple[torch.Tensor, torch.Tensor]: + """Run k-means clustering until convergence or 1000 iterations. + + Args: + dataset: tensor of shape [num_points, dimension]. + num_centers: number of clusters. + device: target device for computation. + + Returns: + Tuple of (centers, codes) where centers has shape + [num_centers, dimension] and codes has shape [num_points]. + """ centers = random_init(dataset, num_centers, device) codes = compute_codes(dataset, centers) num_iterations = 0 @@ -96,7 +153,15 @@ def cluster(dataset, num_centers, device): class GraphConvolution(nn.Module): - def __init__(self, in_features, out_features, bias=True): + """Single-layer graph convolution (Kipf & Welling, ICLR 2017). + + Args: + in_features: size of each input sample. + out_features: size of each output sample. + bias: if ``True``, adds a learnable bias. Default: ``True``. + """ + + def __init__(self, in_features: int, out_features: int, bias: bool = True): super(GraphConvolution, self).__init__() self.in_features = in_features self.out_features = out_features @@ -141,13 +206,14 @@ class GRASPLayer(nn.Module): hidden_dim: hidden dimension of the GRASP layer, default 128. cluster_num: number of clusters, default 12. The cluster_num should be no more than the number of samples. dropout: dropout rate, default 0.5. - block: the backbone model used in the GRASP layer ('ConCare', 'LSTM' or 'GRU'), default 'ConCare'. + block: the backbone model used in the GRASP layer + ('ConCare', 'LSTM' or 'GRU'), default 'ConCare'. Examples: >>> from pyhealth.models import GRASPLayer - >>> input = torch.randn(3, 128, 64) # [batch size, sequence len, feature_size] + >>> x = torch.randn(3, 128, 64) # [batch, seq_len, feature_size] >>> layer = GRASPLayer(64, cluster_num=2) - >>> c = layer(input) + >>> c = layer(x) >>> c.shape torch.Size([3, 128]) """ @@ -158,7 +224,7 @@ def __init__( static_dim: int = 0, hidden_dim: int = 128, cluster_num: int = 2, - dropout: int = 0.5, + dropout: float = 0.5, block: str = "ConCare", ): super(GRASPLayer, self).__init__() @@ -221,17 +287,30 @@ def gumbel_softmax(self, logits, temperature, device, hard=False): y_hard = (y_hard - y).detach() + y return y_hard - def grasp_encoder(self, input, static=None, mask=None): + def grasp_encoder( + self, + input: torch.Tensor, + static: Optional[torch.Tensor] = None, + mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Encode patient sequences with backbone + cluster-aware GCN. + Args: + input: tensor of shape [batch_size, seq_len, input_dim]. + static: optional static features [batch_size, static_dim]. + mask: optional mask [batch_size, seq_len]. + + Returns: + Tensor of shape [batch_size, hidden_dim]. + """ if self.block == "ConCare": hidden_t, _ = self.backbone(input, mask=mask, static=static) else: _, hidden_t = self.backbone(input, mask) - hidden_t = torch.squeeze(hidden_t, 0) centers, codes = cluster(hidden_t, self.cluster_num, input.device) - if self.A_mat == None: + if self.A_mat is None: A_mat = np.eye(self.cluster_num) else: A_mat = kneighbors_graph( @@ -287,72 +366,80 @@ def forward( class GRASP(BaseModel): - """GRASP model for EHR-based prediction tasks. - - GRASP (Generic framework for health status Representation learning - bAsed on incorporating knowledge from Similar Patients) uses graph-based - clustering to capture patient similarity and enhance temporal modeling. + """GRASP model. Paper: Liantao Ma et al. GRASP: generic framework for health status representation learning based on incorporating knowledge from similar patients. AAAI 2021. - Note: - We use separate GRASP layers for different feature_keys. - The model automatically handles different input formats through the - EmbeddingModel. + This model applies a separate GRASP layer for each feature, and then + concatenates the outputs. The concatenated representations are fed into + a fully connected layer to make predictions. + + The GRASP layer encodes patient sequences with a backbone (ConCare, GRU, + or LSTM), clusters patients via k-means, refines cluster representations + with a 2-layer GCN, and blends cluster-level knowledge back into + individual patient representations via a learned gating mechanism. Args: - dataset: The dataset to train the model. It is used to query certain - information such as the set of all tokens. - static_key: The key in samples to use as static features, e.g. - "demographics". Default is None. Only numerical static features - are supported. - embedding_dim: The embedding dimension. Default is 128. - hidden_dim: The hidden dimension. Default is 128. - **kwargs: Other parameters for the GRASP layer (cluster_num, block, - dropout). + dataset (SampleDataset): the dataset to train the model. It is used + to query certain information such as the set of all tokens. + static_key (str): optional key in samples to use as static features, + e.g. "demographics". Only numerical static features are supported. + Default is None. + embedding_dim (int): the embedding dimension. Default is 128. + hidden_dim (int): the hidden dimension. Default is 128. + **kwargs: other parameters for the GRASPLayer + (e.g., cluster_num, dropout, block). Examples: - >>> from pyhealth.datasets import SampleDataset + >>> from pyhealth.datasets import create_sample_dataset >>> samples = [ ... { ... "patient_id": "patient-0", ... "visit_id": "visit-0", - ... "list_codes": ["505800458", "50580045810", "50580045811"], - ... "list_vectors": [[1.0, 2.55, 3.4], [4.1, 5.5, 6.0]], - ... "demographic": [0.0, 2.0, 1.5], + ... "conditions": ["cond-33", "cond-86", "cond-80"], + ... "procedures": ["proc-12", "proc-45"], ... "label": 1, ... }, ... { - ... "patient_id": "patient-0", + ... "patient_id": "patient-1", ... "visit_id": "visit-1", - ... "list_codes": ["55154191800", "551541928", "55154192800"], - ... "list_vectors": [[1.4, 3.2, 3.5], [4.1, 5.9, 1.7]], - ... "demographic": [0.0, 2.0, 1.5], + ... "conditions": ["cond-12", "cond-52"], + ... "procedures": ["proc-23"], ... "label": 0, ... }, ... ] - >>> dataset = SampleDataset( + >>> dataset = create_sample_dataset( ... samples=samples, - ... input_schema={"list_codes": "sequence", "list_vectors": "sequence"}, + ... input_schema={ + ... "conditions": "sequence", + ... "procedures": "sequence", + ... }, ... output_schema={"label": "binary"}, - ... dataset_name="test" + ... dataset_name="test", ... ) - >>> from pyhealth.models import GRASP + >>> + >>> from pyhealth.datasets import get_dataloader + >>> train_loader = get_dataloader(dataset, batch_size=2, shuffle=True) + >>> >>> model = GRASP( ... dataset=dataset, - ... static_key="demographic", - ... embedding_dim=64, + ... embedding_dim=128, ... hidden_dim=64, ... cluster_num=2, ... ) - >>> from pyhealth.datasets import get_dataloader - >>> train_loader = get_dataloader(dataset, batch_size=2, shuffle=True) + >>> >>> data_batch = next(iter(train_loader)) + >>> >>> ret = model(**data_batch) - >>> print(ret["loss"]) - tensor(..., grad_fn=) + >>> print(ret) + { + 'loss': tensor(...), + 'y_prob': tensor(...), + 'y_true': tensor(...), + 'logit': tensor(...) + } """ def __init__( @@ -361,34 +448,26 @@ def __init__( static_key: Optional[str] = None, embedding_dim: int = 128, hidden_dim: int = 128, - **kwargs, + **kwargs ): - super(GRASP, self).__init__(dataset=dataset) - + super(GRASP, self).__init__( + dataset=dataset, + ) self.embedding_dim = embedding_dim self.hidden_dim = hidden_dim self.static_key = static_key # validate kwargs for GRASP layer - if "feature_size" in kwargs: - raise ValueError("feature_size is determined by embedding_dim") - - cluster_num = kwargs.get("cluster_num", 12) - if len(dataset) < cluster_num: - raise ValueError( - f"cluster_num ({cluster_num}) must be no larger than " - f"dataset size ({len(dataset)})" - ) + if "input_dim" in kwargs: + raise ValueError("input_dim is determined by embedding_dim") - assert len(self.label_keys) == 1, ( - "Only one label key is supported for GRASP" - ) + assert len(self.label_keys) == 1, "Only one label key is supported" self.label_key = self.label_keys[0] + self.mode = self.dataset.output_schema[self.label_key] - # EmbeddingModel handles all feature embedding automatically self.embedding_model = EmbeddingModel(dataset, embedding_dim) - # Determine static dimension + # Determine static feature dimension self.static_dim = 0 if self.static_key is not None: first_sample = dataset[0] @@ -403,19 +482,18 @@ def __init__( else: self.static_dim = 1 - # Get dynamic feature keys (excluding static key) + # Dynamic feature keys (exclude static key) self.dynamic_feature_keys = [ - k for k in self.feature_keys - if k != self.static_key + k for k in self.feature_keys if k != self.static_key ] - # GRASP layers for each dynamic feature + # one GRASPLayer per dynamic feature self.grasp = nn.ModuleDict() for feature_key in self.dynamic_feature_keys: self.grasp[feature_key] = GRASPLayer( input_dim=embedding_dim, static_dim=self.static_dim, - hidden_dim=self.hidden_dim, + hidden_dim=hidden_dim, **kwargs, ) @@ -427,52 +505,47 @@ def __init__( def forward(self, **kwargs) -> Dict[str, torch.Tensor]: """Forward propagation. + The label `kwargs[self.label_key]` is a list of labels for each + patient. + Args: **kwargs: keyword arguments for the model. The keys must contain all the feature keys and the label key. Returns: Dict[str, torch.Tensor]: A dictionary with the following keys: - - loss: a scalar tensor representing the final loss. + - loss: a scalar tensor representing the loss. - y_prob: a tensor representing the predicted probabilities. - y_true: a tensor representing the true labels. - logit: a tensor representing the logits. + - embed (optional): a tensor representing the patient + embeddings if requested. """ patient_emb = [] + embedded = self.embedding_model(kwargs) - embedded, masks = self.embedding_model(kwargs, output_mask=True) - - # Get static features if available + # Extract static features if configured static = None if self.static_key is not None and self.static_key in kwargs: - static_data = kwargs[self.static_key] - if isinstance(static_data, torch.Tensor): - static = static_data.float().to(self.device) - else: - static = torch.tensor( - static_data, dtype=torch.float, device=self.device - ) + static = kwargs[self.static_key] + if isinstance(static, (list, tuple)): + static = torch.tensor(static, dtype=torch.float) + static = static.to(self.device) for feature_key in self.dynamic_feature_keys: x = embedded[feature_key] - mask = masks[feature_key] + mask = (torch.abs(x).sum(dim=-1) != 0).int() x = self.grasp[feature_key](x, static=static, mask=mask) patient_emb.append(x) patient_emb = torch.cat(patient_emb, dim=1) + # (patient, label_size) logits = self.fc(patient_emb) - - # Compute loss and predictions + # obtain y_true, loss, y_prob y_true = kwargs[self.label_key].to(self.device) loss = self.get_loss_function()(logits, y_true) y_prob = self.prepare_y_prob(logits) - - results = { - "loss": loss, - "y_prob": y_prob, - "y_true": y_true, - "logit": logits, - } + results = {"loss": loss, "y_prob": y_prob, "y_true": y_true, "logit": logits} if kwargs.get("embed", False): results["embed"] = patient_emb return results @@ -485,35 +558,24 @@ def forward(self, **kwargs) -> Dict[str, torch.Tensor]: { "patient_id": "patient-0", "visit_id": "visit-0", - "list_codes": ["505800458", "50580045810", "50580045811"], - "list_vectors": [[1.0, 2.55, 3.4], [4.1, 5.5, 6.0]], - "list_list_codes": [["A05B", "A05C", "A06A"], ["A11D", "A11E"]], + "conditions": ["cond-33", "cond-86", "cond-80"], + "procedures": ["proc-12", "proc-45"], "label": 1, - "demographic": [1.0, 2.0, 1.3], }, { - "patient_id": "patient-0", + "patient_id": "patient-1", "visit_id": "visit-1", - "list_codes": [ - "55154191800", - "551541928", - "55154192800", - "705182798", - "70518279800", - ], - "list_vectors": [[1.4, 3.2, 3.5], [4.1, 5.9, 1.7], [4.5, 5.9, 1.7]], - "list_list_codes": [["A04A", "B035", "C129"]], + "conditions": ["cond-12", "cond-52"], + "procedures": ["proc-23"], "label": 0, - "demographic": [1.0, 2.0, 1.3], }, ] dataset = create_sample_dataset( samples=samples, input_schema={ - "list_codes": "sequence", - "list_vectors": "sequence", - "list_list_codes": "sequence", + "conditions": "sequence", + "procedures": "sequence", }, output_schema={"label": "binary"}, dataset_name="test", @@ -523,13 +585,13 @@ def forward(self, **kwargs) -> Dict[str, torch.Tensor]: model = GRASP( dataset=dataset, - static_key="demographic", - embedding_dim=64, - hidden_dim=64, + embedding_dim=32, + hidden_dim=32, cluster_num=2, ) data_batch = next(iter(train_loader)) ret = model(**data_batch) print(ret) + ret["loss"].backward() diff --git a/pyhealth/models/rnn.py b/pyhealth/models/rnn.py index 4c9ba0550..3393d7287 100644 --- a/pyhealth/models/rnn.py +++ b/pyhealth/models/rnn.py @@ -92,7 +92,9 @@ def forward( Args: x: a tensor of shape [batch size, sequence len, input size]. mask: an optional tensor of shape [batch size, sequence len], where - 1 indicates valid and 0 indicates invalid. + 1 indicates valid and 0 indicates invalid. Samples with all-zero + masks are clamped to length 1 to prevent pack_padded_sequence + from receiving zero-length sequences. Returns: outputs: a tensor of shape [batch size, sequence len, hidden size], @@ -109,10 +111,13 @@ def forward( ) else: lengths = torch.sum(mask.int(), dim=-1).cpu() + # Clamp lengths to at least 1 to handle empty sequences, + # matching TCNLayer (tcn.py:186). + lengths = torch.clamp(lengths, min=1) # Ensure tensor is contiguous for cuDNN compatibility x = x.contiguous() x = rnn_utils.pack_padded_sequence( - x.contiguous(), lengths, batch_first=True, enforce_sorted=False + x, lengths, batch_first=True, enforce_sorted=False ) outputs, _ = self.rnn(x) outputs, _ = rnn_utils.pad_packed_sequence(outputs, batch_first=True) diff --git a/pyhealth/processors/sequence_processor.py b/pyhealth/processors/sequence_processor.py index 7792709bb..47339eefd 100644 --- a/pyhealth/processors/sequence_processor.py +++ b/pyhealth/processors/sequence_processor.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Iterable +from typing import Any, Dict, List, Iterable, Optional, Tuple import torch @@ -8,25 +8,60 @@ @register_processor("sequence") class SequenceProcessor(FeatureProcessor, TokenProcessorInterface): + """Feature processor for encoding categorical sequences. + + Encodes medical codes (e.g., diagnoses, procedures) into numerical + indices. Supports single or multiple tokens and can build vocabulary + on the fly if not provided. + + Args: + code_mapping: optional tuple of (source_vocabulary, target_vocabulary) + to map raw codes to a grouped vocabulary before tokenizing. + Uses ``pyhealth.medcode.CrossMap`` internally. For example, + ``("ICD9CM", "CCSCM")`` maps ~128K ICD-9 diagnosis codes to + ~280 CCS categories, and ``("NDC", "ATC")`` maps ~940K drug + codes to ~5K ATC categories. When None (default), codes are + used as-is with no change to existing behavior. + + Examples: + >>> proc = SequenceProcessor() # no mapping, same as before + >>> proc = SequenceProcessor(code_mapping=("ICD9CM", "CCSCM")) """ - Feature processor for encoding categorical sequences (e.g., medical codes) into numerical indices. - Supports single or multiple tokens (e.g., single diagnosis or list of procedures). - Can build vocabulary on the fly if not provided. - """ - - def __init__(self): + def __init__(self, code_mapping: Optional[Tuple[str, str]] = None): self.code_vocab: Dict[Any, int] = {"": self.PAD, "": self.UNK} self._next_index = 2 + self._mapper = None + if code_mapping is not None: + from pyhealth.medcode import CrossMap + self._mapper = CrossMap.load(code_mapping[0], code_mapping[1]) + + def _map(self, token: str) -> List[str]: + """Map a single token through the code mapping, if configured. + + Returns the token unchanged (as a single-element list) when no + mapping is configured or when the token has no mapping. + """ + if self._mapper is None: + return [token] + mapped = self._mapper.map(token) + return mapped if mapped else [token] def fit(self, samples: Iterable[Dict[str, Any]], field: str) -> None: + """Build vocabulary from samples, applying code mapping if set. + + Args: + samples: iterable of sample dicts. + field: key whose values are token lists. + """ for sample in samples: for token in sample[field]: if token is None: continue # skip missing values - elif token not in self.code_vocab: - self.code_vocab[token] = self._next_index - self._next_index += 1 + for mapped in self._map(token): + if mapped not in self.code_vocab: + self.code_vocab[mapped] = self._next_index + self._next_index += 1 def process(self, value: Any) -> torch.Tensor: """Process token value(s) into tensor of indices. @@ -39,10 +74,13 @@ def process(self, value: Any) -> torch.Tensor: """ indices = [] for token in value: - if token in self.code_vocab: - indices.append(self.code_vocab[token]) - else: - indices.append(self.code_vocab[""]) + if token is None: + continue # skip missing values, consistent with fit() + for mapped in self._map(token): + if mapped in self.code_vocab: + indices.append(self.code_vocab[mapped]) + else: + indices.append(self.code_vocab[""]) return torch.tensor(indices, dtype=torch.long) @@ -50,14 +88,12 @@ def remove(self, tokens: set[str]): """Remove specified vocabularies from the processor.""" keep = set(self.code_vocab.keys()) - tokens | {"", ""} order = [k for k, v in sorted(self.code_vocab.items(), key=lambda x: x[1]) if k in keep] - self.code_vocab = { k : i for i, k in enumerate(order) } def retain(self, tokens: set[str]): """Retain only the specified vocabularies in the processor.""" keep = set(self.code_vocab.keys()) & tokens | {"", ""} order = [k for k, v in sorted(self.code_vocab.items(), key=lambda x: x[1]) if k in keep] - self.code_vocab = { k : i for i, k in enumerate(order) } def add(self, tokens: set[str]): diff --git a/pyhealth/tasks/base_task.py b/pyhealth/tasks/base_task.py index 888c7e2e1..395686ed7 100644 --- a/pyhealth/tasks/base_task.py +++ b/pyhealth/tasks/base_task.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import Dict, List, Union, Type +from typing import Dict, List, Optional, Tuple, Union, Type import polars as pl @@ -9,6 +9,41 @@ class BaseTask(ABC): input_schema: Dict[str, Union[str, Type]] output_schema: Dict[str, Union[str, Type]] + def __init__( + self, + code_mapping: Optional[Dict[str, Tuple[str, str]]] = None, + ): + """Initialize a task with optional code mapping. + + Args: + code_mapping: optional dict mapping feature keys to + ``(source_vocab, target_vocab)`` tuples. For example:: + + code_mapping={ + "conditions": ("ICD9CM", "CCSCM"), + "procedures": ("ICD9PROC", "CCSPROC"), + "drugs": ("NDC", "ATC"), + } + + When provided, the corresponding ``input_schema`` entries + are upgraded from ``"sequence"`` to + ``("sequence", {"code_mapping": (src, tgt)})`` so that the + ``SequenceProcessor`` maps raw codes at fit/process time. + """ + if code_mapping is not None: + schema = dict(self.input_schema) + for field, mapping in code_mapping.items(): + if field in schema: + base = schema[field] + if isinstance(base, tuple): + base, kwargs = base + kwargs = dict(kwargs) + else: + kwargs = {} + kwargs["code_mapping"] = mapping + schema[field] = (base, kwargs) + self.input_schema = schema + def pre_filter(self, df: pl.LazyFrame) -> pl.LazyFrame: return df diff --git a/pyhealth/tasks/drug_recommendation.py b/pyhealth/tasks/drug_recommendation.py index 5fdcb1ca7..660343bed 100644 --- a/pyhealth/tasks/drug_recommendation.py +++ b/pyhealth/tasks/drug_recommendation.py @@ -89,7 +89,7 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: return_df=True, ) drugs = ( - prescriptions.select(pl.col("prescriptions/drug")).to_series().to_list() + prescriptions.select(pl.col("prescriptions/ndc")).to_series().to_list() ) # ATC 3 level (first 4 characters) diff --git a/pyhealth/tasks/mortality_prediction.py b/pyhealth/tasks/mortality_prediction.py index 9ecf81ea9..249f717f3 100644 --- a/pyhealth/tasks/mortality_prediction.py +++ b/pyhealth/tasks/mortality_prediction.py @@ -63,7 +63,7 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: conditions = [event.icd9_code for event in diagnoses] procedures_list = [event.icd9_code for event in procedures] - drugs = [event.drug for event in prescriptions] + drugs = [event.ndc for event in prescriptions if event.ndc] # Exclude visits without condition, procedure, or drug code if len(conditions) * len(procedures_list) * len(drugs) == 0: @@ -147,7 +147,7 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: ) conditions = [event.icd9_code for event in diagnoses] procedures_list = [event.icd9_code for event in procedures] - drugs = [event.drug for event in prescriptions] + drugs = [event.ndc for event in prescriptions if event.ndc] # Extract note text - concatenate if multiple exist text = "" for note in notes: @@ -279,7 +279,7 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: [getattr(event, "icd_code", None) for event in procedures_icd] ) drugs = self._clean_sequence( - [getattr(event, "drug", None) for event in prescriptions] + [getattr(event, "ndc", None) for event in prescriptions] ) # Exclude visits without condition, procedure, or drug code @@ -387,8 +387,13 @@ class MultimodalMortalityPredictionMIMIC4(BaseTask): item for itemids in LAB_CATEGORIES.values() for item in itemids ] - def __init__(self): - """Initialize the multimodal mortality prediction task.""" + def __init__(self, **kwargs): + """Initialize the multimodal mortality prediction task. + + Args: + **kwargs: Passed to :class:`~pyhealth.tasks.BaseTask`, e.g. + ``code_mapping``. + """ self.input_schema: Dict[str, str] = { "conditions": "nested_sequence", # Nested by visit "procedures": "nested_sequence", # Nested by visit @@ -401,6 +406,7 @@ def __init__(self): "image_path": "text", # Image path as text string } self.output_schema: Dict[str, str] = {"mortality": "binary"} + super().__init__(**kwargs) def _clean_sequence(self, sequence: Optional[List[Any]]) -> List[str]: """Clean a sequence by removing None values and converting to strings.""" diff --git a/pyhealth/tasks/readmission_prediction.py b/pyhealth/tasks/readmission_prediction.py index 88ecc0109..e05223d45 100644 --- a/pyhealth/tasks/readmission_prediction.py +++ b/pyhealth/tasks/readmission_prediction.py @@ -37,15 +37,19 @@ class ReadmissionPredictionMIMIC3(BaseTask): output_schema: Dict[str, str] = {"readmission": "binary"} def __init__( - self, window: timedelta = timedelta(days=15), exclude_minors: bool = True + self, window: timedelta = timedelta(days=15), exclude_minors: bool = True, **kwargs ) -> None: - """ - Initializes the task object. + """Initializes the task object. Args: - window (timedelta): If two admissions are closer than this window, it is considered a readmission. Defaults to 15 days. - exclude_minors (bool): Whether to exclude visits where the patient was under 18 years old. Defaults to True. + window: If two admissions are closer than this window, it is + considered a readmission. Defaults to 15 days. + exclude_minors: Whether to exclude visits where the patient + was under 18 years old. Defaults to True. + **kwargs: Passed to :class:`~pyhealth.tasks.BaseTask`, e.g. + ``code_mapping``. """ + super().__init__(**kwargs) self.window = window self.exclude_minors = exclude_minors @@ -64,7 +68,7 @@ def __call__(self, patient: Patient) -> List[Dict]: - 'patient_id': MIMIC3 subject_id. - 'conditions': MIMIC3 diagnoses_icd table ICD-9 codes. - 'procedures': MIMIC3 procedures_icd table ICD-9 codes. - - 'drugs': MIMIC3 prescriptions table drug column entries. + - 'drugs': MIMIC3 prescriptions table NDC (National Drug Code) entries. - 'readmission': binary label. Raises: @@ -110,7 +114,7 @@ def __call__(self, patient: Patient) -> List[Dict]: prescriptions = patient.get_events( event_type="prescriptions", filters=[filter] ) - prescriptions = [event.drug for event in prescriptions] + prescriptions = [event.ndc for event in prescriptions if event.ndc] if len(prescriptions) == 0: continue @@ -171,15 +175,19 @@ class ReadmissionPredictionMIMIC4(BaseTask): output_schema: Dict[str, str] = {"readmission": "binary"} def __init__( - self, window: timedelta = timedelta(days=15), exclude_minors: bool = True + self, window: timedelta = timedelta(days=15), exclude_minors: bool = True, **kwargs ) -> None: - """ - Initializes the task object. + """Initializes the task object. Args: - window (timedelta): If two admissions are closer than this window, it is considered a readmission. Defaults to 15 days. - exclude_minors (bool): Whether to exclude patients whose "anchor_age" is less than 18. Defaults to True. + window: If two admissions are closer than this window, it is + considered a readmission. Defaults to 15 days. + exclude_minors: Whether to exclude patients whose + ``anchor_age`` is less than 18. Defaults to True. + **kwargs: Passed to :class:`~pyhealth.tasks.BaseTask`, e.g. + ``code_mapping``. """ + super().__init__(**kwargs) self.window = window self.exclude_minors = exclude_minors @@ -198,7 +206,7 @@ def __call__(self, patient: Patient) -> List[Dict]: - 'patient_id': MIMIC4 subject_id. - 'conditions': MIMIC4 diagnoses_icd table ICD-9 or ICD-10 codes. - 'procedures': MIMIC4 procedures_icd table ICD-9 or ICD-10 codes. - - 'drugs': MIMIC4 prescriptions table drug column entries. + - 'drugs': MIMIC4 prescriptions table NDC (National Drug Code) entries. - 'readmission': binary label. Raises: @@ -241,7 +249,7 @@ def __call__(self, patient: Patient) -> List[Dict]: prescriptions = patient.get_events( event_type="prescriptions", filters=[filter] ) - prescriptions = [event.drug for event in prescriptions] + prescriptions = [event.ndc for event in prescriptions if event.ndc] if len(prescriptions) == 0: continue @@ -308,14 +316,16 @@ class ReadmissionPredictionEICU(BaseTask): } output_schema: Dict[str, str] = {"readmission": "binary"} - def __init__(self, exclude_minors: bool = True) -> None: - """ - Initializes the task object. + def __init__(self, exclude_minors: bool = True, **kwargs) -> None: + """Initializes the task object. Args: - exclude_minors (bool): Whether to exclude patients whose age is less than 18. - Defaults to True. + exclude_minors: Whether to exclude patients whose age is + less than 18. Defaults to True. + **kwargs: Passed to :class:`~pyhealth.tasks.BaseTask`, e.g. + ``code_mapping``. """ + super().__init__(**kwargs) self.exclude_minors = exclude_minors def __call__(self, patient: Patient) -> List[Dict]: @@ -452,15 +462,19 @@ class ReadmissionPredictionOMOP(BaseTask): output_schema: Dict[str, str] = {"readmission": "binary"} def __init__( - self, window: timedelta = timedelta(days=15), exclude_minors: bool = True + self, window: timedelta = timedelta(days=15), exclude_minors: bool = True, **kwargs ) -> None: - """ - Initializes the task object. + """Initializes the task object. Args: - window (timedelta): If two admissions are closer than this window, it is considered a readmission. Defaults to 15 days. - exclude_minors (bool): Whether to exclude visits where the patient was under 18 years old. Defaults to True. + window: If two admissions are closer than this window, it is + considered a readmission. Defaults to 15 days. + exclude_minors: Whether to exclude visits where the patient + was under 18 years old. Defaults to True. + **kwargs: Passed to :class:`~pyhealth.tasks.BaseTask`, e.g. + ``code_mapping``. """ + super().__init__(**kwargs) self.window = window self.exclude_minors = exclude_minors diff --git a/test-resources/core/mimic4demo/hosp/prescriptions.csv b/test-resources/core/mimic4demo/hosp/prescriptions.csv index 3cd4672fa..be65abe68 100644 --- a/test-resources/core/mimic4demo/hosp/prescriptions.csv +++ b/test-resources/core/mimic4demo/hosp/prescriptions.csv @@ -23,9 +23,9 @@ subject_id,hadm_id,starttime,stoptime,drug,ndc,prod_strength,dose_val_rx,dose_un 10009,20014,2152-03-20 08:00:00,2152-03-23 13:00:00,Atorvastatin,00378377710,20 mg,20,MG,PO 10010,20015,2150-08-01 11:00:00,2150-08-05 14:00:00,Insulin Lispro,00002751001,100 unit/mL,8,UNIT,SC 10010,20015,2150-08-01 11:00:00,2150-08-05 14:00:00,Potassium Chloride,00338001404,20 mEq/100mL,40,MEQ,IV -1,1,,,,,,,, +1,1,2150-01-01 00:00:00,2150-01-01 01:00:00,TestDrug,00000000001,,,, 1,2,,,,,,,, -2,3,,,,,,,, -2,4,,,,,,,, +2,3,2150-01-01 00:00:00,2150-01-01 01:00:00,TestDrug,00000000001,,,, +2,4,2150-01-16 00:00:00,2150-01-16 01:00:00,TestDrug,00000000001,,,, 2,6,,,,,,,, 3,7,,,,,,,, diff --git a/tests/core/test_code_mapping.py b/tests/core/test_code_mapping.py new file mode 100644 index 000000000..c18509f4a --- /dev/null +++ b/tests/core/test_code_mapping.py @@ -0,0 +1,313 @@ +import unittest +from unittest.mock import patch, MagicMock + +from pyhealth.processors import SequenceProcessor + + +class TestCodeMappingSequenceProcessor(unittest.TestCase): + """Tests for the code_mapping feature in SequenceProcessor. + + Verifies backward compatibility when code_mapping=None (default) + and correct vocabulary collapsing when a mapping is provided. + """ + + # -- Backward compatibility (code_mapping=None) -- + + def test_default_no_mapping_fit(self): + """Without code_mapping, fit builds vocabulary from raw codes.""" + proc = SequenceProcessor() + samples = [ + {"codes": ["A", "B", "C"]}, + {"codes": ["D", "E"]}, + ] + proc.fit(samples, "codes") + vocab_keys = set(proc.code_vocab.keys()) + self.assertTrue({"A", "B", "C", "D", "E"}.issubset(vocab_keys)) + # 5 codes + + + self.assertEqual(proc.size(), 7) + + def test_default_no_mapping_process(self): + """Without code_mapping, process returns correct indices.""" + proc = SequenceProcessor() + samples = [{"codes": ["A", "B", "C"]}] + proc.fit(samples, "codes") + + result = proc.process(["A", "B", "C"]) + self.assertEqual(len(result), 3) + self.assertEqual(result[0].item(), proc.code_vocab["A"]) + self.assertEqual(result[1].item(), proc.code_vocab["B"]) + self.assertEqual(result[2].item(), proc.code_vocab["C"]) + + def test_default_no_mapping_unknown_token(self): + """Without code_mapping, unknown tokens map to .""" + proc = SequenceProcessor() + proc.fit([{"codes": ["A"]}], "codes") + + result = proc.process(["A", "UNKNOWN"]) + self.assertEqual(result[1].item(), proc.code_vocab[""]) + + # -- With code_mapping -- + + def _make_mock_crossmap(self, mapping_dict): + """Helper: create a mock CrossMap that maps according to a dict.""" + mock_cm = MagicMock() + mock_cm.map.side_effect = lambda code: mapping_dict.get(code, []) + return mock_cm + + def test_mapping_collapses_vocabulary(self): + """Multiple raw codes mapping to the same target produce one vocab entry.""" + mapping = { + "428.0": ["108"], + "428.1": ["108"], + "428.20": ["108"], + "250.00": ["49"], + "250.01": ["49"], + } + with patch( + "pyhealth.medcode.CrossMap" + ) as MockCrossMap: + MockCrossMap.load.return_value = self._make_mock_crossmap(mapping) + proc = SequenceProcessor(code_mapping=("ICD9CM", "CCSCM")) + + samples = [ + {"codes": ["428.0", "428.1", "428.20"]}, + {"codes": ["250.00", "250.01"]}, + ] + proc.fit(samples, "codes") + vocab_keys = set(proc.code_vocab.keys()) + # Should contain mapped codes, not raw codes + self.assertIn("108", vocab_keys) + self.assertIn("49", vocab_keys) + self.assertNotIn("428.0", vocab_keys) + self.assertNotIn("250.00", vocab_keys) + # 2 mapped codes + + + self.assertEqual(proc.size(), 4) + + def test_mapping_process_uses_mapped_codes(self): + """process() maps codes before looking up indices.""" + mapping = { + "428.0": ["108"], + "250.00": ["49"], + } + with patch( + "pyhealth.medcode.CrossMap" + ) as MockCrossMap: + MockCrossMap.load.return_value = self._make_mock_crossmap(mapping) + proc = SequenceProcessor(code_mapping=("ICD9CM", "CCSCM")) + + proc.fit([{"codes": ["428.0", "250.00"]}], "codes") + result = proc.process(["428.0", "250.00"]) + + self.assertEqual(result[0].item(), proc.code_vocab["108"]) + self.assertEqual(result[1].item(), proc.code_vocab["49"]) + + def test_unmapped_codes_fall_through(self): + """Codes without a mapping are kept as-is (fallback to raw code).""" + mapping = { + "428.0": ["108"], + # "UNKNOWN_CODE" has no mapping + } + with patch( + "pyhealth.medcode.CrossMap" + ) as MockCrossMap: + MockCrossMap.load.return_value = self._make_mock_crossmap(mapping) + proc = SequenceProcessor(code_mapping=("ICD9CM", "CCSCM")) + + samples = [{"codes": ["428.0", "UNKNOWN_CODE"]}] + proc.fit(samples, "codes") + vocab_keys = set(proc.code_vocab.keys()) + # Mapped code + self.assertIn("108", vocab_keys) + # Unmapped code kept as-is + self.assertIn("UNKNOWN_CODE", vocab_keys) + + result = proc.process(["428.0", "UNKNOWN_CODE"]) + self.assertEqual(result[0].item(), proc.code_vocab["108"]) + self.assertEqual(result[1].item(), proc.code_vocab["UNKNOWN_CODE"]) + + def test_one_to_many_mapping(self): + """A single code mapping to multiple targets expands correctly.""" + mapping = { + "COMBO_CODE": ["TARGET_A", "TARGET_B"], + } + with patch( + "pyhealth.medcode.CrossMap" + ) as MockCrossMap: + MockCrossMap.load.return_value = self._make_mock_crossmap(mapping) + proc = SequenceProcessor(code_mapping=("SRC", "TGT")) + + proc.fit([{"codes": ["COMBO_CODE"]}], "codes") + vocab_keys = set(proc.code_vocab.keys()) + self.assertIn("TARGET_A", vocab_keys) + self.assertIn("TARGET_B", vocab_keys) + self.assertNotIn("COMBO_CODE", vocab_keys) + + result = proc.process(["COMBO_CODE"]) + self.assertEqual(len(result), 2) + self.assertEqual(result[0].item(), proc.code_vocab["TARGET_A"]) + self.assertEqual(result[1].item(), proc.code_vocab["TARGET_B"]) + + def test_vocab_size_reduction(self): + """Demonstrates the real-world impact: many raw codes → few mapped codes.""" + # Simulate 100 raw ICD9 codes all mapping to 5 CCS categories + mapping = {} + for i in range(100): + category = str(i % 5) + mapping[f"ICD9_{i}"] = [category] + + with patch( + "pyhealth.medcode.CrossMap" + ) as MockCrossMap: + MockCrossMap.load.return_value = self._make_mock_crossmap(mapping) + proc = SequenceProcessor(code_mapping=("ICD9CM", "CCSCM")) + + samples = [{"codes": [f"ICD9_{i}" for i in range(100)]}] + proc.fit(samples, "codes") + # 5 CCS categories + + = 7 + self.assertEqual(proc.size(), 7) + + def test_none_tokens_skipped_with_mapping(self): + """None tokens are still skipped when mapping is active.""" + mapping = {"A": ["X"]} + with patch( + "pyhealth.medcode.CrossMap" + ) as MockCrossMap: + MockCrossMap.load.return_value = self._make_mock_crossmap(mapping) + proc = SequenceProcessor(code_mapping=("SRC", "TGT")) + + proc.fit([{"codes": ["A", None, "A"]}], "codes") + # Only "X" + + + self.assertEqual(proc.size(), 3) + + def test_repr_unchanged(self): + """__repr__ still works with or without mapping.""" + proc = SequenceProcessor() + self.assertIn("code_vocab_size=2", repr(proc)) + + def test_remove_retain_add_work_with_mapping(self): + """Existing vocab methods (remove, retain, add) work on mapped codes.""" + mapping = { + "428.0": ["108"], + "250.00": ["49"], + "401.9": ["99"], + } + with patch( + "pyhealth.medcode.CrossMap" + ) as MockCrossMap: + MockCrossMap.load.return_value = self._make_mock_crossmap(mapping) + proc = SequenceProcessor(code_mapping=("ICD9CM", "CCSCM")) + + proc.fit([{"codes": ["428.0", "250.00", "401.9"]}], "codes") + self.assertEqual(proc.size(), 5) # 3 codes + pad + unk + + # Remove mapped code "108" + proc.remove({"108"}) + self.assertNotIn("108", proc.code_vocab) + self.assertIn("49", proc.code_vocab) + + # Retain only "49" + proc.retain({"49"}) + self.assertIn("49", proc.code_vocab) + self.assertNotIn("99", proc.code_vocab) + + # Add new code + proc.add({"NEW"}) + self.assertIn("NEW", proc.code_vocab) + + +class TestTaskCodeMappingInit(unittest.TestCase): + """Tests for passing code_mapping as a task __init__ argument.""" + + def test_no_code_mapping_leaves_schema_unchanged(self): + """Tasks without code_mapping keep simple string schema.""" + from pyhealth.tasks import MortalityPredictionMIMIC3 + + task = MortalityPredictionMIMIC3() + self.assertEqual(task.input_schema["conditions"], "sequence") + self.assertEqual(task.input_schema["procedures"], "sequence") + self.assertEqual(task.input_schema["drugs"], "sequence") + + def test_code_mapping_upgrades_schema_to_tuples(self): + """code_mapping converts string schema entries to (type, kwargs) tuples.""" + from pyhealth.tasks import MortalityPredictionMIMIC3 + + task = MortalityPredictionMIMIC3( + code_mapping={ + "conditions": ("ICD9CM", "CCSCM"), + "procedures": ("ICD9PROC", "CCSPROC"), + } + ) + # Mapped fields become tuples + self.assertEqual( + task.input_schema["conditions"], + ("sequence", {"code_mapping": ("ICD9CM", "CCSCM")}), + ) + self.assertEqual( + task.input_schema["procedures"], + ("sequence", {"code_mapping": ("ICD9PROC", "CCSPROC")}), + ) + # Unmapped fields stay as strings + self.assertEqual(task.input_schema["drugs"], "sequence") + + def test_code_mapping_ignores_unknown_fields(self): + """code_mapping silently ignores fields not in input_schema.""" + from pyhealth.tasks import MortalityPredictionMIMIC3 + + task = MortalityPredictionMIMIC3( + code_mapping={"nonexistent_field": ("SRC", "TGT")} + ) + # Schema unchanged + self.assertEqual(task.input_schema["conditions"], "sequence") + + def test_code_mapping_does_not_mutate_class_attribute(self): + """code_mapping creates instance schema, doesn't modify class attribute.""" + from pyhealth.tasks import MortalityPredictionMIMIC3 + + task_with = MortalityPredictionMIMIC3( + code_mapping={"conditions": ("ICD9CM", "CCSCM")} + ) + task_without = MortalityPredictionMIMIC3() + + # Class attribute unchanged + self.assertEqual(task_without.input_schema["conditions"], "sequence") + # Instance attribute changed + self.assertIsInstance(task_with.input_schema["conditions"], tuple) + + def test_code_mapping_with_readmission_task(self): + """Tasks with existing __init__ params also accept code_mapping.""" + from pyhealth.tasks import ReadmissionPredictionMIMIC3 + + task = ReadmissionPredictionMIMIC3( + code_mapping={"conditions": ("ICD9CM", "CCSCM")} + ) + self.assertEqual( + task.input_schema["conditions"], + ("sequence", {"code_mapping": ("ICD9CM", "CCSCM")}), + ) + # Original params still work + from datetime import timedelta + self.assertEqual(task.window, timedelta(days=15)) + + def test_code_mapping_mimic4(self): + """code_mapping works with MIMIC4 mortality task too.""" + from pyhealth.tasks import MortalityPredictionMIMIC4 + + task = MortalityPredictionMIMIC4( + code_mapping={ + "conditions": ("ICD9CM", "CCSCM"), + "drugs": ("NDC", "ATC"), + } + ) + self.assertEqual( + task.input_schema["conditions"], + ("sequence", {"code_mapping": ("ICD9CM", "CCSCM")}), + ) + self.assertEqual( + task.input_schema["drugs"], + ("sequence", {"code_mapping": ("NDC", "ATC")}), + ) + self.assertEqual(task.input_schema["procedures"], "sequence") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/core/test_concare.py b/tests/core/test_concare.py index c9247d26b..4c8244a5e 100644 --- a/tests/core/test_concare.py +++ b/tests/core/test_concare.py @@ -295,5 +295,23 @@ def test_single_feature(self): self.assertIn("y_prob", ret) + def test_batch_size_one(self): + """Test that ConCare handles batch_size=1 without crashing. + + Regression test: bare .squeeze() in FinalAttentionQKV removed the + batch dimension when batch_size=1, causing softmax to fail with + 'IndexError: Dimension out of range'. + """ + train_loader = get_dataloader(self.dataset, batch_size=1, shuffle=False) + data_batch = next(iter(train_loader)) + + with torch.no_grad(): + ret = self.model(**data_batch) + + self.assertIn("loss", ret) + self.assertIn("y_prob", ret) + self.assertEqual(ret["y_prob"].shape[0], 1) + + if __name__ == "__main__": unittest.main() diff --git a/tests/core/test_drug_ndc_extraction.py b/tests/core/test_drug_ndc_extraction.py new file mode 100644 index 000000000..79897bba8 --- /dev/null +++ b/tests/core/test_drug_ndc_extraction.py @@ -0,0 +1,100 @@ +"""Tests that MIMIC-3/4 tasks extract NDC codes (not drug names) from prescriptions. + +The fix changed event.drug -> event.ndc in mortality and readmission tasks +so that CrossMap NDC->ATC mapping actually receives valid NDC codes. +""" + +import re +import tempfile +import unittest +from pathlib import Path + +from pyhealth.datasets import MIMIC3Dataset +from pyhealth.tasks.mortality_prediction import MortalityPredictionMIMIC3 +from pyhealth.tasks.readmission_prediction import ReadmissionPredictionMIMIC3 + + +# NDC codes are numeric strings (with possible leading zeros), and may include hyphens, +# but should not contain letters (e.g., "0002-3227-30" is a valid NDC format). +NDC_PATTERN = re.compile(r"^[0-9-]+$") + + +class TestDrugNDCExtraction(unittest.TestCase): + """Verify that task classes extract NDC codes from prescriptions, not drug names.""" + + @classmethod + def setUpClass(cls): + cls.cache_dir = tempfile.TemporaryDirectory() + demo_path = str( + Path(__file__).parent.parent.parent + / "test-resources" + / "core" + / "mimic3demo" + ) + cls.dataset = MIMIC3Dataset( + root=demo_path, + tables=["diagnoses_icd", "procedures_icd", "prescriptions"], + cache_dir=cls.cache_dir.name, + ) + + @classmethod + def tearDownClass(cls): + cls.cache_dir.cleanup() + + def _get_drug_vocab(self, sample_dataset): + """Get the drug vocabulary from the processor, excluding special tokens.""" + proc = sample_dataset.input_processors["drugs"] + return { + k for k in proc.code_vocab.keys() + if k not in ("", "") + } + + def test_mortality_drugs_are_ndc_codes(self): + """MortalityPredictionMIMIC3 drug vocabulary should contain NDC codes, not drug names.""" + task = MortalityPredictionMIMIC3() + sample_dataset = self.dataset.set_task(task) + drug_vocab = self._get_drug_vocab(sample_dataset) + + self.assertGreater(len(drug_vocab), 0, "Should have at least one drug code") + + # NDC codes are numeric; drug names contain letters + non_ndc = [d for d in drug_vocab if not NDC_PATTERN.match(str(d))] + self.assertEqual( + len(non_ndc), + 0, + f"Found drug names instead of NDC codes: {non_ndc[:5]}", + ) + + def test_readmission_drugs_are_ndc_codes(self): + """ReadmissionPredictionMIMIC3 drug vocabulary should contain NDC codes, not drug names.""" + task = ReadmissionPredictionMIMIC3() + sample_dataset = self.dataset.set_task(task) + drug_vocab = self._get_drug_vocab(sample_dataset) + + self.assertGreater(len(drug_vocab), 0, "Should have at least one drug code") + + non_ndc = [d for d in drug_vocab if not NDC_PATTERN.match(str(d))] + self.assertEqual( + len(non_ndc), + 0, + f"Found drug names instead of NDC codes: {non_ndc[:5]}", + ) + + def test_drug_vocab_not_drug_names(self): + """Drug vocabulary should not contain common drug names.""" + task = MortalityPredictionMIMIC3() + sample_dataset = self.dataset.set_task(task) + drug_vocab = self._get_drug_vocab(sample_dataset) + + # These are drug names that would appear if event.drug was used + drug_names = {"Aspirin", "Bisacodyl", "Senna", "Heparin", "Insulin"} + overlap = drug_vocab & drug_names + self.assertEqual( + len(overlap), + 0, + f"Vocabulary contains drug names (should be NDC codes): {overlap}", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/core/test_grasp.py b/tests/core/test_grasp.py index 605613ea4..2f67ce9d4 100644 --- a/tests/core/test_grasp.py +++ b/tests/core/test_grasp.py @@ -1,11 +1,15 @@ -"""Test cases for the GRASP model.""" +"""Test cases for the GRASP model. -import unittest +Description: + Unit tests for the GRASP model implementation covering initialization, + forward pass, backward pass, embedding extraction, and custom configs. +""" +import unittest import torch from pyhealth.datasets import create_sample_dataset, get_dataloader -from pyhealth.models.grasp import GRASP +from pyhealth.models import GRASP class TestGRASP(unittest.TestCase): @@ -13,13 +17,13 @@ class TestGRASP(unittest.TestCase): def setUp(self): """Set up test data and model.""" + torch.manual_seed(42) self.samples = [ { "patient_id": "patient-0", "visit_id": "visit-0", "conditions": ["cond-33", "cond-86", "cond-80", "cond-12"], "procedures": ["proc-1", "proc-2"], - "demographic": [0.0, 2.0, 1.5], "label": 0, }, { @@ -27,7 +31,6 @@ def setUp(self): "visit_id": "visit-1", "conditions": ["cond-33", "cond-86", "cond-80"], "procedures": ["proc-3", "proc-4", "proc-5"], - "demographic": [0.0, 2.0, 1.5], "label": 1, }, { @@ -35,7 +38,6 @@ def setUp(self): "visit_id": "visit-0", "conditions": ["cond-12", "cond-45"], "procedures": ["proc-1"], - "demographic": [1.0, 1.0, 0.5], "label": 0, }, { @@ -43,52 +45,42 @@ def setUp(self): "visit_id": "visit-1", "conditions": ["cond-80", "cond-12", "cond-33", "cond-45"], "procedures": ["proc-2", "proc-3"], - "demographic": [1.0, 1.0, 0.5], "label": 1, }, ] + self.input_schema = { + "conditions": "sequence", + "procedures": "sequence", + } + self.output_schema = {"label": "binary"} + self.dataset = create_sample_dataset( samples=self.samples, - input_schema={ - "conditions": "sequence", - "procedures": "sequence", - }, - output_schema={"label": "binary"}, + input_schema=self.input_schema, + output_schema=self.output_schema, dataset_name="test", ) self.model = GRASP( dataset=self.dataset, - static_key="demographic", - embedding_dim=32, - hidden_dim=32, + embedding_dim=16, + hidden_dim=16, cluster_num=2, ) def test_model_initialization(self): """Test that the GRASP model initializes correctly.""" self.assertIsInstance(self.model, GRASP) - self.assertEqual(self.model.embedding_dim, 32) - self.assertEqual(self.model.hidden_dim, 32) - self.assertEqual(self.model.static_key, "demographic") - self.assertEqual(self.model.static_dim, 3) - self.assertEqual(len(self.model.dynamic_feature_keys), 2) + self.assertEqual(self.model.embedding_dim, 16) + self.assertEqual(self.model.hidden_dim, 16) + self.assertEqual(len(self.model.feature_keys), 2) + self.assertIn("conditions", self.model.feature_keys) + self.assertIn("procedures", self.model.feature_keys) self.assertEqual(self.model.label_key, "label") - def test_model_without_static(self): - """Test GRASP initializes correctly without static features.""" - model = GRASP( - dataset=self.dataset, - embedding_dim=32, - hidden_dim=32, - cluster_num=2, - ) - self.assertEqual(model.static_dim, 0) - self.assertIsNone(model.static_key) - - def test_forward(self): - """Test that forward pass produces correct output keys and shapes.""" + def test_model_forward(self): + """Test that the GRASP model forward pass works correctly.""" train_loader = get_dataloader(self.dataset, batch_size=2, shuffle=True) data_batch = next(iter(train_loader)) @@ -99,12 +91,15 @@ def test_forward(self): self.assertIn("y_prob", ret) self.assertIn("y_true", ret) self.assertIn("logit", ret) - self.assertEqual(ret["loss"].dim(), 0) + self.assertEqual(ret["y_prob"].shape[0], 2) self.assertEqual(ret["y_true"].shape[0], 2) + self.assertEqual(ret["logit"].shape[0], 2) + self.assertEqual(ret["loss"].dim(), 0) + self.assertFalse(torch.isnan(ret["loss"])) - def test_backward(self): - """Test that backward pass computes gradients.""" + def test_model_backward(self): + """Test that the GRASP model backward pass works correctly.""" train_loader = get_dataloader(self.dataset, batch_size=2, shuffle=True) data_batch = next(iter(train_loader)) @@ -115,10 +110,12 @@ def test_backward(self): p.requires_grad and p.grad is not None for p in self.model.parameters() ) - self.assertTrue(has_gradient) + self.assertTrue( + has_gradient, "No parameters have gradients after backward pass" + ) - def test_embed_extraction(self): - """Test that embeddings are returned when requested.""" + def test_model_with_embedding(self): + """Test that the GRASP model returns embeddings when requested.""" train_loader = get_dataloader(self.dataset, batch_size=2, shuffle=True) data_batch = next(iter(train_loader)) data_batch["embed"] = True @@ -127,40 +124,121 @@ def test_embed_extraction(self): ret = self.model(**data_batch) self.assertIn("embed", ret) - expected_dim = len(self.model.dynamic_feature_keys) * self.model.hidden_dim - self.assertEqual(ret["embed"].shape[1], expected_dim) + self.assertEqual(ret["embed"].shape[0], 2) + expected_embed_dim = ( + len(self.model.dynamic_feature_keys) * self.model.hidden_dim + ) + self.assertEqual(ret["embed"].shape[1], expected_embed_dim) - def test_gru_backbone(self): - """Test GRASP with GRU backbone.""" + def test_custom_hyperparameters(self): + """Test GRASP model with custom hyperparameters.""" + torch.manual_seed(42) model = GRASP( dataset=self.dataset, - embedding_dim=32, - hidden_dim=32, + embedding_dim=4, + hidden_dim=4, cluster_num=2, block="GRU", + dropout=0.3, ) + + self.assertEqual(model.embedding_dim, 4) + self.assertEqual(model.hidden_dim, 4) + train_loader = get_dataloader(self.dataset, batch_size=2, shuffle=True) data_batch = next(iter(train_loader)) with torch.no_grad(): ret = model(**data_batch) + self.assertIn("loss", ret) + self.assertIn("y_prob", ret) def test_lstm_backbone(self): - """Test GRASP with LSTM backbone.""" + """Test GRASP model with LSTM backbone.""" model = GRASP( dataset=self.dataset, - embedding_dim=32, - hidden_dim=32, + embedding_dim=16, + hidden_dim=16, cluster_num=2, block="LSTM", ) + train_loader = get_dataloader(self.dataset, batch_size=2, shuffle=True) data_batch = next(iter(train_loader)) with torch.no_grad(): ret = model(**data_batch) + self.assertIn("loss", ret) + self.assertFalse(torch.isnan(ret["loss"])) + + + def test_static_key(self): + """Test GRASP with static features (e.g., demographics).""" + samples_with_static = [ + {**s, "demographics": [0.0, 2.0, 1.5]} for s in self.samples + ] + dataset = create_sample_dataset( + samples=samples_with_static, + input_schema={ + "conditions": "sequence", + "procedures": "sequence", + }, + output_schema={"label": "binary"}, + dataset_name="test", + ) + model = GRASP( + dataset=dataset, + static_key="demographics", + embedding_dim=16, + hidden_dim=16, + cluster_num=2, + ) + self.assertEqual(model.static_dim, 3) + self.assertEqual(len(model.dynamic_feature_keys), 2) + self.assertNotIn("demographics", model.dynamic_feature_keys) + + train_loader = get_dataloader(dataset, batch_size=2, shuffle=True) + data_batch = next(iter(train_loader)) + with torch.no_grad(): + ret = model(**data_batch) + self.assertIn("loss", ret) + self.assertFalse(torch.isnan(ret["loss"])) + + def test_without_static_key(self): + """Test GRASP without static features (default).""" + self.assertIsNone(self.model.static_key) + self.assertEqual(self.model.static_dim, 0) + self.assertEqual( + len(self.model.dynamic_feature_keys), + len(self.model.feature_keys), + ) + + def test_batch_smaller_than_cluster_num(self): + """Test GRASP handles batch_size < cluster_num without crashing. + + Regression test: random_init called random.sample(range(num_points), + num_centers) which raises ValueError when num_centers > num_points. + Fixed by clamping: num_centers = min(num_centers, num_points). + """ + model = GRASP( + dataset=self.dataset, + embedding_dim=16, + hidden_dim=16, + cluster_num=4, # more clusters than batch_size=1 + block="GRU", + ) + + train_loader = get_dataloader(self.dataset, batch_size=1, shuffle=False) + data_batch = next(iter(train_loader)) + + with torch.no_grad(): + ret = model(**data_batch) + + self.assertIn("loss", ret) + self.assertIn("y_prob", ret) + self.assertEqual(ret["y_prob"].shape[0], 1) if __name__ == "__main__": diff --git a/tests/core/test_zero_length_sequence_guard.py b/tests/core/test_zero_length_sequence_guard.py new file mode 100644 index 000000000..d41f2ac2c --- /dev/null +++ b/tests/core/test_zero_length_sequence_guard.py @@ -0,0 +1,121 @@ +"""Tests that RNNLayer and ConCare handle zero-length sequences without crashing. + +When code_mapping collapses vocabularies, some patients may have all codes +map to , producing all-zero embeddings and all-zero masks. These tests +verify that the layers handle this edge case gracefully instead of crashing +with IndexError (pack_padded_sequence) or ZeroDivisionError (covariance). +""" + +import unittest +import torch + +from pyhealth.models.rnn import RNNLayer +from pyhealth.models.concare import ConCareLayer, MultiHeadedAttention + + +class TestRNNLayerZeroLengthGuard(unittest.TestCase): + """RNNLayer should not crash when mask contains all-zero rows.""" + + def setUp(self): + torch.manual_seed(42) + self.input_dim = 4 + self.hidden_dim = 4 + self.batch_size = 2 + self.seq_len = 5 + + def test_gru_all_zero_mask_single_sample(self): + """GRU should handle a batch where one sample has an all-zero mask.""" + layer = RNNLayer(self.input_dim, self.hidden_dim, rnn_type="GRU") + x = torch.randn(self.batch_size, self.seq_len, self.input_dim) + mask = torch.ones(self.batch_size, self.seq_len, dtype=torch.int) + mask[1, :] = 0 + + outputs, last_outputs = layer(x, mask) + + self.assertEqual(outputs.shape[0], self.batch_size) + self.assertEqual(last_outputs.shape, (self.batch_size, self.hidden_dim)) + + def test_gru_all_zero_mask_entire_batch(self): + """GRU should handle a batch where ALL samples have all-zero masks.""" + layer = RNNLayer(self.input_dim, self.hidden_dim, rnn_type="GRU") + x = torch.zeros(self.batch_size, self.seq_len, self.input_dim) + mask = torch.zeros(self.batch_size, self.seq_len, dtype=torch.int) + + outputs, last_outputs = layer(x, mask) + + self.assertEqual(last_outputs.shape, (self.batch_size, self.hidden_dim)) + + def test_lstm_all_zero_mask(self): + """LSTM should handle all-zero masks the same as GRU.""" + layer = RNNLayer(self.input_dim, self.hidden_dim, rnn_type="LSTM") + x = torch.randn(self.batch_size, self.seq_len, self.input_dim) + mask = torch.ones(self.batch_size, self.seq_len, dtype=torch.int) + mask[0, :] = 0 + + outputs, last_outputs = layer(x, mask) + + self.assertEqual(last_outputs.shape, (self.batch_size, self.hidden_dim)) + + def test_normal_mask_unchanged(self): + """Normal (non-zero) masks should produce the same results as before.""" + layer = RNNLayer(self.input_dim, self.hidden_dim, rnn_type="GRU") + x = torch.randn(self.batch_size, self.seq_len, self.input_dim) + mask = torch.zeros(self.batch_size, self.seq_len, dtype=torch.int) + mask[0, :3] = 1 + mask[1, :5] = 1 + + outputs, last_outputs = layer(x, mask) + + self.assertEqual(last_outputs.shape, (self.batch_size, self.hidden_dim)) + + +class TestConCareCovarianceGuard(unittest.TestCase): + """ConCare covariance should not divide by zero on single-element inputs.""" + + def setUp(self): + torch.manual_seed(42) + + def test_cov_single_feature(self): + """Covariance with x.size(1)==1 should not raise ZeroDivisionError.""" + attn = MultiHeadedAttention(1, 4, 0.0) + m = torch.randn(2, 1) + + cov = attn.cov(m) + + self.assertEqual(cov.shape, (2, 2)) + + def test_cov_normal_input(self): + """Covariance with normal inputs should still work correctly.""" + attn = MultiHeadedAttention(1, 4, 0.0) + m = torch.randn(2, 4) + + cov = attn.cov(m) + + self.assertEqual(cov.shape, (2, 2)) + + +class TestConCareLayerZeroMask(unittest.TestCase): + """ConCareLayer should handle all-zero masks without crashing.""" + + def setUp(self): + torch.manual_seed(42) + + def test_all_zero_mask_single_sample(self): + """ConCareLayer should not crash when one sample has an all-zero mask.""" + input_dim = 4 + hidden_dim = 4 + batch_size = 2 + seq_len = 5 + + layer = ConCareLayer(input_dim=input_dim, hidden_dim=hidden_dim) + x = torch.randn(batch_size, seq_len, input_dim) + mask = torch.ones(batch_size, seq_len, dtype=torch.int) + mask[1, :] = 0 + + out, decov_loss = layer(x, mask=mask) + + self.assertEqual(out.shape, (batch_size, hidden_dim)) + + +if __name__ == "__main__": + unittest.main() From 5f63039f33896e7deb6220c91ad9be2539d83955 Mon Sep 17 00:00:00 2001 From: Matt McKenna Date: Sun, 19 Apr 2026 10:39:32 -0700 Subject: [PATCH 06/61] Add PhysioNet De-Identification dataset, NER task, and TransformerDeID model (#981) --- docs/api/datasets.rst | 1 + ...pyhealth.datasets.PhysioNetDeIDDataset.rst | 9 + docs/api/models.rst | 1 + .../pyhealth.models.TransformerDeID.rst | 9 + docs/api/tasks.rst | 1 + docs/api/tasks/pyhealth.tasks.DeIDNERTask.rst | 7 + .../physionet_deid_ner_transformer_deid.py | 193 +++++++++ pyhealth/datasets/__init__.py | 1 + pyhealth/datasets/configs/physionet_deid.yaml | 10 + pyhealth/datasets/physionet_deid.py | 371 ++++++++++++++++++ pyhealth/models/__init__.py | 1 + pyhealth/models/transformer_deid.py | 263 +++++++++++++ pyhealth/tasks/__init__.py | 1 + pyhealth/tasks/deid_ner.py | 116 ++++++ test-resources/core/physionet_deid/id.res | 33 ++ test-resources/core/physionet_deid/id.text | 33 ++ tests/core/test_physionet_deid.py | 248 ++++++++++++ tests/core/test_transformer_deid.py | 199 ++++++++++ 18 files changed, 1497 insertions(+) create mode 100644 docs/api/datasets/pyhealth.datasets.PhysioNetDeIDDataset.rst create mode 100644 docs/api/models/pyhealth.models.TransformerDeID.rst create mode 100644 docs/api/tasks/pyhealth.tasks.DeIDNERTask.rst create mode 100644 examples/physionet_deid_ner_transformer_deid.py create mode 100644 pyhealth/datasets/configs/physionet_deid.yaml create mode 100644 pyhealth/datasets/physionet_deid.py create mode 100644 pyhealth/models/transformer_deid.py create mode 100644 pyhealth/tasks/deid_ner.py create mode 100644 test-resources/core/physionet_deid/id.res create mode 100644 test-resources/core/physionet_deid/id.text create mode 100644 tests/core/test_physionet_deid.py create mode 100644 tests/core/test_transformer_deid.py diff --git a/docs/api/datasets.rst b/docs/api/datasets.rst index b02439d26..8d9a59d21 100644 --- a/docs/api/datasets.rst +++ b/docs/api/datasets.rst @@ -238,6 +238,7 @@ Available Datasets datasets/pyhealth.datasets.BMDHSDataset datasets/pyhealth.datasets.COVID19CXRDataset datasets/pyhealth.datasets.ChestXray14Dataset + datasets/pyhealth.datasets.PhysioNetDeIDDataset datasets/pyhealth.datasets.TUABDataset datasets/pyhealth.datasets.TUEVDataset datasets/pyhealth.datasets.ClinVarDataset diff --git a/docs/api/datasets/pyhealth.datasets.PhysioNetDeIDDataset.rst b/docs/api/datasets/pyhealth.datasets.PhysioNetDeIDDataset.rst new file mode 100644 index 000000000..4e04cd629 --- /dev/null +++ b/docs/api/datasets/pyhealth.datasets.PhysioNetDeIDDataset.rst @@ -0,0 +1,9 @@ +pyhealth.datasets.PhysioNetDeIDDataset +======================================= + +The PhysioNet De-Identification dataset. For more information see `here `_. Access requires PhysioNet credentialing. + +.. autoclass:: pyhealth.datasets.PhysioNetDeIDDataset + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/models.rst b/docs/api/models.rst index 7368dec94..166402e86 100644 --- a/docs/api/models.rst +++ b/docs/api/models.rst @@ -177,6 +177,7 @@ API Reference models/pyhealth.models.GNN models/pyhealth.models.Transformer models/pyhealth.models.TransformersModel + models/pyhealth.models.TransformerDeID models/pyhealth.models.RETAIN models/pyhealth.models.GAMENet models/pyhealth.models.GraphCare diff --git a/docs/api/models/pyhealth.models.TransformerDeID.rst b/docs/api/models/pyhealth.models.TransformerDeID.rst new file mode 100644 index 000000000..d07aa94aa --- /dev/null +++ b/docs/api/models/pyhealth.models.TransformerDeID.rst @@ -0,0 +1,9 @@ +pyhealth.models.TransformerDeID +=================================== + +Transformer-based token classifier for clinical text de-identification. + +.. autoclass:: pyhealth.models.TransformerDeID + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/tasks.rst b/docs/api/tasks.rst index 399b8f1aa..23a4e06e5 100644 --- a/docs/api/tasks.rst +++ b/docs/api/tasks.rst @@ -224,6 +224,7 @@ Available Tasks Sleep Staging v2 Benchmark EHRShot ChestX-ray14 Binary Classification + De-Identification NER ChestX-ray14 Multilabel Classification Variant Classification (ClinVar) Mutation Pathogenicity (COSMIC) diff --git a/docs/api/tasks/pyhealth.tasks.DeIDNERTask.rst b/docs/api/tasks/pyhealth.tasks.DeIDNERTask.rst new file mode 100644 index 000000000..2b7428f6e --- /dev/null +++ b/docs/api/tasks/pyhealth.tasks.DeIDNERTask.rst @@ -0,0 +1,7 @@ +pyhealth.tasks.DeIDNERTask +======================================= + +.. autoclass:: pyhealth.tasks.DeIDNERTask + :members: + :undoc-members: + :show-inheritance: diff --git a/examples/physionet_deid_ner_transformer_deid.py b/examples/physionet_deid_ner_transformer_deid.py new file mode 100644 index 000000000..fdbee4173 --- /dev/null +++ b/examples/physionet_deid_ner_transformer_deid.py @@ -0,0 +1,193 @@ +"""Train and evaluate TransformerDeID on PhysioNet de-identification. + +End-to-end example: load data, train BERT-base on token-level NER for +PHI detection, and report binary (PHI vs non-PHI) precision/recall/F1. + +Paper: Johnson et al. "Deidentification of free-text medical records + using pre-trained bidirectional transformers." CHIL, 2020. + +Script structure follows examples/cardiology_detection_isAR_SparcNet.py. + +Hyperparameters follow the paper (Section 3.4): + - Learning rate: 5e-5 + - Batch size: 8 + - Epochs: 3 + - Weight decay: 0.01 + +Ablation results (3 epochs, 80/10/10 patient split, seed=42): + + Config Precision Recall F1 + BERT, no window 95.1% 70.3% 80.8% + BERT, win=64/32 94.1% 69.0% 79.6% + BERT, win=100/60 86.9% 75.7% 80.9% + BERT, win=200/100 94.7% 69.4% 80.1% + RoBERTa, no window 98.1% 64.7% 78.0% + RoBERTa, win=100/60 82.6% 68.6% 75.0% + + BERT with window=100/60 achieves the best F1 (80.9%), matching the + paper's window configuration. Windowing improves recall by allowing + BERT to see tokens beyond the 512 truncation limit. RoBERTa has + higher precision but lower recall than BERT across configurations. + +Usage: + python examples/physionet_deid_ner_transformer_deid.py \ + --data_root path/to/deidentifiedmedicaltext/1.0 + + # With windowing (paper Section 3.3): + python examples/physionet_deid_ner_transformer_deid.py \ + --data_root path/to/data --window_size 100 --window_overlap 60 + + # With RoBERTa: + python examples/physionet_deid_ner_transformer_deid.py \ + --data_root path/to/data --model_name roberta-base + +Author: + Matt McKenna (mtm16@illinois.edu) +""" + +import argparse +from collections import defaultdict + +import numpy as np +import torch +from sklearn.metrics import precision_score, recall_score, f1_score + +from pyhealth.datasets import PhysioNetDeIDDataset, get_dataloader +from pyhealth.datasets.splitter import split_by_patient +from pyhealth.models.transformer_deid import ( + IGNORE_INDEX, + TransformerDeID, +) +from pyhealth.tasks import DeIDNERTask +from pyhealth.trainer import Trainer + + +def compute_metrics(model, dataloader): + """Binary PHI vs non-PHI token-level metrics with window merging. + + When windowing is used, multiple windows may cover the same token. + We merge by taking the non-O prediction with highest probability + (paper Section 3.3). Without windowing, each token appears once + so no merging is needed. + """ + # Collect per-token gold labels and prediction probabilities, + # keyed by (patient_id, note_id, absolute_token_position). + token_gold = {} + token_preds = defaultdict(list) + + model.eval() + with torch.no_grad(): + for batch in dataloader: + result = model(**batch) + probs = result["y_prob"] # (batch, seq_len, num_labels) + labels = result["y_true"] # (batch, seq_len) + patient_ids = batch["patient_id"] + note_ids = batch["note_id"] + token_starts = batch["token_start"] + + for i in range(len(patient_ids)): + pid = patient_ids[i] + nid = note_ids[i] + start = int(token_starts[i]) + word_idx = 0 + for j in range(labels.shape[1]): + if labels[i, j].item() == IGNORE_INDEX: + continue + key = (pid, nid, start + word_idx) + token_gold[key] = labels[i, j].item() + token_preds[key].append(probs[i, j].cpu().numpy()) + word_idx += 1 + + # Merge overlapping predictions (paper Section 3.3): + # if any window predicts non-O, take the non-O with highest score. + all_true, all_pred = [], [] + for key in sorted(token_gold): + all_true.append(token_gold[key]) + preds = token_preds[key] + # 1 - p[0] = total probability of any PHI class, used to + # rank which window's non-O prediction to keep. + non_o = [(p, 1 - p[0]) for p in preds if np.argmax(p) != 0] + if non_o: + merged = max(non_o, key=lambda x: x[1])[0] + else: + merged = np.mean(preds, axis=0) + all_pred.append(int(np.argmax(merged))) + + # Convert to binary: O (index 0) = 0, any PHI = 1. + true_bin = [0 if t == 0 else 1 for t in all_true] + pred_bin = [0 if p == 0 else 1 for p in all_pred] + return { + "precision": precision_score(true_bin, pred_bin), + "recall": recall_score(true_bin, pred_bin), + "f1": f1_score(true_bin, pred_bin), + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--data_root", + type=str, + required=True, + help="Path to deidentifiedmedicaltext/1.0 directory", + ) + parser.add_argument("--model_name", type=str, default="bert-base-uncased") + parser.add_argument("--epochs", type=int, default=3) + parser.add_argument("--batch_size", type=int, default=8) + parser.add_argument("--lr", type=float, default=5e-5) + parser.add_argument("--window_size", type=int, default=None, + help="Token window size (default: no windowing)") + parser.add_argument("--window_overlap", type=int, default=0) + parser.add_argument("--seed", type=int, default=42) + args = parser.parse_args() + + # 1. Load dataset and set task. + print("Loading dataset...") + dataset = PhysioNetDeIDDataset(root=args.data_root) + task = DeIDNERTask( + window_size=args.window_size, + window_overlap=args.window_overlap, + ) + samples = dataset.set_task(task) + print(f" Patients: {len(dataset.unique_patient_ids)}, Samples: {len(samples)}") + + # 2. Split by patient (80/10/10) so no patient's notes appear in + # both train and test. + train_data, val_data, test_data = split_by_patient( + samples, [0.8, 0.1, 0.1], seed=args.seed + ) + train_loader = get_dataloader(train_data, batch_size=args.batch_size, shuffle=True) + val_loader = get_dataloader(val_data, batch_size=args.batch_size, shuffle=False) + test_loader = get_dataloader(test_data, batch_size=args.batch_size, shuffle=False) + + # 3. Create model. + model = TransformerDeID( + dataset=samples, + model_name=args.model_name, + ) + + # 4. Train using PyHealth's Trainer. + device = "cuda" if torch.cuda.is_available() else "cpu" + trainer = Trainer(model=model, device=device) + trainer.train( + train_dataloader=train_loader, + val_dataloader=val_loader, + epochs=args.epochs, + optimizer_class=torch.optim.AdamW, + optimizer_params={"lr": args.lr}, + weight_decay=0.01, + monitor="loss", + monitor_criterion="min", + ) + + # 5. Evaluate on test set. + print("\n=== Test Set Results (binary PHI vs non-PHI) ===") + metrics = compute_metrics(model, test_loader) + for k, v in metrics.items(): + print(f" {k}: {v:.4f}") + + samples.close() + + +if __name__ == "__main__": + main() diff --git a/pyhealth/datasets/__init__.py b/pyhealth/datasets/__init__.py index 54e77670c..50b1b3887 100644 --- a/pyhealth/datasets/__init__.py +++ b/pyhealth/datasets/__init__.py @@ -61,6 +61,7 @@ def __init__(self, *args, **kwargs): from .mimic4 import MIMIC4CXRDataset, MIMIC4Dataset, MIMIC4EHRDataset, MIMIC4NoteDataset from .mimicextract import MIMICExtractDataset from .omop import OMOPDataset +from .physionet_deid import PhysioNetDeIDDataset from .sample_dataset import SampleBuilder, SampleDataset, create_sample_dataset from .shhs import SHHSDataset from .sleepedf import SleepEDFDataset diff --git a/pyhealth/datasets/configs/physionet_deid.yaml b/pyhealth/datasets/configs/physionet_deid.yaml new file mode 100644 index 000000000..2054ab809 --- /dev/null +++ b/pyhealth/datasets/configs/physionet_deid.yaml @@ -0,0 +1,10 @@ +version: "1.0" +tables: + physionet_deid: + file_path: "physionet_deid_metadata.csv" + patient_id: "patient_id" + timestamp: null + attributes: + - "note_id" + - "text" + - "labels" diff --git a/pyhealth/datasets/physionet_deid.py b/pyhealth/datasets/physionet_deid.py new file mode 100644 index 000000000..790a2bfa7 --- /dev/null +++ b/pyhealth/datasets/physionet_deid.py @@ -0,0 +1,371 @@ +""" +PyHealth dataset for the PhysioNet De-Identification dataset. + +Dataset link: + https://physionet.org/content/deidentifiedmedicaltext/1.0/ + +Dataset paper: (please cite if you use this dataset) + Neamatullah, Ishna, et al. "Automated de-identification of free-text + medical records." BMC Medical Informatics and Decision Making 8.1 (2008). + +Paper link: + https://doi.org/10.1186/1472-6947-8-32 + +PHI category mapping in classify_phi() inspired by the label groupings +in the bert-deid reference implementation by Johnson et al.: + https://github.com/alistairewj/bert-deid/blob/master/bert_deid/label.py + +Task paper: + Johnson, Alistair E.W., et al. "Deidentification of free-text medical + records using pre-trained bidirectional transformers." Proceedings of + the ACM Conference on Health, Inference, and Learning (CHIL), 2020. + +Author: + Matt McKenna (mtm16@illinois.edu) +""" +import logging +import os +import re +import shutil +import tempfile +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +import pandas as pd + +from pyhealth.datasets import BaseDataset + +logger = logging.getLogger(__name__) + +_UNKNOWN_PHI_TAGS: set = set() + +# -- Record parsing regexes -- + +_RECORD_START = re.compile(r"START_OF_RECORD=(\d+)\|\|\|\|(\d+)\|\|\|\|") +_RECORD_END = re.compile(r"\|\|\|\|END_OF_RECORD") +_PHI_TAG = re.compile(r"\[\*\*(.+?)\*\*\]", re.DOTALL) +_PHI_SPLIT = re.compile(r"\[\*\*(?:.+?)\*\*\]", re.DOTALL) + + +def _parse_file(path: Path) -> Dict[Tuple[str, str], str]: + """Parse a PhysioNet record file into {(patient_id, note_id): body}. + + Args: + path: Path to id.text or id.res file. + + Returns: + Dictionary mapping (patient_id, note_id) to note body text. + """ + raw = path.read_text(encoding="utf-8", errors="ignore") + out: Dict[Tuple[str, str], str] = {} + for m in _RECORD_START.finditer(raw): + pid, nid = m.group(1), m.group(2) + body_start = m.end() + end_m = _RECORD_END.search(raw, body_start) + body = raw[body_start : end_m.start() if end_m else len(raw)] + out[(pid, nid)] = body.strip() + return out + + +def classify_phi(raw: str) -> str: + """Map raw [**...**] tag text to one of the 7 PHI categories. + + Args: + raw: The text inside a [**...**] tag. + + Returns: + One of: AGE, DATE, CONTACT, LOCATION, ID, PROFESSION, NAME. + """ + t = re.sub(r"[^a-z0-9 ]+", " ", raw.strip().lower()).strip() + + if any(k in t for k in ("year old", " yo ", " age ")): + return "AGE" + if any(k in t for k in ("date", "month", "day", "year", "holiday")): + return "DATE" + if re.fullmatch(r"[\d]{1,2}[ \-/][\d]{1,2}([ \-/][\d]{2,4})?", t): + return "DATE" + if re.fullmatch(r"[\d]+", raw.strip()): + return "DATE" + if any(k in t for k in ("phone", "fax", "email", "pager", "contact")): + return "CONTACT" + if any( + k in t + for k in ( + "hospital", + "location", + "street", + "county", + "state", + "country", + "zip", + "address", + "ward", + "room", + ) + ): + return "LOCATION" + if any( + k in t + for k in ( + "mrn", + "medical record", + "record number", + "ssn", + "account", + "serial", + "unit no", + "unit number", + "identifier", + ) + ): + return "ID" + if " id " in f" {t} ": + return "ID" + if any( + k in t + for k in ( + "doctor", + " dr ", + " md ", + "nurse", + "attending", + "resident", + "profession", + "service", + "provider", + ) + ): + return "PROFESSION" + if any( + k in t + for k in ( + "name", + "initial", + "alias", + "patient", + "first name", + "last name", + ) + ): + return "NAME" + if raw not in _UNKNOWN_PHI_TAGS: + _UNKNOWN_PHI_TAGS.add(raw) + logger.warning( + "classify_phi: no keyword match for tag '%s', defaulting to NAME", + raw, + ) + return "NAME" + + +def phi_spans_in_original( + orig: str, deid: str +) -> List[Tuple[int, int, str]]: + """Find PHI character spans in orig by anchoring on non-PHI chunks. + + Uses non-PHI text from the de-identified version as anchors to locate + where the original PHI text appears in the original note. + + Args: + orig: Original note text (with real PHI). + deid: De-identified note text (PHI replaced with [**...**] tags). + + Returns: + List of (char_start, char_end, phi_category) tuples. + """ + parts = _PHI_SPLIT.split(deid) + tags = _PHI_TAG.findall(deid) + + spans: List[Tuple[int, int, str]] = [] + pos = 0 + + for i, tag_inner in enumerate(tags): + before = parts[i] + if before: + idx = orig.find(before, pos) + pos = (idx + len(before)) if idx != -1 else (pos + len(before)) + + phi_start = pos + + after = parts[i + 1] + if after: + idx = orig.find(after, phi_start) + phi_end = idx if idx != -1 else phi_start + else: + phi_end = len(orig) + + if phi_end > phi_start: + spans.append((phi_start, phi_end, classify_phi(tag_inner))) + + pos = phi_end + + return spans + + +def bio_tag( + text: str, spans: List[Tuple[int, int, str]] +) -> List[Tuple[str, str]]: + """Whitespace-tokenize text and assign BIO labels from char-level spans. + + Args: + text: Original note text. + spans: List of (char_start, char_end, phi_category) tuples. + + Returns: + List of (word, label) tuples. + """ + char_label = ["O"] * len(text) + for start, end, cat in spans: + for i in range(start, min(end, len(text))): + char_label[i] = cat + + result: List[Tuple[str, str]] = [] + for m in re.finditer(r"\S+", text): + w_start, w_end = m.start(), m.end() + word = m.group() + # Collect PHI categories for this token's characters, ignoring O's. + # If empty, the token has no PHI and we label it O. + cats = [c for c in char_label[w_start:w_end] if c != "O"] + if not cats: + result.append((word, "O")) + continue + # Pick the most common category. Handles rare cases where a token + # spans two PHI types, e.g. "Smith01/15" -> chars are NAME+DATE, + # majority wins (DATE). + cat = max(set(cats), key=cats.count) + # B = beginning of a new entity, I = continuation of the same one. + # Use "I" only if the previous token was the same category, + # e.g. "Tom"=B-NAME "Garcia"=I-NAME. Otherwise start a new "B". + prev_label = result[-1][1] if result else "O" + prefix = ( + "I" + if prev_label not in ("O",) and prev_label.endswith(cat) + else "B" + ) + result.append((word, f"{prefix}-{cat}")) + + return result + + +class PhysioNetDeIDDataset(BaseDataset): + """Dataset class for the PhysioNet De-Identification dataset. + + This dataset contains 2,434 nursing notes from 163 patients. + Each note has original text with PHI (protected health information) + and a de-identified version with [**...**] tags marking PHI spans. + + The dataset parses both files to produce token-level BIO labels + for 7 PHI categories: AGE, CONTACT, DATE, ID, LOCATION, NAME, + PROFESSION. + + Data access requires PhysioNet credentialing: + 1. Create a PhysioNet account at https://physionet.org + 2. Complete the required CITI training + 3. Sign the data use agreement + 4. Download from + https://physionet.org/content/deidentifiedmedicaltext/1.0/ + + Attributes: + root (str): Root directory containing id.text and id.res files. + dataset_name (str): Name of the dataset. + + Example:: + >>> dataset = PhysioNetDeIDDataset(root="./data/physionet_deid") + """ + + def __init__( + self, + root: str = ".", + config_path: Optional[str] = str( + Path(__file__).parent / "configs" / "physionet_deid.yaml" + ), + **kwargs, + ) -> None: + """Initializes the PhysioNet De-Identification dataset. + + Args: + root: Root directory containing id.text and id.res files. + config_path: Path to the configuration file. + + Raises: + FileNotFoundError: If id.text or id.res not found in root. + + Example:: + >>> dataset = PhysioNetDeIDDataset(root="./data") + """ + self._verify_data(root) + self._tmp_dir = tempfile.mkdtemp(prefix="pyhealth_deid_") + self._index_data(root, self._tmp_dir) + + super().__init__( + root=self._tmp_dir, + tables=["physionet_deid"], + dataset_name="PhysioNetDeID", + config_path=config_path, + **kwargs, + ) + + def __del__(self): + shutil.rmtree(self._tmp_dir, ignore_errors=True) + + def _verify_data(self, root: str) -> None: + """Verify that required data files exist. + + Args: + root: Root directory to check. + + Raises: + FileNotFoundError: If id.text or id.res is missing. + """ + for fname in ("id.text", "id.res"): + path = os.path.join(root, fname) + if not os.path.isfile(path): + raise FileNotFoundError( + f"Required file '{fname}' not found in {root}" + ) + + def _index_data(self, root: str, output_dir: str) -> pd.DataFrame: + """Parse id.text and id.res into a CSV for BaseDataset to load. + + Reads data files from root but writes the metadata CSV to + output_dir so the data directory can be read-only. + + Args: + root: Root directory containing the data files. + output_dir: Directory to write the metadata CSV to. + + Returns: + DataFrame with columns: patient_id, note_id, text, labels. + """ + root_path = Path(root) + orig_records = _parse_file(root_path / "id.text") + deid_records = _parse_file(root_path / "id.res") + + rows = [] + for key in sorted( + orig_records, key=lambda k: (int(k[0]), int(k[1])) + ): + pid, nid = key + orig = orig_records[key] + # Missing key yields empty string (no deid version). + deid = deid_records.get(key, "") + spans = phi_spans_in_original(orig, deid) + tagged = bio_tag(orig, spans) + + tokens = " ".join(w for w, _ in tagged) + labels = " ".join(lbl for _, lbl in tagged) + + rows.append( + { + "patient_id": pid, + "note_id": nid, + "text": tokens, + "labels": labels, + } + ) + + df = pd.DataFrame(rows) + df.to_csv( + os.path.join(output_dir, "physionet_deid_metadata.csv"), + index=False, + ) + return df diff --git a/pyhealth/models/__init__.py b/pyhealth/models/__init__.py index 5233b1726..deabaf95c 100644 --- a/pyhealth/models/__init__.py +++ b/pyhealth/models/__init__.py @@ -1,6 +1,7 @@ from .adacare import AdaCare, AdaCareLayer, MultimodalAdaCare from .agent import Agent, AgentLayer from .base_model import BaseModel +from .transformer_deid import TransformerDeID from .biot import BIOT from .cnn import CNN, CNNLayer from .concare import ConCare, ConCareLayer diff --git a/pyhealth/models/transformer_deid.py b/pyhealth/models/transformer_deid.py new file mode 100644 index 000000000..964942f5a --- /dev/null +++ b/pyhealth/models/transformer_deid.py @@ -0,0 +1,263 @@ +""" +PyHealth model for transformer-based clinical text de-identification. + +Performs token-level NER to detect PHI (protected health information) +in clinical notes using a pre-trained transformer with a classification +head. + +Paper: Johnson, Alistair E.W., et al. "Deidentification of free-text + medical records using pre-trained bidirectional transformers." + Proceedings of the ACM Conference on Health, Inference, and + Learning (CHIL), 2020. + +Paper link: + https://doi.org/10.1145/3368555.3384455 + +Model structure (dropout + linear head) follows PyHealth's +TransformersModel (pyhealth/models/transformers_model.py), adapted +for token-level classification instead of sequence-level. + +Subword alignment follows the standard HuggingFace token +classification pattern (see BertForTokenClassification). + +Author: + Matt McKenna (mtm16@illinois.edu) +""" + +import logging +from typing import Dict, List + +import torch +import torch.nn as nn +from transformers import AutoModel, AutoTokenizer + +from ..datasets import SampleDataset +from .base_model import BaseModel + +logger = logging.getLogger(__name__) + +# 7 PHI categories with BIO prefix, plus O for non-PHI. +LABEL_VOCAB = { + "O": 0, + "B-AGE": 1, "I-AGE": 2, + "B-CONTACT": 3, "I-CONTACT": 4, + "B-DATE": 5, "I-DATE": 6, + "B-ID": 7, "I-ID": 8, + "B-LOCATION": 9, "I-LOCATION": 10, + "B-NAME": 11, "I-NAME": 12, + "B-PROFESSION": 13, "I-PROFESSION": 14, +} + +# Cross-entropy ignores positions with this index (PyTorch convention). +IGNORE_INDEX = -100 + + +def align_labels( + word_ids: List[int | None], + word_labels: List[int], +) -> List[int]: + """Align word-level labels to subword tokens. + + BERT/RoBERTa tokenizers split words into subwords. For example, + "Smith" might become ["Sm", "##ith"]. This function assigns the + word's label to the first subtoken and IGNORE_INDEX to the rest, + so the loss function skips non-first subtokens. Special tokens + ([CLS], [SEP], padding) have word_id=None and also get + IGNORE_INDEX. + + Args: + word_ids: Output of tokenizer.word_ids(). None for special + tokens, integer word index for real tokens. + word_labels: Label index for each word in the original text. + + Returns: + List of label indices, one per subtoken. Non-first subtokens + and special tokens are set to IGNORE_INDEX (-100). + """ + aligned = [] + prev_word_id = None + for word_id in word_ids: + if word_id is None: + # Special token ([CLS], [SEP], padding). + aligned.append(IGNORE_INDEX) + elif word_id != prev_word_id: + # First subtoken of a word: use the word's label. + aligned.append(word_labels[word_id]) + else: + # Non-first subtoken: ignore during loss computation. + aligned.append(IGNORE_INDEX) + prev_word_id = word_id + return aligned + + +class TransformerDeID(BaseModel): + """Transformer-based token classifier for clinical text de-identification. + + Uses a pre-trained transformer encoder with a linear classification + head to predict BIO-tagged PHI labels for each token. + + Args: + dataset: A SampleDataset from set_task(). + model_name: HuggingFace model name. Default "bert-base-uncased". + max_length: Maximum token sequence length. Default 512. + dropout: Dropout rate for the classification head. Default 0.1. + + Examples: + >>> from pyhealth.datasets import PhysioNetDeIDDataset + >>> from pyhealth.tasks import DeIDNERTask + >>> from pyhealth.models import TransformerDeID + >>> dataset = PhysioNetDeIDDataset(root="/path/to/data") + >>> samples = dataset.set_task(DeIDNERTask()) + >>> model = TransformerDeID(dataset=samples) # BERT + >>> model = TransformerDeID(dataset=samples, model_name="roberta-base") + """ + + def __init__( + self, + dataset: SampleDataset, + model_name: str = "bert-base-uncased", + max_length: int = 512, + dropout: float = 0.1, + ): + super(TransformerDeID, self).__init__(dataset=dataset) + + assert len(self.feature_keys) == 1, ( + "TransformerDeID expects exactly one input feature (text)." + ) + assert len(self.label_keys) == 1, ( + "TransformerDeID expects exactly one label key." + ) + self.feature_key = self.feature_keys[0] + self.label_key = self.label_keys[0] + + self.model_name = model_name + self.max_length = max_length + self.label_vocab = LABEL_VOCAB + self.num_labels = len(LABEL_VOCAB) + + # add_prefix_space=True is required for RoBERTa when using + # is_split_into_words=True in the forward pass. + self.tokenizer = AutoTokenizer.from_pretrained( + model_name, add_prefix_space=True + ) + self.encoder = AutoModel.from_pretrained( + model_name, + hidden_dropout_prob=dropout, + attention_probs_dropout_prob=dropout, + ) + hidden_size = self.encoder.config.hidden_size + self.dropout = nn.Dropout(dropout) + self.classifier = nn.Linear(hidden_size, self.num_labels) + + def forward(self, **kwargs) -> Dict[str, torch.Tensor]: + """Forward pass. + + Args: + **kwargs: Must contain self.feature_key (list of + space-joined token strings) and self.label_key + (list of space-joined BIO label strings). + + Returns: + Dict with keys: loss, logit, y_prob, y_true. + """ + texts: List[str] = kwargs[self.feature_key] + label_strings: List[str] = kwargs[self.label_key] + + # Tokenize with is_split_into_words=True so the tokenizer + # knows word boundaries and word_ids() works correctly. + words_batch = [t.split(" ") for t in texts] + encoding = self.tokenizer( + words_batch, + is_split_into_words=True, + padding=True, + truncation=True, + max_length=self.max_length, + return_tensors="pt", + ) + + # Convert word-level label strings to indices, then align + # to subword tokens. Positions that should be ignored during + # loss (special tokens, non-first subtokens, padding) get + # IGNORE_INDEX (-100), which cross-entropy skips. + aligned_labels = [] + for i, label_str in enumerate(label_strings): + word_labels = [ + self.label_vocab[lbl] for lbl in label_str.split(" ") + ] + word_ids = encoding.word_ids(batch_index=i) + aligned_labels.append(align_labels(word_ids, word_labels)) + + labels = torch.tensor(aligned_labels, dtype=torch.long) + + # Move to device + input_ids = encoding["input_ids"].to(self.device) + attention_mask = encoding["attention_mask"].to(self.device) + labels = labels.to(self.device) + + # Encoder -> dropout -> classifier (per-token logits) + hidden_states = self.encoder( + input_ids=input_ids, + attention_mask=attention_mask, + ).last_hidden_state + logits = self.classifier(self.dropout(hidden_states)) + + # Token-level cross-entropy, ignoring padded/special positions. + # We can't use BaseModel.get_loss_function() because it assumes + # one label per sample. Instead we call cross_entropy directly + # with ignore_index to skip special tokens and non-first subtokens. + # Flatten + ignore_index pattern from HuggingFace's + # BertForTokenClassification.forward(). + loss = nn.functional.cross_entropy( + logits.view(-1, self.num_labels), + labels.view(-1), + ignore_index=IGNORE_INDEX, + ) + + # Per-token probabilities via softmax. + y_prob = torch.softmax(logits, dim=-1) + + return { + "loss": loss, + "logit": logits, + "y_prob": y_prob, + "y_true": labels, + } + + def deidentify(self, text: str, redact: str = "[REDACTED]") -> str: + """Replace PHI in a clinical note with a redaction marker. + + Args: + text: Raw clinical note as a string. + redact: Replacement string for PHI tokens. + + Returns: + The note with PHI tokens replaced. + + Example:: + >>> model.deidentify("Patient John Smith was seen") + 'Patient [REDACTED] [REDACTED] was seen' + """ + words = text.split() + # Forward pass with dummy labels (all O) since we only + # need predictions, not loss. + dummy_labels = " ".join(["O"] * len(words)) + self.eval() + with torch.no_grad(): + result = self(text=[text], labels=[dummy_labels]) + + preds = result["logit"][0].argmax(dim=-1) + y_true = result["y_true"][0] + + # Map predictions back to words using the non-ignored positions. + word_idx = 0 + output = [] + for j in range(len(preds)): + if y_true[j].item() == IGNORE_INDEX: + continue + if preds[j].item() != 0: # non-O = PHI + output.append(redact) + else: + output.append(words[word_idx]) + word_idx += 1 + + return " ".join(output) diff --git a/pyhealth/tasks/__init__.py b/pyhealth/tasks/__init__.py index 797988377..a32618f9c 100644 --- a/pyhealth/tasks/__init__.py +++ b/pyhealth/tasks/__init__.py @@ -12,6 +12,7 @@ from .chestxray14_binary_classification import ChestXray14BinaryClassification from .chestxray14_multilabel_classification import ChestXray14MultilabelClassification from .covid19_cxr_classification import COVID19CXRClassification +from .deid_ner import DeIDNERTask from .dka import DKAPredictionMIMIC4, T1DDKAPredictionMIMIC4 from .drug_recommendation import ( DrugRecommendationEICU, diff --git a/pyhealth/tasks/deid_ner.py b/pyhealth/tasks/deid_ner.py new file mode 100644 index 000000000..215305f7f --- /dev/null +++ b/pyhealth/tasks/deid_ner.py @@ -0,0 +1,116 @@ +""" +PyHealth task for NER-based de-identification of clinical text. + +Converts PhysioNet De-Identification dataset records into token-level +BIO-tagged NER samples for PHI detection. + +Dataset link: + https://physionet.org/content/deidentifiedmedicaltext/1.0/ + +Task paper: (please cite if you use this task) + Johnson, Alistair E.W., et al. "Deidentification of free-text medical + records using pre-trained bidirectional transformers." Proceedings of + the ACM Conference on Health, Inference, and Learning (CHIL), 2020. + +Paper link: + https://doi.org/10.1145/3368555.3384455 + +Author: + Matt McKenna (mtm16@illinois.edu) +""" + +from typing import Dict, List, Optional, Type, Union + +from pyhealth.data import Event, Patient +from pyhealth.processors.text_processor import TextProcessor +from pyhealth.tasks import BaseTask + + +class DeIDNERTask(BaseTask): + """Token-level NER task for clinical text de-identification. + + Each sample contains a list of tokens and their BIO labels over + 7 PHI categories: AGE, CONTACT, DATE, ID, LOCATION, NAME, + PROFESSION. + + Supports optional overlapping windowing (paper Section 3.3) to + handle notes longer than BERT's 512 token limit. + + Args: + window_size: If set, split notes into overlapping windows of + this many tokens. Default None (no windowing). + window_overlap: Number of tokens shared between consecutive + windows. Default 0. + + Attributes: + task_name (str): The name of the task. + input_schema (Dict[str, Union[str, Type]]): The schema for the task input. + output_schema (Dict[str, Union[str, Type]]): The schema for the task output. + + Examples: + >>> from pyhealth.datasets import PhysioNetDeIDDataset + >>> from pyhealth.tasks import DeIDNERTask + >>> dataset = PhysioNetDeIDDataset(root="/path/to/data") + >>> task = DeIDNERTask() + >>> samples = dataset.set_task(task) + >>> task_windowed = DeIDNERTask(window_size=100, window_overlap=60) + >>> samples = dataset.set_task(task_windowed) + """ + + task_name: str = "DeIDNER" + input_schema: Dict[str, Union[str, Type]] = {"text": TextProcessor} + output_schema: Dict[str, Union[str, Type]] = {"labels": TextProcessor} + + def __init__( + self, + window_size: Optional[int] = None, + window_overlap: int = 0, + ): + self.window_size = window_size + self.window_overlap = window_overlap + + def __call__(self, patient: Patient) -> List[Dict]: + """Generate NER samples from a patient's clinical notes. + + Args: + patient: A Patient object with physionet_deid events. + + Returns: + List of dicts, each with 'text' (str) and + 'labels' (str) keys. Both are space-joined strings. + """ + events: List[Event] = patient.get_events( + event_type="physionet_deid" + ) + + samples = [] + for event in events: + note_id = event["note_id"] + words = event["text"].split(" ") + labels = event["labels"].split(" ") + + if self.window_size is None: + # No windowing: one sample per note. + samples.append({ + "patient_id": patient.patient_id, + "note_id": note_id, + "token_start": "0", + "text": event["text"], + "labels": event["labels"], + }) + else: + # Overlapping windows (paper Section 3.3). + step = self.window_size - self.window_overlap + idx = 0 + while idx < len(words): + end = min(idx + self.window_size, len(words)) + samples.append({ + "patient_id": patient.patient_id, + "note_id": note_id, + "token_start": str(idx), + "text": " ".join(words[idx:end]), + "labels": " ".join(labels[idx:end]), + }) + idx += step + + return samples diff --git a/test-resources/core/physionet_deid/id.res b/test-resources/core/physionet_deid/id.res new file mode 100644 index 000000000..4be4e7883 --- /dev/null +++ b/test-resources/core/physionet_deid/id.res @@ -0,0 +1,33 @@ +START_OF_RECORD=10||||1|||| +Patient [**First Name 101**] [**Last Name 102**] was admitted on [**Date 103**] to [**Hospital 104**]. She is [**Age 105**] years old. Contact phone [**Phone 106**]. MRN [**Medical Record Number 107**]. +||||END_OF_RECORD +START_OF_RECORD=10||||2|||| +Seen by [**Doctor First Name 201**] [**Doctor Last Name 202**] in radiology. No acute findings. +||||END_OF_RECORD +START_OF_RECORD=20||||1|||| +Assessment unchanged. Vitals stable overnight. Continue current plan. +||||END_OF_RECORD +START_OF_RECORD=60||||1|||| +[**First Name 301**] [**Last Name 302**] presented to [**Hospital 303**] on [**Date 304**] for follow-up. He works as a [**Profession 305**]. Patient ID [**Medical Record Number 306**]. +||||END_OF_RECORD +START_OF_RECORD=70||||1|||| +[**First Name 401**] [**Last Name 402**] was seen on [**Date 403**] at [**Hospital 404**]. Age [**Age 405**]. No complaints. +||||END_OF_RECORD +START_OF_RECORD=80||||1|||| +Labs reviewed. WBC normal. Continue antibiotics. Follow-up in 3 days. +||||END_OF_RECORD +START_OF_RECORD=90||||1|||| +[**Doctor First Name 501**] [**Doctor Last Name 502**] evaluated patient [**First Name 503**] [**Last Name 504**] on [**Date 505**] at [**Hospital 506**] for chest pain. +||||END_OF_RECORD +START_OF_RECORD=100||||1|||| +[**First Name 601**] [**Last Name 602**] age [**Age 603**] admitted to [**Hospital 604**] for shortness of breath. Contact email [**Contact 605**]. +||||END_OF_RECORD +START_OF_RECORD=110||||1|||| +Stable vitals. Patient resting comfortably. Plan to discharge tomorrow. +||||END_OF_RECORD +START_OF_RECORD=120||||1|||| +[**First Name 701**] [**Last Name 702**] presented to [**Hospital 703**] on [**Date 704**]. He is a retired [**Profession 705**] from [**Location 706**] [**Location 707**]. +||||END_OF_RECORD +START_OF_RECORD=130||||1|||| +[**Doctor First Name 801**] [**Doctor Last Name 802**] discharged patient on [**Date 803**]. Follow-up at [**Hospital 804**] in 2 weeks. +||||END_OF_RECORD diff --git a/test-resources/core/physionet_deid/id.text b/test-resources/core/physionet_deid/id.text new file mode 100644 index 000000000..71048d22d --- /dev/null +++ b/test-resources/core/physionet_deid/id.text @@ -0,0 +1,33 @@ +START_OF_RECORD=10||||1|||| +Patient Jane Doe was admitted on 03/12/2098 to Springfield General Hospital. She is 72 years old. Contact phone 555-867-5309. MRN 00112233. +||||END_OF_RECORD +START_OF_RECORD=10||||2|||| +Seen by Dr. Robert Wells in radiology. No acute findings. +||||END_OF_RECORD +START_OF_RECORD=20||||1|||| +Assessment unchanged. Vitals stable overnight. Continue current plan. +||||END_OF_RECORD +START_OF_RECORD=60||||1|||| +Tom Garcia presented to Lakewood Clinic on 11/05/2097 for follow-up. He works as a plumber. Patient ID 99887766. +||||END_OF_RECORD +START_OF_RECORD=70||||1|||| +Susan Park was seen on 06/15/2098 at Valley Medical Center. Age 58. No complaints. +||||END_OF_RECORD +START_OF_RECORD=80||||1|||| +Labs reviewed. WBC normal. Continue antibiotics. Follow-up in 3 days. +||||END_OF_RECORD +START_OF_RECORD=90||||1|||| +Dr. Linda Chen evaluated patient Henry Adams on 01/22/2097 at Riverside Hospital for chest pain. +||||END_OF_RECORD +START_OF_RECORD=100||||1|||| +Maria Santos age 65 admitted to Cedar Grove Medical for shortness of breath. Contact email msantos@example.com. +||||END_OF_RECORD +START_OF_RECORD=110||||1|||| +Stable vitals. Patient resting comfortably. Plan to discharge tomorrow. +||||END_OF_RECORD +START_OF_RECORD=120||||1|||| +James Wilson presented to Oakdale Clinic on 09/03/2098. He is a retired teacher from Portland Oregon. +||||END_OF_RECORD +START_OF_RECORD=130||||1|||| +Dr. Paul Kim discharged patient on 12/01/2097. Follow-up at Greenfield Hospital in 2 weeks. +||||END_OF_RECORD diff --git a/tests/core/test_physionet_deid.py b/tests/core/test_physionet_deid.py new file mode 100644 index 000000000..5f4c3e43d --- /dev/null +++ b/tests/core/test_physionet_deid.py @@ -0,0 +1,248 @@ +""" +Unit tests for the PhysioNetDeIDDataset and DeIDNERTask classes. + +Author: + Matt McKenna (mtm16@illinois.edu) +""" +import logging +import os +from pathlib import Path +import tempfile +import unittest + +from pyhealth.datasets import PhysioNetDeIDDataset +from pyhealth.datasets.physionet_deid import ( + bio_tag, + classify_phi, + phi_spans_in_original, +) +from pyhealth.tasks import DeIDNERTask + + +class TestPhysioNetDeIDDataset(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.root = ( + Path(__file__).parent.parent.parent + / "test-resources" + / "core" + / "physionet_deid" + ) + cls.cache_dir = tempfile.TemporaryDirectory() + cls.dataset = PhysioNetDeIDDataset( + root=str(cls.root), cache_dir=cls.cache_dir.name + ) + cls.task = DeIDNERTask() + cls.samples = cls.dataset.set_task(cls.task) + + @classmethod + def tearDownClass(cls): + cls.samples.close() + cls.cache_dir.cleanup() + + def test_num_patients(self): + self.assertEqual(len(self.dataset.unique_patient_ids), 10) + + def test_patient_ids(self): + ids = set(self.dataset.unique_patient_ids) + self.assertEqual( + ids, + {"10", "20", "60", "70", "80", "90", "100", "110", "120", "130"}, + ) + + def test_patient_10_has_two_notes(self): + events = self.dataset.get_patient("10").get_events() + self.assertEqual(len(events), 2) + + def test_patient_20_has_one_note(self): + events = self.dataset.get_patient("20").get_events() + self.assertEqual(len(events), 1) + + def test_patient_60_has_one_note(self): + events = self.dataset.get_patient("60").get_events() + self.assertEqual(len(events), 1) + + def test_patient_10_note1_has_tokens_and_labels(self): + events = self.dataset.get_patient("10").get_events() + note1 = events[0] + self.assertIn("text", note1) + self.assertIn("labels", note1) + + def test_patient_10_note1_token_count(self): + """Note 1 for patient 10 should have the right number of tokens.""" + events = self.dataset.get_patient("10").get_events() + note1 = events[0] + tokens = note1["text"].split(" ") + self.assertEqual(len(tokens), 21) + + def test_patient_20_no_phi(self): + """Patient 20's note has no PHI, all labels should be O.""" + events = self.dataset.get_patient("20").get_events() + labels = events[0]["labels"].split(" ") + self.assertTrue(all(lbl == "O" for lbl in labels)) + + def test_patient_60_has_name_labels(self): + """Patient 60's note starts with NAME.""" + events = self.dataset.get_patient("60").get_events() + labels = events[0]["labels"].split(" ") + self.assertEqual(labels[0], "B-NAME") + self.assertEqual(labels[1], "I-NAME") + + def test_patient_60_has_location_label(self): + """Patient 60's note has LOCATION.""" + events = self.dataset.get_patient("60").get_events() + tokens = events[0]["text"].split(" ") + labels = events[0]["labels"].split(" ") + lakewood_idx = tokens.index("Lakewood") + self.assertEqual(labels[lakewood_idx], "B-LOCATION") + self.assertEqual(labels[lakewood_idx + 1], "I-LOCATION") + + def test_patient_60_has_date_label(self): + """Patient 60's note has DATE.""" + events = self.dataset.get_patient("60").get_events() + tokens = events[0]["text"].split(" ") + labels = events[0]["labels"].split(" ") + date_idx = tokens.index("11/05/2097") + self.assertEqual(labels[date_idx], "B-DATE") + + def test_patient_60_has_profession_label(self): + """Patient 60's note has PROFESSION.""" + events = self.dataset.get_patient("60").get_events() + tokens = events[0]["text"].split(" ") + labels = events[0]["labels"].split(" ") + prof_idx = tokens.index("plumber.") + self.assertEqual(labels[prof_idx], "B-PROFESSION") + + def test_stats(self): + self.dataset.stats() + + def test_tmp_dir_cleaned_up_on_del(self): + """Temp directory should be removed when dataset is deleted.""" + cache_dir = tempfile.TemporaryDirectory() + dataset = PhysioNetDeIDDataset( + root=str(self.root), cache_dir=cache_dir.name + ) + tmp_dir = dataset._tmp_dir + self.assertTrue(os.path.isdir(tmp_dir)) + del dataset + self.assertFalse(os.path.exists(tmp_dir)) + cache_dir.cleanup() + + # -- Task tests -- + + def test_task_sample_count(self): + """11 notes total across 10 patients.""" + self.assertEqual(len(self.samples), 11) + + def test_task_sample_has_text_and_labels(self): + sample = self.samples[0] + self.assertIn("text", sample) + self.assertIn("labels", sample) + + def test_task_text_and_labels_same_length(self): + for sample in self.samples: + tokens = sample["text"].split(" ") + labels = sample["labels"].split(" ") + self.assertEqual(len(tokens), len(labels)) + + def test_task_labels_are_valid_bio(self): + valid = {"O"} + for cat in ("AGE", "CONTACT", "DATE", "ID", "LOCATION", "NAME", "PROFESSION"): + valid.add(f"B-{cat}") + valid.add(f"I-{cat}") + for sample in self.samples: + for label in sample["labels"].split(" "): + self.assertIn(label, valid) + + def test_task_sample_has_patient_id(self): + self.assertIn("patient_id", self.samples[0]) + + +class TestPhiSpanAlignment(unittest.TestCase): + """Tests for phi_spans_in_original with repeated non-PHI text.""" + + def test_repeated_word_across_phi_boundary(self): + """Non-PHI word 'at' appears before and after PHI tag.""" + orig = "seen at Mercy Hospital at noon" + deid = "seen at [**Hospital**] at noon" + spans = phi_spans_in_original(orig, deid) + tagged = bio_tag(orig, spans) + words = [w for w, _ in tagged] + labels = [l for _, l in tagged] + self.assertEqual(words, ["seen", "at", "Mercy", "Hospital", "at", "noon"]) + self.assertEqual(labels[0], "O") # seen + self.assertEqual(labels[1], "O") # at (before PHI) + self.assertIn("LOCATION", labels[2]) # Mercy + self.assertIn("LOCATION", labels[3]) # Hospital + self.assertEqual(labels[4], "O") # at (after PHI) + self.assertEqual(labels[5], "O") # noon + + def test_repeated_chunk_between_two_phi_tags(self): + """Same non-PHI text separates two different PHI spans.""" + orig = "Dr. Smith and Dr. Jones" + deid = "[**Doctor Name**] and [**Doctor Name**]" + spans = phi_spans_in_original(orig, deid) + tagged = bio_tag(orig, spans) + words = [w for w, _ in tagged] + labels = [l for _, l in tagged] + self.assertEqual(words, ["Dr.", "Smith", "and", "Dr.", "Jones"]) + self.assertNotEqual(labels[0], "O") # Dr. + self.assertNotEqual(labels[1], "O") # Smith + self.assertEqual(labels[2], "O") # and + self.assertNotEqual(labels[3], "O") # Dr. + self.assertNotEqual(labels[4], "O") # Jones + + +class TestClassifyPhiFallback(unittest.TestCase): + """Test that classify_phi logs a warning on unknown tags.""" + + def test_unknown_tag_logs_warning(self): + with self.assertLogs("pyhealth.datasets.physionet_deid", level=logging.WARNING) as cm: + result = classify_phi("xyzzy gibberish tag") + self.assertEqual(result, "NAME") + self.assertTrue(any("no keyword match" in msg for msg in cm.output)) + + +class TestDeIDNERTaskWindowing(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.root = ( + Path(__file__).parent.parent.parent + / "test-resources" + / "core" + / "physionet_deid" + ) + cls.cache_dir = tempfile.TemporaryDirectory() + cls.dataset = PhysioNetDeIDDataset( + root=str(cls.root), cache_dir=cls.cache_dir.name + ) + cls.task = DeIDNERTask(window_size=10, window_overlap=5) + cls.samples = cls.dataset.set_task(cls.task) + + @classmethod + def tearDownClass(cls): + cls.samples.close() + cls.cache_dir.cleanup() + + def test_windowing_produces_more_samples(self): + """Windowing should produce more samples than the 11 notes.""" + self.assertGreater(len(self.samples), 11) + + def test_window_size_respected(self): + """Each window should have at most window_size tokens.""" + for sample in self.samples: + tokens = sample["text"].split(" ") + self.assertLessEqual(len(tokens), 10) + + def test_window_text_and_labels_same_length(self): + for sample in self.samples: + tokens = sample["text"].split(" ") + labels = sample["labels"].split(" ") + self.assertEqual(len(tokens), len(labels)) + + def test_window_has_patient_id(self): + self.assertIn("patient_id", self.samples[0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/core/test_transformer_deid.py b/tests/core/test_transformer_deid.py new file mode 100644 index 000000000..a796663c1 --- /dev/null +++ b/tests/core/test_transformer_deid.py @@ -0,0 +1,199 @@ +""" +Unit tests for the TransformerDeID model. + +Author: + Matt McKenna (mtm16@illinois.edu) +""" + +import unittest + +import torch + +from pyhealth.datasets import create_sample_dataset +from pyhealth.models.transformer_deid import ( + IGNORE_INDEX, + LABEL_VOCAB, + TransformerDeID, + align_labels, +) +from pyhealth.processors.text_processor import TextProcessor + + +def _make_dataset(): + """Create a minimal in-memory dataset matching DeIDNERTask output.""" + samples = [ + { + "patient_id": "p1", + "text": "Patient John Smith was seen", + "labels": "O B-NAME I-NAME O O", + }, + { + "patient_id": "p2", + "text": "Admitted on 01/15/2024 to clinic", + "labels": "O O B-DATE O O", + }, + ] + return create_sample_dataset( + samples=samples, + input_schema={"text": TextProcessor}, + output_schema={"labels": TextProcessor}, + dataset_name="test_deid", + task_name="DeIDNER", + in_memory=True, + ) + + +class TestLabelVocab(unittest.TestCase): + def test_vocab_size(self): + """O + 7 categories * 2 (B/I) = 15.""" + self.assertEqual(len(LABEL_VOCAB), 15) + + def test_o_is_zero(self): + self.assertEqual(LABEL_VOCAB["O"], 0) + + def test_all_categories_present(self): + for cat in ("AGE", "CONTACT", "DATE", "ID", "LOCATION", "NAME", "PROFESSION"): + self.assertIn(f"B-{cat}", LABEL_VOCAB) + self.assertIn(f"I-{cat}", LABEL_VOCAB) + + +class TestAlignLabels(unittest.TestCase): + def test_no_subword_splits(self): + """When every word is a single token, labels pass through.""" + # word_ids: None=CLS, 0, 1, 2, None=SEP + word_ids = [None, 0, 1, 2, None] + word_labels = [0, 11, 12] # O, B-NAME, I-NAME + result = align_labels(word_ids, word_labels) + self.assertEqual(result, [IGNORE_INDEX, 0, 11, 12, IGNORE_INDEX]) + + def test_subword_split(self): + """Non-first subtokens should get IGNORE_INDEX.""" + # "Smith" split into 2 subtokens (word_id=1 twice) + word_ids = [None, 0, 1, 1, 2, None] + word_labels = [0, 12, 0] # O, I-NAME, O + result = align_labels(word_ids, word_labels) + self.assertEqual( + result, + [IGNORE_INDEX, 0, 12, IGNORE_INDEX, 0, IGNORE_INDEX], + ) + + def test_all_special_tokens(self): + """All-None word_ids should produce all IGNORE_INDEX.""" + word_ids = [None, None] + result = align_labels(word_ids, []) + self.assertEqual(result, [IGNORE_INDEX, IGNORE_INDEX]) + + +class TestTransformerDeIDInit(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.dataset = _make_dataset() + cls.model = TransformerDeID(dataset=cls.dataset) + + @classmethod + def tearDownClass(cls): + cls.dataset.close() + + def test_feature_key(self): + self.assertEqual(self.model.feature_key, "text") + + def test_label_key(self): + self.assertEqual(self.model.label_key, "labels") + + def test_num_labels(self): + self.assertEqual(self.model.num_labels, 15) + + def test_classifier_output_dim(self): + self.assertEqual(self.model.classifier.out_features, 15) + + def test_encoder_hidden_size(self): + """BERT-base has hidden_size=768.""" + self.assertEqual(self.model.encoder.config.hidden_size, 768) + + +class TestTransformerDeIDForward(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.dataset = _make_dataset() + cls.model = TransformerDeID(dataset=cls.dataset) + cls.model.eval() + # Run a forward pass with raw strings (same format as task output). + with torch.no_grad(): + cls.result = cls.model( + text=[ + "Patient John Smith was seen", + "Admitted on 01/15/2024 to clinic", + ], + labels=[ + "O B-NAME I-NAME O O", + "O O B-DATE O O", + ], + ) + + @classmethod + def tearDownClass(cls): + cls.dataset.close() + + def test_output_has_required_keys(self): + for key in ("loss", "logit", "y_prob", "y_true"): + self.assertIn(key, self.result) + + def test_loss_is_scalar(self): + self.assertEqual(self.result["loss"].dim(), 0) + + def test_logit_shape(self): + """logit should be (batch, seq_len, num_labels).""" + logit = self.result["logit"] + self.assertEqual(logit.shape[0], 2) # batch size + self.assertEqual(logit.shape[2], 15) # num labels + + def test_y_prob_shape_matches_logit(self): + self.assertEqual( + self.result["y_prob"].shape, self.result["logit"].shape + ) + + def test_y_prob_sums_to_one(self): + """Softmax probabilities should sum to ~1 along label dim.""" + sums = self.result["y_prob"].sum(dim=-1) + self.assertTrue(torch.allclose(sums, torch.ones_like(sums), atol=1e-5)) + + def test_backward(self): + """Loss backward should produce gradients.""" + # Need train mode and fresh forward pass for gradients. + self.model.train() + result = self.model( + text=["Patient John Smith was seen"], + labels=["O B-NAME I-NAME O O"], + ) + result["loss"].backward() + has_grad = any( + p.requires_grad and p.grad is not None + for p in self.model.parameters() + ) + self.assertTrue(has_grad) + self.model.eval() + self.model.zero_grad() + + def test_deidentify_returns_string(self): + result = self.model.deidentify("Patient John Smith was seen") + self.assertIsInstance(result, str) + + def test_deidentify_same_word_count(self): + """Output should have same number of words (redacted or not).""" + text = "Patient John Smith was seen" + result = self.model.deidentify(text) + self.assertEqual(len(result.split()), len(text.split())) + + def test_deidentify_custom_redact_marker(self): + result = self.model.deidentify("Patient John", redact="[PHI]") + self.assertNotIn("[REDACTED]", result) + # Every word should be either an original word or the custom marker. + for word in result.split(): + self.assertTrue( + word in ("Patient", "John") or word == "[PHI]", + f"Unexpected word in output: {word}", + ) + + +if __name__ == "__main__": + unittest.main() From 144c06a9d359df07cb0753ef0a1967cdb1432262 Mon Sep 17 00:00:00 2001 From: Kobe Guo <89816161+KobeGuo99@users.noreply.github.com> Date: Sun, 19 Apr 2026 12:47:47 -0500 Subject: [PATCH 07/61] dl4h final project kobeguo2 - CaliForest (#999) * dl4h final project kobeguo2 - CaliForest * Update CaliForest to require explicit fit before inference * Remove unused logit_scale from CaliForest --- docs/api/models.rst | 1 + .../api/models/pyhealth.models.califorest.rst | 7 + examples/mimic4_califorest.py | 190 ++++++++++++++ pyhealth/models/__init__.py | 1 + pyhealth/models/califorest.py | 233 ++++++++++++++++++ tests/core/test_califorest.py | 150 +++++++++++ 6 files changed, 582 insertions(+) create mode 100644 docs/api/models/pyhealth.models.califorest.rst create mode 100644 examples/mimic4_califorest.py create mode 100644 pyhealth/models/califorest.py create mode 100644 tests/core/test_califorest.py diff --git a/docs/api/models.rst b/docs/api/models.rst index 166402e86..7c3ac7c4b 100644 --- a/docs/api/models.rst +++ b/docs/api/models.rst @@ -205,3 +205,4 @@ API Reference models/pyhealth.models.TextEmbedding models/pyhealth.models.BIOT models/pyhealth.models.unified_multimodal_embedding_docs + models/pyhealth.models.califorest diff --git a/docs/api/models/pyhealth.models.califorest.rst b/docs/api/models/pyhealth.models.califorest.rst new file mode 100644 index 000000000..69ee1ff9b --- /dev/null +++ b/docs/api/models/pyhealth.models.califorest.rst @@ -0,0 +1,7 @@ +pyhealth.models.califorest +========================== + +.. automodule:: pyhealth.models.califorest + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/examples/mimic4_califorest.py b/examples/mimic4_califorest.py new file mode 100644 index 000000000..01d26de2d --- /dev/null +++ b/examples/mimic4_califorest.py @@ -0,0 +1,190 @@ +from __future__ import annotations + +import os + +import numpy as np +import torch +from sklearn.ensemble import RandomForestClassifier +from sklearn.metrics import brier_score_loss, roc_auc_score + +from pyhealth.datasets import ( + MIMIC4EHRDataset, + create_sample_dataset, + get_dataloader, +) +from pyhealth.models import CaliForest +from pyhealth.tasks import InHospitalMortalityMIMIC4 + + +# Set your MIMIC-IV dataset path via environment variable before running: +# export MIMIC4_ROOT=/your/path/to/mimiciv/3.1 +ROOT = os.getenv("MIMIC4_ROOT") + + +def evaluate(y_true: np.ndarray, y_prob: np.ndarray) -> dict[str, float]: + """Compute AUROC and Brier score.""" + y_true = np.asarray(y_true).reshape(-1) + y_prob = np.asarray(y_prob).reshape(-1) + return { + "auroc": float(roc_auc_score(y_true, y_prob)), + "brier": float(brier_score_loss(y_true, y_prob)), + } + + +def run_califorest( + X_train: np.ndarray, + y_train: np.ndarray, + X_test: np.ndarray, + y_test: np.ndarray, + calibration: str, +) -> dict[str, float]: + """Train and evaluate CaliForest on tabularized features.""" + train_samples = [] + for i in range(len(X_train)): + train_samples.append( + { + "patient_id": f"train-{i}", + "visit_id": f"train-{i}", + "features": X_train[i].tolist(), + "label": int(y_train[i]), + } + ) + + test_samples = [] + for i in range(len(X_test)): + test_samples.append( + { + "patient_id": f"test-{i}", + "visit_id": f"test-{i}", + "features": X_test[i].tolist(), + "label": int(y_test[i]), + } + ) + + train_dataset = create_sample_dataset( + samples=train_samples, + input_schema={"features": "tensor"}, + output_schema={"label": "binary"}, + dataset_name=f"mimic4_train_tabular_{calibration}", + ) + test_dataset = create_sample_dataset( + samples=test_samples, + input_schema={"features": "tensor"}, + output_schema={"label": "binary"}, + dataset_name=f"mimic4_test_tabular_{calibration}", + ) + + train_loader = get_dataloader( + train_dataset, batch_size=len(train_dataset), shuffle=False + ) + test_loader = get_dataloader( + test_dataset, batch_size=len(test_dataset), shuffle=False + ) + + test_batch = next(iter(test_loader)) + + model = CaliForest( + dataset=train_dataset, + n_estimators=100, + calibration=calibration, + random_state=42, + ) + model.fit(train_loader) + + with torch.no_grad(): + ret = model(**test_batch) + + cali_probs = ret["y_prob"].detach().cpu().numpy().reshape(-1) + return evaluate(y_test, cali_probs) + + +def main(): + if not ROOT: + raise ValueError( + "MIMIC4_ROOT is not set. Example:\n" + "export MIMIC4_ROOT=/your/path/to/mimiciv/3.1" + ) + + print("=" * 80) + print("Loading MIMIC-IV EHR dataset") + print("=" * 80) + + dataset = MIMIC4EHRDataset( + root=ROOT, + tables=["diagnoses_icd", "procedures_icd", "labevents"], + ) + + task = InHospitalMortalityMIMIC4() + sample_dataset = dataset.set_task(task) + + print(f"Total samples: {len(sample_dataset)}") + + subset_size = 2000 + raw_subset_samples = [sample_dataset[i] for i in range(subset_size)] + + clean_subset_samples = [] + for sample in raw_subset_samples: + clean_subset_samples.append( + { + "patient_id": str(sample["patient_id"]), + "visit_id": str(sample["admission_id"]), + "labs": sample["labs"].tolist(), + "mortality": int(sample["mortality"].item()), + } + ) + + subset_dataset = create_sample_dataset( + samples=clean_subset_samples, + input_schema={"labs": "tensor"}, + output_schema={"mortality": "binary"}, + dataset_name="mimic4_mortality_subset", + ) + + loader = get_dataloader(subset_dataset, batch_size=subset_size, shuffle=False) + batch = next(iter(loader)) + + X = batch["labs"].detach().cpu().numpy() + y = batch["mortality"].detach().cpu().numpy().reshape(-1) + + X = X.reshape(X.shape[0], -1) + + print("Flattened feature matrix:", X.shape) + print("Labels:", y.shape) + + split = int(0.8 * len(X)) + X_train, X_test = X[:split], X[split:] + y_train, y_test = y[:split], y[split:] + + print("=" * 80) + print("Baseline Random Forest") + print("=" * 80) + + rf = RandomForestClassifier( + n_estimators=100, + random_state=42, + bootstrap=True, + ) + rf.fit(X_train, y_train) + rf_probs = rf.predict_proba(X_test)[:, 1] + rf_metrics = evaluate(y_test, rf_probs) + print("RF metrics:", rf_metrics) + + print("=" * 80) + print("CaliForest (isotonic calibration)") + print("=" * 80) + isotonic_metrics = run_califorest( + X_train, y_train, X_test, y_test, calibration="isotonic" + ) + print("CaliForest isotonic metrics:", isotonic_metrics) + + print("=" * 80) + print("CaliForest (logistic calibration)") + print("=" * 80) + logistic_metrics = run_califorest( + X_train, y_train, X_test, y_test, calibration="logistic" + ) + print("CaliForest logistic metrics:", logistic_metrics) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/pyhealth/models/__init__.py b/pyhealth/models/__init__.py index deabaf95c..4c168d3e3 100644 --- a/pyhealth/models/__init__.py +++ b/pyhealth/models/__init__.py @@ -45,3 +45,4 @@ from .sdoh import SdohClassifier from .medlink import MedLink from .unified_embedding import UnifiedMultimodalEmbeddingModel, SinusoidalTimeEmbedding +from .califorest import CaliForest \ No newline at end of file diff --git a/pyhealth/models/califorest.py b/pyhealth/models/califorest.py new file mode 100644 index 000000000..bdc539eaf --- /dev/null +++ b/pyhealth/models/califorest.py @@ -0,0 +1,233 @@ +""" +Author: Kobe Guo +NetID: kobeg2 + +Paper: CaliForest: Calibrated Random Forests for Healthcare Prediction +Link: https://joyceho.github.io/assets/pdf/paper/park-chil20.pdf + +Description: +Implementation of CaliForest, a calibrated random forest model that applies +post-hoc calibration (isotonic or logistic) to improve probability estimates +for healthcare prediction tasks. +""" + +from __future__ import annotations + +from typing import Dict, List, Optional + +import numpy as np +import torch +import torch.nn as nn +from sklearn.ensemble import RandomForestClassifier +from sklearn.isotonic import IsotonicRegression +from sklearn.linear_model import LogisticRegression + +from pyhealth.datasets import SampleDataset +from pyhealth.models import BaseModel + + +class CaliForest(BaseModel): + """CaliForest model for calibrated probability prediction. + + This model wraps a RandomForestClassifier and applies a post-hoc + calibration step using out-of-bag (OOB) predictions and prediction + variance to improve probability estimates. + + Important: + CaliForest is fit once on the full training set using fit(train_loader). + After fitting, forward() should be used only for inference/evaluation. + This implementation currently supports binary classification only. + + The overall procedure is: + 1. train a random forest classifier, + 2. compute OOB probabilities for each training sample, + 3. estimate prediction uncertainty using variance across tree outputs, + 4. fit a calibration model using uncertainty-weighted samples. + + Args: + dataset: the dataset used to initialize feature and label schemas. + n_estimators: number of trees in the random forest. Default is 100. + max_depth: maximum depth of each tree. Default is None. + calibration: calibration method. Supported values are ``"isotonic"`` + and ``"logistic"``. Default is ``"isotonic"``. + random_state: random seed for reproducibility. Default is 42. + **kwargs: additional compatibility arguments. + + Example: + model = CaliForest(dataset=dataset, n_estimators=10) + model.fit(train_loader) + ret = model(**batch) + print(ret["y_prob"].shape) + """ + + def __init__( + self, + dataset: SampleDataset, + n_estimators: int = 100, + max_depth: Optional[int] = None, + calibration: str = "isotonic", + random_state: int = 42, + **kwargs, + ): + super(CaliForest, self).__init__(dataset) + + assert len(self.label_keys) == 1, "Only one label key is supported" + self.label_key = self.label_keys[0] + + self.n_estimators = n_estimators + self.max_depth = max_depth + self.calibration = calibration + self.random_state = random_state + + if self.calibration not in {"isotonic", "logistic"}: + raise ValueError(f"Unsupported calibration: {self.calibration}") + + self.rf = RandomForestClassifier( + n_estimators=self.n_estimators, + max_depth=self.max_depth, + bootstrap=True, + oob_score=True, + random_state=self.random_state, + ) + + self.calibrator = None + self.is_fitted = False + + + def _build_feature_matrix(self, **kwargs) -> np.ndarray: + """Convert PyHealth batch into NumPy feature matrix.""" + features: List[np.ndarray] = [] + + for key in self.feature_keys: + x = kwargs[key] + + if isinstance(x, torch.Tensor): + arr = x.detach().cpu().numpy() + else: + arr = np.asarray(x) + + if arr.ndim == 1: + arr = arr.reshape(-1, 1) + elif arr.ndim > 2: + arr = arr.reshape(arr.shape[0], -1) + + features.append(arr.astype(np.float32)) + + return np.concatenate(features, axis=1) + + def _build_labels(self, **kwargs) -> np.ndarray: + y = kwargs[self.label_key] + if isinstance(y, torch.Tensor): + y = y.detach().cpu().numpy() + else: + y = np.asarray(y) + return y.reshape(-1) + + def fit(self, train_loader): + """Fit CaliForest on the full training dataloader""" + X_list = [] + y_list = [] + + for batch in train_loader: + X_list.append(self._build_feature_matrix(**batch)) + y_list.append(self._build_labels(**batch)) + + X = np.concatenate(X_list, axis=0) + y = np.concatenate(y_list, axis=0) + + self.fit_model(features=X, labels=y) + return self + + def fit_model(self, **kwargs) -> None: + """Fit RF + calibration model.""" + if "features" in kwargs and "labels" in kwargs: + X = kwargs["features"] + y = kwargs["labels"] + else: + X = self._build_feature_matrix(**kwargs) + y = self._build_labels(**kwargs) + + unique_labels = np.unique(y) + if set(unique_labels.tolist()) != {0, 1}: + raise ValueError( + "CaliForest currently supports binary classification only. " + f"Got labels: {unique_labels.tolist()}" + ) + self.rf.fit(X, y) + + if not hasattr(self.rf, "oob_decision_function_"): + raise RuntimeError("OOB predictions not available.") + + oob_probs = self.rf.oob_decision_function_[:, 1] + + tree_probs = np.stack( + [t.predict_proba(X)[:, 1] for t in self.rf.estimators_], + axis=0, + ) + variances = np.var(tree_probs, axis=0) + + # CaliForest uses inverse tree-level variance so more stable + # predictions have greater influence during calibrator fitting. + weights = 1.0 / (variances + 1e-6) + + if self.calibration == "isotonic": + calibrator = IsotonicRegression(out_of_bounds="clip") + calibrator.fit(oob_probs, y, sample_weight=weights) + self.calibrator = calibrator + else: + calibrator = LogisticRegression() + calibrator.fit( + oob_probs.reshape(-1, 1), + y, + sample_weight=weights, + ) + self.calibrator = calibrator + + self.is_fitted = True + + def predict_proba_numpy(self, **kwargs) -> np.ndarray: + """Predict calibrated probabilities.""" + if not self.is_fitted: + raise RuntimeError("Model must be fitted first.") + + X = self._build_feature_matrix(**kwargs) + rf_probs = self.rf.predict_proba(X)[:, 1] + + if self.calibration == "isotonic": + calibrated = self.calibrator.predict(rf_probs) + else: + calibrated = self.calibrator.predict_proba( + rf_probs.reshape(-1, 1) + )[:, 1] + + return calibrated.reshape(-1, 1) + + def forward(self, **kwargs) -> Dict[str, torch.Tensor]: + """PyHealth forward pass.""" + if not self.is_fitted: + raise RuntimeError( + "CaliForest must be fitted before inference. " + "Call model.fit(train_loader) first." + ) + + y_prob_np = self.predict_proba_numpy(**kwargs) + + y_prob = torch.tensor( + y_prob_np, dtype=torch.float32, device=self.device + ) + + eps = 1e-6 + logits = torch.log( + torch.clamp(y_prob, eps, 1 - eps) / + torch.clamp(1 - y_prob, eps, 1 - eps) + ) + + y_true = kwargs[self.label_key].to(self.device) + loss = self.get_loss_function()(logits, y_true) + + return { + "loss": loss, + "y_prob": y_prob, + "y_true": y_true, + "logit": logits, + } \ No newline at end of file diff --git a/tests/core/test_califorest.py b/tests/core/test_califorest.py new file mode 100644 index 000000000..58d89a8be --- /dev/null +++ b/tests/core/test_califorest.py @@ -0,0 +1,150 @@ +import unittest +import torch + +from pyhealth.datasets import create_sample_dataset, get_dataloader +from pyhealth.models import CaliForest + + +class TestCaliForest(unittest.TestCase): + """Test cases for the CaliForest model.""" + + def setUp(self): + """Set up synthetic data, dataset, and model.""" + self.samples = [ + { + "patient_id": "patient-0", + "visit_id": "visit-0", + "features": [1.0, 2.0, 3.0, 4.0], + "label": 0, + }, + { + "patient_id": "patient-1", + "visit_id": "visit-1", + "features": [2.0, 1.5, 0.5, 3.0], + "label": 1, + }, + { + "patient_id": "patient-2", + "visit_id": "visit-2", + "features": [0.5, 0.7, 1.2, 1.8], + "label": 0, + }, + { + "patient_id": "patient-3", + "visit_id": "visit-3", + "features": [3.1, 2.9, 4.0, 1.2], + "label": 1, + }, + ] + + self.input_schema = {"features": "tensor"} + self.output_schema = {"label": "binary"} + + self.dataset = create_sample_dataset( + samples=self.samples, + input_schema=self.input_schema, + output_schema=self.output_schema, + dataset_name="califorest_test", + ) + + self.model = CaliForest( + dataset=self.dataset, + n_estimators=10, + calibration="isotonic", + random_state=42, + ) + + self.loader = get_dataloader(self.dataset, batch_size=4, shuffle=False) + + def test_model_initialization(self): + """Test that the model initializes correctly.""" + self.assertIsInstance(self.model, CaliForest) + self.assertEqual(self.model.n_estimators, 10) + self.assertEqual(self.model.calibration, "isotonic") + self.assertEqual(self.model.label_key, "label") + self.assertFalse(self.model.is_fitted) + + def test_model_forward(self): + """Test that forward pass works and returns expected keys.""" + batch = next(iter(self.loader)) + self.model.fit(self.loader) + + with torch.no_grad(): + ret = self.model(**batch) + + self.assertIn("loss", ret) + self.assertIn("y_prob", ret) + self.assertIn("y_true", ret) + self.assertIn("logit", ret) + + self.assertEqual(ret["y_prob"].shape, (4, 1)) + self.assertEqual(ret["y_true"].shape, (4, 1)) + self.assertEqual(ret["logit"].shape, (4, 1)) + self.assertEqual(ret["loss"].dim(), 0) + + def test_probability_range(self): + """Test that predicted probabilities are in [0, 1].""" + batch = next(iter(self.loader)) + self.model.fit(self.loader) + + with torch.no_grad(): + ret = self.model(**batch) + + y_prob = ret["y_prob"] + self.assertTrue(torch.all(y_prob >= 0.0).item()) + self.assertTrue(torch.all(y_prob <= 1.0).item()) + + def test_forward_before_fit_raises(self): + """Test that calling forward before fit raises a clear error.""" + batch = next(iter(self.loader)) + + with self.assertRaises(RuntimeError): + self.model(**batch) + + def test_logistic_calibration(self): + """Test the logistic calibration option.""" + model = CaliForest( + dataset=self.dataset, + n_estimators=10, + calibration="logistic", + random_state=42, + ) + + batch = next(iter(self.loader)) + model.fit(self.loader) + + with torch.no_grad(): + ret = model(**batch) + + self.assertIn("y_prob", ret) + self.assertEqual(ret["y_prob"].shape, (4, 1)) + + def test_isotonic_and_logistic_differ(self): + """Test that isotonic and logistic calibration produce different outputs.""" + iso_model = CaliForest( + dataset=self.dataset, + n_estimators=10, + calibration="isotonic", + random_state=42, + ) + log_model = CaliForest( + dataset=self.dataset, + n_estimators=10, + calibration="logistic", + random_state=42, + ) + + iso_model.fit(self.loader) + log_model.fit(self.loader) + + batch = next(iter(self.loader)) + + with torch.no_grad(): + iso_probs = iso_model(**batch)["y_prob"] + log_probs = log_model(**batch)["y_prob"] + + self.assertFalse(torch.allclose(iso_probs, log_probs)) + + +if __name__ == "__main__": + unittest.main() From 1262a760e44b3d22142117e3ba64a2ddaadee492 Mon Sep 17 00:00:00 2001 From: Logic <38597904+Logiquo@users.noreply.github.com> Date: Sat, 25 Apr 2026 18:22:51 -0400 Subject: [PATCH 08/61] Fix drug recommandation drug code and padding. (#1138) * Fix Drug Recommandation NDC/ATC3 code * Fix padding behaviour * remove .codex file * Change test from FakePatient to demo dataset --- .gitignore | 1 + pyhealth/models/embedding.py | 4 +- pyhealth/tasks/drug_recommendation.py | 68 +++++++- tests/core/test_drug_recommendation_atc3.py | 174 ++++++++++++++++++++ tests/core/test_embedding_model_padding.py | 58 +++++++ 5 files changed, 293 insertions(+), 12 deletions(-) create mode 100644 tests/core/test_drug_recommendation_atc3.py create mode 100644 tests/core/test_embedding_model_padding.py diff --git a/.gitignore b/.gitignore index 9993737db..086c8da3f 100644 --- a/.gitignore +++ b/.gitignore @@ -137,6 +137,7 @@ data/physionet.org/ # VSCode settings .vscode/ +.codex # Model weight files (large binaries, distributed separately) weightfiles/ \ No newline at end of file diff --git a/pyhealth/models/embedding.py b/pyhealth/models/embedding.py index 83a3a78c0..4232b2788 100644 --- a/pyhealth/models/embedding.py +++ b/pyhealth/models/embedding.py @@ -165,15 +165,13 @@ def __init__( ): vocab_size = len(processor.code_vocab) - # For NestedSequenceProcessor and DeepNestedSequenceProcessor, don't use padding_idx - # because empty visits/groups need non-zero embeddings. if isinstance( processor, (NestedSequenceProcessor, DeepNestedSequenceProcessor) ): self.embedding_layers[field_name] = nn.Embedding( num_embeddings=vocab_size, embedding_dim=embedding_dim, - padding_idx=None, + padding_idx=0, ) else: self.embedding_layers[field_name] = nn.Embedding( diff --git a/pyhealth/tasks/drug_recommendation.py b/pyhealth/tasks/drug_recommendation.py index 660343bed..ec113e1dd 100644 --- a/pyhealth/tasks/drug_recommendation.py +++ b/pyhealth/tasks/drug_recommendation.py @@ -1,11 +1,57 @@ -from typing import Any, Dict, List +from typing import Any, Dict, Iterable, List, Optional import polars as pl from pyhealth.data import Patient, Visit +from pyhealth.medcode import CrossMap from .base_task import BaseTask +_NDC_TO_ATC3_MAPPER = None +_NDC_TO_ATC3_CACHE: Dict[str, List[str]] = {} + + +def _get_ndc_to_atc3_mapper(): + global _NDC_TO_ATC3_MAPPER + if _NDC_TO_ATC3_MAPPER is None: + _NDC_TO_ATC3_MAPPER = CrossMap.load("NDC", "ATC") + return _NDC_TO_ATC3_MAPPER + + +def _is_missing_ndc(code: Any) -> bool: + if code is None: + return True + code = str(code).strip() + return code == "" or code == "0" or code.lower() in {"nan", "none", ""} + + +def _map_ndc_list_to_atc3( + ndc_codes: Iterable[Any], + mapper: Optional[Any] = None, +) -> List[str]: + """Maps MIMIC prescription NDCs to stable, deduplicated ATC-3 labels.""" + mapper = _get_ndc_to_atc3_mapper() if mapper is None else mapper + drugs: List[str] = [] + seen = set() + + for ndc in ndc_codes: + if _is_missing_ndc(ndc): + continue + ndc = str(ndc).strip() + if ndc not in _NDC_TO_ATC3_CACHE: + _NDC_TO_ATC3_CACHE[ndc] = mapper.map(ndc, target_kwargs={"level": 3}) + mapped_codes = _NDC_TO_ATC3_CACHE[ndc] + for code in mapped_codes: + if code is None: + continue + code = str(code).strip() + if code and code not in seen: + drugs.append(code) + seen.add(code) + + return drugs + + class DrugRecommendationMIMIC3(BaseTask): """Task for drug recommendation using MIMIC-III dataset. @@ -35,6 +81,10 @@ class DrugRecommendationMIMIC3(BaseTask): } output_schema: Dict[str, str] = {"drugs": "multilabel"} + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.cache_version = "ndc_to_atc3_v1" + def __call__(self, patient: Any) -> List[Dict[str, Any]]: """Process a patient to create drug recommendation samples. @@ -92,8 +142,7 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: prescriptions.select(pl.col("prescriptions/ndc")).to_series().to_list() ) - # ATC 3 level (first 4 characters) - drugs = [drug[:4] for drug in drugs if drug] + drugs = _map_ndc_list_to_atc3(drugs) # Exclude visits without condition, procedure, or drug code if len(conditions) * len(procedures) * len(drugs) == 0: @@ -173,6 +222,10 @@ class DrugRecommendationMIMIC4(BaseTask): } output_schema: Dict[str, str] = {"drugs": "multilabel"} + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.cache_version = "ndc_to_atc3_v1" + def __call__(self, patient: Any) -> List[Dict[str, Any]]: """Process a patient to create drug recommendation samples. @@ -240,8 +293,7 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: prescriptions.select(pl.col("prescriptions/ndc")).to_series().to_list() ) - # ATC 3 level (first 4 characters) - drugs = [drug[:4] for drug in drugs if drug] + drugs = _map_ndc_list_to_atc3(drugs) # Exclude visits without condition, procedure, or drug code if len(conditions) * len(procedures) * len(drugs) == 0: @@ -332,8 +384,7 @@ def drug_recommendation_mimic3_fn(patient: Patient): conditions = visit.get_code_list(table="DIAGNOSES_ICD") procedures = visit.get_code_list(table="PROCEDURES_ICD") drugs = visit.get_code_list(table="PRESCRIPTIONS") - # ATC 3 level - drugs = [drug[:4] for drug in drugs] + drugs = _map_ndc_list_to_atc3(drugs) # exclude: visits without condition, procedure, or drug code if len(conditions) * len(procedures) * len(drugs) == 0: continue @@ -413,8 +464,7 @@ def drug_recommendation_mimic4_fn(patient: Patient): conditions = visit.get_code_list(table="diagnoses_icd") procedures = visit.get_code_list(table="procedures_icd") drugs = visit.get_code_list(table="prescriptions") - # ATC 3 level - drugs = [drug[:4] for drug in drugs] + drugs = _map_ndc_list_to_atc3(drugs) # exclude: visits without condition, procedure, or drug code if len(conditions) * len(procedures) * len(drugs) == 0: continue diff --git a/tests/core/test_drug_recommendation_atc3.py b/tests/core/test_drug_recommendation_atc3.py new file mode 100644 index 000000000..1adb4937f --- /dev/null +++ b/tests/core/test_drug_recommendation_atc3.py @@ -0,0 +1,174 @@ +import csv +import gzip +import shutil +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +import pyhealth.tasks.drug_recommendation as drug_rec +from pyhealth.datasets import MIMIC3Dataset, MIMIC4Dataset +from pyhealth.tasks import DrugRecommendationMIMIC3, DrugRecommendationMIMIC4 + + +class FakeNDCToATC3Map: + def __init__(self): + self.calls = [] + self.mapping = { + "11111111111": ["A10B"], + "22222222222": ["C03C", "C03C"], + "33333333333": ["N02B"], + } + + def map(self, ndc, target_kwargs=None): + self.calls.append((ndc, target_kwargs)) + return self.mapping.get(ndc, []) + + +class TestDrugRecommendationATC3(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.resources_root = Path(__file__).parents[2] / "test-resources" / "core" + + def setUp(self): + drug_rec._NDC_TO_ATC3_MAPPER = None + drug_rec._NDC_TO_ATC3_CACHE.clear() + self.mapper = FakeNDCToATC3Map() + patcher = patch( + "pyhealth.tasks.drug_recommendation.CrossMap.load", + return_value=self.mapper, + ) + self.addCleanup(patcher.stop) + self.crossmap_load = patcher.start() + self.temp_dirs = [] + + def tearDown(self): + drug_rec._NDC_TO_ATC3_MAPPER = None + drug_rec._NDC_TO_ATC3_CACHE.clear() + for temp_dir in self.temp_dirs: + temp_dir.cleanup() + + def _copy_demo(self, demo_name): + temp_dir = tempfile.TemporaryDirectory() + self.temp_dirs.append(temp_dir) + source = self.resources_root / demo_name + target = Path(temp_dir.name) / demo_name + shutil.copytree(source, target) + return target, temp_dir + + def _rewrite_prescription_ndcs(self, path, replacements): + opener = gzip.open if path.suffix == ".gz" else open + with opener(path, "rt", newline="") as f: + reader = csv.DictReader(f) + rows = list(reader) + fieldnames = reader.fieldnames + + if fieldnames is None: + raise ValueError(f"No CSV header found in {path}") + + counts = {hadm_id: 0 for hadm_id in replacements} + templates = {} + rewritten_rows = [] + for row in rows: + hadm_id = str(row["hadm_id"]) + if hadm_id in replacements: + templates.setdefault(hadm_id, row.copy()) + index = counts[hadm_id] + ndcs = replacements[hadm_id] + row["ndc"] = ndcs[index] if index < len(ndcs) else "99999999999" + counts[hadm_id] += 1 + rewritten_rows.append(row) + + for hadm_id, ndcs in replacements.items(): + if hadm_id not in templates: + raise ValueError(f"No prescription rows found for hadm_id={hadm_id}") + while counts[hadm_id] < len(ndcs): + row = templates[hadm_id].copy() + if "row_id" in row: + row["row_id"] = str(10_000_000 + len(rewritten_rows)) + row["ndc"] = ndcs[counts[hadm_id]] + rewritten_rows.append(row) + counts[hadm_id] += 1 + + with opener(path, "wt", newline="") as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rewritten_rows) + + def _assert_atc3_samples(self, samples, first_hadm_id, second_hadm_id): + by_visit = {str(sample["visit_id"]): sample for sample in samples} + self.assertIn(first_hadm_id, by_visit) + self.assertIn(second_hadm_id, by_visit) + + first_sample = by_visit[first_hadm_id] + second_sample = by_visit[second_hadm_id] + self.assertEqual(first_sample["drugs"], ["A10B", "C03C"]) + self.assertEqual(second_sample["drugs"], ["N02B"]) + self.assertNotIn("1111", first_sample["drugs"]) + self.assertNotIn("2222", first_sample["drugs"]) + self.assertNotIn("3333", second_sample["drugs"]) + self.assertNotIn("0", first_sample["drugs"]) + self.assertNotIn("9999", first_sample["drugs"]) + + def test_mimic3_demo_drug_recommendation_maps_ndc_to_atc3(self): + demo_path, cache_dir = self._copy_demo("mimic3demo") + self._rewrite_prescription_ndcs( + demo_path / "PRESCRIPTIONS.csv.gz", + { + "142582": [ + "11111111111", + "22222222222", + "11111111111", + "0", + "99999999999", + ], + "122098": ["33333333333", "", ""], + }, + ) + dataset = MIMIC3Dataset( + root=str(demo_path), + tables=["diagnoses_icd", "procedures_icd", "prescriptions"], + cache_dir=cache_dir.name, + ) + + samples = DrugRecommendationMIMIC3()(dataset.get_patient("10059")) + + self.crossmap_load.assert_called_once_with("NDC", "ATC") + self._assert_atc3_samples(samples, "142582", "122098") + self.assertTrue( + all(kwargs == {"level": 3} for _, kwargs in self.mapper.calls) + ) + + def test_mimic4_demo_drug_recommendation_maps_ndc_to_atc3(self): + demo_path, cache_dir = self._copy_demo("mimic4demo") + self._rewrite_prescription_ndcs( + demo_path / "hosp" / "prescriptions.csv", + { + "20001": [ + "11111111111", + "22222222222", + "11111111111", + "0", + "99999999999", + ], + "20002": ["33333333333", "", ""], + }, + ) + dataset = MIMIC4Dataset( + ehr_root=str(demo_path), + ehr_tables=["diagnoses_icd", "procedures_icd", "prescriptions"], + cache_dir=cache_dir.name, + num_workers=1, + ) + + samples = DrugRecommendationMIMIC4()(dataset.get_patient("10001")) + + self.crossmap_load.assert_called_once_with("NDC", "ATC") + self._assert_atc3_samples(samples, "20001", "20002") + self.assertTrue( + all(kwargs == {"level": 3} for _, kwargs in self.mapper.calls) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/core/test_embedding_model_padding.py b/tests/core/test_embedding_model_padding.py new file mode 100644 index 000000000..16d0f5fdd --- /dev/null +++ b/tests/core/test_embedding_model_padding.py @@ -0,0 +1,58 @@ +import unittest + +import torch + +from pyhealth.datasets import create_sample_dataset +from pyhealth.models import EmbeddingModel + + +class TestEmbeddingModelPadding(unittest.TestCase): + def setUp(self): + samples = [ + { + "patient_id": "patient-0", + "visit_id": "visit-0", + "conditions": [["cond-1", "cond-2"], ["cond-3"]], + "deep_codes": [[["deep-1"], ["deep-2", "deep-3"]]], + "label": 1, + }, + { + "patient_id": "patient-1", + "visit_id": "visit-1", + "conditions": [["cond-4"]], + "deep_codes": [[["deep-4"]]], + "label": 0, + }, + ] + self.dataset = create_sample_dataset( + samples=samples, + input_schema={ + "conditions": "nested_sequence", + "deep_codes": "deep_nested_sequence", + }, + output_schema={"label": "binary"}, + dataset_name="embedding-padding-test", + ) + + def test_nested_sequence_embeddings_use_zero_padding(self): + model = EmbeddingModel(self.dataset, embedding_dim=8) + + for field in ["conditions", "deep_codes"]: + embedding = model.embedding_layers[field] + self.assertEqual(embedding.padding_idx, 0) + self.assertTrue(torch.equal(embedding.weight[0], torch.zeros(8))) + + def test_nested_sequence_padding_row_does_not_receive_gradients(self): + model = EmbeddingModel(self.dataset, embedding_dim=8) + embedding = model.embedding_layers["conditions"] + token_index = self.dataset.input_processors["conditions"].code_vocab["cond-1"] + + output = embedding(torch.tensor([[[0, token_index]]])) + output.sum().backward() + + self.assertTrue(torch.equal(embedding.weight.grad[0], torch.zeros(8))) + self.assertGreater(embedding.weight.grad[token_index].abs().sum().item(), 0) + + +if __name__ == "__main__": + unittest.main() From 3fcf8d2ed0190e001d650c3f78d91c09896cf9f1 Mon Sep 17 00:00:00 2001 From: Logic <38597904+Logiquo@users.noreply.github.com> Date: Sun, 26 Apr 2026 11:48:00 -0400 Subject: [PATCH 09/61] Fix sparsemax in AdaCare (#1139) --- pyhealth/models/adacare.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyhealth/models/adacare.py b/pyhealth/models/adacare.py index ab8602b77..09d8f3da2 100644 --- a/pyhealth/models/adacare.py +++ b/pyhealth/models/adacare.py @@ -38,7 +38,10 @@ def forward(self, input): zs = torch.sort(input=input, dim=dim, descending=True)[0] range = torch.arange( - start=1, end=number_of_logits + 1, dtype=torch.float32 + start=1, + end=number_of_logits + 1, + dtype=input.dtype, + device=input.device, ).view(1, -1) range = range.expand_as(zs) From cb75b5c4143296f299d0c7f0dad52f821c707eb7 Mon Sep 17 00:00:00 2001 From: John Wu <54558896+jhnwu3@users.noreply.github.com> Date: Mon, 4 May 2026 11:04:46 -0700 Subject: [PATCH 10/61] reinitialize tutorials for documentation as old tutorials died with UIUC purge (#1143) literally just updating the examples/ no need to waste reviewer time. --- .../tutorials/tutorial_pyhealth_data.ipynb | 219 ++++++++++++++++++ .../tutorials/tutorial_pyhealth_medcode.ipynb | 202 ++++++++++++++++ .../tutorials/tutorial_pyhealth_metrics.ipynb | 172 ++++++++++++++ .../tutorials/tutorial_pyhealth_model.ipynb | 148 ++++++++++++ .../tutorials/tutorial_pyhealth_trainer.ipynb | 174 ++++++++++++++ 5 files changed, 915 insertions(+) create mode 100644 examples/tutorials/tutorial_pyhealth_data.ipynb create mode 100644 examples/tutorials/tutorial_pyhealth_medcode.ipynb create mode 100644 examples/tutorials/tutorial_pyhealth_metrics.ipynb create mode 100644 examples/tutorials/tutorial_pyhealth_model.ipynb create mode 100644 examples/tutorials/tutorial_pyhealth_trainer.ipynb diff --git a/examples/tutorials/tutorial_pyhealth_data.ipynb b/examples/tutorials/tutorial_pyhealth_data.ipynb new file mode 100644 index 000000000..c02516d0a --- /dev/null +++ b/examples/tutorials/tutorial_pyhealth_data.ipynb @@ -0,0 +1,219 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": "# PyHealth Data API Tutorial\n\nThis notebook covers **`pyhealth.data`** — the foundational layer of PyHealth for representing longitudinal patient records.\n\nYou will learn:\n- How to create and work with **`Event`** objects representing individual clinical events\n- How to build a **`Patient`** object from a structured polars DataFrame\n- How to query a patient's event history using **`get_events()`** with powerful filters\n\n---" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Overview\n\nIn PyHealth, a patient's medical record is modeled as a collection of **events** over time. Each event belongs to a typed category (e.g., `'diagnosis'`, `'lab'`, `'note'`) and carries arbitrary attributes (ICD codes, numeric values, free text, etc.).\n\nThe two core classes are:\n\n| Class | Description |\n|-------|-------------|\n| `Event` | A single timestamped clinical occurrence with typed attributes |\n| `Patient` | A patient identified by `patient_id`, holding all events in a polars DataFrame |\n\nThese classes are optimized for efficient time-range and attribute filtering, using binary search on sorted timestamps and pre-built event-type partitions." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "from datetime import datetime\nimport polars as pl\nfrom pyhealth.data import Event, Patient" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Part 1: The `Event` Class\n\nAn `Event` is a **frozen dataclass** (immutable after creation) with three fields:\n\n```\nEvent(\n event_type: str, # category label, e.g. 'diagnosis', 'lab', 'note'\n timestamp: datetime, # when the event occurred\n attr_dict: dict # arbitrary key-value attributes (set via **kwargs)\n)\n```\n\n### Creating Events\n\nYou can create events for any clinical modality — diagnoses, labs, clinical notes, procedures, medications, etc." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- Diagnosis event (ICD-10-CM code) ---\ndiagnosis_event = Event(\n event_type=\"diagnosis\",\n timestamp=datetime(2024, 1, 15, 10, 30),\n icd_code=\"E11.9\",\n description=\"Type 2 diabetes mellitus without complications\",\n source=\"outpatient\"\n)\n\n# --- Lab event (numeric result) ---\nlab_event = Event(\n event_type=\"lab\",\n timestamp=datetime(2024, 1, 15, 11, 0),\n name=\"HbA1c\",\n value=8.5,\n unit=\"%\",\n status=\"abnormal\"\n)\n\n# --- Clinical note event (free text) ---\nnote_event = Event(\n event_type=\"note\",\n timestamp=datetime(2024, 1, 15, 14, 0),\n note_type=\"Discharge Summary\",\n text=(\n \"Patient is a 58-year-old male with Type 2 diabetes mellitus. \"\n \"HbA1c measured at 8.5%, above the target threshold of 7.0%. \"\n \"Medication regimen adjusted: metformin increased to 1000mg BID.\"\n )\n)\n\nprint(\"Created 3 events:\")\nprint(f\" {diagnosis_event.event_type} at {diagnosis_event.timestamp}\")\nprint(f\" {lab_event.event_type} at {lab_event.timestamp}\")\nprint(f\" {note_event.event_type} at {note_event.timestamp}\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "### Accessing Event Attributes\n\nPyHealth supports three access styles — pick whichever is most readable in context:" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# Style 1: subscript (dict-like)\nprint(\"[subscript] icd_code:\", diagnosis_event[\"icd_code\"])\nprint(\"[subscript] timestamp:\", diagnosis_event[\"timestamp\"])\n\n# Style 2: dot notation (attribute-like)\nprint(\"[dot] description:\", diagnosis_event.description)\nprint(\"[dot] source:\", diagnosis_event.source)\n\n# Style 3: containment check\nprint(\"\\nAttribute existence:\")\nprint(\" 'icd_code' in diagnosis_event:\", \"icd_code\" in diagnosis_event) # True\nprint(\" 'severity' in diagnosis_event:\", \"severity\" in diagnosis_event) # False\nprint(\" 'event_type' in diagnosis_event:\", \"event_type\" in diagnosis_event) # always True" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# The attr_dict is the underlying storage for all custom attributes\nprint(\"Lab event attr_dict:\", lab_event.attr_dict)\nprint(\"Lab value:\", lab_event.value)\nprint(\"Lab unit:\", lab_event.unit)" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "### Creating an Event from a Dictionary\n\n`Event.from_dict()` is used internally by `Patient.get_events()` when converting polars DataFrame rows back to `Event` objects. You can also use it directly.\n\nThe dictionary must follow the column naming convention used by `Patient` (explained in Part 2):" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# Event.from_dict expects the {event_type}/{attr} column naming convention\nrow_dict = {\n \"event_type\": \"lab\",\n \"timestamp\": datetime(2024, 3, 10, 9, 30),\n \"lab/name\": \"Blood Glucose\",\n \"lab/value\": 320.0,\n \"lab/unit\": \"mg/dL\",\n \"lab/status\": \"critical\",\n # columns from other event types are simply ignored\n \"diagnosis/icd_code\": None,\n}\n\nglucose_event = Event.from_dict(row_dict)\nprint(\"Reconstructed event type:\", glucose_event.event_type)\nprint(\"Glucose value:\", glucose_event.value, glucose_event.unit)\nprint(\"Status:\", glucose_event.status)" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Part 2: The `Patient` Class\n\nA `Patient` holds all events for a single patient in a **polars DataFrame** (`data_source`). Using polars enables fast vectorized filtering and binary search on sorted timestamps.\n\n### Column Naming Convention\n\nThe DataFrame must use a specific column structure:\n\n```\nColumn name format: {event_type}/{attribute_name}\n```\n\nFor example:\n- `diagnosis/icd_code` — the `icd_code` attribute of a `diagnosis` event\n- `lab/value` — the `value` attribute of a `lab` event\n- `note/text` — the `text` attribute of a `note` event\n\nRows belonging to a different event type will have **null** values in columns that don't belong to them. This sparse layout allows a single unified DataFrame to store heterogeneous event types efficiently.\n\n```\n event_type | timestamp | diagnosis/icd_code | lab/name | lab/value | note/text\n -----------|-------------|--------------------|-----------|-----------|-----------\n diagnosis | 2024-01-15 | E11.9 | null | null | null\n lab | 2024-01-15 | null | HbA1c | 8.5 | null\n note | 2024-01-15 | null | null | null | Patient...\n```" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# Build a DataFrame representing a diabetic patient's record across two visits\ndata = pl.DataFrame(\n {\n # ── required columns ──────────────────────────────────────────────────\n \"event_type\": [\n \"diagnosis\", \"diagnosis\",\n \"lab\", \"lab\", \"lab\",\n \"note\",\n ],\n \"timestamp\": [\n datetime(2024, 1, 15, 10, 30), # Visit 1 – diagnosis\n datetime(2024, 3, 10, 9, 0), # Visit 2 – diagnosis\n datetime(2024, 1, 15, 11, 0), # Visit 1 – HbA1c\n datetime(2024, 3, 10, 9, 30), # Visit 2 – blood glucose\n datetime(2024, 6, 5, 10, 0), # Visit 3 – blood glucose\n datetime(2024, 1, 15, 14, 0), # Visit 1 – discharge note\n ],\n # ── diagnosis columns ────────────────────────────────────────────────\n \"diagnosis/icd_code\": [\"E11.9\", \"E11.65\", None, None, None, None],\n \"diagnosis/description\": [\n \"Type 2 DM without complications\",\n \"Type 2 DM with hyperglycemia\",\n None, None, None, None,\n ],\n # ── lab columns ───────────────────────────────────────────────────────\n \"lab/name\": [None, None, \"HbA1c\", \"Blood Glucose\", \"Blood Glucose\", None],\n \"lab/value\": [None, None, 8.5, 320.0, 175.0, None],\n \"lab/unit\": [None, None, \"%\", \"mg/dL\", \"mg/dL\", None],\n \"lab/status\": [None, None, \"abnormal\", \"critical\", \"high\", None],\n # ── note columns ─────────────────────────────────────────────────────\n \"note/note_type\": [None, None, None, None, None, \"Discharge Summary\"],\n \"note/text\": [\n None, None, None, None, None,\n \"58-year-old male with T2DM. HbA1c 8.5%. Metformin increased to 1000mg BID.\",\n ],\n }\n)\n\nprint(\"DataFrame shape:\", data.shape)\nprint()\ndata" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# Instantiate the Patient\npatient = Patient(patient_id=\"P001\", data_source=data)\n\nprint(\"Patient ID:\", patient.patient_id)\nprint(\"Total rows in data_source:\", len(patient.data_source))\nprint(\"Event type partitions:\", list(patient.event_type_partitions.keys()))" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Part 3: Querying Events with `get_events()`\n\n```python\npatient.get_events(\n event_type: Optional[str] = None, # filter by category\n start: Optional[datetime] = None, # inclusive start time\n end: Optional[datetime] = None, # inclusive end time\n filters: Optional[List[tuple]] = None, # attribute filters: [(attr, op, val), ...]\n return_df: bool = False, # True → polars DataFrame; False → List[Event]\n)\n```\n\n**Performance notes:**\n- Filtering by `event_type` uses a pre-built partition dict → **O(1)**\n- Filtering by time range uses binary search on sorted timestamps → **O(log n)**\n- Attribute `filters` are applied afterwards → **O(k)** for k matching rows" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# 3a. Get ALL events (returns a List[Event])\nall_events = patient.get_events()\nprint(f\"Total events: {len(all_events)}\")\nfor e in all_events:\n print(f\" [{e.event_type}] {e.timestamp}\")" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# 3b. Filter by event_type\ndiagnoses = patient.get_events(event_type=\"diagnosis\")\nprint(f\"Diagnosis events: {len(diagnoses)}\")\nfor dx in diagnoses:\n print(f\" {dx.timestamp.date()} — {dx['icd_code']}: {dx['description']}\")" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# 3c. Filter by time range (inclusive on both ends)\nlabs_q1 = patient.get_events(\n event_type=\"lab\",\n start=datetime(2024, 1, 1),\n end=datetime(2024, 3, 31),\n)\nprint(f\"Lab events in Q1 2024: {len(labs_q1)}\")\nfor lab in labs_q1:\n print(f\" {lab.timestamp.date()} — {lab['name']}: {lab['value']} {lab['unit']} ({lab['status']})\")" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# 3d. Attribute filters: only abnormal or critical lab results\n# Format: [(attribute_name, operator, value), ...]\n# Supported operators: '==', '!=', '<', '<=', '>', '>='\nabnormal_labs = patient.get_events(\n event_type=\"lab\",\n filters=[(\"status\", \"!=\", \"high\")] # exclude 'high', keep 'abnormal' and 'critical'\n)\nprint(f\"Non-high abnormal labs: {len(abnormal_labs)}\")\nfor lab in abnormal_labs:\n print(f\" {lab['name']}: {lab['value']} {lab['unit']} — {lab['status']}\")" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# 3e. Multiple filters (AND logic)\ncritical_glucose = patient.get_events(\n event_type=\"lab\",\n filters=[\n (\"name\", \"==\", \"Blood Glucose\"),\n (\"value\", \">\", 200.0),\n ]\n)\nprint(f\"Critical glucose readings (>200 mg/dL): {len(critical_glucose)}\")\nfor lab in critical_glucose:\n print(f\" {lab.timestamp.date()} — {lab['value']} mg/dL\")" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# 3f. Return a polars DataFrame instead of Event objects\nlab_df = patient.get_events(event_type=\"lab\", return_df=True)\nprint(\"Lab events as DataFrame:\")\nprint(lab_df.select([\"timestamp\", \"lab/name\", \"lab/value\", \"lab/unit\", \"lab/status\"]))" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Part 4: Realistic Longitudinal Example\n\nLet's build a richer patient record — a 63-year-old with Type 2 diabetes, chronic kidney disease, and polyneuropathy, tracked across three hospital admissions over 18 months." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# A more complete patient record with three admissions\nfull_data = pl.DataFrame(\n {\n \"event_type\": [\n # Admission 1 (Jan 2023)\n \"admission\",\n \"diagnosis\", \"diagnosis\",\n \"lab\", \"lab\", \"lab\",\n \"note\",\n # Admission 2 (Jul 2023)\n \"admission\",\n \"diagnosis\", \"diagnosis\", \"diagnosis\",\n \"lab\", \"lab\",\n # Admission 3 (Jan 2024)\n \"admission\",\n \"diagnosis\",\n \"lab\",\n \"note\",\n ],\n \"timestamp\": [\n # Admission 1\n datetime(2023, 1, 10, 8, 0),\n datetime(2023, 1, 10, 9, 0),\n datetime(2023, 1, 10, 9, 5),\n datetime(2023, 1, 10, 10, 0),\n datetime(2023, 1, 10, 10, 5),\n datetime(2023, 1, 10, 10, 10),\n datetime(2023, 1, 12, 14, 0),\n # Admission 2\n datetime(2023, 7, 20, 8, 0),\n datetime(2023, 7, 20, 9, 0),\n datetime(2023, 7, 20, 9, 5),\n datetime(2023, 7, 20, 9, 10),\n datetime(2023, 7, 20, 10, 0),\n datetime(2023, 7, 20, 10, 5),\n # Admission 3\n datetime(2024, 1, 5, 8, 0),\n datetime(2024, 1, 5, 9, 0),\n datetime(2024, 1, 5, 10, 0),\n datetime(2024, 1, 7, 15, 0),\n ],\n # ── admission columns ──────────────────────────────────────────────────\n \"admission/hadm_id\": [\n \"ADM001\", None, None, None, None, None, None,\n \"ADM002\", None, None, None, None, None,\n \"ADM003\", None, None, None,\n ],\n \"admission/admit_type\": [\n \"EMERGENCY\", None, None, None, None, None, None,\n \"ELECTIVE\", None, None, None, None, None,\n \"URGENT\", None, None, None,\n ],\n # ── diagnosis columns ──────────────────────────────────────────────────\n \"diagnosis/icd_code\": [\n None, \"E11.9\", \"E11.65\", None, None, None, None,\n None, \"E11.22\", \"E11.42\", \"N18.3\", None, None,\n None, \"E11.65\", None, None,\n ],\n \"diagnosis/description\": [\n None,\n \"T2DM without complications\",\n \"T2DM with hyperglycemia\",\n None, None, None, None,\n None,\n \"T2DM with chronic kidney disease\",\n \"T2DM with polyneuropathy\",\n \"Chronic kidney disease, stage 3\",\n None, None,\n None,\n \"T2DM with hyperglycemia\",\n None, None,\n ],\n # ── lab columns ────────────────────────────────────────────────────────\n \"lab/name\": [\n None, None, None,\n \"HbA1c\", \"eGFR\", \"Creatinine\",\n None,\n None, None, None, None,\n \"HbA1c\", \"eGFR\",\n None, None,\n \"HbA1c\",\n None,\n ],\n \"lab/value\": [\n None, None, None,\n 8.5, 52.0, 1.8,\n None,\n None, None, None, None,\n 9.1, 44.0,\n None, None,\n 7.8,\n None,\n ],\n \"lab/unit\": [\n None, None, None,\n \"%\", \"mL/min/1.73m²\", \"mg/dL\",\n None,\n None, None, None, None,\n \"%\", \"mL/min/1.73m²\",\n None, None,\n \"%\",\n None,\n ],\n # ── note columns ───────────────────────────────────────────────────────\n \"note/note_type\": [\n None, None, None, None, None, None,\n \"Discharge Summary\",\n None, None, None, None, None, None,\n None, None, None,\n \"Progress Note\",\n ],\n \"note/text\": [\n None, None, None, None, None, None,\n \"Patient admitted for hyperglycemia management. HbA1c 8.5%. eGFR 52. Discharged on insulin glargine.\",\n None, None, None, None, None, None,\n None, None, None,\n \"HbA1c improved to 7.8%. eGFR stable. Continue current regimen.\",\n ],\n }\n)\n\nrichPatient = Patient(patient_id=\"P999\", data_source=full_data)\nprint(\"Patient P999 record:\")\nprint(f\" Total events: {len(richPatient.get_events())}\")\nprint(f\" Event types present: {list(richPatient.event_type_partitions.keys())}\")" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# Query: track HbA1c trend across admissions\nhba1c_labs = richPatient.get_events(\n event_type=\"lab\",\n filters=[(\"name\", \"==\", \"HbA1c\")]\n)\nprint(\"HbA1c Trend:\")\nfor lab in hba1c_labs:\n print(f\" {lab.timestamp.date()} — HbA1c: {lab['value']} {lab['unit']}\")" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# Query: get all diagnoses within a specific year (2023)\ndiagnoses_2023 = richPatient.get_events(\n event_type=\"diagnosis\",\n start=datetime(2023, 1, 1),\n end=datetime(2023, 12, 31, 23, 59, 59),\n)\nprint(\"Diagnoses in 2023:\")\nfor dx in diagnoses_2023:\n print(f\" {dx.timestamp.date()} — {dx['icd_code']}: {dx['description']}\")" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# Query: get declining kidney function (eGFR < 50)\nlow_egfr = richPatient.get_events(\n event_type=\"lab\",\n filters=[\n (\"name\", \"==\", \"eGFR\"),\n (\"value\", \"<\", 50.0),\n ]\n)\nprint(\"eGFR readings below 50 (worrisome kidney function):\")\nfor lab in low_egfr:\n print(f\" {lab.timestamp.date()} — eGFR: {lab['value']} {lab['unit']}\")" + }, + { + "cell_type": "markdown", + "id": "26be2c24", + "source": "---\n## Summary\n\n| Concept | Key API |\n|---------|----------|\n| Create an event | `Event(event_type, timestamp, **kwargs)` |\n| Access event attribute | `event[\"key\"]`, `event.key`, `\"key\" in event` |\n| Build from raw dict | `Event.from_dict(dict)` |\n| Create a patient | `Patient(patient_id, data_source=pl.DataFrame(...))` |\n| Get all events | `patient.get_events()` |\n| Filter by type | `patient.get_events(event_type=\"diagnoses_icd\")` |\n| Filter by time | `patient.get_events(start=..., end=...)` |\n| Filter by attribute | `patient.get_events(event_type=\"prescriptions\", filters=[(\"route\", \"==\", \"IV\")])` |\n| Return as DataFrame | `patient.get_events(return_df=True)` |\n| Load from MIMIC-III | `MIMIC3Dataset(root=..., tables=[\"diagnoses_icd\", ...])` |\n\n### Table name = event type\n\nWhen using a dataset loader like `MIMIC3Dataset`, the `event_type` on every event equals the **table name** from `mimic3.yaml` — not a generic category like `\"diagnosis\"`. The available attributes on each event are exactly the columns listed under that table's `attributes` key in the YAML.\n\n```\nTable name → event_type → example attributes\n─────────────────────────────────────────────────────────────────\ndiagnoses_icd → \"diagnoses_icd\" → icd9_code, hadm_id, seq_num\nprescriptions → \"prescriptions\" → drug, ndc, dose_val_rx, route\nnoteevents → \"noteevents\" → text, category, description\nadmissions → \"admissions\" → hadm_id, admission_type, hospital_expire_flag\nicustays → \"icustays\" → icustay_id, first_careunit, outtime\n```\n\nWhen writing a custom task, always check the relevant YAML config to know the exact attribute names available on events from each table.", + "metadata": {} + }, + { + "cell_type": "code", + "id": "ada1de03", + "source": "from pyhealth.datasets import MIMIC3Dataset\n\nroot = \"https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III\"\n\ndataset = MIMIC3Dataset(\n root=root,\n dataset_name=\"mimic3\",\n tables=[\n \"diagnoses_icd\", # ICD-9 diagnosis codes per admission\n \"prescriptions\", # medication orders\n \"noteevents\", # clinical notes\n ],\n)\n\nprint(f\"Loaded {len(dataset.patients)} patients\")\n\n# Grab one patient to explore\npatient_id = list(dataset.patients.keys())[0]\npatient = dataset.patients[patient_id]\nprint(f\"\\nPatient ID: {patient.patient_id}\")\nprint(f\"Event types present: {list(patient.event_type_partitions.keys())}\")\nprint(f\"Total events: {len(patient.get_events())}\")", + "metadata": {}, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "id": "65a34af1", + "source": "# Pull diagnosis events — event_type matches the table name exactly: \"diagnoses_icd\"\ndiagnoses = patient.get_events(\"diagnoses_icd\")\nprint(f\"Diagnosis events: {len(diagnoses)}\")\nprint()\n\n# Each event's attributes come from the 'attributes' list in mimic3.yaml\n# hadm_id, icd9_code, seq_num\nfor dx in diagnoses[:5]:\n print(f\" [{dx.timestamp.date()}] hadm={dx['hadm_id']} \"\n f\"ICD-9={dx['icd9_code']} seq={dx['seq_num']}\")", + "metadata": {}, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "id": "8c9817cc", + "source": "# Pull prescription events and filter to a specific route (e.g. IV medications)\nprescriptions = patient.get_events(\"prescriptions\")\nprint(f\"Total prescription events: {len(prescriptions)}\")\n\n# Attribute filters work the same way on real data\niv_meds = patient.get_events(\n event_type=\"prescriptions\",\n filters=[(\"route\", \"==\", \"IV\")],\n)\nprint(f\"IV medications: {len(iv_meds)}\")\nfor rx in iv_meds[:5]:\n print(f\" [{rx.timestamp.date()}] {rx['drug']} dose={rx['dose_val_rx']} ndc={rx['ndc']}\")", + "metadata": {}, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "id": "8892105c", + "source": "# Clinical note events — attribute 'text' holds the full note body\n# Attributes available: text, category, description, hadm_id, storetime (from mimic3.yaml)\nnotes = patient.get_events(\"noteevents\")\nprint(f\"Note events: {len(notes)}\")\n\ndischarge_notes = patient.get_events(\n event_type=\"noteevents\",\n filters=[(\"category\", \"==\", \"Discharge summary\")],\n)\nprint(f\"Discharge summaries: {len(discharge_notes)}\")\nif discharge_notes:\n first_note = discharge_notes[0]\n print(f\"\\n Date: {first_note.timestamp.date()}\")\n print(f\" Category: {first_note['category']}\")\n print(f\" Text preview: {str(first_note['text'])[:200]}...\")", + "metadata": {}, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Summary\n\n| Concept | Key API |\n|---------|----------|\n| Create an event | `Event(event_type, timestamp, **kwargs)` |\n| Access event attribute | `event[\"key\"]`, `event.key`, `\"key\" in event` |\n| Build from raw dict | `Event.from_dict(dict)` |\n| Create a patient | `Patient(patient_id, data_source=pl.DataFrame(...))` |\n| Get all events | `patient.get_events()` |\n| Filter by type | `patient.get_events(event_type=\"lab\")` |\n| Filter by time | `patient.get_events(start=..., end=...)` |\n| Filter by attribute | `patient.get_events(event_type=\"lab\", filters=[(\"value\", \">\", 7.0)])` |\n| Return as DataFrame | `patient.get_events(return_df=True)` |\n\nIn practice you won't create `Patient` objects manually — `MIMIC3Dataset` and other dataset loaders build them from raw EHR tables and expose them through the `set_task()` pipeline. But understanding the underlying API helps when writing custom tasks or debugging." + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.9.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/examples/tutorials/tutorial_pyhealth_medcode.ipynb b/examples/tutorials/tutorial_pyhealth_medcode.ipynb new file mode 100644 index 000000000..04c545f10 --- /dev/null +++ b/examples/tutorials/tutorial_pyhealth_medcode.ipynb @@ -0,0 +1,202 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": "# PyHealth Medical Code Ontology Tutorial\n\nThis notebook covers **`pyhealth.medcode`** — a medical code ontology library for looking up codes, exploring hierarchies, and translating between code systems.\n\nYou will learn:\n- How to load a medical code system using **`InnerMap`**\n- How to look up diabetes codes in **ICD-10-CM** with detailed explanations\n- How to traverse the **code hierarchy** (ancestors and descendants)\n- How to **translate codes** between systems using **`CrossMap`** (e.g., ICD-9 → ICD-10, ICD-10 → CCS)\n- Practical patterns for preprocessing EHR datasets\n\n---" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "from pyhealth.medcode import InnerMap, CrossMap" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Background: Medical Code Systems\n\nEHR data uses several overlapping code systems:\n\n| System | Domain | Example |\n|--------|--------|----------|\n| **ICD-10-CM** | Diagnoses (current US standard) | `E11.9` = Type 2 DM |\n| **ICD-9-CM** | Diagnoses (legacy, pre-2015) | `250.00` = Type 2 DM |\n| **ICD-10-PCS / ICD-9-PROC** | Procedures | |\n| **ATC** | Drug classification hierarchy | `A10BA02` = Metformin |\n| **RxNorm** | Drug concepts (US) | `860975` = Metformin 500mg tablet |\n| **NDC** | Drug product codes (package level) | |\n| **CCSCM** | Clinical Classifications Software — Diagnoses | `49` = Diabetes mellitus |\n| **CCSPROC** | CCS — Procedures | |\n\n**Why code mapping matters in ML:**\n- MIMIC-III uses ICD-9 codes (pre-2015 data), while MIMIC-IV uses ICD-10\n- Vocabularies differ in specificity: ICD-10 has ~70,000 codes vs ICD-9's ~14,000\n- ML models trained on ICD-9 codes cannot directly generalize to ICD-10 datasets without mapping\n- CCS groups 70,000 ICD-10 codes into ~300 categories — much better for small datasets" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Part 1: Loading a Code System\n\n`InnerMap.load(vocabulary)` downloads and caches the code system on first call, then loads from the local cache on subsequent calls." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# Load ICD-10-CM (Clinical Modification) — the US standard for diagnosis codes\nicd10cm = InnerMap.load(\"ICD10CM\")\n\n# Print statistics\nicd10cm.stat()" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# What attributes are available for each code?\nprint(\"Available attributes:\", icd10cm.available_attributes)" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# Check code existence\nprint(\"'E11.9' in ICD10CM:\", \"E11.9\" in icd10cm)\nprint(\"'E11.99' in ICD10CM:\", \"E11.99\" in icd10cm) # non-existent code\nprint(\"'XYZ' in ICD10CM:\", \"XYZ\" in icd10cm)" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Part 2: Diabetes Code Deep Dive\n\nDiabetes mellitus is encoded in ICD-10-CM chapter E08–E13. Here's the full taxonomy:\n\n```\nE08 Diabetes mellitus due to underlying condition\nE09 Drug or chemical induced diabetes mellitus\nE10 Type 1 diabetes mellitus\nE11 Type 2 diabetes mellitus\nE13 Other specified diabetes mellitus\n```\n\nEach category expands into complication subtypes. The E11 (Type 2) branch has **87 billable codes** in 2026 ICD-10-CM — a testament to how granular modern clinical coding is:\n\n```\nE11 Type 2 DM\n├─ E11.00 / E11.01 with hyperosmolarity (without / with coma)\n├─ E11.10 / E11.11 with ketoacidosis (without / with coma)\n├─ E11.2 with kidney complications\n│ ├─ E11.21 with diabetic nephropathy\n│ └─ E11.22 with diabetic chronic kidney disease\n├─ E11.3 with ophthalmic complications\n│ ├─ E11.311 / E11.319 unspecified retinopathy (with / without macular edema)\n│ ├─ E11.321x–E11.359x nonproliferative/proliferative retinopathy\n│ │ (each further split: right eye / left eye / bilateral / unspecified)\n│ └─ E11.36 with diabetic cataract\n├─ E11.4 with neurological complications\n│ ├─ E11.40 with diabetic neuropathy, unspecified\n│ └─ E11.42 with diabetic polyneuropathy\n├─ E11.5 with circulatory complications\n│ ├─ E11.51 with peripheral angiopathy without gangrene\n│ └─ E11.52 with peripheral angiopathy with gangrene\n├─ E11.6 with other specified complications\n│ ├─ E11.65 with hyperglycemia\n│ └─ E11.69 with other specified complication\n├─ E11.8 with unspecified complications\n├─ E11.9 without complications (most common at initial diagnosis)\n└─ E11.A without complications, in remission ← NEW in FY2026\n```\n\n> **2026 addition — `E11.A`:** *\"Type 2 diabetes mellitus without complications in remission\"* — confirmed valid for HIPAA transactions in the FY2026 ICD-10-CM release. This distinguishes patients who have achieved sustained normoglycemia (remission — e.g., post-bariatric surgery or sustained lifestyle intervention) from those still actively managed (E11.9).\n\n> **Note on retinopathy codes:** The E11.3x subcategory is highly specific. Real-world MIMIC data often uses the unspecified form (`E11.319`) because the laterality (left/right/bilateral) was not recorded at the time of billing. ML models typically collapse these to the 3–4 character level (e.g., using CCS or grouping all `E11.3xx` together)." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- Type 1 Diabetes Mellitus (E10) codes ---\ntype1_codes = [\"E10.9\", \"E10.65\", \"E10.319\"] # E10.3- = retinopathy\n\nprint(\"=\" * 60)\nprint(\"TYPE 1 DIABETES MELLITUS (E10)\")\nprint(\"=\" * 60)\nfor code in type1_codes:\n if code in icd10cm:\n name = icd10cm.lookup(code, attribute=\"name\")\n print(f\" {code:10s}: {name}\")\n else:\n print(f\" {code:10s}: [not found in this ICD10CM version]\")" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- Type 2 Diabetes Mellitus (E11) codes ---\ntype2_codes = [\n \"E11.9\", # without complications\n \"E11.65\", # with hyperglycemia\n \"E11.22\", # with diabetic chronic kidney disease (CKD)\n \"E11.42\", # with polyneuropathy\n \"E11.319\", # with unspecified diabetic retinopathy without macular edema\n]\n\nprint(\"=\" * 60)\nprint(\"TYPE 2 DIABETES MELLITUS (E11)\")\nprint(\"=\" * 60)\nfor code in type2_codes:\n if code in icd10cm:\n name = icd10cm.lookup(code, attribute=\"name\")\n print(f\" {code:10s}: {name}\")\n else:\n print(f\" {code:10s}: [not in this ICD10CM version]\")" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- Other diabetes types ---\nother_codes = [\n \"E08.9\", # diabetes due to underlying condition, without complications\n \"E09.9\", # drug/chemical-induced diabetes, without complications\n \"E13.9\", # other specified diabetes mellitus, without complications\n]\n\nprint(\"=\" * 60)\nprint(\"OTHER DIABETES TYPES\")\nprint(\"=\" * 60)\nfor code in other_codes:\n if code in icd10cm:\n name = icd10cm.lookup(code, attribute=\"name\")\n print(f\" {code:10s}: {name}\")\n else:\n print(f\" {code:10s}: [not in this ICD10CM version]\")" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- 2026 New Code: E11.A ---\n# Confirmed valid for HIPAA transactions in FY2026 ICD-10-CM (verified against live 2026 dataset).\n# Official description: \"Type 2 diabetes mellitus without complications in remission\"\n#\n# Clinical context:\n# E11.9 = T2DM without complications (still being managed / monitored)\n# E11.A = T2DM without complications IN REMISSION (achieved normoglycemia)\n#\n# Remission criteria (ADA 2021 Consensus): HbA1c < 6.5% for at least 3 months\n# without the use of glucose-lowering pharmacotherapy.\n# Common after bariatric surgery or significant sustained lifestyle intervention.\n\nnew_2026_code = \"E11.A\"\nif new_2026_code in icd10cm:\n name = icd10cm.lookup(new_2026_code)\n print(f\"[{new_2026_code}] {name}\")\n print()\n print(\"Contrast with E11.9:\")\n print(f\" [E11.9 ] {icd10cm.lookup('E11.9')}\")\n print(f\" [E11.A ] {name}\")\n print()\n print(\"Clinical significance:\")\n print(\" E11.9 → patient still has diabetes, actively managed\")\n print(\" E11.A → patient has achieved remission (ADA criteria: HbA1c < 6.5% for ≥3 months,\")\n print(\" no glucose-lowering medication)\")\nelse:\n # Fallback if the PyHealth cache predates FY2026\n print(\"E11.A not yet in local ICD10CM cache.\")\n print(\"Official 2026 description: 'Type 2 diabetes mellitus without complications in remission'\")\n print(\"Update the cache with: InnerMap.load('ICD10CM', refresh_cache=True)\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Part 3: Hierarchy Exploration\n\nICD codes form a tree where more specific codes are children of broader parent codes. PyHealth exposes this hierarchy via:\n- `get_ancestors(code)` — returns parent codes, ordered from closest to farthest\n- `get_descendants(code)` — returns child codes, ordered from closest to farthest\n\nThis hierarchy is stored internally as a **directed graph** using NetworkX." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- Ancestor lookup: trace E11.22 up the hierarchy ---\n# E11.22 = Type 2 diabetes mellitus with diabetic chronic kidney disease\ncode = \"E11.22\"\nancestors = icd10cm.get_ancestors(code)\n\nprint(f\"Ancestors of {code} ({icd10cm.lookup(code) if code in icd10cm else 'T2DM with CKD'}):\")\nprint(f\" [{code}] (starting code)\")\nfor anc in ancestors:\n if anc in icd10cm:\n name = icd10cm.lookup(anc)\n print(f\" ↑ [{anc}] {name}\")\n else:\n print(f\" ↑ [{anc}] (category node)\")" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- Descendant lookup: find all Type 2 DM subtypes ---\n# The live 2026 ICD-10-CM dataset has 87 billable E11 codes —\n# reflecting the high granularity of modern diabetes coding (retinopathy\n# laterality, ketoacidosis severity, complication type, etc.)\n\nparent_code = \"E11\"\ndescendants = icd10cm.get_descendants(parent_code)\n\nprint(f\"Descendants of {parent_code} (Type 2 Diabetes Mellitus):\")\nprint(f\" Total subtypes in this ICD10CM version: {len(descendants)}\")\nprint(f\" (FY2026 live dataset has 87 billable codes)\")\nprint()\n\n# Show the first 20 for readability — the full list is much longer\nprint(\"First 20 (sorted by code):\")\nfor desc in sorted(descendants)[:20]:\n if desc in icd10cm:\n name = icd10cm.lookup(desc)\n print(f\" [{desc}] {name}\")" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- Practical use: find all CKD-related diabetes codes ---\nckd_diabetes = [\n desc for desc in icd10cm.get_descendants(\"E11\")\n if desc in icd10cm and \"kidney\" in icd10cm.lookup(desc).lower()\n]\n\nprint(\"Type 2 DM codes involving kidney disease:\")\nfor code in ckd_diabetes:\n print(f\" [{code}] {icd10cm.lookup(code)}\")" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- Practical use: find all hyperglycemia-related diabetes codes ---\nhyperglycemia_codes = [\n desc for desc in descendants\n if desc in icd10cm and \"hyperglycemia\" in icd10cm.lookup(desc).lower()\n]\n\nprint(\"Hyperglycemia-related diabetes codes:\")\nfor code in hyperglycemia_codes:\n print(f\" [{code}] {icd10cm.lookup(code)}\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Part 4: Cross-System Code Translation with `CrossMap`\n\n`CrossMap` translates codes from one vocabulary to another. This is essential when:\n- Combining MIMIC-III (ICD-9) with MIMIC-IV (ICD-10) cohorts\n- Reducing ICD-10's 70,000 codes to CCS's ~300 categories for modeling\n- Mapping to clinical groupings used in published benchmarks\n\n```python\nCrossMap.load(source_vocabulary, target_vocabulary)\ncm.map(source_code) -> List[str] # returns a list (may be 1-to-many)\n```" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- ICD-9-CM → ICD-10-CM ---\n# MIMIC-III uses ICD-9; MIMIC-IV uses ICD-10.\n# The GEM (General Equivalence Mappings) crosswalk handles this translation.\n\ncm_9to10 = CrossMap.load(\"ICD9CM\", \"ICD10CM\")\n\n# ICD-9 diabetes codes (legacy MIMIC-III style)\nicd9_diabetes = {\n \"250.00\": \"Diabetes mellitus without mention of complication, type II\",\n \"250.10\": \"Diabetes with ketoacidosis, type II\",\n \"250.40\": \"Diabetes with renal manifestations, type II\",\n \"250.60\": \"Diabetes with neurological manifestations, type II\",\n}\n\nprint(\"ICD-9-CM → ICD-10-CM Diabetes Code Mapping:\")\nprint()\nfor icd9_code, icd9_name in icd9_diabetes.items():\n icd10_codes = cm_9to10.map(icd9_code)\n print(f\" ICD-9 {icd9_code} — {icd9_name}\")\n print(f\" → ICD-10: {icd10_codes}\")\n for c in icd10_codes:\n if c in icd10cm:\n print(f\" [{c}] {icd10cm.lookup(c)}\")\n print()" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- ICD-10-CM → CCS (Clinical Classifications Software) ---\n# CCS groups ~70,000 ICD-10 codes into ~300 clinically meaningful categories.\n# CCS category 49 = \"Diabetes mellitus without complication\"\n# CCS category 50 = \"Diabetes mellitus with complications\"\n\ncm_10toCCS = CrossMap.load(\"ICD10CM\", \"CCSCM\")\n\nprint(\"ICD-10-CM → CCS Category Mapping:\")\nprint()\ntype2_sample = [\"E11.9\", \"E11.65\", \"E11.22\", \"E11.42\", \"E10.9\"]\nfor code in type2_sample:\n ccs_cats = cm_10toCCS.map(code)\n icd_name = icd10cm.lookup(code) if code in icd10cm else \"(not found)\"\n print(f\" [{code}] {icd_name}\")\n print(f\" → CCS: {ccs_cats}\")\n print()" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- ICD-9-CM → CCS (direct, for MIMIC-III) ---\ncm_9toCCS = CrossMap.load(\"ICD9CM\", \"CCSCM\")\n\nprint(\"ICD-9-CM → CCS (useful for MIMIC-III preprocessing):\")\nprint()\nfor icd9_code in [\"250.00\", \"250.10\", \"250.40\", \"250.60\"]:\n ccs_cats = cm_9toCCS.map(icd9_code)\n print(f\" {icd9_code} → CCS: {ccs_cats}\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Part 5: Loading Other Code Systems\n\nPyHealth supports many vocabularies beyond ICD-10-CM." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- ICD-9-CM for legacy MIMIC-III data ---\nicd9cm = InnerMap.load(\"ICD9CM\")\nicd9cm.stat()\nprint(\"Example lookup:\", icd9cm.lookup(\"250.00\"))\nprint()" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- ATC (Anatomical Therapeutic Chemical) for drug classification ---\n# ATC hierarchy: Level1 (organ system) → Level2 (main group) → Level3 → Level4 → Level5 (substance)\n# A10BA02 = Metformin\natc = InnerMap.load(\"ATC\")\natc.stat()\n\nmetformin_code = \"A10BA02\"\nif metformin_code in atc:\n print(f\"ATC code {metformin_code}: {atc.lookup(metformin_code)}\")\n ancestors_atc = atc.get_ancestors(metformin_code)\n print(\"Ancestors (drug class hierarchy):\")\n for anc in ancestors_atc:\n if anc in atc:\n print(f\" ↑ [{anc}] {atc.lookup(anc)}\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Part 6: Practical ML Preprocessing Patterns\n\nHere are the most common patterns for using `medcode` when preparing datasets for ML." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# Pattern 1: Filter codes to a clinical domain (e.g., all diabetes codes)\n# Useful when restricting a cohort to patients with a specific condition.\n\nall_diabetes_codes = set(icd10cm.get_descendants(\"E08\")) | \\\n set(icd10cm.get_descendants(\"E09\")) | \\\n set(icd10cm.get_descendants(\"E10\")) | \\\n set(icd10cm.get_descendants(\"E11\")) | \\\n set(icd10cm.get_descendants(\"E13\"))\nall_diabetes_codes |= {\"E08\", \"E09\", \"E10\", \"E11\", \"E13\"} # include root codes\n\nprint(f\"Total ICD-10-CM diabetes codes: {len(all_diabetes_codes)}\")\nprint(\"Sample:\", sorted(list(all_diabetes_codes))[:10])" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# Pattern 2: Translate a patient's ICD-9 code list to CCS before modeling\n# This dramatically reduces vocabulary size.\n\ndef translate_codes_to_ccs(icd9_codes, crossmap):\n \"\"\"Map a list of ICD-9 codes to CCS categories.\"\"\"\n ccs_codes = []\n for code in icd9_codes:\n mapped = crossmap.map(code)\n ccs_codes.extend(mapped)\n return list(set(ccs_codes)) # deduplicate\n\n# Example patient from MIMIC-III\npatient_icd9_codes = [\"250.00\", \"401.9\", \"428.0\", \"585.3\"]\n# = Type 2 DM, Essential hypertension, Heart failure, CKD stage 3\n\npatient_ccs_codes = translate_codes_to_ccs(patient_icd9_codes, cm_9toCCS)\nprint(\"ICD-9 codes:\", patient_icd9_codes)\nprint(\"CCS codes: \", patient_ccs_codes)\nprint(f\"Vocabulary reduction: {len(icd9cm.graph.nodes)} ICD-9 codes → ~300 CCS categories\")" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# Pattern 3: Code validation — remove invalid/unknown codes before training\n# Real EHR data often contains typos, deprecated codes, and free-text entries.\n\nraw_codes_from_ehr = [\"E11.9\", \"E11.99\", \"DIAB\", \"E11.65\", \"999.999\", \"E10.9\"]\nvalid_codes = [c for c in raw_codes_from_ehr if c in icd10cm]\ninvalid_codes = [c for c in raw_codes_from_ehr if c not in icd10cm]\n\nprint(\"Raw codes from EHR:\", raw_codes_from_ehr)\nprint(\"Valid ICD-10-CM codes:\", valid_codes)\nprint(\"Invalid / unknown codes:\", invalid_codes)" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Summary\n\n| Task | API |\n|------|-----|\n| Load code system | `icd10cm = InnerMap.load(\"ICD10CM\")` |\n| Look up a code | `icd10cm.lookup(\"E11.9\")` |\n| Check if code exists | `\"E11.9\" in icd10cm` |\n| Print system stats | `icd10cm.stat()` |\n| See available attributes | `icd10cm.available_attributes` |\n| Get parent codes | `icd10cm.get_ancestors(\"E11.22\")` |\n| Get child codes | `icd10cm.get_descendants(\"E11\")` |\n| Translate between systems | `cm = CrossMap.load(\"ICD9CM\", \"ICD10CM\"); cm.map(\"250.00\")` |\n| Reduce to CCS groups | `cm = CrossMap.load(\"ICD10CM\", \"CCSCM\"); cm.map(\"E11.9\")` |\n\n### Supported vocabularies\n\n| Vocabulary name | Description |\n|-----------------|-------------|\n| `ICD9CM` | ICD-9-CM diagnosis codes (MIMIC-III) |\n| `ICD10CM` | ICD-10-CM diagnosis codes (MIMIC-IV, current US standard) |\n| `ICD9PROC` | ICD-9 procedure codes |\n| `ICD10PROC` | ICD-10-PCS procedure codes |\n| `ATC` | Anatomical Therapeutic Chemical drug classification |\n| `NDC` | National Drug Code (US drug packaging) |\n| `RxNorm` | RxNorm drug concepts |\n| `CCSCM` | CCS diagnosis categories |\n| `CCSPROC` | CCS procedure categories |\n| `UMLS` | Unified Medical Language System concepts |" + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.9.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/examples/tutorials/tutorial_pyhealth_metrics.ipynb b/examples/tutorials/tutorial_pyhealth_metrics.ipynb new file mode 100644 index 000000000..bbb5ba657 --- /dev/null +++ b/examples/tutorials/tutorial_pyhealth_metrics.ipynb @@ -0,0 +1,172 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": "# PyHealth Metrics Tutorial\n\nThis notebook covers **`pyhealth.metrics`** — a collection of evaluation functions for clinical prediction tasks.\n\nYou will learn:\n- **Binary classification** metrics: AUC-ROC, AUC-PR, F1, ECE, and more\n- **Multiclass classification** metrics: accuracy, macro/micro F1\n- **Multilabel classification** metrics: hamming loss, sample-level AUC\n- **Fairness metrics**: disparate impact and statistical parity difference\n- How to call `trainer.inference()` to get raw predictions for custom evaluation\n\n> **Design note:** All metric functions in PyHealth accept raw numpy arrays, so they can be used independently of the Trainer or with any ML framework.\n\n---" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "import numpy as np\nfrom pyhealth.metrics import (\n binary_metrics_fn,\n multiclass_metrics_fn,\n multilabel_metrics_fn,\n)\nfrom pyhealth.metrics.fairness import fairness_metrics_fn\n\n# Set random seed for reproducibility\nnp.random.seed(42)" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Part 1: Binary Classification Metrics\n\nBinary classification is the most common setup in clinical prediction:\n- In-hospital mortality (alive / deceased)\n- Readmission within 30 days (yes / no)\n- Disease onset (positive / negative)\n\n```python\nbinary_metrics_fn(\n y_true: np.ndarray, # shape (n_samples,), values in {0, 1}\n y_prob: np.ndarray, # shape (n_samples,), values in [0, 1]\n metrics: Optional[List[str]] = None, # default: [\"pr_auc\", \"roc_auc\", \"f1\"]\n threshold: float = 0.5, # decision boundary for accuracy/F1/etc.\n)\n```\n\n### Supported metrics\n\n| Metric | Description |\n|--------|-------------|\n| `roc_auc` | Area under ROC curve — threshold-free measure of discrimination |\n| `pr_auc` | Area under Precision-Recall curve — better for imbalanced datasets |\n| `f1` | Harmonic mean of precision and recall |\n| `accuracy` | Fraction of correct predictions |\n| `balanced_accuracy` | Accuracy adjusted for class imbalance |\n| `precision` | TP / (TP + FP) |\n| `recall` | TP / (TP + FN) |\n| `cohen_kappa` | Agreement beyond chance |\n| `jaccard` | Intersection over union for positive class |\n| `ECE` | Expected Calibration Error (calibration quality) |\n| `ECE_adapt` | Adaptive ECE (equal-mass bins) |" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- Simulate a binary classification scenario: mortality prediction ---\n# 300 patients, ~20% mortality rate (realistic for ICU cohorts)\nn = 300\ny_true_binary = np.random.binomial(1, 0.20, size=n).astype(np.float32)\n\n# Simulate model probabilities: a reasonably well-calibrated model\n# True positives get higher probabilities on average\ny_prob_binary = np.where(\n y_true_binary == 1,\n np.random.beta(5, 2, size=n), # higher probs for true positives\n np.random.beta(2, 5, size=n), # lower probs for true negatives\n).astype(np.float32)\n\nprint(f\"Samples: {n}\")\nprint(f\"Positive rate: {y_true_binary.mean():.1%}\")\nprint(f\"Mean predicted prob (positives): {y_prob_binary[y_true_binary == 1].mean():.3f}\")\nprint(f\"Mean predicted prob (negatives): {y_prob_binary[y_true_binary == 0].mean():.3f}\")" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- Default metrics ---\nresults_default = binary_metrics_fn(y_true_binary, y_prob_binary)\nprint(\"Default binary metrics:\")\nfor metric, value in results_default.items():\n print(f\" {metric:15s}: {value:.4f}\")" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- Full metric suite ---\nresults_full = binary_metrics_fn(\n y_true_binary,\n y_prob_binary,\n metrics=[\n \"roc_auc\",\n \"pr_auc\",\n \"f1\",\n \"accuracy\",\n \"balanced_accuracy\",\n \"precision\",\n \"recall\",\n \"cohen_kappa\",\n \"jaccard\",\n \"ECE\",\n \"ECE_adapt\",\n ],\n threshold=0.5,\n)\n\nprint(\"Full binary metric suite (threshold=0.5):\")\nfor metric, value in results_full.items():\n print(f\" {metric:20s}: {value:.4f}\")" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- Impact of threshold on threshold-dependent metrics ---\nprint(\"Effect of decision threshold on F1 and precision/recall:\")\nprint(f\" {'threshold':>10} {'precision':>10} {'recall':>10} {'f1':>10}\")\n\nfor threshold in [0.2, 0.3, 0.4, 0.5, 0.6, 0.7]:\n m = binary_metrics_fn(\n y_true_binary, y_prob_binary,\n metrics=[\"precision\", \"recall\", \"f1\"],\n threshold=threshold,\n )\n print(f\" {threshold:>10.1f} {m['precision']:>10.4f} {m['recall']:>10.4f} {m['f1']:>10.4f}\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "### When to use PR-AUC vs ROC-AUC\n\n- **ROC-AUC** is appropriate when the classes are roughly balanced. It measures discrimination across all thresholds.\n- **PR-AUC** is better for **imbalanced** datasets (like in-hospital mortality, where death is rare). It focuses on performance on the positive class and is not inflated by the large number of true negatives.\n\nIn clinical tasks with positive rates < 20%, always report PR-AUC alongside ROC-AUC." + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Part 2: Multiclass Classification Metrics\n\nMulticlass is used when predicting among 3+ mutually exclusive outcomes, for example:\n- Primary discharge diagnosis category (e.g., CCS groups)\n- Length-of-stay bucket (short / medium / long)\n- Triage acuity level (1–5)\n\n```python\nmulticlass_metrics_fn(\n y_true: np.ndarray, # shape (n_samples,), integer class indices\n y_prob: np.ndarray, # shape (n_samples, num_classes), sum to 1\n metrics: Optional[List[str]] = None, # default: [\"accuracy\", \"f1_macro\", \"f1_micro\"]\n)\n```\n\n### Macro vs Micro averaging\n\n| Averaging | Computes | Best for |\n|-----------|----------|----------|\n| `macro` | Mean of per-class metric | When all classes are equally important |\n| `micro` | Global TP/FP/FN counts | When overall performance matters more than per-class balance |\n| `weighted` | Weighted by class support | When you want to account for class frequency |" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- Simulate multiclass: 4 diagnostic categories ---\nn_classes = 4\nn_mc = 400\n# Ground truth: somewhat imbalanced (class 0 is most common)\nclass_probs = [0.45, 0.25, 0.20, 0.10]\ny_true_mc = np.random.choice(n_classes, size=n_mc, p=class_probs).astype(np.int64)\n\n# Model outputs: softmax probabilities\n# Simulate a decent model by perturbing a one-hot representation\ny_prob_mc = np.zeros((n_mc, n_classes))\nfor i, label in enumerate(y_true_mc):\n probs = np.random.dirichlet([0.5] * n_classes)\n probs[label] += 1.5 # boost the true class\n probs /= probs.sum()\n y_prob_mc[i] = probs\ny_prob_mc = y_prob_mc.astype(np.float32)\n\nprint(f\"Samples: {n_mc}, Classes: {n_classes}\")\nprint(f\"Class distribution: {np.bincount(y_true_mc)}\")" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- Default multiclass metrics ---\nresults_mc = multiclass_metrics_fn(y_true_mc, y_prob_mc)\nprint(\"Default multiclass metrics:\")\nfor metric, value in results_mc.items():\n print(f\" {metric:20s}: {value:.4f}\")" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- Extended multiclass metrics ---\nresults_mc_full = multiclass_metrics_fn(\n y_true_mc,\n y_prob_mc,\n metrics=[\n \"accuracy\",\n \"f1_macro\",\n \"f1_micro\",\n \"f1_weighted\",\n \"precision_macro\",\n \"recall_macro\",\n \"roc_auc_macro_ovr\", # one-vs-rest ROC-AUC\n \"ECE\",\n ],\n)\n\nprint(\"Extended multiclass metrics:\")\nfor metric, value in results_mc_full.items():\n print(f\" {metric:25s}: {value:.4f}\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Part 3: Multilabel Classification Metrics\n\nMultilabel is used when each sample can have **multiple simultaneous labels**, for example:\n- Drug recommendation: prescribe multiple drugs simultaneously\n- Comorbidity prediction: patient can have several conditions\n- Procedure recommendation: multiple procedures per visit\n\n```python\nmultilabel_metrics_fn(\n y_true: np.ndarray, # shape (n_samples, n_labels), binary\n y_prob: np.ndarray, # shape (n_samples, n_labels), in [0, 1]\n metrics: Optional[List[str]] = None, # default: [\"pr_auc_samples\"]\n threshold: float = 0.3, # note: lower default than binary!\n)\n```\n\nNote the **threshold=0.3** default (vs 0.5 for binary). In drug recommendation, it is better to recommend slightly more drugs than to miss necessary ones." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- Simulate multilabel: drug recommendation scenario ---\n# 200 patients, 50 possible drugs, each patient prescribed 3-8 drugs on average\nn_ml = 200\nn_labels = 50\n\n# True prescriptions: sparse binary matrix\ny_true_ml = (np.random.rand(n_ml, n_labels) < 0.12).astype(np.float32) # ~12% = ~6 drugs/patient\n\n# Model probabilities\ny_prob_ml = np.zeros((n_ml, n_labels), dtype=np.float32)\nfor i in range(n_ml):\n # True drugs get higher predicted probability\n y_prob_ml[i] = np.where(\n y_true_ml[i] == 1,\n np.random.beta(4, 2, size=n_labels),\n np.random.beta(1, 6, size=n_labels),\n )\n\nprint(f\"Samples: {n_ml}, Labels: {n_labels}\")\nprint(f\"Mean labels per patient: {y_true_ml.sum(axis=1).mean():.1f}\")" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- Default multilabel metrics ---\nresults_ml = multilabel_metrics_fn(y_true_ml, y_prob_ml)\nprint(\"Default multilabel metrics (threshold=0.3):\")\nfor metric, value in results_ml.items():\n print(f\" {metric:25s}: {value:.4f}\")" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- Extended multilabel metrics ---\nresults_ml_full = multilabel_metrics_fn(\n y_true_ml,\n y_prob_ml,\n metrics=[\n \"pr_auc_samples\", # average PR-AUC per sample (most common)\n \"pr_auc_macro\", # macro-averaged PR-AUC across labels\n \"roc_auc_macro\", # macro-averaged ROC-AUC across labels\n \"f1_macro\",\n \"f1_micro\",\n \"f1_samples\", # per-sample F1, then averaged\n \"hamming_loss\", # fraction of label-sample pairs incorrectly classified\n \"accuracy\", # exact match: all labels must be correct\n ],\n threshold=0.3,\n)\n\nprint(\"Extended multilabel metrics:\")\nfor metric, value in results_ml_full.items():\n print(f\" {metric:25s}: {value:.4f}\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "### Interpreting multilabel metrics\n\n| Metric | Clinical interpretation |\n|--------|------------------------|\n| `pr_auc_samples` | How well the model ranks drugs for each patient — the primary metric for drug recommendation |\n| `hamming_loss` | Fraction of all (patient, drug) pairs incorrectly classified — penalizes false positives equally |\n| `accuracy` (exact match) | Very strict — 1 only if all drugs are exactly correct |\n| `f1_macro` | Per-drug average F1 — gives equal weight to rare and common drugs |" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Part 4: Fairness Metrics\n\nFairness metrics assess whether a model's performance is **equitable across subgroups** defined by sensitive attributes (e.g., race, sex, age group). This is crucial in clinical AI to avoid perpetuating historical health disparities.\n\n```python\nfairness_metrics_fn(\n y_true: np.ndarray, # (n_samples,) true labels\n y_prob: np.ndarray, # (n_samples,) predicted probabilities\n sensitive_attributes: np.ndarray, # (n_samples,) 1=protected group, 0=unprotected\n favorable_outcome: int = 1, # which label value is considered positive\n metrics: Optional[List[str]] = None, # default: both below\n threshold: float = 0.5,\n)\n```\n\n### Supported fairness metrics\n\n| Metric | Formula | Interpretation |\n|--------|---------|----------------|\n| `disparate_impact` | P(ŷ=1 | protected) / P(ŷ=1 | unprotected) | Should be ≥ 0.8 (80% rule). 1.0 = perfect parity |\n| `statistical_parity_difference` | P(ŷ=1 | protected) − P(ŷ=1 | unprotected) | Should be close to 0. Negative = protected group predicted positive less often |" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- Simulate a biased binary classifier ---\n# Scenario: predicting ICU readmission\n# Protected group (attr=1): elderly patients (age >= 65)\n# Unprotected group (attr=0): younger patients (age < 65)\n\nn_fair = 400\nnp.random.seed(99)\n\n# 40% of patients are elderly\nsensitive = np.random.binomial(1, 0.40, size=n_fair) # 1 = elderly\n\n# True outcomes — elderly have slightly higher readmission rate\ny_true_fair = np.where(\n sensitive == 1,\n np.random.binomial(1, 0.35, size=n_fair), # elderly: 35% readmission\n np.random.binomial(1, 0.25, size=n_fair), # younger: 25% readmission\n).astype(np.float32)\n\n# Biased model: under-predicts readmission for elderly\n# (e.g., trained on historical data that under-tested elderly patients)\ny_prob_fair = np.where(\n sensitive == 1,\n np.random.beta(2, 5, size=n_fair), # lower probs for elderly (biased)\n np.random.beta(3, 4, size=n_fair), # higher probs for younger\n).astype(np.float32)\n\nprint(f\"Patients: {n_fair}\")\nprint(f\"Protected (elderly) group: {sensitive.sum()} ({sensitive.mean():.1%})\")\nprint(f\"True readmission rate — elderly: {y_true_fair[sensitive==1].mean():.1%}\")\nprint(f\"True readmission rate — younger: {y_true_fair[sensitive==0].mean():.1%}\")\nprint(f\"Mean predicted prob — elderly: {y_prob_fair[sensitive==1].mean():.3f}\")\nprint(f\"Mean predicted prob — younger: {y_prob_fair[sensitive==0].mean():.3f}\")" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- Compute fairness metrics ---\nfairness_results = fairness_metrics_fn(\n y_true=y_true_fair,\n y_prob=y_prob_fair,\n sensitive_attributes=sensitive,\n favorable_outcome=1, # readmission = 1 is the \"positive\" outcome\n metrics=[\"disparate_impact\", \"statistical_parity_difference\"],\n threshold=0.5,\n)\n\nprint(\"Fairness metrics:\")\nfor metric, value in fairness_results.items():\n print(f\" {metric:35s}: {value:.4f}\")" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- Interpret the results ---\ndi = fairness_results[\"disparate_impact\"]\nspd = fairness_results[\"statistical_parity_difference\"]\n\nprint(\"Interpretation:\")\nprint()\nprint(f\" Disparate Impact = {di:.4f}\")\nif di >= 0.8:\n print(\" ✓ Above 0.8 threshold — model passes the 80% rule\")\nelse:\n print(\" ✗ Below 0.8 threshold — model fails the 80% rule (legally significant in US)\")\nprint()\nprint(f\" Statistical Parity Difference = {spd:.4f}\")\nif abs(spd) < 0.05:\n print(\" ✓ Close to 0 — model predictions are roughly equally distributed across groups\")\nelif spd < 0:\n print(f\" ✗ Negative ({spd:.4f}): protected group is predicted positive {abs(spd):.1%} less often\")\nelse:\n print(f\" Protected group is predicted positive {spd:.1%} more often\")" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- Compare against a fairer model (calibrated equally for both groups) ---\ny_prob_fair2 = np.where(\n y_true_fair == 1,\n np.random.beta(4, 2, size=n_fair),\n np.random.beta(2, 4, size=n_fair),\n).astype(np.float32)\n\nfairness_results2 = fairness_metrics_fn(\n y_true=y_true_fair,\n y_prob=y_prob_fair2,\n sensitive_attributes=sensitive,\n threshold=0.5,\n)\n\nprint(\"Fairer model fairness metrics:\")\nfor metric, value in fairness_results2.items():\n print(f\" {metric:35s}: {value:.4f}\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "### Clinical context for fairness metrics\n\nIn the healthcare domain, fairness is particularly important because:\n\n1. **Historical bias in training data:** If a hospital historically provided less aggressive treatment to a subgroup, the training labels may reflect these disparities — and the model will learn to replicate them.\n\n2. **Feature proxies:** Features like ZIP code, insurance type, or language can serve as proxies for race/ethnicity. A model may be technically race-unaware yet still exhibit disparate impact.\n\n3. **Regulatory considerations:** The US 2021 Algorithmic Accountability Act and emerging EU AI Act both require documentation of bias audits for high-risk AI (which includes clinical decision support).\n\n**PyHealth's approach:** Report fairness metrics alongside clinical performance metrics. A model with excellent AUC-ROC but poor disparate impact is not ready for deployment." + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Part 5: Integration with Trainer — Getting Raw Predictions\n\nIn practice you'll get predictions from a trained model and then compute any of the above metrics. The Trainer provides two ways:\n\n**Option A — `trainer.evaluate(loader)`:** Returns a dict of metrics using the model's default metric function. Convenient, but limited to the default metrics.\n\n**Option B — `trainer.inference(loader)`:** Returns raw numpy arrays `(y_true, y_prob, loss)`. Use this when you want to compute custom metrics, fairness analysis, or calibration plots.\n\n```python\n# After training (see tutorial_pyhealth_trainer.ipynb)\ny_true, y_prob, loss = trainer.inference(test_loader)\n\n# Now compute any metric combination you want\nbinary_metrics_fn(y_true, y_prob, metrics=[\"roc_auc\", \"pr_auc\", \"ECE\"])\nfairness_metrics_fn(y_true, y_prob, sensitive_attributes=race_labels)\n```\n\nThe decoupling of **inference** from **metric computation** means you can:\n- Cache the raw predictions and recompute metrics without re-running the model\n- Apply post-hoc calibration (Platt scaling, temperature scaling) and re-evaluate\n- Run bootstrap confidence intervals on any metric" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Summary\n\n| Task type | Function | Default metrics |\n|-----------|----------|-----------------|\n| Binary | `binary_metrics_fn(y_true, y_prob)` | pr_auc, roc_auc, f1 |\n| Multiclass | `multiclass_metrics_fn(y_true, y_prob)` | accuracy, f1_macro, f1_micro |\n| Multilabel | `multilabel_metrics_fn(y_true, y_prob)` | pr_auc_samples |\n| Fairness | `fairness_metrics_fn(y_true, y_prob, sensitive_attributes)` | disparate_impact, statistical_parity_difference |\n\n### Choosing the right metric for clinical tasks\n\n| Clinical Task | Recommended Primary Metric | Why |\n|---------------|---------------------------|-----|\n| Mortality prediction | `pr_auc` | Rare events (imbalanced); PR-AUC better than ROC-AUC |\n| Readmission (30d) | `roc_auc` | Moderate prevalence; standard in literature |\n| Drug recommendation | `pr_auc_samples` | Multi-label; need sample-level ranking quality |\n| Disease severity (3+ levels) | `f1_macro` | Multi-class; equal weighting across severity levels |\n| Any model in clinical use | + fairness metrics | Always audit for disparate impact |" + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.9.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/tutorials/tutorial_pyhealth_model.ipynb b/examples/tutorials/tutorial_pyhealth_model.ipynb new file mode 100644 index 000000000..71a5affd2 --- /dev/null +++ b/examples/tutorials/tutorial_pyhealth_model.ipynb @@ -0,0 +1,148 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": "# PyHealth Models Tutorial — RNN Deep Dive\n\nThis notebook walks through **`pyhealth.models`** from first principles.\n\nYou will learn:\n- The **`BaseModel`** contract every PyHealth model must satisfy\n- How to create synthetic test data with **`create_sample_dataset()`**\n- The internal architecture of **`RNNLayer`** and **`RNN`** — reading the source code line by line\n- How to run forward and backward passes\n- How **`MultimodalRNN`** handles mixed input types\n\n---" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "import torch\nimport torch.nn as nn\nfrom pyhealth.datasets import create_sample_dataset, get_dataloader\nfrom pyhealth.models import RNN, BaseModel\nfrom pyhealth.models.rnn import RNNLayer, MultimodalRNN" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Part 1: The `BaseModel` Contract\n\nEvery PyHealth model inherits from `BaseModel`, which itself inherits from both `ABC` (abstract base class) and `nn.Module` (PyTorch module).\n\n```python\nclass BaseModel(ABC, nn.Module):\n def __init__(self, dataset: SampleDataset):\n ...\n self.feature_keys = list(dataset.input_schema.keys())\n self.label_keys = list(dataset.output_schema.keys())\n\n def forward(self, **kwargs) -> dict[str, torch.Tensor]:\n # Subclasses implement this\n raise NotImplementedError\n```\n\n### What `BaseModel` provides\n\n| Method | Description |\n|--------|-------------|\n| `device` property | Returns the device the model lives on (CPU / CUDA) |\n| `get_output_size()` | Returns the FC head size (1 for binary, num_classes for multiclass) |\n| `get_loss_function()` | Returns the appropriate loss: BCE for binary/multilabel, CrossEntropy for multiclass |\n| `prepare_y_prob(logits)` | Applies sigmoid (binary/multilabel) or softmax (multiclass) to produce probabilities |\n\n### The `forward()` output contract\n\nEvery `forward(**kwargs)` call returns a dict:\n```python\n{\n \"loss\": torch.Tensor, # scalar — backpropagatable\n \"y_prob\": torch.Tensor, # predicted probabilities\n \"y_true\": torch.Tensor, # ground truth labels\n \"logit\": torch.Tensor, # raw logits before activation\n \"embed\": torch.Tensor, # (optional) patient embeddings, only if embed=True in kwargs\n}\n```\n\nThis uniform interface means any model can plug into the `Trainer` without modification." + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Part 2: Creating Test Data with `create_sample_dataset()`\n\n`create_sample_dataset()` is a convenience helper that:\n1. Accepts a list of raw sample dicts\n2. Fits tokenizers / processors based on the provided schema\n3. Returns an `InMemorySampleDataset` (no disk I/O) ready for model instantiation\n\nThis is exactly how PyHealth's own unit tests create datasets — no MIMIC or real EHR data required." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- Define raw samples ---\n# Each sample is a dict. Keys must match the schemas below.\nsamples = [\n {\n \"patient_id\": \"patient-0\",\n \"visit_id\": \"visit-0\",\n \"conditions\": [\"E11.9\", \"E11.65\", \"I10\"], # Type 2 DM, hypertension\n \"procedures\": [\"99213\", \"36415\"], # Office visit, blood draw\n \"label\": 1,\n },\n {\n \"patient_id\": \"patient-1\",\n \"visit_id\": \"visit-1\",\n \"conditions\": [\"E11.9\"],\n \"procedures\": [\"99213\"],\n \"label\": 0,\n },\n {\n \"patient_id\": \"patient-2\",\n \"visit_id\": \"visit-2\",\n \"conditions\": [\"E11.22\", \"N18.3\", \"E11.42\"], # DM with CKD and neuropathy\n \"procedures\": [\"99213\", \"86900\", \"81001\"],\n \"label\": 1,\n },\n {\n \"patient_id\": \"patient-3\",\n \"visit_id\": \"visit-3\",\n \"conditions\": [\"E11.65\", \"E11.9\"],\n \"procedures\": [\"36415\"],\n \"label\": 0,\n },\n]\n\n# --- Define schemas ---\n# Processor aliases:\n# 'sequence' → SequenceProcessor (tokenizes a list of codes to integer IDs)\n# 'multi_hot' → MultiHotProcessor (binary vector over vocabulary)\n# 'timeseries' → TimeseriesProcessor (continuous time series)\n# 'tensor' → TensorProcessor (fixed-size dense vector)\n# 'binary' → BinaryLabelProcessor (0 or 1)\ninput_schema = {\"conditions\": \"sequence\", \"procedures\": \"sequence\"}\noutput_schema = {\"label\": \"binary\"}\n\n# --- Create the dataset ---\ndataset = create_sample_dataset(\n samples=samples,\n input_schema=input_schema,\n output_schema=output_schema,\n dataset_name=\"diabetes_demo\",\n task_name=\"mortality\",\n)\n\nprint(\"Dataset type: \", type(dataset).__name__)\nprint(\"Input schema: \", dataset.input_schema)\nprint(\"Output schema: \", dataset.output_schema)\nprint(\"Num samples: \", len(dataset))" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# Inspect the fitted processors\nfor key, proc in dataset.input_processors.items():\n print(f\" {key}: {type(proc).__name__}, vocab size = {proc.size()}\")\nprint(\" label:\", type(dataset.output_processors['label']).__name__)" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Part 3: `RNNLayer` Architecture\n\n`RNNLayer` is a low-level building block that wraps PyTorch's native RNN/LSTM/GRU with:\n- **Dropout** before the recurrent computation\n- **Variable-length sequence support** via pack/pad operations\n- **Bidirectional support** with a down-projection to maintain hidden_size\n\n```python\nclass RNNLayer(nn.Module):\n\n def __init__(\n self,\n input_size: int,\n hidden_size: int,\n rnn_type: str = \"GRU\", # one of \"RNN\", \"LSTM\", \"GRU\"\n num_layers: int = 1,\n dropout: float = 0.5,\n bidirectional: bool = False,\n ):\n ...\n self.dropout_layer = nn.Dropout(dropout)\n rnn_module = getattr(nn, rnn_type) # nn.GRU, nn.LSTM, or nn.RNN\n self.rnn = rnn_module(\n input_size, hidden_size,\n num_layers=num_layers,\n dropout=dropout if num_layers > 1 else 0,\n bidirectional=bidirectional,\n batch_first=True,\n )\n if bidirectional:\n self.down_projection = nn.Linear(hidden_size * 2, hidden_size)\n\n def forward(\n self,\n x: torch.Tensor, # shape: (batch, seq_len, input_size)\n mask: Optional[torch.Tensor] = None, # shape: (batch, seq_len), 1=valid\n ) -> Tuple[torch.Tensor, torch.Tensor]:\n x = self.dropout_layer(x)\n\n # Compute actual sequence lengths from mask (or assume full sequences)\n lengths = torch.sum(mask.int(), dim=-1).cpu() if mask is not None else ...\n lengths = torch.clamp(lengths, min=1) # avoid zero-length sequences\n\n # Pack → RNN → Unpack (cuDNN optimization for variable-length batches)\n x = rnn_utils.pack_padded_sequence(x, lengths, batch_first=True, enforce_sorted=False)\n outputs, _ = self.rnn(x)\n outputs, _ = rnn_utils.pad_packed_sequence(outputs, batch_first=True)\n\n # Extract final hidden state at each sample's actual last position\n last_outputs = outputs[torch.arange(batch_size), (lengths - 1), :]\n\n if self.bidirectional:\n # Concatenate forward/backward final states, then project back to hidden_size\n last_outputs = self.down_projection(last_outputs)\n\n return outputs, last_outputs\n # outputs: (batch, seq_len, hidden_size) — all time steps\n # last_outputs: (batch, hidden_size) — final hidden state\n```\n\n### Key design decisions in `RNNLayer`\n\n1. **`pack_padded_sequence`** — tells cuDNN to skip padding positions, which is both faster and numerically correct. Without this, the RNN would process padding tokens and corrupt the final hidden state.\n\n2. **`lengths = clamp(lengths, min=1)`** — `pack_padded_sequence` raises an error for length-0 sequences (empty visits). Clamping to 1 is a safe fallback.\n\n3. **Bidirectional down-projection** — bidirectional outputs have `2 × hidden_size` channels; the linear layer projects back to `hidden_size` so downstream code sees a consistent dimension regardless of directionality." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- Demonstrate RNNLayer standalone ---\nbatch_size = 4\nseq_len = 10\ninput_size = 32\nhidden_size = 64\n\nlayer = RNNLayer(input_size=input_size, hidden_size=hidden_size, rnn_type=\"GRU\")\n\nx = torch.randn(batch_size, seq_len, input_size)\n\n# Mask: batch items have different actual lengths (simulating variable-length visits)\nmask = torch.zeros(batch_size, seq_len)\nmask[0, :8] = 1 # 8 valid tokens\nmask[1, :5] = 1 # 5 valid tokens\nmask[2, :10] = 1 # all 10 valid\nmask[3, :3] = 1 # 3 valid tokens\nmask = mask.int()\n\noutputs, last_outputs = layer(x, mask)\n\nprint(\"outputs shape (all time steps):\", outputs.shape)\nprint(\"last_outputs shape (final hidden):\", last_outputs.shape)" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# Bidirectional RNN: output shape is the same thanks to down_projection\nbidi_layer = RNNLayer(input_size=input_size, hidden_size=hidden_size, bidirectional=True)\noutputs_b, last_b = bidi_layer(x, mask)\nprint(\"Bidirectional outputs shape:\", outputs_b.shape) # still hidden_size after projection\nprint(\"Bidirectional last shape: \", last_b.shape)" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Part 4: `RNN` Model Architecture\n\nThe `RNN` class sits one level above `RNNLayer`. It applies **separate** embedding and RNN layers for each input feature, then concatenates the final hidden states and passes them through a shared fully-connected head.\n\n```python\nclass RNN(BaseModel):\n\n def __init__(\n self,\n dataset: SampleDataset,\n embedding_dim: int = 128,\n hidden_dim: int = 128,\n **kwargs # forwarded to RNNLayer (rnn_type, num_layers, dropout, ...)\n ):\n super().__init__(dataset=dataset)\n\n # One embedding model shared across all features\n self.embedding_model = EmbeddingModel(dataset, embedding_dim)\n\n # One independent RNN layer per feature key\n self.rnn = nn.ModuleDict()\n for feature_key in self.feature_keys:\n self.rnn[feature_key] = RNNLayer(\n input_size=embedding_dim, hidden_size=hidden_dim, **kwargs\n )\n\n # Final FC: concatenation of all hidden states → output\n output_size = self.get_output_size() # 1 for binary\n self.fc = nn.Linear(len(self.feature_keys) * hidden_dim, output_size)\n```\n\n### `RNN.forward()` step by step\n\n```python\n def forward(self, **kwargs):\n patient_emb = []\n\n # 1. Extract value tensors and masks from each feature\n for feature_key in self.feature_keys:\n # Feature is a tuple: (value_tensor, mask_tensor, ...)\n # Schema tells us which tuple index is 'value' and which is 'mask'\n inputs[feature_key] = value\n masks[feature_key] = mask\n\n # 2. Embed all features (tokenized codes → dense vectors)\n embedded = self.embedding_model(inputs, masks=masks)\n # embedded[key] shape:\n # SequenceProcessor → (B, seq_len, D)\n # NestedSequenceProcessor → (B, num_visits, num_codes, D)\n # TimeseriesProcessor → (B, T, D)\n\n # 3. Handle dimensionality:\n for feature_key in self.feature_keys:\n x = embedded[feature_key]\n if x.dim() == 4: # nested: (B, V, C, D) → sum-pool codes → (B, V, D)\n x = x.sum(dim=2)\n elif x.dim() == 2: # static single value: (B, D) → (B, 1, D)\n x = x.unsqueeze(1)\n # Now x is always (B, T, D)\n\n # 4. Run per-feature RNN, take final hidden state\n _, x = self.rnn[feature_key](x, mask)\n patient_emb.append(x) # each x: (B, hidden_dim)\n\n # 5. Concatenate all features' hidden states\n patient_emb = torch.cat(patient_emb, dim=1) # (B, num_features * hidden_dim)\n\n # 6. Project to label space\n logits = self.fc(patient_emb)\n\n # 7. Compute loss, probabilities\n y_true = kwargs[self.label_key]\n loss = self.get_loss_function()(logits, y_true)\n y_prob = self.prepare_y_prob(logits)\n\n return {\"loss\": loss, \"y_prob\": y_prob, \"y_true\": y_true, \"logit\": logits}\n```\n\n**Why separate RNNs per feature?** Different clinical features have different sequence semantics. Diagnosis codes have a different distributional structure than procedure codes. Separate RNNs let each feature develop its own specialized temporal representation before combining." + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Part 5: Instantiating and Running `RNN`" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- Instantiate the RNN model ---\nmodel = RNN(\n dataset=dataset,\n embedding_dim=64,\n hidden_dim=64,\n # RNNLayer kwargs:\n rnn_type=\"GRU\", # try \"LSTM\" or \"RNN\" too\n num_layers=1,\n dropout=0.3,\n bidirectional=False,\n)\n\nprint(model)\nprint()\nprint(\"Feature keys:\", model.feature_keys)\nprint(\"Label key: \", model.label_key)\nprint(\"Mode: \", model.mode) # 'binary'\nprint(\"Output size: \", model.get_output_size()) # 1\nprint(\"Device: \", model.device)" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- Create DataLoader and get a batch ---\ntrain_loader = get_dataloader(dataset, batch_size=2, shuffle=False)\ndata_batch = next(iter(train_loader))\n\nprint(\"Batch keys:\", list(data_batch.keys()))\nfor k, v in data_batch.items():\n if isinstance(v, torch.Tensor):\n print(f\" {k}: tensor shape {v.shape}\")\n elif isinstance(v, (tuple, list)):\n print(f\" {k}: tuple/list of {len(v)} items\")\n else:\n print(f\" {k}: {v}\")" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- Forward pass (inference mode) ---\nmodel.eval()\nwith torch.no_grad():\n output = model(**data_batch)\n\nprint(\"Forward pass output:\")\nprint(f\" loss: {output['loss'].item():.4f}\")\nprint(f\" y_prob: {output['y_prob'].squeeze().tolist()}\") # probabilities in [0, 1]\nprint(f\" y_true: {output['y_true'].tolist()}\")\nprint(f\" logit: {output['logit'].squeeze().tolist()}\")" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- Requesting patient embeddings ---\n# Pass embed=True to get the concatenated hidden state before the FC layer\nwith torch.no_grad():\n output_with_embed = model(**data_batch, embed=True)\n\nprint(\"With embed=True:\")\nprint(f\" embed shape: {output_with_embed['embed'].shape}\")\nprint(f\" Expected: (batch_size=2, num_features={len(model.feature_keys)} × hidden_dim={model.hidden_dim} = {len(model.feature_keys) * model.hidden_dim})\")\nprint()\nprint(\"These embeddings can be used for:\")\nprint(\" - Patient similarity search\")\nprint(\" - Visualization with UMAP / t-SNE\")\nprint(\" - Downstream tasks (transfer learning)\")" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- Backward pass (training mode) ---\nmodel.train()\n\n# Re-fetch batch (gradients cleared)\ndata_batch = next(iter(train_loader))\n\noptimizer = torch.optim.Adam(model.parameters(), lr=1e-3)\noptimizer.zero_grad()\n\noutput = model(**data_batch)\noutput[\"loss\"].backward() # compute gradients\noptimizer.step() # update weights\n\nprint(f\"Training loss: {output['loss'].item():.4f}\")\nprint(\"Backward pass completed. Weights updated.\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Part 6: `MultimodalRNN` — Mixed Input Modalities\n\n`MultimodalRNN` extends `RNN` to handle **heterogeneous inputs**. It automatically classifies each feature into:\n\n- **Sequential** (gets its own `RNNLayer`): `SequenceProcessor`, `NestedSequenceProcessor`, `TimeseriesProcessor`, ...\n- **Non-sequential** (embeddings only, no RNN): `MultiHotProcessor`, `TensorProcessor`\n\nThe architecture:\n```\nconditions (seq) → Embed → RNNLayer → hidden_cond ┐\nvitals (tensor) → Linear → embed_vitals ├→ Concat → FC → logit\nrace (multi_hot) → Linear → embed_race ┘\n```\n\nThe final FC input size is `(num_sequential × hidden_dim) + (num_non_sequential × embedding_dim)`." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- Build a multimodal dataset ---\nmultimodal_samples = [\n {\n \"patient_id\": \"p0\",\n \"visit_id\": \"v0\",\n \"conditions\": [\"E11.9\", \"I10\"], # sequential codes\n \"demographics\": [\"female\", \"age_65_79\"], # multi-hot categorical\n \"vitals\": [120.0, 80.0, 37.0], # fixed-size dense vector: SBP, DBP, temp\n \"label\": 1,\n },\n {\n \"patient_id\": \"p1\",\n \"visit_id\": \"v1\",\n \"conditions\": [\"E11.65\"],\n \"demographics\": [\"male\", \"age_50_64\"],\n \"vitals\": [135.0, 88.0, 36.8],\n \"label\": 0,\n },\n {\n \"patient_id\": \"p2\",\n \"visit_id\": \"v2\",\n \"conditions\": [\"E11.22\", \"N18.3\"],\n \"demographics\": [\"female\", \"age_80plus\"],\n \"vitals\": [110.0, 72.0, 37.2],\n \"label\": 1,\n },\n {\n \"patient_id\": \"p3\",\n \"visit_id\": \"v3\",\n \"conditions\": [\"E11.9\"],\n \"demographics\": [\"male\", \"age_65_79\"],\n \"vitals\": [125.0, 85.0, 36.9],\n \"label\": 0,\n },\n]\n\nmm_dataset = create_sample_dataset(\n samples=multimodal_samples,\n input_schema={\n \"conditions\": \"sequence\", # will get RNNLayer\n \"demographics\": \"multi_hot\", # embedding only\n \"vitals\": \"tensor\", # embedding only\n },\n output_schema={\"label\": \"binary\"},\n dataset_name=\"multimodal_demo\",\n)\n\nprint(\"Input processors:\")\nfor key, proc in mm_dataset.input_processors.items():\n print(f\" {key}: {type(proc).__name__}\")" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- Instantiate MultimodalRNN ---\nmm_model = MultimodalRNN(\n dataset=mm_dataset,\n embedding_dim=32,\n hidden_dim=32,\n rnn_type=\"GRU\",\n)\n\nprint(\"Sequential features (have RNNLayer): \", mm_model.sequential_features)\nprint(\"Non-sequential features (embed only):\", mm_model.non_sequential_features)\nprint()\n\n# FC input dimension explained:\nseq_dim = len(mm_model.sequential_features) * mm_model.hidden_dim\nnon_seq_dim = len(mm_model.non_sequential_features) * mm_model.embedding_dim\nprint(f\"FC input: {seq_dim} (seq) + {non_seq_dim} (non-seq) = {seq_dim + non_seq_dim}\")\nprint(f\"FC out: {mm_model.get_output_size()} (binary)\")" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# --- Run a forward pass ---\nmm_loader = get_dataloader(mm_dataset, batch_size=4, shuffle=False)\nbatch = next(iter(mm_loader))\n\nmm_model.eval()\nwith torch.no_grad():\n mm_out = mm_model(**batch)\n\nprint(\"MultimodalRNN output:\")\nprint(f\" loss: {mm_out['loss'].item():.4f}\")\nprint(f\" y_prob: {mm_out['y_prob'].squeeze().tolist()}\")\nprint(f\" y_true: {mm_out['y_true'].tolist()}\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Summary\n\n| Concept | Key API |\n|---------|----------|\n| Create synthetic dataset | `create_sample_dataset(samples, input_schema, output_schema)` |\n| Batch iteration | `get_dataloader(dataset, batch_size=32, shuffle=True)` |\n| Instantiate RNN | `RNN(dataset, embedding_dim=128, hidden_dim=128, rnn_type=\"GRU\")` |\n| Forward pass | `model(**batch)` → `{loss, y_prob, y_true, logit}` |\n| Request embeddings | `model(**batch, embed=True)` → adds `embed` key |\n| Backward pass | `output[\"loss\"].backward()` |\n| Mixed modalities | `MultimodalRNN(dataset, embedding_dim=128, hidden_dim=128)` |\n\n### Choosing hyperparameters\n\n| Hyperparameter | Guidance |\n|----------------|----------|\n| `rnn_type` | GRU is a good default; LSTM has more parameters but can model longer dependencies |\n| `embedding_dim` | 64–256 depending on vocabulary size |\n| `hidden_dim` | Usually equal to `embedding_dim`; increase for more complex patterns |\n| `num_layers` | 1–2; deeper RNNs need dropout > 0 between layers |\n| `dropout` | 0.3–0.5; reduces overfitting on small datasets |\n| `bidirectional` | Only meaningful when the full sequence is available at inference time |" + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.9.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/tutorials/tutorial_pyhealth_trainer.ipynb b/examples/tutorials/tutorial_pyhealth_trainer.ipynb new file mode 100644 index 000000000..c5ee70f56 --- /dev/null +++ b/examples/tutorials/tutorial_pyhealth_trainer.ipynb @@ -0,0 +1,174 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": "# PyHealth Trainer Tutorial — End-to-End Training with MIMIC-III\n\nThis notebook covers **`pyhealth.trainer`** — the training loop that ties datasets, models, and metrics together.\n\nYou will learn:\n- How to load a public **synthetic MIMIC-III** dataset (no credentials required)\n- How to apply a **mortality prediction task** to generate model-ready samples\n- How to split, batch, and feed data to an **RNN model**\n- How to use **`Trainer`** for training with validation, early stopping, and checkpointing\n- How to **evaluate** a trained model on a held-out test set\n\n> **Dataset Note:** The dataset used here is a fully synthetic MIMIC-III replica hosted by Google Cloud Storage. No PhysioNet account or data use agreement is needed.\n\n---" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "import torch\nfrom pyhealth.datasets import MIMIC3Dataset, get_dataloader, split_by_patient\nfrom pyhealth.models import RNN\nfrom pyhealth.tasks import MortalityPredictionMIMIC3\nfrom pyhealth.trainer import Trainer" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Step 1: Load the Synthetic MIMIC-III Dataset\n\n`MIMIC3Dataset` loads structured EHR tables from a root directory (local path or URL). The Google Cloud Storage path below hosts a synthetic copy — the schema is identical to real MIMIC-III but no real patient data is present.\n\nDefault tables always loaded: `[\"patients\", \"admissions\", \"icustays\"]`\nAdditional tables you can specify:\n- `\"diagnoses_icd\"` — ICD-9 diagnosis codes per admission\n- `\"procedures_icd\"` — ICD-9 procedure codes per admission\n- `\"prescriptions\"` — Medication orders (NDC codes)\n- `\"labevents\"` — Lab measurements\n- `\"noteevents\"` — Clinical notes (discharge summaries, radiology reports, etc.)" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "root = \"https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III\"\n\ndataset = MIMIC3Dataset(\n root=root,\n dataset_name=\"mimic3\",\n tables=[\n \"diagnoses_icd\", # needed for mortality task (conditions)\n \"procedures_icd\", # needed for mortality task (procedures)\n \"prescriptions\", # needed for mortality task (drugs)\n \"noteevents\", # clinical notes (not used by basic task, but loaded for reference)\n ],\n)\n\nprint(\"Dataset loaded.\")\nprint(f\" Number of patients: {len(dataset.patients)}\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Step 2: Apply the Mortality Prediction Task\n\nA **task** is a callable that transforms raw `Patient` objects into model-ready sample dicts. `MortalityPredictionMIMIC3` predicts whether a patient will die in the hospital during a subsequent admission.\n\n**Task schema:**\n```python\ninput_schema = {\"conditions\": \"sequence\", \"procedures\": \"sequence\", \"drugs\": \"sequence\"}\noutput_schema = {\"mortality\": \"binary\"}\n```\n\n**Label definition:** `mortality = 1` if `hospital_expire_flag == 1` in the NEXT admission, else 0.\n\n**Filtering:** Samples with no conditions, no procedures, OR no drugs are excluded.\n\n`dataset.set_task(task)` iterates over all patients, calls the task on each, and returns a `SampleDataset` with fitted processors." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "task = MortalityPredictionMIMIC3()\nsamples = dataset.set_task(task)\n\nprint(\"Task applied.\")\nprint(f\" Task name: {samples.task_name}\")\nprint(f\" Total samples: {len(samples)}\")\nprint(f\" Input schema: {samples.input_schema}\")\nprint(f\" Output schema: {samples.output_schema}\")" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# Inspect the fitted processors (tokenizers / label encoders)\nprint(\"Input processors:\")\nfor key, proc in samples.input_processors.items():\n print(f\" {key}: {type(proc).__name__}, vocab size = {proc.size()}\")\n\nprint(\"\\nOutput processors:\")\nfor key, proc in samples.output_processors.items():\n print(f\" {key}: {type(proc).__name__}, size = {proc.size()}\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Step 3: Split the Dataset\n\n`split_by_patient` partitions samples so that **no patient appears in more than one split** — this is the correct way to split clinical data. Splitting by sample (the naive approach) would allow data leakage: a model could see one visit from patient X in training and another visit from the same patient in test.\n\nReturns three `SampleDataset` objects sharing the same fitted processors." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "train_ds, val_ds, test_ds = split_by_patient(\n dataset=samples,\n ratios=[0.7, 0.15, 0.15],\n seed=42,\n)\n\nprint(f\"Train: {len(train_ds)} samples\")\nprint(f\"Validation: {len(val_ds)} samples\")\nprint(f\"Test: {len(test_ds)} samples\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Step 4: Create DataLoaders\n\n`get_dataloader` wraps a `SampleDataset` in a `DataLoader` with PyHealth's custom `collate_fn_dict`, which handles variable-length sequences by padding and producing mask tensors." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "train_loader = get_dataloader(train_ds, batch_size=32, shuffle=True)\nval_loader = get_dataloader(val_ds, batch_size=32, shuffle=False)\ntest_loader = get_dataloader(test_ds, batch_size=32, shuffle=False)\n\nprint(f\"Train batches: {len(train_loader)}\")\nprint(f\"Val batches: {len(val_loader)}\")\nprint(f\"Test batches: {len(test_loader)}\")\n\n# Inspect a batch\nbatch = next(iter(train_loader))\nprint(\"\\nBatch keys:\", list(batch.keys()))" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Step 5: Build the Model\n\nThe `RNN` model takes the `SampleDataset` as its first argument. It reads:\n- `dataset.input_schema` to know which features exist and their types\n- `dataset.input_processors` to know vocabulary sizes (for embedding tables)\n- `dataset.output_schema` to determine the number of output classes and the loss function\n\nYou never need to specify vocabulary sizes or output sizes manually." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "model = RNN(\n dataset=samples, # pass the full SampleDataset (with fitted processors)\n embedding_dim=64,\n hidden_dim=64,\n rnn_type=\"GRU\",\n num_layers=1,\n dropout=0.3,\n)\n\nprint(model)\nprint()\nprint(f\"Feature keys: {model.feature_keys}\")\nprint(f\"Label key: {model.label_key}\")\nprint(f\"Mode: {model.mode}\")\nprint(f\"Output size: {model.get_output_size()}\")\nprint(f\"Loss function: {model.get_loss_function()}\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Step 6: Setup the Trainer\n\n```python\nTrainer(\n model,\n checkpoint_path = None, # load from checkpoint if provided\n metrics = None, # default metrics per mode (e.g., pr_auc, roc_auc, f1)\n device = None, # auto-detects CUDA; falls back to CPU\n enable_logging = True, # writes log.txt to output_path/exp_name/\n output_path = \"./output\", # directory for logs and checkpoints\n exp_name = None, # defaults to current datetime string\n)\n```\n\n`Trainer` automatically moves the model to the detected device." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "trainer = Trainer(\n model=model,\n output_path=\"./output\",\n exp_name=\"mortality_rnn_demo\",\n enable_logging=True,\n)\n\nprint(f\"Training on device: {trainer.device}\")\nprint(f\"Logs written to: {trainer.exp_path}\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Step 7: Train the Model\n\n```python\ntrainer.train(\n train_dataloader,\n val_dataloader = None,\n test_dataloader = None,\n epochs = 5,\n optimizer_class = torch.optim.Adam,\n optimizer_params = {\"lr\": 1e-3},\n weight_decay = 0.0,\n max_grad_norm = None, # gradient clipping (e.g., 1.0)\n monitor = None, # metric name to track for best model\n monitor_criterion = \"max\", # \"max\" or \"min\"\n load_best_model_at_last = True, # restore best weights after training\n patience = None, # early stopping patience in epochs\n)\n```\n\n**Monitoring:** When `monitor` is set, Trainer saves the checkpoint with the best value of that metric on the validation set, and optionally restores it at the end of training.\n\n**Early stopping:** Set `patience=N` to stop training if the monitored metric does not improve for N consecutive evaluation epochs." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "trainer.train(\n train_dataloader=train_loader,\n val_dataloader=val_loader,\n test_dataloader=test_loader,\n epochs=5,\n optimizer_class=torch.optim.Adam,\n optimizer_params={\"lr\": 1e-3},\n weight_decay=1e-4,\n monitor=\"roc_auc\", # track ROC-AUC on validation set\n monitor_criterion=\"max\", # higher ROC-AUC = better\n load_best_model_at_last=True,\n patience=3, # stop if no improvement for 3 epochs\n)" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Step 8: Evaluate on the Test Set\n\n`trainer.evaluate(dataloader)` runs inference on the given dataloader and returns a dict of metric scores using the model's default metric function for its mode.\n\nFor binary classification the defaults are: `pr_auc`, `roc_auc`, `f1`." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "test_metrics = trainer.evaluate(test_loader)\n\nprint(\"Test set results:\")\nfor metric, value in test_metrics.items():\n print(f\" {metric:15s}: {value:.4f}\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Step 9: Checkpoint Saving and Loading\n\nPyHealth's Trainer saves checkpoints as standard PyTorch `.pt` files containing the `model.state_dict()`." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "import os\n\n# Save manually\nckpt_path = \"./output/mortality_rnn_demo/manual_checkpoint.pt\"\nos.makedirs(os.path.dirname(ckpt_path), exist_ok=True)\ntrainer.save_ckpt(ckpt_path)\nprint(f\"Checkpoint saved to: {ckpt_path}\")" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# Load a checkpoint into an existing Trainer / model\n# This is useful for resuming training or loading a pretrained model for inference\nnew_model = RNN(dataset=samples, embedding_dim=64, hidden_dim=64)\n\nnew_trainer = Trainer(\n model=new_model,\n checkpoint_path=ckpt_path, # loads weights immediately on init\n enable_logging=False,\n)\n\n# Verify the loaded model produces the same test metrics\nloaded_metrics = new_trainer.evaluate(test_loader)\nprint(\"Metrics from loaded checkpoint:\")\nfor metric, value in loaded_metrics.items():\n print(f\" {metric:15s}: {value:.4f}\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Step 10: Raw Inference with `trainer.inference()`\n\nIf you need the raw predictions (not just aggregated metrics), use `trainer.inference()`. This is useful for computing custom metrics, error analysis, or feeding into a calibration step." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "y_true_all, y_prob_all, loss_mean = trainer.inference(test_loader)\n\nprint(f\"y_true shape: {y_true_all.shape}\")\nprint(f\"y_prob shape: {y_prob_all.shape}\")\nprint(f\"Mean test loss: {loss_mean:.4f}\")\nprint()\nprint(\"First 10 predictions:\")\nfor i in range(min(10, len(y_true_all))):\n print(f\" true={int(y_true_all[i])} pred_prob={y_prob_all[i].item():.3f}\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Summary\n\n### Full pipeline at a glance\n\n```python\n# 1. Load EHR data\ndataset = MIMIC3Dataset(root=root, tables=[\"diagnoses_icd\", \"procedures_icd\", \"prescriptions\"])\n\n# 2. Define prediction task\nsamples = dataset.set_task(MortalityPredictionMIMIC3())\n\n# 3. Split by patient (prevents leakage)\ntrain_ds, val_ds, test_ds = split_by_patient(samples, ratios=[0.7, 0.15, 0.15])\n\n# 4. Create DataLoaders\ntrain_loader = get_dataloader(train_ds, batch_size=32, shuffle=True)\n\n# 5. Build model (reads schema and processors from dataset)\nmodel = RNN(dataset=samples, embedding_dim=64, hidden_dim=64)\n\n# 6. Train\ntrainer = Trainer(model, output_path=\"./output\", exp_name=\"my_run\")\ntrainer.train(train_loader, val_loader, monitor=\"roc_auc\", epochs=20, patience=5)\n\n# 7. Evaluate\ntrainer.evaluate(test_loader)\n```\n\n### Key Trainer parameters\n\n| Parameter | Effect |\n|-----------|--------|\n| `monitor` | Metric name to track for best model checkpoint |\n| `monitor_criterion` | `\"max\"` for metrics like AUC/F1; `\"min\"` for loss |\n| `patience` | Early stopping: stop after N epochs without improvement |\n| `load_best_model_at_last` | Restore best checkpoint weights after training ends |\n| `weight_decay` | L2 regularization (passed to optimizer) |\n| `max_grad_norm` | Gradient clipping (useful for LSTM stability) |" + }, + { + "cell_type": "markdown", + "id": "b8313bc0", + "source": "---\n## API Reference: Available Metric Strings\n\nThe `metrics` argument to `Trainer.__init__` and the `monitor` argument to `trainer.train()` are plain strings drawn from a fixed list. The exact list depends on **`model.mode`**, which is set automatically from the task's output schema:\n\n```python\nprint(model.mode) # → \"binary\" | \"multiclass\" | \"multilabel\" | \"regression\"\n```\n\n`Trainer` uses `model.mode` to select the right metrics function, then passes your `metrics` list to it. Any string you pass to `monitor` must appear in that same list — otherwise evaluation will raise a `KeyError`.\n\nTo compute a non-default set of metrics and track a specific one:\n```python\ntrainer = Trainer(\n model=model,\n metrics=[\"roc_auc\", \"pr_auc\", \"balanced_accuracy\", \"ECE\"], # computed every eval epoch\n)\ntrainer.train(..., monitor=\"pr_auc\", monitor_criterion=\"max\")\n```\n\n---\n\n### Binary classification — `mode = \"binary\"`\n**Source:** `pyhealth.metrics.binary_metrics_fn` \n**Defaults when `metrics=None`:** `[\"pr_auc\", \"roc_auc\", \"f1\"]`\n\n| Metric string | Description | `monitor_criterion` |\n|---|---|---|\n| `\"pr_auc\"` | Area under the Precision-Recall curve | `\"max\"` |\n| `\"roc_auc\"` | Area under the ROC curve | `\"max\"` |\n| `\"f1\"` | F1 score at `threshold` (default 0.5) | `\"max\"` |\n| `\"accuracy\"` | Fraction of correct predictions | `\"max\"` |\n| `\"balanced_accuracy\"` | Accuracy adjusted for class imbalance | `\"max\"` |\n| `\"precision\"` | Precision at `threshold` | `\"max\"` |\n| `\"recall\"` | Recall at `threshold` | `\"max\"` |\n| `\"cohen_kappa\"` | Cohen's kappa (agreement beyond chance) | `\"max\"` |\n| `\"jaccard\"` | Jaccard similarity coefficient | `\"max\"` |\n| `\"ECE\"` | Expected Calibration Error (20 equal-width bins) | `\"min\"` |\n| `\"ECE_adapt\"` | Adaptive ECE (20 equal-size bins) | `\"min\"` |\n\n---\n\n### Multiclass classification — `mode = \"multiclass\"`\n**Source:** `pyhealth.metrics.multiclass_metrics_fn` \n**Defaults when `metrics=None`:** `[\"accuracy\", \"f1_macro\", \"f1_micro\"]`\n\n| Metric string | Description | `monitor_criterion` |\n|---|---|---|\n| `\"accuracy\"` | Overall accuracy | `\"max\"` |\n| `\"balanced_accuracy\"` | Accuracy adjusted for class imbalance | `\"max\"` |\n| `\"f1_macro\"` | F1, macro-averaged across classes | `\"max\"` |\n| `\"f1_micro\"` | F1, micro-averaged across classes | `\"max\"` |\n| `\"f1_weighted\"` | F1, weighted by class support | `\"max\"` |\n| `\"roc_auc_macro_ovo\"` | ROC-AUC, macro, one-vs-one | `\"max\"` |\n| `\"roc_auc_macro_ovr\"` | ROC-AUC, macro, one-vs-rest | `\"max\"` |\n| `\"roc_auc_weighted_ovo\"` | ROC-AUC, weighted, one-vs-one | `\"max\"` |\n| `\"roc_auc_weighted_ovr\"` | ROC-AUC, weighted, one-vs-rest | `\"max\"` |\n| `\"jaccard_micro\"` | Jaccard, micro-averaged | `\"max\"` |\n| `\"jaccard_macro\"` | Jaccard, macro-averaged | `\"max\"` |\n| `\"jaccard_weighted\"` | Jaccard, weighted | `\"max\"` |\n| `\"cohen_kappa\"` | Cohen's kappa | `\"max\"` |\n| `\"brier_top1\"` | Brier score for the top predicted class | `\"min\"` |\n| `\"ECE\"` | Expected Calibration Error (20 equal-width bins) | `\"min\"` |\n| `\"ECE_adapt\"` | Adaptive ECE (20 equal-size bins) | `\"min\"` |\n| `\"cwECEt\"` | Classwise ECE with threshold = min(0.01, 1/K) | `\"min\"` |\n| `\"cwECEt_adapt\"` | Classwise adaptive ECE | `\"min\"` |\n| `\"hits@n\"` | HITS@1 / HITS@5 / HITS@10 (produces 3 dict keys) | `\"max\"` |\n| `\"mean_rank\"` | Mean rank + mean reciprocal rank | `\"min\"` |\n\n---\n\n### Multilabel classification — `mode = \"multilabel\"`\n**Source:** `pyhealth.metrics.multilabel_metrics_fn` \n**Defaults when `metrics=None`:** `[\"pr_auc_samples\"]` \n**Note:** threshold defaults to `0.3` (not `0.5`) — lower thresholds are common in drug recommendation tasks.\n\n| Metric string | Description | `monitor_criterion` |\n|---|---|---|\n| `\"pr_auc_samples\"` | PR-AUC, averaged across samples | `\"max\"` |\n| `\"pr_auc_micro\"` | PR-AUC, micro-averaged | `\"max\"` |\n| `\"pr_auc_macro\"` | PR-AUC, macro-averaged | `\"max\"` |\n| `\"pr_auc_weighted\"` | PR-AUC, weighted | `\"max\"` |\n| `\"roc_auc_samples\"` | ROC-AUC, samples-averaged | `\"max\"` |\n| `\"roc_auc_micro\"` | ROC-AUC, micro-averaged | `\"max\"` |\n| `\"roc_auc_macro\"` | ROC-AUC, macro-averaged | `\"max\"` |\n| `\"roc_auc_weighted\"` | ROC-AUC, weighted | `\"max\"` |\n| `\"f1_samples\"` | F1, samples-averaged | `\"max\"` |\n| `\"f1_micro\"` | F1, micro-averaged | `\"max\"` |\n| `\"f1_macro\"` | F1, macro-averaged | `\"max\"` |\n| `\"f1_weighted\"` | F1, weighted | `\"max\"` |\n| `\"precision_micro\"` / `\"_macro\"` / `\"_weighted\"` / `\"_samples\"` | Precision variants | `\"max\"` |\n| `\"recall_micro\"` / `\"_macro\"` / `\"_weighted\"` / `\"_samples\"` | Recall variants | `\"max\"` |\n| `\"jaccard_micro\"` / `\"_macro\"` / `\"_weighted\"` / `\"_samples\"` | Jaccard variants | `\"max\"` |\n| `\"accuracy\"` | Element-wise accuracy | `\"max\"` |\n| `\"hamming_loss\"` | Hamming loss | `\"min\"` |\n| `\"ddi\"` | Drug-drug interaction rate (drug recommendation only) | `\"min\"` |\n| `\"cwECE\"` | Classwise ECE (20 equal-width bins) | `\"min\"` |\n| `\"cwECE_adapt\"` | Classwise adaptive ECE | `\"min\"` |\n\n---\n\n### Regression — `mode = \"regression\"`\n**Source:** `pyhealth.metrics.regression_metrics_fn` \n**Defaults when `metrics=None`:** `[\"kl_divergence\", \"mse\", \"mae\"]`\n\n| Metric string | Description | `monitor_criterion` |\n|---|---|---|\n| `\"mae\"` | Mean Absolute Error | `\"min\"` |\n| `\"mse\"` | Mean Squared Error | `\"min\"` |\n| `\"kl_divergence\"` | KL divergence between true and reconstructed distributions | `\"min\"` |", + "metadata": {} + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.9.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file From acaf7b742521a0d3b97083eb22840e1a86527092 Mon Sep 17 00:00:00 2001 From: Jane Du Date: Tue, 12 May 2026 17:22:52 -0500 Subject: [PATCH 11/61] Colab Tutorial re-link (#1146) * add back backups of original tutorials * Backup lost tutorials * generate new tokenizer tutorial * update with pip install d4rl install pyhealth and rename * update colab references --- README.rst | 16 +- chat-assistant/corpus/pyhealth-code.txt | 16 +- chat-assistant/corpus/pyhealth-text.txt | 24 +- chat-assistant/corpus/pyhealth.txt | 40 +- docs/_static/external_links.js | 9 + docs/api/data.rst | 2 +- docs/api/datasets.rst | 2 +- docs/api/tasks.rst | 2 +- docs/conf.py | 1 + docs/tutorials.rst | 16 +- .../tutorial_stagenet_comprehensive.ipynb | 4 +- .../orig_tutorial_pyhealth_datasets.ipynb | 1613 ++++++++ .../orig_tutorial_pyhealth_tasks.ipynb | 3559 +++++++++++++++++ .../tutorials/tutorial_pyhealth_data.ipynb | 45 +- .../tutorials/tutorial_pyhealth_medcode.ipynb | 35 +- .../tutorials/tutorial_pyhealth_metrics.ipynb | 27 +- .../tutorials/tutorial_pyhealth_model.ipynb | 27 +- .../tutorial_pyhealth_tokenizer.ipynb | 210 + .../tutorials/tutorial_pyhealth_trainer.ipynb | 15 +- 19 files changed, 5545 insertions(+), 118 deletions(-) create mode 100644 docs/_static/external_links.js create mode 100644 examples/tutorials/orig_tutorial_pyhealth_datasets.ipynb create mode 100644 examples/tutorials/orig_tutorial_pyhealth_tasks.ipynb create mode 100644 examples/tutorials/tutorial_pyhealth_tokenizer.ipynb diff --git a/README.rst b/README.rst index c3f3e6355..3940f774d 100644 --- a/README.rst +++ b/README.rst @@ -354,23 +354,23 @@ Module 5: We provide the following tutorials to help users get started with our pyhealth. Please bear with us as we update the documentation on how to use PyHealth 2.0. -`Tutorial 0: Introduction to pyhealth.data `_ `[Video] `__ +`Tutorial 0: Introduction to pyhealth.data `_ `[Video] `__ -`Tutorial 1: Introduction to pyhealth.datasets `_ `[Video (PyHealth 1.16)] `__ +`Tutorial 1: Introduction to pyhealth.datasets `_ `[Video (PyHealth 1.16)] `__ -`Tutorial 2: Introduction to pyhealth.tasks `_ `[Video (PyHealth 1.16)] `__ +`Tutorial 2: Introduction to pyhealth.tasks `_ `[Video (PyHealth 1.16)] `__ -`Tutorial 3: Introduction to pyhealth.models `_ `[Video] `__ +`Tutorial 3: Introduction to pyhealth.models `_ `[Video] `__ -`Tutorial 4: Introduction to pyhealth.trainer `_ `[Video] `__ +`Tutorial 4: Introduction to pyhealth.trainer `_ `[Video] `__ -`Tutorial 5: Introduction to pyhealth.metrics `_ `[Video] `__ +`Tutorial 5: Introduction to pyhealth.metrics `_ `[Video] `__ -`Tutorial 6: Introduction to pyhealth.tokenizer `_ `[Video] `__ +`Tutorial 6: Introduction to pyhealth.tokenizer `_ `[Video] `__ -`Tutorial 7: Introduction to pyhealth.medcode `_ `[Video] `__ +`Tutorial 7: Introduction to pyhealth.medcode `_ `[Video] `__ The following tutorials will help users build their own task pipelines. diff --git a/chat-assistant/corpus/pyhealth-code.txt b/chat-assistant/corpus/pyhealth-code.txt index 5af6889c0..0ecef087d 100644 --- a/chat-assistant/corpus/pyhealth-code.txt +++ b/chat-assistant/corpus/pyhealth-code.txt @@ -26786,7 +26786,7 @@ Here is the code content for tutorial_5_pyhealth_metrics.py: Automatically generated by Colaboratory. Original file is located at - https://colab.research.google.com/drive/1Mrs77EJ92HwMgDaElJ_CBXbi4iABZBeo + https://colab.research.google.com/drive/1bO0h5BR62_kQ7zFOgzQmt5vb8jqJ0rV-?usp=drive_link ### **Preparation** - install pyhealth alpha version @@ -27611,7 +27611,7 @@ Here is the code content for tutorial_0_pyhealth_data.py: Automatically generated by Colaboratory. Original file is located at - https://colab.research.google.com/drive/1y9PawgSbyMbSSMw1dpfwtooH7qzOEYdN + https://colab.research.google.com/drive/17nOzjIjKiAbC8bsntZ3h9xy2Vq4bKpuv """ !pip install pyhealth @@ -28478,7 +28478,7 @@ test_loader = get_dataloader(test_ds, batch_size=64, shuffle=False) """### **Step 2: Select a ML model** - In this tutorial, we use Transformer as the example. -- please check the [Tutorial 2](https://colab.research.google.com/drive/1LcXZlu7ZUuqepf269X3FhXuhHeRvaJX5?usp=sharing) for more instructions on how to initialize a model. +- please check the [Tutorial 2](https://colab.research.google.com/drive/1cUTSfFL1wLUXDBtJGTAWntolvcmxrDGo?usp=drive_link) for more instructions on how to initialize a model. """ from pyhealth.models import Transformer @@ -28532,7 +28532,7 @@ Here is the code content for tutorial_6_pyhealth_tokenizer.py: Automatically generated by Colaboratory. Original file is located at - https://colab.research.google.com/drive/1bDOb0A5g0umBjtz8NIp4wqye7taJ03D0 + https://colab.research.google.com/drive/1jhJ11MLUafhflQAz8HSrWiOEYlIhhvc_ ### **Preparation** - install pyhealth alpha version @@ -29093,7 +29093,7 @@ Here is the code content for tutorial_7_pyhealth_medcode.py: Automatically generated by Colaboratory. Original file is located at - https://colab.research.google.com/drive/1xrp_ACM2_Hg5Wxzj0SKKKgZfMY0WwEj3 + https://colab.research.google.com/drive/1Tw1AUS53fotH1EYr4Abp7qYN3zDBeUbC?usp=drive_link ### **Preparation** - install pyhealth alpha version @@ -29625,7 +29625,7 @@ Here is the code content for tutorial_3_pyhealth_models.py: Automatically generated by Colaboratory. Original file is located at - https://colab.research.google.com/drive/1LcXZlu7ZUuqepf269X3FhXuhHeRvaJX5 + https://colab.research.google.com/drive/1cUTSfFL1wLUXDBtJGTAWntolvcmxrDGo?usp=drive_link ### **Preparation** - install pyhealth alpha version @@ -30421,7 +30421,7 @@ Here is the code content for tutorial_4_pyhealth_trainer.py: Automatically generated by Colaboratory. Original file is located at - https://colab.research.google.com/drive/1L1Nz76cRNB7wTp5Pz_4Vp4N2eRZ9R6xl + https://colab.research.google.com/drive/1up_SL0BxxHPO9pmjKQ98w1GbpiB7LySp?usp=drive_link ### **Preparation** - install pyhealth alpha version @@ -30457,7 +30457,7 @@ To initialize a trainer instance, the following environments should be specified - `load_best_model_at_last`: whether to load the best model during the last iteration. ### **Step 1 & 2 & 3: Prepare datasets, task, and model** -- Example: We use **MIMIC-III dataset** and **RETAIN** model for **readmission prediction** task. Refer to [Tutorial 1](https://colab.research.google.com/drive/18kbzEQAj1FMs_J9rTGX8eCoxnWdx4Ltn?usp=sharing), [Tutorial 2](https://colab.research.google.com/drive/1r7MYQR_5yCJGpK_9I9-A10HmpupZuIN-?usp=sharing), and [Tutorial 3](https://colab.research.google.com/drive/1LcXZlu7ZUuqepf269X3FhXuhHeRvaJX5?usp=sharing). +- Example: We use **MIMIC-III dataset** and **RETAIN** model for **readmission prediction** task. Refer to [Tutorial 1](https://colab.research.google.com/drive/18kbzEQAj1FMs_J9rTGX8eCoxnWdx4Ltn?usp=sharing), [Tutorial 2](https://colab.research.google.com/drive/1r7MYQR_5yCJGpK_9I9-A10HmpupZuIN-?usp=sharing), and [Tutorial 3](https://colab.research.google.com/drive/1cUTSfFL1wLUXDBtJGTAWntolvcmxrDGo?usp=drive_link). """ # load dataset diff --git a/chat-assistant/corpus/pyhealth-text.txt b/chat-assistant/corpus/pyhealth-text.txt index 5d0d59992..705fed027 100644 --- a/chat-assistant/corpus/pyhealth-text.txt +++ b/chat-assistant/corpus/pyhealth-text.txt @@ -326,23 +326,23 @@ Module 5: We provide the following tutorials to help users get started with our pyhealth. -`Tutorial 0: Introduction to pyhealth.data `_ `[Video] `__ +`Tutorial 0: Introduction to pyhealth.data `_ `[Video] `__ `Tutorial 1: Introduction to pyhealth.datasets `_ `[Video] `__ `Tutorial 2: Introduction to pyhealth.tasks `_ `[Video] `__ -`Tutorial 3: Introduction to pyhealth.models `_ `[Video] `__ +`Tutorial 3: Introduction to pyhealth.models `_ `[Video] `__ -`Tutorial 4: Introduction to pyhealth.trainer `_ `[Video] `__ +`Tutorial 4: Introduction to pyhealth.trainer `_ `[Video] `__ -`Tutorial 5: Introduction to pyhealth.metrics `_ `[Video] `__ +`Tutorial 5: Introduction to pyhealth.metrics `_ `[Video] `__ -`Tutorial 6: Introduction to pyhealth.tokenizer `_ `[Video] `__ +`Tutorial 6: Introduction to pyhealth.tokenizer `_ `[Video] `__ -`Tutorial 7: Introduction to pyhealth.medcode `_ `[Video] `__ +`Tutorial 7: Introduction to pyhealth.medcode `_ `[Video] `__ The following tutorials will help users build their own task pipelines. @@ -1206,21 +1206,21 @@ Tutorials We provide the following tutorials to help users get started with our pyhealth. -`Tutorial 0: Introduction to pyhealth.data `_ `[Video] `_ +`Tutorial 0: Introduction to pyhealth.data `_ `[Video] `_ `Tutorial 1: Introduction to pyhealth.datasets `_ `[Video] `_ `Tutorial 2: Introduction to pyhealth.tasks `_ `[Video] `_ -`Tutorial 3: Introduction to pyhealth.models `_ `[Video] `_ +`Tutorial 3: Introduction to pyhealth.models `_ `[Video] `_ -`Tutorial 4: Introduction to pyhealth.trainer `_ `[Video] `_ +`Tutorial 4: Introduction to pyhealth.trainer `_ `[Video] `_ -`Tutorial 5: Introduction to pyhealth.metrics `_ `[Video] `_ +`Tutorial 5: Introduction to pyhealth.metrics `_ `[Video] `_ -`Tutorial 6: Introduction to pyhealth.tokenizer `_ `[Video] `_ +`Tutorial 6: Introduction to pyhealth.tokenizer `_ `[Video] `_ -`Tutorial 7: Introduction to pyhealth.medcode `_ `[Video] `_ +`Tutorial 7: Introduction to pyhealth.medcode `_ `[Video] `_ The following tutorials will help users build their own task pipelines. `[Video] `_ diff --git a/chat-assistant/corpus/pyhealth.txt b/chat-assistant/corpus/pyhealth.txt index 10cd5456e..309f791de 100644 --- a/chat-assistant/corpus/pyhealth.txt +++ b/chat-assistant/corpus/pyhealth.txt @@ -326,23 +326,23 @@ Module 5: We provide the following tutorials to help users get started with our pyhealth. -`Tutorial 0: Introduction to pyhealth.data `_ `[Video] `__ +`Tutorial 0: Introduction to pyhealth.data `_ `[Video] `__ `Tutorial 1: Introduction to pyhealth.datasets `_ `[Video] `__ `Tutorial 2: Introduction to pyhealth.tasks `_ `[Video] `__ -`Tutorial 3: Introduction to pyhealth.models `_ `[Video] `__ +`Tutorial 3: Introduction to pyhealth.models `_ `[Video] `__ -`Tutorial 4: Introduction to pyhealth.trainer `_ `[Video] `__ +`Tutorial 4: Introduction to pyhealth.trainer `_ `[Video] `__ -`Tutorial 5: Introduction to pyhealth.metrics `_ `[Video] `__ +`Tutorial 5: Introduction to pyhealth.metrics `_ `[Video] `__ -`Tutorial 6: Introduction to pyhealth.tokenizer `_ `[Video] `__ +`Tutorial 6: Introduction to pyhealth.tokenizer `_ `[Video] `__ -`Tutorial 7: Introduction to pyhealth.medcode `_ `[Video] `__ +`Tutorial 7: Introduction to pyhealth.medcode `_ `[Video] `__ The following tutorials will help users build their own task pipelines. @@ -1206,21 +1206,21 @@ Tutorials We provide the following tutorials to help users get started with our pyhealth. -`Tutorial 0: Introduction to pyhealth.data `_ `[Video] `_ +`Tutorial 0: Introduction to pyhealth.data `_ `[Video] `_ `Tutorial 1: Introduction to pyhealth.datasets `_ `[Video] `_ `Tutorial 2: Introduction to pyhealth.tasks `_ `[Video] `_ -`Tutorial 3: Introduction to pyhealth.models `_ `[Video] `_ +`Tutorial 3: Introduction to pyhealth.models `_ `[Video] `_ -`Tutorial 4: Introduction to pyhealth.trainer `_ `[Video] `_ +`Tutorial 4: Introduction to pyhealth.trainer `_ `[Video] `_ -`Tutorial 5: Introduction to pyhealth.metrics `_ `[Video] `_ +`Tutorial 5: Introduction to pyhealth.metrics `_ `[Video] `_ -`Tutorial 6: Introduction to pyhealth.tokenizer `_ `[Video] `_ +`Tutorial 6: Introduction to pyhealth.tokenizer `_ `[Video] `_ -`Tutorial 7: Introduction to pyhealth.medcode `_ `[Video] `_ +`Tutorial 7: Introduction to pyhealth.medcode `_ `[Video] `_ The following tutorials will help users build their own task pipelines. `[Video] `_ @@ -28027,7 +28027,7 @@ Here is the code content for tutorial_5_pyhealth_metrics.py: Automatically generated by Colaboratory. Original file is located at - https://colab.research.google.com/drive/1Mrs77EJ92HwMgDaElJ_CBXbi4iABZBeo + https://colab.research.google.com/drive/1bO0h5BR62_kQ7zFOgzQmt5vb8jqJ0rV-?usp=drive_link ### **Preparation** - install pyhealth alpha version @@ -28852,7 +28852,7 @@ Here is the code content for tutorial_0_pyhealth_data.py: Automatically generated by Colaboratory. Original file is located at - https://colab.research.google.com/drive/1y9PawgSbyMbSSMw1dpfwtooH7qzOEYdN + https://colab.research.google.com/drive/17nOzjIjKiAbC8bsntZ3h9xy2Vq4bKpuv """ !pip install pyhealth @@ -29719,7 +29719,7 @@ test_loader = get_dataloader(test_ds, batch_size=64, shuffle=False) """### **Step 2: Select a ML model** - In this tutorial, we use Transformer as the example. -- please check the [Tutorial 2](https://colab.research.google.com/drive/1LcXZlu7ZUuqepf269X3FhXuhHeRvaJX5?usp=sharing) for more instructions on how to initialize a model. +- please check the [Tutorial 2](https://colab.research.google.com/drive/1cUTSfFL1wLUXDBtJGTAWntolvcmxrDGo?usp=drive_link) for more instructions on how to initialize a model. """ from pyhealth.models import Transformer @@ -29773,7 +29773,7 @@ Here is the code content for tutorial_6_pyhealth_tokenizer.py: Automatically generated by Colaboratory. Original file is located at - https://colab.research.google.com/drive/1bDOb0A5g0umBjtz8NIp4wqye7taJ03D0 + https://colab.research.google.com/drive/1jhJ11MLUafhflQAz8HSrWiOEYlIhhvc_ ### **Preparation** - install pyhealth alpha version @@ -30334,7 +30334,7 @@ Here is the code content for tutorial_7_pyhealth_medcode.py: Automatically generated by Colaboratory. Original file is located at - https://colab.research.google.com/drive/1xrp_ACM2_Hg5Wxzj0SKKKgZfMY0WwEj3 + https://colab.research.google.com/drive/1Tw1AUS53fotH1EYr4Abp7qYN3zDBeUbC?usp=drive_link ### **Preparation** - install pyhealth alpha version @@ -30866,7 +30866,7 @@ Here is the code content for tutorial_3_pyhealth_models.py: Automatically generated by Colaboratory. Original file is located at - https://colab.research.google.com/drive/1LcXZlu7ZUuqepf269X3FhXuhHeRvaJX5 + https://colab.research.google.com/drive/1cUTSfFL1wLUXDBtJGTAWntolvcmxrDGo?usp=drive_link ### **Preparation** - install pyhealth alpha version @@ -31662,7 +31662,7 @@ Here is the code content for tutorial_4_pyhealth_trainer.py: Automatically generated by Colaboratory. Original file is located at - https://colab.research.google.com/drive/1L1Nz76cRNB7wTp5Pz_4Vp4N2eRZ9R6xl + https://colab.research.google.com/drive/1up_SL0BxxHPO9pmjKQ98w1GbpiB7LySp?usp=drive_link ### **Preparation** - install pyhealth alpha version @@ -31698,7 +31698,7 @@ To initialize a trainer instance, the following environments should be specified - `load_best_model_at_last`: whether to load the best model during the last iteration. ### **Step 1 & 2 & 3: Prepare datasets, task, and model** -- Example: We use **MIMIC-III dataset** and **RETAIN** model for **readmission prediction** task. Refer to [Tutorial 1](https://colab.research.google.com/drive/18kbzEQAj1FMs_J9rTGX8eCoxnWdx4Ltn?usp=sharing), [Tutorial 2](https://colab.research.google.com/drive/1r7MYQR_5yCJGpK_9I9-A10HmpupZuIN-?usp=sharing), and [Tutorial 3](https://colab.research.google.com/drive/1LcXZlu7ZUuqepf269X3FhXuhHeRvaJX5?usp=sharing). +- Example: We use **MIMIC-III dataset** and **RETAIN** model for **readmission prediction** task. Refer to [Tutorial 1](https://colab.research.google.com/drive/18kbzEQAj1FMs_J9rTGX8eCoxnWdx4Ltn?usp=sharing), [Tutorial 2](https://colab.research.google.com/drive/1r7MYQR_5yCJGpK_9I9-A10HmpupZuIN-?usp=sharing), and [Tutorial 3](https://colab.research.google.com/drive/1cUTSfFL1wLUXDBtJGTAWntolvcmxrDGo?usp=drive_link). """ # load dataset diff --git a/docs/_static/external_links.js b/docs/_static/external_links.js new file mode 100644 index 000000000..1cc5cb605 --- /dev/null +++ b/docs/_static/external_links.js @@ -0,0 +1,9 @@ +// Open every external link (http/https, different host) in a new tab. +document.addEventListener("DOMContentLoaded", () => { + for (const a of document.querySelectorAll('a[href^="http"]')) { + if (!a.href.includes(window.location.host)) { + a.target = "_blank"; + a.rel = "noopener noreferrer"; + } + } +}); diff --git a/docs/api/data.rst b/docs/api/data.rst index c6e940a68..411d4857a 100644 --- a/docs/api/data.rst +++ b/docs/api/data.rst @@ -8,7 +8,7 @@ Getting Started New to PyHealth's data structures? Start here: -- **Tutorial**: `Introduction to pyhealth.data `_ | `Video `_ +- **Tutorial**: `Introduction to pyhealth.data `_ | `Video `_ This tutorial introduces the core data structures in PyHealth: diff --git a/docs/api/datasets.rst b/docs/api/datasets.rst index 8d9a59d21..1875698ae 100644 --- a/docs/api/datasets.rst +++ b/docs/api/datasets.rst @@ -6,7 +6,7 @@ Getting Started New to PyHealth datasets? Start here: -- **Tutorial**: `Introduction to pyhealth.datasets `_ | `Video (PyHealth 1.6) `_ +- **Tutorial**: `Introduction to pyhealth.datasets `_ | `Video (PyHealth 1.6) `_ This tutorial covers: diff --git a/docs/api/tasks.rst b/docs/api/tasks.rst index 23a4e06e5..69e5aa592 100644 --- a/docs/api/tasks.rst +++ b/docs/api/tasks.rst @@ -18,7 +18,7 @@ Getting Started New to PyHealth tasks? Start here: -- **Tutorial**: `Introduction to pyhealth.tasks `_ - Learn the basics of defining and using tasks +- **Tutorial**: `Introduction to pyhealth.tasks `_ - Learn the basics of defining and using tasks - **Code Examples**: Browse all examples online at https://github.com/sunlabuiuc/PyHealth/tree/master/examples - **Pipeline Examples**: Check out our :doc:`../tutorials` page for complete end-to-end examples including: diff --git a/docs/conf.py b/docs/conf.py index 1591cdd47..72d742b7b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -190,6 +190,7 @@ } html_css_files = ["css/override.css", "css/sphinx_gallery.css"] +html_js_files = ["external_links.js"] html_show_sphinx = False # -- Options for HTMLHelp output --------------------------------------------- diff --git a/docs/tutorials.rst b/docs/tutorials.rst index fcdab84ea..9193c86c0 100644 --- a/docs/tutorials.rst +++ b/docs/tutorials.rst @@ -4,21 +4,21 @@ Tutorials We provide the following tutorials to help users get started with our pyhealth. Please bear with us as we update the documentation on how to use pyhealth 2.0. -`Tutorial 0: Introduction to pyhealth.data `_ `[Video] `_ +`Tutorial 0: Introduction to pyhealth.data `_ `[Video] `_ -`Tutorial 1: Introduction to pyhealth.datasets `_ `[Video (PyHealth 1.16)] `_ +`Tutorial 1: Introduction to pyhealth.datasets `_ `[Video (PyHealth 1.16)] `_ -`Tutorial 2: Introduction to pyhealth.tasks `_ `[Video (PyHealth 1.16)] `_ +`Tutorial 2: Introduction to pyhealth.tasks `_ `[Video (PyHealth 1.16)] `_ -`Tutorial 3: Introduction to pyhealth.models `_ `[Video] `_ +`Tutorial 3: Introduction to pyhealth.models `_ `[Video] `_ -`Tutorial 4: Introduction to pyhealth.trainer `_ `[Video] `_ +`Tutorial 4: Introduction to pyhealth.trainer `_ `[Video] `_ -`Tutorial 5: Introduction to pyhealth.metrics `_ `[Video] `_ +`Tutorial 5: Introduction to pyhealth.metrics `_ `[Video] `_ -`Tutorial 6: Introduction to pyhealth.tokenizer `_ `[Video] `_ +`Tutorial 6: Introduction to pyhealth.tokenizer `_ `[Video] `_ -`Tutorial 7: Introduction to pyhealth.medcode `_ `[Video] `_ +`Tutorial 7: Introduction to pyhealth.medcode `_ `[Video] `_ Data Access Guide diff --git a/examples/tutorial_stagenet_comprehensive.ipynb b/examples/tutorial_stagenet_comprehensive.ipynb index 7e1a9a3fd..de7956e27 100644 --- a/examples/tutorial_stagenet_comprehensive.ipynb +++ b/examples/tutorial_stagenet_comprehensive.ipynb @@ -35,7 +35,7 @@ "\n", "The result is a single dataframe where each row represents one patient and their features.\n", "\n", - "For more details on PyHealth datasets, see [this resource](https://colab.research.google.com/drive/1voSx7wEfzXfEf2sIfW6b-8p1KqMyuWxK#scrollTo=NSrb2PGFqUgS).\n", + "For more details on PyHealth datasets, see [this resource](https://colab.research.google.com/drive/1vI_oljc7rU5ocsC26ITM7HUgD5SGZkFE#scrollTo=NSrb2PGFqUgS).\n", "```" ] }, @@ -573,7 +573,7 @@ "\n", "Here, each feature will also need have its own corresponding time intervals. As defined by the StageNet paper, each time interval is defined as the difference in time between the current visit and the previous visit. \n", "\n", - "To define a task, specify the `__call__` method, input schema, and output schema. For a detailed explanation, see [this tutorial](https://colab.research.google.com/drive/1kKKBVS_GclHoYTbnOtjyYnSee79hsyT?usp=sharing).\n", + "To define a task, specify the `__call__` method, input schema, and output schema. For a detailed explanation, see [this tutorial](https://colab.research.google.com/drive/1QB0acnGb-wOuK53UNSgHxjCW74QeYjUl?usp=sharing).\n", "\n", "### Helper Functions\n", "\n", diff --git a/examples/tutorials/orig_tutorial_pyhealth_datasets.ipynb b/examples/tutorials/orig_tutorial_pyhealth_datasets.ipynb new file mode 100644 index 000000000..635d9fa80 --- /dev/null +++ b/examples/tutorials/orig_tutorial_pyhealth_datasets.ipynb @@ -0,0 +1,1613 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "source": [ + "# **Table of Contents**\n", + "In this tutorial, we will go over the following:\n", + "\n", + "0. **PyHealth datasets are not tasks.**\n", + "1. **Using an existing dataset (MIMIC3) to explore data efficiently**\n", + "2. **How data (events) is pre-loaded in PyHealth**\n", + "3. **Deep dive on how to implement and contribute your own PyHealth dataset**\n", + "\n", + "\n" + ], + "metadata": { + "id": "oVqm_kYbpJDn" + } + }, + { + "cell_type": "markdown", + "source": [ + "## **PyHealth datasets are not the same thing as tasks**\n", + "Before we start, we need to conceptualize how pyhealth.datasets work within the wider framework. Specifically, there's commonly a **misconception where people try to contribute dataset code that are more suitable for [pyhealth.tasks](https://github.com/sunlabuiuc/PyHealth/blob/master/pyhealth/tasks/base_task.py)**. While this might be functionally equivalent in some cases, it's not always true.\n", + "\n", + "![Image description](https://drive.google.com/uc?export=view&id=1hHJcavXqisH9JEMqEVtE4TqEg_E489l5)\n", + "\n", + "For instance, in PyHealth and in general, MIMIC3/4 will often have a variety of tasks that are based on the same dataset (i.e mortality prediction, readmission prediction, medical coding, etc.)\n", + "\n", + "While other datasets were originally intended to serve one purpose like SHHS serving the purpose of sleep staging classification, **many datasets serve as more of a pool of data than an annotated benchmark.** The pyhealth.datasets serves to make preprocessing this pool of data easier and more reproducible for the wider-community.\n", + "\n", + "\n", + "**For contributors**: This means that any dataset contribution has to be something that isn't already implemented in PyHealth (i.e MIMIC3/MIMIC4, etc.). So, if you're trying to do synthetic data generation with MIMIC3, please see the pyhealth.tasks [tutorial](https://colab.research.google.com/drive/1QB0acnGb-wOuK53UNSgHxjCW74QeYjUl?usp=sharing) instead of implementing your own synthetic data generation dataset." + ], + "metadata": { + "id": "YuvhioBXxWZC" + } + }, + { + "cell_type": "markdown", + "source": [ + "### Installation Procedure: To be changed to a pip install when a stable version is released." + ], + "metadata": { + "id": "0UEVz4s0K-nx" + } + }, + { + "cell_type": "code", + "source": [ + "!pip install pyhealth" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "-hT1DVoMIPkz", + "outputId": "5a069b25-d9b1-48b7-fd95-1c12a2779fba" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Requirement already satisfied: pyhealth in /usr/local/lib/python3.12/dist-packages (2.0.0)\n", + "Requirement already satisfied: accelerate in /usr/local/lib/python3.12/dist-packages (from pyhealth) (1.12.0)\n", + "Requirement already satisfied: dask~=2025.11.0 in /usr/local/lib/python3.12/dist-packages (from dask[complete]~=2025.11.0->pyhealth) (2025.11.0)\n", + "Requirement already satisfied: einops>=0.8.0 in /usr/local/lib/python3.12/dist-packages (from pyhealth) (0.8.2)\n", + "Requirement already satisfied: linear-attention-transformer>=0.19.1 in /usr/local/lib/python3.12/dist-packages (from pyhealth) (0.19.1)\n", + "Requirement already satisfied: litdata~=0.2.59 in /usr/local/lib/python3.12/dist-packages (from pyhealth) (0.2.61)\n", + "Requirement already satisfied: mne~=1.10.0 in /usr/local/lib/python3.12/dist-packages (from pyhealth) (1.10.2)\n", + "Requirement already satisfied: more-itertools~=10.8.0 in /usr/local/lib/python3.12/dist-packages (from pyhealth) (10.8.0)\n", + "Requirement already satisfied: narwhals~=2.13.0 in /usr/local/lib/python3.12/dist-packages (from pyhealth) (2.13.0)\n", + "Requirement already satisfied: networkx in /usr/local/lib/python3.12/dist-packages (from pyhealth) (3.6.1)\n", + "Requirement already satisfied: numpy~=2.2.0 in /usr/local/lib/python3.12/dist-packages (from pyhealth) (2.2.6)\n", + "Requirement already satisfied: ogb>=1.3.5 in /usr/local/lib/python3.12/dist-packages (from pyhealth) (1.3.6)\n", + "Requirement already satisfied: pandas~=2.3.1 in /usr/local/lib/python3.12/dist-packages (from pyhealth) (2.3.3)\n", + "Requirement already satisfied: peft in /usr/local/lib/python3.12/dist-packages (from pyhealth) (0.18.1)\n", + "Requirement already satisfied: polars~=1.35.2 in /usr/local/lib/python3.12/dist-packages (from pyhealth) (1.35.2)\n", + "Requirement already satisfied: pyarrow~=22.0.0 in /usr/local/lib/python3.12/dist-packages (from pyhealth) (22.0.0)\n", + "Requirement already satisfied: pydantic~=2.11.7 in /usr/local/lib/python3.12/dist-packages (from pyhealth) (2.11.10)\n", + "Requirement already satisfied: rdkit in /usr/local/lib/python3.12/dist-packages (from pyhealth) (2025.9.5)\n", + "Requirement already satisfied: scikit-learn~=1.7.0 in /usr/local/lib/python3.12/dist-packages (from pyhealth) (1.7.2)\n", + "Requirement already satisfied: torchvision in /usr/local/lib/python3.12/dist-packages (from pyhealth) (0.22.1)\n", + "Requirement already satisfied: torch~=2.7.1 in /usr/local/lib/python3.12/dist-packages (from pyhealth) (2.7.1)\n", + "Requirement already satisfied: tqdm in /usr/local/lib/python3.12/dist-packages (from pyhealth) (4.67.3)\n", + "Requirement already satisfied: transformers~=4.53.2 in /usr/local/lib/python3.12/dist-packages (from pyhealth) (4.53.3)\n", + "Requirement already satisfied: urllib3~=2.5.0 in /usr/local/lib/python3.12/dist-packages (from pyhealth) (2.5.0)\n", + "Requirement already satisfied: click>=8.1 in /usr/local/lib/python3.12/dist-packages (from dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth) (8.3.1)\n", + "Requirement already satisfied: cloudpickle>=3.0.0 in /usr/local/lib/python3.12/dist-packages (from dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth) (3.1.2)\n", + "Requirement already satisfied: fsspec>=2021.09.0 in /usr/local/lib/python3.12/dist-packages (from dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth) (2025.3.0)\n", + "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.12/dist-packages (from dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth) (26.0)\n", + "Requirement already satisfied: partd>=1.4.0 in /usr/local/lib/python3.12/dist-packages (from dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth) (1.4.2)\n", + "Requirement already satisfied: pyyaml>=5.3.1 in /usr/local/lib/python3.12/dist-packages (from dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth) (6.0.3)\n", + "Requirement already satisfied: toolz>=0.10.0 in /usr/local/lib/python3.12/dist-packages (from dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth) (0.12.1)\n", + "Requirement already satisfied: lz4>=4.3.2 in /usr/local/lib/python3.12/dist-packages (from dask[complete]~=2025.11.0->pyhealth) (4.4.5)\n", + "Requirement already satisfied: axial-positional-embedding in /usr/local/lib/python3.12/dist-packages (from linear-attention-transformer>=0.19.1->pyhealth) (0.3.12)\n", + "Requirement already satisfied: linformer>=0.1.0 in /usr/local/lib/python3.12/dist-packages (from linear-attention-transformer>=0.19.1->pyhealth) (0.2.3)\n", + "Requirement already satisfied: local-attention in /usr/local/lib/python3.12/dist-packages (from linear-attention-transformer>=0.19.1->pyhealth) (1.11.2)\n", + "Requirement already satisfied: product-key-memory>=0.1.5 in /usr/local/lib/python3.12/dist-packages (from linear-attention-transformer>=0.19.1->pyhealth) (0.3.0)\n", + "Requirement already satisfied: lightning-utilities in /usr/local/lib/python3.12/dist-packages (from litdata~=0.2.59->pyhealth) (0.15.2)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.12/dist-packages (from litdata~=0.2.59->pyhealth) (3.24.2)\n", + "Requirement already satisfied: boto3 in /usr/local/lib/python3.12/dist-packages (from litdata~=0.2.59->pyhealth) (1.42.53)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.12/dist-packages (from litdata~=0.2.59->pyhealth) (2.32.4)\n", + "Requirement already satisfied: tifffile in /usr/local/lib/python3.12/dist-packages (from litdata~=0.2.59->pyhealth) (2026.2.16)\n", + "Requirement already satisfied: obstore in /usr/local/lib/python3.12/dist-packages (from litdata~=0.2.59->pyhealth) (0.8.2)\n", + "Requirement already satisfied: decorator in /usr/local/lib/python3.12/dist-packages (from mne~=1.10.0->pyhealth) (4.4.2)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.12/dist-packages (from mne~=1.10.0->pyhealth) (3.1.6)\n", + "Requirement already satisfied: lazy-loader>=0.3 in /usr/local/lib/python3.12/dist-packages (from mne~=1.10.0->pyhealth) (0.4)\n", + "Requirement already satisfied: matplotlib>=3.7 in /usr/local/lib/python3.12/dist-packages (from mne~=1.10.0->pyhealth) (3.10.0)\n", + "Requirement already satisfied: pooch>=1.5 in /usr/local/lib/python3.12/dist-packages (from mne~=1.10.0->pyhealth) (1.9.0)\n", + "Requirement already satisfied: scipy>=1.11 in /usr/local/lib/python3.12/dist-packages (from mne~=1.10.0->pyhealth) (1.16.3)\n", + "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.12/dist-packages (from ogb>=1.3.5->pyhealth) (1.17.0)\n", + "Requirement already satisfied: outdated>=0.2.0 in /usr/local/lib/python3.12/dist-packages (from ogb>=1.3.5->pyhealth) (0.2.2)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.12/dist-packages (from pandas~=2.3.1->pyhealth) (2.9.0.post0)\n", + "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.12/dist-packages (from pandas~=2.3.1->pyhealth) (2025.2)\n", + "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.12/dist-packages (from pandas~=2.3.1->pyhealth) (2025.3)\n", + "Requirement already satisfied: polars-runtime-32==1.35.2 in /usr/local/lib/python3.12/dist-packages (from polars~=1.35.2->pyhealth) (1.35.2)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.12/dist-packages (from pydantic~=2.11.7->pyhealth) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.33.2 in /usr/local/lib/python3.12/dist-packages (from pydantic~=2.11.7->pyhealth) (2.33.2)\n", + "Requirement already satisfied: typing-extensions>=4.12.2 in /usr/local/lib/python3.12/dist-packages (from pydantic~=2.11.7->pyhealth) (4.15.0)\n", + "Requirement already satisfied: typing-inspection>=0.4.0 in /usr/local/lib/python3.12/dist-packages (from pydantic~=2.11.7->pyhealth) (0.4.2)\n", + "Requirement already satisfied: joblib>=1.2.0 in /usr/local/lib/python3.12/dist-packages (from scikit-learn~=1.7.0->pyhealth) (1.5.3)\n", + "Requirement already satisfied: threadpoolctl>=3.1.0 in /usr/local/lib/python3.12/dist-packages (from scikit-learn~=1.7.0->pyhealth) (3.6.0)\n", + "Requirement already satisfied: setuptools in /usr/local/lib/python3.12/dist-packages (from torch~=2.7.1->pyhealth) (75.2.0)\n", + "Requirement already satisfied: sympy>=1.13.3 in /usr/local/lib/python3.12/dist-packages (from torch~=2.7.1->pyhealth) (1.14.0)\n", + "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.6.77 in /usr/local/lib/python3.12/dist-packages (from torch~=2.7.1->pyhealth) (12.6.77)\n", + "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.6.77 in /usr/local/lib/python3.12/dist-packages (from torch~=2.7.1->pyhealth) (12.6.77)\n", + "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.6.80 in /usr/local/lib/python3.12/dist-packages (from torch~=2.7.1->pyhealth) (12.6.80)\n", + "Requirement already satisfied: nvidia-cudnn-cu12==9.5.1.17 in /usr/local/lib/python3.12/dist-packages (from torch~=2.7.1->pyhealth) (9.5.1.17)\n", + "Requirement already satisfied: nvidia-cublas-cu12==12.6.4.1 in /usr/local/lib/python3.12/dist-packages (from torch~=2.7.1->pyhealth) (12.6.4.1)\n", + "Requirement already satisfied: nvidia-cufft-cu12==11.3.0.4 in /usr/local/lib/python3.12/dist-packages (from torch~=2.7.1->pyhealth) (11.3.0.4)\n", + "Requirement already satisfied: nvidia-curand-cu12==10.3.7.77 in /usr/local/lib/python3.12/dist-packages (from torch~=2.7.1->pyhealth) (10.3.7.77)\n", + "Requirement already satisfied: nvidia-cusolver-cu12==11.7.1.2 in /usr/local/lib/python3.12/dist-packages (from torch~=2.7.1->pyhealth) (11.7.1.2)\n", + "Requirement already satisfied: nvidia-cusparse-cu12==12.5.4.2 in /usr/local/lib/python3.12/dist-packages (from torch~=2.7.1->pyhealth) (12.5.4.2)\n", + "Requirement already satisfied: nvidia-cusparselt-cu12==0.6.3 in /usr/local/lib/python3.12/dist-packages (from torch~=2.7.1->pyhealth) (0.6.3)\n", + "Requirement already satisfied: nvidia-nccl-cu12==2.26.2 in /usr/local/lib/python3.12/dist-packages (from torch~=2.7.1->pyhealth) (2.26.2)\n", + "Requirement already satisfied: nvidia-nvtx-cu12==12.6.77 in /usr/local/lib/python3.12/dist-packages (from torch~=2.7.1->pyhealth) (12.6.77)\n", + "Requirement already satisfied: nvidia-nvjitlink-cu12==12.6.85 in /usr/local/lib/python3.12/dist-packages (from torch~=2.7.1->pyhealth) (12.6.85)\n", + "Requirement already satisfied: nvidia-cufile-cu12==1.11.1.6 in /usr/local/lib/python3.12/dist-packages (from torch~=2.7.1->pyhealth) (1.11.1.6)\n", + "Requirement already satisfied: triton==3.3.1 in /usr/local/lib/python3.12/dist-packages (from torch~=2.7.1->pyhealth) (3.3.1)\n", + "Requirement already satisfied: huggingface-hub<1.0,>=0.30.0 in /usr/local/lib/python3.12/dist-packages (from transformers~=4.53.2->pyhealth) (0.36.2)\n", + "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.12/dist-packages (from transformers~=4.53.2->pyhealth) (2025.11.3)\n", + "Requirement already satisfied: tokenizers<0.22,>=0.21 in /usr/local/lib/python3.12/dist-packages (from transformers~=4.53.2->pyhealth) (0.21.4)\n", + "Requirement already satisfied: safetensors>=0.4.3 in /usr/local/lib/python3.12/dist-packages (from transformers~=4.53.2->pyhealth) (0.7.0)\n", + "Requirement already satisfied: psutil in /usr/local/lib/python3.12/dist-packages (from accelerate->pyhealth) (5.9.5)\n", + "Requirement already satisfied: Pillow in /usr/local/lib/python3.12/dist-packages (from rdkit->pyhealth) (11.3.0)\n", + "Requirement already satisfied: hf-xet<2.0.0,>=1.1.3 in /usr/local/lib/python3.12/dist-packages (from huggingface-hub<1.0,>=0.30.0->transformers~=4.53.2->pyhealth) (1.2.0)\n", + "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.12/dist-packages (from matplotlib>=3.7->mne~=1.10.0->pyhealth) (1.3.3)\n", + "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.12/dist-packages (from matplotlib>=3.7->mne~=1.10.0->pyhealth) (0.12.1)\n", + "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.12/dist-packages (from matplotlib>=3.7->mne~=1.10.0->pyhealth) (4.61.1)\n", + "Requirement already satisfied: kiwisolver>=1.3.1 in /usr/local/lib/python3.12/dist-packages (from matplotlib>=3.7->mne~=1.10.0->pyhealth) (1.4.9)\n", + "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.12/dist-packages (from matplotlib>=3.7->mne~=1.10.0->pyhealth) (3.3.2)\n", + "Requirement already satisfied: littleutils in /usr/local/lib/python3.12/dist-packages (from outdated>=0.2.0->ogb>=1.3.5->pyhealth) (0.2.4)\n", + "Requirement already satisfied: locket in /usr/local/lib/python3.12/dist-packages (from partd>=1.4.0->dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth) (1.0.0)\n", + "Requirement already satisfied: platformdirs>=2.5.0 in /usr/local/lib/python3.12/dist-packages (from pooch>=1.5->mne~=1.10.0->pyhealth) (4.9.2)\n", + "Requirement already satisfied: colt5-attention>=0.10.14 in /usr/local/lib/python3.12/dist-packages (from product-key-memory>=0.1.5->linear-attention-transformer>=0.19.1->pyhealth) (0.11.1)\n", + "Requirement already satisfied: charset_normalizer<4,>=2 in /usr/local/lib/python3.12/dist-packages (from requests->litdata~=0.2.59->pyhealth) (3.4.4)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.12/dist-packages (from requests->litdata~=0.2.59->pyhealth) (3.11)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.12/dist-packages (from requests->litdata~=0.2.59->pyhealth) (2026.1.4)\n", + "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.12/dist-packages (from sympy>=1.13.3->torch~=2.7.1->pyhealth) (1.3.0)\n", + "Requirement already satisfied: botocore<1.43.0,>=1.42.53 in /usr/local/lib/python3.12/dist-packages (from boto3->litdata~=0.2.59->pyhealth) (1.42.53)\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /usr/local/lib/python3.12/dist-packages (from boto3->litdata~=0.2.59->pyhealth) (1.1.0)\n", + "Requirement already satisfied: s3transfer<0.17.0,>=0.16.0 in /usr/local/lib/python3.12/dist-packages (from boto3->litdata~=0.2.59->pyhealth) (0.16.0)\n", + "Requirement already satisfied: distributed==2025.11.0 in /usr/local/lib/python3.12/dist-packages (from dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth) (2025.11.0)\n", + "Requirement already satisfied: bokeh>=3.1.0 in /usr/local/lib/python3.12/dist-packages (from dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth) (3.7.3)\n", + "Requirement already satisfied: msgpack>=1.0.2 in /usr/local/lib/python3.12/dist-packages (from distributed==2025.11.0->dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth) (1.1.2)\n", + "Requirement already satisfied: sortedcontainers>=2.0.5 in /usr/local/lib/python3.12/dist-packages (from distributed==2025.11.0->dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth) (2.4.0)\n", + "Requirement already satisfied: tblib>=1.6.0 in /usr/local/lib/python3.12/dist-packages (from distributed==2025.11.0->dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth) (3.2.2)\n", + "Requirement already satisfied: tornado>=6.2.0 in /usr/local/lib/python3.12/dist-packages (from distributed==2025.11.0->dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth) (6.5.1)\n", + "Requirement already satisfied: zict>=3.0.0 in /usr/local/lib/python3.12/dist-packages (from distributed==2025.11.0->dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth) (3.0.0)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.12/dist-packages (from jinja2->mne~=1.10.0->pyhealth) (3.0.3)\n", + "Requirement already satisfied: hyper-connections>=0.1.8 in /usr/local/lib/python3.12/dist-packages (from local-attention->linear-attention-transformer>=0.19.1->pyhealth) (0.4.9)\n", + "Requirement already satisfied: xyzservices>=2021.09.1 in /usr/local/lib/python3.12/dist-packages (from bokeh>=3.1.0->dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth) (2025.11.0)\n", + "Requirement already satisfied: torch-einops-utils>=0.0.20 in /usr/local/lib/python3.12/dist-packages (from hyper-connections>=0.1.8->local-attention->linear-attention-transformer>=0.19.1->pyhealth) (0.0.30)\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "## **Exploring [pyhealth.datasets](https://pyhealth.readthedocs.io/en/latest/api/datasets.html) (e.g., MIMIC-III, COVID19CXR)**\n", + "- **[README]**: The PyHealth dataset module is used to process the unstructured raw data into a structured dataset object. Here, we showcase how pyhealth datasets work (i.e how to work with the BaseDataset class) and how it simplifies much of the heavy lifting in doing experimental research while being minimally invasive.\n", + "- **[Arguments]**:\n", + " - `root` is the arguments directing to the data folder, e.g., \"mimiciii/1.4/\".\n", + " - `tables` is a list of table names from raw databases, which specifies the information that will be used in building your dataset.\n", + " - ``dev``: whether to enable dev mode (only use a small subset of the data)\n", + " Default is False.\n", + "\n", + "- **[Functionality]**: currently, we provide the api for:\n", + " - [MIMIC3Dataset](https://pyhealth.readthedocs.io/en/latest/api/datasets/pyhealth.datasets.MIMIC3Dataset.html)\n", + " - [MIMIC4Dataset](https://pyhealth.readthedocs.io/en/latest/api/datasets/pyhealth.datasets.MIMIC4Dataset.html)\n", + " - [eICUDataset](https://pyhealth.readthedocs.io/en/latest/api/datasets/pyhealth.datasets.eICUDataset.html)\n", + " - [OMOPDataset](https://pyhealth.readthedocs.io/en/latest/api/datasets/pyhealth.datasets.OMOPDataset.html): any OMOP-CDM based databases." + ], + "metadata": { + "id": "_1S5rqae7FhB" + } + }, + { + "cell_type": "markdown", + "source": [ + "![image.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAABlAAAAGoCAYAAAA5EqguAAAACXBIWXMAABYlAAAWJQFJUiTwAAAgAElEQVR4nOzdeVyU1f4H8M8wgKyisnhBZRFlUcRcyQUdxDT3LAvc7dpiVnptMft1b3lbfi1mZmXXe7uVpl5zSc3dTMHMBRUFFTFFQZRNRGSHYWbO7w9/zPWZBQYYGJbP+/Xilc/znOec7znPBDPPd85zZEIIASIiIiIiIiIiIiIiItKysnQARERERERERERERERETQ0TKERERERERERERERERDqYQCEiIiIiIiIiIiIiItLBBAoREREREREREREREZEOJlCIiIiIiIiIiIiIiIh0MIFCRERERERERERERESkgwkUIiIiIiIiIiIiIiIiHUygEBERERERERERERER6bC2dABERERERERERER1oVarkZKSgqSkJFy8eBFJSUlQqVQICAhAUFAQhg0bBj8/vxrruXz5Mtzc3ODm5tYIUTe+lt6/mly7dg0rVqyAk5MTFi5cCE9Pz2bZRlFREa5fv47evXubve6mpjX1lZo2JlCIiIiIiIiIiKjZOXfuHGbOnImkpCSjZWxsbPD888/j7bffhru7u8Eyc+bMwdq1a9GmTRusXLkSzz//fEOFbBEtvX+mePnll7Fv3z4AQHFxMb766qtm18b58+cxdOhQFBUVISQkBMePH4ezs7NZ22gqWlNfqenjI7yIiIiIiIiIiKhZWblyJcLCwqpNngBAZWUlvvrqKwwaNAh3797VO15cXIx169YBACoqKrB8+fIGiddSWnr/TPXbb79p/3306NFm2caGDRtQVFQEALh48SJ27dpl9jaaitbUV2r6mEAhIiIiIiIiIqJmIzk5Ga+88goqKysl+7t3745nn30Wc+fO1Xt80rVr1/D000/r1VVYWAiNRqPdNpRkac5aev9ModFoUFJSot2uqKholm3k5+dLtlvytWxNfaWmj4/wIiIiIiIiIiKiZmP16tWSpEDXrl2xb98+BAQEaPeVl5cjIiICJ0+e1O7bu3cviouL4eTkpN2nUqkkdetuN3ctvX+tSWu6lq2pr9T0cQYKERERERERERE1G5cuXZJs/+Uvf5EkTwDAzs4OP/74I9q1a6fdp1KpkJGRUed2VSoVbt++LdmnVCpx8+ZNg+UrKiqQm5tbY705OTl6s2mMUSqVSEtLw5UrV0w+x1TmiLe2Y6SrtLQUSUlJyM7OhhDCpHMeVF5ejqysrFqfZ4k2VCoVrly5gsuXLyMrK0uSFDS3vLw8XLhwAWlpadpHY9X2/LS0NCiVyjq1X9/rSmRJTKAQEREREREREVGzoftt9NjYWIM3n318fPCPf/xDuz1w4EAEBgYCAIQQ+J//+R9MmDBBck5BQQFCQkIwfPhwbN++Xbv/5MmT6Nq1K7y8vLBw4UKUl5dj1qxZcHd3h7e3N3x8fPDMM88gLy8PALB//3507txZW96YZ555Bl5eXvD29ja6boZSqcSqVavQt29f2NnZwc/PD4GBgXBwcEBwcDD+/ve/o7i4WHJObftnjnhrO0ZVrl27hqlTpyIgIADOzs4ICQmBp6cnHBwcMG3aNJw9e9ZoPMD9xM+bb76J3r17w9nZGV5eXvDw8MDYsWOxc+fOas81lbna+O233/DUU08hJCQEjo6OCAwMRHBwMLy8vODq6opJkyZJZk0BQFxcHCZNmiS5XgDwwQcfoFevXpg7d65eUqSgoABvvvkmIiIi0LFjR7i5uSE0NBR+fn5wcXFBaGgo/v73v6OsrMxorD/99BMmTpwIb29vuLm5wc/PD3Z2dvDy8sK8efNw4cKFavtal+tal74SNThBRERERERERETUTMydO1cAkPyMGDFCHDhwQCiVSr3yZ86cEdu2bROVlZXafcePH9erQ/fH19dXW37RokXa/XZ2diIqKsrgOU8++aQQQognnnhCu8/Gxkbcu3dPL67s7GxhZWWlLTd37ly9MiqVSkyZMqXGWN3d3cXRo0fr3D9zxFvbMRJCiJUrVwoHB4dq43RwcBAxMTGGXgri+vXrok+fPkbPlclkYsmSJZJ9gYGBBusyxhxtKJVKsWTJEsn4VVffli1btOeOGjWqxnPWrl2rLX/w4EHxpz/9qcZzAIj+/fuLiooKSayVlZWSa2nsx8rKSnzzzTcGx6yu17W2fSVqDJyBQkREREREREREzcbs2bP19h0+fBijR49Ghw4dMH78eHz11Ve4du0aAKBfv36YPHkyrK3/uxRwaGgovL29jbZhZWWFoUOHarfPnTun/Xd5eTk2bdpk8Lyqxa8vXryo3VdZWYk//vhDr2xSUpJk5syD5wD3Z5E8++yz2Lp1q9E4q+Tm5mLChAnadmrbP3PEW9sx2rBhAxYuXIjS0lLtsarZEY6Ojtp9paWlGD9+PH7//Xe9uubMmSNpV5cQAh999JHR46YwRxsbNmzARx99JBk/a2trBAcHIygoCFZW/71FK4TA/PnzoVarAQBjx46FXC43WrerqyseeeQR7bnPPvsssrOzJWXc3NzQt29fuLq6SvafOXMG//znPyX7duzYgRUrVmi327Rpg4iICEyZMgWenp7a/RqNBs899xzS0tL0+lrX61qbvhI1FiZQiIiIiIiIiIio2QgPD8eHH34ouelcpbi4GHv27MHLL7+Mbt26ITg4GN98843emh2Ojo5IS0vDgQMHJPvt7e1x/vx5FBUVYd26ddr9hhax7ty5MxYvXozFixdj2LBhGDhwIN577z0z9RJ477338P3330v2zZs3D4mJiUhKSsKKFSskN6Tv3buHVatW1al/5lCbMSouLsbixYslZefPn4+cnBwkJiYiOzsb06ZN0x4rKSlBVFSUpPzOnTvx22+/SfYpFAr88ssvyM7Oxtq1a+Hh4VGvPpmrjfXr10u23333XRQXF+PSpUtITk5GbGys5Hhubi7i4uIAAAsXLkROTg7GjRsnKbNo0SKkpaXhzp072sTGsWPHJAkNf39/JCcnIzc3F/Hx8cjIyMDYsWMl9ezatUuyrfv4rN27d+Pw4cPYsmUL/vjjD4wcOVJ7TAgheeRYfa9rbfpK1Fisay5CRERERERERETUdCxZsgQPPfQQpk2bpp3RYMjly5fx3HPP4YMPPsC2bdvQt29f7TGZTIagoCBJeVtbW/Tq1avG9q2trbFr1y489NBDde9ENdRqNb7++mvJvkWLFuGzzz7Tbvfo0QM9evTA+PHjtQmipKQk7fH69M8cqhujFStWIDMzU7vdt29ffPnll9qkmJOTE7799lucOHECqampAIDMzExcunQJPXr0AAC9mRM+Pj7Ytm0b2rdvDwCYNWsWfHx8MHLkSIPJHVOYq40HZ1W8+eab+Nvf/iY5Hh4ejsDAQMnMn9TUVAwePBjA/ZkXuomaqnVljLXj7u6OI0eOoFOnTtp9bdq0wcyZM7F3715JOw/SnX30YELG2dkZO3fuxNSpU7Fr1y706dMHjz76qPb4N998U+/rampfiRoLZ6AQEREREREREVGz8+ijjyI9PR0bN27E448/Dnt7e6Nlb9y4gfHjxyM9Pd0sbUdGRjZY8gS4v5h2Tk6OdtvBwQFvvfWWXrlRo0Zh7dq1aNOmDdzd3fHqq682WEy1Vd0YJSYmSrZnzpypN6PIzs4Ojz32mGTfoUOHtP++evWq5Ng777yjTWxUGT58uN7MldowVxurV6/G4sWL8cEHH+Ddd9/V7lepVDh16hT+93//F4WFhZJz7t69W+t4Bw0ahB9++AEvvvgidu7cKUme5OTk4Mcff8RPP/1UbTsTJkyQbD/77LPo27cv3nvvPRw/fhw2NjbYsWMHSktLcebMGbRr105b1hzXlaip4QwUIiIiIiIiIiJqlpycnBAdHY3o6GiUlJTgyJEjOHjwIPbs2aN38zsrKwvvvPOO3mOx6qJfv371rkMIYfTYjRs3JNuhoaF661dUmTp1KiZNmgQbGxvY2NjUOy5jqovXkOrGKDk5WbK9bNkybN68Wa/c9evXJdtVsxbUarXe2ht9+vQx2Fbfvn2xYcMGU0KWMGcbfn5++Pjjj5Gfn49t27bh1KlTiIuLQ3x8PMrKyoy2XxczZ87EzJkzcf78eXz22Wc4ffo0Tp48qdcXY+3MmDEDX3zxhSSxcu7cOZw7dw5vv/02XF1dER0djdmzZ2PAgAGSc+t7XYmaIiZQWqG7d+/iX//6F5KTkzF37lwMGzasxnOuXbuGVatWQaVS4YUXXkBwcHAjRFp7RUVFuH79Onr37m3pUBpES+9fTTQaDdavX4+9e/ciKioKkydPtnRIzVZjjeXly5fh5uYGNze3BqmfiIiIiIhan8TERPzrX/+Ct7c3FixYoJ154ujoiLFjx2Ls2LFYvnw5Vq9ejddff12ymHXVuhL11bVr13rXUVFRYfSY7iLgD84kMMTBwaHe8dSkungNqW6MdG/mZ2ZmSh79ZMygQYMA3F+IXnddm27duhk8JzAwsMZ6DTFnG6WlpVi4cCHWrVtn8jjKZDLTAtVx7do1vPXWW9i8ebNJSS/ddvz9/ZGYmIiZM2fqrc0CAHl5eVi1ahVWrVqFRYsWYdmyZdpHh9X3uhI1RUygtEJLly7Fl19+CQDYsGEDcnJyjH6Locr06dO1bzIOHDig9zzEpuD8+fMYOnQoioqKEBISguPHj8PZ2dnSYZlNS++fKc6cOYPZs2cDADZv3oycnBy4u7tbOKrmqTHGcs6cOdqp5CtXrsTzzz9v1vqJiIiIiKj1UalUGDNmDLKysgDcXzj9ww8/1CtnZWWF+fPn4/z585J1LK5cuYLKysp6z9SobcKiuLhYb9+DiR1dup/PanqcU0FBAW7fvo3u3bvXKi5jahuvIdWNkZeXF27fvq3d7t27t9HY7e3tERAQgP79+2vX23B1dYW9vb1k9kZaWhpCQkL0ztedzWMqc7YxYcIEHD58WLLP09MTQ4cOxcMPP4zw8HB89913WL16tfZ4XRIoMTExGD16tCTxY2Njg379+mHQoEEYPHgwevbsqV1vxFg7nTt3RkxMDOLi4rBx40bs2rVLb9YIcH8tGwDatXnqe12JmiImUFqhPXv2aP+tVqtx4MABTJs2zWj5vLw8nD59Wrt95coVXLlyBQEBAQ0aZ21t2LABRUVFAICLFy9i165d1faruWnp/TPFkSNHtP8WQuD333/nLJQ6auixLC4uxrp16wDc/5bS8uXLmUAhIiIiIqJ6S0tL0yZPgPs3bufNm2d0gemBAwfqLQRuDra2ttUe170p/WDMVa5du2b0fN3F35OSkowmfk6dOgWFQoGysjIsXrwYH3/8cbWxNUS8hlQ3RsHBwUhISNBuR0dHY8mSJUbLK5VKybZMJkPXrl2RlJSk3ZeYmGgwuaG7LoepzNXGvn37JMkTuVyOtWvXYtq0aZJx/+STTyTn6a4dYoolS5ZIkieRkZH4/vvv0aVLF+2+B/tjqJ2cnBxs3boVPXv2hEKhQFhYGD7//HOkpKRgx44dWLFihWRWycaNG7F8+XLIZLJ6X1eipoiLyLdCubm5ku0HM8OG5OXlQaPR1OocS8jPz5ds12WxraaspffPFFUJpCq1nT5M/9XQY1lYWCj5vdEaX69ERERERGR+Pj4+sLOz024rlUq8/vrrevctAKCyshI//PCDZJ+vr68kCaGbkDC2HoWuB2MwxMXFRbL922+/Sbbz8/Px73//2+j5QUFBkpvrt2/fNrh2S0ZGBqZMmaKNe+3atZLjpvavvvEaUt0Y6T4afv/+/UYfN/XJJ5/A2dkZHh4eklkQukmmTz75RG+WTEpKCrZs2VKruB9kjjbi4+Ml2wqFAtOnT5dc34qKCsTExEjK6SaxarqWarVakrwA7j+F5sHkCXB/rB+Un58vuScwduxYvPTSSxgxYoTkS9jdunXDa6+9pv2yZJWcnBztmJjjuprSV6LGxAQKtRgqlara7eaupfePWha+XomIiIiIqCHY2Nhgzpw5kn1btmzBhAkTcOjQIeTn5yM/Px8HDhxARESEZPY9AL1zPTw8JN/AVyqV2keY3717t9YLp1fx9PSUbH/77bfYvHkzSktLcezYMTz22GNISUkxer6TkxNmzZol2feXv/wF69atg0ajgRACCQkJePTRR3Hz5k1tGd21OEztX33jra25c+eibdu22u0jR47g9ddfl5QpKyvDc889hzfeeANKpRIFBQW4ePGi9vjLL78sKX/+/HlMnToVFy5cgEqlwu+//44JEybofSG1NszRhu7j1xMSEiQJgaKiIowfPx55eXmScpcvX5Zs616j48ePQ6VSQalUorCwEDKZDE5OTnplHrR9+3b89a9/lezTaDS4cuUKgPtPqKhax0QIgaioKJw8ebLa/oSGhsLR0RGAea6rKX0lakxMoJDZFBcX681uAf77y7cu0/Ly8vLqfG51srOzoVarJfsyMzONZrQzMzMNfpvlQWq1Wm+Rt+rcvXsXSUlJuHPnjsnnmMoc8dZ2jHSVlpYiKSkJ2dnZdXrDmZ2djfLy8lqfVx2VSqU3e0qpVErebNZ0/pUrV3D58mVkZWVVO8a3b9/Wm+VhqExJSUm1Ze7evVtjPTUx11jWpv/NoR0iIiIiImqePvjgA/Tr10+yb+/evRg5ciQ6dOiADh064NFHH8WxY8ckZfr06YOXXnpJsk8ul8PDw0Oyb9CgQWjbti1cXV0xYMCAOn2m1V3TQa1WIyoqCo6Ojhg6dKjeDA9Dli1bhg4dOmi3y8rKMGvWLLi4uMDd3R19+vSR3HiWyWR45ZVX6tQ/c8RbG15eXnj//fcl+5YvX46QkBAsWLAA8+fPh4+PD7755hvtcQ8PD0RGRmq3hw8fLtkGgJ07dyI0NBT29vYIDw/XS0LUljnaGD16tGQ7Ly8P/v7+ePHFFxEVFYXu3bvj119/1TuvKtFVRTepsHv3brRv3x729vZwc3PD9u3b8cgjj0jKvPnmm3jkkUfwl7/8BYMGDcKUKVMM3h84c+YMgPuvoejoaO3+kpISREZG4tFHH8Vbb72FadOm6fVnzJgx2n+b47qa0teffvpJrw9EDUZQq+Ps7CwAaH9WrFhRbfk//vhDUh6AOHr0qKTMli1btPV++OGHQggh9u/fL8aNGyc6dOggAAg7OzsxfPhw8e9//7va9rZu3SomTJggunTpom1PJpMJT09P8fzzz4vz589Lyp88eVJMnDhRtGvXThKjm5ubCAkJEX/+859FYWGhtvzcuXOFlZWV8PX1FefPnxe7du0SvXv3FgCEra2tGD58uPjxxx+15ceNGycAaMsbEhcXJzp37ixkMpmYOnWq0b4lJCSIP//5z6Jt27aSWF1dXcWjjz4qfv/9d71zats/c8Rb2zGqkpKSIqKjo0X37t2FlZWVNlY7OzsxdepUER8fb3RshBDi+PHjYvTo0cLd3V0AENbW1qJ3797ipZdeEvfu3RN/+9vfJGOwcePGaut70IkTJ0SXLl2EXC4XCxYsEGVlZWLmzJnaa+Ht7S3mzp0r7ty5IznvyJEj4sknnxQ9e/YUtra2kvbbtWsnJk6cKE6cOCE5Z8mSJUIul4v27duLY8eO6cVSWVkpxo8fL2QymbCzsxPvv/++wZg//fRTYWVlJRwdHcXWrVtN7qsQ5hvL2vZfo9GIN998U4SGhur93ujZs6cYNmyY2LZtW73bISIiIiKi1u3evXvi4Ycf1vvcYexn+vTporS01GBdCxcurPbcnJwcMWvWLMm+S5cuVRtfUVGRGDx4cLX1vvLKK9p7JgDEpEmT9Oo5ePCg8PLyqrF/crlcLF++vM79M0e8tR0jtVotFixYILl/YOzHxcXF4D2Fa9euieDg4GrPfeaZZ8Sf/vQn7bZCoag2roZoY/r06dWeb21tLWbMmKF3zy47O1tbR1pamnBxcTFaxwsvvCBOnTqld/9I96dHjx4iMjJSsm/+/Pnadm7duiUGDRpk0v9XgYGB4saNG2a/rqb0laixMIHSCjVEAmXs2LHaY507dxbffPONsLa2NvqLbuXKlXrtVFZWikWLFtX4y9XKykp888032vNGjRpV4zlr164VQgiRn58v+QU+ZcoU4ejoqFdeJpOJCxcuiAsXLkj2L1682OAYzZ8/X1IuPT1dr8zx48cNtqX7M2PGDKFSqerUP3PEW9sxqrJy5Urh4OBQbZwODg4iJibGYEzLli2r9jXTvXt38fjjj0v21SaB8uBry87OTkRFRRls58knnxRCCKFUKsWSJUtM+oMvk8nEli1bhBD33yjY2dlpj0VFRenF8sMPP+idb+gNh4eHh7bMiBEjTO6rOcayrv0/fvx4jeV9fX3r3Q4REREREVFRUZH49NNPRY8ePYx+jujbt69YvXp1tfWUl5eLESNG6J1b9QU8IYTYs2eP9nNW//79TYqvpKREREVFCXt7e0m93t7eYs2aNUIIIebNm6f9vLN+/XqD9eTn54tnn31WuLq66sVob28vHn/8cZGcnFyv/pkj3rqMkRD3v/AYFhYmbGxs9GJ0dHQUr7zyisjJyTF6fkFBgcG4vby8xD/+8Q8hhPSewNdff21ybOZqQ6PRiNdee00vKWBnZydGjRqlvb+yadMm7TgMGTJEaDQaST1btmwxmFjw9fUViYmJQgghzp8/L0JDQ4VMJpOU6dy5s/jkk0+EUqkUZWVlIiwsTPs62Lt3r6SdyspK8fbbb+t9Qbbq2vv4+IglS5aIsrIyo2NW3+tqSl+JGgMTKK1QQyRQAgMDa7z5qfvL1tAslgfLtGnTRkRERIgpU6YIT09PvfNTU1OFEEJ8/vnnQi6XG23L1dVVZGZmCiGEiI2NNTnG33//Xfz444+SfWPGjDE4RuHh4ZJy+/btkxw/d+5cjd8AePDnwUx6bfpnjnhrO0ZCCLF+/Xq9Yy4uLiI0NFQv+eLo6Kh37Q8fPlyr10/VT20SKAqFwqQ6R44cKYQQ4vvvv9c7Zm1tLYKDg0VQUJDeGwh3d3dt4uvB2Rd2dnaipKREEsvkyZP16v7ss88kZXQTEc8//7xJ/TTXWNa1/8XFxcLb29toO1ZWVmLGjBn1boeIiIiIiOhBGRkZ4tSpU2LHjh3i559/FidOnBBZWVm1qiM1NVVs375dbN26VRw+fFjk5eVJjt+6dUscPHhQqNXqWtWrVCrFqVOnRExMjLh69areZ5pjx46Jy5cvm1RXTk6OOHLkiIiNjRVpaWl6N9irU1P/zBFvXcdIiPs37ZOTk8XPP/8sfv31V3Hjxo1a9U+pVIq4uDixfft2ceHCBVFRUSE5fvLkSZGUlFTruMzZhlqtFhcuXBDbtm0TFy9eFJWVlXpl0tPTxaFDh4yOYUVFhYiJiRGbN28We/bsEQkJCQbH6e7du2L//v3i0KFDek/bqIrll19+ERkZGdX2uaSkRJw6dUr89NNP4uzZs3r3OGpSn+tqal+JGhITKK1QYyVQQkJCxHfffSeSk5PFqlWr9L4R/9JLL0nqmDZtmuT4wYMHtccKCwvFyJEjjd7wvXPnjvbRVVU/ixYtEmlpaZI2fv31V4M3dSdNmiTef/99MWPGDBEYGKj9BsbGjRvrnZDIzc2VzCQAIDp16iS2b98u0tPTxe7du8WYMWP0bjLfvn271v0zR7y1HaOioiK96cTz588X5eXl2uO619bLy0vbnkajEf369ZMct7GxEcuXLxdpaWkiLi5OzJgxo94JlKFDh+qd37lzZ7F48WKxePFiMWzYMDFw4EDtY6J0p7O+++672j4JIcRvv/2mV1/V47peffVVyf6dO3dqz6uoqBBOTk565+rOMHnrrbckxzdt2lRjH805lvXpv0ajEQcOHJAcs7e3F+fPn9d7o1WfdoiIiIiIiIiIiBqSNYgaQLt27bB79274+PgAAIKCgnD69GmsWbNGW+batWuSc/744w/Jdlpamvbfzs7O2LlzJ6ZOnYpdu3ahT58+kgXOXF1d9RZF8/b21rZfnSlTpmDLli2mdq3WNm3aJFm43MnJCXFxcejUqRMAoEuXLnjkkUcwfvx4HDx4EACg0Whw+fJluLu7A6hf/8yhujH65ptvkJmZqd3u27cvvvzyS1hZWQG4399vv/0WJ06cQGpqKoD7i9FfunQJPXr0wJkzZxAfHy+p86OPPtIufOfj44O1a9ciNzcXBw4cMFufrK2tsWvXLjz00EMGj8vlcu2/33zzTfztb3+THA8PD0dgYKDkdZuamorBgwfjySefxPLly7X79+zZgwkTJgAAfv/9dxQXF+u1V7XfyckJwP0F0qo4ODhg3LhxNfbJnGNZn/7LZDIEBQVJytva2qJXr15mbYeIiIiIiIiIiKghMYFCDeKNN97Qu7mvUCgkCZTCwkLJ8QkTJkhu/j777LP4+uuvMXnyZERGRmLgwIHYsWMHKioq0KZNG7PF+uqrr5qtLkN+/vlnyfaLL76oTZ5UsbW1xbZt2zBq1CicPHkSY8eOxZAhQxo0rtqobowSExMl2zNnztQmT6rY2dnhsccew4oVK7T7Dh06hB49euDq1auSsj4+Ptob/lWsrKzw1VdfoXv37nXtgp7IyEijyRMAWL16NVavXg0XFxcsXrxYu1+lUuHs2bP49ddf9V7Dd+/eBQCEhYXBx8cHN27cAADs3btXW+bBfz9IqVTi0KFDmDRpEm7duiUZ1/Hjx8PR0bHGPplzLOvT/9porHaIiIiIiIiIiIhqiwmUVsjGxkayrdFoqi1v6LjuDXJdhr5prjuDQqVSSbZnzJiBL774QnJz9Ny5czh37hzefvttuLq6Ijo6GrNnz8aAAQOqbd9UcrkcvXv3rnc9Qgijx6puoleJiIgwWM7JyQnHjx9HQUEBXFxc6h1TdaqLV1dNY5ScnCzZXrZsGTZv3qxX7vr165LtqtkoujORjCU1/P394eLigoKCApPirkm/fv2qPe7n54ePP/4Y+fn52LZtG06dOoW4uDjEx8ejrKzM4DlqtVr776eeegrLli0DANy8edc3fKsAACAASURBVBMXLlxAr169sG/fPm2Z5557DocOHdKOwb59+zBp0iTs2bNHUm9UVJRJfTLnWNa3/6ZqrHaIiIiIiIiIiIhqiwmUVsjNzU2SpLh161a15XNzc/X26SZDdDk7O+vtq2nWiL+/PxITEzFz5kzExsbqHc/Ly8OqVauwatUqLFq0CMuWLZM8/qcuPD09YW9vX686AKCiosLosezsbMm27uwTXQ2dPAGqj1dXTWP04KPWgPuP53rwkV7GDBo0CACQk5Mj2d+5c2eD5WUyGbp3744zZ87UWLcpunbtWu3x0tJSLFy4EOvWrTN5vGQymfbfDyZQgPuP8XJxccGlS5e0+8aMGQM7Ozt88cUXAKBNrjyYQHF2dsbYsWNNat+cY1nf/puqsdohIiIiIiIiIiKqLSZQWiE/Pz9cuXJFu607M0CXboJFJpOhS5cu1Z5T10dsde7cGTExMYiLi8PGjRuxa9cug/FVPQrqs88+q1M7VRwcHGpV3tDaFcD9m8DGuLu7Sx5BVNPjh5KSktC1a1ezJHbqEq+umsbIy8tLssZL7969jT4eyt7eHgEBAejfv792DRvdm/y6CacH6c7mqY+a+jVhwgQcPnxYss/T0xNDhw7Fww8/jPDwcHz33XdYvXq19viDN/b79++Prl27al+/e/bsQfv27bXHbWxsEBkZCQcHB20CJT09HfHx8Th06JC23MSJE2FnZ2dSn8w5lvXtv6kaqx0iIiIiIiIiIqLaYgKlFfL19ZVsx8fHo6SkxOgaC7preHh4eNSYIKnLDc6cnBxs3boVPXv2hEKhQFhYGD7//HOkpKRgx44dWLFihWRmw8aNG7F8+fJ63Uy1tbWt9rhu3VlZWXplNBqN3iyMBwUFBUkerXTu3DkMGzbMYNknn3wSW7duhbu7O86cOQNvb+9q42uIeHXVNEbBwcFISEjQbkdHR2PJkiVGyyuVSsl2t27dJNu6a6pUycrKMjgbqq6q69e+ffskN/XlcjnWrl2LadOmScb4k08+kZyn+2i7p556Ch999BEA4MSJE5JzhwwZAmdnZwwfPhyOjo4oKSkBACxevFiS4IqOjja5T+YaS3P1vyaN1Q4REREREREREVFd8C5UKxQUFCTZvnXrFubNm2ew7NGjR7F161bJPn9//waJa+zYsXjppZcwYsQIySOMunXrhtdeew3r1q2TlM/JyZHcaNZd28XY+gkPqumb/bqP07px44beN/e//vpr7c1vQ4KDgyXbK1asQGVlpV65d999VzvWubm5krUyANP6Z454ddU0Rrr9279/v9E1Vj755BM4OzvDw8NDOzND9/WYkpKC9evX65374YcfmhyzKarrV3x8vGRboVBg+vTpkpv6FRUViImJkZTTTVg9uHaJWq3G0aNHtdtjxowBcH+21siRI7X7H0wotGvXDqNGjTKlOwDMN5bm6L8pr1dzjTMREREREREREVFDYAKlFXrmmWfg6ekp2bd+/XoMGzYMf/3rX/Hzzz9j3bp1eOmllxAREaG3YPPTTz9t9piEENpZEUIIREVF4eTJk5IyuuuqhIaGSmbN6Pbp+PHjUKlUUCqVkkdo1YZunZWVlXjhhReQkpKC3NxcfPnll1i0aFG1dTzzzDOS2Q43btzAmDFjtDeB7927h08//RTvvPOO5LyAgIBqYzHUP3PEW1tz585F27ZttdtHjhzB66+/LilTVlaG5557Dm+88QaUSiUKCgpw8eJFAEBISAgUCoWk/LPPPosNGzagqKgIWVlZePfdd/Hll1+aNe7q6L7WEhISJAmAoqIijB8/Hnl5eZJyly9flmw/9NBDetexStUjzABg3LhxBstMnjy5xhlADzLXWJqj/x4eHpKZIkqlEnFxcQDuP8ZOCGG2cSYiIiIiIiIiImoIfIRXK+Tk5ISPPvoIs2fPluw/evSo5BvyhgwZMgSzZs0ye0wymQzR0dH4+uuvAQAlJSWIjIxEeHg4+vXrh9TUVOzfv19yTtU3+KvoJg92796N9u3bo7S0FHK5HBs3bkS7du1qFVevXr3QqVMnZGRkaPft27fP6BofhgQGBuKNN97Ae++9p9136NAheHl5oVOnTsjOztZLUoWFhWHo0KGSfab0b/LkyfWOt7a8vLzw/vvvY8GCBdp9y5cvx/79+zFixAioVCps3bpV8sgoDw8PREZGarffe+89hIeHa7fLy8sxY8YMyOVyaDQaozNaGsro0aMl23l5efD398fkyZNx584dHDlyRG/BdgDaBMGDoqKiJNceuL8OUWhoqHZ73LhxkMlkev2cOnVqrWM3x1iao/9yuRweHh6SdVgGDRoEJycnFBUVoV+/fnqzY+ozzkRERERERERERGYnqFXSaDTizTffFG3atBEATPp59NFHRU5OjsH6xowZoy1nZWVlsFxqaqqkvujoaMnxW7duiUGDBpkUS2BgoLhx44bk/LS0NOHi4mL0nBdeeEFcv35dsu+pp56qcaw2bdokrK2tjdbr6ekp5s+fL9l37tw5SR1lZWVi5syZJvctOztbLw5T+meOeOsyRmq1WixYsEBYWVnV2D8XFxcRHx+vV8cHH3wgZDKZ0fPc3NzE888/L9kXGxtbY2xVZs2aJTn30qVL1ZafPn16tf2wtrYWM2bMEM7OzpL9utfu6tWreuPy3nvv6bX38MMPS8p07txZqNVqk/v3IHOMpTn6v3DhwmrryMnJMds4ExERERERERERmRsTKK3c9evXRVRUlHBycjJ489LKykr06tVL7Nq1q9p61q9fL+RyuQAgRo8ebbRceHi4ACDkcrnYvn273vHKykrx9ttvi+7du+vddJbJZMLHx0csWbJElJWVGax/y5YtBpMMvr6+IjExUQghxODBg7U3Znfs2GHSOO3evVt07dpVL57p06eLGzduiGvXrgkHBwcBQAQEBBitZ+/evaJXr14GExy+vr5i1apVQqlUGj3flP6ZI966jJEQQpw4cUKEhYUJGxsbvRgdHR3FK6+8YjQJJ4QQO3bsEH5+fnpxP/HEEyIlJUXcunVL+1rt1KmTqKysNDm2PXv2aMe9f//+NZbXaDTitdde0xtvOzs7MWrUKHHhwgUhxP2EVVV/hwwZIjQajV5dDyYSQkJCREZGhl6ZI0eOaOOTyWRi48aNJvfNkPqOpTn6X15eLkaMGKH3WpDL5WLBggVmH2ciIiIiIiIiIiJzkgnRyM/GoSbr1q1buHz5MvLz8+Hk5AR/f3/4+vqavAZDamoq0tPTMWzYMMki0A8SQiAmJgbdu3dHly5dqq2vtLQUSUlJuHnzJvz8/BAYGAgHB4ca41AqlTh+/Dhyc3Ph6OiITp06ITQ0VBuTRqPB4cOHERQUhM6dO5vUtypZWVm4dOkSnJ2d0b17d7Rv3157LD8/HydPnkRkZGSNY6ZUKnH16lVkZmbC1dUVAQEBcHJyMimGmvpnjnjrM0YAoFKpkJKSgitXrsDR0VF7vY29LgzFffr0abi4uCAoKAgdO3bUHisoKMCxY8cwYsSIGhe415WRkYHk5GSMGDFCsj5HdTQaDS5duoSrV68iICAAgYGBsLaWPv3w5s2buHr1KhQKhcF6hRA4ffo0SkpKoFAojI5DXl4ejh07hv79+8PLy6tWfTOmvmNpjv6npaUhISEBarUaHTp0QO/evdGhQwezt0NERERERERERGROTKAQERERERERERERERHp4Fd4iYiIiIiIiIiIiIiIdDCBQkREREREREREREREpIMJFCIiIiIiIiIiIiIiIh1MoBAREREREREREREREelgAoWIiIiIiIiIiIiIiEgHEyhEREREREREZBKZTFbtT1pamt45S5cuNVp+6dKleuXT0tKqbcOQOXPm1KqN2NhYo+V9fX0NtqFQKIyes2bNGr3ya9asMVpeoVAYbMPX19foObGxsbUa2zlz5hhsg9eP14/XT2GwDV4/Xj9eP1+DbTS362dsrOqKCRQiIiIiIiIiIiIiImr2Zs+ebdb6ZEIIYdYaiYiIiIiIiKhFqvpWJ28lEBERUWtgbekAiIiIiIiIiKh5iImJsXQIRERERI2GM1CIiIiIiIiIiIiIiIh0cA0UIiIiIiIiIiIiIiIiHUygEBERERERERERERER6WAChYiIiIiIiIiIiIiImjWZTAaZTGbWOplAISIiIiIiIiKTLF26FEuXLrV0GERERESNgovIExEREREREZFJqr7VyVsJRERE1NQ0xPsUzkAhIiIiIiIiIiIiIiLSwQQKERERERERERERERGRDiZQiIiIiIiIiIiIiIiIdDCBQkREREREREREREREpMPa0gEQERERERERERERERHVR2pqqtnrZAKFiIiIiIiIiEwye/ZsS4dAREREZJCvr6/Z65QJIYTZayUiIiIiIiIiIiIiImrGuAYKERERERERERERERGRDiZQiIiIiIiIiIiIiIiIdDCBQkREREREREREREREzdrSpUuxdOlSs9bJNVCIiIiIiIiIiIiIiKhZk8lkAABzpjwaZAaKTCYz+pOWlqZXfunSpUbLG8oYpaWlVduGIXPmzKlVG7GxsUbL+/r6GmxDoVAYPWfNmjV65desWWO0vEKhMNiGr6+v0XNiY2NrNbZz5swx2AavH68fr5/CYBu8frx+vH6+Btvg9Wt514+IiMgY/q0gIiKi1oSP8CIiIiIiIiIiIiIiItJh9kd4VX2r09A3TomIiIio6ar6RjGf8EpERMbwbwURERE1VQ3xPsXsCRS+mSIiIiJqnvg+joiIasK/FURUGxkZGYiLi8PEiRNhbW1t6XCIqIVjAoWIiIiIGkzVDGJj68QQERHxMz8RVUcIgXv37uHq1atYvnw5zp49C41Gg8rKSnh7e0OhUOCRRx5B586d4eLiAmdnZ7Rp08bSYRNRC8EEChERERERERFZDD/zE5ExeXl5iImJwbZt23Du3DmEh4dj6tSp6NatGy5duoSEhARcuHABaWlpKCoqQteuXRESEoLAwEAEBgaia9eucHV1tXQ3iKgZW7p0qeS/5sAEChERERERERGZhJ/5iUiXUqnEjz/+iB9++AF3797F+PHjMXHiRAQFBcHJyUlStri4GLdv30ZGRgZSUlJw5swZnDlzBoWFhfDw8EBISAj69euH8PBw+Pn58bFfRGRxTKAQERERERERkUnS0tIAAL6+vhaNg4iaht27d+PVV1/FlStXsGjRIrz++uvo2LEjrKysajxXCAGNRgO1Wo27d+9i586d2LFjB/bt2weZTAZ/f3+MHTsW06dPR58+fWBjY9MIPSIikmIChYiIiIiIiIiIiGqk0Whw584dJCYm4rPPPkNSUhLGjRuHxYsXw8/PzyxtqNVqHDlyBIcPH0ZMTAwyMzMhhNDOTBkyZAg8PDzQtm1btG3bFnK53CztEhEZYvYESkM8Z4yIiIiIGh4XkSciIiIiY7KysnDgwAFs2rQJN27cwLhx4zB9+nQ89NBDDdamEAKpqak4c+YMEhMTcfnyZSQnJ8PNzQ2dOnXCQw89hICAAAQGBsLHxweOjo4NFgsRtU5mT6AQERERUfPEmcREREREpKugoABr167Fjz/+CLVajaeeegqjRo1CYGAgbG1tGy0OjUaDwsJCpKenIzc3FxcuXMDx48eRmpqKsrIydOnSBSEhIRgxYgRCQkLQpUuXRouNiJqGhnjUKBMoRERERASACRQiIiIi+i+NRoP169fj9ddfR35+Pj744APMnTsX7du3175vtCQhBNRqNQDg4sWL2Lt3L7Zu3Ypz586hTZs2CA0NRXR0NCIiItCnTx8LR0tEjaEhPtMygUJEREREAJhAISKimvGx3UQtm0qlwu3bt3HixAksX74cOTk5iIqKwhtvvAEXFxdLh2eSnJwcHDt2DPv378eZM2eQk5MDd3d39OnTB+Hh4Rg0aBCcnZ3Rrl07ODg4mLTgPRE1D0ygEBEREVGDYQKFiIhqwr8VRC3XtWvXsGfPHmzfvh2FhYWYNGkSoqOjERAQYOnQ6qy8vByJiYlISkpCQkICLl26hMLCQtjY2CA0NBQhISEICAhAt27d4OPjw2QKUTPXLBIosbGxAACFQmHOaomIiIiogfGmGBER1YR/K4hanps3b+Kf//wn9u/fD3t7e8yePRsKhQK+vr6wtra2dHhmUzW7Jjc3F+np6YiPj0dCQgKSkpLQtm1b+Pj4YODAgRgwYADCw8MbdX0XIjKPZpFA4ZspIiIiouaJ7+OIiKgm/FtB1HKUlpZi5cqV+OSTT1BRUYHVq1cjOjoaNjY2TWKNk4YmhIBGo9HOUlm3bh12796NW7duwd7eHhEREXjssccwZ84c2NjYWDpcIjIBEyhERERE1GCqZhBXzSgmIiLSxc/8RM1bRUUFsrKycPDgQXz11VcQQuDPf/4zFixYwMdX4f4sldTUVBw4cABHjx7FxYsXUVxcjI4dO0KhUEChUKBr165o37492rZtCzs7u1aRbCJqLphAISIiIiIiIiKL4Wd+ouYrISEBu3fvxvbt2+Ho6IjHH38cU6ZMQefOnS0dWpOVm5uLixcvIiEhARcuXMD169eRm5uLrl27Ijg4GMHBwQgKCoK/vz88PDwsHS5Rq8cEChERERERERFZDD/zEzU/ycnJWLVqFY4ePQovLy88/fTTePjhh+Ht7W3p0JqV0tJS5ObmIjMzE6mpqTh16hTOnj2L/Px8uLi4oHv37ggLC8Pw4cPRrVs3PvaLqIVgAoWIiIiIiIiITLJ06VLJf4mo6SooKMBbb72F7777Dm5ubvj6668xbtw4PnLKTDQajXYNlW3btuHnn3/G7t27oVQq0alTJ4wbNw7R0dEYMmQIF6QnasaYQCEiIiIiIiIiImoBSktLkZGRgf/85z/4z3/+g/bt2+OFF17A7NmzLR1aq6BSqRAXF4eYmBjExsYiPT0dpaWl6NevHx5++GEMGTIEnTp1Qrt27eDi4gJra2tLh0xENTB7AsXX1xcAkJaWZs5qTVZUVGSRdomIiIhqy9nZ2dIhEBEREVELcezYMezcuRO//PILPD09MWXKFEycOBFubm6WDq3VSk9Px9mzZ3HhwgUkJSXh6tWrsLe3R8eOHdG7d28EBQUhKCgIvr6+aNu2raXDJSIDzJ5AsTQmUIiIiKi5aGoJFEt/EYaIiIiIau/48eP45z//ifj4ePTs2RMzZ87EgAED0LFjR0uHRv9Po9GguLgYmZmZuHPnDhITE3Hq1Cn88ccfKCgogKenJ3r06IERI0agd+/e8Pf3t3TIRPT/mEAhIiIispCmlkDho1iJiIiImo/09HQsWbIEmzdvRt++fbFixQoMHjyYa5w0A0II7c/Vq1dx4MABbNy4EXFxcQCA0NBQPPHEExg9ejTCwsIsHC1R8zFnzhwAwJo1a8xWJxMoRERERBbCBAoRETU3VbMUq2YtElHjEUKguLgYKSkp+P7777Fnzx50794dL730EsaPH2/p8MgM7ty5g1OnTmH//v04ffo0cnJy4OTkhJ49e2L48OEYOHAgOnTogPbt28PJyQlyudzSIRM1KQ3xmZYJFCIiIiILYQKFiIiaG/6tILIMjUaDX375BT///DNOnjyJoKAgPPHEExgzZgwcHR0tHR41AKVSiStXriAxMRHnzp3DpUuXkJeXB41Gg5CQEISEhCAoKAgBAQHw8/PjgvREYALFJEygEBERUXPBBAoRETU3/FtB1Lg0Gg0OHjyIL7/8EikpKVAoFHjyyScxcODAJvdekhqOWq3GnTt3kJubi1u3bmnXUElOToaNjQ28vb3Rv39/hIWFISIiAm3atLF0yEQW0SwSKAqFAgAQGxtrzmpNxgQKERERNRdN7UMvb4oREVFN+LeCqPEkJydj/vz5iI2Nxfjx47Fy5Up07drV0mFRE1D1O7i8vBwpKSn49ttvsXv3bly7dg0ymQzh4eF48sknMWPGDLRr187C0RI1nmaRQLH0mykmUIiIiKi5YAKFiIiaG/6tIGo4QggUFBQgKSkJX3zxBU6ePIn+/ftjyZIlGDBggKXDoyZOrVYjOzsbe/fuxbFjx5CYmIiCggI4Oztj+PDhCA8PR1BQEFxdXeHi4gIHBwft73SiloIJFBMwgUJERETNRVNLoFTNIK6aUUxERKTL0p/5iVqqsrIy7Nu3Dzt27MC5c+fw8MMPIyoqCsOHD4eNjY2lw6NmKD8/H0lJSTh//jwuXLiAq1evIicnB506dUJAQAB69OiB4OBgdO/eHZ6enkymUIvABIoJmEAhIiKi5qKpJVCIiIhqYunP/EQtTWlpKbZv345vv/0WWVlZmDhxIiZPnozQ0FA4ODhYOjxqIcrLy3Hnzh1kZWXh1q1bOHHiBOLj45GTkwNHR0f4+/tj4MCBUCgUCA4O5hoq1GwtXbpU8l9zYAKFiIiIyEKYQCEioubG0p/5iVqSkydPYt68ebh48SLmzZuHd955B+7u7pYOi1oJIQSUSiV27tyJn3/+Gdu2bUNZWRlcXFwwduxYzJgxAxEREbC3t7d0qEQWxQQKERERkYUwgUJERETUuhQUFOD48eNYuXIl/vjjD0RGRmLhwoXo1auXpUOjVk6tVuPs2bOIjY3FkSNHcP36dRQVFaFXr17o378/wsPD4e3tjfbt26Ndu3awtbW1dMhEjYIJFCIiIiILYQKFiIiIqHU5cOAAPvroI/Tr1w9TpkxBWFgY156gJikzMxMJCQnadVSuXr0KKysruLq6IjQ0FMHBwejRowd8fX3RoUMHS4dL1GDMnkBZs2YNAGDOnDnmrNZkTKAQERFRc9HUEigN8bxYIiIiIvqv/Px85OXlwc/PD3K53NLhENVICIGSkhJkZ2cjLy8P58+fR3x8PC5duoScnBx4eHggKCgIERER6Nu3L4KCgiwdMpFZmT2BYmlMoBAREVFz0dQSKJaeSUxERERERM3HjRs3cPDgQfzwww84duwYNBoNfH19MWXKFEyaNAlDhw61dIjUysTGxgIAFAqF2epkAqUeLl++jLi4OFRUVMDOzg7Tp0/ntweIiIjIZEygEBERERFRS5Cfn4+zZ8/il19+QVxcHDIzM2FtbY0XX3wRL774oqXDo1aiIT7TWputplaksLAQhw4dwvXr17X7ioqKoNFomEAhIiIiIiKiFqvqcd1Vj+8mIiICgPbt2yMyMhKRkZFQqVS4du0aEhISEBAQYOnQiOqFM1BqQQiBhIQEHD16FJWVlXrHFyxYABsbmwZrn4iIiFoWzkAhIqLmhn8riIiIqKlqFjNQLL2IfEPaunUr0tPTLR0GERERERERERERERE1MCtzV/j000/j6aefNne1FqdUKrXJE5lMhj59+uCpp56ycFRERERERERERERERNQQuAaKieRyOWxsbODp6QmFQgF3d3fk5ORYOiwiIiIis5k9e7alQyAiIiKiVkatVkMIAblcrn38TnOnUqkgk8m4VjJRC8AEionkcjkWLFhg6TCIiIiIGgwXBCYiIiJqmiorK7U35FtKkqGkpATZ2dlISUmBUqlEly5d4OPjAxcXF1hZmf2hOQ1Oo9GgoKAAGRkZuHHjBmxtbeHn5wcvLy/Y29u3mOtG1JT5+PiYvU4mUIiIiIiIiIiIiJqg0tJSZGRkIDExETKZDEFBQfD29oazs7OlQ6uXnJwcfPnll9i1axf8/f1hZ2eHq1evwt/fH2+88QYeeuihZpVwUKvViI+Px8cff4w//vgDvXv3RlFREdLT0zFq1CgsXLgQnTp1snSYRC1eWlqa2etkAoWIiIiIiIiITPLOO+9YOgSiVkGtViMuLg6rV6/GgQMHUFRUBJlMBjs7OwwfPhwLFixAWFgY7O3tLR1qrQghcOXKFaxcuRKenp7YsWMHPDw8IJPJUFpaigMHDmDZsmWYOnUqRo8eDVtbW0uHXCONRoODBw9iy5YtiIqKwpgxYyCXyyGEQF5eHn744Qd8/PHHeOaZZxASEtIsZ9eoVCqUl5ejoKAAMpkMzs7OsLe3h7U1by1TyycTQgizVvj/2WEzV2uyoqKiRmsrJycH69ev124vWLAANjY2jdY+ERERNW/N/ZuDRERERGR+Go0GcXFxWLRoEU6dOqV3j00mk8HX1xeffvopJkyY0KzuRd25cweffvop2rVrh5dffhmOjo6S4xqNBrGxsdi4cSOee+45DBgwwEKRmi41NRWff/45Ro8ejUceeUTveiiVSqxbtw6ZmZmYP38+XF1dLRRp3RQWFuLo0aM4duwY0tLSIJPJ0KlTJwwaNAgjR47kZxpq8cyeJhw+fLi5qyQiIiIiIiIiImoVMjIy8Pbbb+P06dMGv6AshEBqaipWrlyJ7t27o1evXhaIsvY0Gg2SkpJw8+ZNLFmyRC95AgBWVlYYPnw4EhMTcfr0aQQHB8PJyckC0ZpGpVLh8OHDcHJywogRIwwms2xtbREREYFvv/0Wly9fxpAhQywQad0UFhbi888/x5o1a5CZmQmVSgXg/lrRmzZtwhNPPIHXXnsNXl5eFo6UqOGYfc5YbGwsYmNjzV0tERERETUwmUzWrJ41TURERNTSaDQanD17FocOHYJGo6m2bFJSEk6cOIGKiopGiq5+ysvLcf36dfTq1Qvt2rUzWk4ul6Nv377Iy8vDvXv3GjHC2svPz8elS5fQu3dv2NnZGS3n7u6OLl264MqVK40YXf1UVlbi+++/x7vvvovU1FRUVFRArVZDrVZDqVQiPT0dK1aswFdffYWSkhJLh0vUYJrfQ/eIiIiIiIiIiIhaoLKyMiQmJppcNjk5udncvNZoNCgvL0fbtm1rLOvk5ASVSqWd8dBUKZVKlJeX1/gYK7lcDltbW5SWljZSZPWXmJiI5cuXQ61WV1tu06ZNiIuLqzHhR9QYFAoFFAqFWetkAoWIiIiIiIiITMKnThA1LI1GY3JCRKPRoLS0tMYb3E2FtbU1CZJT1gAAIABJREFUnJ2dkZGRUWPZ27dvw9bWttpZHU2Bvb09nJ2dkZ2dXW25iooKFBQUoH379o0UWf2Ul5fj9OnTyMrKqrHsnTt3cOrUqWaVHKKW68iRIzhy5IhZ62QChYiIiIiIiIhMEhERgYiICEuHQdRiWVtbo2PHjiaXdXNzg62tbQNHZR5t2rRBt27dcPnyZaSnpxstp1KpcOLECfzpT39q8gkHFxcX9O7dG4mJiSgoKDBaLjs7G7du3UJISEgjRld3SqUSaWlpBtfg0VVZWYmbN2+ivLy8ESIjanxMoBARERERERERETUB9vb2GDBgAKytrWss6+DggD59+jTpRdYfJJPJ0LNnTwQFBeHvf/87iouL9coIIbBp0yZcu3YNgwcPRps2bSwQqenkcjkiIyNRWlqK7du3G3zkmFKpxLZt2+Dm5gZfX9/GD7IOTEmcPIjrKFJLVvNv41qq+kWQlpZm7qqJiIiIiIiIiIhatJ49e+K5557Dv//9b6MLxFtZWWHkyJEYOHAg5HJ5I0dYd87Ozpg3bx6WLVuGDz74AKNGjYKPjw+sra2Rm5uLkydP4vTp03jmmWcQHBxs6XBN4uHhgenTp2PDhg0oKSnB4MGD0aFDB6jVaqSnpyMmJgbl5eWYN2+eSeu/NAX29vYICAiAlZVVjY+Ia9OmDfz9/eHg4NBI0RE1LrMnUG7cuGHuKomIiIiIiIiIiFqFDh06YPHixbh37x52796NwsJCyYwAJycnDB48GIsWLYK3t7cFI62bLl264K233sLOnTtx6NAhaDQaCCFgZWWF9u3b45VXXkFoaKilw6yVIUOGwMnJCYcOHcLmzZu1+62srNClSxeMHz8enTt3tmCEtWNra4uwsDAEBAQgKSmp2rJdunRBWFgY7O3tGyk6osYlE7Wdk1VThf8/ZcvM1ZqsqKioweo+cuQI8vPztdulpaWSxZT8/PxgZXX/qWhyuRwjR47kLw8iIiIyytnZ2dIhSFTNIG4ujxYgIqLGZ+nP/EStSXp6OrZt24adO3fi1q1bAICOHTtizJgxmDp1Kry9vZvV7JMqQggUFRXh7t27yMjIQF5eHjQaDVxcXODp6QkPDw+0bdvWpMeYNRUqlQoFBQXIzc3FrVu3UFJSArlcjg4dOqBTp07o0KEDHB0dtfcNmwOlUonNmzfj1Vdfxe3btw2WcXJywrvvvovnn3+eM1CoSWiI9ylMoNTCypUrDT7L0JhZs2bB3f3/2Lvz+Kiq8/Hjn9knmcm+b0BCSEIIJIQECFtYZVUQBRVtC361VX9qrV3Uai3Uamut+rVq3VBR6wYiWFFZZUcgkBCyA9nIvk2SmSST2X9/8Jr5MiRA0EBAz/sf8c655557J4R7z3PP8wRdtvEIgiAIgnBtu9oCKIIgCIJwMQP9zC8IP1Xt7e1IJJJrJgVUb4xGI6WlpRQWFpKTk0NNTQ06nc5VC8XT0xM/Pz8iIyOJi4sjKSmJuLi4q7qQfFNTE6dOnSIvL4/CwkIaGxtpbW3FZDIhk8nQaDQEBAQQHR1NQkICCQkJDBs27Kqv7XK2NWvW8NRTT1FeXu72uz8qKoqHH36Y++67D6VSOYAjFIT/s2bNGgCWL1/eb32KAMoleOedd9xWoFyIVCpl+fLlV/UveUEQBEEQBpYIoAiCIAjXmqu17qnNZqOuro7W1lbXfIREIkEmk6FWq4mMjLzkCb6Ojg6ampoIDQ11yy5RWlqK0WgkOjoajUbTr+fh1N3dTX19Pf7+/hedMLfb7VRUVNDY2MjIkSMv25guxmAwcOzYMQICAkhISLim3rS/mtjtdrq7u+nq6uL06dPU1NTQ2tpKV1cXcKbehJ+fH2FhYQwaNAhvb2/UavVVuxLFYrHQ2NjI4cOHycvLo6mpCbvdTllZGcnJydxxxx14eXkhkUjo6OigubmZsrIySkpK0Ov1+Pj4MGzYMMaNG8eQIUPw8PAY8ILl3d3dlJWVcejQIU6cOEF7eztGo5G6ujruvvtukpKS8PDwwGazodfrqa+v58SJE5SVlWG1WgkKCiI5OZnU1FRCQ0NRKBQDej4XY7VaOXr0KFu3bnX97o+MjGTWrFlMnDhxwL8PQbjcRABFEARBEARhgIgAiiAIgiD0j7q6Ov7nf/6HvLw8t3oKarWagIAAFi5cyPLlywkPD+9TfzabjQ8//JDXX3+dpUuX8tBDDwFn5hwmT55MZ2cnL7zwAtdff32/n4vD4WDnzp386U9/Yvbs2Tz55JMXbN/Z2cmKFSvYsGEDGzZsYMGCBf0+pr5Yu3Ytt9xyC2PHjmXjxo2EhYUNyDiuZW1tbZw8eZK9e/eSlZVFQ0MDXV1dnDx5kra2NhwOB2q1Gl9fX8LCwoiJiWHUqFFMmjSJkSNH4u/vf1UFrjo6Oti3bx/bt29HqVQyatQoxowZw9ChQ1m/fj2FhYX8+te/xtfXt9f96+rqyMnJ4dixYzQ0NJCSksLcuXMJDQ29wmdyhsPhoL6+nq+//pq8vDxCQkIYPXo0Y8aMQSKR8Oqrr5Kens68efN63d9qtVJaWsrRo0cpLCzEYrEwY8YMMjMzr+oVKUajEaPRiMlkorOzE4fDgUajQalU4uHhgaenpwiiCD9q104yQUEQBEEQBEEQBEEQhF60tbWxefNmHA4HoaGhBAYGotfrOX36NKWlpeTl5dHZ2cnjjz/epzz93d3dHD58mO+++w6DweAKoEilUvR6PTab7bJNeDocDnJzczlw4ABtbW0XDaBYrVbKy8uxWq2cOnXqsoypL5zHbmlpoampSQRQLoHZbCYvL4+PP/6YLVu2UF1dTVtbGwsXLuTRRx/lxRdfZPPmzcD/TWY7gwubN2/mvffeY9y4cSxfvpyMjAy0Wu0An9GZv0Nffvkl+/btY/78+aSlpREYGOgK8IwZM4asrCxOnTpFWlpar32EhYURFhbG5MmTKSkpYcOGDaxevZpf/epXVzxlvsPhoKamhjfffBO9Xs/tt99OQkICWq0WiUSC2WwmOTmZnTt3njeAIpfLiY+PJy4ujsbGRg4ePMhnn31GTU0Nt9xyy1VVQ8RkMlFbW+sK+BQUFFBXV4derwfO1D4JDw8nISGB0aNHExsbS2Rk5FV1DoLQX/o9gLJz587+7lIQBEEQBEG4AlauXOn2X0EQBEG4VjgcDlcmjJdffpmbb74ZgLy8PNLS0jAajWzfvp3ly5czbNiwi/YnlUqx2+3AmYlgJ41GQ1lZ2WU4g96PbTKZ+rSPs73zvwPBef0dDseAjuNao9fr+fTTT/njH/9Ic3Oz22dGo5GYmBiWLl3qCqCczeFw0NHRQUdHB6WlpWzevJnf/va33H333QNekzc/P5+vvvqKxx57jBEjRvT4PCIigvDwcHJzc0lOTr5gGisvLy/S0tIYOnQoq1at4pNPPuH//b//d0VX23R3d7Nu3TpsNhv/+7//2+NzpVLJyJEj+frrr8nOziY1NfW8fUkkEkJCQli4cCHx8fE88MADhIeHM3v27Mt5Cn1WWlrK119/zYcffsihQ4cICQkhPj6e48eP09bW1us+SUlJLF26lEWLFpGYmHjVppQThO+j33/TTJ06lalTp/Z3t4IgCIIgCMJltmrVKlatWjXQwxAEQRCEH8Rqtbr+PHLkSK677joAmpubMRgMmM1mvvzySxYvXkxcXByhoaGMGDGCu+66i6ysLAD+/e9/uyasa2trueWWW9ixYwcmk4m5c+cyefJkjhw54jpOc3Mzb731FpmZmURERDB8+HBWrFjBvn37XG2ysrLIzMzkjjvuYPv27SxevJjo6GiSk5N57rnn0Ol0AKxevZr33nsPOJPCaNGiRXz00UeXdA1sNhulpaU8+uijjBo1iqioKFJTU3n88cddK0WKi4tJT09n6dKlbvVem5qaWLZsGT/72c8wm804HA5qa2t57rnnSEtLIyoqiqSkJB544AHy8vIGLIX7j4Fer+eNN97gscce6xE8AcjNzSUrK4u0tLQ+Bf50Oh1///vfeeGFF1w1UwaC2Wzm0KFDJCUlkZiY2GsbhUJBWloahYWFtLe396lfPz8/Jk2aREVFhevvy5XS1NREVVUVN91003nbREREkJqayrvvvuv2e+hCEhISmDRpEuXl5RiNxv4a7iVzphd74YUXuP3223n44Yc5dOgQALNnz+aDDz5g2bJl503VlZ+fz5NPPslNN93EU089RUFBAWaz+UqegiAAZ4rIOwvJ95erJzGiIAiCIAiCIAiCIAjCD3T2W+kFBQUcP34cOLN6RK1W89lnn3Hbbbfx1VdfERQUREZGBgaDgbfffpsHHngAOBNccL5p7SwYXVtbS319PZs3byYnJ4ejR4+6jvPMM8/wwAMPkJubS2pqKiqVijVr1rBs2TIOHDgAwNGjR8nOzubDDz9kxYoVHDlyBJPJxPHjx1m5ciWffvopNpuNgoICGhsbgTMrUCorKykvL7+ka1BaWsqvfvUrnn32WXQ6HSNGjKC+vp5nnnmG5cuXc+rUKSoqKigsLGTLli1s2bLFte+hQ4f4+OOP2b17NzabjdbWVh555BH++Mc/Ul5eTlJSEhaLhVdeeYUbbriB7Ozs7/EtCXAm7dl//vMfWlpaev28ra2NnTt34uvry+23396nFRcGg4F169aRl5fX38Pts5aWFnJzc0lPTz/vhLtUKiU2NhaVSuWaqO8LlUqFTCbrc4Civ1RWVuLp6XnB+itqtZoxY8ZQW1tLUVFRn/t2pr0aqJVbVquVw4cP8+ijj/LEE09w6NAht+t7/PhxamtrmTNnzkVTdJ08eZJnn32W3//+9+zZs0cEWIUrbsWKFaxYsaJf+xQBFEEQBEEQBEEQBEEQ+uRayDrx4osvcvPNN3PdddexZMkSqqqqkEqljBkzhqioKL744gssFgt33nkna9as4Y033uBvf/sbMpmMrKwsmpqaePzxx5k1axZwpg7Dxx9/zI033ojNZgPOTHRaLBYAsrOzeeutt5BIJPzzn//krbfeYvXq1cyYMYOqqiruu+8+txRjALGxsXz22Wds3LgRX19furq6OHjwIB0dHTz++OPccccdAISGhrJ+/Xruu+++Pp+/1Wrl888/Z8+ePfj4+LB+/XreeecdXnvtNby8vMjKyuKDDz4gIyODoKAgOjo62Llzpytd2Pvvvw/AvHnzUCqV7Nixg7Vr16JWq1m7di2rV6/mzTffZMSIEVRUVPCPf/zjh39pP1FlZWWcOHHivJ+bzWb2799PQ0MDc+fOJTo6uk/91tfXU1NT01/DvGRNTU1UVFSQnJx8wXYBAQEkJyfzzTffXHJA5EoXLa+ursbHxwc/P78Lths+fDgjRoxg165d10zwoK6ujldffZX//ve/va6CKS8v5+DBgyQmJpKZmXnR/rq7u9m2bRsvv/wytbW1l2PIgnBFiQCKIAiCIAiCIAiCIAh9snv3bnbv3j3Qw7iggwcPsn79erZt20ZRUREOh4OJEyeycuVKvLy8ePnll/nggw945ZVXCAkJYe/evaxbtw44Exiprq4mICDANVGqVquJjY09b2HuL7/8ko6ODlJSUpg4cSK+vr4kJiYyceJE5HI5ubm51NXVudpLpVJ+85vfMHbsWMaOHesK1LS1tWGxWPD393fVr1CpVMTExODn54fNZuPrr7/mpZde4l//+herV6+mqampx3hMJhOHDx/GYrGwePFiUlJS8PPzY/78+URGRmI2mykoKEChUHDnnXdit9vJzs6moqICo9HIunXrUKvVLFq0CJlMxu7duzGbzWRmZjJu3Dj8/PyYMGGCq65Ffn5+n1MwCf/HYrGQm5vrVmPnXA6Hg6qqKrZt28bYsWOZMWNGn/q22+2uYN+VZrPZKCkpITg4mMDAwAu2VSgUJCcnYzAY+rwKRS6XI5VKr2hwwmKxUFVVRXh4OGq1+oJttVot06ZNo7S0lKqqqj71L5fLr3hA6GwlJSVs2bLlvCm3Ojo62Lp1KwqFgsWLF/epvonVamXv3r3k5+f393AF4Yrr9yLyovioIAiCIAiCIAiCIAgDZcmSJYwcORK5XE5QUBCxsbFMmTLFlf7IaDRy+PBhnnzySaqqqtBqtRedFL0QZ6qeI0eOMH78eNdEqEQiwdvbm8DAwB7Bl5CQENeffX19AXqsUjmXs3D22cGKLVu28O6777q1s1qtrre+P/roIz7//HPXZxKJhKCgIAYNGoRcLue+++7jn//8J4WFhRQXF7N+/XoAxo0bx/DhwwFc6cO2bt1KZGSkW1/+/v7ExMTg4eHRl0slnKW9vd2V3u1CDAYDmzdv5v7772fq1Kls3LjRleLtamQwGDhy5EifC6JHR0eTmJjIBx98gFwuv2AdEJlMxrFjxzAajVe0gHxTUxMNDQ19DmClpaWxadMmNm3axKhRoy64ukapVFJcXMyoUaP6a7iXxOFwUFpaet40cnAmKJafn09BQQFTp05l7NixfPfddxftu7u7W9RBEX4U+j2A4iw8KgIogiAIgiAIgiAIgiBcaTfffDNLly7t9TOj0eiqSzJ8+HD+8Ic/MHLkSMxmM3fffTcdHR2X/Ca4syZCZGQk9913n2v1iFQqpa2tjaSkJLy9vd32USgUrj/39XhjxozhzjvvpL6+HolEglKpZN68eT3qJshkMnx8fAAYPXo09957r6uNXC6nra2NmTNnolQqCQwM5NZbb+Wtt95i48aN7Nq1C6VSyeTJk13BkoCAAADi4uJ4+OGHXRPXMpmMhoYGpkyZglKpvKRrJkBzczPHjh3rU9vKykqysrKYMGECixYtctX16e1np7Ozk9ra2gFLH6XX6ykuLub+++93297W1obJZHILHtrtdioqKqipqaGqqorNmzdfcHWDVColPz/f9fN9pTQ0NNDe3s7QoUNd19xms9HY2Iifn59bANZsNnP8+HHq6upobW11BSZ6+64cDgcKhYKysjLXiq4rzWAwcPDgwYu2a2xsZNOmTbz88svMmDGDrKysK16HRhAGSr8HUARBEITvLyMjA7PZfM3kShV+XEQBUOHPf/7zQA9BEARBEH6wCxViPn36NAcPHkShUHDjjTfyxz/+EYVCwSuvvEJHRwcAtbW1xMbGugIFzmLy5zN79mxeeukl2traGDVqlOvN+6NHj/KnP/2JDRs2MH36dNc9/oUCJs7PnJPIzjEBJCQk8NRTT7lqlcjlcrRaLQaDwa1PpVJJUlISW7dupaamhjlz5hAcHAzAp59+ypo1a+jq6iIhIQGAO++8k/fff5/169fT0dFBaGgoCxYscI1h/PjxfPDBBzQ3N5OZmcnQoUMB2LZtG//61784efIkY8eOveA1EtxZLBaOHDmCXq+/aFt/f38WLlxIdHQ0kZGRrFy5kq6uLrc2Z3//RUVF/PWvfx2QguQ2m42Kigp8fHyIiIhwbbfb7WzZsoXu7m5+8YtfAGeCBxUVFbz33nsMGzaM+++/H41Gc8GVJRKJhO3bt3Ps2LEr9sxsMpnIyckhPDzcLXDT3NzMhx9+yOLFi4mJiQHOnOeBAwd44403WLx4MWlpaahUqgv+nZdKpRiNRreg6pWk0+nYv3//RduZTCb27t1LSUkJM2fO5LPPPqO4uPgKjFAQBp4IoAiCIFxFzGYzr7/+ep+LAwqCIPQnsYJYEARB+LELDAzEbrdjt9v57LPP8PLy4sSJE25psPbu3cvMmTPRaDQAtLS0kJ6eTkREBPfeey+A2+Tt3LlzmTJlCnv27OGWW27hwQcfxGq18vrrr9Pa2uoKLjj3OXfi1znR7UzhJZFIXBO1jY2NpKen4+fnx8qVK5kwYYJrXOfrQ6lUctNNN7F27VqqqqoYN24cv/zlLzl69Cjr169HLpezaNEi177Dhg1j2rRpbN68GThTBHvcuHGuzxcuXMjrr79OXl4eY8aM4aGHHqKxsZH33nuPrq4uEhMTez0v4fy6urrYsmXLRYMcSqWS66+/nieeeMK1siksLOyC+7S1taFQKAbk+zCbzeTn55OWluYWCGlpaSE7O9vt58pisbB9+3Y8PDz41a9+1edVTD4+Ple0Boper2fnzp0sXbrULRBSWVlJS0sLcvn/Ta3W1dWxdu1aFixYwJIlS/p8DE9PzwGrgVJdXU1ZWVmf2nZ2dlJWVsbUqVN57LHHOHnyZK/jlkgk1NfXs3379gEJ5Ak/bZmZmf3epwigCIIgXEUcDgcREREigCIMCIPBMNBDEARBEARB+F48PT1JTExEp9O5Vlb0JiAggI8//pg///nPtLS08NJLL+Hl5cU999xDd3c333zzDV1dXSiVSjIzM9m2bRuVlZWUl5fT3d2Nr68vUVFRKJVKwsPDXf2++uqrrFy5kpycHF577TUkEgl+fn7MmDGDt956CzgTqIiIiOiRxmjSpEls2LCB6Oho19vqY8aMYfz48Zw8eZKysjKCgoLOuxJGoVCQkpLC6dOniY2NBWDs2LG88MILPP/889TW1vL888+jUCiIj49n/vz5PPzww679fX19ufXWW8nPz8dqtfK73/3Orf+wsDBeeeUVnnnmGYqLi3n11VeRSqVERESQnp7OSy+9BEBSUhIBAQFERkZetHj4T53BYGDfvn09JpclEolbYMDLy4uZM2de8vV0BuKutO7ubk6cONEjeNDQ0EB+fj433XSTW9vTp08zbdq0S1p9YbfbkUgkfSpk3h9aW1vJycnh73//u9v28vJybDYbnp6ewJlrXl1djclkYtmyZZd0DOc5XenvrKuri2+//faiQQ6pVMqIESN47LHHmDJlChqNhp///OcX3Cc7O5uioiIRWBWuuF27dvV7nyKAIgiCcJVyOBxYrVasVqu46RAuC5lMhlwuv2IPH4IgCIIgXPvOLVh+tQgPD+eNN97AYDCQnJx8wba33norw4cPp7CwEI1Gw+DBg0lOTqauro5FixaRkZEBwJw5c4iIiCAvL88VfEhKSuLjjz9Gp9O5veWalJTE22+/TWFhIadPn0YqlRIVFcXo0aNRqVQATJw4keeeew5PT0+39EY33XQTnp6eJCcn4+XlBUBycjLvvPMOx44dw2azER0dTWpqaq/n4+Hhwe9+9zsWLFjArFmzgDPpvZYsWUJGRgbFxcU0Nzfj4eFBTEwMI0eOdNtfJpNx4403EhwcjEQi6VH8WyqVMmXKFNc1q6+vRy6X9xjTrFmz+Ne//sXgwYPdis0L7ux2OydOnOhRtFupVOLp6ekWKPP09MTf3/+SJtatVit2u/2Kp4RyOBw0NTVht9vdvn9nkfLy8nK3MXV1dWG32/H29r6k82tpaUGlUvW6Equ/2Ww2Tp06RWRkpGsFEJw5p5KSEtra2lxjt9vttLe34+Pjc8mBkLq6Oldg9krq6Ohgx44dF51v8PDw4JZbbmHhwoWugNHFOFf6CcKPgQigCIIgXKWsVislJSXk5ORgNBoHejjCj4zD4SAyMpKUlBS3B3hBEARBEIQLWb58+UAPoVdKpZJJkyb1uX1ycnKPQEtYWBgLFy50/b9cLmf06NGMHj3ard3EiRN77dPHx4eMjAxXAOZcGo2G66+/vsd2rVbb4419mUzG8OHDGT58+EXPRSKRkJiY6EqldbbIyMg+BTO8vb2ZO3fuBdsEBQVdMDWKRqO55Dfvf4rMZjOHDh3CbDa7bff19SUxMdHt7WmpVHrBmiDnstvt1NXVuVYIXUnOmiaBgYGuQCCcSfvU22obu91+yefX0NBAcXExqampfZ7I/yG6u7vJyclh4cKFbi+dtbS0kJOT0yOo4nA43FJ69cWxY8eoqakhNjb2kvf9oRobG3vUMXGuhDn7+/Ly8iIpKemSrnlfaj4JwrWi3/9mOotBCYIgCD+M1WqlqKiITz/9lNbW1oEejvAjY7fbSU9PJyQkRARQBJeKigoAhgwZMqDjEARBEARB+LEym81kZ2f3CKAEBAQwfvz4HulnLiUbQVtbG+vXrycqKuq8K5Yup9OnTxMSEuIWQNHr9ezbt4+QkJAe5yKXy10rtC6mo6ODN998E7vdzpw5c/p13OfT3d1Nfn4+zz//vFugp6GhgYKCAhYsWODWXiKRXNLKn/z8fJ599lkyMzOv+PdlsVg4duwYnZ2dbts1Gg0BAQFUVla6tqnVajw8PC6pf5PJhM1mQ61W98t4BWEg9XsAZc2aNf3dpSAIwk+Sw+HAZDKh1+tpb28f6OEIPzJ2u52uri4sFstAD0W4ijjrL4m0gYIgCIIgCP3P4XBQV1dHTU2N2/2WRCJh2LBhREVFubW32Wy0tLRQWVl5wfszh8NBW1sb7777LhUVFfzhD3+44umgDAYDDQ0NpKSkuFZSOBwOamtrsVgsjB492m1Vg1QqRafTsXr1aoKDg897fhKJBJPJhE6nIzg4mLvuugsfH5/Lfj52u53Kykq8vLwIDQ11raRwZorQarUEBwe7jVMul1NUVMTzzz9Pd3f3efuWSCQYDAYMBgMzZsxg6dKlV/z7MplMHDlypEcgLyQkhLS0NLcAyqVyrtzx8fFh6NChP3SogjDgRAovQRCuKZWVlRw9epTs7GxKSkp44oknLprjWBCuJWq1GpVKRXd3NyaT6bIe69wilYIgCIIgCIIgXD52u53i4mJqa2vdtmu1WmbOnNmjNmFTUxPPPfccb7/9dq/37WdP6lssFsLCwnjwwQeZPn365TuJ82hpaUGn07mtZDaZTBQXFzNy5EhUKpXbOfj6+rJ06VLq6+svmsZLIpEQEhLCsGHDCAwMvCJpoZwZIeLi4tzG197ezqFDh0hOTkYul7vOSSqVkpiYyPLly7HZbBd9zlKpVERGRhIXF3fJqzv6g8FgoLi4GKvV6rY9PDyclJQUPv30U7ftfa2babVa2bVrF59//jkzZswgJiam38YsCH3h/B3kzK7QH0QARRCEq1ZdXR3Z2dnk5ORw9OhRcnJy0Ol0wJn1GuhbAAAgAElEQVRlpevWrRPBk350uSfTnTe5l3oMZw5WqVTqKkTn/H9nf30tTnf2fna7fcCDB+dec4fDQWxsLLGxsRQXF1NUVHRZHw6c11IQBEEQBKGvnFknrtZaKIJwNXMWJW9qanLb7u3tzYwZM9i9e7fbdpPJxPHjxy96z+5wOLjxxhv5xz/+QVRUVJ/TYvWn+vp6jEYjgwYNco3XaDRSXFzMxIkTOXbsmFt7lUrF+PHjL/os19rayqFDh4iIiHCrOXK5WSwWysrKGDVqlNv2lpYWjhw5ws0334xer3dtl0gkBAcHM3fu3Is+Z1ZVVVFeXs7gwYMHJHhit9spLy/vsRJKqVSSmpraY4WPwWDg66+/pqKi4oLnZrVaqaqqYseOHYwaNYpbbrnlkmrcCEJ/+CGrp85HBFAEQbgq6HQ6t0BJdnY2dXV1vbZ1Bk8upUjkT5lUKkUmk+FwONzeLnFOnqvVajQaDQqFgpaWFldKJ2dBP+ckv81mc+0rl8tdheUuFIhwblcqlXh7eyOVStHr9a7lzH15EIiKimLJkiXMnDmTN998kx07dhASEsLtt99OZGQkmzZtYuPGjRftSyKREB8fz29+8xsMBgOffvophw4dGrAbOg8PDzQaDTabzVXjxtPTk4ceeoiUlBS++OIL3n//fSoqKkSQQxAEQRCEq8aKFSsAEUARhO+jo6ODkydP9kjvFBISQlxcXI/6J059efErISGB2NjY/hjmJXM4HDQ0NKBWq9Fqta7t7e3t6HQ6brjhBo4ePeq2jzPl1cXodDo2bdqEr68vgwcP7vexn4/JZKK9vZ3AwEC37dXV1XR3dxMXF8fhw4fdPnM+Q19MZWUlO3fuZNiwYfj6+vbruPvCbrdTUlLSY87F29ub6dOnU1VV5ba9paWFF154oU99R0VF8fDDD7Ns2TK3FGeCcC0TARRBEAbE6dOn2bhxI9nZ2Rw9erTPEWIRPOk7h8OBt7c3U6ZM4Re/+AUGg4F77rnHFSAZNmwYM2fOZNKkSa5l0B0dHezevZsvv/wSgEWLFpGWlsbWrVt55513kMvlaDQa7r77bjIyMti3bx/vv/9+rzVaJBIJgwcPZubMmcyYMYOAgABXbt49e/awdetWSktLL/owoFQq8fLyIiQkBE9PT2QyGSqVCh8fH3x9fdFoNH26HhKJBJVKRUBAgCtwNJBmzpxJZmYmJSUlvPnmm0gkEsxmM2VlZQQGBlJXV0dnZ+dlDZ5IpdIBX4UjCIIgCIIgCD8Vzc3NlJSUuN2Dq1Qq0tPTf/C9+bmpmK4kZ4otvV5PbW0t4eHhWCwWjhw5QkhICBqN5ns/18hkMtRqdZ+CLf1JpVLh6+vLiRMnGDduHDKZjKamJvbs2cOtt976g17Ek8lkKJXKAXtRzmw2U1paSltbm9t2f39/xo4d+4Pe4F+6dCn333//Ff++BOFy6vef5u+bokUQhJ8WuVzO008/jdFo7PM+InjSkzMQEBAQgJ+fHyaTiaamJgwGAxaLBalUilarJSIiwi3IER0dzY033si0adNwOByuN0wSExMJDQ1FrVZjt9vx9/cnPDzctYTX4XAglUpd2/38/HrNhepMRbVgwQJXLt/S0lJUKhVDhgwhMjLS9WaSQqHA29ubwMBA1Go1bW1tNDc3YzQaXate5HI5CoXC7SbVue1sUqmUoKAgAgMDkUqltLW1odPp6OzsdF0vhUKBh4cH/v7+DBs2DKVSSUtLC83NzW7Lx51tnOeu0+lobm525bPVarVotVqsVit6vZ6QkBDCwsIoKSmhu7sbHx8fAgMDkclk6PV6GhsbMRqNyGQyBg0aRFpaGqNGjaKrq4vBgwfT3d1NY2Mj27dvp7CwkLKyMiwWCwEBAWi1WoxGI01NTTgcDuRyOX5+fnh4eNDZ2YlOp3MFzAICAvD09KSzs5OWlhY6OjouuEJIrG4RBEEQBEEQhIGjVqsZN27cNZ/qKC4ujsjISF5//XWSkpLQ6/UcP36chQsXolarsVqt32uu0JlJ4UrPM6rVaqZOncpHH31Ed3c3/v7+rqDDb37zGw4dOtTnVNLnOjfDw5Wm0+koKipyG79MJnPVmPkh19rb21sET4QfHfETLQjCgAgPD+eee+7hxRdf7FN7T09PETw5h1QqJTo6mkmTJpGSkoKfnx8Wi4XS0lK2bdtGYWFhrzc+UqmUMWPGkJGRgcViYfPmzRw6dAiA0aNHU1paSkNDA/7+/hcMil/oM6lUSlJSEpMnT8bhcLB27VqOHDmCUqkkISGBmpoaqqur8fLyIiUlhenTpxMREYFCoaC9vZ28vDz27NnDiRMnekzyOxwO1zZnejHn6pIpU6YwadIkwsPDkcvl6HQ6srOz2bJlCy0tLa4+IiIiWLx4MXPmzEEmk1FTU8OePXvYu3cvVquVqKgoJk2aRGpqKv7+/jgcDpqbm/nuu+/YvXs3bW1tDB8+nMzMTKRSKSdPnmTChAkEBwfz73//m9DQUFJTU4mIiEAqldLa2kpBQQGffPIJfn5+3HzzzUyYMIGAgADGjRuHl5cXBQUFvPXWWwwfPpxRo0ahVCoxmUzEx8czbdo0ampqeO211+jq6iIsLIz58+cTFRXFgQMH2LFjBwkJCcyaNYvBgwfj6elJR0cHeXl57N27l+Li4l5/hsTLDoIgCIIgCIJw5YSEhJCZmcmRI0fo6OhAJpMxatQoxo4dO9BD+8H8/f35+c9/zrfffktRUREqlYrFixczfvx49Ho9ycnJ+Pn5XXK/Wq2WlJQU/P39L8Ooz08mk5Geno5MJmPv3r00NDQQHBzMjTfeiI+PD0FBQa4X8i5VYGAgCQkJA1L/BECv11NdXe22Ta1Wu873h/i+QSVBuJqJAIogCAPm4YcfZs2aNa76D+cjgic9ORwOwsPDmTlzJrNmzcLhcFBcXExgYCDTp09HqVTS0dFBbW1tj31VKhWJiYkEBQVx6NAhduzYQUlJCRKJhIqKCrq6ujCbzT/oBtXb25vY2FhCQkLIzc1l8+bNVFdXI5FIOHnyJGazGYvFQlpaGjfffDPJyclUVlbS1tZGQkICMTExyGQyDAZDn47nDKCkpaWRnJyMwWBAqVQyZMgQgoODqa2t5dtvv3W112q1eHt743A4CAwMJCkpibCwME6ePIler2f69OnccMMNeHp6Ul1djUwmY9KkSQwdOpS2tjaysrIIDg4mPT2dsLAwqqurCQ4OxmazERQURFpaGiNGjMBkMmG32xkzZgyJiYlkZ2fT0dGBSqVCoVC4AkPOWjN2u52EhAQmTpxIS0sLhYWFaDQaxo4di9Fo5JNPPsFkMhETE8OkSZPQaDTk5uYyePBgbr/9dsaMGcOJEydoamoiKiqKOXPmoFAoqK6uxmAwiNUmwkWJoJogCIIgCMLl4+3tzT333IOXlxdbt24lJiaGFStWkJCQAPywyWez2dxfw/xepFIpMTExxMTE9PhMpVLx85///HsFDEJDQ1m2bBkqlao/hnlJVCoVGRkZZGRk9PhsxIgRxMfHf6/U0PHx8URHRw9YWmmNRkNQUJDbvb+npyfp6enimVEQeiECKIIgDBitVktmZiYbN248bxtn8GTy5MlXcGRXP4lE4ppo9/LyYtOmTWzfvp2wsDB+//vfk5aWxpEjR2hsbOyxr1qtJjg4GIVCQWNjI42Nja7l4jqdzrWq44dwpq+SSCQ0NjbS3NzsepPFmUrMx8eHkSNHkpqaSmNjI6tXr6a5uZnFixczc+ZM0tPTyc/P71HY7nysVitFRUU0NTVRVlaGh4cH8+bNIyEhgeHDh7Njxw5XW51Ox7fffktubi7Dhw9n0aJFjBkzhqFDh9La2sr48eMJDw9n586dbNiwAW9vb+644w7S09NJT0/n1KlTrpUvcrmczs5OPv/8czo7O6msrESj0VBWVkZNTQ1KpZJ58+YxefJkEhMT2blzJ/v27SMmJobhw4dz/PhxPvzwQwwGg2t1jfPByWAwUFFRQX19PTExMURHR9PS0sKwYcPw9/fnxIkT1NTUMHLkSKZNm0ZtbS1ff/01VVVVZGZmMmfOHFJTU9m2bRt6vV7cDAuCIAiCIAjCAJJIJISGhnLvvfeydOlSNBoNfn5+rvv0yZMn849//OOSVwHYbLZeJ/mvFlKpFC8vL7dtzvqPR44coaCgAJ1OR2JiouuFusDAQDw9PVEqlW6F6a8WSqXye60+gTNprM9NR30lhYWFsWzZMqqrq6murkar1bJo0SJSU1MHbEyC0F927tzZ732KAIogCAPiv//9L6tWreLkyZPnbSOCJ+enUCiIiooiPDwcs9mMVColJSXFdePt5eWFj49Pj5syh8PhKsIOZ25az8692l8T7CqVCpVKhdVqpaOjo8cxHA4H/v7+hIWFIZFIOHXqlOsfuaCgIFJSUggICCAgIKBPARS73U5XVxdbt24lPDwctVqNVqultbUVpVKJr6+vW/v6+npycnLYvXs39fX1xMbGMnfuXKKiovD19cXf35/m5mays7M5dOgQAQEBxMbGkpaWRmRkJJ6enq4gU0NDAxs2bGDLli2usVRVVeHn54enpyehoaGuGiW+vr6YTCbKy8vR6XRYLBbq6urIycnpNXBlsVhobGykpKSEuLg4Ro8ezalTp1xvKzlX7UycONFVPyY0NNT1ICaVSl11UQRBEARBEPpDZmbmQA9BEK55np6eeHp69tielJREfHz8JT+XORyOAZ2Qv1T5+fls27aNDRs2UF9fz5AhQ5BIJOzevRuj0UhISAjjxo0jOjqaoUOHMmLECIKCgvDy8rrma8VcDZRKJQsXLiQqKor8/HxCQkKYMGECISEhwJlUc2lpaZdcy8RmsxEZGXk5hiwIfTZ16tR+71MEUARBuKIOHDjAn/70J7KysgAYOnQoU6dO5e2333ZrJ4InFyaXy9FqtXh4eODn58eCBQtcQQqJREJTU9N5i4c7VzhIpVKUSmWfbrQv9QbeZrNhs9lQKBR4enoik8l6LEf38PBAo9FgsVhoamrCbrcjk8nQ6XQYjUZ8fX1RqVR9OrZEIkGpVDJixAiuu+46hg4d6spLe75r4NxuMplcaeR8fHxcfel0OlfdFJvNRlNTEzabDaVS6XbT3t3dTUtLCxaLBalUikqlIj4+nilTppCQkIC/v7+rjsqFasmc77x0Oh25ubmu1SS5ubmEhYW5Vqd0dXW5AkTDhg0jLCzM7WehsbGR7u7ui15DQRAEQRCEvti1a9dAD0EQfrTkcvmPtgC3M+30J598wvbt2zl16hRjxozhkUceISMjA6VSSX5+PlVVVZw6dYqsrCw+/vhj1Go18fHxhIeHExsby8SJE4mNjSU0NHTAUmD9GGg0GiZMmOCqe3L2vMD06dMZNmzY9wrkhYeH9/dQBWHA/Th/KwuCcNUpLCxk5cqVbN68GYDg4GAeeeQRVqxYgVwu57vvvqOwsBA4EzxZu3atCJ5cgM1mo7u7G7PZzOnTp3nttddoaGhw3eBYrVaampp67CeRSDCbzbS3tyOTyQgNDSUgIMAVQJBKpW4T/Q6HwxUUcO4vk8mQyWQ4HA4sFovb6hKnjo4O9Ho9crmcsLAw10oSZ+F35xgtFgtyuRwfHx+3MZ79+bnjP/smzvlnhUJBREQEf/vb31x1QfLy8oiLiyMqKuq8N37OFFxqtRq73U5NTQ1yuRy73Y6XlxdeXl496pSYzebz5iZ21jB58MEHGTZsGIWFheTl5TF06FC3pfm9jeN8Ojo6KC4uprGxkREjRpCRkUFwcDAnTpzg1KlT2Gw2urq6cDgc7Nq1iy+++IK2tjZXv11dXdTV1Yk3tQRBEARBEARBGBAmk4m//vWvvP/++5w+fZpx48bx9ttvM378eAIDA13tYmNjgTPPVc70UiUlJaxbt47//Oc/wJk6MsHBwcTFxTFv3jzmz5/PkCFDBuK0rnnOZ/tzBQUFERQUNAAjEoSrU78HUMrLy/u7S0EQrmHV1dU8/fTTfPzxx9jtdrRaLQ8++CAPPvig25LlVatWsWTJElfwZMqUKQM46quf2WymqamJtrY2tFotoaGhHDx4EIBBgwZhMBjo6urqtUhfV1cX5eXltLW1MXLkSGbPnk17ezsmk4nx48dTVVVFZWUl3d3ddHV1odVqSU5OZtiwYTQ0NJCYmMiQIUPo7u6mvr6erq6uHsdobW2lpqaGjo4OEhISuO2221i9ejUAY8aMoaGhAaPRSEtLCyqViujoaKKiomhqamLEiBEEBgZSV1dHc3Ozq9aIRCJBo9Egl8tdwQCNRkNgYCAeHh4EBwcTFBREVlYWb7zxBq2trfzsZz9j8ODBKBQKtze55HI5CoUCrVbL0KFDSUlJQa/XU1paikajoauri4SEBOLj4/H29sbX15fk5GRkMhl1dXUYjcZegx52u53ExET8/PzIzc3l/fffp7q6mqVLlzJq1Cg8PDzcblAVCgUajQYvLy9kMlmvQS+z2UxDQwN5eXnMnj2bKVOm4OvrS2VlJTU1NchkMqqrq5FIJCQmJvLRRx9RXl6Op6cn3t7emEymAS8oKVw7li9fDsCaNWsGdByCIAiCIAjCtcvhcGA0Gjlx4gRff/0169evp7m5mcTERFauXMnMmTOJiIg470teUqmUQYMGMWjQIMaOHcv8+fNpamqirq6O/fv3s3PnTqqrq3nllVd45ZVX8PLyIiUlhWnTpjFkyBCCgoIIDg52ZUMQtSAFQfgh+j2AIqK+giAAtLW18c9//pM333yT7u5ulEold955J3/4wx/c3jBxmj17NjNnzuShhx4SwZM+ysnJIS4ujuuvv5677rqLqVOnYrfbCQ8P58CBA2zcuJGGhgbAfeWGzWZj9+7dxMTEkJmZydKlS5k8eTJms5moqChyc3NZvXo1J0+eJC8vj/T0dBISEnjppZdoamoiNDQUHx8f9u/fT05ODlartcfYLBYL3333HUOGDOG6665j8eLFpKWlYbfbCQkJ4dixY6xdu5b8/HwqKiqIi4vjxRdfdBVIVygUHD58mKKiItRqNS0tLUgkEpYsWcLp06eprKxEr9fj5+fHxIkTqa6uprCwkPb2doYPH85DDz2E3W4nMjIStVrNyJEjSU9PdwUSYmNj+eUvf8ltt91GeHg4Hh4ebNiwgdLSUuRyOXl5eURGRjJ37lwSExMBiI+Pp6amhn379tHc3Oy6pueuiKmtraWjo4MRI0Zw33330dnZSVRUFA6Hg0WLFrF9+3aqqqpobGxEKpUyffp0oqOj0ev1PPHEE72+AdTZ2cn+/fuZOnUqYWFhtLS0UFFR4QpAHT16lOPHjxMfH8+qVauor6/Hw8MDqVTKgQMHWLNmDSaTqd9/BoUfn/feew8QARRBEARBEATh+zGZTBw/fpxNmzbxzTff0NjYyIQJE7j//vuZO3cu/v7+l1R8XS6XExwcTHBwMImJiWRkZHDPPffQ3t5OQUEBe/fupa6ujtLSUnbs2IHZbCYyMpKJEycSHR3N4MGDGTVqFIMGDbqMZy0IwtVi5cqVbv/tDyKFlyAI/cpoNPLGG2/w/PPP097ejkQi4eabb+bJJ5+8aIB13bp1vU4eC72rq6tj06ZNtLe3k5KSglarxeFwUF1dTWlpKXq9HpvNRnNzM3l5eW41UcrKyvjkk0+orq5mxIgReHt7I5VKKSgoYN++fTQ1NWE2m8nKysLhcDB58mSio6Px9PSkurqab775hgMHDnDy5Mnzjq+srIx169bR0NDA6NGj8fLywmKxUFRUxOHDh6murqaiooJ3332XadOmERERgZ+fH6WlpRw5coQ9e/bQ1taGSqVi//79xMbGEhQU5KpHsm/fPte4TSYT1dXVvP/++4wfPx6VSkVFRQXZ2dkEBQXh6+uL1WqloaGBAwcOEBcXh1qtRqVSUVRURG5uLnv37nWtLNm4cSNtbW2kpKTg4+ODxWLhwIED7Nixg8LCQrq7u2ltbaWkpAS73U5HRwdw5k2pvLw8vvrqKyZMmIBWq6W5uZmsrCxiY2MZPHgwVquVjo4Odu3ahaenJ8OHD0ehUGAymZDJZJw+fZq8vDyqq6tdAZ/u7m6OHTvGd999h6+vL6WlpVRUVLjSqJWXl/Pyyy9z3XXXERkZiZeXF1arlYqKCleaL0EQBEEQBEEQhMvB4XBgs9nIycnhww8/ZM+ePdTU1DB//nzmzZvHpEmTLjlw0htnVgKNRkNISAjR0dHMmDEDi8VCQ0MDubm5VFdXU1VVxbZt26iurkar1ZKQkEB0dDSDBg1iypQpjBs3zjX3IFIdC8KPy6pVq4D+DaBIHL1VtL2GGQyGgR6CIPwk2Ww2PvroI55++mlqa2uBM4XHVq1aRXJy8gCP7tqRmprKpk2biI+Pp6uri88//5w333wTnU7Xa3ulUklAQABhYWGuAIrBYKCurg6dTofD4cDf35+oqCisVivHjx93BVFUKhVBQUGEhISg1Wqx2+20t7dTW1tLe3u7qyi6VqslPDycwMBAFAoFer2e+vp6mpubL7iqweFwoFKpCAwMJDw8HK1Wi81mo729nbq6Otra2rBarXh7exMREYG/vz9yudw1Bp1Oh8ViQSKRoNVqiYmJwdfXl5MnT9LU1IS3tzdRUVGoVCqqqqqoq6sjJCSEQYMGoVKpaGlpob293ZUiq6ysjM7OTgYNGoSfnx8eHh7Y7XZaW1upr693XS8485ZTUFCQ67ra7XZ0Oh2nT5+mq6sLu93uuu4Oh4OqqipXwFAikRAaGkpERAQeHh60t7fT1NSEn58f/v7+5OXlodfr0Wg0hIeHExISgkQiob29naKiIgYNGoSPjw/Nzc3U19djMplche0TEhJQKpXo9Xrq6urcjqlUKomMjMTf3x9PT08sFgs6nY7GxkZaW1t7FLC32+1kZGRw1113kZGRAYh/QweCl5fXQA/BjXNF1Y/s9lAQBEHoR86XoioqKgZ0HIIgXD3y8/N54403WLt2LY2NjcyfP58//elPJCUlodForvh4LBYLVVVVFBYWUlJSwtq1azl8+DAA/v7+hIaGEhcXx4033sjChQvdanIKgnBtuxzPtCKAIgjCD/b111+zcuVKiouLAUhOTuYvf/kL06ZNG+CRXXsuNYACuIq+O9+csdvtbqmlzi4Kf+7bNWdvdzgcPfY9u42znbOAel/zyDqP7yw839v4Lta/c5uzD+d+zn6cfTq3nduv09ltne16O+ez9z9fu7Ova2/XrLd2znM8t3+JRILNZnN9D2ef29nOLl5/vs+d2y/Uj7OtCKAMPBFAEQRBEK414t8KQRAcDgdtbW0UFhbyxRdf8MUXX2AymUhPT+f2229nwoQJBAcHD/QwsdvtmM1m2tvbqayspKqqiu+++459+/bR2dmJxWLBw8MDpVJJRkYGs2bNIiQkBB8fHyIiIlCr1ed9nhIE4ep0Oe5T+j2F1+XIMyYIwtXp4MGDPPnkk67i5dHR0Tz55JPcdNNNAzyyn4az/zE4O3hwbpDkQjd85/7D0tvy5fMFBvrq7An93o5xsf49PDwICAjAZrPR0NDQI7Bx9j7n1iM59x/MswNNTudbsn3uOHoLblzsusKZ1SxarRYfHx86Ojpob2/HZrP1OG/nOC7U78WWl5/9ubjRFwRBEARBEAShPzkcDvR6Pfv27WPz5s3s2rWLzs5OZs2axaxZs5g5cyZeXl5XTWpuqVSKWq1GrVYTEhLCmDFjmDNnDjqdjoaGBk6ePMnBgwcpKiri6NGj/Pe//8XhcDBkyBCmTJnCoEGDCA8PJz09nYCAAKDvLxIKgvDj0e8BlMuRZ0wQhKtLcXExq1at4quvvgIgKCiIRx55hBUrVqBQKAZ4dD8Nvr6+aLVa5PIzv8YdDgdmsxm9Xo/RaHRbqXAtk0gkREVFsWDBAvR6PR999BFGo7FPbxKoVCp8fHzw9PR0bbPZbHR2dtLZ2YnZbL6kNxKkUinh4eFYrVZaW1v7VJTd4XCg0WhISkpi0qRJ5Obmsm/fPlfNlIEmbv4FQRAEQRAEQbgYh8NBd3c3+/fvd9U46ejoYNmyZcydO5e0tDS8vLyu+vkAmUzmqqESERFBcnIy8+bNo7Gxkc7OTgoKCsjNzaW1tZVPPvkEnU6Hj48P8fHxJCQkMGjQIDIzMxkxYoQry8LVEiwSBOHyEUXkBUHos9raWp555hk+/PBDbDYbGo2GBx54gF//+tcDktf0p0oikZCWlkZKSgp+fn7Y7XZsNhttbW3k5uaSn59Pa2srVqu1x34XChicnQ7rQm3g/Esh+9LH2e3OTW917jglEgmDBg3i1ltvpaGhgQ0bNmAymS5aFN3hcBAeHs64ceMYNWoUdrsdu91Od3c35eXlFBUVcfr0adra2npNi9Vb2i2tVsuyZctobGxkx44dnD59+oKrVJy0Wi2jRo3iZz/7GWq1mmPHjtHZ2XnB69hbP+deo7PP9UJtLvadi/Qbwtn+/Oc/D/QQBEEQBEEQhKuI3W6nsLCQe++9l7y8PAwGA8uWLeOpp54iLCwMlUo10EP8XqRSKVKpFB8fH1cNlJSUFJYuXYrNZqOmpoasrCxOnjzJe++9xzfffINEIiEgIIDhw4cTHx/PggULmDZtGiqVSgRTBOFHTARQBEG4qPb2dl544QVef/11jEYjCoWCO++8k0ceeeSqyGv6UyORSIiPj2fevHkEBwfT3d2NVCpFqVQC8Oyzz/LNN9/Q1NTkCgjIZDKUSiUWiwWbzdajfodzf7lcjtFodLU5+5hSqRSVSoXVasVms2G323vU8VCpVEilUsxmM1artccxnMd1HstqtbqKpDvH6dx+bgDI6ewaIhcKMvj4+JCRkcGMGTOwWq2YzWY8PDyQSCRUV1fz7rvvsmnTJlcwQyqVIpfLUSgUWK1WLBaL23ijo6O54YYbOHz4MIcOHXIbh1QqRSaToVarXefuvD7npvrzW6EAACAASURBVBhzLiO32WyuY5x9neVyOUql0pWv9+zr7GyjUCiQy+WuNr3VY1EoFEgkEtf31dt1ctZMEQQnsYJYEARBEAThp81Zt7KxsZG8vDw+/fRTdu7ciVQqZcmSJSxbtozU1NQfbeF1hUKBQqFg6NChDBo0CKvVyi9/+UtKSkooKCggJyeH3Nxc9uzZw7fffotWqyU4OJi0tDSmTZtGSEiIa4WL85nsYqmYBUHoP7/4xS/6vU8RQBGuKmazGaPRiNVqRa1W4+HhIf6hGUDd3d28+eabPP/887S2tiKRSFi8eDFPPvkkMTExAz28nzTnTW1ubi4bN27EYDBw2223kZqayrRp08jJyaGhoQF/f39SU1OZMWMGUVFRGI1Gdu7cyZ49e6ivr8fhcBAWFsbs2bOZNGkSCoWC6upqdu7cydGjR2lubsbLy4uUlBRuuOEGQkJCMBgMHDx4kL1791JVVYVUKiU0NJSbbrqJ5ORkZDIZNTU17Nixg6ysLAwGAxEREfz2t78lOzubkpISpk+fjo+PD9u3b2fXrl0EBwezYMECkpOTUavVnD59mp07d3Ls2DFXAEImkzFs2DBGjx5NVFQUlZWV7N69m6KiovMGARwOB42NjWzevJmtW7cyePBgbr/9dmJjY5kzZw4NDQ3s2rULHx8fpk+fzvTp0/Hy8qKjo4P9+/eza9cu2tvbSU1N5YEHHsDb25vRo0fz6KOPcuTIEXbs2EFVVRVpaWksXLiQgIAAurq62L9/P99++y2VlZVu44mLi+Oxxx5DpVLR2trKvn372LNnD+3t7chkMjIyMpg9ezaRkZFYLBby8vL49ttvKSgocJ3PrFmzmDp1KhEREXR0dJCbm8v27dspKyvDZrORkJDAzJkzGTlyJEqlkrKyMr799lsOHjwogiWCIAiCIAiCIJyX3W6nubmZ7du3880337Bnzx60Wi2LFi3iuuuuY9KkSXh6ev5k5mmcwRQPDw+CgoIYP348ZrOZpqYmSktLKS8vJzc3l8LCQjZs2MAnn3yCv78/AQEBTJ48mdjYWFftFY1GI4IpgnAFrFmzpt/7FAEUYUBYLBYOHjxITk4OBQUFFBQUUFFRgV6vd2snk8kICgpi8ODBxMfHM3r0aMaPH8+IESMGaOQ/HR9++CFPP/001dXVAGRmZvKXv/yF0aNHD/DIBMCVJkuv11NZWUlVVRUhISEkJyfj4+ODUqkkLCyMG264gQULFiCTySgtLSU09P+zd9+BTdf548efSZomado03XsPWlrasvdeAoJb4VRAPQUXfk9x3rm94Trx5NTz3L87EWUoS9lDllCgUEpLaaF77zZpkzTJ749eP1JaRrGaIu/HP834jNdntE0+r8/79fJn3rx5+Pn58e233yKTybj++uu59tprpRJXYWFh0gV+hULBxIkTue2229BqtWRmZuLj48OcOXMIDg5m5cqVWK1W7rzzTqZMmUJ9fT0ymYzw8HACAgJQq9Vs2rQJrVbLkCFDiImJwWw24+XlRXl5OampqfTp04f77ruPgQMH0tLSgtVqJSIiAkBK3CkUCvz8/Hj22WdxcnJCo9EwYsQIgoKC+M9//kN2dnanfaRQKKRtKisr48SJE+Tk5GCxWJg/fz7R0dEkJCSQnp7OmDFjePDBB6XavmFhYcTFxeHl5cWqVavw8vLC19cXmUyGWq3G09MTNzc3fH19CQkJ4b777sPLy4uamhqCg4OJjo5Gp9OxYsUK4KcyWrGxsZSVlaFSqYiLiyMqKgpfX1+++uor4uLiePTRR3Fzc6O5uRmNRkNUVBT+/v4sXbqUsrIyrr32WubOnYunpycWi4WgoCD8/f0xm83k5+fTv39/HnzwQYKDg6msrMRisTBs2DDi4+MBOHjwYIeSZe2jggRBEARBEC7V9u3bHR2CIAg9zGazYTQa+f777/niiy9ITU1FLpczf/58Jk+eTEJCAjqd7qouUdVetcHZ2RmtVktoaChjxozBaDRSWlpKVVUVeXl5HDhwgGPHjvHpp59iNpvRarXExMSQkpJC3759GThwICEhIVLVAJFQEYTeTyRQhF+NzWZjx44d/Oc//2HTpk2dkiVdsVqtlJWVUVZWxo8//sjnn38OQEBAANOnT+fWW29l2LBh4q7qHrRx40aef/55Tpw4AUBSUhIvvvgiEydOdHBkwtnaR2Wo1Wo8PDywWq3ExsaiUCgoLS2lubmZfv36MXLkSABWrVrF3r178fX15amnnmLIkCGkpaVhtVpJSUlBqVSycuVKjhw5QmxsLGfOnOH06dPExsYyceJEtFoty5YtY9++fYSFhTF37lwSExNJT0+XRmqoVCq2bNmCk5MTN998MxEREURFRaFWq5HJZLi5uaHRaDh8+DB79uzhzJkzlJaWkpKSwsCBAzEajbzzzjtYLBb8/Pw4ffo0hYWF+Pv7A21/D3bv3s2JEydITk5m9OjRREZGEhYWxsmTJzv9HWgvT2W32zGbzRiNRkwmE2lpaZSWlkpJDpPJRFZWFps2bSI3N5eqqiomTpwoLd/d3Z2MjAw2btzInDlzyM7OZv369aSnp1NZWYmnpycbN27EbreTnp7OyJEjmTp1KhEREfj7+1NWVibFlJ6ezurVq7HZbEyfPp2UlBQSExPZsmULeXl57Nq1i6qqKoqLi+nbty+TJk0iMjKS8PBwysvLGTFiBMHBwRw+fJhdu3ZJo/VSU1NRKBTcdNNN9OnThx9++IHNmzdjNpsZN24ckyZNYuLEiRw5cgSz2dzpPBIEQRAEQbhU48aNc3QIgiD0kPbvSgcOHGDBggWcOXMGlUrF3XffzRNPPIGPj89VnTQ5n/ab/NpLZev1egBGjx7NLbfcgslkoqKigu3bt7N9+3YOHDjAxo0bkcvl6HQ6Bg8ezNChQxk/fjz9+vVDpVLh5OSEk5O4TCsIvVGP/2aK5qPCuYxGI5999hlLly6lsLCwy2nUajURERHo9Xp0Oh0uLi6YzWbq6+spKysjPz8fi8UiTV9aWspHH33ERx99RGxsLAsXLuTOO++8YpuX9QYHDx7kueeeY8+ePQCEhYXx7LPPcuuttzo4MuF8ZDIZkZGRzJkzB7PZTGRkJDk5OWzcuJGqqiqGDx8u1Wz19/dn6NChUmk8pVKJp6cnVVVVWK1WtFotCQkJVFZWcuLECbKysqivr8ff35/w8HCUSiU+Pj4MGTIEDw8P3NzccHZ2xtvbG6PRSGpqKpWVlVitVtzd3TEYDDg7O+Pq6ip9CLTb7TQ3N/P999+TmppKbW0twcHBREVFoVQqOXz4MDt37qS1tRUXFxeam5tpbm4G2pIh9fX1rFu3jry8PAwGA7GxsWg0GlxcXLrcP+3JgXOb0tfX10t9T5RKJWazmdOnT7Ns2TJUKhUuLi40NDRIcTg7O1NSUkJOTg52u52amhrS09PJysoCoLGxkbVr1+Lu7o6TkxMNDQ1YrVZp3rOTFMXFxaSmplJfX4+vry9RUVF4eXnh6upKfn4+X331FWq1Gq1WS2NjIyaTCbVajZubGzabDYvFgkwmIyQkhPDwcNLS0jh27Bj5+fnodDqSkpLQarXodDri4uIA8Pf3R6PREBMTg5OTU4e+K2L0iXCuHTt2AOLimCAIgiAIwm+V3W7HarVSUlLCwYMHWb58Ofv370en07Fo0SJuuukmEhIS0Gq1jg71iqRWq1Gr1eh0OsLCwpgzZw5Go5G0tDT27NlDfn4+2dnZfPrpp3z44YcEBAQQHR1NUlIS/fv3JygoCE9PT/R6veihIgi9RI8nUETzUaGd1Wrlk08+4S9/+QtVVVUd3vPz82Pq1KlStj0qKuqCdzW0trZy6tQpDh48yObNm9m8eTNGoxGA7OxsHn30Ud544w2eeeYZ7rzzTnFHdTecOnWKF198kTVr1gDg5eXFE088wT333CM1JRd6L7VaTWhoKDqdDrvdzrfffsvBgwcxGo3SiI/20SkRERHIZDIaGhqor6/HaDRKfUQ8PT0ZPHgw/v7+nDhxAr1ez/Hjx3F1dcXFxQWZTEZMTAwxMTEoFApaW1upqamhsbERjUZDREQEI0aMICQkBI1GQ2xsLCqVSkpgtGtpaSEnJ4eioiJMJhPBwcG4ublhNpspLi7GaDRitVql3+/2D4rtpbiKi4sxGAzU19dLZa668/tut9ulOrbtCQm5XI6HhwcjRowgNjYWnU4n7VP4qVya1WqVlmG1WrFarTg5OeHp6cnQoUNJSEjAw8MDPz8/PDw8yMvL67R+i8VCa2srTU1NVFZW0tTUhJOTEyqVCldXVwYMGEBcXBw+Pj74+fnh7+9PbW0tcrkcm83Gpk2b8PLyIikpialTpxIdHU1qaiq7d++mqakJrVaLXC4nICAAlUqFXC5Ho9FQUlJCZWWlSJgIFzV+/HhAJNcEQRAEQRB+i1pbWykrK2PDhg189913HDhwgMDAQO6++26mTJlCSkpKt79jCV2TyWRSyS83NzcmT57M+PHjsVgslJSUkJaWRllZGVlZWRw8eJD9+/cDEBISQlBQEPHx8fTr14/w8HAiIiJQq9UoFAqRTBEEBxBjw4RfxJEjR3jggQekxsfQ1o9g1qxZzJs3j3HjxnXrj76TkxPx8fHEx8czd+5cmpub2bJlC5999hmbNm0CoKSkhIceeogvvviCJUuWSHdfC10rKyvjr3/9K59//rk0AuHBBx/kkUcewc3NzdHhCZeoqKiIEydOSD07NBqNNOKjtbUVq9VKfX09R44cobq6Gmj7INfU1EROTg6lpaVs27aNlpYWkpKSiIqKYvLkyfj5+WE2m6Vkgclk4sCBAzQ3N0sjKhobG8nMzMTPz49bb72VqVOncuzYMWpqaqRkw/mcnVix2WzI5fIOI0kUCkWnC7jnPu9u4qRdZGQk3t7eNDc3U1dXh1arZeTIkdx///3U1NSQn59Pa2vrBUdp2O127HY7bm5uDB06lPnz56PVajl27BgWi6VDn5Hzbb9arcbZ2ZmmpibkcjkpKSksXLgQmUwmjbo7ezk2m42DBw/i5OREbm4u8fHxxMTEEB4ejkqlYvXq1VitVux2O7m5uWRmZkoj96xWK+Xl5R1GnwiCIAiCIAiCcHVobW2lqqqKNWvW8NVXX5GVlYVer2fx4sWMHz+eyMhIXF1dxcX5X9DZPVSio6OJiIjAZrNhMBgoLCykvr6e06dPs3nzZqlks0qlws3NjZCQEIYMGUJKSgr9+vXD09NTSqiIZJcgdNT+O9GT1z5EAkXoUTabjVdffZXXXntNuoCqUCi4/fbbWbx4MeHh4T2yHo1Gw8yZM5k5cybHjx9nyZIlrFixApvNxt69exk1ahRPP/00jz32WI+s77ekoaGBJUuW8O6772I0GnFycuKee+7hqaeews/Pz9HhCd1UVlbG1q1b8ff3Jy4ujptvvpns7Gx27dpFRUUFtbW1mEwmTp48yaFDh5DJZHh6elJbW0tdXR0eHh74+Phw6tQpDh8+zLBhw7jrrrsIDg5Gp9NRW1tLVVUVKpWK7OxsqWyVTqejsbGRqqoqBg8ezPDhw7FYLKxZs4aCggJmz56NXq9Ho9FccOi30Wikrq4OZ2dn4uLiiI6OxmKxSOs+u3/I5X4wbC8/1v6lYOrUqYSEhFBQUMDp06fx8PBg8ODBeHt7s379erZt28bw4cPx8vJCo9FII1HakxNubm74+/vT2NhIUFAQiYmJ+Pv78+OPP/Lxxx8zaNAgfH19cXFxQafToVQqpVi0Wq1Usis2NhYXFxeysrJoaWlhzJgxhIaGsmPHDlavXo23tzc33HAD7u7uuLu7o1arSUhIwGAw8N1335GVlcWsWbNITk4mKioKg8FAaWkper2e8vJyDhw4QGVlJRqNBrVaLSWGBEFwDHNtKQ0n9+ORPAmFpvfcqGBprKY+Yxf6fuNx0uovaR67zUb98R0otHrcogb8ovHVZ/yA3FmNW8zgX3Q9giBcuvaqE6L6hCD0fna7nbq6OrZs2cIzzzxDQUEBAQEBPPjggzzwwAPodDpxAd4BZDKZdOOjs7MzHh4eQFsPldtuu43W1lYKCwtZvXo1GzZsIDU1la1bt0ols8eMGcNNN91EbGwsISEhuLu7I5fLRb8aQfiFiASK0GNqa2u5++672bp1q/TaiBEjePPNN0lISPjF1puYmMiHH37Igw8+yKJFizh69Chms5kXX3yRo0eP8v7775+3R8LVxGQy8eGHH/L6669TU1MDwHXXXcfzzz9PdHS0g6MTLpdMJqO2tpa8vDxOnDjBoEGDmD59OqdPnyY3N5esrCyGDh3K3LlzCQ8PRyaT0bdvX3bt2sWePXuIj49n9uzZqFQq9u/fj0qlwmazUV1dTWNjI2VlZaSnpzNhwgQWLVrE1q1bpd4rR44cYePGjdIIFTc3N+Li4oiKiiIkJAS1Wk1wcDCxsbGUlJR0GXt1dTXZ2dkYDAYSEhJYsGABJpOJwMBA9uzZw/r163/2XQN6vZ4xY8YQEhJCQEAAERERWCwWdu/eTVpaGlqtFpPJRGtrK6GhoYwYMYI+ffrg4uJCQEAAiYmJHDlyhNraWhobG4mMjGTWrFmUlZVRW1srjdTx8fFh6NChUh8SFxcX+vbtS1lZmZR8iY6OZvLkyfj6+pKcnExVVRWpqamUl5fT2tqKxWLBx8eHlJQUfH198fHxQaVSkZyczO7du5k3bx4BAQFkZmbS0NAgjWApKyvDYDCwY8cOqUSij48PBQUFeHt74+rqyrvvvktBQYEYgSIIDmBrNXN48VBsLU1oI1JIfmWbo0MC2i6qHH16DObaUtT+UQx486D0Xn3GD5z8x3yctB70WfQJ2vB+0nvF6/5BwfKXAIhb/CWe/af8IvFV7PyCnA8eAqDvkyvQJ024rOVcaFt6q6y/30Htsa34T5hHxNy/OTocQejgxRdfBEQCRRB6K7vdjslkIj8/n927d/PNN99w6NAhIiMjueOOO7jzzjsJDg4WJbt7KbVaDUBcXByLFy/mkUceoaGhgQMHDrB161aKi4spKiri8ccfx2q1EhkZyfjx4/H19aVPnz7ExMTg6ekpNaYXSRVB+Pl6PIHSXvO9p0YaCFeGoqIirr/+erKzswFQqVS8/PLLLFiw4Fe7m6F///7s2LGDf/7zn7z44otYLBa++eYbcnNz+eqrrwgKCvpV4uiNvvzyS15++WUKCwsBGDVqFC+//DIDBw50cGTC5WrvyWGz2bDZbFRVVfH111/Tt29fkpKSSE5OZtOmTXzzzTe4uLgwZMgQkpOTAairqyM/P5+MjAxpGX369JHOh7y8PDZs2EB6ejoVFRXI5XLc3d0ZPXo0DzzwAAD19fXU1tbi4eEhlQGbM2cOd9xxBzU1NWRmZlJaWoqPjw8RERGUlpZKfUPObei+b98+fH19mT17NpMnTwagsrISrVYrfXhsn7fd2X1IzpcQaJ9GpVJJJQCNRiOnT59mw4YNbNu2jdLSUnQ6HTt37mT48OGMGzeOUaNGceDAAYqLiwkNDSUsLAyLxcKJEydIS0tj0KBBTJs2jYaGBr788kv279/PsGHD6NevH0lJSRw6dIjKykoCAwMJCwvDzc2NhoYGGhsbiY6OJjY2FrPZTHZ2tnRHkdlsZtu2bYwZM4bY2Fj69evHiRMnKCoqIjw8nJiYGNzd3ampqaFv377ccMMNKBQKamtr2bt3L+vWraOlpYUvvvgCLy8vJk6cyPTp01EoFJjNZrKysggKCqKwsFAkUATBAQz5x7G1NLU9zjuK1dyMwlnj4Kigpfw05trStsdluZhqSlB5BgJQseu/tDbV0tpUS8UPy4g4K+nQkLVPetyYvf8XS6A0ZP/403pyD112AuVC29IbtVTkUXNoAwClGz8g5OZncHLROTgqQRAE4UrQ0tJCQUEB69atY926dWRnZ5OYmMjixYuZOXMm4eHhHUbIC73X2T1UtFotM2fOZPr06VJy7MiRIxQXF3P69Gm+/fZbysrKcHd3Jy4ujqCgIMLDwxkyZAgxMTHo9XppWYIgdJ/M3sNXUn6JOmPd0djY6JD1Xs3y8vKYNm0axcXFAISGhvLf//5XuljrCHv37uXOO++ksrISgOjoaL777rurrkTV5s2bef755zl+/DjQNlrnhRdeYMqUX+ZCh/DzDRgwgHXr1tGnTx+MRiOrVq3igw8+kEYNtfP19cXDwwOTyURFRQXNzc2oVCoiIyORy+VUV1dTU1NDa2srbm5uBAQE4O3tLdW+LS0txWAwIJfLcXV1JSAgAF9fX4xGI2VlZVRVVdHS0gK0leHTarUEBQXh4eGB1WqltraW8vJyGhoaAHB1dSU6OhoPDw+Ki4upqKiQSnhVVlZKyQOLxUJhYaHUSwXa/m+oVCr8/f0JDAzEarVSUVFBRUUFBoMBnU5HcHAwFouF06dPSz17fH19USgUVFdXU1tb22lfuri44OXlhbu7u/SayWSSRte09weBtmHTgYGBhIeHSwkmlUqFt7c3RqORM2fOAODp6UlYWBiurq40NDRQWFhIY2Mjfn5+RERE0NLSwpkzZ1AoFHh5edHS0kJJSQkKhYKAgAD0ej1qtZr6+nppBEt7HHK5HH9/f8LDw2ltbaW4uBiTyYSXlxdyuZycnBxkMpl0rNoTKOXl5dTX10v7U6VS4evri6+vLyqVCoPBQEVFBeXl5Z16qgwfPpzf//73DB8+HBD/Qx2ht/WccvTnuN+qhpP7Of7SdOn5kA/O4KR1v8Acvw5jUSZpT46Ung946whq3zAAst66k5rU9QD4jruD6Hv/IU2X8dcbqD++E4CAqQuImPvXXyS+U+8tpHL3VwAE3/A4oTc/fVnLudC29EaGggyOPj1aej7wH+movK7eG4GE3kf8rxCE3sdkMlFQUMCKFStYu3YtRUVFBAUFsWDBAsaMGUNQUBDOzs6iXNdvhN1ul26qbGhooKSkhOrqanJycli/fj2HDx9GoVCg0WhwdXUlODiYCRMmMHz4cAICAnB1dRU9b4TfLNEDReh1ioqKmDFjhpQ8SU5OZuXKlfj6+jo0rhEjRrBz505uueUWMjIyyMnJ4brrrmPDhg14eno6NLZfw+HDh3n22Wf54YcfAAgJCeFPf/oTc+bMcXBkQk+prq6mrq5O+tDUPkw7JycHoMPrDQ0NGI1G8vLyOo3csFqtNDY20tzcTH5+vjTf2aM9rFYrTU1N5ObmSh+wzh39YTAYOHHiBAqFAovFIi1XLpdLo1zaR6id28i8PfaioiKp54nVapX6dbQ3vG+PF6C5uZmioiJp2q40NzdTWlpKeXl5h3W1x3M2i8VCUVGRlGSwWCzIZDJqamqkhvHQVqqwvem7zWajtbUVm81GaWkpVVVV2O12KSHSPm9raysymYy8vDypyV/7/js7dpvNRnl5uZQsa29kX19fj0wmk54XFxdL29TVKByz2UxpaSkVFRXIZDIphos1thcEgLCwMEeH4DB2mw1ZN75Enp0Evpz3L2e9drsdu7UVudOl3zlqs5iQOV3GBZOzv3Bc4pePS93mnma32bDbWpE7neeuym5uS3f32eWcOxc8jp1ivMT93804zjffxZZzOeuxtZrPf3y6Wkc3z/XLOQbiIqIgCL8FVquVkpISVq9ezVtvvUVxcTHx8fG8/vrr3HbbbeLv3W+UTCZDoVCgUCjw9vbG29sbgHHjxjF//nyqq6spKytj2bJlrFmzhtTUVDZv3ozZbMbLy4trr72W3/3ud1LvUy8vL1HqSxAuQCRQhMtmNBq57bbbpLJQgwcP5ttvv8XV1dXBkbUJDg5m7dq1TJs2jZMnT3LixAluuukmNm7c+Jsdtpibm8tLL73E6tWrgba75RcvXsy9996LSqVycHRCTzr34ju0XQwwm82dprXZbF2+fqnvX+oyTCZTp9fOdqH52y/yd9XkvKt1t5cuu5D2ZV6K863/3H3c1XYCF523/Q6hs0e9dKWr43ru8/Ptp4ttiyBcivZSrFebmkPfcepfD6DyDiHxj2tpPPUjZVs+oTH7R2RKNR5JE/Aaeh0eKW1lBusz93LyrTtQuOiIX7wMl+D4DsszFmVy/OVrAej79OpO6zPXlpD3xXPUHtmIpb4CTVAf9P3GEzb7eeTKjv+vbRYTxWuWUJe+HWNRJjZzM66RA9DFj8B7xM1oQ/p2mL7V2EDx2rdpyj2EsfgklrpykMlw1vuj6zuKiDv/itLt/DeTWJpqyf/vszSc/KlMV82hDRx/JQ/vYTfgP+nuTvNU7l1ByYZ/YizKQu7kjFvMEAJnPIQ+cexF9nzb9hWueo269G0YCzNx1vuijehP0IyHLjhfQ9Y+yjZ/hLE4i+bSHOytZuRKNS5hiQRMuRefkbdc8rZczj5rOLmf0o0fYMhPp6X8NAqNGyrvELwGXYv/5HtQ6rw7beelHMfKfSsp2/jvDvNm/f0O1H6RhN/+Eiqv4C73R3fP4Xan3ruf6h+/JXDmItyiBlKw4i8Y8o/jGtkfvwnz8Bt7u7S9Jd+9h7HwBC3lp1G6+6INScBz8Az8JszvdHHO2mKgcvdyGrL20XByH+aaEpw9g9CG9yNo5iPoYod22obunuvdOQY2cwtF375JQ9ZeDPkZWE0GVJ4BuIQm4j/5HjySJna5XwVBEHojm82G0Wjk1KlTbNu2jfXr15OTk0NycjKPPPIIc+bMwcfHB/j1b2oQHEsmk6FUKvHz88PPz4+EhASefPJJDAYDu3fvZseOHdTV1ZGWlsbatWtxcnKib9++jBs3jpCQEPz9/enbty/u7u4oFAqcnZ3FKBVBQJTwEn6GO+64gzVr1gDQr18/NmzY0KFUTm9RWlrKNddcI5Xguf/++3n11VcdHFXPKi8v59VXX+XTTz+ltbUVjUbDAw88wB/+8Ad0OlEz+0pyqSW8BOHnEiW8eofeVsLrapW1ZB41B9cCoIsfSUPW3i5HAPqrOAAAIABJREFUKsQ89G98ht9Ezr8foWLH/wPAe/hNxD7U8YL32aWnQm97Dl2fYR1KeCk0blibO/++6ZMnE//Yf5Ep2u5xai47Tdabv6O5JLvLuBUaHX2fXoVb1AAAWsrPkP7S9LYEwHloAmNJeGY1zh4BXZbwajp9mOx37ul6fS46hv47r0MJr/ORKVXEP74cfcKY807Tamwg45WZGPLTO8+vUCJTOGEzNwMdS3gVfft3Cr7+8wVHk4Tf/grOngEX3Zbu7jOA8p3/JffD/wNb1yMgVd4hJD63QSq71Z3jeOTxYeedLmL+6wRM7np7unsOA1gaqjh4f6z0nlylxWYySM9lCiX930yldOO/KP3u3S7XC6CLG0Gf//sMpZsX0JYIyXxj9gXPkeDrFxN6yzPS8+6e6905Bq2GejL+ch2GvGPnjSfk5mcIuWHxed8XOnL0d35BuJo1NTWRlZXFmjVr+O677ygrK2Pw4MFcf/31TJs2TUqcCMK52is/WCwWcnJyOHLkCPn5+eTn57Nv3z7q6+vx9vYmNjZWKm89evRoYmJiUCgUuLi4iP45whXhl+jPLkagCJflo48+kpInfn5+LF++vFcmTwACAgJYsWIFY8aMwWAw8N577zFmzBhmzJiB2WyWehgYDAZkMhlubm54eHgQGBh4RWTa9+3bx4033ojBYEChUDB//nyefvppAgICHB2a0EPaS0iJu4eEnnR2aTJBuNpZmxukxw2ZewBw6zMMJxd36jN2SRfxc957AJVXMNrwJGn6mtT1WJsbUWjakmHWliaqD6yV3neN7N/F+hqRq13RBERhOHNUer3u6GZqj27Fc8BUAE5/8thPF5RlMjxSJqN096Vq/zfYWpqwNjdw4m83kvyXXah9Qinb+omUCJAr1fiMno0mMIaWslzKtn4CdjvNJdlU7l153hEeHsmT8J90N1X7VtFqqANAqfNB13fUBe/S1wTFYWmopLWxGgC7xUT+sufRv7L9vPMUr1nSIXni5OaFLnYoTWfSMNeUYLd2HrXXaqijcOWrUnLANWog+uSJKNSuVOz8L83FJwEoWPk3Bi/NuOi2XM4+K1r1qnTh3nv4TXgNmYmx+CRFq1/Hbm3FVFVIxc7/EnLjE90+jmGzn6d4/Ts0ntwvbbMubjguoYn4jLj5vPuyu+ewLnYo5prSDstoT57IFE7Yra3IFE7UH9/RIXmi1Pvh3nc0pqpCGrN/bFtf1l7y/vNHYu5/H7vNRvY793RInriEJeKs96fh5H5sLU1t+/CbN9DFj5RGKXX3XO/OMahJXS8lT5Q6b0JuegqFi46yTR/SeOoAAMXf/p2gax/uNAJM6Nq8efMcHYIgXHUaGxspKiri448/Zt26dbS0tJCUlMSf/vQnBg8ejK+vL05O4hKfcH4ymQy1Wo1arSYlJYV+/frR0tJCc3MzxcXF5OXlkZuby4YNGzh8+DAAn376Kd7e3vj5+TFjxgyGDh2Ki4sLer0eV1dXcY1C6JV6MnHSTvx1FbotPz+fZ55pu2NMLpfz6aefEhzcdTmB3iImJoa33nqL++67D4B7770XLy8vCgsLz1sGSK1WExUVRf/+/Rk3bhxjx47tlU3o+/fvj4eHBxMmTOD5558nNjb24jMJVwSZTIaTkxMajQYXFxdHhyP8xlitVlQqlah1KwhdCJ39HMEz/w8AQ146x16Ygt1iwm61ULn7K0Jvfpq8//cMdqsFm6WFmsPf4zPyFqCtlFL7xeq2i82jaDx1sMPy1QHRJL20BScXHY2nDpD+wjXSe025qXgOmEpd+vYOF6Gj71uK75i2XmahNz9N6qIksFmxGhuoObSBwGsWYizMBNpGDsQ/8RXufUdJ8xuLMmnI2ic9Ph+Fxo3Iu97AXFtGzaENAHj0n0L0fe+cd57Iu9/Ef+JdWM3NZC/9PbWHvmvbd2eOShfiz2Uzt1D6/fvSc5VvOMl/3oGTiw5bq4WMP8+SLtCfrbk0R0qseKRMoc8f/p/UJ8M1PJmMv1zXtvyWJixNdRfdlu7uM1NVEaaqImka/ym/Rxc7FC/ANSKZrL/fid1qQeUdAnBZx1HlHcLRZ34auRPzwL/OW7rrfC52DndVQkup9yfppc046/0w5Kfj7BHA0T9NkN7XBMSQ9MpWFOq2cr15/32Wkg3/BKBy91f4T7kPuVIl7WuAgGkPEHHHKwBY6itJf/EaWsrbRoXXH9+BPnFst/eR16Bru3UMGrL2StO6hCbgN34uMoUTXoOuJfPNOdQf34mzZ0CX56nQtU8//dTRIQjCVcNsNnPq1Ck++ugj/vnPf+Lk5MSIESN4+OGHmTVrlqPDE65Q7dca2pvJ+/j4kJKSAsCjjz7KyZMnyczMZPPmzWzdupWTJ0+ydu1a5HI5Hh4ezJkzh+uvvx4fHx/0ej3e3t4imSL8polPiUK3PfroozQ3t10YeOyxxxg5cuRF5ugdZs+ezfbt21m2bBlNTU00NTVdcPqWlhYyMjLIyMjgP//5D3K5nLFjx3L77bdz3XXX9ZqeImq1mr1796LX6x0ditDD5HI5gYGBDB8+nMbGRvGBROhRNpuNPn36iL8dgnAOtV8kQTMWSc+14f0ImHIvJeuXAtB0+jBKnTeeg2ZQ/eM3AFTtWyklUCr3rJDm9Rl1GzJ55yRlxNy/4uTSVmLTLWYISr2fNAqi1VAPQGNOqjS9QqPDa+h10qgxpd4ffeI46o5tBaA+YxeB1ywkbPbzqHxC0SeOxb3vKGzmFgz5x6g/sQdTVaG0vOaSnJ+5l37iGj0I/4l3tcXprMEjZbKUQIG2Ml1d9VxpqczHZmmRnkfc8Wdpn8idlETf+w+OPN75Ir9b9CDCZj+PubaM4BufQO6kxFRdREP2j9Slbe64jtJTqH1CLhh/d/eZs1cQzl7BmKvbLuAff2kGHimT0SdNQJ80kUHvZmEzN6PyDAQu7zj+XJdyDnclcNpCqeyYa2R/WioLsNSVSe+Hz/2LlDwB8Jt4l5RAAWg8dQC5s7rjMqc/ID1WuvvQ96lVZPzlOuzWVjwHt/UJ6u4+Cpi6oFvHwC12KBW7vmib//hODj86EM/BM9EnjiH+sWWYqgpR+0V0+bsqCILgCFarlYaGBo4fP86mTZtYvXo1TU1N3HTTTUydOpUZM2bg5eXl6DCF3yiZTEZcXBx9+vRh1qxZlJWVUVBQQGZmJnv27KG0tJTNmzezbNkyXF1diYmJYdy4cQQGBhIYGEh8fDwuLi6ih4rwm9LjCRRRDuS3bcuWLWze3PblNDo6mieffNLBEXXP66+/zs6dOykpKcHPz4+YmBgiIyPx8PBAq9Vis9loaGigqqqKU6dOkZ2djcHQVs7AZrOxfft2tm/fznPPPccf/vAH5s2bh1qtvshaf3niAuhvk7OzM4MHD6Zfv34XbZguCJdDqVT2ir9hQu8xbtw4AHbs2OHQOBzJNWoAsnO+6GnD+kmPjQUZ2Cwm/CbMkxIodce20Wqow26zUpe+TZq2vQH3uVyC+nR8HhxPfXsPDnvb3/vmklPS+9bmBn68+/wjEKzG+v/FmYjfhHlU/vAlRWuWYCzI6LIMVlevXa6utqXjyrr+/9VSntfhuWtkSofn6oBoFBpdh9JU7fwn/56qfSvJ+eAhmnIPn7d/ic3aepHou7/PZDIZoTc9Re5Hf2h7zW6j9shGao9sbNuO6EEEzXgI1ZC2u4Iv5zj+XJd6DneaL6Jjubnm4qyO74cnd3iu8Y9E5RuGqSL/f9OfxN76075zCekrJTHaqX3DGLgkreN6urmPunsMvIZeR/m2T2k6fQQAU1Uhpd+9S+l376Jw0eE9/CZCb/ljl4k+QRCEX1tdXR2HDx9m+fLl7N27l8bGRsaMGcMNN9zA2LFj8fQUf6uEX4dMJkOhUBAUFERQUBDDhw/nlltuoaamhvLyco4cOUJmZianT5/m/fffx2w2o9friY+PJyQkhPDwcEaMGEFYWJjUQ0WUmROuVOLMFS6Z3W7npZdekp6//vrrODs7OzCi7tPpdGzcuBG1Wn1J5bisVitpaWls376dlStXkpGRAUBJSQmPP/44S5cu5a233mLSpEm/dOjCVUgmk6HRaNBoNI4ORRCEq8TOnRduDH41kDlduDmm3dqKzWLCPWEMKt9wTBV52K2tVB9Y23Yx9399GdxihqAJjOlyGXKVtuNzZefPU11d4O4Uq1KFxj+KgKkLACjb9imnP36sQ/NwmVKFa0R/rMZ6qQxVT45olKs6lpi83B4SMmXHZK5MJkOuVGFt7jhdq7GB9BemSr1O2jl7BaOLG07Vnq/PWsbF13s5+8x37O9wCUuk6Js3qE3bjP2sY9WUk8rJt+cTeuuzBF/3h8s6jj/XpZ7D51K6d2w8bDO3dHgud+78eUTu9NPxttvtmOsrpeeac5Jr53M5+6g7x8DJRUfi899TsuGfVOz8Ly1ludK0VmMD5Vs/oS59G4nPru+U8BEEQfi11NTUkJOTw4cffsjmzZtxcnJi/Pjx3HzzzQwYMAAPDw9ReldwODc3N9zc3AgNDaV///4YDAaph0p6ejqZmZns27eP1NRUWltbcXFxIS4ujoCAAMaOHUtKSgoajQZPT09Rplz4xbzwwgsdfvYEkUARLtn27dtJS2u7Y2zUqFFMnHj+RqK9WVhY2CVPq1AoGDhwIAMHDmTx4sUcPXqUd955h5UrV2K1WsnPz+fGG29k9uzZLFmyRPwDEH629mZtjY2NDo5EEATh6mQsyOj02tl3yCt1PlKpKb/xcylY3nZzSdX+VR0uBPueZ/QJgEx28VIGmoAo6bHaL5KU1/Z2mkYmV2BtMeDkosNus1G0+k0pEaCLG07oLX/CNXogcidn8r54/qfeJ5ew/kt1KdvSFbVfeIfnxoKMDv1HLA1VWBoqOVf1/tVS8kSmVBF267N4DbselWcgVpOxQwLlYtt5ufusIWsf5roy+iz6BJu5mbrjO6nat4raQ99JZcmK175N4IyHun0ce8KlnsPnpi1kTh0TeZrAjn31jMVZuEUNlJ5bGqp+avwOqH3CUDhrqDvavs5szmWzmMh8Yw5Npw8Tfd9SvAZfe1n7qDvHwGYyUHPoO7wGX0vwrP/DWJRFTep6Kvd8LcVoqsinas/XBM18pNO6BUEQfkktLS0cPXqU9957j88++wwPDw+uueYa7r33XsaPH+/o8AShSzKZDKVSiV6vR6/XExAQwKBBg4C2GyrS0tJIS0tj27ZtHDx4kG3btvHee+8RGhqKv78/06dPZ8qUKej1enx9ffHw8HDwFgm/JS+++CIgEiiCg7z77rvS4yeeeMKBkThOcnIyH374IU899RRPP/00Gze2lQr48ssvOXHiBMuWLSMk5MK1tgVBEARB6L0MeceoPrAGr/+V/zFVF1G29RPpfU3wT3fV+46ZQ8HXfwabtUMTbLnKBe9h1/+sOFyC+0qPW8pP01ycjTYsUXrN0lhN5uuzaco9ROitz6KLG465plh6P+x3L0kXu23mFmrP6g9iPk/Jq7OdfTHdZmq+wJSXR+0bjlypli52F678K26x30gN4QtW/K3L+ar2r5Yeu/cd3aHHRvWBtR2mbd/O821L46kD3d5nxuKTHH95BgCB0x8k/PaX8Ro0A69BMyjb8jGnP1kMtJWispmbu30cg6/7Q6cRJN3d/905h8927ughtX9Uh2NUtOp14hYvk0bj1KSu7zC9NiwRc02J9NxYkEHNkU149p8CtF1Qyfn3IuqP7wBoi3Hwtd3eR56DpnfrGOQte4GK7Z/j5OZF0ktbcAmOwyU4jsBrF3H4D/2lmC/l90Jo034OiPLdgnB5LBYL1dXVHDlyhE2bNknNuRcuXMi0adMYPXq0uKAsXLFkMhn9+/enf//+zJ07l8LCQtLT08nJyeHYsWNkZ2fz+eef8/nnn6PVaunTpw8jR44kLCyM4OBgIiIiUKlUKJVKlEql6AUr9AoigSJcktLSUrZs2QJAXFycVCP9ahUdHc3XX3/NN998wyOPPEJtbS3Hjh1j0qRJrFu3jpiYrkt2CIIgCILQ+5185x78xs/DSetO1b6VtDZWS+8FTv2pybez3g/PgdOoObiuw/xeQ2ah0Lj9rBi8hs6iaPXrNJe2jRw48dotRM57DW1ECk05B8lb9oLURNtqMqDUeXeYv/S795HP+gOWhkoKV73aoZ+FuboIa/OFRzo6632lx7XpWyle9w9aDfW4J4z5WdvVTq5U4T/5HqkJeUPWPo6/PAN90gQMZ45K/SzOpdT9VGaq4eR+qlPXo/GPou7YNvK/fLHDtO0jVc63LWr/qA7TX8o+kyl+Sm6UbPgnMoUT3iNvQSaT05D508gJTWAsTi66bh/Htng7lpnN/+oV3KIGYGu1EDhtYYdG7udzqedwB+dcoJA7KQm+4XEKvnq5bd+lbSLz9dl4DpiKqaqwQwN51+hB6JMnYTMZKVz9hpSYOrlkLgHXLETlGUht2mapGTyANjQB6P653t1j0F7eq7Wxmoy/3kDkvFdxCemL4cxRWptqpendYgZfdL8KgiD8XOXl5ezbt48VK1Zw4MABFAoF06dPZ+bMmQwfPhxX14v/jReEK4VCoSA8PJzw8HAA6uvrKSwspLKyUmpKX1xczLvvvotMJsPNzY3o6Gji4+OJiooiOTkZf39/tFotKpVKJFMEh+nxBMr8+fMB+PTTT3t60YIDrVq1SmpiPXfuXAdH03tcf/31JCcnM2fOHE6cOEFpaSnTp09n/fr1xMbGXnwBgiAIgiD0PjYr5Vs/7vSyPmkinoOmd3jNb8K8TgkU37F3dHjeoXeETN5phMHZ77f3FJE7ORN179ucePUWbCYDlrpyTr49r1NMKt8wAqbeh7O7Ly5hiRjzjwNQtW8lVftWdljv2Q3dG3MPowmI7hiH6qc4zm4GbzXUk7/shbbl7l+FS1Bcl/MAyM/uZfK/PibnEzTzEap+/Fa6ON6Uk0pTTmqH+dvLa7Uvx2vY9dJ22VqaOPnWneddfuP/lnW+bVH5hnV7n+kTx+I5aIY0+qJ47dsUr32707q9R9zcFnc3jyOAk1aPUu+Ppa4MgJqDa6k52Da6RuUdgu/o2867zZJLOIc7HbsuepwEzniI2rTNNGbvB6Du6Gbqjm7uMI2TqwdR9yxpazar1hJ1z5tkvTUXe6sZe6uZknX/6LRc1+jBBFzTlsi5nHO9O8fAf9JdVB9Yg83cjKkij8zXO+8/J1dP9P0mdHpdEAShJ9jtdsrKykhNTWX58uXs3LkTvV7PlClTWLhwIcHBweh0OuTyniuxKQi9kbu7O+7u7gCMHDmSW265BbPZTGFhIT/88ANZWVlkZmZy6NAhDAYDer2ehIQEBg0aREJCAjExMXh7e6PVaq+4nszCla3H/zp/9tlnfPbZZz29WMHB1q1ruzAgk8m44YYbHBxN7xIREcGmTZsYPLjtrrXy8nJuuukmqqurLzKnIAiCIAi9TcS81/Cf/HvkZ93lL1eqCZzxEHGPfdFpen3ieJTuP41wUPtF4B4/osM02tAEaeSEe8IYFOc0Xtcnjmt7IJPjkTxZel3XZxgpr+7Fo//UDnfdQ1tZKv9Jd9Pv+e9x/t/6E55ejS5+ZIeRBHKVFv9JdzPkXznSBXqZUoXKOxiVVxAuIW3lk1yjBkrLAfAddyceKVM6rlPhhNeQ69An/e9Cs1yBPmlSh2k0QX1w9gxqiz9+5AVH4ih13iS/vBX3xHEdYlbqvIm85+/0WfSptF/cE8cC4DVoBlG/fxvFOf1CXEL6Ev/EV/R/MxUn17ayJ+0JogttS3f3GUCfRz4j9NY/4dxFw3GVbzjRC98j5IbF0mvdPY4AMfe/1+EcBFB5B1/SKIlLPYc1AdFoAtpGTbv1GYbStXO5GLmTksRn1xI2+3mc3Lw6vqdU4zXsBvq/th9t6E9luDxSppD811249x2NTNHxfj0nNy9Cb32WhGdWd0iudXcfdecYuMUMIenP29HFjUB2TkJPrnbFa8gsUv72A05a9y72piAIws/T1NTE999/zwMPPMCsWbPYtWsX06dP5+uvv2bp0qUkJiai1+tF8kS46jg7O+Pj40NQUBDDhg3j8ccf56OPPmL37t289957LFy4kISEBI4ePcozzzzDjBkzuPbaa3niiSd477332L9/P4WFhRiNRkdvinAVkNl7uHCpo+uhisbLPc9oNBISEoLFYiE5OZkffvjB0SH1Sk1NTcycOZNDhw4BMGLECNatW4eTk6iUJwiCIHTNze3nlXnqae0jiNtHFF8tMv56g9TDJPLuN/GfeBd2mxVD/nFkMjkuIfGdLgS3M9eWcuiRFOxWCwCht/yR4Osf6zSd1WTEWJCBa9RAZF1cJGkuOYVcpUHlFdzleuw2Ky3lZzDXlqHyCkLlE4pMruhy2lZjA8aCDJR6P9R+ER3KHRjy0nH2CkLp5gmArdWM4UwarpEDutzGloo8WioLcHb3Qe0fLfUoMRafRKFxQ9XFBWybuQVD/rH/bWvXMZ7L2tyIIf84Tq76tt4b/+tb0lyag1yplpIX0v6w22kuPom1uQFNUFyHBuytxgZaynJxjex/SdvS3X12tlZDHcaiLBQqLerAaBRdjOLoEHc3jqPNYqLpTBrYbGgCYzqVaTvb5Z7DNosJQ97RSz5W5rpymktzcNb7ofaL7PJc7rC91laay09jt5hx9vC/4DZI83RjH0H3joHdZqW5LBdLfSVqv4guz1/h4hz9nV8QejuTyURxcTEHDx7k+++/Z+fOneh0OqZMmcL111/P0KFDUSgu7f+jIFzNLBYLp0+f5sCBAxQWFpKVlUVqaiqNjY24uroSEhJCYGAgAwcOJDExEV9fX/z9/XFxccHZ2Vn8nl2lfonPKSKBIlzU9u3bue666wBYtGgRr7zyioMj6r0qKioYN24cRUVtpSj++Mc/8uSTTzo4KkEQBKG36m0JlKtVVxefL8bSWI1cqSLn349Q/b/G5jKlikFvH0Pp7nORuQWhZ13OOSwIl8vR3/kFoTerrKxk3bp1fPvttxw7dgwvLy+uueYapk2bxsCBA1Gpzl/aUhCE87Pb7VRXV5Obm0tNTQ3Z2dn88MMPFBQUYDAY0Gg0ODs7ExkZyeDBg4mNjSUsLAxfX1/c3d1RKpUXX4nwm/DCCy90+NkTxK3xwkUdO3ZMejxo0CAHRtL7+fr6smzZMiZMmIDFYuG1115j2rRpJCUlOTo0QRAEQRDOQyb76e60S7kLP/2l6TSe3N/pdd8xt4vkieAQ3T2HBUEQhJ5nt9s5ceIEr776Knq9nttvv5277roLHx8fXF1dRQNsQfgZZDIZ3t7eeHt7Y7fbGT9+PHPmzKGlpYW8vDx27NhBamoqeXl50igVT09PhgwZwoABA4iIiCAmJobQ0FCRyPyN68nESTuRQBE6qa6u5tixY5SWllJRUcGKFSuk90Qi4OKSk5N58skneeWVV7BYLDz11FNs2LDB0WEJgiAIgnAeav8ISG97rA3rd9HprcaGTq+5Rg8i7LY/9XRognBJunsOC8LPcebMGUeHIAi9kkwmo1+/fjz++ONMnjyZ0NBQR4ckCL9JMpkMtVqNWq0GIDQ0lDFjxgBgMBjYunUr27Zto6CggF27dvHxxx8jk8kYOHAgkydPJioqioSEBOLi4nB3dxfJTeGiRAkvAaPRyJYtW1izZg379++noKCgy+lUKhXl5eWiudklsFqtDB8+nKysLACWL1/OtGnTHByVIAiC0NuIEl69g6m6iJIN7+ISHI/v2Nsv2tfBWHySyt3LsTRUo3T3wTU8CX3ypE7N4QXh19Ldc1gQBEEQBOG3rrm5mZMnT3LgwAFKSkrIzs7mwIEDNDc34+vrS2RkJP7+/sTFxTFo0CCCgoLQaDS4ublJyRlBgF8ggfJL1BnrDpFAuXSZmZm88847rFq1CqPReNHp4+Pj+fHHH3+FyH4bNm7cyC233AK0lT7btm2bgyMSBEEQepvelkC5WpvIC4IgCIIgCILw29Xa2kplZSW5ublUVFSQm5vLjh07KCgoQKFQoFQq0el0BAYGMnr0aBITE4mIiCAgIMDRoQu9QI8nUBxNJFAu7uTJk/zxj39k06ZNnd5zc3Ojf//+Un1APz8/fHx80Gg0eHh4EBQU5ICIr1zjxo3j8OHDAGzfvp2BAwc6OCJBEAShN+ltCRRHjyQWBEEQBEEQBEH4JdlsNkwmE3V1dTQ0NFBYWMi2bds4cuQIDQ0NVFZWAm2DA373u985OFqhu/Ly8gAIDw/vsWWKBMpVxGAw8Morr/Cvf/2L1tZW6XUPDw9uvvlmZs2axahRo1AoROPJnrJ8+XLuvfdeAObOncvSpUsdHJEgCILQm4gEiiAIgiAIgiAIPa3987zo73HpTCYTZWVl7Nu3j5MnT3LDDTeIXtBXoF/iO61IoFwljh8/zrx58zh16pT0WlhYGIsWLeL222/HxUXU7P4lmM1mIiMjaWhowMvLi5ycHJGgEgRBECQigSIIgiBcaRxdtlsQerOmpiYaGxvx8fHBycnJ0eFckex2O1VVVdjtdnx8fEQC4DLU1taycuVKZDIZN954Ix4eHo4O6YrT2tqK3W5HqVQ6OhShm0QC5RKIBEpnX375JYsWLaKlpQUArVbLY489xsMPP4xKpXJwdL99v//97/nqq68A+O677xg5cqSDIxIEQRB6C5FAEQRBEK404n+FIHStsbGRDz/8kIyMDBYsWMCgQYPExf/LkJuby9tvv41MJuPhhx8mKipK7MduqK2t5V//+hcHDhwAYOjQodx3330iiSJcNX6JzynyHlvS/+zYsYMdO3b09GKFy/T++++zYMECKXkycOBAfvzxRxYvXiySJ7+SadOmSY/37dvnwEgEQRAEQRAEQRAEQehpNTU1vP766+zcuROtVss777zD0aNHHR3WFcVut5OTk8Pbb7+NyWTCZDLx9tu/3vGaAAAgAElEQVRvc/r0aUeHdsWoqKjgb3/7G2lpaTz99NM89dRTHDp0iNdee03q6yFcGpvNhtlsxmq1OjoUoRfo8REojr4bRYxA+cmSJUt47rnnpOcLFy7kz3/+sxh+9isrKCggMTERgBkzZrBs2TIHRyQIgiD0FmIEiiAIgnClEf8rBKGjmpoa/v73v3Po0CHeeOMNwsLCePPNN9m1axdvvPEG/fv3d3SIV4SSkhJefvllFAoFL730EjabjRdeeAGbzcZzzz2Hv7+/o0Ps1aqqqnjttdfIzs7mzTffxGQyIZPJUCqVLF68mLi4OB5//HG8vLwcHWqvY7Vaqaqq4siRI2RkZFBYWEhNTQ1msxmlUom7uztBQUHExcUxcOBAgoKCkMvlYmRUL3VFlPBy9IcpkUBps2rVKu666y7pODz//PM89thjDo7q6hUZGUlVVRXOzs4MGzaMpKQkxo4dy+jRo0X/GUEQhKtYb0ugjBs3DkCMJhYEQRDOy9Hf+QWhNyktLeWjjz4iPz+fRYsWYTAYSE9PZ9y4caxYsYK0tDQWLFjAqFGjcHZ2dnS4vZLNZiM7O5ulS5eiVqu56667OHbsGDabjZSUFD755BOMRiMPP/wwffr0QS7v8WI6V7zi4mI++OADKisreeihh6ipqeHNN99EJpPx6KOPotfrWbp0KYGBgdx3330iGUVb0qSiooLMzEz27NlDUVERGo0GvV6PTqdDp9OhUqmwWCw0NTVRX19PXV0dDQ0N+Pr6MnToUJKTkwkKChL9jnoZkUC5BCKBAocPH+aaa66Rynb95S9/4aGHHnJwVFe3a6+9ll27dnV6XaPRMGvWLO655x6GDRvmgMgEQRAER+ptCRRBEARBuBhHf+cXhN6ipKSEJUuWYDAYuO+++6isrOSdd97BZrMxcuRI5s+fz5o1a/jhhx+44447mDJlirhj/Rw2m41Dhw7xzjvvEBQUxNy5c9m3bx+rV69GLpczc+ZMRo4cyWeffUZ5eTkPPvggAwYMEEmUsxQUFPDGG28AsGDBAoqKinj//feZMGECAFu2bOH+++8nMDCQDz74AKVSyaOPPkpISIgjw3aopqYmdu3axaZNmzCZTCQmJpKUlERQUBCenp5otdoO1XusVisGg4G6ujrKysrIyMjg2LFjtLS0MHbsWCZNmoS3t7cDt0g4m0igXIKrPYHS0tLCqFGjyM7OBtrKdr322msOjkpYvnw5X3/9NdnZ2eTn53f5+zFq1ChefPFFBg8e7IAIBUEQBEcQCRRBEAThSuPo7/yC0Bvk/3/27js+inJr4Phvtmaz2fRGekggCQECCUV6EylKR0CaRBSFKxexXLFcBctVUSy8KigKiFJVinTBAqKA0pWWBEhCSCAJJED6lnn/yN25ibQkJCzl+X4+QMrMM8/MLpvNc+ack5bGK6+8glqtZvLkyaSmpjJjxgyGDh1Ks2bNmDlzJvXr12fChAmsX7+eNWvW8Nhjj9GtWzcRRPkvWZbZvXs3H3zwAdHR0YwdO5aNGzeybt06HnvsMSRJ4pNPPqFHjx707t2bOXPmkJSUxKRJk2jevLm4jkBKSgqvvPIKJpOJJ554gqSkJGbOnMnQoUMZOnQoUL4etWzZMv75z38SERHBjBkzsFqtvPjii4SHhzv4DG68c+fO8cknn3D48GE6duxI9+7d8fHxwWAwVPk5VVJSwrlz59i2bRsbNmzAy8uLRx55hAYNGojn5W1KPXXq1Km1OeC0adMAqOVhq6ysrMwhx71ZvP3226xatQqAu+66i3nz5t2WkfmFCxfSvn173njjDUJCQmjatKmjp3RVjRs3ZsiQIYwfP55//OMfdOzYEXd3dzIyMigoKADK7xr48ssvycrKomPHjiK9VxAE4Q6g1+sdPQVBEARBqLbOnTsrZR8F4U5z8uRJpk2bhiRJvPrqq2RlZTF9+nSGDRvGQw89RFBQEA0aNGDWrFmcPn2axMREzGYzb7zxBqGhoURERIhFVuDo0aO89tprNGrUiKeeeooff/yRL7/8kkcffZS7776b0NBQTCYTn3/+Oe7u7owaNYojR47w3Xff0axZMzw9PR19Cg514sQJpk6dipubG6+99honTpzgnXfeYfjw4SQmJqLT6dDpdDRv3hyr1crnn39Oy5YtGTp0KFu2bOGnn34iISEBV1dXR5/KDZOZmcnzzz/P6dOnefLJJ7n33ntxd3dHq9VW6/+kRqPBZDIRGxtLfHw8O3bsYN26dTRu3Fj0mLlNiQDKbSQzM5OHHnoIs9mMRqNh6dKl+Pn5OXpadeLPP/9k7dq1QHl5rJs9gFKRXq8nPDyc7t27M378eBo1akRycjI5OTkA7Nu3j5UrV9K+fXt8fX0dPFtBEAShLokAiiAIgnCrEcET4U6WnJzM//3f/2E0Gnn22WdJTU3l448/pmfPnowcOVK5EdLPz4+EhARWr15Neno6AwcOxNfXl8WLF6NWqwkLC6tUIuhOUlZWxu+//84777xDkyZNGD16NBs3bmTRokUkJibSo0cPNBoNkiQREhKCq6sr8+fPR6VS0a9fP06fPs2yZcsIDAzEz88PtVrt6FO64S5evMhrr72GRqPh+eefJzk5mVmzZtGrVy+GDx9+yXOrYcOGmM1mvvnmGyIiIrjnnnvYtGkTx48fp23btrd9Dw+bzcaRI0d46623cHV15bnnniM6OrpWxnZzcyM+Pp6srCwWL15MSEgI/v7+t+XN7HeyWn80Q0NDCQ0Nre1hhSp4//33KSoqAmD8+PHExsY6eEbCtWg0GgYMGMCvv/7Khx9+iJubGwDHjh3jnnvu4ccff3TwDAVBEARBEARBEARBAPjiiy9ITk4mMTGRo0ePMmPGDNq1a8fQoUMxGAyVtm3SpAn/+te/OHHiBAsXLqRfv35ER0fz1VdfcebMGQedgePl5OQopeZHjRrFDz/8wNdff83YsWPp06dPpRuMdDod9957LxMmTGD16tX8+OOPDBgwgMzMTBYuXEhhYaGjTsOh1Go17u7ulJWVsWnTJt58803at29f6XlYscyis7MzQ4cOpUOHDvznP/9h5cqVmM1mGjdufNsHT6C84sucOXMICgrihRdeuOq6tdVq5dChQ6xcuZIvv/ySr7/+mt27d1NcXHzFfby9vXn00Udp27YtM2fO5MCBA9hstro4FcFBar0HiqPdqT1Q8vPziYqKori4GGdnZ44cOYK7u7ujp1VnFi5cyPjx4wGYNWsWI0aMcPCMakdmZiYPPvggO3fuBMrvTF6yZAndunVz8MwEQRCEunCz9UAJCwsDIDU11aHzEARBEARBuBlt3LiRTz75hLCwMP744w9GjBjBiBEjlPd0sixjs9mUrAir1cqBAwd488038fT0JC0tjWHDhjFo0CCMRqMjT8VhCgsLmTdvHt999x1RUVFkZGQwYcIEunTpoizmW61WJElS7uK3Wq389NNPzJo1C4DS0lKefvppOnTocEdmoMiyTGZmJh999BELFy4kMjKSRYsWKVVorFYrW7ZsQa/X06ZNG+U6njlzhqFDh5KTk8OUKVMYMGAALi4ujjyVOldaWspHH31EVlYWTz31FP7+/lfc9tSpU3z66aekpqbi5uaGXq/HYrFw4cIFvL29eeihh4iKirri/iUlJcycOZPMzEyeffZZ6tWrVxenJDiAyCe6TSxcuFCJho4YMeK2Dp7czgICAli7di33338/UP5CP3z4cPbv3+/gmQmCIAh3grS0NNLS0hw9DUEQBEEQhJtS9+7defDBB1m3bh05OTn07dsXk8mEzWZDlmX27dvH+++/T2FhoRJIiY2NJTw8nCVLljBo0CCGDx9+xwZPAIxGI4899hj9+/fn66+/xtnZmXbt2qHRaJQA1Oeff84XX3wBlAcL1Go1rVq1wtnZmdTUVJ577jk6d+58RwZPACRJIjAwkKeeeorhw4fj6emJTqfDarXy448/cujQIXbu3MmePXs4cOAA27Ztw2q1otPp8PT0ZNSoUTzwwAO3ffAEysvk79ixgwceeOCqwZOkpCSeeuopTp48SWJiIi+99BKvv/46U6dO5bHHHsNqtTJ58mR27959xTGcnJxITEyksLCQn376SWShOMiYMWMYM2ZMrY55++dp3SGWLFkClL+ITpgwwcGzEa6HTqdjzpw5qNVqlixZQnFxMQ888ADbtm2745ukCYIgCIIgCILgWPYsRXvWoiDcSVQqFT179qSwsJAFCxZgNpuRZZnt27eTlpZGSUkJ27ZtIzQ0FLPZzMCBA4H/Law+8MADd0TJpGvRaDSMHj0as9nMiRMnsNlsmM1mfvnlFy5cuMD27dvRarX4+PhgMplo3749NpsNPz8/OnXqxF133eXoU6h1paWlnDhxgvPnzyPLMlUpGCRJEk5OTpW+lp6ezsyZMzl16hRqtZqff/6ZoUOHVmqSXlZWxq5du6p8DEmScHNzIzw8/Jbq4Zidnc1nn31Gu3btrto7+dSpU8yZM4eYmBgmTpxYae1Np9PRsmVLoqKi+Pzzz3nvvfd4/vnniYmJuWzjeS8vLwYMGMCqVato27at+FnpAPbg6/z582ttTPGqfRtIT09XMhSaNWtGRESEg2ckXC+VSsVHH31EZmYmW7duJSMjg8mTJysvAoIgCIIgCIIgCI4QHh4OUKWFtxutuLiYffv24eHhcdkGwTabjT///JOysjJatmwJlP8+LctytXu5Hj16lHr16uHq6qp8raysDK1We9lFtb/bu3cvTk5ONGzY0GF30VssFrZt24a3tzeNGzd2yBwczWq18ttvv3Ho0CHMZrNSOupqJEkiJSWFgoIC5XODwcCuXbvYunUrmZmZ6PV6Bg0ahEqlwmq1YjabSUlJ4dNPP61Sc2l7+Sp3d3c6d+5MYGBgrZxvXSkuLmbdunWcOXMGq9VapdcHWZb5888/lee//bqvWbOGH3/8EZVKhUaj4YEHHlC+V1RUxPbt25X+v9div44+Pj7ce++9N3Xmz8WLF1mxYgUrV64kODgYb2/vKmUwFBcXExcXh06nQ6VSkZCQwA8//MDBgweRJImmTZvStGlTJElCp9PRrFkzkpOTycjIuObYKpWKs2fPkpaWxoABAxg3btwtE0CxWq3s2LGDY8eOMXXq1CsGLq1WKzt37qSsrIyHH374ijcuu7q68uijj/Liiy+yadMmQkJCLpvBo1KpaNOmDevWrWPLli0EBgai1Wpr9dyEG08EUG4DFRuN9+rVy4EzEWqTVqvliy++oHXr1mRnZ7NixQqGDx9Ojx49HD01QRAEQRAEQRCEm86GDRt46qmnsFgs7NmzB29v70rf//3335k0aRKpqamsXbsWnU7HpEmTSE5OZvPmzZcNulzOrFmzmD59OsOGDeOFF17AxcWF33//nfHjx3Pfffcxbdq0q+5/9uxZ+vfvT3BwMIsWLSIkJKTG53w91q9fz+OPP46fnx/Lly8nKCjIIfNwtJycHBYvXoyHhwdNmzbFYrFcM4hiNBoZOHCgUj7d1dWVoqIijh07RnFxMWfOnMHDwwOVSoUkSXTo0EE51rWoVCry8vLYsGED/fv3p3379td/knVMlmUyMjJYvHgxDRo0qHJA0s/Pj6ZNmyqL/0ajkXPnzpGTk4MkSZw7dw6j0ahkWnTu3JkDBw6QnZ19zflIkkRaWhrJyckMGTLkpgz6VuTh4UGXLl34/fffGTFiBG3atMFisVxzP3sAz2AwYLVa+eGHH9Dr9fTu3Ru9Xo/VamXr1q1ERkZiMBj4xz/+cdWG6BWp1Wp27NjBl19+SdeuXeusXUBBQQHOzs5VCi5WVX5+PuvWraNfv35XDUBaLBb2799PQkICvr6+Vx3T2dmZgQMH8tVXX5Gbm3vFEmj2x3Lx4sXcfffdN30AVLi2Wg+gdO7cGYCff/65tocWruC3335TPhaL67cXLy8v3nvvPUaMGAHACy+8wN13333H1vkUBEEQBEEQBEG4kry8PE6dOkVZWRlLlizh8ccfV75ns9n47bff2L17N1arlby8PNRqNWlpaZw6darKd7QD7N+/n1OnTpGdnY3VagVg9+7d7Nmzh4KCgmsGUNLT00lPTyc/P1/JYnCElJQUsrKyKC0tJTMz844MoKjVajp06MD27duJjY1lzJgxymN6LfbsBpvNxq5du1Cr1Tz//PNs2bKF+Ph4Nm/eTJs2bTAajdxzzz107969SuOqVCqSkpI4fPgwvXr1qnZ2VFUUFhYqC+9VyZi6FmdnZ4YMGUJKSgp9+/ala9euSJKkBDL+/q+d/XOVSkVpaSkHDhygSZMmODk5oVKpiIqKqvS1wYMHM2jQoKuOXfHfH374gdWrVzN8+PA66fdhtVopKirCYDBcd2k2tVqNm5sbRqMRPz+/q/bruBJJkujfvz9Dhw5l8eLFmEwm+vTpg9lsRq1WI0kSXl5e1RrT398fo9GIm5tbna1FpaWlUa9ePTw8PGrl+SjLMunp6Zw4cYK33nrrqtsWFRVx9uxZunbtWqVMkZiYGMrKyjh//vxVt7v77rt56623SE5OFgGU20CtB1C2bNlS20MK12Av36XRaO7YtNvbWZ8+fejYsSNbt24lKSmJ7777jgEDBjh6WoIgCIIgCIIgCDeVxo0bExoaSnJyMsuXL+exxx5TFjXPnj2rNFL28vKiXbt2aLVa3nvvPcxmM/Hx8co4siyTm5uL2WzGYDDg4eFR6TgzZsygVatWtGvXDjc3N6C8fFfFf6vC3lugIlmWOXv2LCUlJTg5OeHp6anclW02mzl37hw+Pj6V7tSWZZn8/HycnZ0rldcpKyvj3LlzWCwWTCYTrq6ulxyvNhYrbwdqtVpZHLaX3VKpVKhUKqWxub2ZuSRJ2Gw2pbySSqWiR48e3HPPPSQnJ5Ofn8/EiROxWq0YDAaln4Usy8rz0b5/xWPYy4fZAwr2f+tCWVkZqampBAcHV3qOXS/788l+TvZghlqtrnQd7Nfafh3s5aUGDx6MSqXi008/RavVMm7cOEpLSys1mK94XezBrorBLFmWletq/15dPc9lWebcuXMUFBQQERFxST+SmrKfx/Hjx0lJSSEqKoqQkBDOnj3LoUOH0Gg0tGjRAo1Gw+7du7l48SJxcXF4eXkRFhaGLMvcd999qNVq/Pz8lIBSTk4O+/fvx9XVlYSEBCwWC7t27cJisRAbG4unpyfp6ekcPXqUyMhI6tevr5Rkq8sMnrKyMk6ePIkkSZe83tZ0vEOHDhEREaG8Rl+JwWDg4YcfJjg4uMrj2wN+fw8IVuTm5kaLFi3YtWsXrVq1wtnZuVrnINxcRAmvW5zNZuPYsWMA1K9fX9TVu029+OKL3HPPPUB5urgIoAiCIAh14aeffnL0FARBEAShxmJjY4mNjSU5OZnk5GQOHDigBEbOnDnDtm3bAPjnP/+Js7MzJ06cYP369WRlZdGjRw9MJhMpKSl8/PHHHDt2jJKSEkwmEw0bNuTBBx8kKioKgK1bt7JixQp0Oh0REREsXLiQpUuXApCbm8tDDz1Ely5dGDVq1GXneaUFtzNnzjB37lx27NhBUVERzs7OxMXFMWHCBPz9/fnmm29YtGgRgwYNYsyYMcp+Bw8e5KWXXmLgwIEMHToUrVbL0aNH+fzzzzl48CClpaV4e3vTsWNHxo4dWynIcrOXNbpRKi4QHzt2jDVr1hAdHU3Xrl25ePEi69evJzc3l1GjRuHh4cHPP//Mzp076dGjB3Fxccqib0xMDD4+Pnh6eipBg9zcXNatW8e5c+cYN24cOp2OjRs3cvDgQbp06UKLFi1ISUlh2bJlNGrUiD59+lwyp7pQVFREeno6UF794nqDDPb97XP+888/2bhxI506daJFixZkZ2ezatUqXF1dGThwIGq1mvXr13Py5En69OlDSEiIch3tTc+dnZ2Vhee0tDTWrl1LQEAAffr0obS0lFWrVnH+/Hn69u2Lr6+v0ofmnnvuoWnTppWCNnVBkiQloy01NZXQ0FAMBkOtjG2xWFixYgXvvvsuL7zwAo888gh79+7lqaeewtXVleXLl2MymXjllVc4fPgws2fPpmvXrkqQLzIyUpkjlK8f7tu3j3HjxtGkSROWLFnChQsXePrppyksLOSdd96hU6dOrFmzhjfeeIPJkyczadKkGs29uLgYs9lcpeeUSqXCYrFQWlpKamoqAO7u7tf1fCwrK+P48eNXbRxv5+TkpPSIqYrz589js9kwmUzX3KdTp058//33yuu5cOsSAZRb3OnTp5U7XOxv5oTbz1133UVcXBz79+9nx44dpKWl1UkaryAIgnBns5diFQRBEIRbkdFopEePHqxZs4azZ8/y888/KwGUbdu2kZOTg8FgYPz48QAcPnyY1atXk5mZye7du2ncuDGTJ09m3bp1ODs7ExwczIkTJ1i1ahW7du1i/vz5BAQEsGDBAtasWYPRaKRXr15s3LiR7du3A+W1/OfNm0dJSckVAyiXW8w9deoUTz75JN999x0lJSV4enpy/vx5NmzYwLp161i9ejW//fYba9eu5a+//mLIkCHKgtx7773HihUr8PX1ZeDAgWRnZzN48GCOHj2KLMt4eHiQk5PDmjVrOHDgALNnz1bmITJQLpWUlMS7777LfffdR/v27cnJyWHu3LkcPnyYHj164Obmxvfff89HH32Eh4cHjRs3VjIqTCbTJQurZ86c4fPPP+f48eM88MADmEwmli9fztdff41eryc+Pp5Dhw7xxhtvcN999yml2av72JjNZsxm8zW3kyRJWUcqKioiNTUVlUp1xebZVVXxeS3LMrt37+att95CrVbTrFkzMjIy+OCDDwgODqZXr17odDpWrFjBr7/+SmxsbKVeQJfLBjhx4gQzZ86kZcuW9OrVi4KCAhYsWMDx48eJj4/H09OTX3/9lenTp+Ph4UGTJk1qfB4lJSXXDLrYgycWi0UJlAGEh4fXSqN1lUpFbGws/fv3JyYmBpVKRUBAAH379sVgMODk5IRGo6Fz5840aNCAgICASs+Zv2cVSZJEYGAg/fr1IyQkBI1Gg7OzM71796a0tJTAwEDUajUxMTH069eP2NjYGmcmZWVlkZ+fX+XAVUlJCVarlYKCAo4dO0Z4eDienp41fn2yWCycOXOG1q1bV2n76pznzz//jKenZ5X+vzRt2pS5c+dSWlpa5fGF6/fyyy/X+pgigHKLO3funPLx9f6wE25uw4YNU8q1rV69ulI9X0EQBEEQBEEQbk/5+fl11rj3dtS/f3+efPJJiouL2blzp3L9vv76a6C8b6iPjw9QvshmLwFUXFzMvn372LdvHzabjaVLl9K7d2+lhHJaWhppaWkEBAQofUvKysqwWq0sWrSIuLg4pkyZQr169Th+/PhVS/lcblHwhx9+4JtvvgFg48aN3HPPPSxdupTHHnuMvXv3MnPmTO666y4WLFjAmTNn2Lx5M3379qWwsJB58+ah1Wpp06YNzs7OPP744/z1119ERUWxadMmgoODefvtt3nhhRf45JNPGDFihNLUXGSgXComJoann36a6OhodDodvr6+jBs3jrNnz+Ln54dKpaJnz554eHjQpk2bSn0hLvfY+vv7M27cOPLy8jCZTOh0OoYMGUKTJk3o2LEjarWaxo0b8+KLL9KoUSN0Oh1Q/cfm9OnTlfryXI3ValWCLYWFhSQnJxMTE4Orq2u1jlnR38+9VatWPP/883Ts2BGNRkNwcDBPP/00rq6uODs7o1arGTRoEK1bt1ayJa40FkBERASTJ08mICAAjUaDi4sLY8eO5fz584SEhKDVaunYsSMArVu3vq7F9yNHjlQpGAXlgSt7ya2cnBwkSaJBgwbXXRZNrVbTtWtX2rdvj16vR61WEx0dzfPPPw+g9K8ZP348Npvtmk3YVSoVDRs25NVXX0WtVqPVatFoNEyePBlJktDr9Wg0Gjp06ECrVq3Q6XQ17nlSWlpKSUkJFovlmtvaA1FQ/rwsLCxUgno1zUQxm80UFxdfsyl8daWmpvLtt9/yyCOP4O3tfc3tQ0JCKC4uFgGUG2zq1Km1PmbdFFMUbpiKje5qK01QuDndd999ysei15AgCIIgCIIg3Bnmz59Pfn6+o6ehqOuyQtfL39+ffv36AbBv3z5SU1M5ffo0v/zyC2q1mv79+19xX3d3d4xGIwD//ve/+eCDDzCZTHz55ZfMmjWLli1bApdf3LXv5+TkpARP0tPT+f7771m7di1r167lxx9/pKCg4JLrZ7Va2bx5MzabjdatWxMSEsKBAweIjo7Gz88Pm83GgQMH6NChA0FBQZSWlrJ+/XpkWWbevHnIskyjRo1o0aIFsiyzePFiAHr27Elubi4HDhwgPDxcqWKwfv165TxEBsql3N3dadOmDZGRkajVagwGA02aNKF169bKonV4eDgdOnTA39//qovW9jJUcXFx3HXXXcqidIMGDWjbti316tUDym+I7dixIzExMcqidXUfG3tAsKSkhNLS0qv+qRgcsGdcHD58mLNnzyq9Xaqr4vNakiR8fX1p164dgYGBqFQqTCYTCQkJxMbGolarlSbxLVu2rFLgxs3NjZYtWxIVFYVKpUKr1RIbG0t8fDwuLi5KhkW7du2Uvh81PQ97Salr/bEHCeznbrPZyM7OJikpiZKSkhod385isbBs2TL69OnDqlWrsFgs7N69m8TERCZOnEh+fj6lpaVMmTKFYcOGsXv37qsGz2w2G7t372bIkCFMmTKF0tJS8vLymDhxIg899BC7d++mrKyMVatW0bdvX77++usqBUCudryq/Pn7nGVZprCwkLS0tBr/7LP/X7jeTCD7z7uioiK2b9/O1KlTadu2LZ07d65ScEmn0yFJkgig3AZEBsotruIPqNpIERRuXqGhofj7+3P69Gn27Nnj6OkIt4COkz9z9BSEG2Drew87egqCIAiCINSh/IGEypQAACAASURBVPx85s+fz5gxY0QmShU988wzLFmyhOPHj3Po0CFWrVqF2WwmOjqa5s2bX3G/Jk2a8OyzzzJt2jT27NnDnj178PT0JCoqimHDhtG6dWulmfW1lJSU8PHHHzNz5kylXJLBYODrr79WFs0rsveiOHz4MAMGDFDKa9l7Ari4uBAQEMDAgQN57bXX2LNnD0eOHOGrr75CkiRat25Nw4YNycrKUhZuly1bxoYNG4DybJmMjAyASndO38zBMEc5cOAAzzzzDD179mTKlClkZWXx+uuvc+zYMRYtWkRoaCiLFy9m7ty5TJs2jcGDB1+xH60sy2RkZPDKK68oPTyMRiNz5sxh9erVPPPMM4wcOZLdu3fz2GOP0atXL95++21l3+qw38lfkwCIPYiSmpqKLMs16olScXtZltm6dStTp07liSeeYPTo0Rw/fpxnnnmGevXq8eGHH6LVapk9eza7du1i+vTp1yy3dPjwYaZMmUJcXBzTp0+noKCAt956i4yMDN59912ioqJYt24d77//Pi+99BKDBg2q9nWoOH97VklN9rNnooSEhFzXzc4FBQWcPn2aoqIiZFmmtLSUnJwcpeSVvYn9mTNnrrlIL8syZWVlZGdn4+Xlpcw1NzeXwsJCZf/CwkLOnDlz2WDvjWKz2SgoKCA7Oxs3N7dqZ/PYg8M1DQYCXLx4kb1795KUlERKSgo5OTm0bt2aIUOGVPlnsf34Nc3kEW4etR5AmTdvXm0PKVxFxbTgwsJCB85EuBHi4uI4ffo0OTk5nDt3TpRtEwRBEGqVPd25LtKeBUEQhJoTQZTqiY+PJyEhgd27d7N69Wq2bt0KlJf1qV+//hX30+v13H///XTr1o0dO3awatUq1q1bx/bt2zl69ChGo5GxY8dWaWFZp9MRHx9PfHw8Fy9eVO6ODw8Pp7i4+JLtXVxcAIiNjWXcuHHKXdlqtRqTyUSrVq3QaDSMHz+e1157jaSkJObMmUNKSgpGo5F+/fqh1Wor3cnfo0cPOnTooMxXq9Xi5+dXaaFaZKBcSq/X4+XlhaurK5IkodVq8fLy4uLFi2g0GiRJwmQy4ePjU6XG0FqtFg8PDwoLC5UG325ubvj5+WE0GpEkCScnJ3x8fCqVLLrRj439zv/09HT0ej0mk6na+1dkMBjw8fFRzlGr1eLj46P0trBfRy8vL6Vs2dXodDq8vLxwc3NDkiSlb0tpaSlarRZJkjAajXh7ezu0Qossy1itVs6ePYtGoyE0NBSNpvrLrxqNhv79+9OyZUulRFmzZs2YOXMmarUad3d31Go1L730EkVFRURERFyzhFfTpk2ZM2cORqNRKdn11ltvYbVaCQ0NRavV0rt3bxo3bkxgYOAVA4N1TaVS4eTkVOM+KDqdDo1Go5RbrImsrCzmzp2LVqulbdu2DBkyhJiYmGo9t+wZNOKG91tfrQdQxowZU9tDClfh4eGhfHzmzBkHzkS4ESo2VTt16pQIoAiCIAi1atq0aYAIoAiCINyMRBClekaPHs3u3btZsmQJUF7+p2PHjkqg4nK2bdvGsGHDaN++PQsXLmTIkCGkp6cTGhrKuXPnSElJUe76/jv7wqW9NJIsywwZMoQhQ4Zcsu2+ffsqfW7vmQDlj/Po0aOV76WlpTFy5EgGDRrEE088QUBAACNHjuSrr77iww8/VDJrevXqBZQHYgIDAzl16hTu7u6MGDFCWbxbtWoVzz33HB9//DGtWrWq8rW80zRv3pz58+cr5dgCAgKYNm0aFosFT09PVCoVo0ePZsCAAXh4eFx1kVmSJIKDg3njjTewWCxKgGTChAkkJiZiMplQqVS0bNmS5cuX4+zsrNwo64i7/1UqFa6urlft4XMlFRe6JUmiW7dutGjRApPJhFarJTIyUln8t5fcmjx5MmazuUqvaY0bN2b27NlotVolWPjSSy8p11Wj0dCvXz+6du2Km5ubQ+/6tweMXF1dazwPm81Geno6v/zyC127dsXLy4uzZ8+yY8cOdDodoaGh6PV69uzZQ05ODh4eHtcMeuXl5fHLL7/g5+dH/fr1KSsr4/fff8diseDi4oLRaCQjI4MtW7bQvn17/Pz8ajT36yFJEgaDgZCQELy9vWsUQNFoNLi7u5ORkVHlRvJ/V1JSgiRJDBs2jG7dutVojGPHjuHl5SUCKDfYzz//DEDnzp1rbUxRwusWZ6+3abPZRADlDlAx1TovL8+BMxEEQRAEQRAE4UYyGo0iiFIN3bt3x9fXl+zsbAACAgLo2rVrpW0q9gCx16lXqVQsXboUT09P+vTpw/Hjx4HywERYWBhqtfqyC6L2xyMrK0tp0nzffffRtGnTS7a1B1vsx1apVAwcOJBvv/2WgwcPcv/99/PAAw+QlZXFO++8Q1pamtL0HWDSpEl8/fXXSsmdJ598stIi46uvvsqECROYNWsWbm5uJCQksGvXLmbNmkV+fr7yu2TFcxeZKP+TnJzM/PnzSUhIYODAgeTl5bFo0SKysrJ46qmn8PX15fvvv2fTpk2MGDGCtm3bolKpLgl42K9pTk4OCxYsIDs7m5dffhm9Xs+qVav4448/GDBgAJ07d+bo0aN89NFHtGjRQrkx+UY/Jmq1Gj8/P4KDg2uUeVDx/GVZZs+ePSxatIh+/frRpUsXTp06xdy5c/Hw8GDcuHFoNBq+/fZbUlJSGDNmDA0bNrxsjyX78/PEiRN88cUX1K9fnwcffJDi4mIWLFhAbm4uDz30EIGBgfz666989913DB06lLZt2173NakJe0ZRSEhIjUqh2VmtVrZs2cJbb72Fk5MTjRs3Jikpiffffx+TyUTPnj2RJIkFCxZw8OBBYmJiCAoKIicnh8zMTOU62rPfvL29SU5O5u2336Zp06b07duXgoICZs2aRVFRESEhIQQEBPDbb7/xxhtvMGXKFOLj42s0d61Wi8FgqFIQUJIkCgsLsdlsSuZJcHBwjYMn9uMHBgaSlJRUo/0rjlOTYKLdH3/8QXh4+HWNIVRfly5dgNoNQosAyi1Op9MRFBREeno6KSkpjp6OUMcqvlG/nlqOgiAIgiAIgiDcWu6++242b97s8CCKfXF3/vz5N/zY1REYGMj999/PrFmzsNlsDBs2jLCwsErb+Pr6KiWY6tWrR0REBI8//jjvvPMOs2bNYs6cOVgsFrRaLY8++ih9+/YFICEhgVWrVuHp6aksjMXGxhIdHc2RI0d46aWXcHFxITw8/LIBlKCgIJycnPDw8FBKF7Vo0YLp06fz5ptv8s033/DNN98A5ZkzEyZMYMKECcr+jRs3pnfv3qxYsYKYmBiGDx9eafxhw4Zx+vRpZs+ezbRp05AkCVmWiYiI4Nlnn1UCSWFhYciyjEajwcfHpxau+u3h1KlTfPPNN5jNZvr27cv58+f5/vvvOXLkCA8//DDe3t5KcCAhIYHWrVuTmZnJ77//TnFxsbLoGxERQUJCAnl5eWzcuJFjx47xr3/9C7Vaza+//sq3335LdHQ0nTp1Ii0tjYULF1JUVMSIESOA6i/+2fvmVKVclL3MlJ1araZevXoEBwfXyt3ysiyTkpLC0qVLiYyMpFOnTpw9e5aVK1cSFBSkvI78+uuv/Pbbb/Tq1YsGDRqQlJTEgQMHlL5BOp2OJk2aEB0dzenTp1m5ciWtWrVi1KhRlJaWsnnzZlJTU+nbty/+/v4cPHiQpUuXEh8fT5s2ba5r/lXtd2Rvhg7lwQBnZ2fCwsLw8vKqdu+OitRqNe3bt0eSJFq1aoVarSYyMpLHH38cnU6H0WhEq9UycuRIsrOziYiIQJZl1q9fz/LlywkMDAQgIyODwYMHM3LkSOrXr8+TTz6Jv78/Wq0Wo9HII488gsViISIiArVaTatWrXj22Wdp165djbNn/P39q1x+S6VScfToUUpLSzEYDISGhl73tdPpdERFRV33z6ma9MGxs1qtbN68mSFDhmA0Gq9rHoLjiQDKbSAuLo709HTOnTtHenp6pTJPwu2lYlOwqtQIFQRBEARBEATh9mA0Gm+KIMoXX3wB3PwBFJPJxNSpU+nduzfOzs60aNHikm2aN2/OZ599htVqpVGjRmi1WiZOnMj9999PcnIyubm5mEwmoqKiCAgIUMp/TZ48maZNm9KkSROlZE6jRo1Yt24de/fupaSkhAYNGtCoUaPLzs3T05P169crd6lDeb+IwYMH06FDB3Jycjh+/Dgmk4nQ0NBKx4byevoffPABiYmJREdHX7I4ZzAYmDx5MsOHDycjI4P09HSCgoIICQmhXr16SnZBjx49WLZsmZJ1IJRr0aIFCxYswNfXFycnJ4KCgnj99dcpKioiKCgItVpNYmIiXbt2JTo6GrVazf79+/nwww9p0KABOp2O48eP06hRI5o2bUpoaChvvfUWxcXFeHh4oFarmTRpEsOGDSMyMhKVSkWbNm1YtWoVfn5+SlCuunffe3l5Kb1ArsZ+x//p06eVu/4DAgJq3KvjclQqFd27d2fZsmXUr18fjUZDgwYNmD17Nnq9Xinh9eSTT5KYmEhsbCxms5lt27axYsUKQkNDkWWZkydP0q9fPyIjI2nSpAmffvopbm5uaLVa3NzcmDZtGsXFxTRs2BCdTsfAgQOJi4sjKiqqxgvwarWaoKCgKm1rb+Juz+pydnYmPDz8ujJP7FQqFQ0aNMBkMlGvXj1UKhW+vr507doVtVqNk5OT8twpKSnB19cXQGkSbw+6zpgxQ6lY4+vrS48ePTAYDMoYnTt3xmq14ufnh1qtJiIiAoPBoFS8qQmj0VjloIG93JlGo7nuzBM7jUZDdHQ0paWl/PHHH7Rs2bLaY0iSpPQsqonDhw+Tn59P8+bNHdZLRqg9tR5Asb+JEr1QbpxWrVqxevVqALZu3crIkSMdPCOhrtjTz6Fy/xtBEARBEARBEG5/N0sQ5VYgSRLe3t707t37itvodDql1IedwWAgPDyc8PDwK+7n4uJCv379Kn1Nq9Vec7+KLlebXaPREBgYSGBgIM2aNbvivva+GlcLejg5OREaGkpoaCjt2rW77DYGg4H+/ftXab53EpVKpTShrvi5xWJRFpTVajU6nU65Q7+4uBgfHx9eeOEF3NzcmDt3LsePH1cCFFqttlLGh06nQ6/XK/vbx6sYAKnune8mk6lKjd8lSSIvL4/s7GzUajU+Pj4EBwfXWvAEyueuVquVRuX2pu96vR69Xq+U5bKXSFKr1ciyjNlspkWLFkycOBGr1crs2bOVbBT7gn/FxWidTocsy8rjotFoKl3XmlCr1QQEBFRpW6vVSmlpKRcuXECn0ymZJ7VRfs1sNrNgwQJmzJjBiy++yNixY9m+fTvPPPMMJpOJb7/9FpPJxKRJkzh48CCffvqpUurPw8ODqKgoZFnGw8MDSZKw2Wzs2LGDRx55hKZNm7JkyRIuXrzI6NGjKSoq4u2336Zjx44sWbKE119/naeffpqJEyfWaO7VPX+tVouPj0+tXTsoDxa1bNmSTz75hISEhGoHgyIiIvjXv/6lBKaqQ5ZlNmzYQMuWLUVw+jZR83yoK0hMTCQxMbG2hxWuouIbrw0bNjhuIkKdS01NVT4ODQ113EQEQRAEQRAEQXAIexClYk+U/Px8R09LEG4bf/zxB6NGjeL//u//KC4u5uTJkzz77LMMHz6c9PR0rFYrc+fOpX///qxduxaz2QyglEQymUxKdoAsy6SmpvL0008zYsQI8vLyKCkpYcaMGQwcOJAVK1ZgtVr59ddf6devH2+88QYlJSVA9Reh7UGJa/2xU6lUeHt7ExwcXOsVLmRZZuPGjQwePJhvv/0Ws9mslEB77rnnuHjxIiUlJbzzzjskJiayf/9+ZX56vR6j0YiLiws6nQ6VSoUkSezfv5+HHnqI6dOnU1ZWxvnz55XAwpEjRzCbzXzzzTfcf//9rFu37rrKnlfnWtrLptlLT9VWAECSJHx8fGjUqJFS0sqeEdegQQM0Gg0qlYqwsDBiYmIqBc9kWVbKT1UMxJlMJmJiYggPD0elUqHRaGjYsCFRUVGYTCblOdGoUSN8fHxuWB+eoKAgvL29r6ts198ZjUbuvfdejh07xs6dO6u9vz1DsCblt1JSUvjrr7/o378/bm5u1d5fuPmIEl63gbi4OIKDgzl58iSbNm3i4sWLVbrrQLi1yLLMgQMHgPIfLvZavYIgCIJQWx588EFHT0EQBEGoApGJIgh1JzAwkMGDBxMXF6eUiurRowfNmjXD3d0dSZJISEhgxIgRREVFKdkO9sXqiovWkiTh4eFBr169yM3NxWAwoNFoaNeuHc7OzjRq1AiVSkVoaCgjR46kefPmSiZIbTZA/jt72a6KJcNqkyRJREZGMmzYMGJjY5WF+QEDBii9fzQaDe3bt6devXr4+/sr+9oDH/bzt19Pf39/BgwYQFhYmJKN0r17d3Jzc/Hx8UGtVhMbG8uwYcNo2LDhDVv8t5ft8vHxqdUAgEajoW/fvrRv3x5vb280Gg3NmjXjnXfeQa1WK8/F5557jrKyMvz8/K56fJVKRVxcHLNmzUKv1ysZT6+//jo2m005Rq9evWjVqhUeHh5K9lBdc3Nzq5PjREVF0a9fP5YtW0bDhg3x8vKq8r6pqamsWLGCu+++m7i4uCrvV1JSwoYNGwgNDa3WfsLNTQRQbhODBw/mvffeo7i4mMWLFzNu3DhHT0moZQcOHFDqat51110Ono0gCMLNa8aMGY6ewi3L3lx36tSpDp2HIAiCcG0iiCIIdSMqKoopU6ag1WrR6XT4+PgwduxYbDYbLi4uqFQqevToQefOnXF2dq5ULupyi8B+fn6MGzdO2V+SJAYOHEifPn0wGAyoVCpiYmJ45ZVX0Ol0SjZIXS5cu7i44OLicl2lrq7GHmSKiYlRgkaBgYFMmjQJlUqFwWAAyteyLBYLzs7OlUqcXU54eDhPPvkkarUajUaD0Whk9OjR2Gw2jEaj0nQ9ISEBZ2fnG7Lwr1ar8fb2vq5eGZdjH8vd3b3Sa7rBYLik73HFcmNWq7VSZkzF8SRJwmAwKO/37ezN5u1cXV1xdXW9ZC51qa6OYe8t9dprr7FhwwaGDh1a5VJ1Bw8eZMaMGRgMhmoFQvbv38/27duZOHFipeso3Dh1UbFHBFBuE4mJibz//vvIsswHH3zAgw8+iF6vd/S0hFq0fv165eOOHTs6cCaCIAiCIAiCINwMRBBFEGqfvVQS/C8LomIFCJvNpvTysH9u385ms1X63P5vxTJAsizj5OSkZH7YbDbUarVS6udypZdqW10ETiougttsNjQajbKAbO8F4+LiAvzvulTMfrGXQrNv//frqtFolP3tWSp/f1y0Wq3SI+V6SnhVlb18V20qKysjKSkJk8mkBJWqGmCw2WxkZmaSk5PDn3/+CUBubi4qlYoDBw5cMUPl7881+/HUajVJSUmYzeYbltFT24KCghgwYABz584lOjqahISEKu3Xtm1bFi5cSMOGDat8rKysLN5880169epF8+bNb9lrdqur2P6gtogAym0iLCyMPn368N1333Hy5ElmzZrFE0884ehpCbVElmUWLlwIlKdx9unTx8EzEgRBuPkNHz7c0VMQBEEQhDp3o4MoL7/8cp2MKwiOZLVa2bZtG1arFYvFUu399+7dS2pqKosXL8bZ2ZmdO3dy5swZvvrqK7Ra7TWDIRUbx6tUKk6fPk1BQUGtloS6EQoLC/n+++9JS0ur9r42m43ffvuNwsJCvvzyS2w2G3v27MFgMNQow0OSJA4dOkRhYeEts5Dt6upKVFQUK1euZN26dZd8v2JpuMuRZVkJukyfPh0of26npaXxxhtvVHmcisrKymjatOkt3Sqgffv2pKWl8e677zJp0iTi4+OvGfjy9PSkU6dOVbpGVquVw4cP8+abbxIXF8f9999fJ6XxBMeR5FoOaVd80XeEixcvOuS4N4NDhw7Rrl07rFYrrq6u7N27Fx8fH0dPq04sXLiQ8ePHAzBr1ixGjBjh4BnVre+++46RI0cC0KtXL5YuXergGQm3go6TP6v2PgaNhElrQ6uqzmu4RLFV4lxJtQ8n1IKt7z3s6CncdOwlvEQARRAEQbiTFBYWsnnzZgoLC3F3dxeZKIJQRUVFRWzZsoW9e/de11qWLMuVFlv//nl1xzKZTPTv379OytHUhYKCAtauXUtycvJ1nTdcurZ4PeNFRkbSp0+fGjUDv9GsViuFhYWUlpbWeIwrXbvreW7r9XqlTNqt6sKFCyxZsoR169bx4IMP0qNHj1rpLVxWVsbOnTuZN28eUVFRPPzww9XqtSLcGmo9A6VTp061PaRQRY0aNeKhhx5izpw5XLhwgXHjxrF8+fJbJtIuXJ7VauXNN99UPn/88ccdOBvhdiYDvk4WOjcJpp63OzJVe4NltdrIyD7Lr4ezSC/UUpuvOOXv8WQqDSpLXO5lTbbv8LfvSRW+oJyT/N/vVNhW/vu+VzjOZef1t23lCn8jX/4Nv/zfYzppNfi4u+DspOVCYSk55wuwWGxI0o2pNysIgiAIwu1BlPMShJoxGAzcfffddO7c2dFTqUSlUinlqG4FRqORAQMGXLOXyY2mVqtvmeuoVqtF34w64urqyqhRowgKCuLzzz8nOTmZkSNHUq9evRr93i3LMhcuXGDlypUsX76c/v37M2jQIPH43aZqPYDy888/1/aQQjW8/PLLbNq0idTUVH744QdmzJjB008/7ehpCdfhs88+46+//gKgXbt2dOjQwcEzEm5nBo2NhNgImkRHVvkOFVmWyTl7DpPzPtbsSiWzSF1rQZSmEfW4t0003m7ldwtJksSmP5L49a9ULhb9764cWZYJ8fPgnpYNiQ3zKw9wSHAkPZuVvxwkO68AXw8jdyc0ICEqCKtVJvlULnPW7EQCPFwMPDW0E0768gCQDKSdzuPdZVvQairfZWO12XhySCcCvV1RqcrPNCv3Ikt/2k9GTj42WWb0PQk0axCISoK9yZls2p1EVu5FJcgiSVDf35POzSOJCfXFzeiERq2i1GzlQlEJp3LOs/NwOnuTMykzV7+EgCDUVFRUFABHjx518EwEQRCEmhBBFEGoPkmSKvXOEGpGkiR0Op2jpyEIV2QwGOjduzfBwcFMnz6dxx9/nEceeYS4uDi8vb3RaDRIknT5GyD/uz5itVrJy8vj4MGDfPbZZ+Tm5jJlyhTatWsnXkNuY6IHym3G1dWV+fPn0717d8xmM6+//joREREMGDDA0VMTaiApKUmpMaxSqSploghCXZAAF6Ox2r9ku7q6IkkSxSWlbP7rNFlFqloJovh5utC2cTjBPm7K10pKzRw9mcOFotL/JYDI0DDIm56toogM9Fa2ddZp2fxHMtnIOOt1xIb70y2+AVabDRdnfXmkRAInvYYuzSNwdvrfG/6LRaX83/JtleYjyxDs687dCZEE+fzvGqWcymXD70fIyCl/Y9U43J9u8ZEAWKw2th9KAy4AEhq1ilbRwQzuHEeT+v54uTpfkup/oagUPw8TB1KyauEqCoIgCIJwJxFBFEEQBEG4ssaNGzNz5kzWrFnD0qVLWb16NdHR0dSvX5/AwEC8vLyUkmU2m42ioiLy8vLIzMwkNTWVI0eOkJeXx1133cWQIUPw8fER1SNuIvZswtpM8hABlNtQfHw8r776KlOmTMFqtTJ27Fi0Wi333Xefo6cmVEN+fj7Dhw+nqKgIgIkTJxIXF+fgWQl3gppURtVoNISFBNOzgwWz9Xc2/ZXNudLaaHgoXZIJ06R+Pfw8XDieeRZ7SodaJRFez5Ng38oLA/+t1PW3L/y3BFelYaVLztvkrKdFdBB/HM5QMkdkWaZlVDAuBn3l41zrov33+zabTJMG/gy/uzmtYkLRasqv0cWiUs5dKMLZSYu3mwtatYriUjNllpsr/V0QBEEQhFtDXQZR7AsSN1u5I0EQBEGoCkmS8PDwYMSIEfTs2ZN9+/Zx6NAhtmzZwoULFyguLkalUqFSqZBlGavVil6vx2Qy4e/vT9euXYmLiyMgIOCazeiFG2/Lli21PqZ4lG9TEyZMIDMzk5kzZ2KxWBgzZgzvvfceo0aNcvTUhCooKChgyJAhJCUlAdCsWTP+/e9/O3hWwp3Icu4spakpqF1c0YdHIl0lJVWn09EgIowuRSUkZ/zIuTN1M6dAbzeCfd3Zm5xJqdmCLMsE+boTEeCFXlu7P9a6xTfgj8MnsUdhrLJMp2YRGPTVT82VAU9XZzo0qU9CVDBajYqiUjO7jpzk+11JnC8owUmnIdDbDT9PF7b9mYrVZqvV8xEEQRAE4c5RV0GULl26ANfXkFgQBEEQHE2lUuHj40O3bt3o0KEDFy9epLCwkOLiYoqLizGbzWg0GgwGA05OThiNRlxdXdHr9ahUtXHDqHCrEAGU29hrr71GWVkZs2fPpqysjH/84x/s2rWL6dOno9frrz2A4BDZ2dkMGzaMXbt2ARAYGMjSpUtFLVHhhqmYsFG4ewfFhw4gOTmh/sOE2lShIdp/G6lrvHxwbhKP2s0dnU6Pv58PJr0aqN3sCbPFilqlQqWSaBjkg7e7kYzs88gyNAjyITzACyjvUSLLoFFX4w3N3xYASs0W9FoNXZpF8ObCH5VN6nmaiArxQafRUGaxolWrlFTdayahyDL1AzxpXN8fJ135j98jadnMW7+LfSmnsMkyEmDQ6/B2cya/oLjq8xcEQRAEQbiMvwdR3n//fZ544glRzksQhJteUUkBJzNTwepBYJAfeq0KjVqFqJQk1DaVSoWTkxNOTk74+Pg4ejrCTajWAyhhYWEApKam1vbQQg1Mnz4dk8nEO++8gyzLzJs3j507d/L222+LZuQ3oe3btzN27FgyMjIA8PPzY/ny5dSrV8/BMxPuVLbSYsxnTqF290Tj4YnKaKwQKZBBBpWTE1S8+0KWqYt3tZm5F/B0dcbkrKdxfX8C9DQVVwAAIABJREFUvFw5mZ2PSiUREeBFiK87BcVllJSZMTnrqxdAqcAmyyRn5NI43B8/TxPRIb4kZeRgs8m0ig7GZNAjSZB19gL+Hib0uqr9KJVlCPP3JLyeJwAFxaXsTT7FgWOZAKj+e81KysyczD4vfjEQBEEQBKFWGI1G5eOwsDARPBEE4aZ2oTCPQ+l7OXD8d9zycrAdLmFFw0ep5+9OiK8zIb4GAn2ccNKpHT1VQRDuELUeQElLS6vtIYXr9O9//5v4+HjGjRvHxYsXOXToEPfeey8DBw5kypQpREdHO3qKd7z8/Hz+85//8Omnn2L7b8mekJAQVq5cSWRkpINnJ9zJdE1aUGbRYfD0wL1ZE9RGFy7NtZBAVfdvXnPOF2Kx2dDrNITX8yTAyxW1SoWfhwv1Azwx6LUcTsumsLSMpvWrGXSsEK2w2WT2JmfSONwfgI5x9Uk+lYvVZqVLfCR6nQaL1cZfJ07j6qyvegAF8DAZ8HR1Lj+f/EKOZ53DYrWhUlWOlojgiSAIgiAItWXz5s0UFhYSFhbGmDFjHD0dQRCEy5Jlmf3HdvD70S24m7zo1rwfIZ7BnFz0Mu0bn+ecexhppwv5aW8B5y6aaRBkJKGhOyF+zo6euiAItzlRwusOce+997J161b++c9/8ssvvwCwfPlyli9fTqdOnXjkkUfo3r07BoPBwTO9s2RmZvLZZ5/x2WefkZ+fr3y9W7duzJ07Fw8PDwfOThAgucyVJNc4jFoNfplWjE4Flb4vA856Df5ehjq/A0iSIDkjFxcnHX6eJiICvfA0GYgK8SXMvzyr41hmLheKSomPDKzxcWyyzJ6kk4y6Jx6ALs0j+Xzt73i7GWlS3x+dRs3h9Gwyss9jjqpGjxJZRqNWKZkmRaVmLhSVVK6ZJggO9uOPPzp6CoIgCEIt2rx5M9nZ2SJ4IgjCTe1iUT5f/fAhJWVF9G0zkhDfSLSa8jLm3m0HcXL528Q9v4LG4SaKS61cKLSwOymf2d+dQK2S6NLch3ZNvNBrRV8KQRBqnwig3EEiIiJYu3YtK1as4IUXXlDKRG3ZsoUtW7ag1+tp27YtnTt3Ji4ujsjISHx9fS/plyLV8Nbo9PR0PvzwQ/7zn/+g0dxcTz2LxcLzzz/P448/TkhISJ0d5+LFixw+fJjt27ezadMmtm3bpmScALi4uPDKK68wduzYGl9nQahNxzIusPtwDh4mPXm+zvi4O12Sf+Jh0uPpqq/zAIpGreLP41kEervi52miUagfQT7uRAZ4EezrRqnZwvGsc5gt1uv6/yPLMmln8jmZfZ5gXzdiQn3x8zQRG+aHyVD+erj9r1SKS8uqN7BUnt1S4VMROxFuOoGBNQ8+CoIgCDcXETwRBOFmV1xayI4jP7H8l7n0b/cg3Zr3u2Qb1+i2SCo15/ZswDO+Jy4GDS4GDQHe/vRp60/SyQLWbj/N/PXp3BXrQfcWvgT5GNBqVGjU4jcuQbjTzJs3r9bHvLlWsYUbYsCAAfTs2ZMvv/ySjz/+mOPHjwNQWlrKTz/9xE8//XTV/U0mE6tWraJFixZVPmZ6ejq9evXi5MmTZGVlMW/evJsmiGKxWEhMTGTVqlWsXbuW9evXVyuIsnfvXsaPH09Z2ZUXUy0WC3l5eZw/f/6y39fr9SQmJvL000/j6+tb7XMQhLoSHeaO2WrB1agjvJ4bRoOmQsN1CZDR69Q3pP6sJEkcO5XLmbxAYmWZiEBvmjUIIDLQGxeDnpRTuZz4f/buOz6qMn/7+OdMb8mkJ6RDKAHpXQUBUSyg2BWVFVHBta6uLvpbXXDtuLr7iL0g6loQEGSxN2ygdKQFCKmk9zLJ9PP8EYhC6IacBL5vX1IyM/e5ZkIyk3PNfd+FlcRFhPzhY6nAdxt3ce3ZTbNQhvRIZGTfLhgNelRgdUY+idFHt364goLH52/eoD7EZiYy1I56nPaMEUIIIcTJ63iWJykpKa06nhDi5NPgcbEtbz0bMldiNdv453WvEB4SddDrp17zCDnvPoCz1wj0Fsc+l3VPctA9qSsN7gCrM6pYtqIYnV4hIdJCcpyNpGgrcRFmeZOqECeJ4/GmkfZxBlu0OavVyrRp07jpppv46aefWLx4MZ9++mnzrJRDqaurY+LEiUdVohQWFlJZWQnARx99xPXXX98uSpTflycAlZWVFBYWHnGBsn79ei688MKDFiOH06tXLyZNmsQ111xDVNTBXywIoZWQ8HocnXKwm22ER5qwGW2o+81BMegU9Pr956W0HuV3v9c2eMgsqGBQ90TCQ6yMHdiNEFvT1O4d+eXs3F1OfFToHz+oCt+sy2wuUMYO7ka/tHiMBj3ZRZXsKqwkNjzkqKeQlFW7KKtykRjjJCbMQXpyDA6rmQa3F0VRmh5Z9XezVORFvhBCCCGO0vGeeZKTk9PqYwohTg6qGmRr3nq+2/gx4SHRjOh9DmnxPTHojYe8nS0pHXtyHyrWfEzMiCsPfB2LnlH9oxjZN5KiSg85RS6yCl2s2laJ368ysHsY/bs6CbUf+lhCCLE/KVBOcoqiMGLECEaMGMHTTz9Nbm4ua9asISMjg9zcXMrKyvD5fAe87TPPPMOcOXOIjIw87HGGDx/OokWLuPTSS3G5XO2iRNm/PLHb7SxatIjhw4cf0e137drFtGnTsFqth907xmAwEB4eTkxMDD169KBPnz6MGjWKxMTEP3w/hDie1hdu4Iecn3BaQil0FRDniN23QFFVnBYn3aK6Emr54zM/DkdRFDZnFzG6fxfCQ6z0TIlBUcDnD5BVVEFBeQ1KKyyMpaKyIbOAercXh8XEsPTk5s3if/w1G7f3wN8XD5c9q7CCzIJyEmOcmE0GBnZP4NyhPfhlWx71jV7MRj2hdgshVjP1jV4yC8r/8H0RQgghxMlDlu0SQrRXbm8D7y9/mcq6Ui449VqSo7tgNh75PrzRoyZR+PHzhPc7C2PIwc9D6XQKCVEWEqIseP1BGtwBiivd/LSpgoXfFZIWb+f8YbF0S3IcdAwhhPi9Vj9zfbjln0T7lpKSctymZJ922mntpkQ5WHly2mmnHfEYaWlprF69+nhFFEIzv59LEmENJz2mB2a9GZvRut+lNC09dZwnSfz+iAqwI7+M0qp60pNjMOibNgnMK6kkZ8/+J63F6wvw85ZczhrUrbk8Afh5Wx6NHl/LcIehKJBXWs3GXYUM6J6A026ha0IU158/lJ4psVTUuLBZjMRGhBBqs7B2+24pUESbmzNnDgC33367xkmEEEIcLSlPhBDtjaqq+AJeNmSuZP7ylxnVbzzXnHlr8wbxR8MSmYg5vBOu7A2E9R17RLcxGXSYHDrCHEbSk0Pw+oKs2FzBnMVZBFWVcYNjGN4rAqfDiFGvoNPJCgBCiJZa/az16NGjW3tIcQJpDyVKa5QnQpwsBkQnEx0oxmaNJjFuKDq9pc0zKPv9paiijuyiSgb1SMRuaXrhnZFXyq7CiqYlr5TfthQ51Mvf/VfH+m2psKY5LDpF4bNftnPWoG7N1ykor2V7XmlTUfO7XeBb9kgtj6wALreXHzZlkxDt5OzB3XHaLSRFO0ka3Xef69a43GzLLTlEenEk3n33Xa0jdDjPPfccwBHNLhVCCNG2rr766oNeJuWJEKK98fo9bM1dz/rMFQTVAPdeOZu48KRjHk9nthHS81Rqfv0WZ6+RKMdQwpiMOkYPiGb0gGh2lzXy89ZK5n2WR3iIkcRoK8mxVhKjrThlmS8hOqx58+YBrbsXiizhJdqcliWKlCdCHB1z7WaS635B54shoHMTNEfus0cHgGJ0oLMno+htrX58V6OHvNJqfP4ABeU1eLx+dDqFjbuK6JoQSXyUE0WBjbuKKCivRVEUaurdZBVWoNMpFFXW4fU1zUrx+QOUVteTVViBPxikqKK2+Th+f4Ds4kpsZhNenx+vP4CiU/hlWy7ZRU37N+kUhS/X7sTt9QNQ5/KQV1JFfYOH/NJqPL6mjysolFY1lTyKolBSVY93z8wYnaKwM7+c977eQG2Dh4Hd4okIsWM1GzEadHh9ARo8Xgr3lERCCCGEEIcj5YkQor3JLNjCl+sWE2oLZ0iPM0hP6o9B/8fP94R0HUz5yg9x5W3G0WXgHxorMdrKZaMSaPQEyCttJLe4gdUZ1Xz2SwlRYWYG9winR5IDg15mpQjRkVx//fWAFCjiBKBFiSLliRBHL+irA72NYMBDsGIVOoOjeQ8UhaYuRWdLwGiOPC4FSlZhBfO/2YDNYqKqroHSaheKovBrVhGNHi9Oh5VAMMiOvDK8Pj86BTZlFfHy/35GURTKq11U1zeiKArVLjffrs9ke34ZwaBKZa2reRZKXaOHuZ+sxqjXEVBVquoa0e0pY15Y8hM6nQ6dorA9vwy314+iKGTkl/HfL9dhMRmpdbkprXY1PS4KfLVuJ1tyikFRKK6opbK2oXlDeJ1OYVdBBS9/tJLUThGkJ0UTFWbHZjZR1+CmrMZFbkk1O/LLWv3xPFn89a9/1TrCEQsJOf57Bx2Nhx56CIBZs2ZpG0QIIUSzQ31P1qI82bvqxPLly9vkeEKIjsPlrmPxT2+SW7KTi06/jrRO6VhMVlpr3We92U7U8IvJW/A4vWYsaJUxrWY9PZIcdE904PYGqG/0sy23jo9+LKS0ysuIvhGcNywWu0VOoQpxspKvfqGZtixRpDwR4tgYo09FMVhRzFEYwvuj6I98k7/WUFJVT3FlffPf9y6VVVHjonxPYdH88T0FRW5JFTnFVXsu+O2leoPby69Zxfy6q7jFZW6vn6/XZu4zHoBer+PLNS0/DlBYXkNBWU2LyxSlaYbMxsyiFsf5/XW9/gA78svYnlfa8o4ryvHeWkYIIYQQHZxWM0++++67NjuWEKJj8Po87CraxoLvX6Nncn9mXPkvDPrjswxWWO/R5C96iupN3xLWZ0yrjasoTWWK1awnOszMGf2iKK3y8NmqEq5/ch0DuzoZOyiG7kkOrGY9RoNOfmYT4iTR6gXK3nfHyDsXxZFoixJFyhMhjp3OloTJduzr1LaG/fcqOdzHD3mb5l+0O07LMeRltxBCCCGOjizbJYRoDzw+NzsLNrMhcyVV9RVcM/ZW0jr1PO7H7XLd4+QtfJzQ7sPQmVt/JYS9YsLN/OmcZP50TjLrd1azOqOaH36tICbMRHKsjeRYG/GRFkxG3XHLIITQXqsXKLL0gzhax7NEkfJEiKPn8/nwuN2oh7/qAXm83hb7pAghhBBCiNYh5YkQoj0orMjjq3WLUVWV/mnDOSV10HGbdbI/e2pfLJ3SqFz/OVHDL26TYw7oFsaAbmFU1HrJLW4gr6SBrbl1NLgD9EhyMKhHGHERljbJIoRoW7KEl2gXjkeJIuWJEEfPG1TYnr0bj9d3zGOUV9bg8gZbMZUQoq3MnDlT6whCCCEOQcoTIYTWvH4Py35+lxVbv+aKUTfSJ3UIFrONtl6EOGbkJIq/eJWwPmMw2MPa7LiRoSYiQ030S3PS4AlQUetl9bYqnp6ficOq59yhsQzrFdFmeYQQx58UKKLdaM0SRcoTIY6eAhQ3Glj443bMhh3HPI4/CNWe1sslhGg7MoNYCCHaLylPhBBa8vm9TfucfPcqneN68NRNb2u6HLElJgVDaBSunF9xnnJGmx9fr1cIsRkIsRlIjbNx6ah4NmfX8r8VxTy/OJuxg6MZ2SeS2AgzFpMevU6WbhaiLYwaNarVx5QCRbQrrVGiSHkixLFz+Zr+55gX8BJCCCGEEMeDlCdCCC34/F6yijNYu/NH6lzVXDF6Gt0T+mi+l6Pe4iC0x3BqtnxPaPppKHptT3HqdAp905z0TXNS6/Lx89YqPvqxCJNRR6coCykxNpJjrUQ5TZo/dkKcyJYvX97qY0qBItqdP1KiSHkihBBCCCGEOBG1l/LkjTfe0DqCEKKNlFQV8OW6D/H6vfTtPJReKQOwmR1ax2oW2n0Y5T8voaFgO/bkU7SO0yzUbmTckBjGDIiiqMJNbkkj2/Pr+HFTOWaTngFdw+ibForVrNc6qhDiCEiBItqlYylRpDwRQgghhBBCnKjaQ3kC7SeHEOL4UVX4av2H/LDpM84ZfDn9ugzDYQ3VOlYLemsIUUMvJO+DR+h5z3tax2nBaNCRHGsjOdaGxxdOoydAVpGLnzZV8NYXefRLC+W84XEkRVu1jiqEOIRWL1Cuu+661h5SnKQOVaLsT8oTIYQQ4o/LyckBmt7lLIQQon1oLzNPhBAnPp/fS27JTj747lUc1lDuuXw2oba226D9WIQPGEf+oieoyViBM739ngMyG3WYjToGdgtjYLcwXG4/yzdU8MQ7OwixGhg7KJr+XZ2E2AyYjXpklS8h2g9FVdUTaqH7uro6rSOIVrZixYrmEgVg4sSJnHXWWdx+++0AzJkzh6+++krKEyH2c8Zdr2kdQbSB7/99o9YRxB8QEhKidYR97F2P+QR7eSiEEEIIIQ4hGAyQU7KDdTtXUF5bwoje4+idOljrWEesLnMtBcuepfstL6MzWbSOc9QyC1ys2lZJYYWbyBAT8VEWUuJsJEZbcVhl8SAhtCYFiugQ9i9RYmJiKC0tbfFnKU+E+I0UKCcHKVA6NilQhBBCCCGElspqivly7SLqGmvo23kovTsPIcTq1DrWUct66z7Ceo0kYvB4raMcs7pGP/kljeSWuCiq8FBV5yUx2sqgHuF0TbBrHU+IDmHvagp7V1doDVKgiA5j/xJlf1KeCLEvKVBODlKgdGxSoAghhOho5s2bB8heKEKcCH7a8gVfrlvC6L7nM7THaKxme/PrwY6mPnsDJd+8RcqkWRhs7W+/lqOhqtDoCVDb4GPjrhp+2FiBy+1n3JBYxgyIwmKSzeeFOJjj8TOtFCiiQzlYiSLliRAtSYFycpACpWOTAkUIIURHI88VQnRs/oCPosp85i9/BQWYPO5OYpydtI71hwUaatm99N+E9TsLZ8/TtY7T6vJLGvn452JWZVTRv6uTUf2iSO1kw2rWYzLotI4nRLshBcoRkALlxLd/iSLliRBCiI5KChQhhBAdjTxXCNFx5ZbsZP2ulewuy2Z4zzEM7jaSE2m38sq1n+LK2UjSJX8D5cQsFTzeIOt3VrM+sxqPVyU2wkxSjJWUWBtxkRaM+hPn8ynEsegQBYrWL6akQDk57C1RAClPhBBCdFhSoAghhOho5LlCiI6n1lXFJ6s/oLq+nFNSBtEvbRihtnCtY7U6f0Mt2W/OIPGiv2Lt1FXrOMdVMKhSVuMlr6SB3JJGSivd+AIqp6SGMCg9nHCHUeuIQmhCCpQjIAXKyWPFihUAUp4IIYTosKRAEUII0dHIc4UQHUcwGGTtzh9YsuItRpwyjhF9zsVuCUF3gs7OAKhYvYyynxaQ/pc3tY7SZvwBFbc3QFGFm5VbKlmxuZKUOBsTTo2lTxen1vGEaFNSoBwBKVCEEEII0VG0twJFCCGEOBytf+YXQhyeP+CjrqGahT+8QbWrnMlj7yAuIlHrWG1m/d9Opeu0OYR0Hax1FE0Eg/DLtko+XllMXYOfEX0iGdYrnEinCYtJj14ny3yJE5cUKEdAChQhhBBCdBRSoAghhOhotP6ZXwhxcEE1SFFFHusyfyIj/1eG9zyT4T3HYNSbtI7WpmozVlL81Vy6Tn8OndGsdRxNFVe6WZ1RzY78OhxWA3ERZlLi7CTHWAkPMZ1IW+AIAcDy5csBGD16dKuNKQWKEEIIIYRGpEARQgjR0ew9IbH3BIUQon0IBAN8tmYBeSWZ9Ezuz4Cup+G0R2gdSzNZb9xL+IBxhPc/W+so7YLHF6SgrJHckgYKyhoprfbitBsY1COMPl2cMitFiEOQAkUIIYQQQiNSoAghhBBCiCNVUJELqkpCVOo+H/81axWvfPI4o/uOZ+yAi3Daw9Hp9NqEbCfqdq6m7McPSJk0E73FoXWcdsXjC+Jy+9mRX89368vJLHQxqn8U5wyJITrsCGbsqID0LeIkIgWKEEIIIYRGpEARQgghhBBHwh/wMeO1PzGw6+lcMXo6ep2esuoiPlu9gOKqfK47+27iIhK0jtlu+F3V7P7o30QOHk9I96Fax2nXalw+Pl9Vyvcby+kUaeGMvpH06hyK3aLHbNK36EqeXbSLcYNj6J7sQCdrgImTgKG1B8zOzm7tIYUQh/Hiiy9y8cUXExcXp3UUIYQQHdiUKVMAmDdvnqY5hBBCCCHEb3x+H69+8gS5JZl4fB46d0qnpr6CnNKddE/ow1VjbsZstGgds10x2J040gZSs+V7QroPQaZMHJzTbuSKMQlcOiqejNw61uyo5peMaqKdJuKjLKTG2UiMtmI166mq8/LZqlK259dz28Vd6JkibwgTJ75Wn4GiNZmBIk42b775JrfffjupqaksXbqU1NRUrSMJIYQ4Qu1tBorWM4mFEEIIIURL3238mOf/9zCqGgQgMboz44dOok/nIcSGy6yTg/HXV5P15t9IueIBzNHJWsfpUGpcPvJKGsktdlFY4aa2wU9avJ2SKg+LvisEoHuSg/uu7k5SjFXjtEL8ZtasWfv83hqkQBGig6uqquKSSy5h7dq1xMXFsWTJEnr16qV1LCGEEEdAChQhhBBCCHEoBRW5PLv4QbKKMpo/pigKj14/l24Jp2iYrGMoW7GIqvWf0f3WV7WO0iEFgypuX5DqOh8/b63k9U9y8fqCzZeHO0w8fWtvKVFEu3E8fqbVtdpIQghNhIeHs2zZMkaPHk1xcTHnnXcea9as0TqWEEIIIYQQ4gSUmpoqs96FaCMen5tv1n1ETvGOfT6uqirPLLwft7dRo2QdR/Twi6nLXIMr51eto3RIOp2CzawnPspCl3g7fv++J6Ur6738880McosbkPdgiROVFChCnADsdjsLFixgwoQJVFVVccEFF7B8+XKtYwkhhBBCCCFOMLm5ueTm5modQ4iTQkF5Nhm7N5IY3YW0+J6kJ/WjT+chDOx6OimxXckrzdQ6Yvun05F2w78p/OxFVL9P6zQd2srNlQRVFVUFg17BaTeSGGXBbNLx46YKGj0BrSMKcVy0+hJex2OdsaMhS3iJk1kgEODWW2/l3XffxWw2M3fuXC644AKtYwkhhDgIWcJLCCFERyPPFUK0nar6cipqSjAazRj1Rox6EwaDEaPeiEFvwmQwodPptY7ZIeyaezeRQycS1nuU1lE6JLc3yNKfiqht8BHtNBMWYsRhNWC36HFYDDhsBkJsBvQ6Reuo4iR3PF6ntHqBovWLKSlQhIAZM2bw4osvotfree6557jmmmu0jiSEEOIApEARQgjR0chzhRCiI6rJWEHFLx+RetVMdGab1nE6HFUFXyCIAuj1CjpFihLRPh2P1ymGVhtJCNFuPPnkk4SFhfH4449zyy23UFNTwy233KJ1LCGEEO3czJkztY4ghBBCCCFEq7Ml9KDKYMK1exshaYO0jtPhKAqYDLIThDg5yQwUIU5gL730EjNmzEBVVe6//37uv/9+rSMJIYT4nfY2A0UIIYQ4HK1/5hdCiGOiqpStWIS3soCEC+7UOo0Q4jiZMmUKAPPmzWu1MaVAEeIE995773HLLbcQCAS4+eabmT17ttaRhBBC7CEFihBCiI5G65/5hRDiWPnqKsieN4PUyY9iCovVOo5oBd9uKSE12k7nGIfWUcQJTOZeCXGCmzRpEm+//TZms5mXXnqJadOm4ff7tY4lhBBCCCGE6IC+/fZbvv32W61jCCHEUTOGRBLWfyy58x/WOopoJX99ex3L1hVqHUOc4KRAEeIkMGHCBBYuXIjdbuf9999n8uTJeDwerWMJIYQQQgghOpjRo0czevRorWMIIcQxiT71Mmo2Lachf5vWUUQraPAE8AWCWsdoM1UuL7MWbOLDVflaRzmptPom8rL5qBDt06hRo1i2bBmXXHIJH3/8MZdeeinz58/HbrdrHU0IIUQ7sXz5cgA5MSaEEEIIIU5IisFI2g3PUPjZS6RNfRpF3+qnRk84gaBKXrkLk0GPy+OnuLqRUKuRHvGhWE16Gr0BdpXU47AYyC1z4bQZ6ZcSjsvjJ7O4jjq3H5tJT7dOIYRajUDTMpA5ZS52VzZi0Cl0iXUQE2pBUaDe7WdHUS0uTwCnzUh6fGjzBvYeX5DN+dV4fAFSYhzsWVUSgNpGH1t319Ar0dl8nFWZFcRHWEmMsJFZXIfXH0SvUyipcTOwcwR2i4HdFQ3klbvQ6xSSo+zEh1sBcHsDbNldQ73bT7jdRK8kJwadwv6qXV4yi+tp8PqJCbXQNS4Ej6/ptinRdmKdluZ8mcX1dItz4LAY2VlcR1mtG4NeR9dYB5EhZgAq6jwU17hJjLCxvbCWQFClR3woEQ4Tm/Nr2FZQC0BylJ3kSBsxTgvF1Y1klbjwB1WSIq0kR9nRHyCrODatvgeK1mQPFCEObdu2bVx00UUUFRUxcOBAPvzwQyIiIrSOJYQQJ6X2tgeKrGsvhBBCCCFOBrte+wvRI64gNP00raO0ew0ePy99mcmW3TVU1ntRUfH5g1w6LImpY9LILXfxwPu/EgiqFNc0khYbwkOX92He8ix+3F6G3WzA7QvQIz6U2Vf3R1EUvt9WyqtfZ9LgDRBUoWdCKHeP70F0qIUH5//K9sI6bGY9JTVuLhmWyNTRaQRVlZkLNrEhpwpVhehQMxtyqpgyugt3j09nbVYl9/x3Pf+6dgCDujSd5xr78NdcMyKVqWPSeGjhZjblVaGiUFrj5pVpQ6lt9PHcZztQUXH7AjitJu6ekE6vRCf/WrqV7zPKiQk1U+Xy8sINg4kLs+7z2FS7vDyyeAu7KxqwmfRUNfi4ZkQKp3aL4saXVzG2dyz3XNATgHd/zGHp2gIevaofm/OqefP7bBxmAx5/gMgQM09e058Qi5GvNxXzxvIsQm0m8spdBIIqfZLDmH1Nf25/Yy1rsiqxGnVEhlq4cUwX+iSH8cwaX/4yAAAgAElEQVTHGeSVubBbDMSFWbjr/HRi9hQ34o+TJbyEOMn07NmTL774gtTUVNatW8e5555LUVGR1rGEEEIIIYQQQggh2kTk8IuoXPMJQZ9b6yjtnqo2zZ7YkFvNLeO68cq0oYxIj2bu8iw8vgD+gMruygYavX7ev/N0HruqLz9sK+V/awt44JLevDZ9GDeemcbarEre+iGbslo376/IxW4x8t/bTuPZKQMZkhaBoigsXp3Pql0V3HJON56fOphLhiaydE0BGQW1fLe1lCWrdzN+QDzv3XkaXWIdBH/3vi+3L0BZrQe377clvYqr3dS7m/YBrmnwkllSz2ndI1l49wjSYh3MXPArkSFm/t+Uwbx041CySuuZvyKXoqpGPt1QzNl9Ynn+hsE8PXkAUXtmiPzeolX57CyqY8bEXrxw4xBO7x7Fol/y8QdUIkPMfLetjNpGHwDv/pSLXqegovLuT7n0SQrjxZuGcP9Fp1BQ2cg7P+TsuR9BskpdeP1B5t48jGtHpvJjRhl55S7uHp9Oz4RQLh6axNybhzHmlFg259ewelclr0wfyv+bMojbzumB02Y6fv8g2jlFUZrfGNhapEAR4iSUkpLCF198Qa9evcjIyGDcuHFkZ2drHUsIIYQQQgjRzs2aNYtZs2ZpHUOIVlXbUE1lXRkAwWCARm9Du5yR6/Y24vN7tY5xQrAnnYKKSkPBdq2jdAiqCuf178RZfeOICbVwavco9IqOrFIXAAoKt53bg5hQC3azgR1FdfROcnJa9yicNiNje8eRFGnj2y2l1DT6yCp1cemwRGxmPclRdi4ZmkRUiJnPNxRhNuipafDy/bZSwu0m6hr9lNW5+TWvGgWFK05NIcxmYualvfEf5f4noRYjt5/bg1inhd2VDRRWNuK0GVmdWc6aXRWkRNkpq/Wg1ylEOEwsW1fI91tLcdpM6PUtT6Ov2F5OiMVAXrmLbzeX4LAYKK/zUNPoY9LpKVS7vKzNquTXvGoKKhs5t18nSqrdlNa6SYtz8P3WUvIrmpYP25Bb1fRYA0aDjtemDSHGaaF3UhjhDhN55Q3YLXoMOgWLSUeIxYDJoCPcbiTEYuBv/91AdqmLMLsRk1FO+bcmeTSFOEnFxcXx2WefMXjwYHJzcxk3bhxbtmzROpYQQgghhBCiHXvooYd46KGHtI4hRKt695vneeL9uwHILc3k9U+forq+QuNULf336zn8kvFtuyx39mpw11HrqtI6xmEZQyOxJ51C7daftI7SYeh1StPZfcBs0GPUK3h9QRRAp1NwmJv2k/EHVFweP+GO32ZB2Mx6jHodLrcfnz9Ig8fXvE/J7+0tH77bWspXm0v4eWcFZ/aOJc5ppW7PTI5QW9Pt9p9lsPdvh/r6MBp0zfupVLu8BFXYWVTHV5tL+GpTCTFOM0PSInHajMy8rDf9U8P5z6fbufvtdeSWuVqMV93gpazOw3dbS/l6cwmZJfWceUocYTYj4/rEYTLoWLOrkm82l6DXKYzr24kGTwCvP8jqXRV8tbmE5VvLSIt1MCQtct/7suf+GfUKekUh8Luy6Pf3fEBqONPGpqEoCn99ex3PLMugql6K1tbU6jsl5eTkAJCamtraQwshWllYWBjLli1j0qRJfPvtt5x33nksWrSIIUOGaB1NCCGEEEIIIYRoM3tPugYCfrx+Nyrtr6RwexsIBANaxzikbzcuI7dkJ7dc+A+toxyaoiO8/9lkvTkDv6sKgz1c60TtmwLZpS48/gAWo56i6kbqPX7iIyzUNjYtkbX3K8Zk0OG0mfgls7z55rsrGqhu8NEjPgSTQU+I1cjOojqGd4tCVZs2Tg+xGkiJsqMoCnee34NOe/Yb0SkKBr1ChMMMqGSV1NMzIZTthbX7lCgWk75prHoPAFvyaw65lFOnMCsKMLxbFH8+uytq091Er1PQ6RT6JIfx8BV9WLGjnH8ty+Cn7WWkRttbjOH1B/nbhT335Gu6/d4N3Mf2juWH7WX4A0HO6BlNbJgFR6kBi1HPxMGJjOoZ03RcBQy6Q89zaMqnoCjQ4Pnt+4BBr+PCwYmM7BnD5xuLeOnLTC4emkSE4+Rdxqu1tXqB0rlzZ0A2HxWio7DZbHzwwQdMnTqV//3vf1x44YW8++67jBkzRutoQgghhBBCCCHEcdHocZFdvJ2EyJR9TrImRnfmylHTcNqaTqgH1SAFZdlU1pcTH5lCtDNun3Eq68ooqSrAaY/AqDditdixmmwUV+4mNjyR2oYqqurLSY3thl6372m4itpSymuKiYtIxGmPaP64ikpu8U4aPS6SY7tit4QAcOmIqdgsjua8JVUFVNSWkBTdhRBb2AHvpwoUludQ21BFl7h0zCYrXr+Xkso84iKSMBqaTvoGggHKa4qJDI3BoDfiC/jILd6Bw+okLiKxeTyXuw63t5HI0Bh2l2ejoCMhKqX5Mc0q3k5mwRaq6sow6I2E2MJQUcku2o4/4CMlthtmY/vY3NrojMZ5yihy5z9M2tRntI7TrinAr3lVTH3xF07tHsWnG4rolRBKVIiF2sb6fa5rMekZkBrGpxsKmf7qaoZ3i+DnnRXklrt4buog7GYD/VLCeOmrTAJBKK1tZFthLbMu7cNNY7ty9ZwVPLMsg3P6daKm0UdGQS03jOnCqF4x/L9Pt/PY4s2c3iOaj9YUYND99rXbNc6BosB/f8ghq6SeT9YXNhcZQItKtFO4leHdo1iyOh+TQUfnGDvrsiuJDrUwsHMEz3++gwmDEqio81Dt8hJub1lIXDosibvfWsdzn+9kVK8Y8stdlNZ6mDq6CzFOCxMGJjB/ZR7BoMpjV/UDoGdCKMlRNmZ+sInp47oS6TCxYns5I9KjOa9//CE/B0a9gsNi5NstpUSHmukaF0J2qYsNOVWcOyCezOI69Lqmwkm0nlYvUIQQHY/ZbOatt97itttu45133uHyyy9n7ty5XHjhhVpHE0II0YZSUlK0jiCEEEIIcVypapCV277h1Y+fQNHpMBlM+PxeIkJiAMjI/5VnFz/I7Jvexmq28/h7d1FYmYdep6fR4+LSkTcw8dTJ6HQ6Fv7wOktX/hez0UKtq4qgGuTcIZczqu/5PDhvGv26DGP77l/x+X3YzHYenvIqseEJVNWV8fzSf5KRvxGz0Uqj18XZAy9h8lm343LX8cLSh9lZuAW9Tk9qbDduOv9+YsI68cT8v3LaKWdz+Rk38tKyx1i55UusZjsqKq/e9WmL+1pSVcBrn84mq2gbep0Bg87A1PPuwWmP4O6XJnHtWbdx5ajpACz4/lV+2fYtd1/2OFlF23hp2WPYLSF4vI306zKMuy57DL3OwJdrP+TLdR8S7Ywnu3g7Hl8jo/tNYNr4+/loxVus3PIlgWCAe165hn5pp3L5yBv418L7qKorQ6fT0bfLMG46bwZWs71FXi3EnHEla25/hPhzb8Ea31XrOO1a3+QwgqrKsnUFJEfZuXt8OooCRr2OxAgrVtNvMyjO6tOJapePd3/K4f0V9USFmHlm8oDmWSXTxnalwRPg3Z9ysJr0TBycQFSoGbvZwMzLevP299n855PtKAqM7hVDqNVIcpSd2dcM4K3vs1i2rpDzB8SzraCmudgItZp45k8DeO6znXy5qYTzByawNb+meUP1mFALiZG2fe7Tk1f356n/bWPJmt0EgyrhdhM3jU0jKdKK3WzgxS92otcpTBgYz5mnxLZ4TE7vEc3dE9L58Jfd/LKzAotRx7kDOmHbs5xZYqSNUT1jcHn89Ep0AhBmN/HgJb15culW3vkhB1WF2DBL8+wWu1m/T06zUU9cmAWrxYDTZuLc/p14+ctMXv0mi79d0JMuMQ6WrSvgnws3EWo1cvPZXekc42itT7sAFLWVp4rsbcG1moFSV1enyXGFOFHcd999vPDCC+j1eubMmcO1116rdSQhhDhhhYSEaB1BCCGEOCpa/8wvxB9VVlPMnCX/QFF0XDv2NjILt/LRircJsTp5atp/2Zj1M88unsmTN76F3RrCZ6s/oE/qEELt4bz5xX+obajizosfxmp2MO3f53H+sEmcNWAi8754hszCrcya/BIen5t/vHkTMWHxXDpyKh6fh5eWPcpFp/2Jq8+8hbmfP83X6z/ipvNnkBrbneUb/8eSn95ixpVPYzFZeX7pQ9w84e+kxHanpGo3neN6YLeEcNeLV3Jqr7M4c8AFTHnqLC46dTKXjJzKzt2bGdaz5SoS7y9/iVXbv+P6cXeTGNWZuZ8/TaPXxc0T/s6st24mMjSGGVc+jUFv5NY5F5Ge2I+LR1zHvxbeR9/Ow7j49D+xs2Arcz97ikln3sLZAy/moxVvM/+7VxjS/QzOGjiRXzKW8/X6j3jutsXodXrmfvYvMgu3MOPKp7Ga7WzYtZJ5n/+b/5v0HyKdsVTWldEjsQ8Gfcv9L7RSsWopNVt+oPN1T6Do9FrHaXdcbj+zl24DBR66vA91jT5CDrB/ycHUu304LAe+fm2DD7NRj3m/Tc9VFercPqxGPUbD/pepBIJg0CuoavNWIc0CQZVAUMVk0DUvy3U4bl8AX0AlxLLvXIN6tx+9TsFqOvS/i2BQpdbtx2E2HNXsD7cvgD+g4rAceo6Dqqr7zJTz+oN4/YHmx1VVVapcPkKtBgwH2Oz+ZHI8Xqec3I+oEKKFJ554gv/7v/8jEAhw66238vzzz2sdSQghhBBCCCGEaBVVe5bcuvDUa+mW0JuzB15Mr5QBB7yu1WRj7ICLqKovZ13mT+h0Ojw+N16/B6+vkaAaJNoZS2x4AiG2MPQ6PWGOpqW4gmqQMf0v4PRTxjG633isJhul1YXUN9aSXbSdfl2GMbrveFJjuzHxtD9hNJj4NfsXwhyRWEx2PvllPtvy1tMlLr15Ca+9bGYH0c44tuZt4Jdt35Ce3P+A+Tdlr8aoN7I5Zw1frV9CIOCjuDIfj7eRi077E6VVheQU72B7/kZKqwsZ2O108kp34Wqsw6g38v2mz8gt2YEKbNz1S/O4Rr2RP1/wAH06D2Vgt9OxmKyUVO0mzBFJqD0cq9lOSmw3YsLiiXZ2wmAw8vGq98kv3UXX+F7tqjwBiBx6IUGfh/pd67SO0i6pQBCVYFBFVdWjKk+Ag5Yn0LQh/P7lCTSVIqFWY4vypOmy35aoOtAWJ3qd0rxR/JFWGRajvkV50pTdcNjyBECnUwizGY966SyLUX/Y8gRosZeLyaDb53FVFIUIh+mkL08AsrOzyc7ObtUx5VEVQrRw33338dRTTwFw//338+ijj2qcSAghhBBCCNEeXHfddVx33XVaxxDimPn8Xjw+N5GhTcvxGPRGzIYD78lR21DNY+/ewcsfP86qjOUUVOQCTe9sDg+Jxmqy8d43L3L3S1exfOMyBqSd1lx2NJ1obtqXRKfoMOpNBIJ+3N4GvH43ceG/7SvitEegoOD1eegc14ObJ9yPy9O0lNecj2ZRUVu6Ty6b2cFT0/5LXEQib389h5lvTsfr9xwwv8/vpay6iKKKPCwmG8N7jsVuCeHM/hNxexvJKsrgxy1fYtCbGJo+Cpe7Dn/Qj8tdR3FlPhW1pfTrMpQBXU9tHlen6LCYmpZiMhksKIoOf8B/wMewX9pwbr9oFkWVeTyz6H7e+PwZ3N7GI/pctaXIYRdSuWYZQb9X6yjtjs2s56Yz07jprLRDbsouRHuQmppKampqq44pe6AIIQ5o+vTpOJ1ObrnlFp588kmqq6ubSxUhhBBCCCHEyWnevHlaRxDiDzEbLdjMdnYVbiU1thv17lpqG6tbXE9RYFvuOipqy7jrkkc5JXUQn676gG82LAVFYVveBty+Rk7vdTZGo5nzhlzJ6P4Tfrv9/u9933Pi2WYJwWZ2sH33Jrx+DyaDmayibQTVILHhiQSCfnok9uPJG99i+caPmfvZUxRX7SYyNKZ5KH/Aj90cwr2Xz2bV9uW8+smTrNjyJaP7TdjnkDHOTkSExnD9OXdjMzv2LH3kR683oKAwsNvp/JLxDUWVuxnT7wLsllDCHJGYDCbOG3oF3RJ67zmeD51y4Pdg7386XVEUPF430LTfjM/vZUDa6QzuNpIPf5rH/1a+w5WjpjUXMO2FPbk31b9+Q2PhDuzJvbWO067oFIXkqPaxZ40QWpACRQhxUFdddRWhoaFMmTKFl19+merqal588UUMBvnWIYQQQgghhBCi44lyxpEa14MlK94iGAyQXbydLTlriXZ2arqC2vwLFpMVf8DH+l0r2F2ezbJf3m2aYaKq2C0hBIMBsooziAlLwOWuo66xmpF9ztszTNN/zfasx28z2+nTeQjvffsi73z9PPGRKazY8gWRoTGcfsrZ/Jq1iq156+kc14PCilxCbGHNZcPe8bKLM/hk1Xz6dRlOSXUBOp2emLD4Fvd1cPczWLLiTT5a8RYpsd2pqivDoDcyss+52MwOBnUfyX8+fACAC069GoBu8b3pFJHE80sf5oLhkzDoTeSVZjKkxyjSk/r9LsXvH67fPhJmj6SyrpQv1i4iwhFNRX0ZNfWVJESlUlZdRERINHp9+9tnxBgWgzWhO7XbVkiBIoTYR6ufBZWN5IQ4sZx//vksXLiQq666ivnz51NXV8e8efOwWA48xVkIIUTHNXr0aACWL1+uaQ4hhBBCiOPFaY/g0hHX89X6Jfy87RuSY9I4d8gV+PYsgeW0RzCw2+kYDWa6J/bl/GFXsS1vAyVVBZw75ArKa4v3zGDZhslgZlj6mSTHdKG0uoiFP7xOIOBndL8JDOh6GrG/KzUGdD2NxJjOAJw96BKsJju/Zq8it3QnidGduXrsrSREpeILeKmpr+SrdUuwmm1cccZNJESmAtC381ASolKJCInBbLTww6ZPMRpMXDj8Gnok9WtxX0f2OQcVlS05a9mxewsOawjD0sc070HSNf4Uxva/EJ1eT0pMVwBiwuOZes49fLNhKSu3fQMqxIR1IiIkGoBOEckM6nb67x7PcPqlDSfcEQnAab3OorS6kB83f86gbiPo0imdLTlrycjfSKjNyaQxt7TY06U9UBQdEQPOJfvt+4kZfQ16s8y4EEI0UdQTrPGoq6vTOoIQJ6R169ZxySWXUFlZyciRI5k/fz4Oh0PrWEII0aGFhLSvHx73rml8gr08FEIIIYTYh4qKP+AjGAyi1+kBBZUgRr2JoBrEH/BjNBhRUAgE/fgD/j0bVxsJBPwYDSZe/WQ2qzK+4Ykb3yIyNAaXu55b50xkdN8JTDnnL3h9Hgx6Ezpd09JXXr8HnaJrLi+ajuNDVVX0On3zx1W1KVsgGECna7r+3uWzfH4vOp0Ovc6AL+AlGAwACka9EZ3uwLM6fn+cvfdh73h7l/QCWmzsvjeDAuh+ly8YDBAIBjAaTM1j+AJejHpT82tJn99LUA1i0BnQ6fRNj7UabL7/7XkfjaIvXsVdkk3nyY9pHUUIcQxmzZq1z++tQQoUIcQRy8jIYOLEiRQVFTFgwAAWL15MRESE1rGEEKLDkgJFCCGEEKJj2pS1mjkfzcRkNBMREk1xVQFOezg3T3iAtE7pWscTxyjgdrH2jt70feRbLDGpWscRQhyl4/EzrRQoQoijkpeXxwUXXEB2djbp6eksWbKE+PiWa612VKGhoVpHEOKkUVtbq3UEzUmBIoQQoqOR5wohfrO7PIfMgi14/R5CbGH0TOpLmCNK61jiDyr7aSH1WevofO0jsGe2jhCiY5AC5QhIgSLE8VdSUsJFF13Eli1bSE5OZunSpXTp0kXrWK0iNDRUTuoK0Qbka62JFChCCCE6GnmuEEKcDDJfuY24s2/A0XmA1lGOSVBVyS9vIK/chT+oEhViplunECzGAy/11lH4AkG2F9ZSVutBr9MRH26hS6wDXTteFu5IVNR52FVST73bj8NiIC3WQWSIWetYHVKHKFCmTJkCwLx581pz2CMmBYoQbaO6uppLL72U1atXExsby+LFi+ndu7fWsf4wOakrRNuQr7UmUqAIIYToaNr7c8XmzZt58sknKSsra3GZqqqEhoby8MMPk56u7RJLzz33HAkJCYwfPx6TyXTI644ZMwav18srr7zCKaec0kYJ97V06VL+8pe/MH36dGbMmKFJBiHaUsXaT3DtWkfSpTNQ9tsfpr3blF/NM8syyCisIxAMggqKTiHMZmLS6clMGdXx3gDb6A3w3x9zePu7bDyBIMGgikLTc1JKlI0/j+vGqF4xHa5IqXJ5mfPZDr7ZXILL07QfkaKAUa/j1O5R/HVCOp3CrBqn7Fg6RIGi9YspKVCEaDsNDQ1cffXVfPPNN4SFhbFw4UKGDh2qdaw/RE7qCtE25GutiRQoQgghOpr2/lxxww03MHfuXACMRmPzBt576XQ67rrrLh599FEt4gGwfft20tPTSU1N5csvv6Rr164HvW5ubi6pqak4HA5ef/11rrjiijZM2sTj8fDwww/z6KOPkpycTG5ubptnEKKteSoKKPzkOWLHXIctsWPsadPoDTDvuyxe+CIT3UF6hKCqMiA1nIcu70NqtIOO0DdkFNbyxJKtrM+pOuT1Lh+exF/OT8dhMbRRsmMXDKpszK1ixrsbKK52Nz+3/p6qQnSomRkTezHmlBiMellO7kgcj9cp7f9flBCi3bLZbHzwwQfccMMNfPTRR0ycOJF33nmHM888U+toQgghjsEbb7yhdQQhhBDiD6murgbA6XRy7bXX0qlTJ6DpRIqiKDgcDi699FItI1JZWQmAy+UiGAwe8ro+nw+AYDB42OseL6qqEggEAPB6vZpkEKKtmcLjsMSmUZuxskMUKI3eAK9/u4s3lmcdtDwB0CkKv+ZW88iHW5h5WR+So2xtF/IYFFe7eXjRZn7Nqz7s7JL5K/Oocnl58JLehNkPPbNPayt2lPO3dzZQ7/EfsDyBppko5XUenvrfNkwGHaN7xbRxSrGXFChCiD/EZDIxb9487rjjDt5++22uuOIKXn/9dSZOnKh1NCGEEEdp71KsQgghREcXFhbGrbfeSs+ePQ94eWVlJXPnzqVbt277/Ozi8/mYPXs2cXFxTJ48uXl5rY0bN7J06VJKSkoICQlh1KhRjBkzBrO5aY36uro6Fi1ahNvt5uabb2bx4sWsXLkSnU7HyJEjGT9+PNA0o2Tp0qUANDY28s4773D22WczYsSIo76PmzZtYtmyZezevZvQ0FCGDRvG+PHjMRqN5OTk8PbbbzNy5EhGjx7d4v6lp6dz4YUXYjQaCQQCrFy5ks8//5zy8nLi4uIYO3Ysw4YNw2jcd+mig53oE+JEo+j0RAw6j5z//p2YMdeia+fLeG3IqWLxqt34A4d/170KrM+pYsnqfG4Z1x2DXpuv68rKSh555BHuvPNOUlJSWlweCKrMXLCJTfk1R7Q0l05R+HpzCd07hTL9rIPP7DveMjIyWL16NRMnTiQ0NLTF5YVVjbyxPAuXx8+RPPIl1W5e/GInZ/SM1nSJspdeeomePXsycuTIFrM725OZM2e2+phSoAgh/jC9Xs/zzz+P0+nkueeeY8qUKTz77LNMnjxZ62hCCCGEEEKIk5CiKBgMBz/lsX37du699146deq0T4Hy+eef88ADD9CjRw8GDhzIgAEDeOGFF/i///s/6urqCAaDKIrCc889x0033cRDDz1ESEgIWVlZPPLII+Tn5/Pzzz/z3nvv4fV60ev1/Oc//+Gxxx7j7rvv5pFHHuG1114DoL6+nkceeYRnnnmG6upq9Poj29xZVVUWLVrEX/7yF4qKippnphiNRsaPH8/ixYv57rvvmDVrFvHx8eTn5zff9v333+cf//gHp59+OgMHDqRz5848/PDDzJ49m8bGxubH7umnn+af//wnd955Z4tjC3GyMEcm4Og6mPwFj5Ny1T+0jnNQHl+ArzYVU17nOeLbBIIqi1btZuqYNBx6bU4Pu91u/v3vfzN//nxuv/127rjjDmy232bE/JJZzvfbSjEZjvxkvarCt1tKOK9/J5Kj7Mcj9mGVlJQwa9Ys7rnnHh5++GFuvPHGfQqHtVmVbN5dc8TjKUrTMmbL1hVw4aDE4xH5iHzzzTfce++9TJgwgdmzZ5OUlKRZlkOZNWtWq4/ZfusiIUSH89hjj/H3v/+dQCDArbfeypw5c7SOJIQQQgghhGhF2dnZZGdnax3jsIqKirjssssYPnz4Pv/ffvvtAISHh5OUlERRUREffPAB0FQOLFiwAGiawZKamsr333/PrbfeSmNjIxdffDHvvvsuf/7zn/H5fMyZM4fPP/8caJrZ0djYiNfr5a233uKcc87h5ptvJiIiAo/Hw4IFCygpKeHuu+/mT3/6E9A0m3/69Om88847R1yeABQXF/O3v/2NwsJCzjrrLBYuXMg999yD3W5nyZIlPPjggyQkJBAREcHu3bubM6qqygsvvEAwGCQhIYHo6Gg++eQTHnroIRRFYdq0aSxevJgrr7ySxsZG7rvvPjZt2rTPrBOZgSJONrGjJ1P85Wt4KnZrHeWg3L4ga7Iqj/p2FXUe1h7D7VpbYWEh999/P/369ePNN99sKoYDfr7fVob+UOuRHURRlZtdJfXHIemR8/v9lJaWMn36dAYOHMjChQspLS3F5faxdXcNbm/gqMdcuDL/8Fc6jlRVpb6+nvfff59u3bpx5513sm3bNhoaGjTN1RZkBooQolXNmDGD8PBw7r33Xv7+979TU1PDAw88oHUsIYQQQgghRCtITU3VOsIR8fl8ZGdn7zMLRVEUPJ6md2h36tSJcePG8frrr/PUU09x2WWXsXPnTtavXw/AhAkTCA8P55VXXgGge/fuPP7443Tu3JlJkyaxa9cuPv/8cxYvXsxll122z7H3lhomkwmPx8Mbb7xBTU0NxcXF9OvXj5tvvpm33noLp9PJXXfdRbdu3fD5fOzevZv6+qaTfhaL5aDv7l20aBHZ2dl07tyZF198keTkZMaOHUtubi4LFy7k7ROkvAkAACAASURBVLffZurUqfTv35+vvvqKV199lXPOOYfVq1ezZcsWHA4H55xzDg6HgwcffBCAkSNH8o9//IPY2FgGDx7Mrl27WL16Na+99hpPPfVU87FPtBkogaCfBl89gaAXnaLHYrRj0lu0jvWHqKi4fQ14Ag2ggklvxmp0oCjyHupjYbA7SZ38GEVfvEbqVTNpj7uu+4Mquysbj/p2Op3C+598z5blJXCAxaT2fr0fr+K0pmbfWRiZmZlMmTKFQYMGcestf2ZTTWd0x1Cg1Lt9LPn0a7Z917QnVlt/39q5c2fz93JoWgLyiiuuYNSoUVx+zRRygt2PadyMkgaef/75g34+Dvf5+qOfz127djX/2ePx8Oyzz7JgwQKuueYa7rjjjnY7I6U1tHqBcjzWGRNCdCzTpk3D6XTy5z//mdmzZ1NVVcXTTz+tdax2y+v1smPHDlwuFzExMcTHxzevpXwyZ/k9n8/H9u3b8fl89O7du8VazCeKnJwcTCYT8fHxWkcRJ6l58+YBsheKEEKIji86OprHHntsn8JHURSio6MBCA0NZezYsSxevJg1a9awadMmNmzYQG5uLgCTJ0+msbGRvLw8AAoKCrjzzjtRFAVFUcjIyACgoqKixbGnTp3avHdK9+5NJ8qCwSB+v7/5z/Db5uyqqvLVV18xc+ZM3G430DQ75a677mLw4MEtxv/5558BqK2tZfr06c3H2rJlC6qq4nK5CAsLY8yYMXz//fesXLmSrKwsli5dSmNjIykpKYwfP57a2lpycnIA2Lx5c/MyMz6fr3nZr+Li4n2OfaLMQPEHvOTVbKegdhduXwMB1Y8OHWaDlWh7Ap0jemM1OrSOedSqG8vIqtpMdWMZ3oAbBTDoTDhMYaSEpxMXkqp1xA4pdvS17Hz5Nlx5W7Cn9NY6Tkuqij8QPKabmswW4uLiOFDH8Puvd1VVURSl1X4Hmr937c9qtRIWFoZaoxzRHiH7C6rgcDiIi9GmDK2srGyxR4hOp8Nutzfdr5ZPG0fE4w3QqVOnw17vUI/5oS4/3O8WS8vH02g0Eh4eftDP5Ymi1QuU47HOmBCi4/n/7d13eFRl2sfx77T0RhICISEJvUgRKaIIhGIDF0VQXBSBVVDAtSyr7uq+CzZUFAuyuyoKcVddYS0oKhZKEEFBgnRDEZIQOumFlCnvHzFjhiSQ4ITJwO9zXVxkZp555j4zgTnn3Od+7jFjxhASEsL48eOZP38+eXl5vPrqq6ddh/hCUl5ezpw5c/joo4/Ys2eP82AKwNfXl+uuu467776bSy+9tNpzZ8+ezYIFCzh06BAWi4V+/fqxePHial9mc+bMYd68eWRlZeHn50f//v1ZvHhxteUBzjaW8vJy7rjjDpYvX+5ydUXl80JDQ2ndujXTpk1zWVe6PhYtWsS8efNITU11Xi3o5+dHjx49SExMZPr06dW+qJctW8Zjjz1Gamqq8+C0UmBgIAkJCQwcOJDp06c7D6ArpaWlMWnSJDZt2kR5eTlhYWHMmzePESNGuIw7dOgQN998M6mpqZSVlREfH88TTzzBDTfccFbbCRXrid54440YjUa++eYbunQ5ux3zXbt2ceeddxIZGcm8efOIiYk565jkwjNx4kRACRQREfF+fn5+9O3bl86dO9f4uMFgoH///rRu3Zrs7Gzmz59PWVkZ+fn53HbbbcTHx1NQUIDNVrHMitVqJScnx7l/GR0dTZcuXbj//vurzV11H7MuJ5UMBgPZ2dn88MMPLvdnZGTQs2fPauMre5XYbDays7Od+/fR0dG0bt2aW2+9lSZNmjBq1CjmzJlDTk4Ob731Ft9//z1Wq5XRo0fTtGlTsrOzndtXXl5Odna280RZQkICl112WbXVBM6HCpTi8gI2Zi7nWOEB7I7qy+gcKzpAem4ql8f/jlC/CA9EeHb2Z+9g86FkrPZyHLh+TlnFhzmQt4s24d3o1mIAJkPdl4yTCk26DyU75XMCWnbCYGxc75/JaCAiyJeswrr3QIGKf88DL+3OjX08Uzlw6NAhl9vt2rXjxRdfZMiQIfj5+fHDf7ew7dDBejdO9/MxcsWlvbi2h2cuTIyIiHD2ugLo0aMHc+bM4fLLL6fMbuSnpT8BufWet0WoDzfeeKMbI62f//73v86fzWYzDz30EPfffz/h4eH1WoayoVVeGODOilnV74lIg7n22mv54IMPCAoKYvHixYwdO9Z5RdWFbMeOHQwaNIhZs2bx008/uSQsoKIU8oMPPuDqq6/mueeeczlIKSsrY9asWc4djfLycpKTk/n666+rzfHCCy84r4grKSnh66+/ZuXKlW6LZdWqVSxZsqRa8qTyeceOHeP7779n3LhxTJo0qV7v0fHjxxk7diyTJk1iy5YtzuRJ5bZ89913PP300wwfPpxjx465PHfevHns3LmzWvIEoKioiB07dvDPf/6Tbt26sWjRIpfH//Of/7B+/XrKy8sByM3N5Zlnnqk2z3vvvcfWrVspKysDID09nZdffrle23iq5ORk55WJa9euPet5FixYwJYtW1ixYgVvv/32b4pJRERExFs5HA7Ky8spLS11+VO5nwcQGxtLYmIiAG+//TZffvklFouFxx57DIDg4GCaNWsGVJzY+/jjj1m/fj3r169n2bJlTJ482fl4VWeq0qh83G63O/evx4wZw+7du1m3bh1r164lNTWVBx98sMYL0C655BIA4uPjWbp0KRs2bGDDhg2sWLGC++67j/j4eAA6dOhAYmIiJ0+e5LXXXmPz5s2YTCamTp0KQHh4OFFRUQD069ePr776ig0bNrB+/Xo+/vhjbrnlFkJDQ+u1bY1dma2ElMzlHC7YX2PyBCp+dwpKc0jet5jck8fPcYT153DY2XPiR9Yf+IJye1m15Eklu8POz9nb2H5kLVZ72TmO0vsFtelBeWE2JUf3eTqUanzMRrrGh9byydfO12Li0raRDRJTfXTr1o1XXnmFTZs2MXz4cOfFob3bRmCz1z9pGxXi57EG8pVCQkLo1q0b//73v1m/fj2DBg3C19eXQF8zHaJD6t3bxQEMu8SzK1WYTCZiY2O5++672bdvH0899RRNmzZtVMkTgFatWtGqVSu3zqkEiog0qP79+/PZZ58RHh7OF198wY033ljjCfcLRVFRESNHjmTr1q3O+5o1a8aIESMYNWoUkZG/7rzY7XaeeOIJ5s+f77yvvLy8xsTAsmXLXG6vW7eOgoKCauMOHjzotlgqy/orjRw5ktGjR3PVVVdVq55YtGgRS5Ysqf6G1GL8+PF8+umnzttRUVGMGDGCkSNH0qRJE+f969evJzEx0SUxt2/fPpfn9enTh4svvthleyq3/95772XPnj3O+2pK8G3fvp3MTNeGgae+31D96pn6qrySEHA5sK+v/Px858+nrikrIiIi8lvNnDmzUa88UXmCPy8vj9dee40nn3ySJ598kqeeeoqnnnqKp59+muTkZOf42267DYPBQF5eHhkZGQwePJjWrVs7Hx85ciRQsfb7yy+/zKZNm1i/fj0PPfQQN9xww1ldsFKZlCgoKOCrr75iyZIl7N27l3bt2nHZZZdx+eWX06FDh1qr96+77joiIyPZtm0b8+bNIyUlhe+++46//OUvjB49mocfftg59s9//jMAR48eJSsri1GjRhEXF+d8/O677wYqjh8WLlzI9u3bWb58OVOmTGHMmDF88sknLq/t7RUoO45+z6H8fdRlYaBSawk/ZH5FidVzDZLtdjs5OTnOC7dqcrzoELuOp2CsQ48Tu8PGz1nbOFKQ7s4w662srKza8nCNnW94DH6RcRTsWu/pUKrx8zEx+KJmBPjU/US2wwGDL2pG0xDPLdkdEBDAE088wbJly7jnnnsICnJdNm/wRc2IDQ+o97zd48Jo2zzYXWHWW2xsLE888QTr1q1j3LhxLkuQG40GLmndhLjI+m1XRJAPv+8X7+5Q62XEiBF88MEH/POf/zyv+53URGvpiEiD69GjB19++SXXX3893377LcOGDeOjjz4iIsJ7yqHdZfbs2S47imPGjOHll18mIKDiy9PhcLBo0SLuuusu58HJ3//+d4YPH37apZiqHgRCRXXIuY7l1Vdfxd/f33n766+/ZtSoUc7bn3/+eZ2WuHr33Xf59ttvnbeHDRvGG2+84dyZstvtTJ8+nTfffBOAzMxMPvvsM5fXqnTTTTfx9NNPO28XFxfz1ltv8eijj2K1Wjl58iSTJk1i+fLlp11ebtWqVYwbNw6AwsJCUlJSzrgdnlI1webtB7giIiLS+FRWZzTWJErlMUZeXh7/+te/ahwzaNAgZ+VJ165dGT58OJ9++ikmk4lbb73VZezo0aPZsWMHc+fO5aWXXuLDDz+kvLycffv20bVrV+dyKr6+vgQGVlzxHBz864m78PBwoGL9+8qTaAkJCURGRnLixAn+/ve/YzKZuP3223nppZeqxVp58ZDJZHLur1500UX87W9/4/HHH+f555/ngw8+wGazkZaWRnh4uMv+72WXXcbAgQNZvXo1FouFP/3pTy7zT5kyhV27drFgwQJmzpzJm2++SV5eHgcOHGDAgAGMHDkSk8nk3KbKbfRGOSePsTdrS72aqeeVZJGRm0r7yEsaMLLaWa1W3n33XbZt28a0adPo2rWry+PltjIO5O3mZHndL1K02kvZfWITsaHt3B1uneXk5HDzzTdz++23c/PNNxMSEuKxWOrKYDIT3ms46e/OIGrgWKjH71FDMxoMXNGhKZe2jWD1T8dq7GdyqtgIf27qG4fF5LntCAsLq7ZMYFWhARae/n137k1KobDEWuu4qqJC/Bif2Bpfi+e2q02bNrRp06bWx9s3D+GGXrG8+PmuOs1nNhkYP7AVkcGe7U87duxYj76+J7k9gVJ5Eq9yZ0REBCrKx7/66itGjBjB5s2bueaaa1iyZMkF1Z/h2LFj/OMf/3De7t69O6+//rpLGbzBYOCWW24hLS2NWbNmARUn/f/3v//VuL5ypczMTPbu3Uvbtm2BMydQGjKWSldeeSUdO3Z0NtjcuXPnGZ9js9lcdqDi4+N5++23XZIbRqOR5557jp9//tn5nbN06dIaEyinCggIYMqUKZSUlDBjxgwANm3axOrVqxkyZEitz6uaQFmzZk21pc7q4z//+Q9ffvklO3bsoHnz5vTo0YMpU6ac9jlpaWksXLiQ1NRUUlNTKSwsJDQ0lIsvvpjJkyfTt29fysrKeOGFF1yW//riiy/Izs5m9OjRXHnllUDFe/zOO++wfv16UlNTSUtLw9fXl5iYGAYPHlxjXxkRERERb/HEE0/Qpk2bGpu7Q0Vz4ptuusl522g08vbbbzNlyhTi4+O59tprq42fNWsWo0ePdp7IDg4O5p577mHs2LHOfifx8fHceuutpKWl0bt3b+fzhw8fztChQ7n88svp2LEjUJGEWLx4Ma+88grHjx+nTZs2TJ48ucZ4IyIiePDBB8nKynL2JDSbzdxzzz1cccUVfPjhh6SkpGCxWLjzzjuZOHFitWXF5s+fz8MPP8wVV1xBjx49qm3f3LlzGTNmDB9++CFbtmyhS5cuXHXVVdxwww3OBM7QoUPZsGGDc5/YG2Xk7sZut9VrGTK7w8bxwkxah3fBbDz3+8gOh4MDBw4wf/58kpKSGDt2LE8++SQtWlQs5VNiLeJE0cFal+2qmYETRYc5WVaAv49nrtK32WysWbOG7777jrlz5/Lss89W+7fXGPlFxROQ0JUDS+bQcuSDng7HRUSwL3+9oTMZJ4rZe7TgtH1D/HxMTLuqHb1ah9PYV+Xr2TqcR0ZexGPvb6ek3FZr7Zjd7qBpqB8Lp1xKbET9q1bOJbPJwO0DW3G8oISk5DTMpto/BKPBwI29WzL60rhax0jDMzjcfHlq5ReRp656rWnJGhFpPI4dO8YNN9zA9u3badmyJUuXLnUpkfe0kJAQlyWQ3GnlypUuFRiLFi2qdSfR4XCQkJBATk4OAL1792bFihUUFRURHR3tHNenTx82bNgAVDSNnzRpEtnZ2bRu3Rq73U5AQABt2rRh27ZtAMydO5cJEya4JZY333yTBx54wDnu6NGjLhUolfFVJlAuu+wyvvzyy9O+R/v27ePiiy923n7xxRe54447ahx77Ngx+vfvz/Hjx/nHP/7B73//ewA6derkXKps2rRpLlfgVcrPz6dt27bOJbtmzZrFPffcw6OPPsorr7zijL3yvW3atCl79+7FYDDw0EMP8eqrr1YbEx0dza5dtV9BYrVaGT9+PEuXLq32WHh4ONHR0ezYscMlHoAPPviAP/7xj7Uufefn50dSUhL+/v5cf/31NY5p1aoVW7ZsIScnhxEjRrBly5Za4xw8eDDvvvuusxKpoTTkvzVvUvUK1cbA0/txIiLS+Om7QrxV8s/vc6zowJkHniLUJ5LOYf3wM537/bbS0lJmz57tPEYBiIyMZOrUqUycOBGfEPj+0KdY7fVfAvii8H5E+bq3T0BdGAwGDh8+TLdu3Vzu79+/P3/961/p06cP4eHhjbbfTnlBFpse6EmP577Hp0lzT4dTTXGpleeW/sS3u05QcLKc0nI7DhxYTEYCfc3EhPszfXhHLmkd7ulQ68zucPDd7ixeW76X9BNFFJdaKbfZMWDAx2wk2N/MJa3Cue/aDsSE+595wkbk4x8ySfpmP8fzSykutWK3OzAaDfj7mAjxt3DL5XGM7ZeAxdx4Kp4au4bYT9ESXiJyTkVFRbFs2TJGjRrFhg0buOqqq/joo4+qlSKXlZWxYcMGrrjiCg9F6n67d+92ud23b99axxoMBrp27co333wDVFQg1KR///7OE/jJyclMmjSJ1atXO5dxuvTSS2vsg9EQsVRVXl7OCy+84EyeQMVSA2dSdTxUNLOsTVRUFDt27CA/P9+5NEJdhYSE0KlTJ3788ccaXxcqkg6HDh0iMzOT48ePs2PHDrp06eKyXFrV9/9MFi5c6JI8adasGf369WPjxo1kZGSQnZ1d7TllZWVMnz7dmTy5/PLLGTRoEMHBwbz77rts3bqVkpISHnzwQVJSUrj77rv56KOPOHr0KFCx9mrfvn0ZPnw4AO+8844zeRIUFMTvf/972rdvz759+3jjjTcoLy9n5cqVfPTRR9WWr5ALw8CBAz0dgoiIiEiDKLOdPPOgGhw7foyUz/7Dydyzr0I/WzabrdrywSdOnODxxx9n6dKlPPXCDGwRZ9c/MXnNSo7sLHJHmPViMBhqvJBqzZo1bNu2jUceeYT77ruv0VbFW4IjiLv5UY4sf5O4mx71dDjVBPiamTG6K7sO5/Pj/hwOZp/EarcTHuhLx5gQ+raN8LqT8UaDgX4dIrk4IYyNP2ez+0g+OYVlGA0GokL96B4fxkWxoZg9uBzZ2bq+dyyXd4jkh5+z2Xu0kOJSKwE+ZhKiAunbLoKoED9PhygogSIiHhAaGsonn3zCrbfeyooVKxg2bBjvv/++syTdZrNxxx13sHbtWn788Udnk0VvVzVpERoa6tIMvSadO3d2Ji2ysrKw2WzVxgwYMIA5c+YAFTucdru92gn+qs3YGzKWq666CqPRSE5ODgcPHnRphG6xWLjttttO+xpAtQqOMzUmM5vN9U6eVKraVL62ypErrriC9957D4DVq1cTGRnpTLYEBATQs2fPOr/e7NmznT9fdNFFrFy5En9/f2w2G6NHj2bFihXOxyuvlEhPT3cmVkaNGsUbb7yByVTRGLBfv370798fgAMHDmC1Wpk9ezZ5eXn897//BeD66693qcDZtGmTM/alS5e6xJ+WlsayZcuAui23JuenU/spiYiIiJwvLKaz6x8QHh7OxdcNxuKo/cr2ul7xXN9xpaWlnDhxgnXr1jkfCw0N5ZZbbuHWW28lpk1TfszOwOaof3Knd89LCb2kRZ3iqS2+s3X06FFefvlll/sqlzYePHjwaftTNgbNh0xk7/x7KT6YSkBMR0+HU6MO0SF0iG78vWXqI9DXzMDOUQzsHOXpUNyqaYgfw3q08HQYchqN+38kETlvBQQEsGjRIu68806WLFnC9ddfzzvvvMOQIUO49957+fjjj4GKpYyeffZZD0frHpUnvsG10Xdtqo6xWCw17qQmJCQQGxtLZmYmubm5bN68uU4JlIaIpbZlofz8/Jg1axaXXFLReLG4uJiUlBSXPiJxcXG0adOG0tJSl+c2ZNl2UdGvV1vV1tNkwIABzgTKqlWrXJIuffv2rfNVUceOHXNWhQA899xzzuXOTCYT//rXv2jfvn2157Vr147Zs2eTkZHBo48+islkoqysjC1btrBy5UqXsbt373a+x7X5y1/+QkREBEOHDnUmTw4fPszatWs5duyYc9zpliITEREREfFGYf5RHCvKxFBrF4WahQQ0oWNc57NOwPwWpaWlLheMjRgxglmzZtGhQwfMZjP5pdkEFARTUJpTz5kddI7vQZBvmHsDrqOqy9jGxMTwzDPPcPPNN9d6rNnYGExmwroOIjtlGf7R7TEYva/yQeR81RBLjCqBIiIe4+Pjw8KFCwkJCeHf//43Y8aMYfDgwXzxxRfOMfPnz2fixInOhoverF27ds6fCwoKOHHihMsJ+VNVTUi0atUKYy07Zf3793dWHCQlJbF//36gojllbSfUGyKWli1bYjKZMBqNBAcHExcXR8eOHZk0aRLNm1esDZuTk0OXLl1q7Fe1YMEC57hKqampp00KfPDBB2zdupXJkycTExNT67ia7Nu3z/nzqY02K1VWeACsXbuWkJBfr+Cpz/JyVV/LYDC49HkBaN68OTExMc7eLVVNnjyZb775hscee4yUlBS2bt1aLdEE1Hjfqdq3b8+f//xnPvzwQ8aNG0dKSgqZmZnVxpWVldVls0REROQCNGPGDE+HIHJWWoa2Z8+JH+v1HKPBRERgtEeSJ1Bx7BAWFsbw4cP529/+Vm3pZT9zIBEBLSgozYV6NJIP84vyWPIEKrardevW/OEPf2DKlClnvaqAJwW37UXB3hRKj6fj1+zc95IRkXNHCRQR8SiTycS8efMIDQ3llVdecUmeQEVlwMMPP+ysSPFmbdu2dbn97bffujRyr8put7N9+/Zan1vVgAEDXBIolfr27YvFYjlnsWzcuLFaE/lTFRQU1NoMPSMjgz59+rjc99NPP9WaQNm0aRMTJ04EKkrAKxu710VWVhZHjhxx3q4t+RIfH0/Lli05cOAARUVFvP/++87HBgwYQG5ubp1er+qSZw6HA1/f6gdgVa/Cqrzqymq1ctNNN7ks7wUVlUf9+vVj8eLFLkulncny5csZN26cS/WNv78/PXv2xOFwsHbt2jrPJSIiIhemmTNnejoEkbMSGdiClmHtycjdVecqlCCfUOLDPHcxn9lsZuLEiTzwwAM1HkP4mHyJDW3HkYI0Sqx162diMphpF9nD3aHWS1hYGMuXL6dVK+9NPPhGxuITHk3B7vVKoIic59xeYxYfH098fLy7pxWR89zp/t9YtWqVS/Ntb9W9e3cCAgKct5966qkae4kAPPPMMy4nuXv37l3rvFWrJKo6XYVEQ8VyJnFxcSxevJipU6dy1113cdddd3H33Xfz/PPPM2nSJC677DKXSpjXXnuNkpKSavOUlJQwbdo05+36Vkw88sgjLrevvfbaWsfW9P4GBgbSo0fdDzpOPTCo2oMGKqpH9u7d67xdWXK6fPlyZ/IkMDCQOXPmsGfPHrZu3cq//vUvl6XYqv586jyVHn/8cednOWTIEFasWEFmZiaff/45V1555WnnEhERERHxdt2jB9A0MAZHHao1LCYferQYRKCP53pyGo1GmjVrVmPypFKzoJa0ieiG3VHz8ZzLfAYj8U06EhPSxp1h1pu/v79XJ08ADCYLEb2uIytlmadDEZEG5vYESlpaGmlpae6eVkTOY4sXL+bBBx887ZhHHnmkxhPp3iQiIoJ7773XeXvXrl2MHz+e/Px8530Oh4O33nqLZ555xnlfTEwMd911V63zxsXF1dhsvbbESkPGUhdXX301zzzzDM899xzPPfccs2fPZvLkyQQHB2MymZgwYYJz7ObNm7n33ntdqixKSkr4y1/+wo4dO+q0rVXZbDZee+01Z8UOQHR0NIMGDar1OTXNfbrqnppER0cTFvZrifypDRPnz59fYx+WxYsXO38eMWIEkyZNci43tnbtWpd/EydOnABw6ctSNbG0Z88eNm/e7Lw9d+5cevfujcViweFwsGrVqmpzyYUnISGBhIQET4chIiIi0iACLMH0ir2SiIDoWpMoDhz4mQPo32okzYLjznGE9WcymukcdSk9WgyiYhmvmrfL7rDRMrQ93aMH4GP2O6cxnq/8o9sQENOBQ5/N83QoItKAtISXiHjUsmXLuPvuu8/Y5Ck9PZ25c+fy0EMPnaPIGsb999/PW2+9xeHDhwH45JNPWLduHX379sXPz49vvvnGpZk3wGOPPeZSLVKTAQMG8M477zhvn67/SUPH8ls9+OCDfPjhh86+Ie+99x6ffvopV1xxBQ6HgzVr1lBcXOwcf/HFFzNu3Djn7apNBxctWsR3330HVCQT0tPTXfqvmEwmXn311dNWXAwYMKDafXVN2FSNaerUqcyaNcu5TVarlaFDh7JlyxZef/31Gp9XtefKmjVr2LZtG23atCE5OZmpU6e6jE1NTWXYsGEu/Vy+/vprlixZQnZ2drU+QgsXLmT69OkcPXqUZ599ltWrVzsf27NnDzabTZUoF6D09HRPhyAiIiLSoEJ8wxnQaiR7szaTkbsbq60Mu8OGwWDEbLQQFRhLh6heBPl4rkdIfRkMBjo07UlEQHNSj28kryQLm73iAi2jwUigTwhtIrrTMqx9nZcvk7ppfuUdbH64H037j8ES0tTT4Yhc8Covyq26xP1vpQSKiHjU8ePHad68eY2NrE/1wgsvcOutt9a7WXhjd+ykQgAAH89JREFUEhAQwOeff87kyZP54YcfgIqr/T/99NNqYwMDA3nqqae4+eabnfdZLBYsFouzIqMymTF06FCXBMrAgQMxm83OearO6a5YqlZUBAYGulQ+/Bb+/v589NFHjB49mj179gBQWFhYrT8OQMeOHVmwYIFzW6Giuqby9+nEiRO1VlP4+fkxe/Zsl+qTmt6rli1b0q5dO2csULH8FeCSTDpTYmnatGm89957zsTQ+++/79JTpVmzZhw9ehTAWaJ/ww03kJSUhN1uJzMzk379+mEwGJwJx5CQEGfV0MaNGwHo1KmTc8709HRuv/12AHr27Em/fv2cfU6ef/555syZU+NcxcXF7Nixg27dup12m0REROTCU7nihCoWxZv5mPzoHNWXDpG9yCvNotxWislgJsgnFD9L4JknaKQiA2PoF9iCorJ8issq9u39zAEE+TbBaHD7IjQC+IQ1I2bE/RxZvpCWN3r3BZ8i54O33noLcG8CRf97iohH3X777WzevJkXX3yR2NjY044tLi7m//7v/85RZA2nTZs2fPXVV8ycOZPu3bvj5+daPh0TE8OIESNYt24df/jDH1we8/HxYfDgwQD06NHDWW0wfPhw54lzi8XisjxXZX+PgICAatUUvyWWwYMHExpasR7wyJEj3Vqt0KpVK1asWMGDDz5IdHS0y2MGg4H4+Hj+9Kc/sWbNmmpN7e+//37Cw8OrzWkwGAgLC6NTp05MnTqVdevWuSwXBnDllVdiNpsxGo1cddVVzvv//Oc/O38eOnQo3bt3Byr6wURERACn76MCFU3ik5OTufbaa10SPi1atGD+/Pm8/vrrGI1GzGaz83NKTEzkzTffrLY93bt357PPPmP9+vU0b94c+LWP0KhRoxg1apTLeB8fH6699loWLVrE0KFDMRorvv4dDgfBwcHcc889pKamOiu8AgMDXXrRiIiIiFRq1aqV1/cuEKlkMpoJ929Gs6A4IgNbeHXypJIBA0E+oUQFtSQqqCUhfhFKnjSw6KsmUXJ0PycP7z3zYBHxOgbHmdbN8TJVl2YREe9SXl7OO++8w/PPP09GRkat47744gsuv/zyBomh6lX454rNZmPv3r3k5ubSsWNHZ1KiNna7nc2bN9OtWzeXE/ElJSVs3bqVDh06VJtj586dxMTEnHHu+sZSXFzMnj17nAmFhmC32zly5AhHjhzBarXSqVMngoODG+z1Dh06RHl5uTMhUSkjI4Pc3NxqVRnFxcXs3bu3XtUaxcXFbN26lSZNmtC6dWtnP5W0tDR8fX2rJY0cDgd79+4lLy+PTp06uVTK1Pb6mZmZ7Nu3j2bNmtGmTRuX35XCwkK2b99O8+bNiY+Pd1n2rK6/K7+VJ/6tNUYN+bt8Nip/F86z3UMREXEjfVeIiFR3bPU7lBfmEDNsKihhJeIxDbGf4vYESmJiIgDJycnunLbOlEAR8X5Wq5X//ve/PP/88+zfv7/a4926dePbb79tkNfWSV2Rc0P/1ioogSIiIt5G3xUiItWdPLyXw1/NJ2bYNHybxnk6HJELVkPsp7g9Jbp69WqXRrQiIvVlNpsZN24cKSkpvPrqq7Rp08bl8a1bt7JgwQIPRSciIiIiIiIi8iu/pnH4hDQlf88Png5FRNxMNWUi0miZzWbGjh3Lxo0bef3112nXrp3zsSeeeILc3FwPRicicv5ZtWoVq1at8nQYIiIiIiJexWD2IbzP78ja8InzvvK84xRnpnowKhFxByVQRKTRM5lM3HLLLaSkpPDmm2/SoUMHsrKymDVrlqdDExE5ryQmJjqXYxURERERkboLiOmAX7NWHFmRxPE177H1/4Zw/Lv3PR2WyAVlxowZzJgxw61zms88RESk8bjpppu46aab+OCDD3jhhRfYuXMnnTt39nRYIiIiIiIiInKBcthtWAuy8AmLYt8b94HRiMNux6B2USLn1MyZM90+pxIoIuKVRo0axahRo8jIyPB0KCIiIiIiFww1jxcRqS53ywoOfjqXgj0bwGj65V6DR2MSEffQEl4i4tXi4uI8HYKIiIiIiIiIXMBMASHYy0tx2G1V7nWgJIqI93N7BcrChQvdPaWIiIiInAOV5c4NUfYsIiIiInK+CunQl47T3+Xn1+8hO+VzDCZLxQMGJVBEvJ3BcZ7V3xYUFHg6BBHxYiEhIeTn53s6DJHznv6tVQgODvZ0CC4MvxzgnWe7hyIiIiIi54TdWkbmh89xdNVblOUdo+XIh4i76RFPhyVywUhOTgYgMTHRbXOqB4qIiIiIiIiIiIjIb2Q0+xB386MEtu5O5kfPV+mHIiLnwqBBgwD3XhSoBIqIiIiIiIiI1MmECRMASEpK8mgcIiKNWfgl1+LfrDXWYlXdi3g7LeElIlJFSEiIp0MQuWBoCS8t4SUiIt5H3xUiIiLSWDXEforbK1Aqr0KpvCpFRMSb6ISuiIiIiIiIiIiIQANUoHj6ahRVoIiIiIi3UAWKiIh4G31XiIiISGPlFRUoIiIiIuKdxo8f7+kQRERERERERBoNVaCIiIiIeEhjq0ARERE5E08f84uIiIjUJiEhAYC0tDS3zakEioiIiIiHKIEiIiLextPH/CIiIiLnkpbwEhEREREREZE6mTFjhqdDEBERETlnVIEiIiIi4iGqQBERERERERFpvNxegTJw4EB3TykiIiIiIiIiIiIiInJOub0CxdNUgSIiIiLeorFVoHi6klhERERERESkMTF6OgAREREREREREREREZHfIjExkcTERLfOqQoUEREREQ9RBYqIiHib5ORkALefnBAROVtp7/wfDrudVuOeOu24vB3fcHTVv2l/zxvnKDIROdca4pjW7T1QREREREREROT8NGjQIEDJdhFpPAr2/oDDZjvjuLLcI+Tt+KZOc9qK8ynOTCUgrjMmv6Czjq284ASFe1MIvWgARh//s57HnUqOp2MtzCEwrgsGk04Ni5yJlvASERERERERERERL2UAw6+3HDYrtuJ8bCcLcNhrTqzYSoqwlRRBTclgh4PCtC2kvjiO4oyd2MtKqjxmx3ayEFtJEQ6H3fVpdhu2kwXYThaAww4OB9kpy9j57E0UpW3DYSsHak4+20uLsZUU4rDbnTHYy0tx2KwucdnLTv66TQ4HttKiatvpsNsqnvtLrPbSYud2Ouw2Dn/xGunvzqAs+5Bzfofdjq2kEFtxfrXtErnQuT3NmJCQAEBaWpq7pxYRERERERERERGpwuHMSxSlbSV70zJKjmVgMJkJbtebZom3/TrS4SB74+fkbF2BwWAg9KIBRPQZ4TJb8cFdHFmZhK20iMNfv4E5MIyEsY9jLysma+NnFO5NAaOJ4Ha9iOh1HSb/YKzFeWSt/5iCvSngsNPk4iuxhESSvfFzMBg4vPxNgtN60vSKMZgDQn+Nx1rGiQ2fULjnB+zWMgJiOtB0wFgMBiPH1y7G6BtAVP9bACg9cYAjq94ioudw/GM6kP3jFxTs+h57eSlBrboT0ed6LCGRFO3fQu6O1ViCIyjc9yNGH3/CL7mG0M79OboiiYJd31Oef4LMT14kpHN/InoNI2v9J+Tv+g6HtZyQjpcTNXBsw39sIl7C7RUo6enppKenu3taERERERERERERkVP8WoFybM17FPy8iYCY9tjLTpKx+MmKpMYvHLZy9i2YjsFg5OThvex/62GK0ra6zGY0WzAYLWC3YTCaMFp8sVvLObLiLTKXzMGvWStMvgEceP8ZsjZ+CkB+6ndkfvIiPuHR+DVvTeH+zRiM5oolsgwGDIDB7IvB4Hoq9mjy2+xPegiHzYolpCmZS+eSvuhxjBYfCn/eRMbiJ51jc7au4PDn/8JgtpC98TMy3nsck28Qvk3jOfjZPzj2zbs4bFaKDuzkwAfPcmjZPzH6BpC3YzUHPnyWkuPpGC1+YDQCDjCaMPn4UZy+g4z3Z2HyD8Y/pj0Fe9Y3yKck4q200J2IiIiIALB//35PhyAiIiIiUj8OB5UZlFbjZjnvLj2RSeoLYynY/T3BbXs6x7a/P4mQ9pdSnPkTu+ZO5PCXr9H2rn84n+fXvA3RV/6BvO3JRF9zF0GtLubk4b2c+P5DYobfQ/Mr78RhK8d2soCs9UuI6v97Sk8cwGAwEtX/FvyiEpxzRRwdTtb3HxF97VSCWnV3CdtedpLDX71OUNuetP7DnIr7Sos4snwB8Tf/jZBO/cja8DE5W1cQ2qk/+T+tw695awJiOpD29qNEXnYj8WP+DgYDRpOZ7I2f0WzweADMAaG0uv0Zwrokkr/nB1LnjKU0K5OoxFspPrSLovRtxN34EJbQKLI2fIzDWkZk3xsIan1JA3xAIt5NCRQRERERAX5dilVERERExGsYDBh+WcKr5HgGJ9a9z8lDe7AWZlNy/IBLDxOjxZeQ9pcC4BvZEktQOCXHM6rPeUqrEnvpSaz5WRxNfpvsTV/icNgpyz6Eb2QsAMHtL+XY6nfY/tQIIvuMICpxHAExHX6Jr+YFgMpyjuAoL+PkwT3sfPamivtyj2AJaYq1OJ/QiwZiCWnK0eVJ+IbHUHxgJ80GjcNeXkZp1iHKco9SlLYNqGhWb/QNcPY68WnSHN/wmIrtbBINDgcOa1mNcQS17olPRCypL95O+CXX0GzwBALjLgKDocbxIo3ZwoUL3T6nEigiIiIiIiIiUifx8fGeDkFExJXDgQOwW8vIWPwk+bu+I/qqyRiMRooP7sK5vpejogcKDjsYjNjLy3DYrJgDQ2uZ1/5rk3lDxZ/QTv0I7tD310RFeAsAguK70m7qa2Rv/Iyjq/5NUfp22t/zRpV5amjM/ktixb95a5oNvt1ZSWO0+ODbpDkGsw/BHfqS/cOn5G7pi7U4n7DuV1ZsiNFAUKseNLnkagxGEwCWkKaY/INreB0DBuMpSRyHo+K9AHwjY2k/bT7ZGz/l6Kp/k797PZ3//B4+ETFnfu9FGpkJEya4fU4lUERERERERESkTtLS0jwdgoiIq18qUGxFuZQc3Y9/i3a0GH4PuVuW47CWO/MnAA5rOT8vmE7czX8jP3UdJcfTibl8dPU5jUZspSexFudjK87HHBCKb1QCpVkHSeg1HAxGyrIPYbdZASg6sAOzfwixI+6nvCCHvG0rsBbmYPDxA4OB0uMZBMZ1AZPJ2QfFJzwaS2gUJ4/uIzChG35N47EW53My8ycMZh8AohJv49jqdzj42SuEdLwMS2hTDCYzAbGdKMs5TEjHy/AJa055YTbWotxqPVZ+2WrXTfPxx1qUi604H3tACCXH0jAYzbQYNg1rSRHHViRRlntUCRSRXyiBIiIiIiIiIiIiIl7JYSvHYbdhCY3CNyKW498uYv0dcdhO5oPdjuOXJIfDYcdgNnP4qzc4/NV8sNsJaNmJFldPrjanyT8YS2hTdj49kvL8HPq9l0vTfjeR9p9HWHtbJD6hzSjLziTmuntJuO0pjq5I4sjyBfiENafkWBoRfW/Ar2k8BpMZW2kxqS/dTnCHS+nwx4X4/pKYMJp9aDn6r+x4YjibHrgES0hTynKP4BcZR8+5FY3tQztejm9ELGXZhwjrkojZPxgMBpoPup1dr9zJxmmd8Y2IpTTrIE37jabNnS+Bw47Dbvu1egYq3oNfbge37UnGoif48cFLaTpwLL7hLcj88Dl8mkRTmpVJcLs+BLTs1NAfm4jXMDgcDseZh9VdcnIyAImJie6cts4KCgo88roiIiIi9RUcXEOJvQfNnDnT5W8RERERkcYuK+VzcNiJ6HUd1uJ8Tqz/CGveCYLb96G8IAv/5m0IjO/KySM/U3zgJwLjLuLEhk8w+QUS0XM4PuHR1eZ02G0U7t9M/o5vMPmH0GzoRAwGIwV7N1Kwez220mL8ohII6zYES3A4ZblHyd60jPLcY/hExBJ56QhMfkEA5O/ZQP5Pa/GPbkeT7kMx+vi5vFbJsTRyNn+NtTAHnybRNOk+xLk0GED+7vWUHN5LWNfBLrEWZewgP3Ud1oIsfCNbEtp1EL7hLTh5+GeK0rcR1m0w5oAQbCVFZP2wlLCLBuAT3gJ7eSk5m7/m5MFdhHUbjF+z1mRv/JTSrIP4hDUnou8NmANCGujTEvE+bk+geJoSKCIiIuItGlsCxfBLo8jzbPdQRERERERELgBJSUmAe3uhKIEiIiIi4iFKoIiIiIiIiIi4R0Mc09bUWUhEREREREREpJrExESPLdktIiIicq6pAkVERETEQ1SBIiIi3kbfFSIiItJYeUUFysyZM9V4VEREREREREREREREvJrbK1A8fTWKKlBERETEW6gCRUREvI2+K0RERKSxaoj9FLPbZhIRERERrzZjxgxPhyAiIiIiIiLSaKgCRURERMRDGlsFioiIyJl4+phfREREpDaJiYkAJCcnu21OJVBEREREPEQJFBER8TaePuYXEREROZe0hJeIiIiIiIiI1MnChQs9HYKIeLmCn1PI+fFrTh5MxVZShDkojMCE7oRfcjX+0e08Hd5ZsZeVkLfzW3K3raTk8M84DOAbEUto5yuI6P07DCbvPAVbcjyDrA2fULR/C9aiHEx+QQQmdCO857UExHQAg9HTIYo0OFWgiIiIiHiIKlBERERE5EJhLczh56SHyFr3Pg6Ho+IcosEADgcOhx2jjz8tb3yY2Osf8HSo9WIvKWLHM6PIT12LwWiu2CZwbpdPaBQdpr9DSLs+ng20Hhw2KweXziXjf0+Cw+6aKHHYMZh9iBnxAHGj/+q5IEXOEbcnUCZMmABAUlKSO6etMyVQRERExFs0tgRKWloaAAkJCR6NQ0RERETOLyeP/MzPbzxA/k/fnn6gw0Fkv5todfvTWIIjzk1wZ8vhIG/nt6S+NA5bcf5ph/qENSfh1ieI6DMCg9lyjgI8O9bCHDLef5qjq/6Nw1pW+0CHg4g+I2j9h+exhDQ9dwGKnGNuT6B4mhIoIiIi4i0aWwLF05XEIiIiInL+sZ0sJPXF28jdtgqD0XTG8Q6blWZDJtB6wmyMFr+GD/AsFWfsYPe/7qY4ffuvVSe1cuAX1Yr2f3yDoNaXnJP4ztbBT14i439P4bDbzjjWYSun+ZWTaDVuFkafxvtZifwW3rkA32k0thMRIiIiIiIiIiIiF6rDy98kb/vqOiVPAAwmM9k/LKVJ9yuJ6PO7Bo6uZmVlZSxZsoSgoCCGDRtW7XF72UmOfbuIkwd31yF5AmCg5Oh+Di17lfbTXnd/wHV0/PhxXnrpJaZMmUJsbGy1x0uzDnLw05frlDwBMJgsZKd8TpNLria8x9XuDrdOSktLee+994iNjWXIkCEeiUEaj8rVFCpXV3AHdfoRERERERERkTpJSkry2JLdIuJ97NYyjq5YWMckw6+sRXlkp3yGw1beQJGdnt1uZ926dQwfPpxevXqxfft2l8dLsw6St2NN/eIzGDi+7n3spSfdHG3dlZeXM2vWLC6++GLmzZtX7fHDX7+JtTC3fnPmHSN360qPfVY2m42VK1cydOhQBgwYwO7duz0ShzQO6enppKenu3VOJVBEREREREREpE4mTpzIxIkTPR2GiHiJ4oyd2EuLz+q5pVkHKcs96uaI6q5yeduUlBT69u3L+PHj+eabb8jLy8NamEPJ0f31nxMHOVu+dneo9ZaVlcUf//hHOnbsyOuvv87+/fux2uzkbV1R72QXQOmxNMrzsxog0roxGitOca9Zs4ZevXoxadIk1q1bR37+6XvTiNTFedcDRURERETOjnqgiIjImei7QkTqI2frCvbMm4S1qH5VDQAF/s3Y1vxqiixNcDgcGAyGc/Z3eXk5K1as4Pvvv3eJKSIigmuuuYY7rumN37LHOJv/Cfc2S+TnkG7UP03x2xUUFPDCCy+43GcwGOjVqxd3TZlKpzUzoLT+/aXzAmLY1uxKis0hvym+s/2sPvvsMzZv3uwyV9OmTRkxYgT33XcfXbt2/U1xifdoiP0Ut/dA0c6UiIiIiIiIiIiIGC1+YDi7BXCCQ5vQu09f7MHN3BzVmZWVlZGamlrt/pCQEHr06EF0i1hyzBaw1n/ZqraduhDbdiB44NxpVlb1KhGLxULLli1p3749lk3BlJ9FAiU0PII+fS/DHhBx1rEZzqLyBaCkpISUlJRq94eEhNCtWzeioqLOOiYROA+byIuIiIiIiIiIiIjn+UUlYDRbzuq5TaLjaTfkGsyBYW6O6sxKSkr44osvnLd9fX2ZPn06jz76KAEBARSlb6MorDmlJw7Ub2KHg+5XjSEw/iI3R1w3hw4dcrndvXt3Xn/9dfr06QNA6g+XkJ19qKannlZkXHtaD74ac8Bvq0A5G8XFxfzvf/9z3vb39+fRRx/l4YcfxmzWqW/57fRbJCIiIiKAKohFRERExL18I2II7tCXrO+X1Ku3htHiS3D7Sz2SPKnk6+tLu3btGDp0KPfddx8dOnRwPubTJJqgNj0pzcqseyWJw0FAy04eS55UhOAgLCyM3r17M3nyZEaOHInJZHI+3mzQeHJSluGw2+o8p9E3gOC2vT2SPKnk6+tLhw4duOaaa/jjH/9ImzZtPBaLnH+UQBEREREREREREZEGEXvDn8nbvhprUQ7UpfOHw4F/dFuaXj6qwWOrjdls5ne/+x233HILF198cbXHLcERRFx6Pfk711BeULfm6UYfP1pcd6+7Q62XoKAg5s2bx7Bhw2jSpEm1x5t0H0JYtyFkb1qGwWiqYYZTOBwEtOxMeK9rGyDaurFYLNx8881MmzaNLl26eCwOaRxWrVrl9jnd3kRePVBEREREREREzk+JiYkAJCcnezQOEfEuJ777kJ/fuB9bSeEZx5qDmtDxgbcJ6XjZOYjs7NnLTnLgw+fIXPI8BtPpr1E3mMxEJY4jfsz/ebSqpi6K0rez99WpFGVsP+NYo28AnR58j9BOV5yDyEQ8QwkUERERERERERERaVBZGz5h72v3YC3Oq7G6wWErx7dpPF0e/Ri/5t6zBNOBj54n473HKpIopy5T5nDgsFuJGngbrSc+j8kv0DNB1lNR+nZ2v/IHig/uxmA0VnvcYbfhE96CLjOW4R+VcO4DFDmHlEARERERERERERGRBleUsYMjyxdQ+PMmbCcLcNitGM0WzEHhhHZJpPngCfiER3s6zHrLT13H4eULOHlwF/aSQhyAyccf36bxRA34PeG9rqsxEdGYlecd4/DyBeRuXYmtKBe7tQyDyYwlKJyQzlfQ/Mo78A2P8XSYIg3O7QmUtLQ0ABISEtw5rYiIiIg0sAkTJgCQlJTk0ThERERE5PxWnn+C0qxM7GUlmPyC8G0a59Em5G7hsFN6IpOy3CMAmIMj8YuKr1svkUbMVlJIydE0bCWFGH388I2MwxIc7umwRM4ZtydQRERERMQ7qZJYREREREREvNXMmTNd/nYHJVBEREREBFACRURERERERLxXQxzTetfieyIiIiIiIiLiMQkJCVqyW0RERC4YqkAREREREUAVKCIicmb6rhAREZHGyisqUGbOnOnWNcZERERERERERERERETONbdXoOhqFBERERHvpP04ERE5E31XiIiISGPVEPspZrfNJCIiIiJebcaMGZ4OQURERERERKTRUAWKiIiIiIiIiNSJjvlFRESksZowYQIASUlJbptTCRQRERERERERqRMd84uIiMiFREt4iYiIiIiIiEidrFq1ytMhiIiIiJwzqkARERERERERERERERE5hdsrUNR8VERERMQ7JScnA5CYmOjROEREREREREQaA7dXoIiIiIiId1IlsYiIiIiIiMivjJ4OQERERERERERERERE5LcwGAzOCwPdRQkUEREREREREamTmTNnMnPmTE+HISIiInJOaAkvEREREQG0hJeIiJyZvitERESksWqI/RS3V6CkpaWRlpbm7mlFRERERERERERERETOGbdXoOhqFBERERHvpP04ERE5E31XiIiISGPVEPspZrfNJCIiIiJeLT4+3tMhiIiIiIiIiDQaqkARERERERERkTrRMb+IiIg0VqpAEREREREREREREREROcX+/fvdPqcSKCIiIiIiIiJSJ+PHj/d0CCIiIiI1SkhIcPucWsJLRERERERERERERETkFEZPByAiIiIiIiIiIiIiItLYuH0JL1WeiIiIiHinxMREAJKTkz0ah4iIiIiIiEhj4PYlvERERETEO2kpVhEREREREfFWM2fOdPnbHZRAERERERFACRQRERERERHxXg1xTKseKCIiIiIiIiJSJwaDwXlyQkREROR8pwSKiIiIiIiIiIiIiIjIKdyeQJkwYQITJkxw97QiIiIiIiIiIiIiIiLnjNt7oGjtbBERERHvpP04ERE5E31XiIiISGPVEPspZrfNJCIiIiJebeHChZ4OQURERERERKTRUAJFRERERAC0DKuIiIiIiIh4rRkzZrh9Ti3hJSIiIiIiIiJ1omN+ERERuZCoAkVERERERERE6mT//v2eDkFERETknFEFioiIiIiIiIiIiIiIyCncXoHSEOuMiYiIiEjDS0pKAtQLRURERERERAQaoAJFRERERLyTKolFRERERETEW6WlpQGQkJDgtjmNbptJRERERERERM57M2fOxGAw1Phn5syZ1canpaXVOr4yeX+qCRMm1Os1kpOTax1f20mUxMTEWp9TWZVZVVJSUq3jExMTa3yNhISEWp+TnJxcr/e2tgrR0723lSeS6voa+vxc6fPT56fPL6HG19Dnp8+vsX5+rVq1olWrVjWOP1tKoIiIiIiIiIiIiIiIiJxCS3iJiIiICKAlvERERERERESqcnsFSnJyco1lOCIiIiIiIiIiIiIiIt7C7RUounJRRERExDtpP05ERERERETkV2ZPByAiIiIijcPAgQM9HYKIiIiIiIhIo/H/7XtUmk5u5CwAAAAASUVORK5CYII=)" + ], + "metadata": { + "id": "MKy5JUKMDIKp" + } + }, + { + "cell_type": "markdown", + "source": [ + "### **Example 1: MIMIC-III**\n", + "- **[Initialize]:** In this example, we load the MIMIC-III data by [pyhealth.datasets.MIMIC3Dataset](https://pyhealth.readthedocs.io/en/latest/api/datasets/pyhealth.datasets.MIMIC3Dataset.html).\n", + "\n", + " - The root of this datasets is in (we use a synthetic MIMIC-III for demo)\n", + " - `https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/`.\n", + " - For each patient's ICU stay, user wants to obtain the `DIAGNOSES_ICD`, and `PROCEDURES_ICD` tables. **Note that, different databases have different raw table names**.\n", + "\n" + ], + "metadata": { + "id": "xfYVXbMHX_cE" + } + }, + { + "cell_type": "code", + "source": [ + "from pyhealth.datasets import MIMIC3Dataset\n", + "dataset = MIMIC3Dataset(\n", + " root=\"https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III\",\n", + " tables=[\"diagnoses_icd\", \"procedures_icd\", \"prescriptions\", \"noteevents\"]\n", + ")" + ], + "metadata": { + "id": "5qKMacix0RE_", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "88de5f57-7ba5-41c8-e0f0-f867770cd601" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "No config path provided, using default config\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.mimic3:No config path provided, using default config\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Initializing mimic3 dataset from https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III (dev mode: False)\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "/usr/local/lib/python3.12/dist-packages/pyhealth/datasets/mimic3.py:50: UserWarning: Events from prescriptions table only have date timestamp (no specific time). This may affect temporal ordering of events.\n", + " warnings.warn(\n", + "INFO:pyhealth.datasets.base_dataset:Initializing mimic3 dataset from https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III (dev mode: False)\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "- **[Output]:** user can check the data loaded by the `.stats()` function. The output `dataset` effectively contains a table of patients, each containing a set of events, defined by the loaded tables." + ], + "metadata": { + "id": "2z2nf68aslx9" + } + }, + { + "cell_type": "code", + "source": [ + "dataset.stats()" + ], + "metadata": { + "id": "wdDj0xEBsfPS", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 974 + }, + "outputId": "c8f4cab3-009a-4121-e1d5-fe864f02dc7b" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "No cache_dir provided. Using default cache dir: /root/.cache/pyhealth/3b3dd072-c3a0-53d4-90cd-e62cb8e1294a\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:No cache_dir provided. Using default cache dir: /root/.cache/pyhealth/3b3dd072-c3a0-53d4-90cd-e62cb8e1294a\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Scanning table: patients from https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/PATIENTS.csv.gz\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Scanning table: patients from https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/PATIENTS.csv.gz\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Downloading https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/PATIENTS.csv to /root/.cache/pyhealth/3b3dd072-c3a0-53d4-90cd-e62cb8e1294a/tmp/48ffef21-7439-4890-8275-18b800666e37/PATIENTS.csv\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Downloading https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/PATIENTS.csv to /root/.cache/pyhealth/3b3dd072-c3a0-53d4-90cd-e62cb8e1294a/tmp/48ffef21-7439-4890-8275-18b800666e37/PATIENTS.csv\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Scanning table: admissions from https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/ADMISSIONS.csv.gz\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Scanning table: admissions from https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/ADMISSIONS.csv.gz\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Downloading https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/ADMISSIONS.csv to /root/.cache/pyhealth/3b3dd072-c3a0-53d4-90cd-e62cb8e1294a/tmp/33c98d9d-0cd8-4e8a-96ad-2cb326142f2e/ADMISSIONS.csv\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Downloading https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/ADMISSIONS.csv to /root/.cache/pyhealth/3b3dd072-c3a0-53d4-90cd-e62cb8e1294a/tmp/33c98d9d-0cd8-4e8a-96ad-2cb326142f2e/ADMISSIONS.csv\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Scanning table: icustays from https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/ICUSTAYS.csv.gz\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Scanning table: icustays from https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/ICUSTAYS.csv.gz\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Downloading https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/ICUSTAYS.csv to /root/.cache/pyhealth/3b3dd072-c3a0-53d4-90cd-e62cb8e1294a/tmp/2a77d8e8-a14d-4116-a2ed-275f2dc9db3a/ICUSTAYS.csv\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Downloading https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/ICUSTAYS.csv to /root/.cache/pyhealth/3b3dd072-c3a0-53d4-90cd-e62cb8e1294a/tmp/2a77d8e8-a14d-4116-a2ed-275f2dc9db3a/ICUSTAYS.csv\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Scanning table: diagnoses_icd from https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/DIAGNOSES_ICD.csv.gz\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Scanning table: diagnoses_icd from https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/DIAGNOSES_ICD.csv.gz\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Downloading https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/DIAGNOSES_ICD.csv to /root/.cache/pyhealth/3b3dd072-c3a0-53d4-90cd-e62cb8e1294a/tmp/b980ed68-ca6b-41f1-b01f-bcf78146df96/DIAGNOSES_ICD.csv\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Downloading https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/DIAGNOSES_ICD.csv to /root/.cache/pyhealth/3b3dd072-c3a0-53d4-90cd-e62cb8e1294a/tmp/b980ed68-ca6b-41f1-b01f-bcf78146df96/DIAGNOSES_ICD.csv\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Joining with table: https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/ADMISSIONS.csv.gz\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Joining with table: https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/ADMISSIONS.csv.gz\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Downloading https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/ADMISSIONS.csv to /root/.cache/pyhealth/3b3dd072-c3a0-53d4-90cd-e62cb8e1294a/tmp/33b458ea-fc2b-437e-a1f2-1a83c9809500/ADMISSIONS.csv\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Downloading https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/ADMISSIONS.csv to /root/.cache/pyhealth/3b3dd072-c3a0-53d4-90cd-e62cb8e1294a/tmp/33b458ea-fc2b-437e-a1f2-1a83c9809500/ADMISSIONS.csv\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Scanning table: procedures_icd from https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/PROCEDURES_ICD.csv.gz\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Scanning table: procedures_icd from https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/PROCEDURES_ICD.csv.gz\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Downloading https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/PROCEDURES_ICD.csv to /root/.cache/pyhealth/3b3dd072-c3a0-53d4-90cd-e62cb8e1294a/tmp/fb9376f3-9f0f-4a9f-a181-59da74fd50e9/PROCEDURES_ICD.csv\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Downloading https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/PROCEDURES_ICD.csv to /root/.cache/pyhealth/3b3dd072-c3a0-53d4-90cd-e62cb8e1294a/tmp/fb9376f3-9f0f-4a9f-a181-59da74fd50e9/PROCEDURES_ICD.csv\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Joining with table: https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/ADMISSIONS.csv.gz\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Joining with table: https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/ADMISSIONS.csv.gz\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Downloading https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/ADMISSIONS.csv to /root/.cache/pyhealth/3b3dd072-c3a0-53d4-90cd-e62cb8e1294a/tmp/c4463a4f-b179-43b4-8185-6e863a36426a/ADMISSIONS.csv\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Downloading https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/ADMISSIONS.csv to /root/.cache/pyhealth/3b3dd072-c3a0-53d4-90cd-e62cb8e1294a/tmp/c4463a4f-b179-43b4-8185-6e863a36426a/ADMISSIONS.csv\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Scanning table: prescriptions from https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/PRESCRIPTIONS.csv.gz\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Scanning table: prescriptions from https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/PRESCRIPTIONS.csv.gz\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Downloading https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/PRESCRIPTIONS.csv to /root/.cache/pyhealth/3b3dd072-c3a0-53d4-90cd-e62cb8e1294a/tmp/e0fb7026-fed3-4c3d-829e-071026f00607/PRESCRIPTIONS.csv\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Downloading https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/PRESCRIPTIONS.csv to /root/.cache/pyhealth/3b3dd072-c3a0-53d4-90cd-e62cb8e1294a/tmp/e0fb7026-fed3-4c3d-829e-071026f00607/PRESCRIPTIONS.csv\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Joining with table: https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/ADMISSIONS.csv.gz\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Joining with table: https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/ADMISSIONS.csv.gz\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Downloading https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/ADMISSIONS.csv to /root/.cache/pyhealth/3b3dd072-c3a0-53d4-90cd-e62cb8e1294a/tmp/5356bbfc-1bfc-45f0-a6f3-691bff93fe5a/ADMISSIONS.csv\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Downloading https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/ADMISSIONS.csv to /root/.cache/pyhealth/3b3dd072-c3a0-53d4-90cd-e62cb8e1294a/tmp/5356bbfc-1bfc-45f0-a6f3-691bff93fe5a/ADMISSIONS.csv\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Scanning table: noteevents from https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/NOTEEVENTS.csv.gz\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Scanning table: noteevents from https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/NOTEEVENTS.csv.gz\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Downloading https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/NOTEEVENTS.csv to /root/.cache/pyhealth/3b3dd072-c3a0-53d4-90cd-e62cb8e1294a/tmp/d497ae08-d725-4c9e-a556-1f339a6054b6/NOTEEVENTS.csv\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Downloading https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/NOTEEVENTS.csv to /root/.cache/pyhealth/3b3dd072-c3a0-53d4-90cd-e62cb8e1294a/tmp/d497ae08-d725-4c9e-a556-1f339a6054b6/NOTEEVENTS.csv\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Preprocessing table: noteevents with preprocess_noteevents\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Preprocessing table: noteevents with preprocess_noteevents\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Joining with table: https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/ADMISSIONS.csv.gz\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Joining with table: https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/ADMISSIONS.csv.gz\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Downloading https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/ADMISSIONS.csv to /root/.cache/pyhealth/3b3dd072-c3a0-53d4-90cd-e62cb8e1294a/tmp/ccb26ae6-3af4-4471-9671-57f3e9549279/ADMISSIONS.csv\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Downloading https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/ADMISSIONS.csv to /root/.cache/pyhealth/3b3dd072-c3a0-53d4-90cd-e62cb8e1294a/tmp/ccb26ae6-3af4-4471-9671-57f3e9549279/ADMISSIONS.csv\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Caching event dataframe to /root/.cache/pyhealth/3b3dd072-c3a0-53d4-90cd-e62cb8e1294a/global_event_df.parquet...\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Caching event dataframe to /root/.cache/pyhealth/3b3dd072-c3a0-53d4-90cd-e62cb8e1294a/global_event_df.parquet...\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Dataset: mimic3\n", + "Dev mode: False\n", + "Number of patients: 49993\n", + "Number of events: 2176707\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "patient1 = dataset.get_patient('1')\n", + "patient1" + ], + "metadata": { + "id": "PgYrkEDEZ2fA", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "b14503b9-dd82-40c9-b818-9bbbd1cf6e17" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Found 49993 unique patient IDs\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Found 49993 unique patient IDs\n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": {}, + "execution_count": 4 + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "### **We can get a patient's events by calling the .get_patient() function with their get_events()**\n", + "\n", + "**One key thing to understand is that:**\n", + "\n", + "\n", + "\n", + "1. All events are loaded directly from the tables defined above.\n", + "2. Everything is lazily loaded, meaning no computation is done until data access is required.\n", + "3. The events available are all defined in the .yaml file. **We'll go into the inner-workings of how datasets work later in the deep dive section below.**\n", + "\n", + "\n", + "![Image description](https://drive.google.com/uc?export=view&id=1h3Ija0CqcEBH5ui0upRqRo6Ym1Zt_mTT)\n", + "\n", + "\n" + ], + "metadata": { + "id": "bgRxP_AlPEFl" + } + }, + { + "cell_type": "code", + "source": [ + "dataset.get_patient('1').get_events()" + ], + "metadata": { + "id": "0JEBSOezsDhy", + "collapsed": true, + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "584c8896-0672-4dc5-e773-7819672f50ea" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "[Event(event_type='patients', timestamp=datetime.datetime(2026, 2, 20, 18, 7, 0, 445569), attr_dict={'gender': 'M', 'dob': '2102-09-07', 'dod': None, 'dod_hosp': None, 'dod_ssn': None, 'expire_flag': '0'}),\n", + " Event(event_type='noteevents', timestamp=datetime.datetime(2024, 12, 2, 0, 0), attr_dict={'hadm_id': '100000', 'text': '**Discharge Summary**\\n\\n**Patient Information:**\\n- Name: John Doe\\n- Age: 60\\n- Gender: Male\\n- Medical Record Number: 123456\\n\\n**Admission Details:**\\n- Admit Date: 03/15/2023\\n- Admit Time: 14:30\\n- Discharge Date: 03/22/2023\\n\\n**Chief Complaint:**\\nShortness of breath and mild chest pain after a spinal manipulation therapy session.\\n\\n**History of Present Illness:**\\nPatient Mr. John Doe, a 60-year-old male with a past medical history significant for Type 2 diabetes mellitus, uncontrolled, with background diabetic retinopathy, secondary malignant neoplasm, transplanted kidney, and a history of drug abuse in remission, presented to the ED with acute onset of shortness of breath and mild chest pain. The patient reported feeling pain in his back after undergoing lumbar spinal manipulation therapy earlier in the day. He also noted a mild cough and some difficulty breathing.\\n\\n**Past Medical History:**\\n- Diabetes Mellitus, type II, poorly controlled\\n- Lumbago with previous sprain\\n- Diabetes with retinopathy\\n- History of kidney transplant\\n- Hypoxemia\\n- Secondary malignant neoplasm\\n- Contusion of lung (suggestive of post-procedural complications)\\n- History of drug abuse in remission\\n- Overweight (BMI 28.5)\\n\\n**Pertinent Findings:**\\n- Physical examination revealed normal oxygen saturation at rest and mild tenderness over the lumbar spine.\\n- Chest CT scan showed a contusion of the lung without mention of an open wound into the thorax.\\n- Arterial blood gas (ABG) indicated mild hypoxemia.\\n- EKG and troponin levels were normal.\\n\\n**Hospital Course:**\\nThe patient’s management included supplemental oxygen during hospitalization, continuous monitoring, and rest. Pain relief with NSAIDs and anti-diabetic medication regimen adjusted for better glycemic control. The patient also received education on preventive measures to avoid complications post-spinal manipulation procedures.\\n\\n**Discharge Plan:**\\n1. Continue with follow-up with his primary care physician and endocrinologist for diabetes management.\\n2. Follow-up with the nephrology team for assessment of kidney function post-transplant.\\n3. Avoided high impact activities and to gradually return to normal activities over the next 2-4 weeks under guidance.\\n4. Monitor oxygen saturation levels at rest and with activity.\\n5. Strict adherence to the anti-diabetic medication regimen and avoid spinal manipulation therapies until healed.\\n6. Follow-up with an ophthalmologist for diabetic retinopathy.\\n7. Encourage lifestyle modifications for weight control.\\n\\n**Follow-Up Appointments:**\\nPrimary Care Physician and Endocrinologist in 2 weeks for blood sugar and weight management. Follow-up nephrology evaluation in 4 weeks for kidney function. The next ophthalmology appointment is in 3 months.\\n\\n**Discharge Medications:**\\nMetformin 500mg PO bid, Lisinopril 20mg PO QD, and Oxybutynin 5mg PO TID.\\n\\n---', 'category': 'Discharge summary', 'description': 'Discharge summary', 'storetime': None}),\n", + " Event(event_type='prescriptions', timestamp=datetime.datetime(2103, 6, 7, 0, 0), attr_dict={'hadm_id': '100000', 'drug': 'HYDROmorphone (Dilaudid)', 'drug_type': 'MAIN', 'drug_name_poe': 'HYDROmorphone (Dilaudid)', 'drug_name_generic': 'HYDROmorphone PCA', 'formulary_drug_cd': 'HYD12.5PCA', 'gsn': '004101', 'ndc': '00074233411', 'prod_strength': '12.5mg/50mL Syringe', 'dose_val_rx': '12.5', 'dose_unit_rx': 'mg', 'form_val_disp': '1', 'form_unit_disp': 'SYR', 'route': 'IVPCA', 'enddate': '2103-06-07'}),\n", + " Event(event_type='prescriptions', timestamp=datetime.datetime(2103, 6, 7, 0, 0), attr_dict={'hadm_id': '100000', 'drug': 'Potassium Chloride', 'drug_type': 'MAIN', 'drug_name_poe': None, 'drug_name_generic': None, 'formulary_drug_cd': 'KCL20PM', 'gsn': '045309', 'ndc': '00338070341', 'prod_strength': '20mEq/50ml Premix', 'dose_val_rx': '20', 'dose_unit_rx': 'mEq', 'form_val_disp': '1', 'form_unit_disp': 'BAG', 'route': 'IV', 'enddate': '2103-06-07'}),\n", + " Event(event_type='prescriptions', timestamp=datetime.datetime(2103, 6, 7, 0, 0), attr_dict={'hadm_id': '100000', 'drug': 'Calcium Gluconate', 'drug_type': 'MAIN', 'drug_name_poe': 'Calcium Gluconate', 'drug_name_generic': 'Calcium Gluconate', 'formulary_drug_cd': 'CALG1I', 'gsn': '001356', 'ndc': '00517391025', 'prod_strength': '1g/10mL Vial', 'dose_val_rx': '2', 'dose_unit_rx': 'gm', 'form_val_disp': '2', 'form_unit_disp': 'VIAL', 'route': 'IV', 'enddate': '2103-06-07'}),\n", + " Event(event_type='prescriptions', timestamp=datetime.datetime(2103, 6, 7, 0, 0), attr_dict={'hadm_id': '100000', 'drug': 'Heparin', 'drug_type': 'MAIN', 'drug_name_poe': 'Heparin', 'drug_name_generic': 'Heparin Sodium', 'formulary_drug_cd': 'HEPA5I', 'gsn': '006549', 'ndc': '00641040025', 'prod_strength': '5000U/ML VIAL', 'dose_val_rx': '5000', 'dose_unit_rx': 'UNIT', 'form_val_disp': '1', 'form_unit_disp': 'ml', 'route': 'SC', 'enddate': '2103-06-07'}),\n", + " Event(event_type='prescriptions', timestamp=datetime.datetime(2103, 6, 7, 0, 0), attr_dict={'hadm_id': '100000', 'drug': 'D5W', 'drug_type': 'BASE', 'drug_name_poe': None, 'drug_name_generic': None, 'formulary_drug_cd': 'D5W250', 'gsn': '001972', 'ndc': '00338001702', 'prod_strength': '250ML BAG', 'dose_val_rx': '250', 'dose_unit_rx': 'ml', 'form_val_disp': '250', 'form_unit_disp': 'ml', 'route': 'IV DRIP', 'enddate': '2103-06-07'}),\n", + " Event(event_type='prescriptions', timestamp=datetime.datetime(2103, 6, 7, 0, 0), attr_dict={'hadm_id': '100000', 'drug': 'Pantoprazole Sodium', 'drug_type': 'MAIN', 'drug_name_poe': None, 'drug_name_generic': None, 'formulary_drug_cd': 'PANT40I', 'gsn': '047635', 'ndc': '00008092355', 'prod_strength': '40mg Vial', 'dose_val_rx': '40', 'dose_unit_rx': 'mg', 'form_val_disp': '1', 'form_unit_disp': 'VIAL', 'route': 'IV', 'enddate': '2103-06-07'}),\n", + " Event(event_type='prescriptions', timestamp=datetime.datetime(2103, 6, 7, 0, 0), attr_dict={'hadm_id': '100000', 'drug': 'Acetaminophen', 'drug_type': 'MAIN', 'drug_name_poe': 'Acetaminophen', 'drug_name_generic': 'Acetaminophen', 'formulary_drug_cd': 'ACET325', 'gsn': '004489', 'ndc': '00182844789', 'prod_strength': '325mg Tablet', 'dose_val_rx': '325-650', 'dose_unit_rx': 'mg', 'form_val_disp': '1-2', 'form_unit_disp': 'TAB', 'route': 'PO', 'enddate': '2103-06-07'}),\n", + " Event(event_type='prescriptions', timestamp=datetime.datetime(2103, 6, 7, 0, 0), attr_dict={'hadm_id': '100000', 'drug': 'Docusate Sodium', 'drug_type': 'MAIN', 'drug_name_poe': 'Docusate Sodium', 'drug_name_generic': 'Docusate Sodium', 'formulary_drug_cd': 'DOCU100', 'gsn': '003009', 'ndc': '63739008901', 'prod_strength': '100mg Cap', 'dose_val_rx': '100', 'dose_unit_rx': 'mg', 'form_val_disp': '1', 'form_unit_disp': 'CAP', 'route': 'PO', 'enddate': '2103-06-07'}),\n", + " Event(event_type='prescriptions', timestamp=datetime.datetime(2103, 6, 7, 0, 0), attr_dict={'hadm_id': '100000', 'drug': 'Hydromorphone', 'drug_type': 'MAIN', 'drug_name_poe': 'Hydromorphone', 'drug_name_generic': 'Hydromorphone HCl', 'formulary_drug_cd': 'HYDR2', 'gsn': '004110', 'ndc': '00074241512', 'prod_strength': '2mg Tablet', 'dose_val_rx': '2', 'dose_unit_rx': 'mg', 'form_val_disp': '1', 'form_unit_disp': 'TAB', 'route': 'PO', 'enddate': '2103-06-07'}),\n", + " Event(event_type='prescriptions', timestamp=datetime.datetime(2103, 6, 7, 0, 0), attr_dict={'hadm_id': '100000', 'drug': 'Verapamil', 'drug_type': 'MAIN', 'drug_name_poe': 'Verapamil', 'drug_name_generic': 'Verapamil', 'formulary_drug_cd': 'VERA120', 'gsn': '000564', 'ndc': '51079068320', 'prod_strength': '120 mg Tab', 'dose_val_rx': '240', 'dose_unit_rx': 'mg', 'form_val_disp': '2', 'form_unit_disp': 'TAB', 'route': 'PO', 'enddate': '2103-06-07'}),\n", + " Event(event_type='prescriptions', timestamp=datetime.datetime(2103, 6, 7, 0, 0), attr_dict={'hadm_id': '100000', 'drug': 'Lisinopril', 'drug_type': 'MAIN', 'drug_name_poe': 'Lisinopril', 'drug_name_generic': 'Lisinopril', 'formulary_drug_cd': 'LISI20', 'gsn': '000391', 'ndc': '00310013239', 'prod_strength': '20MG TAB', 'dose_val_rx': '40', 'dose_unit_rx': 'mg', 'form_val_disp': '2', 'form_unit_disp': 'TAB', 'route': 'PO', 'enddate': '2103-06-07'}),\n", + " Event(event_type='prescriptions', timestamp=datetime.datetime(2103, 6, 7, 0, 0), attr_dict={'hadm_id': '100000', 'drug': 'Amoxicillin-Clavulanic Acid', 'drug_type': 'MAIN', 'drug_name_poe': 'Amoxicillin-Clavulanic Acid', 'drug_name_generic': 'Amoxicillin-Clavulanic Acid', 'formulary_drug_cd': 'AUGM500', 'gsn': '008992', 'ndc': '00029608031', 'prod_strength': '500mg Tab', 'dose_val_rx': '500', 'dose_unit_rx': 'mg', 'form_val_disp': '1', 'form_unit_disp': 'TAB', 'route': 'PO', 'enddate': '2103-06-07'}),\n", + " Event(event_type='prescriptions', timestamp=datetime.datetime(2103, 6, 7, 0, 0), attr_dict={'hadm_id': '100000', 'drug': 'Sertraline', 'drug_type': 'MAIN', 'drug_name_poe': 'Sertraline', 'drug_name_generic': 'Sertraline', 'formulary_drug_cd': 'SERT50', 'gsn': '046228', 'ndc': '52959036100', 'prod_strength': '50mg Tablet', 'dose_val_rx': '50', 'dose_unit_rx': 'mg', 'form_val_disp': '1', 'form_unit_disp': 'TAB', 'route': 'PO', 'enddate': '2103-06-07'}),\n", + " Event(event_type='prescriptions', timestamp=datetime.datetime(2103, 6, 7, 0, 0), attr_dict={'hadm_id': '100000', 'drug': 'Milk of Magnesia', 'drug_type': 'MAIN', 'drug_name_poe': 'Milk of Magnesia', 'drug_name_generic': 'Milk Of Magnesia', 'formulary_drug_cd': 'MOM30L', 'gsn': '003026', 'ndc': '66689036430', 'prod_strength': '30mL UD Cup', 'dose_val_rx': '30', 'dose_unit_rx': 'ml', 'form_val_disp': '1', 'form_unit_disp': 'UDCUP', 'route': 'PO', 'enddate': '2103-06-07'}),\n", + " Event(event_type='prescriptions', timestamp=datetime.datetime(2103, 6, 7, 0, 0), attr_dict={'hadm_id': '100000', 'drug': 'Nicotine Patch', 'drug_type': 'MAIN', 'drug_name_poe': 'Nicotine Patch', 'drug_name_generic': 'Nicotine Patch', 'formulary_drug_cd': 'NICO21P', 'gsn': '016427', 'ndc': '00766145010', 'prod_strength': '21mg/24Hr Patch', 'dose_val_rx': '21', 'dose_unit_rx': 'mg', 'form_val_disp': '1', 'form_unit_disp': 'PTCH', 'route': 'TD', 'enddate': '2103-06-07'}),\n", + " Event(event_type='prescriptions', timestamp=datetime.datetime(2103, 6, 7, 0, 0), attr_dict={'hadm_id': '100000', 'drug': 'Haloperidol', 'drug_type': 'MAIN', 'drug_name_poe': 'Haloperidol', 'drug_name_generic': 'Haloperidol', 'formulary_drug_cd': 'HALD5I', 'gsn': '003968', 'ndc': '00045025501', 'prod_strength': '5mg/mL Vial', 'dose_val_rx': '1-5', 'dose_unit_rx': 'mg', 'form_val_disp': '0.2-1', 'form_unit_disp': 'VIAL', 'route': 'IV', 'enddate': '2103-06-07'}),\n", + " Event(event_type='prescriptions', timestamp=datetime.datetime(2103, 6, 7, 0, 0), attr_dict={'hadm_id': '100000', 'drug': 'Ceftriaxone', 'drug_type': 'MAIN', 'drug_name_poe': None, 'drug_name_generic': None, 'formulary_drug_cd': 'CEFX1F', 'gsn': '009156', 'ndc': '00004200278', 'prod_strength': '1GM FROZ BAG', 'dose_val_rx': '1', 'dose_unit_rx': 'gm', 'form_val_disp': '1', 'form_unit_disp': 'BAG', 'route': 'IV', 'enddate': '2103-06-07'}),\n", + " Event(event_type='prescriptions', timestamp=datetime.datetime(2103, 6, 7, 0, 0), attr_dict={'hadm_id': '100000', 'drug': 'Oxycodone-Acetaminophen', 'drug_type': 'MAIN', 'drug_name_poe': 'Oxycodone-Acetaminophen', 'drug_name_generic': 'Oxycodone-Acetaminophen', 'formulary_drug_cd': 'PERC', 'gsn': '004222', 'ndc': '00406051201', 'prod_strength': '5mg/325mg Tablet', 'dose_val_rx': '1-2', 'dose_unit_rx': 'TAB', 'form_val_disp': '1-2', 'form_unit_disp': 'TAB', 'route': 'PO', 'enddate': '2103-06-07'}),\n", + " Event(event_type='prescriptions', timestamp=datetime.datetime(2103, 6, 7, 0, 0), attr_dict={'hadm_id': '100000', 'drug': 'Pneumococcal Vac Polyvalent', 'drug_type': 'MAIN', 'drug_name_poe': 'Pneumococcal Vac Polyvalent', 'drug_name_generic': 'Pneumococcal Vac Polyvalent', 'formulary_drug_cd': 'PNEU25I', 'gsn': '048548', 'ndc': '00006494300', 'prod_strength': '25mcg/0.5mL Vial', 'dose_val_rx': '0.5', 'dose_unit_rx': 'ml', 'form_val_disp': '1', 'form_unit_disp': 'VIAL', 'route': 'IM', 'enddate': '2103-06-07'}),\n", + " Event(event_type='prescriptions', timestamp=datetime.datetime(2103, 6, 7, 0, 0), attr_dict={'hadm_id': '100000', 'drug': 'Ranitidine', 'drug_type': 'MAIN', 'drug_name_poe': 'Ranitidine', 'drug_name_generic': 'Ranitidine HCl', 'formulary_drug_cd': 'RANI150', 'gsn': '011673', 'ndc': '00904526161', 'prod_strength': '150mg Tablet', 'dose_val_rx': '150', 'dose_unit_rx': 'mg', 'form_val_disp': '1', 'form_unit_disp': 'TAB', 'route': 'PO', 'enddate': '2103-06-07'}),\n", + " Event(event_type='prescriptions', timestamp=datetime.datetime(2103, 6, 7, 0, 0), attr_dict={'hadm_id': '100000', 'drug': 'Hydromorphone', 'drug_type': 'MAIN', 'drug_name_poe': 'Hydromorphone', 'drug_name_generic': 'Hydromorphone', 'formulary_drug_cd': 'HYDR2I', 'gsn': '004103', 'ndc': '00074131230', 'prod_strength': '2mg/mL Syringe', 'dose_val_rx': '0.5-1', 'dose_unit_rx': 'mg', 'form_val_disp': '0.25-0.5', 'form_unit_disp': 'SYR', 'route': 'SC', 'enddate': '2103-06-07'}),\n", + " Event(event_type='prescriptions', timestamp=datetime.datetime(2103, 6, 7, 0, 0), attr_dict={'hadm_id': '100000', 'drug': 'NIFEdipine CR', 'drug_type': 'MAIN', 'drug_name_poe': 'NIFEdipine CR', 'drug_name_generic': 'NIFEdipine CR', 'formulary_drug_cd': 'ADAL90', 'gsn': '012061', 'ndc': '00085172202', 'prod_strength': '90 mg CR Tab', 'dose_val_rx': '90', 'dose_unit_rx': 'mg', 'form_val_disp': '1', 'form_unit_disp': 'TAB', 'route': 'PO', 'enddate': '2103-06-07'}),\n", + " Event(event_type='prescriptions', timestamp=datetime.datetime(2103, 6, 7, 0, 0), attr_dict={'hadm_id': '100000', 'drug': 'Hydralazine HCl', 'drug_type': 'MAIN', 'drug_name_poe': 'Hydralazine HCl', 'drug_name_generic': 'Hydralazine HCl', 'formulary_drug_cd': 'HYDZ20I', 'gsn': '000283', 'ndc': '00517090125', 'prod_strength': '20mg/mL Vial', 'dose_val_rx': '10', 'dose_unit_rx': 'mg', 'form_val_disp': '0.5', 'form_unit_disp': 'VIAL', 'route': 'IV', 'enddate': '2103-06-07'}),\n", + " Event(event_type='icustays', timestamp=datetime.datetime(2103, 6, 7, 0, 0), attr_dict={'icustay_id': '200000', 'first_careunit': 'SICU', 'dbsource': 'carevue', 'last_careunit': 'SICU', 'outtime': '2103-06-07'}),\n", + " Event(event_type='procedures_icd', timestamp=datetime.datetime(2103, 6, 7, 12, 0), attr_dict={'hadm_id': '100000', 'icd9_code': '8472', 'seq_num': '3'}),\n", + " Event(event_type='procedures_icd', timestamp=datetime.datetime(2103, 6, 7, 12, 0), attr_dict={'hadm_id': '100000', 'icd9_code': '544', 'seq_num': '1'}),\n", + " Event(event_type='procedures_icd', timestamp=datetime.datetime(2103, 6, 7, 12, 0), attr_dict={'hadm_id': '100000', 'icd9_code': '3895', 'seq_num': '0'}),\n", + " Event(event_type='diagnoses_icd', timestamp=datetime.datetime(2103, 6, 7, 12, 0), attr_dict={'hadm_id': '100000', 'icd9_code': '86121', 'seq_num': '0'}),\n", + " Event(event_type='diagnoses_icd', timestamp=datetime.datetime(2103, 6, 7, 12, 0), attr_dict={'hadm_id': '100000', 'icd9_code': 'V4589', 'seq_num': '1'}),\n", + " Event(event_type='diagnoses_icd', timestamp=datetime.datetime(2103, 6, 7, 12, 0), attr_dict={'hadm_id': '100000', 'icd9_code': '25082', 'seq_num': '2'}),\n", + " Event(event_type='diagnoses_icd', timestamp=datetime.datetime(2103, 6, 7, 12, 0), attr_dict={'hadm_id': '100000', 'icd9_code': '79902', 'seq_num': '3'}),\n", + " Event(event_type='procedures_icd', timestamp=datetime.datetime(2103, 6, 7, 12, 0), attr_dict={'hadm_id': '100000', 'icd9_code': '9618', 'seq_num': '2'}),\n", + " Event(event_type='diagnoses_icd', timestamp=datetime.datetime(2103, 6, 7, 12, 0), attr_dict={'hadm_id': '100000', 'icd9_code': '19889', 'seq_num': '4'}),\n", + " Event(event_type='admissions', timestamp=datetime.datetime(2103, 6, 7, 12, 0), attr_dict={'hadm_id': '100000', 'admission_type': 'EMERGENCY', 'admission_location': 'EMERGENCY ROOM ADMIT', 'insurance': 'Private', 'language': None, 'religion': 'CATHOLIC', 'marital_status': None, 'ethnicity': 'UNABLE TO OBTAIN', 'edregtime': None, 'edouttime': None, 'diagnosis': None, 'discharge_location': 'HOME HEALTH CARE', 'dischtime': '2103-06-07 12:00:00', 'hospital_expire_flag': '0'}),\n", + " Event(event_type='diagnoses_icd', timestamp=datetime.datetime(2103, 6, 7, 12, 0), attr_dict={'hadm_id': '100000', 'icd9_code': '36201', 'seq_num': '5'}),\n", + " Event(event_type='diagnoses_icd', timestamp=datetime.datetime(2103, 6, 7, 12, 0), attr_dict={'hadm_id': '100000', 'icd9_code': '27802', 'seq_num': '7'}),\n", + " Event(event_type='diagnoses_icd', timestamp=datetime.datetime(2103, 6, 7, 12, 0), attr_dict={'hadm_id': '100000', 'icd9_code': '99681', 'seq_num': '6'}),\n", + " Event(event_type='diagnoses_icd', timestamp=datetime.datetime(2103, 6, 7, 12, 0), attr_dict={'hadm_id': '100000', 'icd9_code': '30593', 'seq_num': '8'})]" + ] + }, + "metadata": {}, + "execution_count": 5 + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "### If you want to iterate across each patient's events, you can do something like below\n", + "\n" + ], + "metadata": { + "id": "HkcNx2BohaAX" + } + }, + { + "cell_type": "code", + "source": [ + "patient_ids = dataset.unique_patient_ids\n", + "for patient_id in patient_ids[:2]: # Iterate through the first 2 patient IDs\n", + " patient = dataset.get_patient(patient_id)\n", + " print(f\"Patient ID: {patient_id}\")\n", + " for i, event in enumerate(patient.get_events()):\n", + " if i < 2: # Print only the first 2 events for each patient\n", + " print(event)\n", + " else:\n", + " break # Stop iterating through events after printing 2" + ], + "metadata": { + "id": "wfvGSEnkiU0j", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "d6cae20d-5b0d-4543-f1d8-68f5abbecd9f" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Patient ID: 17900\n", + "Event(event_type='patients', timestamp=datetime.datetime(2026, 2, 20, 18, 7, 0, 461446), attr_dict={'gender': 'M', 'dob': '2125-04-11', 'dod': None, 'dod_hosp': None, 'dod_ssn': None, 'expire_flag': '0'})\n", + "Event(event_type='prescriptions', timestamp=datetime.datetime(2125, 6, 22, 0, 0), attr_dict={'hadm_id': '118575', 'drug': 'Gentamicin', 'drug_type': 'MAIN', 'drug_name_poe': None, 'drug_name_generic': None, 'formulary_drug_cd': 'GENT80PM', 'gsn': '009291', 'ndc': '00338050941', 'prod_strength': '80mg Premix', 'dose_val_rx': '80', 'dose_unit_rx': 'mg', 'form_val_disp': '1', 'form_unit_disp': 'BAG', 'route': 'IV', 'enddate': '2125-06-23'})\n", + "Patient ID: 23726\n", + "Event(event_type='patients', timestamp=datetime.datetime(2026, 2, 20, 18, 7, 0, 472646), attr_dict={'gender': 'F', 'dob': '1976-07-10', 'dod': '1976-11-10', 'dod_hosp': '1976-11-10', 'dod_ssn': '1976-11-10', 'expire_flag': '1'})\n", + "Event(event_type='icustays', timestamp=datetime.datetime(1976, 11, 10, 0, 0), attr_dict={'icustay_id': '224634', 'first_careunit': 'SICU', 'dbsource': 'carevue', 'last_careunit': 'SICU', 'outtime': '1976-11-10'})\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "### Let us do some explorations with the events\n", + "\n" + ], + "metadata": { + "id": "6OgF6H_Jrz0S" + } + }, + { + "cell_type": "code", + "source": [ + "# get patient dictionary\n", + "icd_events = dataset.get_patient('1').get_events(\"diagnoses_icd\")\n", + "print(icd_events)\n", + "for icd_event in icd_events:\n", + " print(icd_event.icd9_code)" + ], + "metadata": { + "id": "gZP5AHAfr5y4", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "673f4693-adb6-4210-ad2b-5198bb055307" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "[Event(event_type='diagnoses_icd', timestamp=datetime.datetime(2103, 6, 7, 12, 0), attr_dict={'hadm_id': '100000', 'icd9_code': '86121', 'seq_num': '0'}), Event(event_type='diagnoses_icd', timestamp=datetime.datetime(2103, 6, 7, 12, 0), attr_dict={'hadm_id': '100000', 'icd9_code': 'V4589', 'seq_num': '1'}), Event(event_type='diagnoses_icd', timestamp=datetime.datetime(2103, 6, 7, 12, 0), attr_dict={'hadm_id': '100000', 'icd9_code': '25082', 'seq_num': '2'}), Event(event_type='diagnoses_icd', timestamp=datetime.datetime(2103, 6, 7, 12, 0), attr_dict={'hadm_id': '100000', 'icd9_code': '79902', 'seq_num': '3'}), Event(event_type='diagnoses_icd', timestamp=datetime.datetime(2103, 6, 7, 12, 0), attr_dict={'hadm_id': '100000', 'icd9_code': '19889', 'seq_num': '4'}), Event(event_type='diagnoses_icd', timestamp=datetime.datetime(2103, 6, 7, 12, 0), attr_dict={'hadm_id': '100000', 'icd9_code': '36201', 'seq_num': '5'}), Event(event_type='diagnoses_icd', timestamp=datetime.datetime(2103, 6, 7, 12, 0), attr_dict={'hadm_id': '100000', 'icd9_code': '27802', 'seq_num': '7'}), Event(event_type='diagnoses_icd', timestamp=datetime.datetime(2103, 6, 7, 12, 0), attr_dict={'hadm_id': '100000', 'icd9_code': '99681', 'seq_num': '6'}), Event(event_type='diagnoses_icd', timestamp=datetime.datetime(2103, 6, 7, 12, 0), attr_dict={'hadm_id': '100000', 'icd9_code': '30593', 'seq_num': '8'})]\n", + "86121\n", + "V4589\n", + "25082\n", + "79902\n", + "19889\n", + "36201\n", + "27802\n", + "99681\n", + "30593\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "# get patient dictionary\n", + "icd_events = dataset.get_patient('1').get_events(\"procedures_icd\")\n", + "print(icd_events)\n", + "for icd_event in icd_events:\n", + " print(icd_event.icd9_code)" + ], + "metadata": { + "id": "sv7sR4qn0oLY", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "4d3c8fca-9599-4619-ecd2-d055bb1e8130" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "[Event(event_type='procedures_icd', timestamp=datetime.datetime(2103, 6, 7, 12, 0), attr_dict={'hadm_id': '100000', 'icd9_code': '8472', 'seq_num': '3'}), Event(event_type='procedures_icd', timestamp=datetime.datetime(2103, 6, 7, 12, 0), attr_dict={'hadm_id': '100000', 'icd9_code': '544', 'seq_num': '1'}), Event(event_type='procedures_icd', timestamp=datetime.datetime(2103, 6, 7, 12, 0), attr_dict={'hadm_id': '100000', 'icd9_code': '3895', 'seq_num': '0'}), Event(event_type='procedures_icd', timestamp=datetime.datetime(2103, 6, 7, 12, 0), attr_dict={'hadm_id': '100000', 'icd9_code': '9618', 'seq_num': '2'})]\n", + "8472\n", + "544\n", + "3895\n", + "9618\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "**And, if we wanted to pick an event at specific interval (i.e a hospital visit/admission), we can:**" + ], + "metadata": { + "id": "U3pGdWfdlKNr" + } + }, + { + "cell_type": "code", + "source": [ + "admissions = dataset.get_patient('1').get_events(\"admissions\")\n", + "start = admissions[0].timestamp\n", + "procedures = dataset.get_patient('1').get_events(\"procedures_icd\", start=start)\n", + "print(procedures)\n" + ], + "metadata": { + "id": "zL0y8q6clJuc", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "a8621ab1-942a-484a-dcf9-918ed4db9965" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "[Event(event_type='procedures_icd', timestamp=datetime.datetime(2103, 6, 7, 12, 0), attr_dict={'hadm_id': '100000', 'icd9_code': '8472', 'seq_num': '3'}), Event(event_type='procedures_icd', timestamp=datetime.datetime(2103, 6, 7, 12, 0), attr_dict={'hadm_id': '100000', 'icd9_code': '544', 'seq_num': '1'}), Event(event_type='procedures_icd', timestamp=datetime.datetime(2103, 6, 7, 12, 0), attr_dict={'hadm_id': '100000', 'icd9_code': '3895', 'seq_num': '0'}), Event(event_type='procedures_icd', timestamp=datetime.datetime(2103, 6, 7, 12, 0), attr_dict={'hadm_id': '100000', 'icd9_code': '9618', 'seq_num': '2'})]\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "## **Going Deeper: What event types and attributes are available?**\n", + "\n", + "Below, we showcase an example of our config YAML files that pre-define what is pre-loaded for use in MIMIC3. For a full look of the config file as well as where this is located in PyHealth, see [here](https://github.com/sunlabuiuc/PyHealth/blob/master/pyhealth/datasets/configs/mimic3.yaml). For brevity's sake, we showcase the tables we used above below:\n", + "\n" + ], + "metadata": { + "id": "pv9VUbg8zeV0" + } + }, + { + "cell_type": "markdown", + "source": [ + "\n", + "\n", + "```\n", + "version: \"1.4\"\n", + "tables:\n", + " patients:\n", + " file_path: \"PATIENTS.csv.gz\"\n", + " patient_id: \"subject_id\"\n", + " timestamp: null\n", + " attributes:\n", + " - \"gender\"\n", + " - \"dob\"\n", + " - \"dod\"\n", + " - \"dod_hosp\"\n", + " - \"dod_ssn\"\n", + " - \"expire_flag\"\n", + "\n", + " admissions:\n", + " file_path: \"ADMISSIONS.csv.gz\"\n", + " patient_id: \"subject_id\"\n", + " timestamp: \"admittime\"\n", + " attributes:\n", + " - \"hadm_id\"\n", + " - \"admission_type\"\n", + " - \"admission_location\"\n", + " - \"insurance\"\n", + " - \"language\"\n", + " - \"religion\"\n", + " - \"marital_status\"\n", + " - \"ethnicity\"\n", + " - \"edregtime\"\n", + " - \"edouttime\"\n", + " - \"diagnosis\"\n", + " - \"discharge_location\"\n", + " - \"dischtime\"\n", + " - \"hospital_expire_flag\"\n", + "\n", + " diagnoses_icd:\n", + " file_path: \"DIAGNOSES_ICD.csv.gz\"\n", + " patient_id: \"subject_id\"\n", + " join:\n", + " - file_path: \"ADMISSIONS.csv.gz\"\n", + " \"on\": \"hadm_id\"\n", + " how: \"inner\"\n", + " columns:\n", + " - \"dischtime\"\n", + " timestamp: \"dischtime\"\n", + " attributes:\n", + " - \"icd9_code\"\n", + " - \"seq_num\"\n", + "\n", + " procedures_icd:\n", + " file_path: \"PROCEDURES_ICD.csv.gz\"\n", + " patient_id: \"subject_id\"\n", + " join:\n", + " - file_path: \"ADMISSIONS.csv.gz\"\n", + " \"on\": \"hadm_id\"\n", + " how: \"inner\"\n", + " columns:\n", + " - \"dischtime\"\n", + " timestamp: \"dischtime\"\n", + " attributes:\n", + " - \"icd9_code\"\n", + " - \"seq_num\"\n", + "\n", + "```\n", + "\n" + ], + "metadata": { + "id": "1oWUKr_HqKkY" + } + }, + { + "cell_type": "markdown", + "source": [ + "One key thing you'll notice is the name of the **tables** that are available in the loading of our dataset. In this case, **\"diagnoses_icd\"** and **\"procedures_icd\"** are defined with the following required definitions in the YAML file above:\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "* **patient_id**, uses to define which column \"subject_id\" that represents the identifier for the patient_ids. Effectively, this defines what a \"sample\" is in our dataset. Left null, it indicates that every row in each \"table\" (file.csv, etc.) is a sample in our dataframe.\n", + "\n", + "* **file_path**, defines what file in the **root* directory that should be read by PyHealth's polar backend to load the data from.\n", + "\n", + "* **timestamp**, defines what table column to look at for time parsing\n", + "\n", + "* **attributes**, defines what columns to read for each row to store in our Event data structure. In this case, for each table, we can see we have **icd9_code** that we explored above.\n", + "\n", + "* **join**, we can also optionally define joins if we want to merge tables into the main **ADMISSIONS** tables based on their hospital admissions id **hadm_id** for task-specific purposes (i.e mortality prediction/readmission prediction/etc. We discuss more of this in a later tutorial)." + ], + "metadata": { + "id": "NSrb2PGFqUgS" + } + }, + { + "cell_type": "markdown", + "source": [ + "## **Contributing your own dataset**\n", + "We go over how to define your own datasets below. There are essentially 2 steps:\n", + "\n", + "\n", + "1. Defining your dataset_config.yaml file.\n", + "2. Defining your dataset_interface.py\n", + "\n", + "We go over two implementation examples to showcase how to implement **2 types of datasets.**\n", + "\n", + "1. Datasets that are already in a table (i.e MIMIC-III)\n", + "2. Datasets that are in a file directory format (i.e Covid19CXR)\n" + ], + "metadata": { + "id": "YIWZqJrHxFQw" + } + }, + { + "cell_type": "markdown", + "source": [ + "### **Example 1 - MIMIC3**\n", + "\n", + "#### **1. Defining mimic3.yaml**\n", + "\n", + "Every configuration file is defined in **pyhealth/datasets/configs** [here](https://github.com/sunlabuiuc/PyHealth/tree/master/pyhealth/datasets/configs).\n", + "\n", + "Simply create a .yaml file, and define the **tables** that are available like the following:\n", + "\n", + "\n", + "\n", + "```\n", + "version: \"1.4\"\n", + "tables:\n", + " patients:\n", + " file_path: \"PATIENTS.csv.gz\"\n", + " patient_id: \"subject_id\"\n", + " timestamp: null\n", + " attributes:\n", + " - \"gender\"\n", + " - \"dob\"\n", + " - \"dod\"\n", + " - \"dod_hosp\"\n", + " - \"dod_ssn\"\n", + " - \"expire_flag\"\n", + "```\n", + "The PyHealth BaseDataset class will automatically read these tables in with these defined attributes.\n", + "\n" + ], + "metadata": { + "id": "j-CDPfq_6H1m" + } + }, + { + "cell_type": "markdown", + "source": [ + "#### **2. Defining MIMIC3Dataset**\n", + "Once your config.yaml file is defined, we will need to define a class that inherits **BaseDataset**, which does much of the processing underneath the hood. For the ugly details, see [here](https://github.com/sunlabuiuc/PyHealth/blob/master/pyhealth/datasets/base_dataset.py).\n", + "\n" + ], + "metadata": { + "id": "aNvjwQu48ftW" + } + }, + { + "cell_type": "code", + "source": [ + "import logging\n", + "import warnings\n", + "from pathlib import Path\n", + "from typing import List, Optional\n", + "\n", + "import polars as pl\n", + "\n", + "from pyhealth.datasets.base_dataset import BaseDataset\n", + "\n", + "logger = logging.getLogger(__name__)\n", + "\n", + "\n", + "class MIMIC3Dataset(BaseDataset):\n", + " \"\"\"\n", + " A dataset class for handling MIMIC-III data.\n", + "\n", + " This class is responsible for loading and managing the MIMIC-III dataset,\n", + " which includes tables such as patients, admissions, and icustays.\n", + "\n", + " Attributes:\n", + " root (str): The root directory where the dataset is stored.\n", + " tables (List[str]): A list of tables to be included in the dataset.\n", + " dataset_name (Optional[str]): The name of the dataset.\n", + " config_path (Optional[str]): The path to the configuration file.\n", + " \"\"\"\n", + "\n", + " def __init__(\n", + " self,\n", + " root: str,\n", + " tables: List[str],\n", + " dataset_name: Optional[str] = None,\n", + " config_path: Optional[str] = None,\n", + " **kwargs\n", + " ) -> None:\n", + " \"\"\"\n", + " Initializes the MIMIC4Dataset with the specified parameters.\n", + "\n", + " Args:\n", + " root (str): The root directory where the dataset is stored.\n", + " tables (List[str]): A list of additional tables to include.\n", + " dataset_name (Optional[str]): The name of the dataset. Defaults to \"mimic3\".\n", + " config_path (Optional[str]): The path to the configuration file. If not provided, a default config is used.\n", + " \"\"\"\n", + " if config_path is None:\n", + " logger.info(\"No config path provided, using default config\")\n", + " config_path = Path(__file__).parent / \"configs\" / \"mimic3.yaml\"\n", + " default_tables = [\"patients\", \"admissions\", \"icustays\"]\n", + " tables = default_tables + tables\n", + " if \"prescriptions\" in tables:\n", + " warnings.warn(\n", + " \"Events from prescriptions table only have date timestamp (no specific time). \"\n", + " \"This may affect temporal ordering of events.\",\n", + " UserWarning,\n", + " )\n", + " super().__init__(\n", + " root=root,\n", + " tables=tables,\n", + " dataset_name=dataset_name or \"mimic3\",\n", + " config_path=config_path,\n", + " **kwargs\n", + " )\n", + " return\n", + "\n", + " def preprocess_noteevents(self, df: pl.LazyFrame) -> pl.LazyFrame:\n", + " \"\"\"\n", + " Table-specific preprocess function which will be called by BaseDataset.load_table().\n", + "\n", + " Preprocesses the noteevents table by ensuring that the charttime column\n", + " is populated. If charttime is null, it uses chartdate with a default\n", + " time of 00:00:00.\n", + "\n", + " See: https://mimic.mit.edu/docs/iii/tables/noteevents/#chartdate-charttime-storetime.\n", + "\n", + " Args:\n", + " df (pl.LazyFrame): The input dataframe containing noteevents data.\n", + "\n", + " Returns:\n", + " pl.LazyFrame: The processed dataframe with updated charttime\n", + " values.\n", + " \"\"\"\n", + " df = df.with_columns(\n", + " pl.when(pl.col(\"charttime\").is_null())\n", + " .then(pl.col(\"chartdate\") + pl.lit(\" 00:00:00\"))\n", + " .otherwise(pl.col(\"charttime\"))\n", + " .alias(\"charttime\")\n", + " )\n", + " return df" + ], + "metadata": { + "id": "UZbKvKHF8PxW" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "#### **Code Breakdown**\n", + "It looks scary at first, but let's break it down, we can first specify a default config path that **BaseDataset** will use to load tables:\n", + "\n", + "```\n", + "if config_path is None:\n", + " logger.info(\"No config path provided, using default config\")\n", + " config_path = Path(__file__).parent / \"configs\" / \"mimic3.yaml\"\n", + "```\n", + "Next, we define some defualt tables that are always parsed for the dataset, regardless of the user's arguments. **This is key for datasets where there is one metadata table that unifies all other tables like it is in MIMIC-III.** We also offer some warnings about dataset specifics for those who have never used the prescriptions table before.\n", + "\n", + "\n", + "```\n", + "default_tables = [\"patients\", \"admissions\", \"icustays\"]\n", + " tables = default_tables + tables\n", + " if \"prescriptions\" in tables:\n", + " warnings.warn(\n", + " \"Events from prescriptions table only have date timestamp (no specific time). \"\n", + " \"This may affect temporal ordering of events.\",\n", + " UserWarning,\n", + " )\n", + "```\n", + "\n", + "Finally, **we make sure to run the optimized BaseDataset:** that will do the heavy lifting behind the scenes.\n", + "\n", + "```\n", + "super().__init__(\n", + " root=root,\n", + " tables=tables,\n", + " dataset_name=dataset_name or \"mimic3\",\n", + " config_path=config_path,\n", + " **kwargs\n", + " )\n", + " return\n", + "```\n", + "\n", + "\n", + "**You'll also see we offer overrides for how tables are processed in BaseDataset:**\n", + "\n", + "```\n", + "def preprocess_noteevents(self, df: pl.LazyFrame) -> pl.LazyFrame:\n", + " \"\"\"\n", + " Table-specific preprocess function which will be called by BaseDataset.load_table().\n", + " \n", + " Preprocesses the noteevents table by ensuring that the charttime column\n", + " is populated. If charttime is null, it uses chartdate with a default\n", + " time of 00:00:00.\n", + "\n", + " See: https://mimic.mit.edu/docs/iii/tables/noteevents/#chartdate-charttime-storetime.\n", + "\n", + " Args:\n", + " df (pl.LazyFrame): The input dataframe containing noteevents data.\n", + "\n", + " Returns:\n", + " pl.LazyFrame: The processed dataframe with updated charttime\n", + " values.\n", + " \"\"\"\n", + " df = df.with_columns(\n", + " pl.when(pl.col(\"charttime\").is_null())\n", + " .then(pl.col(\"chartdate\") + pl.lit(\" 00:00:00\"))\n", + " .otherwise(pl.col(\"charttime\"))\n", + " .alias(\"charttime\")\n", + " )\n", + " return df\n", + "```\n", + "\n", + "BaseDataset will override how it reads any **table** if you define your own **preprocess_{table}** functions within the class. Here, it adjusts for how dates are represented in the noteevents table, which is different than the usual discharge date columns within the other tables.\n", + "\n", + "\n", + "\n" + ], + "metadata": { + "id": "-Dmt5EFP8TxC" + } + }, + { + "cell_type": "markdown", + "source": [ + "### **Example 2. COVID-CXR Dataset**\n", + "Here, the data is not formatted as a table, so you might be confused on how to contribute a dataset.\n", + "\n", + "**If none of the data is defined in a tabular format, we can simply construct a table such that it can leverage all of the tricks implemented by BaseDataset.**\n", + "\n", + "Let's first start by defining our dataset implementation, located [here](https://github.com/sunlabuiuc/PyHealth/blob/master/pyhealth/datasets/covid19_cxr.py).\n", + "\n", + "#### **1. Defining COVID19CXRDataset**" + ], + "metadata": { + "id": "QQHtI7tcANj4" + } + }, + { + "cell_type": "code", + "source": [ + "import logging\n", + "import os\n", + "from pathlib import Path\n", + "from typing import Optional\n", + "\n", + "import pandas as pd\n", + "\n", + "from pyhealth.tasks import COVID19CXRClassification\n", + "from pyhealth.datasets.base_dataset import BaseDataset\n", + "\n", + "logger = logging.getLogger(__name__)\n", + "\n", + "\n", + "class COVID19CXRDataset(BaseDataset):\n", + " \"\"\"Base image dataset for COVID-19 Radiography Database.\n", + "\n", + " Dataset is available at:\n", + " https://www.kaggle.com/datasets/tawsifurrahman/covid19-radiography-database\n", + "\n", + " Data Sources:\n", + " ------------\n", + " COVID-19 data:\n", + " - 2473 CXR images from padchest dataset[1]\n", + " - 183 CXR images from a Germany medical school[2]\n", + " - 559 CXR images from SIRM, Github, Kaggle & Tweeter[3,4,5,6]\n", + " - 400 CXR images from another Github source[7]\n", + "\n", + " Normal images:\n", + " - 8851 from RSNA [8]\n", + " - 1341 from Kaggle [9]\n", + "\n", + " Lung opacity images:\n", + " - 6012 from Radiological Society of North America (RSNA) CXR dataset[8]\n", + "\n", + " Viral Pneumonia images:\n", + " - 1345 from the Chest X-Ray Images (pneumonia) database[9]\n", + "\n", + " Citations:\n", + " ---------\n", + " If you use this dataset, please cite:\n", + " 1. M.E.H. Chowdhury, T. Rahman, A. Khandakar, et al. \"Can AI help in\n", + " screening Viral and COVID-19 pneumonia?\" IEEE Access, Vol. 8, 2020,\n", + " pp. 132665-132676.\n", + " 2. Rahman, T., Khandakar, A., Qiblawey, Y., et al. \"Exploring the Effect\n", + " of Image Enhancement Techniques on COVID-19 Detection using Chest X-ray\n", + " Images.\" arXiv preprint arXiv:2012.02238.\n", + "\n", + " References:\n", + " ----------\n", + " [1] https://bimcv.cipf.es/bimcv-projects/bimcv-covid19/\n", + " [2] https://github.com/ml-workgroup/covid-19-image-repository/tree/master/png\n", + " [3] https://sirm.org/category/senza-categoria/covid-19/\n", + " [4] https://eurorad.org\n", + " [5] https://github.com/ieee8023/covid-chestxray-dataset\n", + " [6] https://figshare.com/articles/COVID-19_Chest_X-Ray_Image_Repository/12580328\n", + " [7] https://github.com/armiro/COVID-CXNet\n", + " [8] https://www.kaggle.com/c/rsna-pneumonia-detection-challenge/data\n", + " [9] https://www.kaggle.com/paultimothymooney/chest-xray-pneumonia\n", + "\n", + " Args:\n", + " root: Root directory of the raw data containing the dataset files.\n", + " dataset_name: Optional name of the dataset. Defaults to \"covid19_cxr\".\n", + " config_path: Optional path to the configuration file. If not provided,\n", + " uses the default config in the configs directory.\n", + "\n", + " Attributes:\n", + " root: Root directory of the raw data.\n", + " dataset_name: Name of the dataset.\n", + " config_path: Path to the configuration file.\n", + "\n", + " Examples:\n", + " >>> from pyhealth.datasets import COVID19CXRDataset\n", + " >>> dataset = COVID19CXRDataset(\n", + " ... root=\"/path/to/covid19_cxr\"\n", + " ... )\n", + " >>> dataset.stats()\n", + " >>> samples = dataset.set_task()\n", + " >>> print(samples[0])\n", + " \"\"\"\n", + "\n", + " def __init__(\n", + " self,\n", + " root: str,\n", + " dataset_name: Optional[str] = None,\n", + " config_path: Optional[str] = None,\n", + " ) -> None:\n", + " if config_path is None:\n", + " logger.info(\"No config path provided, using default config\")\n", + " config_path = (\n", + " Path(__file__).parent / \"configs\" / \"covid19_cxr.yaml\"\n", + " )\n", + " if not os.path.exists(os.path.join(root, \"covid19_cxr-metadata-pyhealth.csv\")):\n", + " self.prepare_metadata(root)\n", + " default_tables = [\"covid19_cxr\"]\n", + " super().__init__(\n", + " root=root,\n", + " tables=default_tables,\n", + " dataset_name=dataset_name or \"covid19_cxr\",\n", + " config_path=config_path,\n", + " )\n", + " return\n", + "\n", + " def prepare_metadata(self, root: str) -> None:\n", + " \"\"\"Prepare metadata for the COVID-19 CXR dataset.\n", + "\n", + " Args:\n", + " root: Root directory containing the dataset files.\n", + "\n", + " This method:\n", + " 1. Reads metadata from Excel files for each class\n", + " 2. Processes file paths and labels\n", + " 3. Combines all data into a single DataFrame\n", + " 4. Saves the processed metadata to a CSV file\n", + " \"\"\"\n", + " # process and merge raw xlsx files from the dataset\n", + " covid = pd.DataFrame(\n", + " pd.read_excel(f\"{root}/COVID.metadata.xlsx\")\n", + " )\n", + " covid[\"FILE NAME\"] = covid[\"FILE NAME\"].apply(\n", + " lambda x: f\"{root}/COVID/images/{x}.png\"\n", + " )\n", + " covid[\"label\"] = \"COVID\"\n", + " lung_opacity = pd.DataFrame(\n", + " pd.read_excel(f\"{root}/Lung_Opacity.metadata.xlsx\")\n", + " )\n", + " lung_opacity[\"FILE NAME\"] = lung_opacity[\"FILE NAME\"].apply(\n", + " lambda x: f\"{root}/Lung_Opacity/images/{x}.png\"\n", + " )\n", + " lung_opacity[\"label\"] = \"Lung Opacity\"\n", + " normal = pd.DataFrame(\n", + " pd.read_excel(f\"{root}/Normal.metadata.xlsx\")\n", + " )\n", + " normal[\"FILE NAME\"] = normal[\"FILE NAME\"].apply(\n", + " lambda x: x.capitalize()\n", + " )\n", + " normal[\"FILE NAME\"] = normal[\"FILE NAME\"].apply(\n", + " lambda x: f\"{root}/Normal/images/{x}.png\"\n", + " )\n", + " normal[\"label\"] = \"Normal\"\n", + " viral_pneumonia = pd.DataFrame(\n", + " pd.read_excel(f\"{root}/Viral Pneumonia.metadata.xlsx\")\n", + " )\n", + " viral_pneumonia[\"FILE NAME\"] = viral_pneumonia[\"FILE NAME\"].apply(\n", + " lambda x: f\"{root}/Viral Pneumonia/images/{x}.png\"\n", + " )\n", + " viral_pneumonia[\"label\"] = \"Viral Pneumonia\"\n", + " df = pd.concat(\n", + " [covid, lung_opacity, normal, viral_pneumonia],\n", + " axis=0,\n", + " ignore_index=True\n", + " )\n", + " df = df.drop(columns=[\"FORMAT\", \"SIZE\"])\n", + " df.columns = [\"path\", \"url\", \"label\"]\n", + " for path in df.path:\n", + " assert os.path.isfile(path), f\"File {path} does not exist\"\n", + " df.to_csv(\n", + " os.path.join(root, \"covid19_cxr-metadata-pyhealth.csv\"),\n", + " index=False\n", + " )\n", + " return\n", + "\n", + " @property\n", + " def default_task(self) -> COVID19CXRClassification:\n", + " \"\"\"Returns the default task for this dataset.\n", + "\n", + " Returns:\n", + " COVID19CXRClassification: The default classification task.\n", + " \"\"\"\n", + " return COVID19CXRClassification()" + ], + "metadata": { + "id": "xGpZNCqhAVgq" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "#### Code Breakdown\n", + "Taking a closer look, the init() function is practically the same as our MIMIC3 implementation. But, the one difference is we have a **prepare_metadata()** function that constructs a metadata to be used for our BaseDataset. Looking closer at this function, we can see that:\n", + "\n", + "1. **Reading Excel metadata files**: The function reads four separate Excel files containing metadata for each disease category:\n", + " ```python\n", + " covid = pd.DataFrame(pd.read_excel(f\"{root}/COVID.metadata.xlsx\"))\n", + " lung_opacity = pd.DataFrame(pd.read_excel(f\"{root}/Lung_Opacity.metadata.xlsx\"))\n", + " normal = pd.DataFrame(pd.read_excel(f\"{root}/Normal.metadata.xlsx\"))\n", + " viral_pneumonia = pd.DataFrame(pd.read_excel(f\"{root}/Viral Pneumonia.metadata.xlsx\"))\n", + " ```\n", + "\n", + "2. **Constructing file paths**: For each category, it builds the complete file paths by taking the filename from the Excel metadata and prepending the appropriate directory path:\n", + " ```python\n", + " covid[\"FILE NAME\"] = covid[\"FILE NAME\"].apply(\n", + " lambda x: f\"{root}/COVID/images/{x}.png\"\n", + " )\n", + " lung_opacity[\"FILE NAME\"] = lung_opacity[\"FILE NAME\"].apply(\n", + " lambda x: f\"{root}/Lung_Opacity/images/{x}.png\"\n", + " )\n", + " # Special handling for Normal images with capitalization\n", + " normal[\"FILE NAME\"] = normal[\"FILE NAME\"].apply(lambda x: x.capitalize())\n", + " normal[\"FILE NAME\"] = normal[\"FILE NAME\"].apply(\n", + " lambda x: f\"{root}/Normal/images/{x}.png\"\n", + " )\n", + " ```\n", + "\n", + "3. **Adding labels**: Each DataFrame gets a \"label\" column with the corresponding disease category:\n", + " ```python\n", + " covid[\"label\"] = \"COVID\"\n", + " lung_opacity[\"label\"] = \"Lung Opacity\"\n", + " normal[\"label\"] = \"Normal\"\n", + " viral_pneumonia[\"label\"] = \"Viral Pneumonia\"\n", + " ```\n", + "\n", + "4. **Combining and cleaning**: All four DataFrames are concatenated into a single DataFrame, and unnecessary columns are dropped:\n", + " ```python\n", + " df = pd.concat([covid, lung_opacity, normal, viral_pneumonia], axis=0, ignore_index=True)\n", + " df = df.drop(columns=[\"FORMAT\", \"SIZE\"])\n", + " ```\n", + "\n", + "5. **Renaming columns**: The final DataFrame columns are standardized to match the expected schema (below):\n", + " ```python\n", + " df.columns = [\"path\", \"url\", \"label\"]\n", + " ```\n", + "\n", + "6. **Validation**: The function verifies that all image files actually exist on disk:\n", + " ```python\n", + " for path in df.path:\n", + " assert os.path.isfile(path), f\"File {path} does not exist\"\n", + " ```\n", + "\n", + "7. **Saving processed metadata**: The combined and cleaned metadata is saved as a CSV file:\n", + " ```python\n", + " df.to_csv(os.path.join(root, \"covid19_cxr-metadata-pyhealth.csv\"), index=False)\n", + " ```\n", + "\n", + "This preprocessing step transforms the original dataset's scattered Excel metadata files into a single, standardized CSV file that the BaseDataset can easily consume according to the YAML configuration." + ], + "metadata": { + "id": "NG5Prgf3B8Yz" + } + }, + { + "cell_type": "markdown", + "source": [ + "#### **2. Defining covid19_cxr.yaml**\n", + "Once you essentially define your own metadata table, we can define a config.yaml file that takes all of these into account such that we can easily access all of the images in our Events() format.\n", + "\n", + "\n", + "\n", + "```\n", + "version: \"5.0\"\n", + "tables:\n", + " covid19_cxr:\n", + " file_path: \"covid19_cxr-metadata-pyhealth.csv\"\n", + " patient_id: null\n", + " timestamp: null\n", + " attributes:\n", + " - \"path\"\n", + " - \"url\"\n", + " - \"label\"\n", + " ```\n", + "\n", + "You'll see the typical **tables** and other necessary definitions. Taking a closer look at the **attributes:**, we can see 3 key columns that we've created from earlier **path**, **url**, **label** that characterize each X-ray.\n", + "\n", + "These attributes are now directly explorable by any user using the Event.path or Event.url notation from earlier." + ], + "metadata": { + "id": "hmcEoCXnBqIL" + } + }, + { + "cell_type": "markdown", + "source": [ + "If you find it useful, please give us a star ⭐ (fork, and watch) at https://github.com/sunlabuiuc/PyHealth.\n", + "\n", + "Thanks very much for your support!" + ], + "metadata": { + "id": "89v82LIUbTE_" + } + } + ], + "metadata": { + "colab": { + "provenance": [], + "machine_shape": "hm", + "gpuType": "A100" + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "accelerator": "GPU" + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file diff --git a/examples/tutorials/orig_tutorial_pyhealth_tasks.ipynb b/examples/tutorials/orig_tutorial_pyhealth_tasks.ipynb new file mode 100644 index 000000000..38a044829 --- /dev/null +++ b/examples/tutorials/orig_tutorial_pyhealth_tasks.ipynb @@ -0,0 +1,3559 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "Wr6CSIDPiYNi" + }, + "source": [ + "# **Table of Contents**\n", + "In this tutorial, we will go over the following:\n", + "\n", + "1. **Writing your own [pyhealth.task](https://pyhealth.readthedocs.io/en/latest/api/tasks.html)** with two examples\n", + "2. **Using it with the rest of the pipeline.**\n", + "\n", + "As a reminder, **pyhealth.tasks** directly builds off of **pyhealth.datasets**.\n", + "\n", + "![Image description](https://drive.google.com/uc?export=view&id=1hHJcavXqisH9JEMqEVtE4TqEg_E489l5)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Dbz1YJ2nqOSN", + "outputId": "9ab52cfc-df1a-4ca9-f5ea-e007da49b681" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Requirement already satisfied: pyhealth in /usr/local/lib/python3.12/dist-packages (2.0.0)\n", + "Requirement already satisfied: accelerate in /usr/local/lib/python3.12/dist-packages (from pyhealth) (1.13.0)\n", + "Requirement already satisfied: dask~=2025.11.0 in /usr/local/lib/python3.12/dist-packages (from dask[complete]~=2025.11.0->pyhealth) (2025.11.0)\n", + "Requirement already satisfied: einops>=0.8.0 in /usr/local/lib/python3.12/dist-packages (from pyhealth) (0.8.2)\n", + "Requirement already satisfied: linear-attention-transformer>=0.19.1 in /usr/local/lib/python3.12/dist-packages (from pyhealth) (0.19.1)\n", + "Requirement already satisfied: litdata~=0.2.59 in /usr/local/lib/python3.12/dist-packages (from pyhealth) (0.2.61)\n", + "Requirement already satisfied: mne~=1.10.0 in /usr/local/lib/python3.12/dist-packages (from pyhealth) (1.10.2)\n", + "Requirement already satisfied: more-itertools~=10.8.0 in /usr/local/lib/python3.12/dist-packages (from pyhealth) (10.8.0)\n", + "Requirement already satisfied: narwhals~=2.13.0 in /usr/local/lib/python3.12/dist-packages (from pyhealth) (2.13.0)\n", + "Requirement already satisfied: networkx in /usr/local/lib/python3.12/dist-packages (from pyhealth) (3.6.1)\n", + "Requirement already satisfied: numpy~=2.2.0 in /usr/local/lib/python3.12/dist-packages (from pyhealth) (2.2.6)\n", + "Requirement already satisfied: ogb>=1.3.5 in /usr/local/lib/python3.12/dist-packages (from pyhealth) (1.3.6)\n", + "Requirement already satisfied: pandas~=2.3.1 in /usr/local/lib/python3.12/dist-packages (from pyhealth) (2.3.3)\n", + "Requirement already satisfied: peft in /usr/local/lib/python3.12/dist-packages (from pyhealth) (0.18.1)\n", + "Requirement already satisfied: polars~=1.35.2 in /usr/local/lib/python3.12/dist-packages (from pyhealth) (1.35.2)\n", + "Requirement already satisfied: pyarrow~=22.0.0 in /usr/local/lib/python3.12/dist-packages (from pyhealth) (22.0.0)\n", + "Requirement already satisfied: pydantic~=2.11.7 in /usr/local/lib/python3.12/dist-packages (from pyhealth) (2.11.10)\n", + "Requirement already satisfied: rdkit in /usr/local/lib/python3.12/dist-packages (from pyhealth) (2025.9.6)\n", + "Requirement already satisfied: scikit-learn~=1.7.0 in /usr/local/lib/python3.12/dist-packages (from pyhealth) (1.7.2)\n", + "Requirement already satisfied: torchvision in /usr/local/lib/python3.12/dist-packages (from pyhealth) (0.22.1)\n", + "Requirement already satisfied: torch~=2.7.1 in /usr/local/lib/python3.12/dist-packages (from pyhealth) (2.7.1)\n", + "Requirement already satisfied: tqdm in /usr/local/lib/python3.12/dist-packages (from pyhealth) (4.67.3)\n", + "Requirement already satisfied: transformers~=4.53.2 in /usr/local/lib/python3.12/dist-packages (from pyhealth) (4.53.3)\n", + "Requirement already satisfied: urllib3~=2.5.0 in /usr/local/lib/python3.12/dist-packages (from pyhealth) (2.5.0)\n", + "Requirement already satisfied: click>=8.1 in /usr/local/lib/python3.12/dist-packages (from dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth) (8.3.1)\n", + "Requirement already satisfied: cloudpickle>=3.0.0 in /usr/local/lib/python3.12/dist-packages (from dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth) (3.1.2)\n", + "Requirement already satisfied: fsspec>=2021.09.0 in /usr/local/lib/python3.12/dist-packages (from dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth) (2025.3.0)\n", + "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.12/dist-packages (from dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth) (26.0)\n", + "Requirement already satisfied: partd>=1.4.0 in /usr/local/lib/python3.12/dist-packages (from dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth) (1.4.2)\n", + "Requirement already satisfied: pyyaml>=5.3.1 in /usr/local/lib/python3.12/dist-packages (from dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth) (6.0.3)\n", + "Requirement already satisfied: toolz>=0.10.0 in /usr/local/lib/python3.12/dist-packages (from dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth) (0.12.1)\n", + "Requirement already satisfied: lz4>=4.3.2 in /usr/local/lib/python3.12/dist-packages (from dask[complete]~=2025.11.0->pyhealth) (4.4.5)\n", + "Requirement already satisfied: axial-positional-embedding in /usr/local/lib/python3.12/dist-packages (from linear-attention-transformer>=0.19.1->pyhealth) (0.3.12)\n", + "Requirement already satisfied: linformer>=0.1.0 in /usr/local/lib/python3.12/dist-packages (from linear-attention-transformer>=0.19.1->pyhealth) (0.2.3)\n", + "Requirement already satisfied: local-attention in /usr/local/lib/python3.12/dist-packages (from linear-attention-transformer>=0.19.1->pyhealth) (1.11.2)\n", + "Requirement already satisfied: product-key-memory>=0.1.5 in /usr/local/lib/python3.12/dist-packages (from linear-attention-transformer>=0.19.1->pyhealth) (0.3.0)\n", + "Requirement already satisfied: lightning-utilities in /usr/local/lib/python3.12/dist-packages (from litdata~=0.2.59->pyhealth) (0.15.3)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.12/dist-packages (from litdata~=0.2.59->pyhealth) (3.25.2)\n", + "Requirement already satisfied: boto3 in /usr/local/lib/python3.12/dist-packages (from litdata~=0.2.59->pyhealth) (1.42.72)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.12/dist-packages (from litdata~=0.2.59->pyhealth) (2.32.4)\n", + "Requirement already satisfied: tifffile in /usr/local/lib/python3.12/dist-packages (from litdata~=0.2.59->pyhealth) (2026.3.3)\n", + "Requirement already satisfied: obstore in /usr/local/lib/python3.12/dist-packages (from litdata~=0.2.59->pyhealth) (0.9.2)\n", + "Requirement already satisfied: decorator in /usr/local/lib/python3.12/dist-packages (from mne~=1.10.0->pyhealth) (4.4.2)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.12/dist-packages (from mne~=1.10.0->pyhealth) (3.1.6)\n", + "Requirement already satisfied: lazy-loader>=0.3 in /usr/local/lib/python3.12/dist-packages (from mne~=1.10.0->pyhealth) (0.5)\n", + "Requirement already satisfied: matplotlib>=3.7 in /usr/local/lib/python3.12/dist-packages (from mne~=1.10.0->pyhealth) (3.10.0)\n", + "Requirement already satisfied: pooch>=1.5 in /usr/local/lib/python3.12/dist-packages (from mne~=1.10.0->pyhealth) (1.9.0)\n", + "Requirement already satisfied: scipy>=1.11 in /usr/local/lib/python3.12/dist-packages (from mne~=1.10.0->pyhealth) (1.16.3)\n", + "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.12/dist-packages (from ogb>=1.3.5->pyhealth) (1.17.0)\n", + "Requirement already satisfied: outdated>=0.2.0 in /usr/local/lib/python3.12/dist-packages (from ogb>=1.3.5->pyhealth) (0.2.2)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.12/dist-packages (from pandas~=2.3.1->pyhealth) (2.9.0.post0)\n", + "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.12/dist-packages (from pandas~=2.3.1->pyhealth) (2025.2)\n", + "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.12/dist-packages (from pandas~=2.3.1->pyhealth) (2025.3)\n", + "Requirement already satisfied: polars-runtime-32==1.35.2 in /usr/local/lib/python3.12/dist-packages (from polars~=1.35.2->pyhealth) (1.35.2)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.12/dist-packages (from pydantic~=2.11.7->pyhealth) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.33.2 in /usr/local/lib/python3.12/dist-packages (from pydantic~=2.11.7->pyhealth) (2.33.2)\n", + "Requirement already satisfied: typing-extensions>=4.12.2 in /usr/local/lib/python3.12/dist-packages (from pydantic~=2.11.7->pyhealth) (4.15.0)\n", + "Requirement already satisfied: typing-inspection>=0.4.0 in /usr/local/lib/python3.12/dist-packages (from pydantic~=2.11.7->pyhealth) (0.4.2)\n", + "Requirement already satisfied: joblib>=1.2.0 in /usr/local/lib/python3.12/dist-packages (from scikit-learn~=1.7.0->pyhealth) (1.5.3)\n", + "Requirement already satisfied: threadpoolctl>=3.1.0 in /usr/local/lib/python3.12/dist-packages (from scikit-learn~=1.7.0->pyhealth) (3.6.0)\n", + "Requirement already satisfied: setuptools in /usr/local/lib/python3.12/dist-packages (from torch~=2.7.1->pyhealth) (75.2.0)\n", + "Requirement already satisfied: sympy>=1.13.3 in /usr/local/lib/python3.12/dist-packages (from torch~=2.7.1->pyhealth) (1.14.0)\n", + "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.6.77 in /usr/local/lib/python3.12/dist-packages (from torch~=2.7.1->pyhealth) (12.6.77)\n", + "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.6.77 in /usr/local/lib/python3.12/dist-packages (from torch~=2.7.1->pyhealth) (12.6.77)\n", + "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.6.80 in /usr/local/lib/python3.12/dist-packages (from torch~=2.7.1->pyhealth) (12.6.80)\n", + "Requirement already satisfied: nvidia-cudnn-cu12==9.5.1.17 in /usr/local/lib/python3.12/dist-packages (from torch~=2.7.1->pyhealth) (9.5.1.17)\n", + "Requirement already satisfied: nvidia-cublas-cu12==12.6.4.1 in /usr/local/lib/python3.12/dist-packages (from torch~=2.7.1->pyhealth) (12.6.4.1)\n", + "Requirement already satisfied: nvidia-cufft-cu12==11.3.0.4 in /usr/local/lib/python3.12/dist-packages (from torch~=2.7.1->pyhealth) (11.3.0.4)\n", + "Requirement already satisfied: nvidia-curand-cu12==10.3.7.77 in /usr/local/lib/python3.12/dist-packages (from torch~=2.7.1->pyhealth) (10.3.7.77)\n", + "Requirement already satisfied: nvidia-cusolver-cu12==11.7.1.2 in /usr/local/lib/python3.12/dist-packages (from torch~=2.7.1->pyhealth) (11.7.1.2)\n", + "Requirement already satisfied: nvidia-cusparse-cu12==12.5.4.2 in /usr/local/lib/python3.12/dist-packages (from torch~=2.7.1->pyhealth) (12.5.4.2)\n", + "Requirement already satisfied: nvidia-cusparselt-cu12==0.6.3 in /usr/local/lib/python3.12/dist-packages (from torch~=2.7.1->pyhealth) (0.6.3)\n", + "Requirement already satisfied: nvidia-nccl-cu12==2.26.2 in /usr/local/lib/python3.12/dist-packages (from torch~=2.7.1->pyhealth) (2.26.2)\n", + "Requirement already satisfied: nvidia-nvtx-cu12==12.6.77 in /usr/local/lib/python3.12/dist-packages (from torch~=2.7.1->pyhealth) (12.6.77)\n", + "Requirement already satisfied: nvidia-nvjitlink-cu12==12.6.85 in /usr/local/lib/python3.12/dist-packages (from torch~=2.7.1->pyhealth) (12.6.85)\n", + "Requirement already satisfied: nvidia-cufile-cu12==1.11.1.6 in /usr/local/lib/python3.12/dist-packages (from torch~=2.7.1->pyhealth) (1.11.1.6)\n", + "Requirement already satisfied: triton==3.3.1 in /usr/local/lib/python3.12/dist-packages (from torch~=2.7.1->pyhealth) (3.3.1)\n", + "Requirement already satisfied: huggingface-hub<1.0,>=0.30.0 in /usr/local/lib/python3.12/dist-packages (from transformers~=4.53.2->pyhealth) (0.36.2)\n", + "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.12/dist-packages (from transformers~=4.53.2->pyhealth) (2025.11.3)\n", + "Requirement already satisfied: tokenizers<0.22,>=0.21 in /usr/local/lib/python3.12/dist-packages (from transformers~=4.53.2->pyhealth) (0.21.4)\n", + "Requirement already satisfied: safetensors>=0.4.3 in /usr/local/lib/python3.12/dist-packages (from transformers~=4.53.2->pyhealth) (0.7.0)\n", + "Requirement already satisfied: psutil in /usr/local/lib/python3.12/dist-packages (from accelerate->pyhealth) (5.9.5)\n", + "Requirement already satisfied: Pillow in /usr/local/lib/python3.12/dist-packages (from rdkit->pyhealth) (11.3.0)\n", + "Requirement already satisfied: hf-xet<2.0.0,>=1.1.3 in /usr/local/lib/python3.12/dist-packages (from huggingface-hub<1.0,>=0.30.0->transformers~=4.53.2->pyhealth) (1.4.2)\n", + "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.12/dist-packages (from matplotlib>=3.7->mne~=1.10.0->pyhealth) (1.3.3)\n", + "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.12/dist-packages (from matplotlib>=3.7->mne~=1.10.0->pyhealth) (0.12.1)\n", + "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.12/dist-packages (from matplotlib>=3.7->mne~=1.10.0->pyhealth) (4.62.1)\n", + "Requirement already satisfied: kiwisolver>=1.3.1 in /usr/local/lib/python3.12/dist-packages (from matplotlib>=3.7->mne~=1.10.0->pyhealth) (1.5.0)\n", + "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.12/dist-packages (from matplotlib>=3.7->mne~=1.10.0->pyhealth) (3.3.2)\n", + "Requirement already satisfied: littleutils in /usr/local/lib/python3.12/dist-packages (from outdated>=0.2.0->ogb>=1.3.5->pyhealth) (0.2.4)\n", + "Requirement already satisfied: locket in /usr/local/lib/python3.12/dist-packages (from partd>=1.4.0->dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth) (1.0.0)\n", + "Requirement already satisfied: platformdirs>=2.5.0 in /usr/local/lib/python3.12/dist-packages (from pooch>=1.5->mne~=1.10.0->pyhealth) (4.9.4)\n", + "Requirement already satisfied: colt5-attention>=0.10.14 in /usr/local/lib/python3.12/dist-packages (from product-key-memory>=0.1.5->linear-attention-transformer>=0.19.1->pyhealth) (0.11.1)\n", + "Requirement already satisfied: charset_normalizer<4,>=2 in /usr/local/lib/python3.12/dist-packages (from requests->litdata~=0.2.59->pyhealth) (3.4.6)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.12/dist-packages (from requests->litdata~=0.2.59->pyhealth) (3.11)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.12/dist-packages (from requests->litdata~=0.2.59->pyhealth) (2026.2.25)\n", + "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.12/dist-packages (from sympy>=1.13.3->torch~=2.7.1->pyhealth) (1.3.0)\n", + "Requirement already satisfied: botocore<1.43.0,>=1.42.72 in /usr/local/lib/python3.12/dist-packages (from boto3->litdata~=0.2.59->pyhealth) (1.42.72)\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /usr/local/lib/python3.12/dist-packages (from boto3->litdata~=0.2.59->pyhealth) (1.1.0)\n", + "Requirement already satisfied: s3transfer<0.17.0,>=0.16.0 in /usr/local/lib/python3.12/dist-packages (from boto3->litdata~=0.2.59->pyhealth) (0.16.0)\n", + "Requirement already satisfied: distributed==2025.11.0 in /usr/local/lib/python3.12/dist-packages (from dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth) (2025.11.0)\n", + "Requirement already satisfied: bokeh>=3.1.0 in /usr/local/lib/python3.12/dist-packages (from dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth) (3.8.2)\n", + "Requirement already satisfied: msgpack>=1.0.2 in /usr/local/lib/python3.12/dist-packages (from distributed==2025.11.0->dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth) (1.1.2)\n", + "Requirement already satisfied: sortedcontainers>=2.0.5 in /usr/local/lib/python3.12/dist-packages (from distributed==2025.11.0->dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth) (2.4.0)\n", + "Requirement already satisfied: tblib>=1.6.0 in /usr/local/lib/python3.12/dist-packages (from distributed==2025.11.0->dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth) (3.2.2)\n", + "Requirement already satisfied: tornado>=6.2.0 in /usr/local/lib/python3.12/dist-packages (from distributed==2025.11.0->dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth) (6.5.1)\n", + "Requirement already satisfied: zict>=3.0.0 in /usr/local/lib/python3.12/dist-packages (from distributed==2025.11.0->dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth) (3.0.0)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.12/dist-packages (from jinja2->mne~=1.10.0->pyhealth) (3.0.3)\n", + "Requirement already satisfied: hyper-connections>=0.1.8 in /usr/local/lib/python3.12/dist-packages (from local-attention->linear-attention-transformer>=0.19.1->pyhealth) (0.4.9)\n", + "Requirement already satisfied: xyzservices>=2021.09.1 in /usr/local/lib/python3.12/dist-packages (from bokeh>=3.1.0->dask~=2025.11.0->dask[complete]~=2025.11.0->pyhealth) (2025.11.0)\n", + "Requirement already satisfied: torch-einops-utils>=0.0.20 in /usr/local/lib/python3.12/dist-packages (from hyper-connections>=0.1.8->local-attention->linear-attention-transformer>=0.19.1->pyhealth) (0.0.30)\n" + ] + } + ], + "source": [ + "!pip install pyhealth" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1e_FEPyho3zx" + }, + "source": [ + "### **BaseTask**\n", + "\n", + "There's a couple of things to keep in mind here. First, every task inherits the BaseTask class, which looks like\n", + "\n", + "\n", + "\n", + "```\n", + "class BaseTask(ABC):\n", + " task_name: str\n", + " input_schema: Dict[str, str]\n", + " output_schema: Dict[str, str]\n", + "\n", + " def pre_filter(self, df: pl.LazyFrame) -> pl.LazyFrame:\n", + " return df\n", + "\n", + " @abstractmethod\n", + " def __call__(self, patient) -> List[Dict]:\n", + " raise NotImplementedError\n", + "```\n", + "\n", + "\n", + "It defines a couple of key things:\n", + "\n", + "\n", + "1. The call function where we process the extracted data into a usable format from each patient. **(Note: The patient variable here effectively just represents a sample unit defined by the pyhealth.dataset module earlier.)**\n", + "\n", + "\n", + "\n", + "2. The input_schema and output_schema, which define the format of the model input and model output. This can range a wide variety of datatypes. While we offer [processors](https://github.com/sunlabuiuc/PyHealth/tree/master/pyhealth/processors) that take explicitly defined datatypes and processes them for use in training existing models on PyHealth. However, these pre-defined datatypes **are not required** and can effectively be anything for your purposes. However, We **recommend** schemas primarily for documentation purposes, and for direct use with our lightweight trainers.\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "_uHLk9voEXZ8" + }, + "source": [ + "![image.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAABqgAAAIwCAYAAAD6VOrwAAAACXBIWXMAABYlAAAWJQFJUiTwAAAgAElEQVR4nOzdWXCU55n+/0u9t6SW1Npa+4Y2BBKL2YxtMLbxEsoex9skk8lkkkolR1MzVXMyZzOVykFmTqdmflVJpZya2E4qMYnX2MbGYDBgECCxaEVbS2qtrV3d6kX9vv8DB/1NEi8QUAP6fqp0gEpv9/28aknUc/V9PymmaZoCAAAAAAAAAAAAVokl2QUAAAAAAAAAAABgbSGgAgAAAAAAAAAAwKoioAIAAAAAAAAAAMCqIqACAAAAAAAAAADAqiKgAgAAAAAAAAAAwKoioAIAAAAAAAAAAMCqIqACAAAAAAAAAADAqiKgAgAAAAAAAAAAwKoioAIAAAAAAAAAAMCqIqACAAAAAAAAAADAqiKgAgAAAAAAAAAAwKoioAIAAAAAAAAAAMCqIqACAAAAAAAAAADAqiKgAgAAAAAAAAAAwKoioAIAAAAAAAAAAMCqIqACAAAAAAAAAADAqiKgAgAAAAAAAAAAwKoioAIAAAAAAAAAAMCqIqACAAAAAAAAAADAqiKgAgAAAAAAAAAAwKoioAIAAAAAAAAAAMCqIqACAAAAAAAAAADAqiKgAgAAAAAAAAAAwKoioAIAAAAAAAAAAMCqIqACAAAAAAAAAADAqiKgAgAAAAAAAAAAwKoioAIAAAAAAAAAAMCqIqACAAAAAAAAAADAqiKgAgAAAAAAAAAAwKoioAIAAAAAAAAAAMCqIqACAAAAAAAAAADAqiKgAgAAAAAAAAAAwKoioAIAAAAAAAAAAMCqIqACAAAAAAAAAADAqiKgAgAAAAAAAAAAwKoioAIAAAAAAAAAAMCqIqACAAAAAAAAAADAqiKgAgAAAAAAAAAAwKoioAIAAAAAAAAAAMCqIqACAAAAAAAAAADAqiKgAgAAAAAAAAAAwKoioAIAAAAAAAAAAMCqIqACAAAAAAAAAADAqiKgAgAAAAAAAAAAwKoioAIAAAAAAAAAAMCqIqACAAAAAAAAAADAqrIluwAAwNpmGIYikYhGR0c1Pj6ukpISFRQUyOFwJLs0AAAAAAAAALcIARUAIGmGhobU2tqqlpYWjYyMaGxsTEVFRSoqKlJBQYGqq6uVm5ursrIyZWRkJLtcAAAAAAAAADdJimmaZrKLAACsDaZpKhKJaGxsTAMDA/rkk090/Phx9fb2qqioSIlEQpFIRIuLi0pLS1N1dbV8Pp8aGhpUXFyszMxM5ebmKisrS1lZWXI6nUpJSUn2sgAAAAAAAABcJwIqAMAtZ5qmYrGY5ubmNDIyoj/84Q967bXXNDAwoPz8fO3du1f/8A//ILvdrt7eXl26dEkXL15UZ2enZmZmZBiG3G631q1bp61bt6qurk6bN2+Wz+dTamqqHA7HyofFYiG0AgAAAAAAAG5zBFQAgFvONE01NzfrpZde0q9//WtNTk6qsbFRL7zwgr7+9a9r/fr1K6HS1T9LiURC4XBYra2tunDhgj755BO1tLSov79f0WhUNptNpaWl2rp1q+6//35t27ZNmzdvVnp6ejKXCgAAAAAAAOArIKACANwSiURCsVhMf/jDH/T222+rtbVV0WhU1dXVeuyxx7R582aVlZUpOztbqampf3a9aZoyDEOhUEihUEiLi4uam5vTzMyMLl26pOHhYXV0dGh4eFjRaFRut1sej0dVVVWqqalRfX29GhoaVFFRIZfLJavVmoS7AAAAAAAAAOAvIaACANxU8XhcgUBAp0+f1rFjx9Ta2qpwOKzS0lLt2bNHW7duVW1trfLz8+VwOK7rsa+eYTU9Pa35+XmNjo5qdHRUIyMj6u/v18jIiCYnJ7W0tCS73a7CwkLl5eVp/fr1KiwsVE1NjUpKSji/CgAAAAAAAEgyAioAwF/NNE1Fo1H5/X61t7fr5MmTam5u1sjIiNatW6edO3dq3759amhoUF5e3k19bsMwtLi4KL/fr9HRUfX39+vKlSsaHh7W5OSkgsGgPB6PXC6XcnJyVFtbq6KiIpWXlysvL095eXnKysqSx+OhywoAAAAAAABYJQRUAIAbdnUE39jYmAYHB/Xee+/p+PHjmp6els/n0549e/TUU0+poaFhVc+GMgxDk5OTam9vV0dHh9ra2tTd3a2JiQlFo1EtLy+rrKxMRUVFqqurU0NDg4qKipSVlbUyKjA9PV1Op3PVagYAAAAAAADWEgIqAMANMU1TS0tLamlp0c9+9jP9+te/VjQa1T333KNvf/vbeuGFF1RYWJjsMlcYhqHBwUE1Nzfr+PHjamlpUWtrqxYXFyVJ6enp2rZtm3bs2KEdO3bonnvuUXl5uUzTVEpKCuMAAQAAAAAAgJuIgAoAcN3m5+f15ptv6ne/+53OnDmjhYUF3Xffffq7v/s7bd++Xfn5+UpPT5fNZkt2qSsMw1AikVA0GlU0GlU8HtfExIQGBwfV2tqq5uZmjY+Pq6+vT4ZhKDU1VYWFhdqwYYN27NihjRs3qqGhQdnZ2cleCgAAAAAAAHDHI6ACAHwl8XhcQ0NDev/99/XBBx+or69Ppmlqw4YNeuihh7RhwwZVVVUpKyvrtgqmvkg8Hlc4HNbU1JQmJycVjUbV19ensbExdXd3q7e3V9FoVJFIRG63W/n5+SotLVVFRYU2b96sgoICVVZWKi0tLdlLAQAAAAAAAO4oBFQAgC+0tLSk7u5unT59Ws3NzTp79qxsNptqampWxuFt3rxZqampyS71pgiHw5qdndXw8LD6+vo0MTGh3t5eBYNBjY6Oan5+XhaLRdnZ2SooKFB5ebmKi4tVUFCg6upqZWZmyufz3TEhHQAAAAAAAJAMBFQAgD+TSCQ0Pz+vgYEBdXd368iRIzp//rwWFhZUVVWlxx57THv27FF1dbXS09OTXe4tZRiGpqenNTY2pitXrqi9vV39/f3q7+/X4uKiDMOQaZryer1qaGiQz+fT+vXrlZ+fL4/Ho5ycHHk8HqWlpclqtSZ7OQAAAAAAAMBtgYAKALAiHo8rFAppdnZWPT09+vnPf65Tp05paWlJ69ev13PPPafvfOc7Sk1NXdNhi2mampiY0OXLl9Xc3Kzz58+ro6ND4+PjMgxDGRkZ8vl8amho0Pbt27Vu3TrV1tbK5XLJarXK6XTK5XLJZrMpJSUl2csBAAAAAAAAVh0BFQBgxcTEhN555x398pe/1OHDh+V2u3XgwAH9/d//vR544AFlZWXJYrEku8zbgmmaKx/Sp6MBBwcHdfz4cV24cEGffPKJurq6tLS0JEkqKirSxo0btWvXLu3evVtbt25VXl5eMpcAAAAAAAAAJA0BFQCscdFoVIFAQAcPHtTRo0fV29ur9PR07dy5U/v371d1dbWKi4uVkZGxprumvoxhGIpGo5qfn1coFNLCwoKmpqY0PDys5uZmjY+Pq6enR3Nzc3K5XMrMzFRBQYFqa2tVX1+vDRs2qLa2VpmZmcleCgAAAAAAAHDLEVABwBq1sLCgixcv6uOPP9bp06d15coVeTwebd68Wffee682bNig6upqeTwextDdoEQiodnZWY2MjGh2dlajo6MaHR3V0NCQenp6FIvFNDY2JovFotzcXBUVFamgoGAlFKyvr1dpaSldawAAAAAAALjrEFABwBoTDAbV3d2t1tZWffLJJ7p06ZIMw9DmzZu1e/du3X///aqrq5PNZkt2qXel5eVlBQIBtbS0aHp6Wt3d3RoZGdH4+LjGx8clSdnZ2crPz1dZWZmqqqpWwiuv1yufz6fMzEy62QAAAAAAAHBHI6ACgDUgFotpbm5Oo6OjOnv2rN599121tbXJarWqurpaTz/9tB5//HHl5ubSrbPKlpeXFQqF1NfXp7Nnz2pwcFAtLS2amJjQ7Oys4vG4PB6P1q9fr9LSUjU2Nq50tnm9XrlcLqWnp8vpdPK9AwAAAAAAwB2DgAoA1oBAIKC3335b/+///T+1trbK4/HoySef1He/+1098sgjyS4Pf8HIyIhaW1t16tQpnT9/XidPntTs7Kwkyel0qqKiQk8++aTq6uq0e/durVu3Tk6nM8lVAwAAAAAAAF8NARUA3MVaWlr06quv6r333tP4+LicTqeefvppPf3006qoqJDH41FGRgZnTN1mTNNULBZTJBJRNBpVLBZTNBpVf3+/2tra1NbWps7OTvn9fsXjcTkcDnm9XtXU1Gjnzp2qq6vT1q1b5fP5GAUIAAAAAACA2xIBFQDcZaampnTx4kW99dZbunTpkkZGRpSVlaUHHnhADz30kMrLy1VcXKzU1FSCqTuIYRgKhUKam5vT1NSUZmZmNDMzo66uLvX398vv9ysYDEr6tMPK6/WqsLBQ69at0/r161VVVaWSkhJ5PB7OFwMAAAAAAEDSEVABwF0iEAjo8uXLOnPmjJqbm9Xa2qq6ujo1NjZqx44d2rZtm6qrq5NdJm6ysbExjY6Oanh4WH6/X+Pj4+rv71cwGNT09LQMw5DX61V+fr7KyspUUFCg0tJSFRcXq7S0VLm5uXI4HMleBgAAAAAAANYYAioAuINFo1GNjY1pcHBQZ8+e1ZEjR9TT0yOPx6OGhgYdOHBAe/bsUX5+frJLxSqJRCLq7OxUIBBQZ2enuru7NTY2pkAgoGg0KpfLJY/Ho/Lycq1fv15lZWUqKipSVlaW0tPTlZOTo7S0NFmtVjrsAAAAAAAAcMsQUAHAHSgWi2lpaUl+v1/vvfeeXnnlFQ0PDys7O1t79+7V3/7t3+rhhx9Odpm4TSwsLKi1tVVtbW06d+6cWlpaNDg4qFgsJtM0VVtbq6qqKm3atEn33HOPysvL5fV65XQ65XA4ZLfbZbPZCKwAAAAAAABw0xBQAcAd6OzZs3rppZf0+uuva2hoSIWFhfrud7+r559/Xg0NDUpJSZHFYkl2mbhNmKYp0zRlGIYMw1AikdD4+Liam5vV3Nysc+fOqbW1VdPT07JarXI4HNq1a5e2bNmyMh5y3bp1yV4GAAAAAAAA7iIEVABwBzAMQzMzM2pubtZrr72mlpYWzc/Pq6SkRPv27dODDz6o4uJi5efny+12J7tc3OZM01Q8HtfCwoIWFha0uLiomZkZBQIB9fb26sKFC5qcnNT4+LgSiYQyMzOVlZWl9evXq76+XvX19aqtrVVxcXGylwIAAAAAAIA7FAEVANzGEomEhoaGdP78eZ04cULNzc2amZlReXm5du7cqR07dqiurk7FxcWy2+3JLhd3sEQiocXFRU1NTWliYkLBYFB+v19DQ0MaHBzU8PCwpE/PPfN4PCooKFBJSYmqqqpUVVWlkpIS+Xw+eb3eJK8EAAAAAAAAdwICKgC4zVztbhkcHFRnZ6dOnTql5uZmjY+PKzc3V/fee68efPBBbdiwQYWFhckuF3exSCSi6elpjY6Oqq2tTWNjY+ru7tbY2Jimp6c1OzurjIwM+Xw+FRcXa926dSorK1Nubq6KioqUmZmpjIwMpaamJnspAAAAAAAAuM0QUAHAbSKRSCgUCikYDGpkZETvvPOOPvroI01OTio/P1/btm3TN77xDTU2NrLhj6SZnp7W2NiYOjs71dzcLL/fr56eHoXDYUWjUTkcDpWUlKipqUkVFRWqq6uTz+eT0+lUVlaWnE6nUlNT6fgDAAAAAABY4wioAOA2EYlEdP78eb388sv6xS9+oXA4rA0bNuib3/ymnn/+edXW1ia7ROAvmp6eVnd3tz755BN99NFHamlpkd/vlyQ5HA6Vl5dr48aNeuqpp1RRUaGmpiZlZ2cnuWoAAAAAAAAkEwEVACSRaZqKRqN644039MYbb+j06dOKxWKqr6/X888/r507d6qkpERut1sulyvZ5QJ/0fLysmKxmKLRqCKRiCKRiIaHh9Xd3a1Lly6pvb1dPT09mp+fl8PhUEFBgSoqKtTQ0KAdO3aosrJS1dXVcrvdyV4KAAAAAAAAVgkBFQAkgWEYGhgY0NGjR3XkyBF1dnbKNE3V1NRoz5492rp1qyoqKpSdnc0oNNyRlpaWND8/r+npaU1MTGhmZkajo6O6cuWKBgcHNTw8rPn5eaWnp8vj8ai4uFjl5eWqrq5WfX29KioqlJeXJ4vFkuylAAAAAAAA4BYgoAKAVRSPx9XW1qbW1lY1NzertbVVMzMz2rJlizZs2KD77rtPjY2NjD/DXWlxcVFDQ0MaGhrS4OCgBgcHNTk5qf7+fi0uLmppaUlOp1PFxcUqLCxURUWFfD6fysrKVFxcrJKSEjkcjmQvAwAAAAAAADcBARUA3GKGYWhxcVGDg4Pq7e3Ve++9p/Pnz2txcVFlZWW677779MQTT6impkYejyfZ5QKrJhaLaWJiQhcvXlQgEFB7e7t6e3s1MzOj8fFxpaenKzMzUz6fTzU1NWpoaFB+fr6ys7OVkZGhjIwMeb1e2Wy2ZC8FAAAAAAAA14mACgBukUQisdIV0t7erldeeUVvvfWWEomE6uvr9eyzz+qZZ55RWVlZsksFbhvLy8vq7e3VoUOHNDg4qDNnzqinp0fhcFimaSo7O1uNjY3asGGDtm3bpq1bt8rj8cg0TVmtVrlcLjmdTkYDAgAAAAAA3OYIqADgFllcXNRrr72mX/7ylzp27JiWl5e1f/9+fe9739P999+v3NxcWa1WpaSkJLtU4LZiGIYMw5BpmopGo5qfn1d3d7cOHTqknp4enT59WoFAQIZhyO12q6CgQI8++qiampq0Y8cObdq0ia4qAAAAAACA2xwBFQDcRLFYTH6/X2+88YY++ugjdXd3y+l0aseOHXriiSdUVVWl0tJSZWZmsoEOfAWmaSqRSCgcDmtmZkZLS0uamprS5OSk+vr6dPnyZQ0ODmpiYkKxWEzp6enKy8uT1+vVvffeq5KSEjU0NKiiokJOpzPZywEAAAAAAMAfEVABwE0Qi8V0+fJlHT58WC0tLWppaVFmZuY1HR0NDQ1KS0tLdqnAXSESiWhyclKBQEDT09MKBAIaGhrS0NCQRkZGND8/L5vNppSUFBUUFKikpERFRUWqr6+Xz+dTZWWlcnNzk70MAAAAAACANYuACgBukGEYWlxcVEdHh9ra2nTy5Em1t7drcXFR69ev1549e/Tggw+qpqZGDocj2eUCd71gMKjBwUH19vaqt7dXAwMDGh0d1dTUlEKhkFJSUlRcXKySkhKVlpaqrKxMeXl58vl8ys3NVXZ2ttxuN2M3AQAAAAAAVgEBFQBcp1gspoWFBY2MjKizs1O//e1v1dLSIklqamrSI488omeffVbZ2dmM8QOSKB6Pq6+vTz09Pero6NClS5c0MDCgYDCoWCwmq9Uqn8+n+vp61dXVqa6uToWFhfJ4PEpLS5PH45HL5VrpxAIAAAAAAMDNQ0AFANdpcnJSb775pn72s5/pk08+kSQ988wz+sEPfqB9+/bRLQXc5s6cOaPLly/r5MmT+vjjj9XX16d4PC5J8vl82rx5s+677z7t3btX69evV25uLgEVAAAAAADATUZABQBfwfLyssbGxvTiiy/q3Xff1dDQkLKzs7V37149++yzKi0tVXZ2ttLT02W1WpNdLoDPYZqmQqGQwuGw4vG4YrGYQqGQWltb1dHRofb2dvX09GhiYkIOh0NpaWmqqKjQ+vXr1djYqJ07d6qkpEQej0cWiyXZywEAAAAAALhjEVABwBeYmZlRZ2en3n//fX388ccaGxtTTk6OmpqatHfvXjU0NKisrEwul4vNauAOZJqmEomEZmZmNDc3p+npaY2Pj2tyclJdXV0rYdXCwoJM01RBQYE8Ho9qa2tVW1urqqoqVVZWqrCwUFarlU4rAAAAAACAr4iACgA+x9GjR3Xq1CmdO3dO3d3dcjqd2r17tzZt2qRdu3apurqacX7AXSoSiSgQCGhoaEhDQ0Py+/0aHh7W9PS0RkZGZBiGLBaLvF6vKioqVFhYqJKSEvl8PpWVlamwsFBZWVnJXgYAAAAAAMBti4AKAD7Hj370I7311lsyDEONjY3av3+/HnzwQfl8Psb4AWvM1S6rK1eu6NKlSxocHNSVK1cUCAS0uLiomZkZlZeXq7i4WCUlJaqpqVFVVZUyMjLk9XqVkZGh9PR0paWlJXspAAAAAAAAtwUCKgD4HP/3f/+nQCCge+65R/v372d0F4BrRKNRjYyMqL29XR999JGuXLmitrY2jY2NyTAMWa1WNTU16Z577lFTU5M2btyoyspK2e12ORwO2Ww2Wa1WAm8AAAAAALAmEVABwOeIx+OSJIvFwgYygD9jmubKRyKRkGEYmpyc1ODgoC5evKgPPvhAQ0ND6uzs1MLCgtLS0pSfn6+mpiY99NBDamxs1IYNG5Sfn5/spQAAAAAAAKw6AioAAICbJBaLKRqNKhQKaXp6WtFoVIODgxoaGlJvb686Ojo0OjqqaDQqp9OpnJyclbGAu3btUmlpqSoqKpSdnZ3spQAAAAAAANxSBFQAAAC3UCQS0fz8vAKBgEZGRjQ9Pb0SWgUCAQWDQcViMWVnZys1NXXlDKuNGzfqkUceSXb5AAAAAAAAtwQBFQAAwCqbm5vTxMSEBgYG1NXVpb6+Pg0PD2tyclKLi4vKy8vTww8/rH/9139NdqkAAAAAAAC3BAEVAABAkl09v6qjo0OXL1/W4uKi6uvr9fTTTye7NAAAAAAAgFuCgAoAAAAAAAAAAACrypLsAgAAAAAAAAAAALC2EFABAAAAAAAAAABgVRFQAQAAAAAAAAAAYFURUAEAAAAAAAAAAGBV2ZJdAAAAN8o0zWv+nZKSkqRK7lzcQwAAAAAAACQDARUA4I5kmqaWl5eVSCSUkpIiq9Uqm40/a9fDMAzF43GZpimLxSKr1Sqr1ZrssgAAAAAAALAGsJMHAKvkaqDy2TCAbpXrYxiGotGoFhYWFAwGNT4+roWFBVmtVmVmZqqoqEher1dpaWmy2+3c379geXlZS0tLmpubUzAY1OjoqGKxmNxut3JycuTz+eT1euVyuQirAAAAAAAAcMsQUAHALWSapuLxuMLhsObn5zU5OalEIqHU1FRlZWUpMzNTLpdLNpuNMOVLLC8va2JiQh0dHTp16pRmZmZWOqdM05RpmrLb7aqqqtKWLVtUU1Mjj8fDff2MaDQqv9+v1tZWtbW1aW5uTg6HQykpKVpeXpbFYpHH41FdXZ22b9+uwsJCud3uZJcNAAAAAACAu1CK+aeHTwAAborl5WVNTU2pt7dXXV1dGhsb08TExEq3Sl5enoqLi1VXV6eqqiplZmYyou5zxGIx+f1+nT17VhcuXFAwGFRGRoZ8Pp8yMjIUj8c1MzOjqakpmaapkpISbdmyRVu2bFFeXl6yy0860zQVi8V06dIlnTlzRt3d3YpGoytdZ3a7XaFQSNPT01pYWJBpmtq4caO2b9+u2tpaeTyeZC/hjmAYhkzTVEpKiiwWS7LLAQAAAAAAuK2xEwoAN5lpmjIMQ4FAQKdOndKhQ4d04sQJjYyMXLOB7XQ6VVhYqL179+prX/uaNm/eLJ/PJ7vdnuwl3FZM09To6Kg++ugjnTlzRk6nU9/61rdUX18vj8cji8WyMj5xenpaH3zwgc6ePavx8XGZpqkHH3xQdrt9zQYGpmkqEolocHBQb7/9tvr7+1VTU6NHH31UdXV1K6GoaZoKh8Py+/16++23deLECc3Pz0uSNmzYIJfLlcxl3Jau/qwvLy8rHo8rFAopkUjIbrfL5XLJ6XTKarXKYrHQyQcAAAAAAPAn6KACgJtseXlZ4XBYL774ol588UVdunRJhmF84TW7d+/WD37wAz3xxBPKz89fpUpvf1dHJL766qt69913VVpaqh/+8IcqKyv73GtisZiOHDmid955R0tLS/rnf/5nVVZWrtlRdYlEQiMjI3r55Zd16tQpvfDCC3riiSeUnZ39F7/eNE0lEgm9+OKLOnnypOrq6vT8889r3bp1q1z57c8wDIVCIc3MzGhiYkJ9fX0Kh8PyeDwqKSlRcXGxsrKylJqaumYDUgAAAAAAgM9DBxUA3ESJREKTk5N6//339ctf/lK9vb36svcBpKSk6OLFi/rVr34lq9WqAwcOyOv1rlLFt7dEIqG2tja1tLSourpaDz/8sHw+3xdeY7PZtGXLFsViMb333nt67bXX9I//+I9yuVxrsotldnZWly5d0smTJ/W1r31N27dvV0ZGxkon319itVr12GOPKRwOa2hoSGfOnFFlZSUhy2eEQiF1dXXpyJEjOnz4sPx+v+Lx+Mp9TU1NVWFhofbv36+HH35Y1dXVSk1NXZOvQQAAAAAAgL+EgAoAbqJQKKTu7m69/PLL6u3tVSgU+tKAyjRNLS4uqrW1VSUlJSotLdXevXtXqeLbl2EYikaj6urqUigU0saNG9XQ0CCn0/mF11ksFuXm5qqiokIVFRXq6upSIBBQRkaG0tPTV6n624NpmpqZmdHAwICi0ai2bt2qkpKSLzzr7GqAUlhYqJqaGs3Ozmp4eFiBQECFhYVr/pw0wzC0uLio48eP6+2339bHH3+swcFBhUKhla8xTVNWq1UDAwMaHh5Wf3+/Dhw4oO3btysnJyeJ1QMAAAAAANw+1vYuEwDcZLOzs7p8+bJOnDihSCTypeHUZwWDQbW2tmrLli3asWPHmu34uSqRSGh+fl7Dw8NKS0tTWVnZV+4ss1gsysvLU3l5uS5fvqyRkRGVl5evuYAqFotpenpak5OTKi8vV3FxsVJTU7/StXa7XRUVFerr69Pk5KT8fv+Xdq/d7UzT1NLSklpbW/Xqq6/qgw8+UCAQ+Is/54lEQrFYTHNzc5qbm1M8HpfT6dSuXbvW7LhJAAAAAACAz2JWDwDcJPF4XOPj4+ro6FAoFFIikbiu66+OB+zu7tbY2NiXnlt1t7saUM3OziovL09ZWVnXdb3b7VZ+fr7sdrtmZmYUiURuUaW3r0gkorm5OUWjUdXW1srhcFzX9VlZWfJ6vUokEpqYmFjzr8lYLKbR0VG9/vrrOv3uAz0AACAASURBVHLkiIaHh79SCD00NKQPPvhA7733nvx+/ypUCgAAAAAAcPsjoAKAmyQSiWhsbEwDAwM3/BiLi4sKBAIaHh5e82HA1RF/sVhMLpfrusMVq9Uqt9stq9WqSCRy3YHh3eBqF49hGPJ6vbJardd1vcPhkMPhWOkcup6OwLvR4uKiLl++rDfffFOjo6PXdW1/f7+OHDmiM2fO3KLqAAAAAAAA7iwEVABwkyQSCYVCIc3Nzd3wYywvLysUCl1zns1alZKSIpvNJqvVqmg0quXl5eu6/mrAZZqmHA7HdYczdwOLxSKr1SrTNLWwsHDdoWc8Hlc8HldKSoqcTueaHzkZDAZ18eJFjY6OKhaLXdf1hmFoYmJCly5d0uTk5HW/ngEAAAAAAO42BFQAcJOkpKTIarXKZrvx4/1SUlJksVhksfDr2WazKTMzU5mZmZqYmND09PR1XR8OhzU6OqpoNKqcnBy5XK5bVOnty+VyKSsrS06nU52dnYpGo9d1/dTUlKampmS1WlVYWLimX5fRaFTj4+Pq7u6+4XApFAppaGhIgUBA8Xj8JlcIAAAAAABwZ1m7O00AcJNZrValp6df91lJn2W32+XxeOTxeNZ0t4r06f3MyMhQWVmZZmZmNDg4qJmZma98/dTUlHp6emS1WlVSUqL09PRbWO3tyeFwKCcnRzk5Oerp6VEgENDS0tJXujaRSGhwcFCjo6NKS0tTVVXVmuxCuyoej2t+fl7j4+M3PH4zFotpenpa09PTa3LkJAAAAAAAwGcRUAHATeJwOJSXl6eKioobfoyMjAwVFRWpsLBwzQdUFotFbrdbNTU1ysjIUFdXly5evKhIJPKF1yUSCY2Njam9vV1jY2OqqalRQUHBmuygslgsysrK0rp162Sz2XT69Gn5/f4v7AAyTVOmaWpgYEBtbW0yDENlZWXKz89f0x1UiURi5UyvG2WaphKJhJaXl9f8GXMAAAAAAAA3PocKAHANh8OhgoIC1dfXy+PxaGlp6bpGgdntduXn56umpkbFxcVrOgy4ymq1qrq6Whs3btSZM2d07NgxZWVlqbq6WjabTTabbSXIu7rxPzc3p7Nnz+rChQuy2Wx66KGHlJWVtWbvZ0ZGhmpqarRr1y6dPXtWmZmZcjgc8vl815zNdTU8iUQiCoVCOnr0qIaHh1VSUqKmpqY13T0lffpadDgcSk1NveHw2Gq1yul0yuVyrdnXIwAAAAAAwFXsjgDATeT1erVx40bdf//9192xk5mZqfXr12vTpk2y2+1rvoNK+vRMLq/Xq0ceeURVVVVqbm7WT3/6U7W1tWlycnIlBIzH4wqHwxobG9Phw4d18OBBDQ0Nac+ePdqxY4fS0tKSvZSkcTqdqqys1Le+9S25XC79/ve/129+8xtduXJFc3NzisViMgxD8Xhcc3NzunLlin7zm9/o97//vdLS0rRv3z41NDQkexlJ53A4lJmZKZ/Pd8M/m3a7XTk5OcrNzf2rzqoDAAAAAAC4G7A7AgA30dWRdN///vc1MzOjixcvKhwOf+l1aWlp2r17tw4cOKDGxsZVqPTOUlhYqGeffVY+n0/Hjh3TT37yEzU1NamsrEwej0eJRELT09Pq7u7W5OSkKisrtXfvXu3YsSPZpd8W7Ha7ysvL9YMf/EDvv/++Ll26pCtXrqihoUGFhYVyOByKRCIKBAIaGBjQ6OioHnnkEe3bt0+1tbXJLv+24HK5VFRUpKamJr322muKRqMyTfO6HuNqN1tFRYXsdvstqhQAAAAAAODOYP2P//iP/0h2EQBwt0hJSZHD4VB+fr4cDodCoZAWFha0vLysRCLxZ1/vcDiUnZ2thx56SM8995weeOAB5eTk0D31J6xWqzwej/Ly8lRUVKTs7GyFw+GVQGVkZESRSEQ+n0/bt2/XAw88oIaGBmVlZSW79NtCSkqKrFarsrKy5PP5VFRUJJfLpampKfn9fg0MDGh8fFw2m01lZWV66KGHdN9996miokKpqanJLv+2cPVn0jAMdXR0aHp6+rrOo0pPT9eWLVv0ta99TY2NjUpJSeHnHAAAAAAArGl0UAHATXb1LKkDBw4oIyNDlZWVunTpkkZGRhSNRpVIJJSSkqL09HQVFxdr06ZNevjhh7Vjxw4VFBSwaf05LBaLcnNz5XQ65fP51NPTo4mJCYXDYaWkpCg1NVVlZWWqqalRbm6u3G63TNPkfn6G3W5XaWnpyqi63t5eTU9PKx6Py+FwKCsrS5WVlaqurlZ6erocDgf38DPS0tJUU1OjAwcOaG5uTp2dnYrH4196nd1uV21trR588EFt3rxZkrinAAAAAABgzUsxr3c+DQDgK5ufn1d3d7fOnz+vnp4ezc7OyjAMORwO5eTkaN26ddqxY4cqKirkdrvZtP6MRCKhWCymWCympaUlzc/Pa2ZmRqFQSEtLSythn2maslgsstlsstlsstvtSktLU2ZmpjIyMlaCFqfTKZvNJotlbRy/aJqmlpeXFYvFFI1GFYlENDU1pfn5eYVCIUWjUcXj8Wvuod1ul81mk8PhUEZGhjIzM+XxeJSamiqHwyGHwyGbzbamX6exWEydnZ168cUX9f7772tgYECRSOTPOiRTUlJksVjkdDpVVVWlJ598Us8884y2bNkiq9WapOoBAAAAAABuHwRUAHCLGIaxckaNaZqKx+Oan5+XYRhyOp1yu93XdKhYLJY1vfEvfXqfEomE4vG4QqGQJicnNTExodHRUfn9fo2NjWlubk4pKSkKh8OKxWIyDGNltKLD4ZDb7VZGRoays7OVl5engoIC+Xw+5eXlyev1yu1239VBlWEYK8HU3NycJicnNT4+rpGREfn9fk1OTiocDsswDC0tLWl5eVmmacpqta4EeW63W16vV9nZ2fL5fNd8pKWlrYR9a/X1ahiGLly4oIMHD+rQoUMaHBzU7OzsNV9js9mUmpqq8vJyPfXUU3rqqafU0NDA2VMAAAAAAAB/REAFALdIKBTS4uKiQqGQIpGIIpGIYrHYShhls9nkdDrlcrmUlpamjIwMOZ3OuzY4+SqWl5c1OjqqgYEBdXZ2qqOjQ36/X/Pz81pYWJDdbld5ebm+/e1vy+Vyye12y263yzAMhcNhhcNhTU1NrYQxgUBAoVBIxcXFqqqqUn19vWpra1VeXq6MjIy7LmAxDEOxWEyDg4Pq6elRR0eHenp6FAgEtLCwoHA4rLS0NO3YsUP79u1Tenq6XC6XLBaL4vG4lpaWtLCwoGAwqKGhIfX39ysYDMpqtaqwsFB1dXWqr69XTU2NCgoK1vT5VPF4XJOTkzp//rwOHz6s1tbWlfPmbDabcnNzVV9fr8cff1ybN29WTk7Omg71AAAAAAAA/hQB1R9NTU3pwoULOnTo0Mo78q9KSUmRzWaT2+1WVlaWamtrtXPnTuXl5d205+/o6FBLS4tisZgeffRR5eXlXfMu63A4rF//+te6ePGiKioqtG/fPm3atOmmPf9f6/jx4+ru7lZRUZG2bdt2Q/emvb1dBw8e1Pnz51VbW6t///d/l9PpvCNHIY2OjurDDz/Uyy+/rMzMTP3Lv/yLtmzZIofDkezScAtFIhHNzc1pampKwWBQfX19GhgYUCAQ0NTUlBYXF1dGgVkslpVOn5ycHJWWlqq8vFwVFRXKz8+X1+tVVlaW7Hb7mtjQjsViCgaDamtr08WLF9Xb2yvDMFRSUrIShvT396u5uVmJREI//vGPV0b6WSyWlc6r5eVlxeNxRaNRLS0taXFxUVNTUyvfi6mpKWVlZamhoUFNTU2qrKxUZmbmHfl75rNM01QoFNLw8LDa29t1/vx5jY+PKy0tTRUVFaqsrFRRUZGOHTumlpYWbd++Xd/4xjeUmpq60rlnGMY193BpaUlLS0uam5vT2NiYBgYG1NfXp1gspuLiYm3cuFENDQ2qqqq6qzvSvsjy8rIWFhY0NTWlubk5zc3NKR6Py+l0yuPxKDMzU7m5uUpPT5fNxrGfAAAAAAAAn8VuyR9duXJF//u//6vTp0+vjOX67HkSV0cfpaamqqysTMePH9czzzyjpqYmud3uv+q5Q6GQDh06pIMHD8pms2l0dFTf//73V0KeeDyumZkZ/fznP1dvb69KS0tlt9tvi4BqeXlZ0WhUL774olpbW1VVVaWFhQW98MIL1/1Y58+f1x/+8AedO3dOfX19eu6557Rhw4Y78h36fr9f7777rt577z05HA7dd999Ki8vV0FBQbJLwy0QCoUUDAbV39+vtrY2dXV1qa+vTwsLC0pJSVE0GtXk5KQCgYDi8fjKSDq73S6Xy6XU1FR5vV7l5OSshFX19fXauHGjSktLlZ+fL7fbfVcGAKZpamFhQb29vTpx4oT6+/tls9lUU1OjiooKlZeXq6SkRF6vV0VFRZqfn1dXV5eGhoZUX1//lX7/Li0tqaqqSoODgxoaGtLg4KDOnj2r7u5ubd++XVu2bFFhYaFcLtcqrPjmMwxDk5OTunz5ss6ePavh4WF5vV5t27ZNFRUVKisrU2FhobKyshSPxzUxMbESpvp8vi99/EQiobm5Oa1bt041NTUaGhrS0NCQjh07pq6uLu3evVvr169Xdnb2mgphotHoyjloqampstvtSk1NVTwel8vlWhk5GY/HFQ6HVz4HAAAAAACAT62dnaQvMTw8rHfffVehUEiSVsZu2Ww2JRKJlfFc0qfdTufOndPY2Jj+7d/+TdXV1X/VmRIzMzNqaWnR8ePHlZaWplAopBdeeGEloLp6Tsjw8LCmpqaUnp6uxcXFv37RN8Hy8rImJiZ0/Phx9fT0aGRkRKWlpTcUUPn9fgWDwZVzevr6+lRTU3NHBlRTU1Py+/0yDEORSETDw8Oan58noLqLXP3eBoNBdXZ2qrW1VZcvX1Z3d7d6e3sVDAbl8/m0f/9+lZaW6sqVK5qcnFQsFpNpmjJNc2WDe25uTqOjo5Iki8Wi7OxsVVZWqq6uTo2Njdq8ebNqa2vl8/n+6kD8dmKapubm5tTe3n5NF2ZTU5PuuecelZeXX9PZlJOTo4qKCvn9fn3wwQcqKSn5SvfD7Xarurpa1dXVmp+fV3d3t86cOaOOjg6dOHFCCwsL2rVrlyorK++4AME0TU1MTOjUqVM6d+7cyu+ZXbt2qampSbm5udd04JWWlqq0tFRDQ0O6ePGiamtrv/Q5rFarsrOzlZ2draamJo2NjenixYtqaWnR0NCQ3nzzTYXDYW3atEk+n++O70b7PFf/L3B1bOfExITGx8cVDAY1NzenUCi0cp6XJNntdqWlpSkzM3PlLLScnBxlZGTI7XbL7XbftfcKAAAAAADgqyCg+qPl5WUtLS2t/LukpEQNDQ3yer2KRqOamppSIBBYOcdjampK//d//6eHH35YeXl5ys3NveHnDofDisfjkj7dbJyZmblmxKDdbld2drYeffRRdXR0qLGxUbt27brxxd5EV8cbXd2Qi8ViKyHf9UokEiuPY5qmlpeXb1qdq+3qqKyrEonENd9T3Nk+G6J+8skn+uijj3TmzBmNjo5e87otLCzUk08+qd27d+vMmTPy+/1qbW39wteCYRgKBoMKBoNqbm5WaWnpynlBu3btUn19vZxO513RqRKJRNTd3a2jR4+qvb1d27Zt01NPPbXSJfqnMjMzVVlZqY6ODjU3N+upp56S1+u9rnuRkZGhbdu2qampSWfOnNGbb76pkydPKhaLKTU1VaWlpTdzibdUIpFQLBbT2bNn9eGHH8owDO3du1ePP/64PB7PXxwNmZ+fr+rqag0ODqqrq0tLS0tyuVzXNUayoKBABQUFampq0okTJ3Tw4EEdOnRIpmlq586dysnJuZnLTLqrncIzMzMaHh7WwMCA/H6//H6/RkZGND4+romJCQWDwZWOKovFIpfLJY/Ho5ycHBUXF6u8vFyVlZUrIWFZWZmysrLkdrs5mwoAAAAAAKxJd/4O5y3y1FNP6Z/+6Z9UWVm58rkLFy7oxz/+sQ4ePLgSpJw4cUJbtmz5qwIqq9W6MrrramfFZ9/Ff7Wj4mc/+9kNP8etYrVaZbVar9lYuxvHkAGfNTs7q1OnTunVV1/V73//+8/taJyZmVFfX5+2bNmi+vp6PfDAA7p8+fJKIP1VXB2nduLECT322GP67ne/q02bNikjI+OO/1kLBoM6fPiwOjo6tH//fn3nO9/5wq+3Wq3Ky8tTeXm5PvjgA125ckV5eXnKzs6+7ud2OBy6//77VVhYqJdeekknTpyQ1+tVaWmpTNO8I8KCeDyusbExHTx4UFlZWfr617+uPXv2fOE1TqdT5eXl8nq9Gh0dVVdXlzZu3HhDgWdBQYGefPJJ1dbW6r/+67909OhReb1e7dy5845/bX7WzMyMurq6dPbsWZ04cUKnTp1SIBCQw+FQTU2NcnNz5XK5ND8/f811sVhM8/PzCgQCunjx4srnCwsLdc899+iBBx7Qtm3btH79evl8vjviNQcAAAAAAHAzEVD90dXA6apEIqFwOHzN56qrq3X//ffrwoULunLliiQpEAhobm5O0qedD4uLi3r77bfV0tKiwcFBzc/PyzAMuVwuVVRUaNu2bbrvvvtUWVmpRCKhzs5Ovf7662pra5P06Tu1p6am9J//+Z/atm2bdu3apYaGBoVCIf3P//yPLly4oKamJj355JNqaGj4szWMjIzonXfe0ZkzZzQxMaF4PK7MzExVVFRo3759uvfee5Wenr5yTTQa1YULF/Sb3/xGy8vL2r9/v/bs2aPDhw/r6NGjGhoakmEYKi0t1e7du/U3f/M3K++2n5mZ0dmzZ/XGG29odnZW0qcdEadPn9aPfvQj1dfXa9++fSujCr+MYRgrm5pXN+o+22limqbC4bDeeustnTlzRgMDA4pEInK73fL5fNq3b5/uv//+lTF6hmEoEAjopz/9qYaHh7V3715985vflNPpvOZ5g8Ggjh07pt/+9re699579fjjj18z9sowDH388cc6duyYOjo6ND09LbvdLp/Pp61bt+r+++9XY2Pj565F+jS0+2xHFe5cIyMjeu211/S73/1OLS0t13Re/qnJyUk1Nzdr06ZN2rRpkzZv3qzc3Nw/67T6Kqanp3Xo0CH19/frhz/8oR588EEVFhbesZvahmFocHBQwWBQpaWlevzxx7/SdWlpaaqqqlJlZeXKuXc3ElBdVVJSosbGRoVCIQ0NDWl2dlbp6em3fYfa1d+HbW1tCofDeuKJJ77yuYS5ubmqra3V9PS0jh49qvr6+hter81mU319vTZt2qRgMLjSZZyZmXlDj3e7iMfjCgaDam9v19GjR3XixAn19vZqYWFh5f8GdXV1+t73vqedO3fqwoUL+slPfqKxsTFFo9HPfVyLxaKpqSkdP35c58+f17p167Rr1y7t3btXW7ZsuatHJAIAAAAAAPyp23sHbhX9aUBlsVj+bAxXWlqabDbbNZ+PRCJaXl5WLBbTpUuX9OKLL+rcuXMaGRnR4uLiynkzNptNGRkZOnHihC5fvqxnnnlGjY2N+sUvfqEPP/xQPT09kv7/kXlvvPGGPv74Y33/+99XeXm5pqam9MorrygQCMjv98vr9V4TUM3OzurcuXP67W9/q5MnT2piYkJLS0syTVNOp1MZGRk6cuSIDhw4oOeee0719fWSPn2H9+9+9zu9++67mp2dVW9vr86fP6/XX39dk5OTWlhYkGEYSk9P1+nTp+X3+/WNb3xD5eXl6unp0UsvvaTDhw+vvHM8Go2qt7dXr7zyitLT01VSUvJnZ6B80ffgTwOqq8LhsLq7u/WrX/1Kx48f1/DwsBYWFpRIJGS1WuV2u3Xu3DmdOHFCzz33nO69917FYjF9+OGHK50W4XBYtbW12r179zWPPTAwoP/+7/9WV1eX+vv7VVhYqNraWi0vLysUCukXv/iFDh8+rPb2ds3OzioajcpiscjtduvkyZM6deqUnnjiCX3zm/8fe3ce2+ad33n8zfumSJG679OSLFlybMtXnIxj55wkM5lJ05kWm25b7KLALHYX2y0W2/61wAK7BdqiKLpA/+gudntNp5g2aZOZpOnkcmzXRxw7tnVLlKibkiiSIsWbfPYPD59KlhRbsh1byfcFBJnoefjweR49tAe/D7/f7/fVyot8Pr/uGgo/E7tXPp8nm81y+fJl3n33XS5fvryhYuJ2iUSCsbExRkZG6OzsZM+ePXR3d7O6usry8vK23j+dTrOwsEA4HKa2tpaSkhLsdjtOp/NeLuuhyOfzrK6u4vf70ev11NfX33WQbbVaKS8vZ8+ePYyNjTE7O0tTU9OG4PluFf58tNvtZDIZUqkUNpttR8f6MmUyGSKRCH19fZSVlVFVVXXXoVBRURFVVVW43W76+voIBoOUlpbuaJaiVqtFq9XidruJRCKk02nS6fS2j/OoUBSFlZUVRkZGOH/+PD/5yU8YHx9nfn6e1dXVdX+OZzIZcrkcHo9H/ULJP/3TP31hQJXP59V7FIlEiEQiTE9P09/fzxNPPMHp06dpamrC4XB8parQhBBCCCGEEEIIITYjAdXP3b4QlM/n1y145nI5hoaGuHnz5rqFZbfbjcPhYHJykjfeeIM/+7M/IxqNYrfbqa+vp6Kigkwmw+TkJNPT00xNTREMBrFarfT09KizJzZ7b6vVisFgIJfLsbi4yOjoqDrfyefzrXvNzZs3+X//7//x1ltvEQ6HcblctLW1YbFYWFxcZGhoCJ/PRyAQwGq1YrVaqa2tVV87MzNDOBxmZWWF/v5+/H4/DQ0N2O129VvxMzMzBINBuru7qaysxGAwYDKZ1n3buxDQWK1WtQrhbis8dDqdWlVSWAQsvNbn8/FXf/VX/N//+39ZXFzE4XDQ0NCAx+MhFApx/fp15ubmmJmZQVEU3G43zc3NJBIJlpaWCAaDfPbZZ3z88cfrAqpIJMLNmzc5c+aMet8LFTHRaJQf//jH/Nmf/Rl9fX0oikJlZSU9PT0kk0kGBgbo7+9namqKubk5mpqa6OzsxGq1otfrN8yg2snir3h0FCokP/nkE65fv37HcAr+paqxv7+fw4cPU19fz+HDh+nv7992QAW3nqNEIsHly5c5ePAgbW1tuzagisViTE5OYjabaWpquuvFeJ1Oh8vlorOzk5GREfx+Py0tLdTV1e34fAotVXdTkJzNZgmFQvT399Pd3U1paeldv9ZoNFJZWUlFRQX9/f309fVhs9lwuVw7Ph+j0bhuhuButbKywpUrV3jnnXf48MMPuX79+pYtORcXF+nr6+PgwYPs2bNHnTUXiUTuumJ2dXWV8fFxFhYWmJ2dZWFhgRdffJHu7m7cbvf9vDQhhBBCCCGEEEKIR44EVFuYnp7m8uXLBAIBstkssViMjz76iI8++ohQKIRGo0FRFHp6eigpKaG/v59z584Ri8WwWq08/vjjPPfcc3R0dJBMJrl27Rp/8id/wvz8PJOTk1y8eJF8Ps/3v/99HA4HP/zhD7l69apaDfTaa6/R09PD/v370Wq1ZDIZdQG38C3/gpWVFd566y3ee+89VlZWcDqdvPTSS5w6dQqXy8W1a9d46623uHLlCuPj47zxxhtUV1dTU1MDsG4hbXV1FaPRyEsvvcT+/fuJxWKcP3+eS5cukclkGBsbo6+vj+7uburr6/ne976HzWbjL/7iL1haWlJncrz++uvU1tbS1NR0z7+LRCLBuXPn+Lu/+zuWlpYwGAycOHGC559/nrq6Oqampvjrv/5rrl27xvT0NO+99x7V1dX85m/+Jvv378fj8TA6Osri4iLnz58H/qUF38TEBJcvX1YXpffu3Ut1dTW5XI6xsTH+8A//kImJCfL5PJ2dnbzwwgscP36c1dVV3nzzTT766CNmZma4du0a/+t//S/++3//71it1nu+ZvHoyeVyhMNhrl27tq1wqVDl0t/fz759+9i/fz/vvfceMzMzX1hp8UXm5+eZnZ0lGo3u6PUPm6IoRKNRZmdnqamp2Xa4ZDKZaG9v58KFC0xMTDA2NkZNTc2OK04Ks/92k0wmQzgcxufz8a1vfWvb4ZLb7aaxsZGrV69y8eJFmpub7ymgWnv/dmvbyVwux8jICG+99RZ///d/z/j4+BfuH4lE6O/vZ2hoiK6uLvbv3099fT2Li4tbzqXbyurqKtevX2d+fh6dTofdbqerq2vHlYFCCCGEEEIIIYQQu4EEVFv46U9/ygcffKDO+Vj7DWqNRoPJZMLhcHDixAkqKiq4efMmgDrv6bd/+7d5/PHH0Wg0ZLNZvvnNb/LWW2+xuLiotmZaWlqivb0dg8HA559/ztWrVzEYDFRUVPBv/s2/obGxEeCOC13Xrl3j/PnzBAIBLBYLjY2N/N7v/R4lJSVoNBpOnDhBa2srv/RLvwTAhQsXOHbsGK+88sqGRVmTyUR3dzd/8Rd/gdVqRaPR8L//9/9mbGyM+fl5APx+P0tLS3R1dXHkyBG8Xi9vv/02S0tLWK1WDh48yL//9/9+3XHz+Ty5XG5ddYJGo0Gr1aLT6TZd0CxUY42MjHDx4kXGxsbQ6/V4PB7+83/+z5w8eRK41Waxt7eX733ve0SjUYaHh/n444/5zd/8TbVq5fr160QiEQYHBxkfH6eyshKTycTY2BgXL15U3/Pxxx9nz549LCwscOHCBfr7+wHweDy88sor/Nf/+l/VirfOzk7y+Tx//dd/TSgU4oc//CH/8T/+R2pqanbtAq3YWjabZXFxkbm5uQ3z6b5IOp3G7/czMDBAIpGgpaWFzs5OfD4fMzMzOzqXdDpNKpXa9hyrR0UulyMUCrG8vExnZydVVVXber3RaKSmpobGxkY1pEomk/clHN4tQVU6nSYUChEKhaitrd32zCe73U5tbS01NTVcvXqVxcVFamtrH/nZWw+KoijEYjHOnTvHhx9+eMdwCm79mTAzM8PQ0BChUIi27WuIjQAAIABJREFUtjYee+wxfD7ftgMquPW5mJub44MPPqChoYGamhrKysp2cjlCCCGEEEIIIYQQu8LXcyXqLiSTSZLJ5KbbjEYj+/fv57d/+7fZt28fBoOBo0eP8t/+23/jypUrnD59mpaWFhKJBD6fj+vXr/PZZ58xOzurBjSFWVObBTeKomxr4XlkZIRIJAJAcXExTz75JPl8Xq2u0Gq1NDY2YrVa1UXtYDDI0tLShlkrzc3NvPbaa+t+Xl5eTmtrqxpQpVIpNbArzOVZu6h7e4usbDaL3+/n6tWrLC8vq9UKFouF2tpajhw5gsVi2fL6Ci30AMxmMydPnsRutxOLxcjn8yiKQnt7O263G6PRSDqdJhwOMzs7S1VVFY8//jgDAwNcv36dZDLJm2++yb/6V/9KDb/6+vrQ6XQ4HA4OHjxIVVUVAwMDjIyMqOfQ2tpKY2PjumCivLycqqoqLBYL8XhcXVxsb2+/69+d2B0Kn6e+vj5WV1e3HWKsrKwwMDDAlStXOHz4MIcOHeLKlSs7Dqh2s3w+TzKZZGJiArPZjMfj+cLP/2YKAXBzczN9fX3Mzs4yOjrKvn37dnROXxSUP4oKAd/8/DwVFRW4XK5ttxDVaDQ4nU7a29u5evUqw8PDVFdXbzssLNDpdGi12l1zD2+Xz+cJhUJcvnyZqampu37d8vIy169f59q1a3z3u9+lu7ubjz/+GL/fv+NzmZycZHR0lGAwKAGVEEIIIYQQQgghvtIkoNpCbW0t5eXlahWR2WzG7XZTUlJCW1sbPT09tLW14XA4gFuVUw0NDYyMjPCXf/mXBAIB5ufnCYVCRKNRIpEIwWBwXXiz1ayT7c5AmZqaUmdTLS0t8eabb/LZZ5+ps6G0Wi2xWIxkMrluttNmC7J2u53q6up1PzObzep1Fs5v7QL9ndpqnT17lh/+8IecP39ebVVYmPVSXV3NSy+9xA9+8IMNryucWyAQIBgMArfa/X388cf4fL51i9o6nY6RkRE1OFMURb3+J554gjNnzqhzg9544w2+/e1v4/f76evrI5VKYbVaefHFF6msrARuVa1NT0+rxx8cHOR3f/d3+dM//dN15zg+Pr4uyNxNi9zi7mWzWZaXl7l27RrJZHLbAdXq6ipjY2NcuHCBw4cP09zcTGNjoxp4fZ0UWqaOjIxQXl6+rdlJt2toaKCxsZGxsTGuXr1KTU2NGiLf7e/IYDCwtLRENBrFaDTuuE3glymVShEIBAgEAvT09Oy4csxms9HW1kZjYyPDw8NUVVWpIf9m96/ws7V/xmk0Gmw2G0tLSyQSiV1TgXY7RVEIhULMzMxsq3VmIpFgcnKSGzdu8N3vfpeWlhY6OjqYmJhgaWlpR+cSj8c3VG4LIYQQQgghhBBCfBVJQLWFzs5OXn75ZXVOk8FgwGaz4XQ68Xg8VFRUrNt/cnKSd955h3feeQefz4ff7yeTyeByuaiurqajo4OLFy8SCARIp9P39VyTyaQ6RyqXyxGJRPD5fOoiYiGoqaurw2AwUFlZyeHDh7Hb7RsGuRsMBux2+7qf6XS6e2r7FAqFGBoaUtsgrlVo8fVFA+XT6bR6zxRFYXV1lcnJSfW6CjweD5WVlbhcLnX+Ftyqsti7dy9nzpwhGAyq35C/du0afX19apj1ne98R/22ejabXRc8pdNpNXBcS6vV0tLSgt1up7KyktraWoxG465dpBWby2azhEIhrly5sq32fmtfv7CwwJUrV5ienqa8vJx9+/Zx7do1BgcHt3283fx85XI5VlZWGB4e5vDhw/dUIeJyuXA4HMzNzTE3N7cuXNlOQHXz5k1mZmZwOp27ImBeG1A9/fTTWCyWHZ230WikuLgYq9WqzhmcnZ3dUNX7RTQaDRaLhevXrwO3fr+74R6ulc/nSaVSTExMEAqFthUMKYpCMBikv7+fwcFBKioq6Onp4fr16zsOqAqV1bv5cy6EEEIIIYQQQghxNySg+rnbF4JaW1s5duwYXV1dd3xtNpvl3Llz/J//83/49NNP0Wq1dHZ2snfvXlpbW2lqaqK9vZ0f/OAHBINB0un0ugW8fD6vhi2Fn689n9u/ta7VatdtdzgcmM1m4NY34ru6ujh16pTa8imfz6PRaDAYDKyurtLc3Mxjjz2G1WpVWwMWqgb0ev2GxcVMJrMuoNLr9esqlda+vlAZtVZdXR3PPvssBoNBvfbCP/X19Rw5cgSdTkcul1tX9VU4vtlsVuerGAwGGhsbefLJJyktLV13HzQaDfF4nKqqKnp6etR7YrVa6erqor29nY8++ohsNsuZM2e4cOECfr8fs9lMVVUVvb29OJ1O4FYoV1xcrB7b4/Hw+OOP09nZuaF6LJ1Oo9Pp6O7uprKyUp07tjZA0+l0u3ZekLgVUC4vLzM0NLRldcndHGNxcRG/38++ffs4cOAAPp+PfD6vVhVu9vkp0Gq15HI5wuEwyWRS/czsNul0mkgkwtzcHOXl5Xi93g37pFIpEokE+Xx+3eewQFEUcrkcU1NTzM7OkkgkiMfjDAwMoNfrN13c3+r+6nQ6pqenicfj265efVji8TjLy8tEIhG6u7vVP+sKFEUhmUyyurqKxWLZ0MoVbv29sLKygs/nY3FxkVQqhd/vV6vINrsXhT/T1j53hZmMi4uLFBUVbaiw3Q0KoWl/f/+OZkfF43FGRkY4e/YsL774Iu3t7TQ1NdHf37/jL6Ts1s+3EEIIIYQQQgghxHZIQLUFRVHu+lvgy8vL9PX1qeGUTqfj13/91/nFX/xFysrKyOVyamu7woKToijqQvfa91EUhUwm84ULfLe/pr6+Xq16MplMNDY28u/+3b+juLhY3S+TyRAOh/n0009JpVIbru32oGetzfYt/Kyw6FuQz+c3LGw+9thjtLS08PLLLxMOh9X3stlslJWVqW31NBrNptddUVGhLmLrdDrKysr45V/+ZQ4ePLhuv6WlJQYHB1lYWNhwzvv27ePQoUN88skn5HI53njjDaampggGg1RXV/PUU09ht9vVUM/pdKrVc3BrttepU6d4/fXX14V10WiUiYkJhoeH11337deynedJPFoURSEejzM/P8/y8vKOfo96vR6Hw0FJSQn5fB6LxUJXVxfZbJaWlhb0er1a9fNFLeYSiQRXr17ls88+2xWt6G5XCE4WFxfJ5/OUlJSo4fPafYLBID6fj0wmw8mTJ7c8xs9+9jOmpqbU2UklJSVYLJZNZyFt1p5OURT18zw5Obkh/H8UFeahhcNhdDodDQ0NG56FTCbDzMwMIyMj1NXV0dHRsW67oiisrKwwODjIe++9RywW47HHHlPnWZnN5k3v39p7uPbvALPZzMzMjBq27jbZbJZIJEJfX9+22vsVJJNJZmZmOH/+PCdOnKCqqorW1lZKS0vXtYrdDvk7QwghhBBCCCGEEF8HElD93O0LQWurmu4kGAyqrXy0Wi1er5ejR4+qravS6TQ//elPmZ2dJZVKAbcWmgOBAIqiYDAY1IW/bDZLOBzGaDSqxy/878I+ty+g7t27V638WV5e5oMPPmBmZoaioiJ18XVlZYU/+qM/4o//+I8Jh8P81m/9Fv/zf/7PdZVKcOub5LcvMBoMhnXf5M7lcupxC5VZhdcnk8l1rfEKHA4He/fu/cL7uPZb+2tnZTU3N6v3MplM8tFHHzE6OsrevXvXzaH60Y9+xB/90R8xPDzMc889xzvvvKNua25upre3F7vdTiQS4bPPPlO3eb1eXnvttXX33Ov1smfPHvW/h4aGuHr1Ki+//PK6io9Lly7x+7//++p7ff755+zbtw+dTrdh3tjdPk/i0ZLL5VhaWmJkZGTH4UVxcTFHjx7l9ddf59ChQzgcDoqLi6moqOCFF1646+NEIhH+4R/+gampKWDjn1uPukKrRJ/PR1VVFS6Xa8PnIp/PMzY2xk9/+lPMZvOGgCqfzxMKhXjnnXd4//33OXz4MC+99BJtbW07Pq+ysjI+/PDDdX+WPapSqRRzc3PEYjFqamrUcLNAURRSqRRXrlzhZz/7Gd/85jc3BFS5XI6JiQl+9rOfceHCBV5//XVOnTpFSUnJjs9rZWVFrWDbbfL5POFwWJ1TuF2KohCNRrl8+TKDg4Ps37+fAwcOcOHChR0FVIqi7NqwTwghhBBCCCGEEGI7dt9K0pfobhd/S0pK8Hg8aDQacrkcCwsL/MEf/AFPPvkkRqORa9eu8eabbxIIBNTXxONxhoaGOHnyJA6HQw1astksKysr/Kf/9J8oKirC6/XS1dW1YYFxrba2Np577jmmp6cZGRlhYWGB3/iN3+D06dM0NzezsLDA+fPnOXv2rLqoWVlZedeLX1+0YKvX6/F6veoicyqV4uzZs/zar/0a6XSapqYmXnrppQ3VTnf7XoqiUFdXx5NPPsnly5f59NNPyWQy/I//8T/453/+Z/bt20c+n1db9k1PT+PxeGhoaNhw7Pr6ep577jl+9KMfqT+zWCw0Nzdz+PBhtXoKwO1209vby+nTp7l8+TLRaJS///u/Z2FhgZMnT+J2u7l48SLnzp1jeHgYq9VKQ0PDusBMfDVkMhnm5+cZGhra0euLioo4fvw4v/zLv8w3vvENLBbLjhfxM5mM2oruUQ9SNpPJZNTqqK6uLhwOx4Z9IpGIuqi/toqxIBqNMjAwwIcffsihQ4d45plnaGhouKeKk3Q6TTabXfdnwKMqkUgwPT1NOp2mvb19w/ZCtezExARerxePx7Nhn0AgwJUrVwgEAnz7299eN7Nvp9Lp9K6dm5RKpVhYWGBpaWlb86fW0mq1GI1GEokERqOR7u5uvv/971NfX7/uuSpU6W3VgrKvrw+/378rqvmEEEIIIYQQQggh7pUEVD+n1+uxWq3q/Amr1XrXYYPL5eLQoUMcO3aMc+fOkcvl+OCDD7hx4wZarZZwOEwikeDYsWMMDg4yPz9PLBZjdHSUXC6Hy+WiqqqK4uJilpeXAfjoo4/Q6XQ4HA5WVlZ46qmn1MVXvV6/rtrHbDbzzW9+k2g0yg9/+EPGxsb49NNPmZubw+VykU6nmZqaIhqN4vF4eO2113jqqafU15eUlKgLaDqdbsN1W63WdW24jEajusBuMBjwer1UV1fj9/tJJpPMzc3x9ttvoygKZWVltLa23lVAVVRUhNVqXfe+Wq0WvV7P0aNH+ZVf+RVSqRT9/f309/eztLTERx99hKIoTE1NEYlEKCoq4umnn+a1117bcPy6uroNAVVDQwPHjx/HZDKt21en01FTU8N/+S//hT/8wz/k3Llz6j3s7+/HZDIxNzfH4uIiGo2Grq4ufuu3fkutQLj9nplMpl1ZWSBuLfgHAgGGh4eBLw5sN1NWVkZvby9Hjx5VKx3vRaG6YjdKp9MsLS0xPj7OCy+8sOn9WFlZYXJyklwuR3l5+Ybt0WgUv99POBzmyJEjNDY2bvj87sRWFaqPmkQiwczMDJlMRg2o1gZzhSrcsbExmpqaNg2eFhYWmJqawmQycfr0aTwezz1X62zWQnE3yOVyhEIhRkZG1Lln2+VwOOjo6ODll1+mtbUVu91OUVERzz33HIcOHVp3b3O5HBqNZtP7nUgkePvtt3n33XelekoIIYQQQgghhBBfC7Ji/nNVVVU8/fTTXLhwgfLycrq6uiguLr6r1+r1eg4dOsSv/uqv4nQ6mZ6eVkMpt9vN/v376e3tpbm5mQ8//JBz587hdrtpbW1Vv3V94sQJpqenuXr1qlppZTabqauro62tTW0RNj4+TkNDA11dXevOoaWlhV/4hV/A6XTyySefEAgECIVCLC0tYbFYaG9vx+12c+jQIV555ZV1C5tPPvkkExMTOJ1OOjs7qaioWHfs8vJyenp61NCss7NTDWIKi2i/8Au/gN1uZ2hoiFgshl6vx263s2fPHkpLS+/qPnZ2dtLV1UUmk6GxsZG2tjZ14bm6upqXXnoJk8nE+++/z8zMDEtLS4RCITQaDTU1NfT29tLW1sYLL7zA4cOHNxzf5XJx5MgRvvnNbzI6OgrA008/vaGFWIHNZuMb3/gG0WiU5uZmbty4wdLSEpFIhFAohNPppKGhgT179tDb28tLL72E2WwGbj1PBw8eZHh4GKPRSHt7+10/T+LRUZg/FQgEmJub23K/reanaTQaSkpKqK+vv+vPwVdZIpFgeXmZSCRCY2MjNpttwz6RSITJyUk0Gs2GAEtRFGKxGMFgEJvNRl1dnTp/717kcjk1ONDr9Y90yLKyskIwGESv11NXV7dheyaTIRQKMTY2xp49ezZ84SCbzRIMBlldXcXlctHY2Hhfziubzapt6XZTuJLJZFhaWqKvr29H1VNms5nW1lZefvllXn31VcrKyrBYLOqXHDarAtxKKpXixo0bWK3WR/oZFEIIIYQQQgghhLhfJKD6udbWVn7jN36D+vp69uzZw9GjRzdtP7WVmpoavvvd79LQ0MDFixeJRqPYbDaqq6tpaWnhyJEjaDQa6uvr6erqQq/X89xzz6mVUMeOHcPr9fLP//zPjI6OotVqKSoqorOzkwMHDqDT6fi3//bfMjAwQGNjI8eOHVv3/kajkX379lFfX88TTzzB8PAws7OzRKNRjEYjlZWVNDU1cfDgwXXXZTabOX36NMlkkmg0yqFDhzYNqE6dOkU4HMZgMHDy5El1JlTB97//fZqamrh27RoLCwtoNBqKi4s5ceLEpm2oNtPT08N3vvMdWltbaWpqor29XV3012q11NXV8a//9b/m0KFDjIyMMDU1RTgcRqPRUFpaSmNjIz09PRvOv8BgMFBdXc3v/M7vcObMGbUyq7Ozc9P9C4vVr7zyCt3d3YyMjODz+ZifnyedTlNSUkJdXR179uyho6Nj3aJsTU0NL774ItlsluLiYo4dO4bb7b6r+yAeHblcjsXFRebm5kgkEhu2F54Rs9lMPB5fN6sNblXiOZ1ObDbbfVm0VxSFdDqtHns3BQGFSpVwOExxcTHFxcXrKkELFhYWmJ2dpaioaMN8qlwuRyKRIJ1O43K5MBgM92UhPx6Pk0gk8Hq9WCyWRzYcyGazBAIBstkspaWlm1ZHJZNJZmdnmZ2dJZvNbriWTCbD6uoqWq12W3/H3cnKygr5fB6TyXRfKtq+LJlMhsXFRfr7+3fUorC4uJje3l61eupe5HI5MpmMWsX1qFfzCSGEEEIIIYQQQtwrCah+zuPx8Mwzz/DMM8/s+Bgul4unnnpqXfu82x0+fHjT6h6LxUJ3dzfd3d1bvvY73/nOHc/B6XRu+R6bKXwL/wc/+MGW+xiNRrq6ujZUbd3+vqdPn+b06dN39b6b8Xg8vPrqq7z66qvqz25fXDUYDPT09NDT07Oj97Db7Rw9epSjR49u63WNjY3bqjSw2Wz09vbS29u73VMUj5BUKoXf72dycnLT1l86nQ6r1UpFRQXT09Nqi9C1281m830LPDKZDJFIBI1Gg9ls3lUBVTKZZH5+nkgkQkdHB0ajcd19yefzZLNZxsfHiUQi2Gy2DQv0hQV8rVZ73+a9pVIplpaWSKVSOBwOHA7HIxkMFKr5/H4/FouFmpqaDW1Ds9ksoVCI0dFRkskkuVxuw3ObzWbJZrP3NAttrcI8tPn5efR6PTabbVfN4ksmkwSDQYaHh8lms9t+fVVVFXv37mXv3r33fC6F2VSF1ra3B95CCCGEEEIIIYQQXzW7Z3VTCCG+ZMlkEr/fz8TExKYBlV6vx+Vy0dnZuWU1SmHR+V7lcjlisRjDw8OYTCY8Hs99rYB50NYGVD09PRvCkVwuRyQSYWhoiEgksmX4Vgi17leIdOPGDUZGRjCZTGoI/ShWUCmKwurqKn6/X23/evt5plIp5ufnuXnzJvl8ftPr0Gg09/X64vE4H3zwAYuLi5SWllJZWbmh8u1Rlc/nWVlZUZ/LzZ4prVaLTqfb9J7p9XqKiopwuVz37Z6uPYdH8TkUQgghhBBCCCGEuJ8koBJCiC2kUilmZmaYmprasoLK7XbT2dm56TwluLUIfj8W7BcXF7l06RIXLlygqqqKlpYWvF7vPR/3y5JMJllYWGB1dVVtc7p2AT6XyxEOhxkdHSWTyWC1Wjc9TmHel16v33EFWT6fJ5FIcOXKFX76058Sj8fZt28fbW1tOzrel6FQQTUzM4PFYqG2tnbTgGphYYHBwUEcDgcGg2HDPSoEVLlc7p6ey3w+TyAQ4JNPPuFHP/qROrtxOzOXHrZMJkMgEMDn86mVYGtptVrMZjOlpaWbVoXpdDq1xef9srq6SjabVedTCiGEEEIIIYQQQnyVSYs/IYTYhKIoRCIR5ufnCYfDm+5jMBjweDw0NzdvGagU2nXt1Nog4J133iGZTHLgwIEvfM9HTT6fJxKJEA6HMRqNVFdXbzpfamFhgcXFRex2Oy6Xa0NgoNFoyOfzxGIxfD4fly5dori4WK1SK7RE26zipTBfSKPRkEwmWV5eZnh4mHA4TEdHBwcPHsTj8TzYG7FDiqKo86fi8ThFRUUUFxdvuMZkMsni4iJLS0vs3bsXk8m0aVVQLpdjaWmJWCzGhQsX1LlmgDq36vbfz9r7WwjLZmdnGR8fR6fTceTIEdrb27Hb7Q/gDjwYhQDa5/Ntur0wQ66xsZHR0VHi8fi67YWw73602szn8+r8MK1Wi9Pp3DWfbyGEEEIIIYQQQoidkoBKCCFuUwgEpqamWFxcJJPJbNhHo9Fgs9mora3F6/ViMBg27JPP54lGo/j9fvr7+9UZN5sFKIUKDq1Wq24rBDvXr1/n7NmzjI6O8uSTT3L8+HEqKip2TQuwdDrN7OwsqVSK8vJyrFbrukX9fD5PPB5nampKrUrbbBaURqPBZDJhs9mIRqNcvHgRi8WitrMrBIHZbHbDa3U6HVqtllQqRTKZJB6PYzAY2L9/P729vdTX19+XmUwPQuH++Hw+7HY7Xq93Q9VOoUViKBTCarVSXl6OwWDY9B7a7XZ0Oh0zMzN8+OGHZDIZdT+9Xr8ujFr7ukLVWyKRUCt9XC4Xzz//PAcPHsTr9e6quWiF53JiYmLT7YUWnnv27GFxcZFAIPDAziWRSDAyMkJ/fz9Op5Pa2lpcLtcDez8hhBBCCCGEEEKIR8GjuRonhBAPkaIoJJNJhoeHmZ+f33QfvV6Px+Ohra1ty1ZcuVyOsbEx3n77bW7evKmGAJvNpSpUYxRaCWo0GjKZDMFgkKmpKbLZLJ2dnfz6r/86+/bt27Kl4KMoHo/j9/sBaGxs3BBiZLNZwuEwY2NjlJeXY7PZ1KBkLa1Wi9frpauri1gsht1uXxfSFfbfLCwsbDObzTgcDlwuF62trbS1teHxeB7puUm5XI5oNMrg4CBlZWWUl5dvuIfJZJJAIEA0GqWlpQWbzbZpWKTX66murubAgQNMTEyg0WjW3a9Cldlm96Pw3FosFoqLiykvL6e1tZW9e/du2k7wUVaoApubm2Nubm7TfXQ6HS6Xi+bmZm7cuLHpPvc600tRFNLpNBMTE7z77ruMjY3x/PPP09HRgdvt3vFxhRBCCCGEEEIIIXYDCaiEEOI2iqKQSqXUgGqz+VNGo5HS0lL27t275QJ1Pp9nYmJiywqNu+F2u+nt7eX555/nqaeeoqura8fHelgSiQQTExMoikJzczPAunuWTqcJBoP4fD4aGxtJJpNbzvwqLy/nqaee4siRIzs6F41Gg9FoxOl07poKtEJANTAwwDPPPEN5efmGc4/H48zPzxOPx+nu7v7CtpStra3U1NSQSqV2dD6F6sHd3IIun8+zsLBAIBAgmUxuuo/BYKC8vJzq6uotr1Wn093TrKh8Ps/g4CB/+7d/y5//+Z9TUlLC8ePHaWlpeaRDUyGEEEIIIYQQQoj7QQIqIYS4zdqWc6FQaNN9jEYjJSUltLW1MT09fcdj7iQMURSF7u5uvve973Hq1KlHdkbSnUSjUYLBICUlJdTU1Gy4F6lUSg2oent7mZ2d3TSggluBgMViwWQybdhWqJLa6l4X3iMSidDS0kJDQ8OuCAGy2aw6D62srAyPx7PhGldXVwkEAiQSCR5//HHOnDkDsGmlnl6v31HAlM1mmZ+fZ3h4GKvVSlNTE5WVlfd2cQ9JIpFgdHSUmZmZTed0Fdr7FWZ5bVYdls1mGR8f580332RyclJtLVlo1bmZQjtKrVZLJpNhYWGBwcFB/H4/DQ0N/NIv/RKHDx/G6XTe92sWQgghhBBCCCGEeNRIQCWEEGsoikImk2F6eppgMLhplUlhjk9JSQmlpaVMT0/fMYDabBH8bpSWllJTU0NlZeWuCFM2E41GSafTauXS7eLxOMFgkEwmQ2VlJSsrK6TT6U2PVWg/t5N7kUgkGB8fZ35+nqKiIurq6nbFPc1kMqysrJDNZrHb7Vit1nXPWz6fZ3l5mVgshsFgoLGxkU8++QTY/LnTarU7bse3srLCwMAALpeL4uLiXRtQFWZ6zc7ObnqPjEYjbrebrq4uDAbDpp/vXC6H3+9X56EV2iOubdV5O61Wq75fNpvFYDDgcrk4cOAAR44c4eTJk1RUVGzaplIIIYQQQgghhBDiq0YCKiGEWCOfz5NKpRgcHCQcDm+60KzX63G73VRXV2OxWNBqtVsuSN8rrVaL2WzeFUHKVgrztDabvZXJZFheXmZmZoaamho8Hg96vX7Ltmv3ohD0hEKhLQOwR5FGo0FRFHK53KbbU6kUc3NzRKNRKioqKCoqAvjCSp6dKMxmC4fDaDSaLc9nN0gkEvj9fubm5rasoPJ4POzbtw+/3082m92wj6IoRKNRotHojs+jo6ODI0eO8Pzzz7Nnzx5KSkp29WddCCGEEEIIIYQQYjt2z0RzIYT4EuTzebX91+rq6pbVFVVVVTQ2NqrhwYOyWaiz2zgcDux2O+FwGJ/Pp4ZV+XyepaUlhoeHmZ2d5fjx4w90rlGhtZpOp7uvwc2DVqiysdlsTE5OMj8/v6725EfJAAAgAElEQVSd3MzMDP39/QDs2bMH2FlLyTtZe/90Ot2umeG1meXlZebm5lhZWdl0e6GFZ1NTEwaD4YF9BhsaGjhx4gTHjx+nvLxcwikhhBBCCCGEEEJ8reyeFTohhPiS5HI5wuHwllU2JpOJqqoqGhoaHnh4tNvDKQCPx0NzczOBQIC33nqLxcVFEokEsViMGzducO7cOcLhMC+88AIWi+Urcc33k9FopKysjL179/LJJ59w7tw5QqEQiUSCaDTK2bNnuXr1Kna7nZ6eHvW5fRD3sXDM3fo7KrTw9Pv9BIPBTSujAGw2GxUVFRiNxgcaZlqtVmw2G3q9FLQLIYQQQgghhBDi60dWRIQQYg2NRoPJZKK+vn7LsMRisVBTU0NDQ8NDOMPdp6ioiOeeew6NRsP58+f5D//hP+B2u0kmkyQSCcrKyvjWt75FfX094+PjZLNZtULofiq0yctms7uqPZ1er6e8vJzXX3+dH//4x/z4xz/m7bffxmq1EovFSKVSdHZ28tRTT1FWVobf7yeTyZDL5R7IPXxQv58vQ6FCstDCczNGo5GKigpaWlq+lPN5UO1BhRBCCCGEEEIIIR51ElAJIcQaOp0Om83GsWPHOHv2LPPz8+tmzFitVg4dOsRjjz2G3W7/Us5pN7dSg1sBS11dHadPn6a0tJTJyUlSqRSKouDxeGhra6OrqwutVovNZqOxsZFkMklxcfF9PQ+73U59fT0OhwOv17tr2vwV5pB1dHTwrW99i4GBAZaWltR7WFdXR3d3Ny0tLRiNRpxOJ52dndTV1WGz2e7beWg0GtxuN62trdjtdhwOx3079pelEFIuLy9vOefMbDZTWVlJc3Pzl3x2QgghhBBCCCGEEF8vElAJIcQahQqq9vZ2vvWtb2Gz2RgcHGR1dRWj0UhXVxfPP/88+/btw2AwrJun9CB8FSosNBoNVquV5uZmPB6PGvpZLBbcbjder5eioiLg1ryqrq4ucrkcpaWl9/U8nE4nbW1tJBIJSktLd1Xwp9VqcTqd9PT0UFNTowZUZrOZkpISvF4vVquVbDaL2+3m2LFjFBUV4XQ67+s5eL1e9u/fj9FovO8B4pdBo9Gg0+mw2+1bttUzm81UVVVJQCWEEEIIIYQQQgjxgElAJYQQt9Hr9bhcLp599llqamro6+sjGAxit9s5cOAA7e3tlJWVqftbLBZqa2uJRCLo9XoURdnQ/kyj0aiByBdtXxtGZbNZKisrMRqND/Bqvzxms5mKigoqKirIZrPodLoNIZHFYqGurg5gywCh0GYuEokQj8fVSiKLxaLO9NlsdpDFYqG6upp8Po9er981FVRrORwOHA4HdXV15PN5dDrduu06nQ6Hw0FbWxs6ne6+zjYqhGRWqxWtVrvhvXeDQgDd1tZGdXU1fr+fVCqlbjcYDNTV1bFnzx71M64oygN7VnQ63a5qNymEEEIIIYQQQghxP0lAJYQQm9DpdFRXV+PxeOjp6SEej2MymfB4PBgMBjVY0el0lJeX88ILL9DT06MuZK8No77I7fut/e90Os3+/fupqKh4INf4MG0VnBTa2W1GURRSqRSRSITFxUUGBgaYmpoiGAyiKApVVVVUV1dTVVVFSUkJDocDk8mEwWBAp9Pt2lBlM4VKoK1+brVaH8j76vX6+xp6fdm0Wi0mk4kDBw5w5MgRFhcXmZ6eJpPJqJ/lJ554gt7e3i2fQyGEEEIIIYQQQghxf+zeVSYhhPgSWCwWLBbLltu1Wi0VFRX84i/+Itls9r62jcvn85jN5i98/68TjUbD5OQkb775Jn/7t3/L8vIywWCQaDSKRqOhuLiY4uJiSktLaWpqoru7m/3791NXV4fb7d6VM5PEg1FXV8ev/Mqv0NzczLlz55iamsLr9fLEE09w4sQJmpqa1H3z+TzpdPqBnEcymdxVrSaFEEIIIYQQQggh7ieNcqev9wshhPhCiqKobboURdlywfn2KqnNtt++TaPRoNVqv9aL2KlUiosXL3LmzBkuXbrE6Ogos7OzNDQ00NHRQVlZGaurqwwNDTE+Pk4ikUCj0eB2u6mqqqK4uBi3201NTQ3t7e00Nzfj9XpxuVxYLJavTFWVuHuKopBMJlleXiYQCBCJRLDZbOrzYjKZ1GrI/v5+/uZv/oaBgYF11ZOb/d+nO1VOrt2eyWQ4evQozzzzDB0dHQ/iMoUQQgghhBBCCCEeaRJQCSGEeOTk83nm5+f59NNP6evr47PPPqO/v5/V1VVKSkpob2+nt7eXtrY2vF4v8Xgcn8/HzMwMoVCIpaUl5ubmmJ2dZWZmBp1Oh9PppKqqitraWjweD16vl+rqahobG6msrMTpdOJ0OjEYDA/78sWXJJfLkclk1JloRqNxQ2AZDocZGxsjHA6j1WrVOXGbzaW6mxC6sD2fz1NWVkZ1dTUul+t+XpYQQgghhBBCCCHEriABlRBCiEdGNpvF7/czNjbG559/zscff8zg4CAAXq+XPXv20NXVRW9vL83NzbjdboxGI7lcjpWVFeLxOKlUilAoxMTEBAMDAwwNDRGPx4nFYkQiEWKxGNFoFLPZTFlZGa2trTQ2NuJ2uykrK6OsrIyKigo8Hg92ux2j0fiQ74p4mPL5PJlMhnw+v6Ey6vYgajsBFdyaYafX6zcNu4QQQgghhBBCCCG+6iSgEkII8VDlcjkSiQShUIjp6WnOnj3LmTNnGB4eZnV1Fbfbzb59+zhx4gSPPfYYtbW1FBcXr2u3ttkx4/E4kUiESCRCNBplYWGBkZERrl+/zvDwMPF4XJ0tpNVqURQFm81GfX09XV1d7Nmzh/LycoqKirDZbLhcLmw2m1pl83VuuyiEEEIIIYQQQgghxL2SgEoIIcRDUZjdtbKywujoKO+++y7vv/8+IyMjRCIRiouL6e3t5dVXX6Wnp4eamhrsdvuO3y+XyxGLxVheXiYUCrGyssL4+DifffYZFy5cUAMxnU5HUVERRUVFOJ1OSkpKaGlp4ejRo3R0dFBRUYHD4cBoNMqMMCGEEEIIIYQQQgghdkgCKiGEEF86RVEIhUJ89tln/NM//RMff/wxk5OTxONx6urqOHXqFE899RS1tbXU1NRgsVgwGo331ApNURTy+Ty5XI5cLoeiKCQSCZaXl1lcXFRDK5/Px9WrV7lx4wZzc3PkcjlMJhNlZWV4PB5cLhdlZWU0NTVx9OhRGhsbKSkpwWKxSFAlhBBCCCGEEEIIIcRdkoBKCCHEl0JRFFKpFMFgkCtXrnD+/Hk+//xzfD4fKysrVFdXc+rUKTo6Oujs7KShoQGz2YzJZHpgM3oK84XS6bQ6Z2h5eZnp6Wl8Ph9zc3MkEgmCwSAjIyP4/X6i0Sh6vZ6ioiI1nPJ4PJSWllJbW8vevXupq6vDbrevO28Jr4QQQgghhBBCCCGE+BcSUAkhhHjgEokEfr+fzz//nJs3b3L16lUGBwfRaDTU1dXR3t7O3r17OX78uDr3yWg0PpRzzeVypFIpVldXicViapXV6Ogog4ODBAIBEokEkUiEQCBAKBQilUrhcDioqKigo6OD+vp6iouLcTgcVFZW0tTURElJCQaD4aFckxBCCCGEEEIIIYQQjxoJqIQQQtx3iqKQzWaJRqPMz8/j8/m4dOkS586dw+fzYTQaKS0tpauri+PHj3Pw4EFKSkpwOp3o9fqHffobpNNpVldXCYVCRKNR4vE48/PzXLlyhaGhIbXSKpPJkEqlyOfzGI1GSkpKqK+v57HHHqO+vh6n04nNZsPlcuH1enE4HOh0OkAqrIQQQgghhBBCCCHE14sEVEIIIe6bQjAVj8dZWlri5s2bnDlzhgsXLuD3+8lms3i9Xk6cOMGzzz5LV1cXZWVl2O32h33q25LP50mlUmoV1cLCAgsLCwQCAS5cuMDQ0BDLy8vk83m0Wi0mkwmTyYTD4aCqqoquri56e3tpaWnBZDJhMBiwWCzqrK0CCa2EEEIIIYQQQgghxFeVBFRCCCHuq9nZWS5dusR7773Hu+++y/z8PKlUipaWFl544QVeeeUVdXbTw2rjd7/lcjm1eioSiTA9Pc3MzAxzc3OMj4/z/vvvMzIyQjKZBMBut1NaWkpDQwMmk4na2loOHz7MiRMnaGpqeshXI4QQQgghhBBCCCHEgycBlRBCiHuSz+dJp9P4/X4uX77M2bNnuXbtGnNzc4TDYR577DGeffZZ9u/fT1VVFTU1NZhMJoxGI1qt9mGf/n2hKAqKopDP59UZVslkkng8TiwWY35+nqmpKRYWFlheXmZ6epq+vj4WFxfJZDKYTCa8Xi9VVVXU1tai0+moqalh3759dHZ2Ul5ejqIoaLVaNBrNV+a+CSGEEEIIIYQQQoivLwmohBBCbJuiKORyOaLRKD6fj8uXL/P5558zMDCA3+9Hq9XS1NTEE088QWtrKz09PVRWVmIwGDCZTA/79L8U+XxeDawymQzxeJxoNEosFmNhYYGxsTEmJiaYnp4mHo8TDoeZmZkhHo+j0+lwu93U1tbS2NhIdXU1Wq2W6upqmpubqa2txel0otFo1MBK2gEKIYQQQgghhBBCiN1EAiohhBB3rVAltLy8jM/nY3BwkCtXrnD+/HlmZmYoLi6mtraWzs5Ouru7efbZZzGZTFgsFgwGw8M+/Ycul8uRz+fJZDLEYjHC4TBzc3NEo1ECgQADAwOMjY0RiUTIZrOsrKywurqKVqtFq9VSU1NDR0cHra2tVFZWYrVacTqdVFZW4vF4MJvN6ntJaCWEEEIIIYQQQgghHmUSUAkhhLgjRVFIp9NEIhGmpqYYHh7m7NmzXLp0iUAggF6vp6KigieeeIKTJ0/S1tZGUVERRUVFD/vUH2mF+5rL5Uin06ysrBAIBBgfHycajTIxMcH169eZnJwkHo+j0WhQFAWj0YjT6aSoqIiKigp6enpoa2vD4/FgMBiwWCx4PB5sNht6vV59v0LFlRBCCCGEEEJsR3Z24GGfghBC7Dr6yvaHfQqPPAmohBBCfKFCeLKwsMClS5f44z/+Y6ampggGg+RyOZqamvj2t7/Nq6++SmVlJUVFRetCEbF9qVRKnWUVDAbx+/0MDw8TiUS4evUqV65cYX5+nkwmg16vx263U1JSQkVFBWVlZdTX1/Pkk0/S3t6utgLU6/UYDAZ0Op2EVEIIIYQQQohtkYBKCCG2TwKqO5OASgghxKZyuRy5XI6pqSl+8pOf8I//+I/4fD78fj9FRUX09PRw6tQpHn/8caqrqyktLUWn00lruftAURQKfz0XZljFYjG17d/s7CwTExMMDQ0xPz/P8PAwfr+fWCyGTqfDbDZTUlJCVVUV5eXllJaW0tDQwPHjx6mrq8NqtQK3Kqp0Ot3DvFQhhBBCCCHELiABlRBCbJ8EVHcmAZUQQgiVoijkcjnC4TADAwO8//77jIyM0N/fTyAQwG6309bWxtNPP01bWxuNjY2Ul5djMpmkauoBKsz+Kvx+kskksViM5eVlYrEYwWCQmZkZJicnmZ6eZnl5mampKUKhELlcDrPZjNPppK6ujrq6Oqqrq/F4PHg8Hvbv309xcTFGoxG4NbtKQishhBBCCCHEWhJQCSHE9klAdWcSUAkhhFADkIWFBQYGBrhy5Qo3btzg8uXLRKNRnE4nbW1tHDp0iM7OTvbv34/b7cZsNkuY8ZAoikI2myWfz5PNZlldXWVpaYm5uTnC4TDT09NMTk4yNTXF/Pw8wWCQVCqFVqvFbDarM8I6Ojqor6/H6/XicDjwer20trZitVrVajiZXSWEEEIIIcTXmwRUQgixfRJQ3ZkEVEII8TVVCKXS6TShUAifz8f169c5f/48V69eZWVlBY/HQ0NDA11dXRw+fJiDBw/icDgwm81otdqHfQlijcLvM5PJkM1mSaVSalDl8/mYmpoiEAgwMzNDIBAgGo2yuroKgMvlwu124/V6KS8v58CBA1RUVOByubDb7TidTlwuFyaTSUIrIYQQQgghvoYkoBJCiO2TgOrOJKASQoivmcJ8o2Qyqc4zunHjBn/3d39Hf38/sVgMq9VKbW0tL774IidPnqSurg6bzYbJZJJQYhcpVFmlUimy2SzJZJK5uTlGRkYYHBxkfHycqakp5ubmiEajZLNZFEXBbDZTU1NDXV0dtbW1NDY20tbWRklJCWazGbPZjMlkwmKxqK0dC8+FPB9CCCGEEEJ89UhAJYQQ2ycB1Z1JQCWEEF8ziqKQTqcZHx/nJz/5CX/5l3/J2NgYyWQSq9XK4cOHeeWVV3jmmWfweDzYbDZp4/cVUqiaSyaT6r99Ph83btzgxo0bjI2NMTY2xuLiIrlcDp1Oh9lspri4mLa2Npqbm2lqaqKjo4OOjg6Ki4vVaiqdTicBlRBCCCGEEF9BElAJIcT2SUB1ZxJQCSHE10Ch9Vs8HmdsbIw33niDK1euMDk5SSgUwuPxcOjQIY4dO0ZnZyc1NTWUlJSg1+vVlm7iq6FQQVdoCagoCvF4nJWVFSKRCNFolFgsxvj4ODdu3GB0dJTFxUVWV1dJJBJqYFVUVER5eTmNjY1UVlZSXV1NU1MTtbW12O12dDodWq1W/bcQQgghhBBi95KASgghtk8CqjuTgEoIIb7CCsHUwsIC165d4+zZs+qsqdXVVaqqqti7dy+PP/44ra2tNDQ04PF4MBqNEip8jRTCqnw+Ty6XI5/PEw6H1XlVkUiESCSC3+9naGiImZkZVlZWSKfTaLVaioqKKCoqorS0lNraWmpra2loaKCkpITy8nJcLhcWiwWtVqv+I4QQQgghhNg9JKASQojtk4DqziSgEkKIr5i1YcPU1BT9/f18/vnnfPrpp1y+fBlFUSgrK2Pv3r0cPHiQnp4eenp6sFgsGAwGqZYSwK3nKJfLkc1myeVyJJNJAoEAw8PDTE9PEwwGWVpaYnp6munpaVZWVkgmk+j1etxuN7W1tXi9XiorK6mqqqKiooKKigocDgfFxcXrqqw0Go2EVkIIIYQQQjzCJKASQojtk4DqziSgEkKIrwhFUchms8TjcRYXF5mdneXSpUt88MEH9PX1oSgKJSUltLa20tvby9GjR2lpacHhcEgwJe5KLpcjnU6TzWbJZDLEYjGmp6e5fPky09PTLC0tsbCwQCAQIBwOk0wmAXC73dTU1NDS0kJpaSl1dXVUVlZSXFyM0+nEYrHgcDgwmUxqYFX4RwghhBBCCPHwSUAlhBDbJwHVnUlAJYQQXwGKopBOpwmHw4yMjPDhhx/yV3/1V8zNzZHP5/F6vRw8eJBXX32VEydO4HK51DBAiHuhKAqpVEoNr4LBIIODg/zsZz+jv7+fmZkZIpEIq6urpFIpNBoNVqtVbS/Z09NDVVUVbW1tlJWVYbPZMJvN6HQ6dDqdBFVCCCGEEEL8f/bOOzyqauvD77SUmWTSK+khgVBCS6SDIKJYwIuIoOKH/V7bVbn2ir2hXgsqYkUELICFIl16h0BIQnrvPZm0ycx8f+RmkpMzCQGCgO73eXgezt77rL1nJWcys397rXUBIAQqgUAgOH2EQHVqhEAlEAgEFzGtwlRFRQVxcXGsX7+eHTt2kJ+fT2NjI35+fowbN46JEycSHR2Nt7c3Op1ObPwLegyLxSL519zcTGNjI9XV1TQ2NmIwGCguLiYxMZGdO3eSmZlJWVkZDQ0NmM1mVCoVdnZ2uLi4EBQUxJAhQxg2bBg6nY6wsDBrhJVGowFArVaf51csEAgEAoFAIBD8/RAClUAgEJw+QqA6NUKgEggEgouQ1lR+rRv/GzduJDExkezsbOrr6/Hz8+PKK6+kf//+hIeHExgYiKurq0jlJ/hTaK1f1VoPra6ujsrKSgoKCqipqaG6uprCwkJSU1OJj48nMzOTpqYm1Go1Tk5OuLq6olar8fX1JTAw0Pp73Bp55eDggL29PRqNxpoSUCAQCAQCgUAgEJw7hEAlEAgEp48QqE6NEKgEAoHgIsJkMlFfX09GRgZxcXEcPXqUlJQUTpw4QWNjI+Hh4QwdOpTo6GhGjhyJn58fWq3WGjElEJwvzGaz9V9zczNVVVXk5eWRkpJCfn4+BoOB0tJSsrKySE9Pp6CgADs7O1xdXfHx8cHX1xcHBwcCAgLo1asXERERBAQE4Orqik6nQ61Wo1arxe+6QCAQCAQCgUBwDhAC1Zmzc/8hlny/mkH9o7hlxlT0zk7ne0nnhLr6epb+9CsHjhzn5uuvZfyoSzodW2swoFAo0Gm1Xdq8GHzX3NzMyjUb+H3bTmZccwVXThz3p3wnLS4tw9vT45zPc7FwofpDCFSnRghUAoFAcIFjNpsxGo3U1dVRUFBAYmIiBw8eZN++faSmpqJSqQgPDyc0NJTRo0czcuRIgoKCcHBwEBv1gguW1iir5uZmzGYzDQ0NlJSUkJycTHx8PGlpaTQ0NFBZWUlhYSElJSUYDAYcHR3x8/OjT58+hIeH4+npiZubG15eXvTq1QsPDw9rSkCVSoVarRZRVgKBQCAQCAQCwVkiBKoz48TJFAZPnGa9nv2Pa/jmwzfP44rOHY889xoffL7Eer3jl2WMGDZINu6pVxbwzqdf4WBvzwN33MJLTzxk097F4rtFS1Zw3xPzrdebf/qacSNiz9l8KemZXH3z3WRk5zIsuj+L332VAX0jztl8FzoXuj+EQHVqRCEHgUAguECxWCwYjUZqamooLS0lIyODnTt3sm7dOnJyclCr1QQFBREdHc306dMZOHAgnp6e2NnZic14wQWPQqGwRj0BaLVaXF1dCQsLY9KkSZhMJmpra8nMzOTIkSMcPHiQ1NRUmpqaqK6uZuvWraxbtw4AnU5Hnz59iI2NpU+fPnh7e+Pm5oZer8fd3R0nJydJOkAhWAkEgosdc3MThZu+pHTXD2gDo/C/6l60AX/NL7/1henkr/kQQ2YcniOn4zvpDpR2DrJxxqoSDDkJOPeOQeWg69Te+fCdxWzCkHkMhdoeXVC/czrXucDc1EDR1m8o3rEcfZ8R+F91L/YeAed7WQKBQHBRsH7LDsn179t2YDabUSqV52lF5461m/+QXP+2YYtMoMrNL+SthZ8DYKir4/UPFjHj2isZ1L+vzN7F4rstO/dKrjdu23VOBaq3PvqcjOxcAA4dO8Hzb77PT198cM7mu9AR/rj4EQKVQCAQXKAYjUby8vI4cOAAGzZsYNu2bWRmZuLk5ERISAiTJk1i6tSpREdHW1OciU13wcWMUqlEqVSi0WiAFuHJ09OTwYMHc8stt9DU1ERNTQ0nTpxg586d7N692ypa7d+/n127dqFQKHB0dLTeN2HCBKKiovDx8cHFxQUHBwfr8yIQCAQXK5XHt5G55EkAatMPU1+QysDn153nVZ0bMr97lopDLa+tNv0IGlcfvEbNkIypOLqRpHduxmJqRuXoTO97PsIj9hqb9v5s31nMJk68Mo3qpN0A6PuNof8TK1GoLp6/Q6X7fibjmycAMGQcpdlQScQ/F57nVQkEgr87cSeScHd1IbCX3/leSpdk5+ZLrssrqjDU1ePs1PlhiouVzJw8yXVGTq5sTFpWjqwtKzfPpkB1sfguM1v6OotLy87pfGlZ2ZLrrNy8Tkb+PRD+uPi5eD4VCwQCwd8Ao9FIbW0t+fn5HDhwgO3bt3Ps2DHKyspQqVQMGzaMGTNmEBMTQ3BwsDU6RNTdEfwVUSgUqFQqq2jl6OiITqfD2dmZvn37Mm3aNMrLy7FYLKSkpHDo0CESExMpKyvDaDRy+PBhjh8/joODA05OTnh7exMZGcmIESMIDQ3FxcXFalfUahMIBBcTlcc2S65rUg9iajB0GTl0MWJuNlIVLz2NXXFkg0ygyl/3MRZTMwCm+hpyV73VuUD1J/uuJvWgVZwCqE7YScXRjbgPm3JO5jsXVCdIT7BXxm06TysRCAR/d0wmE79u2MoHi79h+96DfP7uq9w687rzvazTxmw2n+8lnDeamppkbbWGum7f/3f2XSsdfXg6/vsrIvxx8SMEKoFAILgAaGhooKqqitzcXOLi4ti3bx/Hjx+npKQEi8VCVFQUY8aMITo6mn79+uHj44OjoyMKhUJsqAv+0nT8HVer1djb2+Pq6kpAQID1w2j//v2JiYkhNzeXsrIyGhsbyczM5MSJE2RnZ1NQUEB2djZJSUkcOHAAd3d39Ho9Xl5ehIeHM2jQILy9vXF0dESpVOLg4CDquAkEggsWY3WptMFswlhd8tcTqBrrMDdKNxmMVcWycU0VhdLryqJObf7Zvuu4NoCmSnnbhUx9Ybrk2lhdisXUfFFFgQkEgoubyqpqvlj2Ewu/XEpWh6gawcWFxWI530u46BE+lCL8cfEjPlEKBALBecJsNtPU1ITBYCApKYnjx49z7NgxEhMTyczMRKVSMXjwYPr160d0dDQxMTEEBQUB8k17geDvRGtklUqlstZc0+l09OrVi6FDh9LY2EhjYyO5ubkkJyeTnp5OeXk59fX15OXlkZGRQWJiIs3Nzej1evz9/dm7dy9eXl64uLjg4eFBUFAQkZGR6HQ6NBqNtV6WRqMRz55AILggsfyNTxRbmhsl1+amhtO7/xz6ztIsPyl+uus7/4iNH4FAcH5ISk3noy++5Zvvf6auvr7b91RV1zB86CBZX0lZOXEnkvBwcyU8JAi9s1Ondhobm9h94DAB/r5EhIWcct78wmKOHE9gYL9Ignr5d2utANl5+aRn5tBkNHLJkGhcXfTdvrc9ZeWVxCclU1xahp+vN8G9/M8o/WF1TS2JKWmUV1RSVVOL1tEBF2dnIsND8fPxOqO1nSt6ynfdoay8kuOJJ3F10dO/T29rWvoz5Xz5uSfntVgspGZkcTI1A73eiUB/PwL8fM7IN83NzSSnZ5KUko6DvT0B/r70Dg1C6+h42rYEFx9CoBIIBII/GZPJRGNjozViKjk5mR07dnDgwAEKCgrQ6XRERUUxYMAApkyZwsCBAyqdpGwAACAASURBVHF1dRXpxwQCG7Q+E+3rVzn+70Osl5cX0dHR1NfX09DQQG1tLSkpKezfv5/U1FTKysowGAxUVlby22+/YTQasbOzw9/fnz59+jBkyBC8vLxwcnLC1dUVHx8fvL29cXBwsKYeVKvVKJVK8WwKBALBeeSCPjl7Ia9NIBAILkAsFgsbtu3k/cVL2LBt52nd+9QrC3hr4ecATBw7kt+Xf05TUxPPvfk+v27YQnJapnWsUqnkuimTeOrf/5TVP6o1GBh/3S0cSzgJwNMP/ZMXHn2w03nTs3KIveJ6qmtq0Wg0rF++mHEjYrtc66q1G3n53YXWOaDlu82AvhE8dPfcbqUuLCop5dH5b7J1114Ki0tl/UMH9uOf/zebubOmn/L7yqbtu3n5nYXsPRyHyWSyOSYiLJjbZl/Pw3fPPe2avoePJ/DkKwvYe/CorO+2fz/JfU/MJzqqD6898x9GxQ7p0lZP+K47GI1Gnn/zfZavXktOfoG13d7ejkH9+vLAnXOYdd3Vp2XzbPz89KvvsGrdRlLSsyTj0zKzcY2IQe+s4x9TLue/rzzTo/N25JvvV7N46fccS0jGUCeNdFcqlVw7eQL3zr2JiWNHdmqjleWr1/D2ws9JTEmjqcko6dNptcz+x9XM+9ft9A4Nlt17Nv4QXFgIgUogEAj+ZGpqaigoKODIkSP8/vvv/Prrr9TV1eHi4kJ4eDgTJkxgzpw5REZGolQqz/dyBYKLjtYvX+2jrFxcXPDx8SE8PJzLL7+c6upqysrKyM/PJz09nU2bNpGcnExFRQUFBQUkJyfz7bffotFocHV1JTIyklGjRjF+/HgCAwPRaDRotVpcXFxwdnZGpVJhsViEUCUQ/AWpL0yj/OAa9H1H4dw7pkNfOjXJ+zA11GLvGYRT2GDsXH1s2rFYLJQfWktDYToel0zFwVv+Rbs9xupSyvb/gsXUjMcl12Ln1r1T0M2GSmpS9tNQlInFZMQpfBhOYUNQauy794IBi6mZ+sI06nISqM9PQamxx86jF9rAfugC+3XbDoCxqoTqk3toqiymubYci8WM2tEZtbMHzuHDcPSPOC17HTE3NcgikszNjTSW5aG016Jxcuu2rZ7wXXssFgvNdVWydmN1KY1ledi5+nSZJs9iNlGbdpi6vJM015Zjqq9FaeeISqvH0ScU58hLUDl0fvq/PaaGWgzZJzBkxWNuqMXO3R97jwAc/SPQ6D3P6PXJXldNGeUH1wDgcck01DqX07bRU89beypPbKcuJxGvUddbX2tTRSEVR37H3GzEpd9otAFRnd5vMZtpKEpveR4K09G4eKEL7IdjQF9Udqd/stvUVE/Nyb00lubSVFmIUm2Ho38kjr0icfAOQaFUnZY9Y005dTkJ1OUkYKwtx87VB3uPXuj7ju526kqLxUJjSRaGzOPUF6ahcnTG3iMAe88AtAFRKLr5naSn7Aj+nhjq6vjm+5/58IslEiGpM8ZcMkwiLFksFr796Rfr9ZYdezgYF8+Tryxg2659svvNZjMr12xgzaZtfP7Oq9x43VXWvk3b90jEj7c//oJ//t9sfL1tR5i8vfBzqmtqgRZRY9nK37oUqOa//SEffL5E1m6xWDiemMwdDz+FyWTittnXd2pj+eo1/PuZlymvkP+daeXw8QTu/s+zrFy7kW8+eAM3V/n7cl19PXfPe5YVP6/t1E4rKelZPPXKO2zavod13312WvsVr7z7MVt27LHZZzabqTXUsfvgEV7/4FN++eaTTu30hO+6Q0Z2Ljf/ax4Hjh6X9TU2NrH/yDHm3Pco+w8fw9jcfEp7Z+vnuBNJvPnR4k7vMdTVYairY+FX3zFn5nXEDBrQI/O2p7GxiX8/8zKff/djpzbMZjM/r9/Mz+s3c99tN/PuS0/Z/I5cVFLKfU/M5+f1m21YaXtNi5f+wPLVa/j83VeZfvVka9+Z+kNwYSIEKoFAIPgTMBqNVFRUUFxczLZt21i3bh1xcXE0NDSg1+sZO3Ysl112GaNHjyYkJAQnJyex0S0QnCOUSiXOzs7odDr8/f0ZNGgQl19+OaWlpZSWllJYWEhKSgrbt28nLy8Pg8FAeno6mZmZ/PDDDzg7O6PVaunduzcjRowgJiYGX19fFAoFWq0WJycnaxSXQCC4uKlO3kf8/CnW65A5r+J/5T8p+H0RuT+/I6uHpFCp8Rh+HSE3vYidm6+kL/PbpylY37LhkrX8BQY8txZ95PBO505691ZqkvcCkLfmQ4a+cxCl2q7T8RaTkazlL1KwcTHmhlrpujT2eI+7mbDb3ury84WpsY7sFS9RuPlLm6npAJzChhBw3X9wHzbFZn8rhqx4Mr55nOok25tRrdi5+xM081m8x97Y5biOmI2NJL45k+qTe7GYpCduLcZGDj04sGW94cPo/8wvXQoJPeG7jpTs/pHMJU9jrC6R9eX98i55v7yLSqsn5JZX8Bl/s/S1NTeRv+ZD8n57H1NddeeTKFV4xF5DyC2vYO9uO51U9cm9pH3+CPX5J21Hc/3Phv9V98kEodOhPj+FE69Np6k8D4D8tQsZ8tbe07LRk89bK+lf/ofCTV8ALc/dkLf2kbt6AcXbpJubGr0XoXPfwHN426n75rpqspY9T8mOFZiNNtIyKhQ4944l7PYF6IL6n/L1GatLyfnpdUp2/Yip3vbP1d47mKAZT+M56vpT/r5VJ+0h7fOHqc9PttmvdHDCe8xMgmY+g1rnanOM2dhIxpKnKd3dxZq8gvCddDu+l9+Jyl57Tu0I/p5kZOey8MulfLHsJ6vI0xm+3p7cesN1zJ01XZZ2r7yiioIi6XvuyKtmnnL+xsYm5tz/KIG9/KyRO846rWzM19+v5vH775Ldb6irY9mq3yRtfj7eXc5pS2DpyD2PPkdYSBDjR8qFruWr1zDnvkdPaaOV9Vu2M+ufj7B+2WLZe8uCj7/slnjRni079rB46Q/cPaf7f7sHD4jil987FyNaGX3JsC77z9Z33cFsNnPDnQ8SdyLplGO7sx44ez+HBQfi7KSjptbQ5T0ajQb3dkJkT/58X3nvY5viVIC/LyVl5TQ2Sj87fvTlUqIiw7nn1lmS9ubmZi6/4TYSU9K6tZ5aQx2z7nmYjd9/yfhRlwCcsT8EFyZCoBIIBIJzSH19PeXl5Zw8eZKdO3dy4MABsrKyqKqqQq/XM2LECCZOnEh0dDRBQUF4enri6OgoxCmB4ByiUCisKQtaUwI6Ozvj4eFBSEgI9fX1xMbGMmnSJHJycigoKLBGWyUkJFBaWkplZSWlpaUkJCSwatUqXFxc0Gq1hIeHM3jwYPr27WtNzWlvb4+joyN2dp1vLAsEgguTisO/S65L96yksSTbKjR1xGJqpnT3j1Qn7Sbq0RWSTeuSnSvaDbSQu+pt+j3+g0071Ul7rOIUQFNZLnU5iTiFyutptJK04CYaijJsr8vYSNHmL7CYmgi/8782P2dUn9xL6qf3dWqjldr0IyS9czP+V99PyE0v2hyTs+ptcla+AWbbKWTa01SeT+on/8JYXUKvq+8/5fhWKuP/oCphxynH1aYdovLoJjwuubbTMWfrO1vkr11oU5xqj6mumoJ1H0sEqobiTBLfmtWp6CDBbKJs38/UpBxk8Os7ZCJE/vpPyFz6bNc/h//ZKDvwG5H3fornyOmnnrcD9QWpxL90jeT11ucnU5+fcloRcj35vLVSfmRD23hjI0lvz6YuN1E2zlhdQspH9+A2aBIqBycqjm4g7fNHaCrP73zBFgs1Kfs59swEek17hIBp81CqbdfdqEk7xMn3/q9re0BjcRYpC++mcPMX9Hv8R5tCjrmpgazl8ynYsKjLFJLmhloKN31BRdwm+j7yHbogafRjY2kuSe/NwZAR1/WaSrLJWvYC5YfWEfXoCtRaaY2XnrIj+Hvy1CsLePvjL7pM1apWq7nqsvHcPvt6rpw4FpXq9KIMATzcXLn3tpvpHRrE9j0HWLl2IxWVbZFHFouFu+Y9TdyWX1Cr1YwdEYOTTkutoS192fc/r7UpUP26YatkHMCUy8adck0KhYJRMUO4YuJYKiqr+OTr5dQ3tInhFouFL777USay1BoMPPbim5I2pVLJY/fdydxZ03HSaflj934eef51ikra0v5t2bGHj79axr233SS5d2m7qDMAraMj7730FFdNGo+rXk9+UTH/XfQ1H325VDLux99+Py2B6vH772RYdH+WrvyF739eJ+m7+vJLmThmBAP6RDBmeNcCFZy577rL0p9+tSlOxQ4eyHVXXY5GrebX37ewY9/B07B5dn52dtJxcMNK1mzaxuvvL6K4tMw6xkmnZd6/7qB3aBCxgwcSFhzYY/O2Z9XajZLr8aMu4ZsP3sTf1xuj0chXK1bxwFMvSVII/rRmg0yg+vCLpTJxKioinHdffJIhA/uRV1jMgo+/kKzdYrFw6wOPkbx7A/b2dmfsD8GFiRCoBAKBoIcxm800NjZSXl5OfHw8hw4d4ujRoyQlJVFeXo6rqyvDhw8nNjaWoUOHEhUVhZeX11kX2RQIBKdP60ajQqHAwcEBBwcHazrAiIgIqqurqaqqora2lrKyMlJSUsjKyqK8vJzKykry8/PJysqitrYWOzs7EhISOHjwIAEBAdb6VQEBAURERODv749Op7OmHbSzszujL9gCgeDPo6FEmtO+NvUgtamn3oxoKs8n6d05DH59p3WDWe3kTnNthXVM5bHNNJblYe/RS3Z/0dZvZG22xknWegphCaB427c4hQ3B97LbJO3GqhIS35rVaeSDLfLXfIiduz/+V/5Tuo7iTHJ+fLXbdlrJXvES3uNuQuPs3q3xTiHRKFQaWfSUDIUCe6+uNybOxned4RwRiyFDXmejI64DJ0iuc356o3viVDuayvPIWfU2obe8bG2rz08hc8lTsrEKjT0qex3NteXSDrOJ5IX34DLg0m7/DAAaSrI58ep1MjHOzs0Pe6+g03odPfm8QUvqu6ayXMlYW+JUKxaTkebaShqKs0lccHO3BNaW+5rJXdmyWRx0/ROy/sbyfBLemIHJ0Hkaro7UnNxL8vu303fed7K0eDmr3qTg90+7bauxJJukBbMZ9Op2SdrFrGUv2BSVNHpPmg1VsmerJnkfOT+9Tugc6fPdU3YEf0/iEk52Kk717R3G3FnTmXPDNLw9Pc54jr69w1j73WcE9mpJlXvT9Gv55//NZszU2ZKIj+S0TLbs3MvkS8dgZ2fHzKlT+GLZT9b+YwknSUpNp2/vMIn9jtFTEWHBxA4eeMp1LXr7JebOajsUMHHMCK6dI/2bumv/Idl9r7+/SBYp9voz/+Hhe+Zar2dOu4qh0f0ZNHGqpK7Pwi+XSgSqwuIS0jKzrdc6rZa1330mqQEVGhTAey8/zdZd+0hITrW2H7SR+q4r7OzsuGrSeNRqlUygmjl1CjdN7/wgSUfO1Hfd5fk335e1Tb96MisWvWe9fvieudz3xHwWLVkhG9uRnvJzWHAgD9wxhxWr10gEGR8vT555+F/nbF5oiXRMSk2XtM267mr8fVuiBTUaDXfdMhMvD3duvPshzGYzAFEdnpeSsnJeXPChpM3f15tdvy3H2aklLa27mytfvf86dhoNXy5vewbzC4tZvW6TNR3n6fpDcOEiBCqBQCDoIZqbm6mrq6O8vJzU1FSOHz/Ovn37iI+Pp6KiAg8PDyZMmMCAAQMYOnQo/fv3x8+v5UOyiJgSCC4sWp9JvV6PXt9ywtdsNhMbG0t5eTllZWVUVlaSm5tLQkICmZmZ1NbWUlNTQ05ODsePH0elUqHVavH39yciIoLw8HDc3d1xcHDAz8+PwMBA3N3d0Wg01qgujUYjas8JBBcJLv3H4XHJVJoNlZTt/wVD5jFJf2NxJrmr3iZ41nMAuA25goJ1CyVjSveukkUNmZrqKftfHZ9WnMKGdrtOkFPYUFyjJ9JsqKRoy1dYTNK6CCU7VshElqzl82XilL1XEKG3voFT+BCaKgrJWfkGFYekG0pZ3z2PxyVTJenlSnbIN2r0UaMJvvFZtIH9MBubqE7aTfoX8ySihsVkpOLoxm6n+rNz82XQ6zuoPLqRrO9fxmJslPQHXv8E9l7BOIUNRturT7dsnonvOiN49gu4RI2haNsSKuM2Sfr0fUfiPuwqtAFROPdtKyBuajBQdkC60alQaQi5+SU8LrkWlc6VpvJ8cle9LY3IA8oPrZEIVB1/hwDC73gXr7GzUGrsqc9PIem9/6M+r90JcbOJqvht3Y6iaqoo4MSr02RRQQqNPRH3fnrGtbtscbrPG3BK8dLeOxhjdZk1raPb4MnYefQi+cM7ZeKUylGPe+zVuPQbS7OhkuKtS2RiV94v7+E1agaOfr0l7WmLHrQpTrkNmYy+72iMVcVUxG2iPu+kpL/i6AaqT+7FJWqUta2+MI38NR/JbPlOvgu/y+9E6aCj8vhWMpc+I5mzsTSH7B9eJmzuW0BLGsmKOOlJeF3wQPo8/C0OXoGYjY0UrP+ErOXzJWMq4zbDnLbrnrIjEHTksfvuZP5jD1qzHpwNC+Y/YRWnWhk8IIoFLzzB/U9KI4F/+HU9ky8dA8Cdt8yUCFTQEkX13Ly2v9vlFZVs2LZLMub/bjz1e+icG6ZJBBaAKyeOI6iXP9l5be+p5ZXy945fN2yVXAf6+/HAHbfIxvUODeb22dfzydfLrW0n0zJIy8wmPKTlAIGvtxeXjx/Nxj9aXsN/X35aIl60J3bIQImAUVNrwGg0/ukHbc/Gd92hoKiEnPwCSVtwgD+L3n5JNva9l57iyPEEm3Wq2nO+/NyT89rbyAby4oIPMZvNzJkxDUdHBwCumzKJjT98yYrVa4kMD2XOjKmSe7bvOSBLy/fMw/daxan2vPzkQxKBCmDt5j8k9eIEfw2EQCUQCARnSXNzM01NTZSWlpKdnU1cXBybNm3iwIED1NfX4+npSWxsLCNGjODaa68lPDwcBweH871sgUDQDdqLxyqVCpVKhZ+fn1VctlgsNDQ0kJ+fT3FxMUVFRSQmJnL06FFycnKora0lOTmZuLg4lEolTk5O6PV6oqKiiI2NpXfv3ri4uGBvb4+7uzvu7u44OjqiUqlQKBQolUqUSqUQsQWCC4xe1z5E0I3PWp/NXtc+RMIbM6iK3yYZV7z9O4JmPoNCqcR77Cy5QLVnpUygqji8XlYHyXP0Dd1aV/DsF+h1zYPWa6fwYaR+Ij1B2nFj35B5nOLt30naVFo9A+dvwM6l5VSsnYs3UY8sJXHBTVQcXm8dZzEZKf5jKYH/aKuDUZslte8ycAJRjyxFadfy2UflAB6x12CsKib9y/9IxnYV3WILrX8kWv9ICjZ+TmNxZtv6HfUETn/stGydie+6QmXniMcl12JuqpcJVO4x1+A/RX6yt6EwHXNj26aNQqWh7yPf4jb4cmubo2844Xe9T9WJ7TRVtG2gNRZnYW5qsPq54+8igMeIf1hFI0f/CAY+v5b4l6+lLvtEm/2Avt16fcbqUk68+g8ai6VRT0o7R/o89DUu/cZ0y053OJPnrStUOheiX9yMo28YFouFuqx4TI0G9H1GULxjOTUp+6U3KFVE/WcZ+nZios+lczj537lUHmurp2JpbiLj6yfo90RbfY76/BQqj2+RrSH01tfxu+Ju63Xw7PmcfP82yg/8KhlXFb9VIlBlLXtBJrz1mvoQwTe2CXM+429GF9SfY89eJkkBWLLze4JnvYDKQUdt2mFM9TUSO/p+Y3D4X7ShUmNPr2v/jdrJnbTF/7aO0QZLi8z3lB2BoCNvfrSYJT/+zNwbp3Pb7OsJDQo4IztREeFWwakjM6dOkQlU8YltEayxgwcyqH9fSaq3FR0Eqp/WbKC5ue0wg1KpZM6MaadcV2dj+kaESUSWjhv5JpOJlAzp++6ImMGYTGZMJnntyJhBA4HlkrbM7FyrQAWw9rvPOHD0OG4uenqHBlvbs3Lz2Hcojh37DrJ5xx5S0qXzAlTXGPBwt13j7lxxpr7rLmlZ2bK2ay6fgIveWdau0WiYOe2qUwpUcP783FPz+vt60yc8lJNpbVHnBUUl3PfEfB55/jVGxQ5l4pgRXHP5BMaNiGXcCNvpFRNT0mVtoy8ZKqtfBeDi7ExoUAAZ2W3R0JnZubJxgosfIVAJBALBWVJdXU16ejqbN29m9erVHDhwwLqJPWbMGKZMmcLkyZMJCQkBENERAsFFTnuxSKFQWGtPhYW1bHRdffXVVFVVkZ2dTVpaGllZWZw4cYLExEQKCwvJzc3l5MmTrFq1Cp1Oh4eHBz4+PsTExDBmzBgiIiLQ6/VoNBqcnJxwcnI6j69WIBB0xNEvgqCZT0vfC5RKIu//jKOPjZJEBRmriqlJ2Y++zwh0wQNwChtKbfpha78h4ygNRRk4+IRa20p3S0+KKlQavLohULkNmSwRWAA8R04n45vHMdW1RUeZjQ2YmupR2TkCUJUkPfkN0OuaB63iVHuCZz0vEagAqhN2QjuBynvs7JZaQhYzdm5+9Hngc6to0h6NDfuytHN/Emfqu55GGzwAXegga7q0oJlPS8SpVizGBuy9giQCFYCxpsyaClLlKN9IO/7CFfS65kHcY65GrdWj1rnS/6nVFG76kqbyfNyHTUEX2E92ny0SXr9elopQpdUT9egK9JHDu2WjO5zp89YVwTOfxdG3JeWQQqFAF9KWiqvqxHbZ+KAbnpaIUwAqBx1hty/gyLwYSaRdVcIOLKZmFKqWrZaS3fJi8q7Rl0nEKetrum8RiW/OlNRWc/SXRv9VJe6UrsNRT8B1UqEXwCl0MB6XTKNs32prm6m+BkPmMfR9R9r8/SjcsBhzYx2+k+5AG9QfhUKBz4Q52Ln7Ubb/Vxx8Q/EeO7vD/D1jR/D3JWbQADbv2COpWdNKQVEJr73/Ka+9/ymXjR3JHTfNYNqVl51WXdeIsJBO+9xcXejl50NeQVHbnMXS1Hl33HQDDz7dFjmTnJbJ0fhEBg+IAmB5h/R+V0wYY0151hUhnQhunu5uXd6XlpmD0SgVqX/4ZR0//LKukzvk2NqPiB08kMSUNF797yfsP3KMA0eOS1Kmdcb5OEN3pr7rLumZObK2AVGRnY4f3L97Bzvg/Pm5p+ZdtOBlZtzxACVl0s9rjY1NbN25l6079/Ls6+8xeEAUt8yYxh03XY+TThoZldSh9hTAoAlTZW2dIVLk/zURApVAIBCcJhaLBYPBQHFxMbm5uWzZsoU//viD7OxsLBYLQ4YMYciQIYwcOZKBAwcSGBiIi4uLEKYEgr8wCoXCunmmVCpxc3PD0dGRkJAQGhsbqampsYpTeXl55Ofnk5qaSl5eHrW1tda2TZs24ePjg7e3N+7u7oSGhjJkyBD8/PzQ6XRoNBp0Oh06nU5EVQkE5wnfK+5GoZR/OdY4e+AyYDylHTak6/NTrBvmPhNvlQhU0BJFFXDdPACaDVVUHJWmy3IfNqVbNYG8Rs+UtSnVGhx8wmS1kMyNbSJLfZ683pHayQ1DZvdqSzSUSjdyPGKvIebDE9TnJeMUPgSVgxMWUzOGnARqUw9SnbyPqhM7MFYWymyZGs7stPPZcqa+62kUCgXRL26mNu0gKq2LNTWhsbqUmtSD1KQcoDppDzWpB2zWRzK1i75y6T+O8g5p/urzTpL66X2w6AGcwobgMmA8niOmE/gPucBxKmxFu7lGX9aj4hSc3fPWGW6DJ3faV5ebJGvzHHm9zbEOXkG4Db2S8nZpGS0mI/VF6Wj9WzYzDZny2kz+HaImW1Fq7Onz8BLSv/wP9QWpeMReg+fIf1j7myqLZKkCNXoPGgrkm31gWzxqLM0BRuLoH4HG1VfyHFpMRoq2fE3Rlq/R6D1x6TcOtyGTcY+5GrdBk2zO0VN2BH9f5j/2IHfPuZGPv/qOz5Z+T3mF7bRsm3fsYfOOPXi4uXLrzOu4/aYZslpQtvD17jo9rpeHu0SgqqiUpru9afo1PP7SW9Q3NFjbvv95LYMHRJGbX8j2vdI6eXO7kd4PQKe1/XdEre568z23Q+q508XN1YVBHQSV/MJibn3gMf7Yvb+Tuy4sztR33aW0vELW5tNFDbSwkK7rXLZyvvzck/OOih3Ckc2ruf2hp9iwbWen447GJ3I0PpGvlv/E6q8XEhzQVkc1N1/++e90uHT0JWd1v+DCRAhUAoFAcBpUV1dTVFREfHw8u3btIi4ujry8PAwGA76+vgwdOpShQ4cSExODv7+/tb6MQCD4+9BaT6p99JPFYiE4OJh+/fpRW1tLbW2tVeTOycmhuLiYgoICsrKyyM7OJiMjwxqdtXnzZgICAnB3d8fNzY3w8HD69euHi4sLGo0GtVqNVqs9rdOkAoHgzOlYX6Y92qB+sFvaZqwqtv7fc+R0Mr59RpLCr71AVbb/F1n6Lu/xN3drXfbeITbbNc62NlXa0n7V55+U9aZ/Ma9bcwIo1fL3HjtXH5prK8j9+V2qk3ZjyIjDbGywcbcUBedHeD9T350LFEol2uCBlB9cQ85Pb1Cbduh/okI37m3nP5+JLennKo5skA+0mKlNO0Rt2iHyfn4HbVB/fCbOxXfS7Wd1+KFs32pqpvwL594xZ2yjI2fzvNlC7eSOvaft0/cWi0UeFebobE1X19311eeetApUTRXyjbiuotTUWj2R9y2y2WdLTG4oyiDu6fGd2uuIQt3yvUSptqP33e+T/MGdsvpz0CKKlu5dSenelSjtdXjEXkPAtEdw9I+QjOspO4K/N738fHj5yYd55uF7+W7Vr3yweAnxSSk2x5ZVVPLup1/x7qdfMTp2KP995RmZ4NKeopLSLuduL04BBPr7Sq5d9M7MnDaFr1essrZ9/8s6Xn16Ht93iFrycHPl2skTupyvFVv1fLpDgL+frM1JpyU6qvNaixo7DQF+vkT368OUy8bh7taWKq6quoZR19wo8wOAt6cHI4YNZmTMYMYMj+HXkCNlpgAAIABJREFU3zfz5keLJWPOxyHcM/VddwnqJfdxYRe/RxnZeae0eb78fC7m9fHyZM3SRew7HMeylb+xbst20rNsf06JT0phxJSZpOzdYI2kCujwjAFcMiQadReRUS4uzkRFhDN4QBQzp07p8jULLk6EQCUQCASnwGg0UldXR3V1Nfv37+f48eMcOnSIpKQkqqqqCAoKYuTIkcTGxhIdHU1ERAReXl7ne9kCgeACQqFQ4ODggIODg+T9oba2ltLSUuu/7OxsMjMzyc3NpaCggLKyMpKSkkhNTUWpVKLVavH19SUyMpJevXrh7OyMm5sbAQEBhIaG4uDggFqtRq1Wo9FoRAoEgeAcYFu0aEGpkh9KaW5Xn0Xl4ITXqOsp2vK1ta0uN5G63ES0AVGU7pGm97Nz88M1emK31qWy19psV9h8H2gTIZoqu97QPxUdaw2ZmxpI++IRSnYs7+SOLjhPkaFn6rtzQVXiLpLfvw1jddebqjZp5z+lWkPUf5ZTsOEzMr97DouxsdPb6rJPkPHVo1Sd+IOIf33SqT9OicVC2uKHiH55K0p1zxzQOpvnzRatKRBtYaqrwtxYJ2mzc5NvVEqR/z60F2ON1fIUShqXM/ue0FQl32A8HZR2jjiFt4mHboMmMei1HaQsvJua5H2d3mduNFCycwXlh9YR+eDnuEVfJunvKTsCgYODPbfPnsHts2ewbdc+3l+8hN82bsVisX0wYNeBw8SdSOpSoEq0kU6slcLiElmqssjwUNm4O266QSJQZeXms/dQnCy9383XX3vOD6eGBQeg0Wgkaf7Gjojhl28+OSN7r7+/SCZejI4dyhvPPcrwoYMk7Ut+WE1H/opZYtrX52rlWII8urY7fa2cLz/39Lwmk4kDR+PpFxnO8KGDGD50EO/xNKkZWazfuoPlq35j32Fpvc7S8gp+37qT66+5AoA+NiIf//vKM8QMErUJ/84IgUogEAhsYLFYMBqNGAwGioqKSE1NJSkpic2bN5OQkIDRaCQoKIiJEycydOhQhgwZQmRkJK6uf26BUIFAcHHTGmXVWqPOaDRSXl5OTk4OWVlZlJSUkJ6eTm5uLoWFhZSVlXHixAkOHTpkTSPo7u5OREQEAwcOxMfHBxcXF1xdXfHw8MDNzc0qWCmVSlQq1V/yi6RA8GfSVFmILtj2l2hjjbyGUscNcZ+J/ycRqKAlisp30h2S2jMAXmNn2UxvZguFjUim7mDvGUBDoXQDz8En1GaNqFbUTm44eIegCxqA5yhp+rPUzx6UpV1rWaACbVB/nHvHoo8cjoNvGMefnywbcz44U9/1NHW5SSS89g9JTaNW1E5uOEfE4hxxCfo+IynatkQuAtrwn9/ku/AceT1lB36hdM9Kak7utWkfoPzAb6TbzyPiXx93a70qnYss5VxdTgL5az4gYNoj3bJxKs72eeuIsgvxTa1zReXojKmdyNVYlovFYuk0ssyWL1UObbUj7Vy9aSqXnq43Vpei0XeedszcbKQmZT/awH5onNpqqth72I78cu4iraLSzhGNswfa4P64DpwoiwZz8ApkwHNrqU7aQ+nuH6k4ulG23lZM9dUkvnkjQ97eh6Nv+DmxIxC0cuno4Vw6ejjpWTks/HIpXyz7iZra008Dm5yWyW8btnKNjcim5avXytr62BCoRsYMpn+f3pw4mWpte+mdjzgSL011OneW7XSgPYlaraZ3SJBEeNt7KK7L96lag4FH57/Jobh45sy8jgfumGPt2773gGSsSqVi2afv4ucjF9L3HZanLK2uqUXvfHr1cm2ts6nJaGPk+cGWQLVu83aqqmtw0UtTp5rNZn5ev/mUNnvazx192Jn/enJeQ10d0ZdOJTsvH51Wy561K4iKaHkP7x0azP2hwdx/+y18+s1y7n/yRYmdPQePWAWqqAi5QLXn4JEuBao1G7fx1keLcXFx5v1XnpGkDITu+0Nw4SIEKoFAIOiAxWLBZDJZU/lt376dTZs2cfDgQRwdHQkLC+OSSy5h0qRJXHnllbi7n7ouhEAgEHQHjUaDj48PPj4+xMS0nHK2WCzk5eVx8uRJUlJSyM3NJT4+ntzcXPLz8zlx4gSbN29GqVQSGhpKYGAgoaGh9OvXj8GDB+Pl5YVOp8PZ2RlnZ2ccHc9N7RSB4O+CIfNYp3VU6rJPyNo6prJyCh2MLiQaQ2bbCdPSPStRO7lDh1Pi3U3v1yU2D563NTr6R1IV/4ekN3jWC3hccu1pT2XIPiEXp5QqAqY+hP9V96HWtR3ksVXjSnGhCein8F1Pk/3DKzLBQxc8kJBbXkEfNVqyAZO39kPZ/e3FTIvZROWxzTQbqnEbMhnfiXPxnTgXU0MtVfHbKT+yntLdP2FuqpfYKD+0DnOz8ZQRUPaegfR/+mfKD64lc+kzkr6cVW/hMXxaj4gPZ/u8dcRWSkrJ/b36UJvaVlPG3FhHY0kWDp2kgbRVi0sbGNVuPZHUph+R9Buy43EdcKlNe8aaMo49M5HG0hyUdo70eehr6+t37CVP4aUN7MfA59fJ2rtDY1keFXEb0QVH4xI1CpeoUf97TUlUxG2iZNf31GXFS2+ymCk/uIZe1zzY43YEAluEBQfy9gtP8MKjD/D196v58PNvSc3IOi0bjzz/Gn16hxIRFmJt23XgME++skA2dvyoWJs27rj5Bh557jXrdcf6O0MH9mNgVORpretMmXzpaIlAVVFZxTOvvcsrT8kPBhQWl/CPufdxMK7lGTyRnMpdN8/EwcH+f/3SaF21WoXj//ras+SHn4k7IY8UOpmabjNlW1d4uLvJ2rpKodfTNDU18dOaDazbvJ2rJo1n+lWXS1Kl652dGDciRlJfLCs3nzn3P8rqrxZKDvs99eo73art1NN+7ujD8soqjEajLIKvJ+f9fetOsvPygRax6rr/u5ftPy/Fx0t64OK2WdNlApVdu3WNih2K1tGRuvq2zx9vfvgZ106eSEig/JDJoiUreOCplzCbzQAMi+7Pc/OktRy7649WktMy+ezbFSiVSu646QYiw0NsjhP8eQiBSiAQCP6HxWKhvr6e/Px8jh07xs6dO9m1axcZGRnY29szYsQIhg0bxoQJExg8eDDe3t5io1cgEPwpeHt74+7uTkxMDM3NzdTW1pKdnU1cXBxpaWlUVFSQkZFBZWUlKSkpHD9+nLVr16LX6wkODsbPz4/g4GD69OlDnz59cHd3t4pV9vb2olaeQHAa5K/7GL/Jd6FylJ6iNeQkyCKgwHaNGp8Jt5L+5X+s1w1FGbJNfuc+I3D0PXUB+LNFa2PTuzppd5cCVfaPr1Gw/lNUjs5E3rcIfd+RANSmHZKN9Yi5mqAbnpa12/KVrXRo3UGhkApb3al39aehkItu5k7S7dWmHZa1Rdz/mbWekfX+ZiM1J/fKxhqrS61CSupnD1KyfRkAutDBDHxhPUq1HSoHJ9xjrsI95ioCpz/O8flTaCrLtdow1VfTVFHYZd0lgAHPrcHeIwC/K+6mePt31OUkWPssxkbSFj/EgGd+7dJGd+iJ503CKURQbQeBCqBw05eE3DRfNrYuN5HKuE1S83aO2Hu2nb539JdvWOf99oFNgcrcbCT5gzusNcfMTfWU7v7RKlBpnNzQ6D0l6R/rchMx1lZIIq3aU5+fQvLCe6jLjsdj+HXW+lYNxZkcfiQGLC0bfn3+/bX1mdcG9EUb0JdeV99Pzqq3yfnxVanNdrWwesqOQHAqnHQ67rvtZu6dexPrt2zn/cVL2LR996lvBDKycxlx1UzuumUm4SFB7Nx3iF9+30xzs/RAwLgRMVw5cZxNG7dcP5UnX1lAY2OTzf4/I3qqlefm3c+Kn9dKxIc3P1qMWq3mzptvILCXH0ajkWWr1vDkKwsoLm372xrg52sVpwBGxQ6xig4AjY1NPP7SWzw37356+fmQmZPHN9+v5qV3PrK5lhPJqVw2btRprd/fRvTOp98sx9PdDaOxmfzCIq6+fAIjhg2ycffZ8+QrC3h/8RIAlq36jQN3zmHB/CclY1547EEmTr9V0rZu83Zir7ieqVdMRKt15PetO7slTkHP+9nfRxrlbqir494n5jNxzAgysnNx0mm5ffb1PTrv0Oj+kr70rByuuukuPnnrRQb374tGoyGvoIi3F34uszOwX9tnTX9fb56bdx9PvPy2ta2wuJQps+/kkzfnMzJmMHZ2diSnZTJ/wQd8/7P0EIatCLfu+sNJp6O4tIzBl02zpsn84PNvyTy4BW/PzlMKC849QqASCAR/e0wmE5WVleTl5ZGUlMSBAwfYuXMnpaWlKBQKoqKiGDZsGNdeey2enp74+fnh4uIiNnQFAsGfgkKhwM7OTnKyrzWFX2hoKDU1NTQ2NlJZWUlmZibHjh0jPz+fqqoqSkpKKCwsJD09nb179+Ls7IyPjw9BQUGEh4fj7e1NYGAg/v7+ODs7o9Vqsbe3x87OTtSvEgg6obmmjOQP7yLivkWotfqWNkMVJ9+9VRaNYu8dgp27/DSo56gZZH73nLTeTYfoKZ/xt/T84m2g7zcGlCowm6xthZs+x23w5bL6Vx3rS5nqqynescwqUDUbKmX2baVAq808Rt6v78na6/PPbMNa7ewOReltczY30VRZhJ2rzxnZ60nUznLhoLEk2+bY7vjPYjaT8fXjNNdWyMbW56fg3Lsl+rb8UFvqKkPGUVI+upvIB76QRKnZe/RC33ckpbt+kNjpTiSbnWvLiWqFSk3YbW8T/+JVkv7qxF0UbfsWn0vP7ve4J5639ijV8pPj7XEfOoXiP5ZK2vLXfIC+70jch15pbWuqKib10wdk97sNmSzxX8cabQBVx7eS99v7kughi8VC6qf3UXViu2SsxWSSXLsMnCD9eVksJH9wO1GPrpBFh1Ul7ubke7fSXNuSCrF0948Ez3oee49elB/+3SoqAaR8/E/s3HxwjrhEYsNr9A0yYam9yNdTdgSC7qJQKJhy2XimXDaexJQ09E7dSy9XXVPLgo+/6HLM688+2mmfm6sLM665kqU//SLrs7e3Y/Y/ru5yzd1ps9Vua5ze2Yl35j/JTf+aJ2l/9b+f8Op/P8HPx4uKqmoaGuSHIZ55+F7J9eRLx7B89RpJ2xfLfuLL5SvROjpiqJPW5evItl37efDONiHHTqOhvt37ltLGIQ1vTw/UarVEIMzNL+Rfjz1vvf7wi2/JPbq9x31nMpn45oefJW1frVjFG88+ilrdtkU+dngMk8aNkomgxxJOcizhpM35u6Kn/ezvK/9889XylXy1fKX1uqq6tkfnDQnsxbiRsWzf05Y28FjCSUZdfSMODvb4eHqQlZsvs+Hv6831V0tTOv/7rltZ+tMvHE9s+9yXmpHFpBvm4ujggN7ZiSIbUXUDoyK5boo8qrq7/nj2kXv5buVvkhpuRqORb75fzX/uvaMrVwjOMUKgEggEf1uMRiPV1dXk5eVx6NAhDh48SHJyMkVFRRgMBvz9/Rk5ciSxsbFEREQQHR0tarcIBIILApVKZa1f1YrZbGbAgAEMGTKEyspKampqyM3NJScnh6SkJEpLS6mtrSUlJYWMjAwOHz6Mo6Mjbm5uBAYGEhAQQFBQEO7u7vj5+eHq6opOp8PR0RGNRiP50iYQ/N2pOLqBo4+PwnXApTTXV1N5bCvmRnltjOBZz9ncIFFr9XiOnE7xtm9t2lfa6/AYPq3H120LrX8k/lfdR/5v71vbLKZmkt67laAbnsY1eiIavRdVJ/4gZ+Vb1OdJU8Boe7UVp7dVC6f88Hpyf30PzxHTMdVVU5Wwg5yfXpfU+WmlsTSH5rpqqxDRXezc5OmFkhbcjL7fGJpKc1HaORDwj0dx8A4+Lbs9gZ2rn6ytZOcKa2RVU0U+boMvx/ey23COHE5V/DbJ2PQv5hEw7RF0oYOoTT1E0davqTiyweZchnYp7/SRwyXjyvb/QvJHd+Fz6S04hQ/DWFlE+eF1lO2VFkVXO7lh5yZfc1fo+4zAe/zNMmEna9nzuMdc3Wl0T3c52+ftdHCPuQq3IVdQceR3SfvJ925F33cUuuCBoFBQuvsnmioKJGOUDk6E3PKKpM05IhavMTe2/MzbkbXsBSqObEAfNZrm2goq47fRUJBKR1w7pDcMuelFKg7/jqm+2tpWFf8HSe/MwXfS7ej7DKepopDCzV9SuPFziXikdnKzirb6PtJn1dxUT8JbNxIyez7OkcOx9+hFddIem0KyNrDtFH1P2REIzoTW+jdd8fYLT/D50h8kKfE6onV0ZOEbzxM7eGCXtu64eYZNgeq6Kyfh6tL5361B/ftKrnuHBndat6nj2M7q8twwdQqNTU3Me+F1yiuktQALikpk41UqFa89PY85N0g/W8y67ip+37qDFT9L63FZLBaJeOHh5tqSZnHFKmu6QICM7BzJfUMGRLH7YFta09ghcp8qlUqe+vc/eXGBPFVtK4a6euobGnvcdyqVCj9vLyqr2t5DnZ10Nvd5ln3yDjf9ax4b/9jV6ToBHr5nLqXlFSxpJ3z16yNNN9vTfr57zkw++XqZTRGnlbLyCp544K4enXfxO68weeZtZOZIaww2NDTaFKdCgwL4duHbkoOW0FJL7bdvF3H/ky/y64Ytkr76hgbqG+SR8P0ie7P2u8/QaeW1JLvrD4AAP7mY1bG2mODPR+w0CASCvxUmk4mmpiZqamooLCzk2LFjHD16lEOHDpGRkYFCoSAsLIwrrriC/v37M2LECCIiIkQkgUAguOBRKpV4enri6dmWB7y+vp6CggKSk5OpqKigvLycrKwscnJySEtLIy8vj9TUVOLi4nB3d8fLywu9Xk9gYCDBwcH4+Pjg5+eHXq/Hw8MDJycnazSXQqFAqVSe9WagQHCx0lSeT/H27zrtd+odg+fw6zrt95lwa6cCleeI61A56Dq9t2NKO+gi6sXWM9rh/sDpj1G2dyWNpW2p3syNdWR+K0/N1x5772C8xt5ovXbuHYNKq8dU17bxg8VM9vIXyV7+og0LcqoTd+E+bErLMtUaFCq1NIrIxmu3d/eXtdWmH6Y2vS1lnsViJuKfC3vcd6fCzl0u9pib6ina3HaSv+LwelwHTsRt0GUygaomeR+Jb91Id6hul/LOa+xsmZBVtncVZXtXdWnD74p75P6w6QdpW/DsFyg/uEYSBdZcW0H5gd/wmTCnW+vvijN53mz9rG2+lg6EzX2LIwk7JBGOFlMzVSe2yyKc2hN0w1M2fxdDbn6J8iPrMRmkm8jVSbupTuo8RZnbkMl4jb5B0mbn6kPQjc+S8ZU00qMybiOVcRu7fF0B0+ahULVsATmFDsaxV1+J4GwyVJG2+KEubagc9XiPm2297ik7AsG5ItDfl22rlnDvE/PZvGOPRJhQqVQMHxLNpwteom/vU6fUHTs8huAAf9lG/NxZ07u87/Lxo3F10VvnvmXG1E7HXjlhLM87vU9NrQGFQsEtMzo/rHLLjGlcMWEsj85/k7Wb/6Cisko2RqPRMHPqFJ595F6bqdE0Gg1LPnqLAH9fvlz+k0zs8nR34/prruDFxx7E3c2VKZeNY9y0m8kvLAbg7jmzpGu6YRp7D8dhNptx0Tsz9YrLbK79qX/fw/HEk6xetwlLhwhyBwd7npt3H57ubufEdw/d83/c+/h8TCYTSqWSx++/26ZA5eqi59cln/Ds6+/x5fKVlJZLI5f79g5j/mMPMv3qyWzavpvlq9diNBrRabVMvUIagd7Tfvb29ODr91/n/x58wqYoM3RgPx66Z26PzxsaFMDRLT+z4OMv+XXDFhKSU2lqMkrGKJVKQoMCmHrFRF549AG0nZTF8Pf1ZuWXH/LTb78z/+0PSU7PxNQhahhaUvo98cDdzLlhWqf7ct31B8C0Ky8jIiyYlPSWWnZhwYHcfP3p110V9CwKS8d3AoFAIPgLYjabMZlMVFVVkZeXx8mTJzl48CAbNmwgMzMTrVZLREQEsbGxjB07lksvvRS9Xi82XgUCwV8Ki8VCdXU1mZmZHDp0iLy8PIqKisjPz6egoIDMzEwMhpZT6a2RVcHBwXh5eREREWGtZ+Xr64tSqcTZ2Rk7OzuUSqX1C4N43xT8lTj5wR2SDX2N3gvvS28m7xd5NEB7nHrH0OfBL7D3COhy3NEnxkhq97Qy4Lm16PuM6PS+3F/fkwg+aic3Yj9OsSm0ZP/wCrmr2wrB23sHM+zdI7JxDcWZpC1+qMsN+PbYewbS/6nVOPiEStorjm7g5H9vk6Vf64hL/3HoggeSv1Za9yDstrfxnXS79frwvFgaCttOvvea9gjBM6X1uhqKs4h7apzNqKxWPEdeT+T9n50T352K9K8fp3DDZ12OGfL2fuzc/Tn537myukYdUTk643/1A5TsWEZDUYa1XaP3IvbjltRDFouFjK8fp3Dj4m6v02vMjYTf+R5KjTQNXsaSpyhY/4n12ilsKNEvyddY/Md3pC6SFi6PvH8xniO73rxt5Vw8b4f+HS0RXoNueJqA6+bJxnXEkH2C1E/uxZB1/JRjlfY6Qm6aj89lt3X6N9CQFU/qovsxZB47pT0A1+jL6PPvL1E5yCMFLBYLhRs/J2vFi5gbak9tTKGg19SHZc+NISeBhNemY6wq7taaNK6+RNz7Ca79pTV6esqO4MKnOT/xfC+hS8rKK/EdKK2HtGLRe0z/X3oxi8VCfFIKcScSCQkKYNjA/jg6OnTbfnlFJSExEyXRHYH+fqTu23jKTCv19Q2s3fwH0f36EBEW0uVYQ10dazf9QczggYQGdf05oj3FpWUkpqSTlZOHj5cHEWEhBAf4n9Zh2+S0TA4di8fNRc+g/lH42agX1dDQyG8btzI0uj9hwfJ6hZk5ef/P3p0HRlWdjR//zpLMmsm+7ysJkEBCWEIAWQRZFAUVRMW9/tpaa6tt39rF2moXba3Vt1rrSt0KLogiOyRAgEAIS0iAkD2BLCRkzySZ/fcH74wMk0CAYAiezz86dzn3zL25w8x57vMc8g8XMv/GG/oNTpzb55y9+XR0dqHVqAnw82VCaorTdbka5+500xly8w8zITWFkKCAfrc7V2lFFfkFRWjVahJio4mPiXS67vWnm9iZu595s6b1m+VlNxjnGc6Oc+09UEBFdQ1Wqw0fL08SYqNJiI26qse1M5vNlFRUcbS4FKlUyoi4GOKjI1Eo3C+4X1+MRiNlVTUUl1ZgNJmIjYogISbqkrKbBno+bDYbOfvysdlg6sRxV71Skjwk6aq2fz0QASpBEK57VquVnp4eamtr2b17N9u3byc3N5eqqio0Gg0jR45k6tSp3HTTTYwbNw6tVotEIhGDrIIgXLdsNhtWqxWbzYbJZKKpqYkTJ06wd+9eamtrqa+vp6GhgcbGRmprazGZTCiVSsLCwkhJSSEtLQ13d3dGjhxJWFgY/v7+eHt7I5fLxfx8wnWlrwHz8f86QcPWd6nf9KbLnEkyjSeB0+8jYslvkMovfi+c/OJvLvOyKIPjSPvbhSfd1p88RuGzcx2D08Fzv0/08j/1uW1n+QGOPr/QETAKv/2XhC/+Rb9tn97xEbVfvkxvY6XLvFgA7j6hhN7yBIEz73OZ88auo2Qf5W//1KUcIBIp6tARhN32lCNoUf7Ok5zOWgGAm86PsS/m4ubxzUTVp776BzWrzgaUpAoNyc+sRxPlWjLozL4vqXj3yT7nZlKFJDDiiRWowxKv6rnrj9Vk4PjflrlkRwFIZG6EzP8hkXednXvDajZR+Z//4cze1c6ZaIBc44V36k1ELnsWd69Auk8dp+i5mx3vOfyOpwlfdF5mTWE2deteQ199BFOH61PFUqUWdVgi4Yt+jvfY2X32v+PEXo79eTFWUy9IpMQ+/HK/WVElrz3KmT2fne2v1odxrxRcMBvwXFfjfqv57M+c+uKvZ7dXeZD8hy2oQxIG1B+r2UTtVy/TuOMjDGdOuqyXqXV4Jk0havmfUfpfeBAPzmZh1a77J6ezVvQ5D5lE5oY2No3wO54eUPDG0HyKyg9+TXvRDqeSfw5SGT7j5hF++y/RhI/ssw2zvo1TX/2DtoKt9NSVuM4ZJ5Gg8AvHK2UWkXf9rt/ym4PVjnBtG+4Bqiv17F9f5Y//eMNp2W+f/CHPPPWjfvYQBEEQAaqBEAEqQRCua729vVRVVZGdnU1eXh6HDx+mpaUFnU5HTEwMycnJTJ8+nejoaPz9/dFoNKKcnyAI3yn2IFVPTw+dnZ0YjUa6urqor6+nvLycw4cP09bWRnNzM52dnfT09NDb23u2dIenJxERESQkJBATE4NSqSQmJgZ/f398fHzw8PBALpe71B0XhOGivwFzO1NnM50n9mEx9aCNHosyMOaSHnAp/vu9tBxwnhcg4q5nCLvlwuWxACw9nbQc2oQ6bCSaiL4Hn+3M3R20HtqEJnrMwAfnjb30NJQ7BpuVAVEog+MuaT4hs76dzrL9mDrOoApJQB2ehMzd9WnqzvKDGJtr8R47G6m769PsPXWldFUW4J0654ID2zaLmY6SfWeDCVYrcq03ysAo1GHOAwNX+9z1x9BSR2fJPiy9eqQKFW4efnjEpiFTuT4dbLPZ6KktpqvyCHKNJ5rI0X1mCFl6u2g5sAFNVArq0BEXPL6pvQn9yaMYmmtR+IaiCknosyRdn/t2tdJ2eAseIyah9HctE+Xot9VKx4lczJ3NeI+d0+f17M/Vut+6Tx1Hf/IYPqk39ZmRNBDm7g66Tx6jt6ECN08/1OEjL5oheSEWQzc9daX0nq5AptSgCo5H4R+BRHp5v0OMrQ1015VgOHMSN50/quBYlP6RjpJ+A2E1m+ipK6X71DEkEimq4HiUIXF93rPfRjvCtee7FqAym80YTSbUKhXbd+/j1vt/SHfPN9nBMpmM8n1bCe1jThtBEAQ7EaC6OBGgEgThumWz2WhrayMnJ4eXX36ZkydPolQqiYqKYtKkSWRmZhISEkJoaChqtfqqp/U2MHMQAAAgAElEQVQKgiAMFxaLhe7ublpbW2lubqa3t5fW1lZqa2spLi6muLiYpqYmev+vxIlUKkUuPzsIFhYWRlRUFPHx8YSFhaFWqwkPD0en06HValGpVCLLShg2LjZgfiX0J49R8MspzgulMtJfLcTdO2hQjiEIw8nVvN8EQbhy36UA1evvfcwzL75Ce0cnSqWC3l6Dyzb33rGQ9175y2X3VxCE7wYRoLq4gT9OIwiCMAzJ5XLUajUqlYoRI0aQmZnJmDFjiIuLIz4+XgSlBEEQ+iCTyfDw8MDDw4OIiLNPyptMJtra2qirq6O6utqRUVVXV0dZWRnl5eW0tbXR0dFBdXU1+fn56HQ6FAoFkZGRREdHExsbS1BQkKNtlUqFSqVyzGMlCNczq7EXfU0RcrUn5p4Oyv7tWhLIN32BCE4JgiAIwmXw9tKhVqmcspzioiMvq63X3vuQ9o6z8xr2FZwKDvTn97/48eV1VBAEQXAiAlSCIFy3JBIJWq2WkSNH8uCDD6JQKJgwYQKBgYFifilBEIRL5Obmhr+/P/7+/owZMwaA7u5uTp8+TVlZGaWlpdTX1zuCVlVVVRQWFtLT00N+fj7BwcFEREQQFBSEj48PgYGBhIWFER4ejq+vL0qlEjc3NxQKBQqFAplMJj6rhSEn4by/wcv8m+yuPcGR387Caui+4HYhNz9+We0LwvVgsO43QRC+m6RSKSkjE9h7oAAAL08dyUmXV5o1OWkEJeVVfa6LjYpgw3/fJiJ0YCVSBUEQhAsTASpBEK5rEomEkJAQ7rzzzqHuiiAIwnVHrVYTHR1NdHQ0s2fPBs7O/VdeXk5BQQGHDh2isrISs9lMdXU127Zto62tDQCVSkVycjLJyclER0cTGBhIaGioI2il0+lEgEoYcu6+oU6vlYExl9VOW8G2iwanAmc+gEfsuMtqXxCuB4N1vwmC8N310b9e4i+vvonNZuPxR5Zf9nfJd//xJzInpFFwtJiW1nbc3d0YMyqR1NFJZKSn4qlznTtQEARBuDwiQCUIgiAIgiAMGnd3d+Li4oiIiODGG2+kp6cHg8FASUkJBQUFlJSU0NjYiNlsprGxkQ0bNtDb24tCocDX15eRI0eSkpJCZGQkPj4+eHl5ERwcTHBwMEqlcqjfnvAdo4kc7fRal5hxee1EpVxwffBN/4+o5X+6rLYF4XoxWPebIAjfXRGhIbz+wrNX3I5apeLxh5dfeYcEQRCEi5LYbDbbUHdCEARBEARBuP7YbDZsNhsWi4Wuri7a2tpoaWmhq6sLo9FIcXExRUVFVFVV0dXVBUBPTw9GoxGpVIpGoyEgIIDExESSkpIcWVahoaH4+voO8bsTvgusJgMnP3+B7pPH8Bozi4Ab7kGmUF9WW21Hd9J6YD3GtkYsvV0o/MLRRCWjjR6LNnrsIPdcEIafwbzfBEEYfOa640PdBUEQhGFHHpI01F245okAlSAIgiAIgvCtslgsjgyquro6GhsbaW9vp6Ojg5KSEoqLi6mrq6O3txe5XI5KpUKn0+Hn50d6ejoLFixg9OjRFz+QIAiCIAiCMChEgEoQBOHSiQDVxYkSf4IgCIIgCMK3SiaTIZPJCA8PJzw8HKvVislkQq/Xc+rUKSorKzl58iRnzpyhtbWVU6dOUVpaSmtrKxERERgMhqF+C4IgCIIgCIIgCIIgXCERoBIEQRAEYdiyl4+zZ9q4u7sjlUqHulvCJZJKpSgUChQKBT4+PqSkpGA0Gunq6qK1tZXa2loOHjyIwWAgMTGRkJCQoe6yIAiCIAiCIAiCIAhXSJT4EwRBEARh2DKbzZw5c4asrCzi4uJISkrCw8NjqLslDBL711SbzYbZbMZoNALg7u6Ou7v7UHZNEARBEAThO0WU+BMEQbh0osTfxYkMKkEQrms9PT3U1NRQWFhIWloaISEhKJXKoe6WcBWZzWaqqqqora1FpVKRlpaGTCZDIpEMddeEQdbd3U1paSmrVq0iPz+f8PBw5s2bx/Tp0/Hz8xvq7gmDwH7fSiQS3N3dkcvlTssFQRAEQRAEQRAEQRi+RIBKEITrll6vp6ioiM2bN5OXl0dxcTELFiwgMTERlUo11N0TrgKr1UpJSQmbNm2ioKAAb29vjEYjycnJ6HQ6Mah9Henu7qawsJCtW7dy8OBBgoODaWtrY+vWrZhMJubMmYO3t7co93edEddTEARBEARBEARBEK4fsmefffbZoe6EIAjCYNPr9RQUFDgGry0WCydPnsRisaDVatHpdKI81HXEZrNhNBopLy9ny5Yt5Ofnc/r0abq6uqivr8fT0xOdTodKpRJBqutAV1eX4/4uLCwkKSmJm2++GU9PT+rq6qipqQHAz88PpVIpghrXGavV6ij9J+5nQRAEQRCEb4e188xQd0EQBGHYkXr4D3UXrnkig0oQhOuK1WrFbDZTVFTExo0bOXHiBLGxsUyePJnt27eTn5+PyWTCZrMxatQo1Gr1UHdZuEI2m42enh5OnTrF119/zcGDBwkMDGTSpEl0dXWRlZWFQqHAYrGQlpaGp6enCFgMUzabDYPBwJEjR9iwYQNlZWXExsbywAMP4OXlRVJSEp6enmRlZbF69Wrc3NwYP348QUFBjtJwwvBgMpkwGo0YDAZMJhNmsxmTyYTJZHIEqORyOW5ubshkMmQyGXK5HIVCgVKpxM3NTQSvBEEQBEEQvqOsVisV1Sc5eqKUouJSSsqrkMtlxMdEkRQfQ8rIRKIjwoa6m8PS4aLjvPPxp0SGhfLgXbfj6+N11Y9pMpk4eqKM1rZ2TGazy3p7OXBfby/ioyNRKIb2YeTGM80E+PkOaR+uJeJ8CBcjRmsEQbiumEwmampqePfdd6muriYjI4O7776b+Ph4xo0bxz//+U927NhBa2sr999/P8nJyUPdZeEKWa1WTp06xeeff87atWuZOHEiS5YsYcKECTQ1NeHm5sbGjRtpa2vDYrEwbdo0MQ/ZMGW1WiktLeWDDz6goqKCjIwMli9fTnR0NJs2bSI+Pp65c+ei0+l46623eP3113n44YeZNWsW/v7iqaXhpKWlhaqqKiorK6mtraWxsZGOjg66u7sxGAwAyOVylEolXl5e+Pr6EhwcTEREBAkJCQQGBuLm5jbE70IQBEEQBEH4NhkMRn7zl5f59/ur6OntveC2d922gOd++ROiwkO/pd4Nf80tbUycdydWqxWArF17Wf/xW1f1mA2NTUxftJzyqpoBbS+RSEiIieLJHzzI/UsWIZPJrmr/zlVaUcWCex6lsuYU41JG8fbLf2J0Yvy3dvxrjTgfwkCJEn+CIFw3Ojs7OXDgAC+//DLt7e3MmTOHefPmoVKp2LdvH2FhYURFRWGz2SgpKaG4uJjQ0FA8PT3FQOYwZTKZKCgo4IsvviA3N5fMzEwWL16Mj48PtbW16PV6pk2bdvYJuooKysrKAIiKikIul4sMi2Gks7OTgwcP8tprr9HR0cGsWbOYP38+crmcd999l/fff5/S0lJ0Oh0pKSnExsZSUVHB8ePHMRgMBAYGotPphvptCP0wmUzU1tayc+dO1qxZw4YNG8jLy6OiooLm5mYMBgMKhQJPT0+8vb3x9vZGo9EglUrp7e2ltbWVU6dOUVRURE5ODgUFBTQ0NCCRSPDw8BCf8YIgCIIgCFfoWi/xd7joOAvueZS1m7Mx95Flc76i4lLe+M9K4qMjGSUGzQdkzYYtfLF+i+P1yboGnvjefVd1+oS/v7GCNRu2XtI+za1tfL05m8/WbiIyLISE2Kir07nzPP38S2Tv3gdA/ekm6k83sfTW+d/Ksa9F4nycJUr8XZzIoBIE4brQ0tJCfn4+69evp6mpiYULFzJ58mRMJhNffvklWVlZzJ49m+nTpzNz5kzc3NzYv38///73v7n77rsZPXo0Xl5XPzVdGDy9vb0cPHiQbdu2ceLECcaOHcuiRYtwd3cnJyeH/Px8vL29uffee5k2bRpubm7k5uayZcsWbDYbM2fOxMPD41t9okq4PM3NzRw8eJDVq1c7glNTpkzBarWyefNmNm7ciK+vLydPnmTz5s1YrVbGjBnDkiVL2LhxI3v27MFkMjF//nyioqKG+u0I5+ju7nYEEktKSmhoaMBsNqNSqYiKiiIgIABfX188PT3RarW4u7s7gk02mw2TyURnZycdHR20tLTQ3NxMbW0tdXV11NfXU1RURExMDCNGjGDUqFFotVpxzwuCIAiCIFxn6hoamb5oOfru7kvaz2Qy8fCTvyYuJoq05JFXqXfXj6qTtU6vTSYTtfWNjIiLvmrHrKg+edn7FpdVcNsDP+T9f77IskU3D2Kv+lZe7ZzlVX2qtp8tvxsGej4KjhYTHxOJWqX6NrolXINEgEoQhGGvubmZ3NxcsrKyaGho4KabbmL69On09PSQk5PDjh07MBgMbNy4EblcTmZmJjfccAMymYwNGzbw1VdfYTQaSUtLE0GqYcBms2GxWDhw4AAbNmygurraUdrN29ub7du3s2vXLk6ePIlWq2XlypXceuutjB8/HqlUyp49e1i7di1SqZSJEycSGBgo5qS6hjU1NZGXl8fWrVtpaGjgxhtvZPr06VgsFnJycti1axdxcXHMmTOHkpISjh07xubNm1EqlaSlpWE2m8nKymLv3r1YLBYWL15MQECAyKgZYgaDgdOnT1NQUEBBQQF1dXVIpVL8/f2JjIwkNDSUgIAAvLy80Gq1aDQa3N3d+7xXjUYj3d3ddHZ20tbWRnt7O7W1tVRXV1NXV0deXh4nTpygurqa5ORkoqKi0Gq1Q/CuBUEQBEEQhKvhb6+/4xKc8vf14Y5bbmL2tEwmpY+lo7OLNRu28oeXXqO7p8exXW+vgaf/+BKbVr7zbXdbGACT2eSyLDoijJDAAADaOzupOllLl77/4ORDP/kVEWEhZI5Pu2r9hLO/S851oT59Fwz0fPzqT38n79ARHlp2Oz988G4iw0TZze8aEaASBGHYslqtdHR0kJuby+bNm2lubmbSpEksWbKEtrY2cnJyyMvLIyAggIyMDLKysti1axcSiYQpU6Ywa9Ysenp6yM/PZ/PmzVgsFtLT0/H29h7qtyb0w2q10tnZSXl5OevWraOqqooRI0Ywe/ZswsLCyM7OZufOnSgUCqZPn05vby979uzBw8OD6dOnk5GRgVQqZcOGDaxduxabzcbEiRMJCAhALhf/JF5rWlpayMvLY8uWLZw+fZobbriBW265Bb1ez86dO9m3bx86nY777ruPMWPGkJycjEqlYu/evXz++efcf//9pKamIpPJ2LZtGzk5OcjlcmbPnk14eDgKhWKo3+J3jtVqpb29ncrKSvbt20dhYSFGo5GQkBBGjRpFQkICkZGReHp6DjjTyd3dHXd3d7y8vAgPDwdAr9dTW1tLaWkpx44do7y8nKysLGpra0lPTycpKQk/Pz+RTSUIgiAIgnAd+HpLttNrD62GnV9+RFx0pGOZv68PT/3gIYIC/Hjgx7902j53/yGMRuNVLVUnDJ5nf/44dy++xWlZc0sbJ8oreelf7/LVpm1O68xmMz9/9gX2rFt1Vftls9muavvDzaWcj7b2Dv7+xnv8483/cMucGTz+yH3ckDH+KvZOuJaI0ThBEIYlq9WKXq9n3759fPrpp3R3dzN9+nQWLlyIRqPhww8/ZNeuXURERHDXXXeRkZFBfHw8b7/9Nps2bcJkMnHrrbfyyCOPIJPJyMnJoaOjA6lUSmZmJgqFQsxPdI2x2Wx0d3dTWlrKihUrqKioYMqUKSxYsAB/f39yc3NZtWoVfn5+zJ07lylTptDU1IReryc7OxuDwcAtt9zCjTfeiEKh4MMPP+Srr77CbDYzffp0fH19RSbVNcJqtWI2m9mzZw+rV6+mvb2dmTNncueddyKRSFi9ejXbt28nMDCQZcuWkZmZCUB8fDwLFy5ELpfz2WefsWLFCpYvX86kSZPQarV8+umnrFixAqlUyuzZs4mMjBQBim+R/aGCgoICNm3aRF5eHklJSdx4441MmDCBiIiIQQsUazQaEhISiI+PZ8KECRw6dIisrCxyc3MpKytjxowZzJgxAz8/P3HfC4IgCIIgDGNms5nqU3VOy8aMTHQKTp3rntsXkrVrL+9/ssaxTCKRYLFYL3qsjs4ujpeW09LaRntnF2qVEk8PDxJiowkOvPg8M4cKjyGTyUgZOcJpudVq5eiJMlra2hmXMhKtRtPn/r29Bg4WHkOhcGfMyBED+u5cXFZBe0cnE9PGuKxram6h4Ggxvt5exEZFoPO4elUGmlvaKCouofFMM8FBAUSGhhAeGjxo7fv6eDHZJ5XJ4/+X3/z5ZV7451tO6/cfLuTrzdncPGfGBdu50mt8uQbzuDabjbLKak6UVaLTaQkPCSYsOPCyqoiYzWZKKqooLq1AqVAQFhJEXHTEVSvHZ7Va+XLjNr7cuI3kpAQef3g5yxbdjFIpHi69nokAlSAIw1JnZye7d+/mH//4B97e3ixcuJDZs2djtVp59dVX2bp1KxMmTGDx4sWkpaUhk8mYOHEiFouFjz/+mPXr16PX63niiSe45557UKvVbNmyhXfeOZvWb5+zSLh2GAwGDh06xCeffMLu3bu57777WLhwIWq1mm3btvHBBx/g4+PDXXfdxaRJk/Dy8sLDw4Of/OQn/Otf/2LPnj3o9Xruuusux1xVq1at4rPPPqO7u5vbbrsNT0/PoX6bAmfnJdqxYwdvvPEGGo2GhQsXMnfuXNzd3XnjjTfYtm0biYmJLFmyhIkTJzr2k0gkREZGsnDhQlQqFW+99RZWq5W7776b9PR0tFotZrOZVatW0dHRweLFi4mNjR3Cd/rd0tHRwebNm1m/fj0tLS3cfPPNzJ07l8jISFQq1VV5KEAikeDn58eMGTNITk5m27ZtbN26lVWrVlFbW8tDDz2EVqsVGZSCIAiCIAjDlEQiQSqVYrV+E2A6cOQopRVVxMdE9bnPK8//mp25+x1zKt2/9DZUKmW/x9i6cw/P//119h4swGKx9LlNfEwkDy67nZ8++kCf3y2XP/ZzVq5ZB8C9dyzkvVf+Qkl5FT/57R/JzT/kKH8mlUoZM3IEv/7pD7h17o0AZO/ayzMvvsKBI8cwmc6WvFMplYxPTeaZJx/jhskT+uzTr/74En99/ewYx8ypGWxa+Q5Go5FnXnyVtZuzKCmvcmwrlUq5bd6N/OqJ7zNmVGK/5+JSnG46w89//yLZu/fS0HjGZX1a8ki+f/8yHrhr8aD+FvjD/zxB9u595B064rT8iw1b+g1QXck1/vWf/s4XG7ZQWlHttH15VQ1e8enoPDQsmjebV/74m0E97vne/2QNb3/0CUeOlbiUvJRKpdwyZwY/fOBuZk7N6LcNu5Vr1vG319/heGk5RqNzmUWNWs2yRQt46gcP9RkIvpLzca7C4yU8+rPf8vQfX+KRe+7k+/cvIywk6KJ9F4Yf2bPPPvvsUHdCEAThUtTX17N161ZWrlyJt7c3t9xyC5MnT6a9vZ0vvviCDRs2cMMNNzBv3jySk5NRq9WOL61eXl54eXmh1+s5fPgwDQ0NxMbGEh0djVqtpra2lry8PLRaLb6+vqjEJI3XhI6ODnbu3MnatWtpaGjg1ltvZf78+QBkZWWRlZWFTqfjnnvuITU1FW9vb6RSKTKZDI1Gg7+/P729vZSVlVFXV0dgYCCjR49Go9HQ0NBAaWkpHR0dxMTE4ObmJjIqhojNZqOuro6tW7fywQcfEBwczNy5c8nMzESv1/Ppp5+ydetWxo0bx80330xKSgoajcbpx4xUKkWhUBAQEIBKpaKkpISamhoUCgUjR44kISGBjo4Ojh8/zunTp/Hy8iIwMHAI3/X1z2q10tLSwpo1a9i+fTtqtZr58+czf/58goKCUCqVV/Wes3/+K5VKgoODCQgIoKenh6KiIqqrqwkODsbDw0MEqQRBEARBEC7A2ukaYLgWSKVSPvzsS1rb2h3LzGYzH61ei8LdncS4aJfsC3d3d+5efAtJ8bH8+Hv384P7l/XZdndPDw8+8TRP//ElTtbVX7BkWUtrO9tycsk9cJh7Ft/i9BulvaOTh376K0cQ7eiJMmIiw7njkR9zrKQMo+mbAIDNZqOh8Qyr128hZWQCOfsOcO9jP6emtt4pCGfPHPvs603ckDHBJRvJZrPx0E+edgS+KmtOMf/GG3jop7/iv6u/prm1zWX746Xl/OeTL4iNjGB0YrzLe8zZl8/23fuclv3wwXvw83GdJmHlmnXcev8P2H+osN+5h+obm1i7OZv9h4uYN3MqKqVrkHD1us0cPVHmtGzR/NkkJ41w2dbOfu7PL/14pqWVnzz6gNOyK73GBUeLuf/H/0NLa3uf+5hMJrr03ew/XMj8G28gJChgUI57LoPByI+e/j3P/u1/OVXX4Ahinstms3GirJIPP/uKltY2bpoxtc+g4OmmMzzw4//hz6/+m9NNZ/rMLDSZTBwsPMZ/Vn1BQkwUSQnfPPR5Oeeju6eHgmPFdHR29blPT28vu/MO8s93P+ToiTJCAgMGNfvuapN6XL3Mu+uFCFAJgjBs2Gw2ampq2L59Ozt37sRqtXLbbbeRnp5Oe3s72dnZ5ObmEhcXx6233sro0aPx9PR0+kdXoVDg7e2NSqWira3NMV9RZGQkkZGRyOVyqqqqKCsrQ6FQ4OnpiVarFeX+hoh9vpqcnByys7Npa2tj3Lhx3HHHHVitVnbu3Elubi7u7u4sWLCAjIwMfHx8HAPNEokEmUzmyKZqb2+nvLycuro6oqOjCQ8PR6vVUl9fT2lpKXq9Hj8/P1QqlRisHiL2+cU2b97MokWLmDRpEiaTiR07drBlyxbi4+OZP38+Y8eOxcvLq8/AhkwmQ61WExQURHd3N+Xl5dTX16PT6UhNTaWqqopDhw6h1+sZMWIEMTExQ/BOvxvMZjOtra1s3LiRnJwcNBoNs2bNYurUqYSEhODm5vatfL7aPws0Gg1+fn7odDo6OjrYs2eP47Nep9OJ+14QBEEQBKEf12qACs4Oqu/KO+C0rNdgYPP2Xbz87xVs27mHmtp6lEoFIYEBSCQS1CoVY0cnERUe2m+7f/nft3j9vY8uqS+VNacIDvRn3JjRjmUHjxzj3Y8/c7y22Wys2bAVg8HYbztWq5VPvtzAui3bnQJT5zOZzXz+9Sb+3313OQXiWlrbefav/+u07TsfferIGuuPxWLhiw1bmDV1sksQYKABqpVr1rH8sZ/T02u44LHsyiqrOVB4jHtvX+jy2+ByAlQA0eGhjuwxu84uPf/vvqVoNWrHsiu9xhq1itdXfOySZXQ+Nzc3nvrBQ3h7eQ7Kcc/1h5f+yatvf+CyfVhIEAaj0SUza//hQgL9/Ug/rx2z2cyUW5a53Ev9MZpMfP71JqZNGu+4jy7nfIxLGcWPH1lORvpYDEYjZZU1ff7N22w2jpWUsWLlatZtyUalVJAYF3PNl+0XAaqLE4+IC4IwbNhsNo4fP862bduoqqoiMzOTCRMm0NXVRVZWFnl5eYSEhHDvvfeSnJzsEpyCs4OU3t7epKWlsWDBAiIiItixYwc5OTlYLBbGjRtHUlISu3bt4sCBAzQ2NoqJLoeQzWajsbGRrVu3cuzYMUJDQ5k9ezYajYacnBxycnKQy+XMnDmT6dOn4+3t3ecAs0qlYuTIkcyZM4fExEQKCgpYs2YNnZ2dJCUlkZiYSH19PZs2baKurg6z2TwE71aQSCQoFAq8vLxQqVTo9XpKSkrYsmULO3fuJCgoiNtvv90pSw7O/p1YrVbHvWpvJyIigrlz55Kenk5TUxOff/45u3fvpqysDJVKRWxsrMieuorsmVP79+9nw4YNqNVqZs6cSUZGBkFBAyvNYLPZMJlMNDc3U1xcTF5eHnv27GHPnj3k5eVx9OhR6uvr6enpGdBntUQiwcvLizFjxrBgwQJGjBjB3r172bVrF6dOnRL3viAIgiAIwjD0k0cfICm+79LdZrOZXXkHeO7vrzHllmUEp2Sy/LGfs2XH7ou2+9HnXzm9VqtUvPm35zh1eCddFYcpyd3MYw/e47LfZ19vcnpto//vqTfPmcFvn/whc6ZPueCDW6mjk/jZDx5m2qR0l3WdXXqydu292Ntx8PX24rdPPsZ//vcFHr77DkfQxNFfm43vPfXry/pu3KXX84s/vOi0TCqV8svHH6V490ZOHd7JR6//jUB/P6dtsnJy+deK/17y8frj7eXpFIiyO9PS6vT6Sq+xh1ZD/ubV/P0PTxPg5+u0jVaj5nc/e5wPXvsrhdvXEhMZPmjHPdcX67c4vb5h8gSqD2yncn8WrSf28/oLz7oEcT5ft9mlnX+++xHHS8udliXFx7Lxv29zumgPB7eu4Z7bFzqtt9ls3Pf4LxzB1ss9H1KplDnTp7Dy3y9Tc3A7f3v2l4waEefSR7uDhcd48ImniR4/i5x9+f1uJwwP4jFRQRCGjbMTl1ro7e3FYrGg1WodT+bn5+cTEBDA4sWLGT9+PBKJxPHl7txBaztvb29SU1Nxc3PjvffeY8eOHRgMBuLi4tDr9bi7u+Pp6elSPkz49tlsNnp7ezGbzcjlcnp7e8nKymLjxo14eHgwffp0Zs2ahbe3t9M+dvbrp9VqSUlJQS6Xo9frycnJQalUEhsbS2dnJwC+vr6o1epr/gmc61l4eDhz587lzJkzHD58mP3799Pc3IyPjw8PPfQQ6enpTvMVWa1WjEYjDQ0NjiwY+/WTyWTExcUxe/ZsOjo6+OyzzygvP/uFe/LkycyfP5+4uP6/9AqXz2az0dnZSXFxMevWrcNoNDJr1iymTJmCr6/vxRvgbPmHzs5O6uvrKSkpoaysjKamJjo6OrBarSiVSvz8/IiNjSUhIYGwsDB8fHxQKC4+ga6HhwejR4/m4Ycf5vXXX+fAgQNotVo8PDwICAi40rcvCIIgCDQGGawAACAASURBVIIgfIt8fbzYs24lDz7xNGs2bL3gti2t7axcs46Va9YxadwYnv3Z48yaNtllu4bGJsqrahyvNWo16z9+i8njUx3LoiPC+MfzvyZ79z6OlXyT5ZN/uHBA/f7tkz/kmad+5Hi97PtP8tnajS7bLV4wh5X/ftnxG2jeskfYunOP0zbl1TUu+/UlMS6G9R+/5ciOunvxLXz//mVMWbjMKaOrpLyKrF17mTN9yoDatfvLq29Sf7rJedlvfsZP/98DjtdLbp1PWsooxsxc6JRp8/p7H/HDB+++pONdSICfr0t5wTMt35Q2HKxrHBMZzuMPL2fVmnU0nml2LA/09+M3P/2BS78G82+rsuYUxWUVTsvuum2Bo5Sgm5sb37t3Cf6+Pix99CeOzKSkOOcqIk3NLfzhpX86LQsJCmD31yvx0GoA8PH2YsWrf8HdzY33Vn7u2K6uoZE1G7ay9Lb5l3U+zufn480T37uPJ753H3mHjvDefz9n1Zfr6ezSu2zbeKaZyupTTJ3oGrgVhg+RQSUIwrAhkUiYMGEC8+fPx9vbm/fff58XXniBVatW4e3tzdKlS5kyZQpSqdQlqHRudoW9LZ1Ox+TJk3nggQew2Wy8//77vPjii6xbt44777yT+fPnExERIQJUQ0gqlRIREcHChQuJiYlh9+7dPP/887zyyit0d3dz8803s2DBApcB7/MzauxUKhUpKSk8+uijREZG8vXXX/Pcc8+xdu1a4uLi+P73v09CQgLKPmpfC98ODw8Pxo0bx1NPPYWXlxcHDhzgxIkT+Pv7k5aW5hScgrNBjPr6el5//XX27t1Lb2+vU3symYyoqCgSEhKw2Wzs3r2b1NRUFi9eTHp6urjWV4nVaqW2tpadO3eSl5fHkiVLmDp16oCDU3B27rkdO3bwwgsv8Ne//pX8/Hz0ej2enp74+Pggk8koKytjxYoVvPDCC6xevZqmpqaLN/x/7J8HS5Ysobu7m23btlFcXCyyZgVBEARBEIYhrUbDJ2+9wt+e/SUJsVED2mfvgQLmLnuEz/vISgkK8Gf2DZmO1688/2unAMK5xqcmO73u7NL3OQ/QuZRKBb947HtOy5YsnOeynUqp5D+vvuD0G+jeOxa6bNfW1nHB49m99PtfupTuGzs6iZee/aXLtp/2ESy7mLWbned9Cg8J5vGH73XZLi46koeW3e607ER5pVPg5krJZK7D3uf+Xrza17g/g3lchbu7yz5/eOmfvPnBKnp6vnmvt827kS2fvsejy5fyt2d/ye9+9iOnfXbm7ncJAP3mpz90BKfO9fzTP3FZtn7bjj77f6UmpKbwrxd/z7Gc9UwaN+aqHEMYeiKDShCEYcXHx4eZM2fi7u7OypUr2bFjB2q1mqioKGJiYlzmo+np6aGwsJB169Zx//33Exoa6vJ0fVxcHOHh4Rw6dAibzcatt97KkiVLCAsLw83N7dt8e8J5JBIJSqWSjIwMZDIZX375JVu3bqWxsZHly5cTFRWFVqt12sdqtXLo0CGOHz+Oj48P8+fPd1ovl8vx9/dnzJgxHDx4kN7eXjIzM7nzzjtJSkoS1/waIJfLCQ8P59FHHyUoKIgDBw6gVqsxmUyO4GN1dTXt7e3odDrMZjP5+fkkJibS0tLCyZMn0ev1jBo1CoVCgcViQSqVEhoaSmpqKkuWLCE2NrbP+auEwdHc3MzevXspLi7mpptuYurUqeh0ugHvX1ZWxvr168nLy0OpVPLAAw8QFxdHQECAY15Ag8FAc3MzZWVlHDp0iNzcXE6dOsXy5cuJjY0dUCYVQHp6OlVVVezcuZPNmzcTGxtLcHCw+PsQBEEQBEEYZiQSiSPzovB4CZ9/vZEv1m91ykDpy/0//h/CQoKYmOY8AL7+47fYf7gQb08dcdGRjuXVp2rZd6CAnH35bMvJpbSi2qXNjk49vj5e/R5z8vg0pzmj4GyGyvnGjx3tsp1vH9v1Gi4+31NSfGy/GVFLFs7jR0//wWlZ0fGSi7Z5LovFQmml87mYlD4Wi8WKxeI631b6mGRgpdOyqppTxEZFXNJx+2K1Wqk+Veey3P+8snNX8xpfyGAdNyQogBGx0Zwor3Ssrz/dxGO//D1P/u7PTB6fxswpk7h59gymTRrPtEnj++zP8dIKl2WZE9L6nCfN08OD6IgwKmtOOZZVnfP/g2nvgQLe/fgzVn25nu6enqtyDGHoiQCVIAjDikwmIzAwkIyMDMfrM2fOoNFocHNzQyKRYLPZKCsrQy6X4+bmRm1tLZs3b+a2227D3d0dg8GAXC4nNDQUuVyOu7s7Go2GiIgIkpKSWLRoEWFhYS6ZGsLQkEgkeHp6kpaWhsViQSaTkZWVhUajwf3/nhYyGo00NzfT1taGr68vNTU1FBQUEBwczE033URVVRVyuRxfX1+0Wi1ubm4olUqCgoIICgripptuYsyYMQMe0BauLolEglwuZ9SoUdTW1tLc3IxcLndktlgsFiorK9m1axft7e2Eh4dTV1dHbm4utbW16PV64uPjiY+Px93dHZvNhlQqxcPDg5SUFMLDw10Cm8Lg6e3t5ejRoxQVFaHRaFi0aBG+vr59zg93PovFwunTp9m4cSNHjhwhNDSUWbNmMWrUKNRqNSqVytGO1WolJCSEqKgoYmNjyc3N5ciRI3z44YcsW7aMuLg4VCrVRY/p4eFBZmYmp0+f5ujRo+Tm5jJ79mw8PDxEkEoQBEEQBGGYSk5KIDkpgWd//mNq60+zdeceNmTtZPW6zS4Z8waDkbc+/MQlQAUwfmwyx0vL+dMrb5B36Aj7DxU6lS3rz8WGEkIC/V2WadSu312D+tquj7mVBiI+Jqrfdd5enoQGB1Jbf9qxrL5x4NUJAMqrTrpkFX361QY+/WrDgNsYrO/fJ+vqncoH2gWdN/cVXL1rfDGDddw3X3qeOx5+nKbmFqflBoOR7F17yd61l9/+5R+MHZ3EvXfcysN3345W45wZVXze3FMAY2a4Zur1ZzCnSWhpbePDz7/i3Y8/4+iJCweX/X19iI4MG7RjC0NDBKgEQRh27NkV7u7u1NfXs2/fPsfgtf2LZmFhITU1NSgUCmpra6mpqWHfvn0YjUakUilJSUkEBgY69pPJZISGhpKWlsaoUaNEFs01RiqV4u/vT3p6OgaDgaqqKtzc3BzX22AwUFtbS3Z2NjqdjvLyckpLS2lqamL16tXU1NSQmJhIWlqaIzBhsVgIDAxkzJgxjB49Gk9Pzwt1QRiAtrY2Kisrqaqqwmw2O8osms1mJBJJnz827BPvnh+8sM85d/ToUerq6ggLC3Nap9Pp0Gg0nDhxgkOHDtHU1EReXh7Nzc0kJCQQFBTkaNPeVkdHB0VFRaxZswaNRuNYf/48dRfrr0Qiwc3NDblcjo+PDzExMURGRrps+13V3NxMfn4+7e3tpKenM3r06AEFp2w2G3q9nry8PI4cOYKvry+zZ89m4sSJeHh49LmPu7s7Wq0Wb29vVCoVNpvNkQWl0+kGVKZVIpEQGRlJamoqJ0+eZNu2bSQnJ6NQKEQJSEEQBEEQhGHgq03bWLHyC6IjQvn5Y48QFOAc1AkNDuT+pYu4f+ki9h8u5JGf/tolq6qveaPqGhq57/FfsGNP3qD32cfL9fenWx/fmVV9fB+93CBOUIBrcOZc/r4+TgGq1gGWDbQ7VVd/Wf2y8/byZMyoxCtqw6680rVUoEQiIcDPx2nZ1bzGFzKYx508PpVD29bw0E9+xebtu/rd7nDRcQ4XHWfFys9Z85/XiQwLdaw7VddwRX2Ynjnhiva32Wzs2JPH2x99yhcbtvQZXDzXmFGJPP7wcu66bQEKhWuZQ2F4EQGqc1itVrq7u2lqasLNzQ1vb280Gtdam+eyWCycOXMGg8GAWq3Gz+/sh73NZqOrqwuDwYBCoeh3YOVS2Gw22trasFqtqFQq1Gq1y3qLxYLFYsHd3f2yMj8MBgNtbW20t7c7yugMZFDpWmU2m2lsbMRkMqHVai9p7gth8LW3tzsmuLdYLACO//b3Bcs+gWNf63t7e+ns7MRsNjsFp+ztVlRUUFNTQ3NzM3q9ni+//BKFQkFCQgIjRoxwastms2EwGGhoaGD//v1IJJI+nwC5UH/s6+0D2zKZDC8vL/z8/C6ptNX1wv6ZdPLkSdrb2x3l2SwWS7+D/xc6vxKJhK6uLpqbmx1tnXvNbTab4/rZj+nm5kZXVxcBAQFER0c7bWvvy5kzZzh69CinT592+dy0l5Oz96mvuc3gm6eFJBIJarWa0NBQtFrtoD5FNBy0tbVx+PBh1q9fT0NDAxqNBrVajVKpxGq1Ou53O3umFOC4j88lk8no7OzEZrPh6+vrCBzbA8pxcXGUlZWxefNmOjs7qa6uJiQkhOjoaOLi4pDJZI6/Na1Wi4eHBxUVFXR2duLm5uYIjp3PHrg+v7/2+7q3t5fW1lZUKhVjx45lwYIFIkDFN4E+e4DYx8eHjIwMR6bjxRiNRhoaGtizZw8ymYxJkyYxadKkAWW7KZVKEhMTsdlsHDx4kMLCQqKjo/Hz87vodzn7/klJSVRVVbFq1SpOnDiBj48PCoVCZNIKgiAIgiBcw46VlHH7Q487XpdWVvPV+2/0u/34scn89Xe/YME9jzotLy6rxGg0Or67tnd0MvnmpU4BG7sAP18mjRtLRvpYpkxMZ+2mbbz42ttO21wsiKQcYPUOzQAqAgzU6aYzF1x//nsNDwm6pPbDQoJdlmk1alKSRvSx9Vlu7m6EBQeRMnIE82ZN67PM4eX4cuM2l2UjYqOdxjiv9jXuz9U4bqC/H+s+epN9Bwv47+qv2ZC1k4rqk30ev6i4lEnzllC6d7Mjkyqsj2s9ITUF+QXGNDw9PUiKj2Xs6KQ+508bqP+s+oI/vfJGv/21k0gk3DJnJj/+3n3ckNF3qUJheBq+kYeroLu7m4KCArKzs/H29iYzM5Pk5OQLDjA2NjaSnZ1NVVUVo0ePZt68ecjlcjo7Ozl+/DhVVVWEhYUxadKkfgdkB8JisdDV1eV4Ijk+Pp6kpCTHP5w2m42Ojg46Ojro7Ox0lCe71CyQuro6srOzOXToEEuXLmXkyJH4+PhcfMdrkMVioampia+++oozZ86QkZHBtGnTRGbMECorK2PDhg3s37+fzs5O5HI5UqkUqVTqGOg/n/2eOX+9fcCwsbHRUcLp3ADC+PHjMZvNnDlzhj179tDZ2cn+/ftZsmQJ06ZNc5prSCKRIJFIqKmpoaSkhB07diCRSBzZHwPpz7l9slqtGAwGdDodU6ZMYc6cOYwdO/ayztlwZrFY6O7u5osvvmDfvn00N59NVbcH0C/n/FqtVrq6umhqaiI9Pd1xzpVKJZGRkcyZM4c1a9Zw6NAhKisrHRkUmZmZpKSk4O3t7WhPoVDQ2NhIYWEhOTk5qFSqPgMo5/apv/7aA2YymYwRI0Zw7733kpycPKDyYtcTDw8PgoOD8fHx4ciRI6SmppKYmIiPj48jeHu+87OXzl8nlUrx8vIiOjoajUbj+Dvo6OigtraW6upqlEoler0euVxOa2sr5eXlJCcnEx4eDpz9mxsxYgRLly6loaEBi8WCzWbrN/DQX5/smVgVFRXU1taiUqkIDAwkICDgis7bYLGf46EKqNjv+cOHD2MymYiPj2fkyJED3r+zs5OysjKqq6uZPHkySUlJl1SKUaVSOT4HsrOzqaysJDIyktjY2AHtHxYWxpgxY1i/fj0HDhwgOjoanU4nSn8KgiAIgiBcwzZs2+nyelN2DjfNmNrvPrOmnp3j+NzffxaLBav1m997f3n1TZcAQub4NF545ucupQA/+HSNyzGuxVLRx/so42bX0NjkUiIuITa6n637FhN5dh7vc8v8TZ2UfsGA4dWwZ/8h/vWf/7osX7b4ZqfXQ3WNB/u4FouF/YeLGJkQy8S0MUxMG8M/+DVlldVszM5h5Rdfs+/gEad9zrS0sil7F7fffBMAI+JiXI7zyh9/Q/qY0Zf1Hi/FJ19tuGBwSueh5cG7FvPYQ/cSHSHK+V2PRIDqHCdPnmTp0qX09vZitVq55557eOaZZ/D3d633ardixQr++9//Ul5eTkpKCmq1mszMTNatW8ebb77Jvn37iI6O5r333iMxMfGysyj0ej3vvvsu77zzDuXl5cyYMYPnn3+etLQ04OyT55988gnPPfccHR0dPP744zz00ENOGQMXYzAYyM7O5sknn8RoNFJUVMQzzzzDjBkzLqvPQ81gMPD222+zcuVKKisrmTlzJj4+PqSmpg51176zfH19CQ0NpaCggJaWFlJSUoiLi0OpVLoEBuzsA63nBwbg7D/K9syKtLQ0p6zClpYWioqKKC4upuf/JlLs7e2lsLCQmJgYoqOjCQo6+4SISqVi+vTpBAQE0Nra6hi4vlAQqq/+yGQyenp6qKysZNu2bWRmZhIWFjZsg7xXSiKR4O7uTnR0NEVFRVRVVeHt7c2MGTOwWq2XfH7t6+1ZN+PGjXMEBuwZsMePH+f48eO0trYilUoxGAwUFxdTVFREbGys45rb95fJZDQ0NDjKtvX3d2Y/Rl/9kUqlHD9+nKNHj+Lh4UFcXBze3t7fuewpAB8fH1JSUmhqaqKmpoYbbriBadOmER4e3m8QeiBkMhlyudyRzWIymTh+/DilpaXExMQwd+5c3n77bTIzMwkICKClpYVdu3aRmJiIQqFALpcTERFBYGBgv1lTA2E/dn5+Pr29vQQEBDBx4kRiYly/zH/bLBYL7e3tjsznofhBbM9aLigoICgoyOlBmoHo7u6mqqoKk8lEYmIioaGhF9/pPBqNhpkzZ7Jjxw7q6upoamoacIDKPsfh5MmTyc3NZeLEiYSHh4sAlSAIgiAIwjWsryyfZ154hTnTp/T74Nb+w0UuYxCB/n4old9879u5d7/TeplMxn///TLBfcwJte9ggcuyjs4udB7X1ry3JeVVfL05m5vnuI7zrVyz3mXZiEsMUMnlcuKiIpwCYXsPFFzw4cAuvZ6f//5FDhQUsXzJbTz+8PJLOub5unt6ePSp3/T52/7uxbc4vR7sa3z+e+yvTN1gHlff3U3K9IXU1NahUavJXb+KpPizv3/ioiP5UXQkP3roXv79/kp+9PQfnNrJzT/kCFAlxbv+ps3NP3TBANW6Ldv562tv4+npwat//I1TyUAY+PnoT1x0JD96+F7uX3Kby5xZwvVFBKjOYbPZaG1txWg0YrFYOHHiBPv372f+/Pl9biuRSNi1axeVlZV0d3ej1+vp7u5GIpHQ2tpKa2srPT099Pb2YjQar6hvEomEhoYGOjo66O3txWAwOP1j2traSnV1NQ0NZ2uG7ty5k8WLF1/SMbq7ux3l/QBqamocA/vDkc1mo66ujpaWFnp6emhtbaWp6dImeBQGV1BQEKNHj6ampgaTycTkyZPJyMi4YPmli2VXwNnsGU9PT5RKpdMcVD09PUyaNImkpCS2bdvmCECfOXOGiooKYmJikMvluLu7M2rUKCIjIx33P/T9NMyF+iOTyWhpaeHAgQMUFhaSkpLCqFGjCAwMvMQzNbiMRiNWq9UxyP9tZVdIpVIUCgVjx46lqqqKnp4eQkNDWbRoEVar9ZLPr329vfyip6eno3yqwWDg1KlTVFdXk5KSgr+/Pw0NDajValJTUx0D32FhYY7Se1FRUfj4+Dh9nvZ13AuVHTz334Kenh5HSbPAwMAhz9a0WCyOJ9fc3d2/lYCFPSipVqtxc3NDoVCg0+nw8PDoN+h4KW3br4+bmxujRo1yBH/d3Nz46KOPSEhIYOrUqfT29iKXy1Gr1Y7MSrlcPihBQ6PRiE6ncwTMVCrVNRGMNJvNnD59mp6eHsd8TN92JpXRaKSiooKmpiYmTZrkUkr1Qmw2Gz09PdTX1+Pm5oafn98lZU/Zubu7Ex4ejoeHB21tbXR3d/f7edMXPz8/Jk+ezLp166itraWrqwsvr8EpMyIIgiAIgiAMvhsmT0ChcMdg+Gbc7WDhMe7+wVO8+/KfUKmc53A6XHScpY8+4dLOrKkZTq8bGp3L4cnlMlRK1weXPvj0SwqOFrssP1FW0WfZtKH25O/+zIi4aOJjohzLdu8/yNN/fMll2xsmX3oZtTnTM50CVK1t7fzmzy/zx1896bJtQ2MTix54jPyCIgCOlpTxvXuWOAUK+9Kl76a9oxP4v+kajEaamlvZtS+fv772DjW1dS773HHLXKLCnQMog32NfX28nda3tJ2d6uD8sYHBPO6m7F2O96vv7ua2+3/Izi8/ItDfeb6xB+9a7BKgcj+nX5PHp6FWqeg+Zxz4xX++xS1zZrqcN4A3P1jF4796zjFeMi5lFM889SOnbQZ6Ps43c2oGTzxyH/NmTRPl1r8jRIDqHEqlklGjRlFcXOwoM7Nv374+A1Q9PT1UV1dTWVmJXq8HQK1WEx8fj1QqJTU1lXnz5hEVFUVMTAwxMTFXNNG2XC5n5syZdHZ2Ul9fT0ZGhqN0EZzNDOnq6sJkMjkyAgwGwyUdw2w2O6Xh9vb29pvVMlzYs+Hg7GDtpZ4TYXAplUp8fHzw9fVFp9MRHBxMZGQknp6eToGJSx3Itv+DZd/XZrM5AhEqlYqSkhJ27drFzJkzkUqlmM1mgoKCHIPK9hJiXl5el3Vs+z5SqRSdTkd9fT0eHh74+/s75jAZSiaTia6uLgC8vLy+tf7Yr4v9mnt6euLr60tMTMwFn6C6lPbtbcjlcnx9fZk4cSLe3t4UFhZy6NAh/P39Wbp0KWVlZYSEhDjeu0QiQavVotVqrzhwAmfLV3p6euLj40NgYCBKpXLIv0jZs8oMBgMqlepbH2S3X59zr5O9tJ69tOe5ZTHtn9Xnz/9m3we+KeMplUoJCwtzZNC1t7c7SncmJCQ4ynXYv/ja5xKzt2Ofm8q+7twApf0Y585TZl9uXzfU17Yv9tKiLS3flOX4NoNUFouFzs5OTpw4gUKhuOTsUZvNhkKhICYmBj8/P/z9/S+r7/ZAlFqtpru7m+7ubsxm84AzudRqNbGxseh0OhoaGmhqaromAs6CIAiCIAhC3wL9/XjswXv4+xvvOS3/bO1G9h44TEZ6KslJCfT2Gjhw5CibsnP6bOfJ7z/o9Hry+FSnQIfBYOR/nvsrzzz1I0KDA6k6Wcv7n6zhub+/1md7R0vKmDVt8hW+u8FXWXOKSf+fvfeOr6M68//ftxfdXnSLereKJcuWZMlFLhhj0wwxxSQQIO0HSdiQsNkf27LJlmwSlrAkmwKhJMDSIbRQjMHGBtvYuBsX2ZYsq/d6pdvv9w/vDPdKcpdsE+b9evkFmpl7zsw5M3Ol53M+z3P5DXzz5hvIyUznw4+38do7743LNFFbXcGyxbVn3P6P7vkuz736ZoII84vfPIJSqeQbX7metBQPoVCIZ/78F/7+P+6ns7tHPC7V4x4nTskY/zfBd+79Cd+59yenfU75OZk8/F//Nm77ZM+x15WY+t03MsK37/0Ji+dV03CsGUOSnq/dtHJS+51ZWpywr76xicu//E1+f9+/MqN4GiqVipa2Dv7rt4+Oa2d60WcLCr3uZH50z3e499//S9zW3tnN8pu+we9/8RNqKmagVqupO3KUn9z/a55/9a2EtnIy08e1f7rjAaDVavjKl67irm/cQnFB3oTXLfHXiyRQxWEwGLj88stpaWkRhaC9e/cyOjo6Ltg4ODjIu+++S39/P7FYDIPBQHZ2Nnl5eSgUCmbMmEEoFKKwsJCMjAy8Xu+4/iKRiOiyisViKJVKDAbDhGkANRoNc+fOxe/34/P5KCwsFFNVwXFxKf7LJBQKMTo6ekarhsdyqsCQ4Ery+/0oFAp0Oh1Go3FczRUhOO73+3G5XCcM7g0NDREIBFCpVJjN5nH7hdozfX19YuAxKSlJXMk+EZMReJaYXOIDwPHCEpDgXBL2jw0sC0FigeN5oqMJ7QJizSfhGSgrKyMnJwePxyMeK9w38X2MDY7HB87j+54oeC1cS/x1XQyBbKGGnSA62+12sQ7U+WCsSAGJafOEcR8rWAhjPHbOo9EokUgkQazQaDSkpKRgs9nQ6XSiG8PlclFeXk56ejoKhUJMAzl2zuP7PtE+4XyE+0EQOS7W90wsFiMQCNDf3y/WfDvfbp/4MRsYGGDnzp309/fj8XjIysrC4XAQi8Wor6/n2LFj+P1+PB4PhYWF6PV6QqEQe/fupbm5Ga1WS15eHl6vF5VKhV6vR6/XE4lEUCgUXHvttaSkpIjuLQHhu/bo0aO0tLSg0WiorKzEYrEQjUZpb29nz549hMNhUlJSyMrKwmKxMDo6SnNzM3V1dahUKjE1qHBdFyujo6NivTeZTHbexNJQKERvby91dXV4PB6cTucJv5tPhNVqpbKykmg0es6pUSf6/jgdFAoFJpOJ3Nxc+vr6aG9vJy8vTxKoJCQkJCQkJCQuYv7zH++hp6+fPz3354Ttza3tvPDaW7zw2lsn+ORx/uMffkBZ8bSEbUsXzuPZV/6SsO2xZ17i8WdfRq/T4RsZOWmb6z7awt9846sAKOTj/wab6O+y0952mu2diMGhYe7/3WMnPeZn//zDcdvUE/xOPPZvDZPRwC9/8vd8+c57Erb/9MHf89MHf4/H5aRvYBC/f/wC8n/6/rfHbXOMceGcKRaziRcf/TVGw/jMPZM5xwBe9/jsOX989mX++OzL4s8Dg8OT2m9mWgq1NZWs3/RZ2sDd+w4y54ob0Wo1uBx2GpvHO8q87mRWXrE0Ydv3vvlV/vel19izv07cdrihkSXX34ZOq8VkNNDR1T22KaYX5nPN8iUT9HF64/EPd9/BEzm/wG6TMld8UZEEqjgMBgNXX301zz77LHK5HL/fz7Fjx9i7dy/l5eUJwWyfz8df/vIX0ZWQnp5OZWWleIzP52NgYICBgQEGBwcTCpf7/X56enpoamriwIEDdHd3E4vF1qIOagAAIABJREFUMJvNOJ1OcnNzSU5Oxmq1iit+hT6Hh4cZGBhgaGhItEW2trayb98+2traxGP9fj/bt29Hq9WSmppKamrqpAWohoeH6ezs5PDhw9TX19PZ2YlarcZut5OVlUVOTg4ul0sUqhoaGti+fTv9/f0sWLCAzMzMBBFLON9NmzbR0tJCZmYmVVVVYtq3aDTK4OAgTU1NNDQ0UF9fj8/nQ61W4/F4yM3NJS0tDYfDMc4ZcjGIAxLjGRssjEaj9Pf309HRgc/nIzk5GYfDgU6nIxAI0N3dTVdXl1gfxGazoVQqCYVC1NfXMzo6isFgwOl0YjKZkMvl2O124HjKqfz8fG677TaSk5MxmUwJgcZYLEZ/fz+dnZ2MjIxgMpnIyMhAoVCI9VR6enqQyWTYbDYx6CoEgTs7O9HpdKSmpiaIy4Lr42IJZAvuhkAggFwuF98v5/MZEcZCEH58Ph/t7e0MDQ1hMBhITU1Fo9GIgkJ3dzeBQCDBlRQIBMT3p9FoJDk5GbPZjEqlwmQyYTKZiEajFBQUYDQaxfeIcD/A8fvN7/fT2toq9i20A9DV1UV3dzfhcBiTyST2HQwG6erqEh0qwpyPFS0vJoRrFdyjHo8HvV5/XkUqmUxGOBymtbWVhx9+mLq6Ompra7nuuuvEMd+0aROvvfYaHR0dLFiwgG9/+9vo9XpGR0d55513eP3113E6ndxyyy3i8y/cuwqFAoPBQEVFBQqFYty1RSIRjh49yp///GdWr16N2Wzm/vvvx2g0EggE2L17N/fffz/Dw8NcdtllXH/99ZhMJvr7+1m/fj1PPvkker2eG264AY/Hc0Y1lU4HQUQ5nZpsJ/tZEOtjsRjDw8Niew6H47w86+FwmL6+Po4cOcIll1xyxo49uVyO0WgU0wKeqbglILxfRkdHRVH2TNoS0kEWFxezbds2Ojs7CQQCZ5VuUEJCQkJCQkJC4vwgl8v5w/3/jlaj4aEnnj3tz+m0Wv71//8ed3/r1nH7Vl1zOe+s3cBzrybWZorFYgkCgt1q4cc/vIs/PfdnMVUdQMOxJvH/ZxRPG5c+raZixrg+p+VmYTGb6B8YFLfNqRxfR32i9uZUzjzV5fJfP76XR//3hYQUfGPR63T89uf/QuWM6eP2lZcUJvxss5rJyUwbd9z1Vy8nEAxyz49/Rm/fQMK+to7xpTcUCgX/+Y/3cMv1K8bti3f5nClXLV3Mb372LxPWdoLJnWOAb91yA7//0zMTijgCPb193HvXNye130d++R8sveF2jja1JGz3+wMTilNZ6ak89dv/Gve3rVKp5I2nHua7f/+vvL76/YR9o34/o37/uLaK8nN58+k/kBRXE17gdMdj7mncuxJ/3UgCVRx6vZ6ZM2fi8XjE+ks9PT2sWbOGkpISMcAhOHlWr14tBiZzc3OpqfksX+2bb77Jgw8+yPbt28nKymL27NmYzWY0Gg2NjY386U9/4r777puwWLvFYuHee+9l1apVZGRkAMcFr8cff5zf/OY3NDc3s3z5ch599FE8Hg8PPPAATz31lFh/KhqN0tbWxt/+7d+iVqv51re+xYMPPjhpwandu3fzwAMP8Morr4w7f51Ox7Jly/jJT37CtGnTkMvl/PGPf+QXv/gFSqWS+fPn8/DDD4ur0OF48PDAgQP88Ic/ZPfu3cyePZuf//znLFiwADi+Kvvdd9/lgQceYNOmTePOx2az8eUvf5l7770Xr9criVKfQwKBAJs3b+bJJ5/k4MGDrFq1ihUrVpCbm0t7ezuvvvoqzz33HFqtlq997WssX74cs9lMb28vP//5zzl48CAzZ85k5cqV1NTUJKTTVKlUZGRkkJqailKpTHDiCIHh7du389RTT3Hw4EGqq6v5x3/8RywWC4ODg7z88su89tprKBQKli9fzqpVq7BarTQ0NIjnlZuby/e//32qqqouxPCdknhX0ujoKE1NTaLgdqFcAbFYjEOHDvHYY4+xdetWysrKuOeee0hPT8fn8/Hee+/x0ksvie+7W2+9lczMTDo7O3n99de5//77mTlzJjfddBMLFixIEKDkcrk45xO9DwKBAPX19TzyyCNs2bKFsrIybr75ZubMmYNMJmPNmjW89NJL9Pf3M2/ePG677TbS0tLo6uripZde4vXXX0cmk/GDH/yA6urqCV2vFwPxzjXBKSSTyfB4POc92C58b+7du5f9+/fj9Xrp7e0VBZXm5mZ27dpFc3MzKSkp+P/vF99QKMTRo0fZuXMnHo+H9vZ2QqHQOCFQJpOdMHWlsMihvr6eHTt2YLPZGBoaEmt0dXV1sW3bNoaHh8nJyREXlcQvUtHpdMyfP/+MUsWdLqFQCJ/Ph9/vP6c6YcFgkFAoJI5PvEjlcrkS0hpOBeFwmOHhYdrb28/6HhvrljwbQqEQ/f39DA4O4vV6sVgsZyzIKhQKcnJy2Lhxo1ibVEJCQkJCQkJC4uJGJpPxP//5I668dCGPPfMSb7y7LqGMRTypXjcrLruE799xGxmp4+vrwPFYwpO/uY9Ur5vHn31pnNDisFlZeeVl/Ovf/Q02q4Xll9RSu+IrtLZ3AvCtW1aJx2q1Gr6y8ir+8NTzAGSkelk0t3pcn0qlkptXXs3/PPYUcNzlsnThvHHHjW0vxeNiSW3NuOPGkuZ1s+7PT/Lte3/Cexs2JQhhCoWC2eWlPHT/vzEtN3vCz9fWVJKVnkrDsWYAbl559Ql/1775uhVctmg+P/zJL3jzvQ/o6x8Yd4xKpeKGq5fzzz/49oQp4gC+dtNKduzZx/OvvTVhGwJGQxI2ixmPy8kl82u44tJFE4psY/ufrDkGSHbY+dOvfsatf3PvhKLMzOlF3P3/3Tbp/Walp7Lz/Ve5/3eP8/rq99lXd5hgMPHel8vlZKWncvVli/nxD+9CPyb7lYDXnczLj/8PL73xDj/5r/+hrv7ohOVfcjLTufeub3HL9StOeA+c7nhISEgC1QQsXLiQpqYmjhw5Qm9vL2vWrOE73/mO6Prp6upi69bj1sloNIpGo6GgoICioiKxjdbWVnHFeiwWY/PmzcydO5e2tjaeeOIJHnvsMXGVb2ZmJhqNht7eXrq6uvD5fDz44IPEYjG+/OUvk56eLqZAEoJNPp+PTz/9VFzNfaL6Vmq1+rSDPYLLK574IGAwGOT111/n17/+Nbt27SIWi6FQKLDb7WJwLRgM8sEHH/DVr36VP/zhD+Tn5wOfraBfv349bW1teDwe8ZyDwSAvvPACAwPHX8gqlYrR/1sFEggE+OlPf8pzzz1Ha2urmH7NbDYzOjrK6OgoQ0NDPP300wwODnL33XdTXl4unnv8Ncnl8glXqEucX+LdhPDZPPX29op13VpbW8XUl36/n46ODg4cOEBSUhI9PT0EAgFisRjBYJCjR4/S0NBAcnIyAwMD4+Z4opSCAoLjZWBggGPHjnHkyBHS0tIIh8PEYjFCoRCdnZ0cOnQIpVJJW1sbwWCQaDSKz+ejo6OD+vp65HI5Q0NDCWnh4vs4F4T0nWd77wquTeGahDo1bW1thMPhCZ2Hk0183Z/48RBSqR45cgSr1crIyIgoGvT09HDkyBGOHTvG9OnTRcHC7/fT2dlJfX09VquV3t7eCYPHggtiIiKRCKOjozQ0NHD06FHx3hHmrbOzk8bGRjo7O8nIyEi43zo6Ojh69CgKhYLe3l7xD574dKqT4aIS+jvbOoAymYxQKCS2IbQjpH4T3IRTLejL5XIikQhKpRK3281Xv/pVmpqaKC0tJTs7G6VSSSwWo7q6GrlczsDAAMXFxaKzSqfTsXjxYkwmE0ajkfLy8jOuq6RUKklLS+Pyyy/H5XKRlJSE1+tFqVSi1WopKSnhu9/9LoFAgJkzZ+L1esXvmXnz5hEIBNDr9VRVVaHRaMT3Rnwa0nPB7/fT29srCmMTtTfWOTXR/mg0Kr6fhG3C8wLHXYQajWbK5lxILdzb2ys6YC8EPp+PdevWEQwGSUlJOavaa3K5HK/XSzAYxOfzSQKVhISEhISEhMTniGWLa1m2uJahYR/HWlppbe+kvbMbtUqFy2nH40qmIDfr1A1x/Hfqn/3T3/Kzf/pb6o4cZdvuvVjNJsqKC8e5cjJSUzj40Tu88e5aZpYWk52R6Cz67c9/zLduuZGWtg6WLZ5/wqD+A//2D9y26ks0tbSxbPH8E/5dG9/eZYvmnXbWAJvVwrMPPUAsFmPvgUPs+nQ/mempzJpejE43cWxRQKVSsf/Dt3jrvfWkp3opPYW7yWm38cdf/QyAzu4e9h+qp7GpBZfTTl52Jhmp3lMuJhOEx//5zx+d1vWdKZM5xwCX1M7h2PZ1bN62i/rGY0SjMWwWM/k5WeTnZE5Zv0l6PT+65zv86J7vEA6Hqas/yqcHDiGXyynIzSYvKwON5vQXW6688jJWXnkZwWCQw0ePceBQPcFQiJzMdPKzMzGbjKfVzumOh8QXG0mgmoCFCxeydu1ajhw5gs/no66ujsbGRnJzc9HpdLS3t7N582bx+JKSEnJzcxNWCwcCgYTA5fDwsOgU2rNnDx0dHWi1Wr785S9zySWX4Ha72b9/P2+88QZvv/02bW1tHD58mGPHjpGefnwVgd/vFwNPkUiEkf+zfd500014PB5efvll1q5di1wux2Qycfvtt1NZWUlBQcE5B6SEVeaPPvoou3btor+/n/z8fFasWMGiRYvw+Xy88cYbbNiwgYaGBsLhMA899BA/+MEP8Hq95OXlsX//fiKRCJ988gmpqamkp6cTjUYZGhrivffeo7u7G5lMhtPpZNq047l/3333XTZs2MDhw4fRarUUFRXxne98B7fbTU9PD6tXr+bVV1+lt7eXt99+m8rKSjweD263e1wKJLi464Z80YifFyGAPHfuXLxeLyUlJZjNZmQyGUajkaKiIpYtW4ZWqyU7OxudTodcLkev1zNnzhxSUlKYNm0aHo9nwl9uTlQLStju9XqpqanB4/Ewffp0MYir1WqZNm0aS5YsQaFQUFxcLKZHs1qtFBcXs3TpUtLS0kSHwmTj8/kYHBzE7/ef9f0bDAZF8UdoY3h4WByT8yFSTYTdbmfmzJmo1Wry8/OxWCyiaJCdnc3ChQvp6OigvLwco/H4Lz8Gg4Fp06Zx2WWXiTX+JhLoT/bOUyqVWCwWqqurxfbia/rl5uZSW1vLwMAAM2bMwGAwiPdbYWEhixYtIhaLkZaWJo7bZL9bIpGI6AI527YFx1wgEBCFj1AoRF9fnzg+UyFSTXS+CoUCh8PB1VdfzcjICGazGYfDIQpUJSUlpKSkEAqFRDEKji+yqK6upqCgAJVKhcvlQqPRnJHLRqFQ4PF4qK2tFd3QycnJKBQK5HI5OTk53HTTTUSjUSwWC3a7XfweLSsrw+12o1QqxefkRKswzxZBlA2Hw+LClrMlXuCKxWKEw2FGRkZEkcpqtZ6xwHe6hMNhRkdHxdSYF+Kd4vf7aW5u5s0338RqtZKdnY3LNT7v+amQy+U4HA7gM2eahISEhISEhITE5wujIYnigjyKC/Impb38nMxTBtS1Wg3XXbXshPtnlBQyY0yavIkoK542ribWubQ3ETKZjOmF+UwvzD+jzykUCq5cuuiM+0t22El22KGm8ow/e76YjDmG439PzKksnzA941T2K6BUKinKz6UoP/e0jj8ZarX6nNs60/GQ+OIhCVQTIKzs/vjjj8WVzZs3byY5ORmdTkdbWxsbN24Ujy8rK6O8vDwhYCasGofPAkZyuZze3l4xUCRsS01NpaKigtLSUtLT01Eqlfh8PgoKCnA6j6vm8Sm64HhAS+ivpKQEjUbDvn37WLt2LQBGo5Err7ySxYsXn/Z1y+XyCdMmwfEg+c6dO/n4448ZGBjAarVSW1vLHXfcQXb2cetveno64XCYo0eP4vP5ePXVV1m1ahXTp0+nrKyM/fv3A7Bu3TpqampIT09neHiYffv2cfjwYUZGRkSBIC3t+GqAN998UxS2XC4Xt956KzfeeCM2m02sTdPQ0MAnn3xCZ2cnmzZtorKyErfbLbpmhDGLRqPnte6KxMQI91n8Kn+lUklBQQFarZaBgQEyMjJwOBzI5XIsFguzZ88mNTUVhUJBRkYGSUlJyOVykpKSuOaaaxgaGsJqtZKWlnZGKesEd1VOTo7Yjs1mSxDAqqurycjIQCaTie4LhUKB2+1mwYIFZGVlYTQaycjIEOvixD9H5xoIDgQCDA0N4fP5ztpNE41Gx7mwhJpUwjkKtbWmInAttBmJRBLa93q9LF++nOrqaqxWqyhY6PV6SktLsdvtjIyM4Ha7sVqPF0Y1m81UVFRgNBqx2+2kp6eLdaZOF0HouOKKK5g9ezYWi0VMpwrH3+kOh4NQKITdbsdqtSKXyzGbzdTU1JCZmUk0GiU/P18M9isUioR7+lwRFjb09/ef9byPrUsktBsIBMQ6WsJzdK5p1eKJv37hvSs4wjo7O+nv7yccDmMwGEhKSiIWizE0NER7e7vo9hO++2KxGL29vbS2tqLVajEYDGLdr9Md51gsJl5zS0sLarUar9eLXq8nGo0yMjIiOgpjsRhGoxGdTiemrGttbUWlUqHRaLDZbAnOqbGOybMdL6EG1YkcVKfLRPX9hBqW3d2fpVSYCpFKGEvh3Xm2NaTOFuH3jw8++ICmpiZWrlxJXl6eKHaeCTKZjKSkJJRKpXhPSkhISEhISEhISEhISEhITD6SQDUBVquVsrIyNm7cyMGDBwmHw6xZs4a5c+diMBg4fPgwhw8fBkCr1VJWVkZu7omV5PggkMPhwG63i+mX3n77bQDq6urIyMjAbDazatUq5HI5M2fOFMWfU6FSqcTAvEwmQ61Wi+l1YrEYIyMjHDhwgOHhYTHQIgSyjEaj6Fg6EQMDA2zZskVMu2a32/F4PPj9flF4ik+bFAwG6erqoqurizlz5lBZWckrr7yC3+9n69atHD16lIqKCtEFJbQ7ffp0ampqxGDpzp07xaCayWSitLSUxsZG2tvbUSgUKJVKcnJy+OSTTwBobGykubmZ2bNnn9a4SVwcyGQyVCoVRqMRmUyGTqcTxUSFQoFWqxWD0vEpquKD60Iw8WxQKpUkJSUhk8nQ6/Xi/Se4qIQaQ1qtVtynUCjQ6XRYLBb0ej0qlWpKxB0haC24LM4GIZA+dpsgUgkiglAn73zUcROEyaSkJKLRqDiPQvBfrVZjMBhQKpWiYAiI94DNZsNoNJ7VuAuCkl6vx2w2i6KjgEajwWg0Eg6Hx/Ut3A+xWAyVSjWpwk48gogbiURE4eRs2hg790K7wuKLWCyGx+NJeOamgkgkQldXF8899xwHDx5kzpw5XHXVVWIqv507d7J69Wq6urpEN6Ner8fv9/Phhx+yevVq7HY7N9xwA1arVXxeT7fv1tZWVq9ezdq1azGbzaSmpmI0GgkGgxw8eJBHHnkEn8/HwoULueKKKzAajQwODrJ161aeffZZDAYDK1aswOPxTJnwMtF8TVa7kUiE4eFhcZtwL08mwjtK+G6eqmdDIF50HRoa4ujRo3z00Uds3bqV0tJSqqurcbvdZ/0+UyqVKBQK8RmUkJCQkJD4IvD7Fz7hjusrLvRpSEhISEhISHyBkASqEzBr1iyKi4tFgWrdunV8+9vfJhQKsWvXLjFQnJWVRUFBgbi6fyLiA05FRUVUVlayZcsW+vv7aW5u5pFHHuGRRx4R013Nnz+fq666ShR7TpcTBbWi0SgtLS3cc8897Nu3j+HhYXHVt1qtpqCggAceeICcnJwTtjE6OkpjY2NCDaxNmzbR3Nyc4Bo4ePBgQjBoZGQEu91OSUkJWVlZ7N+/n+bmZvbv309XVxednZ288847hEIh5HI5ZWVllJWViTVihJX2AB0dHbzwwgtisA2O17A5cuSI2F8wGEyoFXE+Au0S504oFGLv3r28+uqrHDx4kGuuuYYlS5aQkZFBT08P69at46233kKj0XDttddSW1uL0WhkaGiIJ554gvr6eoqKili+fDkzZsw47dRSQrD+4MGDvPLKK9TX11NeXs6dd96J0WjE5/Oxdu1a3n//fRQKBbW1tVx++eWYTCZaW1tZs2YNr7/+OllZWdx+++2UlpaKbU9WsF9wasTXEZoshHZ9Ph+NjY1kZGRgtVpRq08/L/GZMDbdZlNTEy+++CK7d++mqKiI2267jZSUFFHIXr16NW1tbdTW1nLttdeSkpJCX18f69ev55FHHqG0tJQrr7ySysrKM6ozEwwGaW1t5YUXXmD79u0UFhbypS99Saxft2XLFlavXs3g4CCzZs3iS1/6Em63m/7+ft5//33ef/99otEo3/jGN5gxY8YZO7hOl3iRajJFC6Etv99PV1cXMpkMt9udIM5ONoKIsGvXLnbt2oXNZmPu3Lmiw6e5uZnt27fT0tKC3W4Xa46Fw2Hq6+vZunUrbreb2traMx6PaDTKwMAAhw4dYsuWLdjtdrHeUzgcprOzk48//pihoSFSUlIYGhoS6981NTWxbds2DAYDM2fOJBwOn3dn0GQgpPsTXJNyuRyXy3XC9Kdn2wcwZWK9gCCEBYNBwuEwfr+f3bt389Zbb3H48GHy8vK4/fbbycvLu2B1sCQkJCQkJD6v/P6F4ws/JZFKQuLzj9ViQq/TMfJ/Nd4BcrMyTvIJCQkJiQvD5y/Kcp4oKSmhpKSEl19+mWg0SmdnJ3V1dYRCIXbv3i0et3DhQlJSUk67Xa/Xy5133klubi6/+MUv2LNnj7ivp6eHtWvX8sEHH/D4449z7733snLlSux2+zldy/DwMJ9++ikffPDBuH2jo6Ns2bKFHTt2YLFYThhUElIzCWJUW1sbbW1tJ+3XYrGQmppKUlISaWlpLF26VHRbffLJJ2zduhWfz8eOHTsAyM7Opri4GJfLhd/vp7W1NWHVcltbG4899hhwYjEuKysrIVWXxOeDSCRCS0sLH330Ebt37yY/P5+qqioxzdn+/ft555130Ol0lJWVMXv2bJKSkhgZGWHTpk18+umn+P1+Zs6ceUYijiAAtLW1sWXLFvHZDgQCGI1GAoEABw8eZM2aNSiVSpxOJ4sWLSIpKYm+vj4+/fRT1qxZQ35+PldccUVCCr3PC0LgOhqNioKz3W6fMpEqvt/e3l527NjBunXrGB4eFoWgQCBAY2Mj69ev58iRI1itVpYuXQocF8cPHTrEunXrGBwcpLS0NEEYPB0ikQgDAwN88sknbNiwAb/fz7x588T9DQ0NrF+/nu7ublQqFUuXLhXTpO3fv5/3338fuVzOsmXLKCwsnDKBaioR3qGRSISOjg5isRhutxuDwTAl4oJcLsdgMDB9+nTkcjm5ublYLBYx5afX66WsrAyv10tubq4oMiuVStLT05kxYwZ2u53k5OQzTkUp1JPKzs6mvLwcq9WK0WhELpeLz3VFRQU+n4+cnBxxDDQaDSkpKZSWlmI0GvF6T13A9/NAJBIhGAxOSnrCeJRKJSqV6pxcf6dDb28vu3fvZsuWLRw6dIiGhgZGR0fJzs5mxYoV1NbWkpube85zFQ6HiUQiKBSKM0odKyEhISEh8XlHEqkkJP46kMvllBbls3nbLgAsZtMZ15uSkJCQOB9IAtUJMBgM5OfnU1hYKIoqmzdvpru7O8Gxs2DBgtMSqOJTkplMJhYsWEB+fj4NDQ3s3r2bPXv2sH//fo4cOUI4HKalpYXf/e53KJVKbr/99rO6BqFPvV5PSUkJ1113nRjIEfbr9XrS0tKYNWvWSR0IarUal8slrq7Pzs5m/vz54wLDQtourVZLTk4OM2bMACA1NZXFixfz4IMPIpPJ2Lt3L2+88UZCYHfevHkUFBQgl8tRqVSkpqYmBIVyc3O54447xgXUhD5tNhuFhYUnTbcocXEil8ux2WwUFBQQiURIS0tDr9eLKfZSU1MpLS1Fq9XicrnEALVGoyE///gvWFlZWZjN5jNygAjuAavVSk5ODuFwmMzMTPG+U6lUeL1epk+fjkKhICUlBbVaLQbb09LSKCkpIScnB7PZnHBvTrbbaaoR0r51d3cjl8ux2+2T7hQZW5vLYDCQkZFBcXExubm5Ypo/lUqF0+mkqKgIo9FIenq6KFhotVrcbje5ubkUFBTgcDjOOHgs1MjJzs6mq6uLzMxMMdUcQHJyMoWFhfT395ORkYFWq024F4uLi8VUp59HN41AfC24/v5+MS3bZAtusVgMhUKBx+Pha1/7GgMDAzidTvEdH4vFxFpvwWAQh8Mhfh/pdDqWL19OeXk5arU6YT5OF4VCQXp6Otdccw01NTXi94twvSUlJdx9991irUNBiLJarcybN088NjU1Fa1W+7l7tgWEtH5ms1msqzbZApVOp8Pv9zM6Oko4HJ4SQa+/v599+/axd+9etFotl156KW63m8zMTLKzs8XviHNBWBwRCoUSUihLSEhISEh8UZBEKgmJvw7+93f387NfPUwsFuOub9wiZRmSkJC4KPn8RtamGKVSSW5uLhUVFaJAtW7dOnw+HwMDA2KgfNq0aWJ9mtNh7969rF27lo6ODqqrq1m+fDkzZ86kpaWFuro6tmzZwqOPPkowGGTPnj0cOHDgtGofjE3TEx9AU6lUeDwe7r77brq6uhJS4AE4nU6mTZt20n50Oh1paWkJ9Xfy8vJYtWpVwjmEQiEaGxtZu3YtarWaoqIi4DPBb+bMmdTV1dHe3s57772HXq8XPzt37lxRXBJWYptMJpRKJeFwGJPJRFVVFSkpKWKdHJlMxujoKNu3b+fIkSN4PJ7PpYvli45SqaSgoIBVq1bR3d1NUVERDocDuVyO1Wplzpw5ohhQXFwspiIzGAxcd9119Pb24nK5yMnJOSuxIicnR2zH6/WK7ev1empqasRUWFlZWRgMBhQKBS6Xi4ULF+J2u7HZbAnC1ueVWCyGXC5HoVBM+S+uMpkMr9fL1VdfTVVVFS6XK0E+QyhGAAAgAElEQVRsKi0tRa/XMzQ0RGZmpphG1Ww2U11dzd/93d/h8XiYNm2a+B45XVQqFS6Xi6uuuoqZM2eSnJyc4LwU0vYFAgFSUlJEd6nJZGLu3Lm4XC5isRgFBQXodLq/il/yhbpcF8ohJIiXQkpcYUwFEW2y3Djx/Zxs/0THfJ7nWajlZ7VasdvtZ1TD63QRBCqtVkt/fz9+v/+0062eCX6/n6GhIVQqFeXl5SxYsAC73Y7JZEKr1U6KaByLxUSxXqfTfe7f7RISEhISEmeDJFJJSHz+SU/x8tuf//hCn4aEhITESZEEqpOQlpZGTU0NTz75JAD19fXiPofDwbJly7Db7RMG9CKRiBgkEfbHYjG2b9/Oc889x759+9i6datYnykvL4+SkhJcLhePPvqo2IYQmBP+KwhESqUyQYSSyWRiACUWizE8PEx7ezsdHR3IZDKSkpKYO3fuSa+3vb09IWAlk8lEsScpKYmZM2diMBgIBAL09vZy6NAhBgYGKCgoQCaTMTAwQH19PX/5y1948803MRqNFBUViUXKrVYr119/Pb/97W9pamqivr5eDIjm5+dTVFSEzWYT+4bjQeqjR4/S3t7OwMAAGzZs4JZbbiE5ORm5XE53dzf79+/n2Wefpa6ujr6+PpxOJxaLhWg0SjQaTRDVPq8r3/+aiEajyGQycV6EYLTRaCQzMxOHw0FycrIY2FSpVNjtdvLy8kRXgyCgKBQK0tLScDgcGI3GM05PJhwrOHmcTqeY+guO3zNOp1MUQ202m9i3RqMR7229Xi/2LVyfwLkG1oXxmcrguNC+0WjEZrNhMBgmtRaRMAZKpTJhPHQ6HampqZjNZoxGoyg8C8JkZmYmgUAAm82W4GpzOBwUFxdjNpsxm81nLKoIbqiUlBQMBgNGoxGtVivut1gsZGVlicK4Wq0W37EOh0OsC2YymcS+hVRg8dd7rgj9TBXCvGs0GqxWK2azeVIC8fHnLJfLxXdxb28v77zzDo2NjcyYMYP58+djMBgAOHjwIBs3bqS/v5/S0lKcTidarZZgMMiOHTvYvHkzJpOJyy67DLPZLNYWiq/xONG1Cc9kZ2cnmzdvZsuWLRgMBtxuN0lJSQSDQY4ePcqf//xn/H4/lZWVzJs3D71ej8/n49NPPxXTiy5cuFAUzoXv48mYo7HOwrMl/js7fptCoUCtVmOz2bDb7RiNxikRIgVXpNVqpaurC5/Pl+BMnCwikQhqtRqHw8G0adPIz88Xn9HJ7KO5uRm1Wn1W7xgJCQkJCYm/FiSRSkJCQkJCQmKqkQSqk+D1epk2bRper5eOjg5R3BBW0l977bUndE8pFIqEldpCoCwQCBAMBunr62PdunX87ne/46abbiIjI4O+vj4OHDggtpGcnCwGZoWgrdBmvPACxwO9Qs2YaDRKT08PH3zwAR0dHej1ejIzM5k/f/5Jr3ci14Tws9VqZfbs2UyfPp3t27fT09PD+vXrsdvt3HzzzahUKg4dOsSLL77IX/7yF0ZGRkhNTSUQCIhtmc1mvvSlL/H000/T1tYm1nfQ6XRcfvnleDyecf1fccUV1NXV0d3dTVNTEw8//DCpqalMmzYNhULBjh07ePrpp0XH1uzZs8U+x45ZvMAnceGId0YIRKNRmpqa2LhxIy0tLcydO5eZM2ei1WrFGlSbN29GrVYzf/58pk+fjlKpxO/38+GHH9LW1kZGRgZVVVVimrgTBa7jz0E4j7a2NrGdnJwcPB6P2P6+ffvYvn378fzNpaVUV1ejUCjo6+tj586dbNq0CY/Hw7JlyyZ0JUxG0HSqRQohLWdycjJWq1UUiiazDyAheB6Lxejp6WHz5s0cOXKEjIwMli9fjkqlIhgMUl9fz9atW+np6aGsrIx58+ah0+nw+XwcOHCA119/ndzcXKqrq5k2bZqYKu5kYyW8hyORCH19fWzatIkjR46QmZnJnDlzRJfWkSNH+OSTTxgZGSE/P5/58+ej0WhEwWLHjh3EYjGuuOIKsrOzRRfVVMzTVAmTwvtRpVJhtVpxOp0kJSVNSiB+7PMljHlvby/vv/8+n376KYFAgPz8fHJzc4nFYtTV1fHee+/R2tpKMBhk8eLFOJ1OgsEgu3bt4o033sDlcpGfn09JSQkajYZgMMjg4CCjo6PjRB6lUonBYMBkMhGJROjq6mLbtm289tpr2Gw2brjhBlJTUwmFQjQ1NfHmm2/i8/lQKBQUFRWRkZEh3mvCgguv10tNTY0oZk7WfAuuxTOtrTWWWCwm1pYSxl0Qp6xWKw6HQ3SATgUqlYqkpCTcbjft7e0MDw9PST/CGCkUCjQazaSnKozFYkQiEerr69FqtaJILSEhISEh8UVFEqkkJCQkJCQkphJJoDoFHo+HpUuX8vTTT4sClVKpxGw2c9lll50wQCUESyFRGKmpqWHjxo1s3bqVYDDIk08+yXPPPYdWq8Xv9yek36usrGTWrFliUDU+eBWNRhOCTFarFZvNJq5Wj8Vi/Pd//7e4v6qqio8//vik1yrU44gnPhDkdDr5+7//e+655x76+/tpaGjgl7/8Jb/85S/RaDQJYpRareab3/xmQtosrVZLfn4+eXl5NDU10d/fDxwPzq1cuRK32z3unK6++mq2bNnCvn376OzspLGxkVtvvVV0kMWPv81mY+HChWJdLMHxEC+ISALVhUcI5AtzJ6SGPHjwIP/7v//L7t27CQaDpKWlkZycTF9fHxs2bOChhx5Cp9Oh1+vJzc0V0789++yz7N27l5qaGux2O1lZWQluwrEpveKdFXD8WTp06BDPP/88u3fvZvHixVx66aXodDpGRkbYsGEDTz31FCqViuuvv56SkhK0Wi3t7e2sWbOGP/zhDxQVFVFQUEBGRoZ4fZMVMJ1I0JtMhLo0GRkZk+agORHx7rJYLEZLSwuvvPIKH3zwATU1NcyaNQuz2czo6Cg7d+7kiSeeoKGhgeuvv57i4mKcTif9/f18+OGH/OpXv6KiooKkpCTS09MxGo1iuycaKyGQHQqFaG9v59VXX2X9+vVUV1fj9XrJy8tDJpOJfXd3d3PZZZdRXFyM1WoVXZxPP/20mPLR6/Wi0+nEdy9Mrqg0FQKV0KYgTqWlpU26KCkQ/96NxWKEQiECgQB+vz/B0RoKhcTaRWPT0IZCIUZHRxM+E4lEOHz4MB999BGffvopoVBIfPbkcjnJyclUVVWxcOFCcW6EfgOBQML7IBwO4/f7xX3x5x4MBvH7/RO6lsemIzxb1Gq1eP+ei3gUDofp7e0V3dfCd6DZbCYlJQWtVjul34EqlUp0HzY2NjI4ODgl/cSP/VS9FyORCPv27UOn04luPgkJCQkJiS8aM4vS2b7vGCCJVBISEhISEhJThyRQnYKUlBS+/vWvs3HjRo4ePUowGKSwsJBvfetbwImDhyUlJaSmpnLo0CGSkpKYMWMGer2enJwc7rzzTpxOJ8899xxNTU2EQiGx/pNQoP6WW27hpptuory8XNw+b948PvroI9rb28WUZwIKhYKamhpWrVrFM888kxC0SUlJobq6+pTXarVaycjIwOFwiHWA7Hb7uD5+9atf8cwzz/D222/T2NgIIAYUDQYDM2fO5Prrr2flypU4HI5x/Xz1q1+lpaWFjz/+GLvdzqWXXkp+fr6Ysmksd911F9OnT+f555/nnXfewefzJYhTVquVRYsWcfvtt1NVVSUG2JVKJXPmzGH9+vXimOXn559yHCQuDIKbY+yK+HgXgFqtTnD6yeVycbsg4Ar3xbZt29i5cyft7e3jxOLU1FRmzJhBYWGhKMwKdc/ihVqhb5VKhUqlSuhbEI01Gs05Ox9OhXAeZ9uHkCY0vpaP4KAxGo24XC7MZvOk1G45E4QAevz8jR1flUo17n6InzNhXyAQoLW1lQ8//JCWlhYCgYA4j5FIBJfLJc65IFwL95VSqRw3txPNORx/rwjpxCbbuTF2bATxQ61Wn3UgXpj3ePFMSOtns9nweDyTnh5tIlQqFbm5udx3330MDg7idDpJTU0Vr+3aa6+luroav9+P0+nE5XIBx79TvvnNb3LllVeiVqvJzMzEYDAQjUZpa2tjz5491NXVkZubi0KhIBqN0tHRQVtbGyaTiTlz5mAwGCgpKeF73/seN954I2q1moKCAnHuFyxYwB//+EfxPklJSRHrzN14443Mnj0bpVJJWloaSUlJk54qVq1WY7fbMZvNJ03neSoxzO/3Mzw8TDAYTEjd6HK5xPfqVCKkYy0oKOC9996ju7t7UsX6eKZSnIpGo/h8Pvbv309tbS1er1cSqCQkJCQkvpAsnjMNQBKpJCQkJCQkJKYUSaA6BUlJSZSWlnL//fdz6NAhotEoeXl5zJo166Sfq6ys5Lvf/S7Lli0jLS2N9PR0NBoNCoWC4uJiLBYLtbW1NDc309/fTyAQQKPRiGmE8vPzycjIICkpCTjuPlq6dCkAPT09FBcXk5KSIvYnk8koLi7m+9//PkuWLKGpqYlYLEZSUhJpaWkUFhae1vVWVlZy3333UV9fz/z58xMEHZnseC2riooK7HY7y5cvp6+vj87OTkKhECaTCafTSUpKCtnZ2SQnJ0+4Gnzu3Ln88z//MwcPHkSn01FRUYHZbD5h8MzlcrFkyRKysrK48cYb6evro6+vDzheK8blcpGWlkZ2drYY4IPjwbIlS5YQjUbp6+sTRUOJiw+1Wk15eTl33XUX3d3dlJSU4PF4kMvlOJ1Oli1bRnp6OgqFghkzZog1ksxmM3feeSfd3d243W6KiopQqVSiK2rTpk309fXhcDjE7S0tLRw7dgyz2SzWLikpKeGOO+6gu7ubtLQ0sX2DwcCyZcvIyspCLpeTm5uLyWQSg9XXXnsteXl52Gw2Mc2cwGQFZfV6PVarVXwXnA2BQIDh4WFGR0cT6tmZzWacTidWq3VKnVMTIZfLycrK4tZbb+XSSy/F7Xbj9XpRqVQYDAYx5d7g4CA5OTk4nU4A7HY7S5cuxWQy4Xa7KSkpwWg0EgwGaW9vZ+PGjXR2dorp3YLBIJ2dnej1evR6PWlpabjdbtLS0rjttttYunQpLpcrQfCfN28eNpsNv99Penq6WPPObrezbNkysrOzASgvL0ev109JAF4ul2OxWM56XgRn0MjICD6fT1xEINTfstlsJCcno9frp1y4iBfa3G43ZrMZg8EgOs9isRhGo5FYLEY4HMZgMIhiqTAOQgo8oUZcNBolGAwSjUbxeDysWLECjUZDOBxm69at7NmzB5/PJzqJNBqN+B6IFxmF77XU1FQikQgmk0l0kwluaUGwMplMyOXySReoFAqFKNCf670k1HXUaDRYLBYcDgd6vf681FBSKBQYDAYKCwtZs2YNjY2NdHR0TOiOPhfGumAn+/kbHh5m3759BAIBvF4vDodDqkElISEhIfGFxGrSSyKVhISEhISExJQjCVSnQC6XYzKZuPLKK+nu7hYdThqN5qSfc7lcXHrppdTW1oqBUYGkpCTy8/PJz89nZGSEoaEhMT2RyWQSU/3Eo1QqSUlJYcWKFcDxoPXYFb0Wi4WKigpmzJhBR0cHsVgMnU6HwWA45fkKpKWlcf3119Pb24vL5Zqw7kJSUhIlJSWUlJQQDofp6+sTg4oGg+GUwSKHw8GSJUuYM2cOsVgMm812yvOy2+3Y7XaqqqoYHR1lYGBAFBDixzYewSlz7bXXimNxIpeWxIVFLpfj8XjQarUEAgEsFgsmkwmZTCY6DwWRwGKxiAFktVrNzJkzCQaD6HS6hGL2/f39DA8P43Q6qaqqwmAwEA6HWb16NcPDwwwMDIjB6+TkZCoqKggEAuj1elEUUKvV5OXliQFW4VmSyWRYLBYKCwvxeDxoNBrsdvuUBDF1Oh0KhYJIJHLWgVghUB+fvsxkMuFwOMTxvBBYLBbKy8spKChAp9OJ4oNKpSI9PR2LxUIoFMJgMIgCnV6vJzs7G5PJRFJSEmazGbVaTSAQwOfz0dPTg8lkYvr06aSnpzM8PMyOHTs4duwYAwMDBAIBFAoFZrOZ8vJy8vPz0Wq1mM1m8byEviORiPj+lslk6HQ68vPzcbvdxGIxUfCYChQKBUaj8YTvt1MhCFS9vb0Eg0HC4TDRaFR0TjmdTlGIPR8INajeeecdWltbyc/Pp7KykszMTAD279/Pzp07GRwcpKCggMWLF2O1WgkGg2zfvp2dO3diMBiYO3cuJSUlwHHhS6VS4XK5qKioQK/XizUejx49KrrGIpEInZ2dbNu2jb1795KUlMQNN9yA1+slHA7T0NDA22+/TTAYpKSkhIqKClJSUhgeHmbPnj1s3LgRjUZDVVUVFRUVU5Z28VzbFT6vVqsxm83YbLYprTk1ERqNhoyMDNxuNw0NDRw+fHjSBSrh3ZuSkkJycvKkz0dvby8fffQRycnJpKSkYDQap9xhKCEhISEhcbEiiVQSEhISEhISU40kUJ0BE6WrOxlGo3FCsSmeseLVqTgdMUcQs84GQYA7XbeGUqkUnQ1ngkajOeug+JkKTVar9az6kZgaJkrLFI1GaW9vZ8+ePfT09Ig1nWw2GyMjIzQ0NLBv3z5UKhVlZWXk5OSg0WgIBoPs2LGD3t5e3G43hYWFeL1elEol0WgUk8lEcXExS5cuxWKxEAwGaWxsFFNTCmmiOjs72bNnD729vaSkpDBv3jwUCgWhUIj6+nrq6uqQy+VkZ2dTXFyMVqtlYGCAuro69u3bh81mY/bs2Wi1WvH6hOD4uaJQKNDpdCdNlRWftm8iotGomEJPaC85ORmLxYJWqz1vwdf4FIyxWIyBgQF2795Na2srLpeL2bNnYzabCYVCNDc3c+DAAdFBVVpaisViYXR0lKNHj7J582Y8Hg/FxcVkZGSIc6lWq8nPz6e2tpaioiL6+/tRKpUMDQ2Jbp1IJMLg4CC7d++mpaUFl8vF9OnTMRgMADQ1NXHw4EH8fj+pqamUlZVhMpkYHR3lyJEj1NXVATB79mxSUlLQaDSTnspMcDqdiNPpLxQKiWkxhZSIdrsdp9MpioFTjXCOgkj04osvsn//fi655BKSk5NFV+vOnTt5/vnnaWtrY+nSpVRUVGC1WvH7/WzYsIEXX3yR5ORkrFYrubm54veHIGhqtVrxXh6bmjESidDS0sL777/Pa6+9htVqZeHChSQnJxMIBDh06BB/+tOf8Pl8rFixgtTUVNxuN4ODg3zyySc88cQTJCUloVKpmD59esJzfrGhUqnQ6XTYbDaMRuN5d/4olUqSk5MpLi5mx44d7Nu3j/Ly8nNygI7F4XAwe/ZswuGw6GqbLILBIM3NzWzbto3y8nJSU1Ol9H4SEhISEl94JJFKQiKRdR99zFMvvcaCmipuuHo5Gs34hd1nwuDQMC+98Q5mk5Erliw85/YuFJ3dPSQ77Kc+8Dwy7PMdz5pxirjrh1u28eTzr1BWXMjN112NyWg4T2d4elyMYyshMZlIApWEhMR5RQisC0HFWCxGMBjkk08+4Q9/+AN79+7ltttu4ytf+Qpms5mOjg7efvttHnnkEfR6Pd/73vew2+04HA76+vr4/e9/z759+6iurubmm2/G4XCI6cGUSiVarRa9Xo/RaMTv96PRaETXi1Cbac+ePTz00EPs3buXBQsWUFZWhkqlYnBwkLfeeotnn30WlUrFNddcQ0pKCna7ncbGRl555RUef/xxCgoK+PGPf4zD4RDT58Vf77lwOmmsTlcYEdKkeb1eLBbLeak9BJ+NQXywPBaLceTIEZ544gk2bNhARUUFaWlp6HQ6hoaG+Oijj3jiiSc4duwY11xzDW63G4vFQmdnJ++++y4//vGPqaqq4hvf+IaYojBegBPmXXDXCaKF4CRramri8ccfZ9OmTcyePZuvf/3rpKWlAbBhwwaeeeYZurq6uOSSS/B4PCQlJdHT08Obb77Jiy++CMCPfvQj0cEVjUYT7unJYDLmXDhWSFfndrunPK1f/PWPTYknl8vFf/FipXCegpgW30b8duG6BUFSOHbsv/jj4tsR/sX3LbQviFpjPyOXy8fdu0LfMpls0sToc0WhUGC1WjGbzaLz8kKcg8lkYtasWezatYtDhw7R0NAgut4mA61WK7q7J8N5JhCLxWhra2P//v10d3dTWVmJ1+ud0EkuISEhISHxRUMSqSQkjrNz734uveF2AP703J/p6unlB3fcftbtDQ4Nk1dzKb19AwBcMr+Gt599dFLO9XxxqP4oV3zlWzQca2ZWaTGPPPBTSqblnfqDU8w//Mf9/PKhP6LVaLjr6zfzb/fePeFxnx48xKJrbxF/3rxtJ0/8zy/O12melIt1bCUkJhtJoJKQkDivTBS8F2rP+P1+RkZGxNoygFhrZmRkRExbFh9c9vv9jI6O4vf7CYfDJ+1bqL80FqHv0dFRAoFAQvuhUIjR0VHC4TCBQCAhbZhwXqfT94VGLpej1+txOp3YbDbRUXW+mGjcI5EIgUBAHMOJxl2Yk7H3QzAYxO/3EwqFEtoWxIOxfcdvF1xU8e3HiyihUAi/3y/+E/oWxNTR0VFkMtm4vi9GBEeRTqcjIyMDlUp13tL6xaNUKklPT+cHP/gBfX19pKSkkJWVJYrFS5YsITc3F7/fj8fjwW4/vjpMr9ezcuVKqqqqUKvVTJs2Db1ef0aikFKpJC8vj5tvvpna2lrUajWZmZmiqFlRUcG///u/EwqFSEtLIysrC4VCIdYcy8jIQKvVkpWVhU6nu2gEqbEI7qV499iFIiMjg6KiIvbu3cv7778/qQJVJBIhEokQjUbFOZyM641Go+zdu5ctW7YwY8YMcnNzpbTAEhISEhIScUgilYQEvLdhU8LPa9ZvPCeB6tlX/iKKU0L7TS1tpKV4zrrN8819v3mUhmPNAGzb/Sn/8otf8dJjv76g59Tc2s59vz0u9PlGRvjZrx/muquWUVY8bdyxb7+/IeHnd9ZtSFiAeiG5GMdWQmIqkAQqCQmJ84rgeBCCvEItqRkzZvD1r3+d9vZ2Kioq8Hg8yOVykpOTueSSS0THT3V1tVhrymKxcPvtt9PR0UFaWhpFRUUJNYFOJEjF9y2XyykuLua2226jo6ODrKwssTaP0Whk8eLFOJ1O5HI5RUVFWCwWMdh++eWX43K5cDqd5Ofni8H/+AD2hQ4Uw2euBrPZjNFoPO/ilBBAjheBZDIZWVlZ3HDDDVRVVZGWlobL5UKlUmE0Gpk9ezYajYb+/n4KCwvFVKJOp5NFixYRiURIT0+noqICo9EoikXCvSX8MjmR40aj0ZCSksJXvvIV5s6dS2pqKgUFBeK51dTUoNfrGRkZITs7G6fTiUKhwGazsWTJElwuFwBlZWVi3b34eb8Y5hw+SxNot9vR6XRi/bTz0a9ANBoV3U+CsyYUCqHVahNcPmlpaTgcDrFOlpD6VqVSkZeXR1pamlh3UKVSEQwGx7mX4oWKsc+4ULcoMzMTuVyO2WwW0wO6XC5qamqIxWKi804ul4uint1uFwVeoW9hvsc6Ji8kglPuYrj/HA4HVVVVtLW1sWfPHrZs2UJ5efmk1Gw7ePAg69evp6enh0WLFjF79mzRNXu2xGIxDhw4wPbt2wkEAlx99dVifUEJCQkJCQmJz5BEKokvOseaWxN+bm3vOKf2ise4YfQ6HYaks6tDfKE40ngs4efG5pYLdCafcaSxady2xuaWCQWqsXPa2zeAb2QUo2Hy0pSfLRfj2EpITAWSQCUhIXFemSh4qlQqyczMxGKx4Pf7sVgsYm0Rk8lEaWkpWVlZyGSyhLpJOp2ORYsWiWnczGYzSqVynKNmbJq8+JRiMpmM1NRUDAaD2I4QyNdqtQl9GwwGMahut9uZNWuWWA/HbrcnBEkvlqA1kFCHSHAcXCji58LhcDB//nwqKirQ6XSi+CeTycjLy8PlchEOhzEYDJhMJgAMBgPFxcW43W5xznU6HaFQKGF+x15j/D0h1GKaP38+s2bNQqvVYrFYxP35+fm43W4ikYiYHlImk5GUlCTeD3A8CC8EsMemhrsYEGoKCsLKhZp34TlTqVTY7fYJ50c4T+F4AUFMMplMCdtP5JqJ3y78v1B/S61WT9iOUL9q7HkpFAqxJuPY98jF9HzHczGIU3C8zmR+fj6lpaWsXbuWl156idTUVJKTk89ZTGptbWXt2rU0NzfjdrupqKg45zb7+/v58MMPaWhoYNr/Y+++w5uq+jiAf7Oa1ZXuvTerjDLLVgRkqAwFxBcEAQcqKA5UREFeRRyoKILiQJEhsgSUvXfZo6V0772StNnvH30benuTJi2FlPL7PI/P03vOufeeJDeS3G/OOVFR6NKlC6RSaat9nQm5H9TdsCaEtD0UUhHScvrEdcFzT4/Hxu27IZWI8c6rz0Pm7GTrbjWJWq1mbMsVShv15LaGfQKa1q/WMmtGa3xuCbkbKKAihNhcXfhT/0Zw3Y1WgUAAFxcXuLi4MNrX1fn6+jL2abgGjakb1g3/rrs53vDcdWu6yGQyxnmB2huwbm5ucHV1Zd0sby03ievw+fxWM7Ki/nNVt1ZUw+edz+fD0dERDg4OrH3qrgeZTMYKHuuvMWTunHV/C4VC40io+scAYNW569fVDzxbk7pQErBN3+rW+1IqlZDL5SaDKQCNPn/m6uqmYVSr1eDxeFAoFMapIVUqlbFeLpcb15ey9pyW6lUqFRQKBWO6UcImk8nQuXNn5OXl4eDBgzh48CAGDhwILy+vOwp+xGIx/Pz8YGdnBycnpzu6tuum+zx58iQuXrwIBwcH9OnT5477SAihgIqQto5CKkJazrefLMSKj98H0Pq+U1qjNX4nao19ao628jgIsYQCKkKITej1emi1WqjVaqhUKrPtzN3UrqsDmB/i6tprtVrodDrj2lF1/2m1Wuj1euONbIFAYHLNoobHtaZPHA4HarUaGo0GOp2u1XyYaC0fcutuBqtUqglnqiEAACAASURBVEbXjbH0mpsKoOrWpap7zTUajfH6qn8t1LUz9Zo3J6yoq6+7rlobW7z2da+tRqNBamoqHB0d4e7uDr1eb3Jkm6WRb6ZCQK1Wi8TERBQUFEAoFOLs2bMQCoXQarVISkpCYWEh9Ho9zp07Z5yyz5rj1t82d03odDokJSWhvLwcvr6+reb91doIBAIEBwejd+/eSE9Px5YtW+Do6IjevXtDJpM1OwCKjo7GlClTUFNTAz8/v2aPnjIYDFAqlcjIyMDWrVthMBjQv39/dOzY8Y5HZBFCCCEPAgqpyP2svKISVxJvQqFQIiYyDAG+Pi1yXJ1Oh6uJySguKYW/rw+C/H1gZ2dncb/T5y/DxdkJEaFBVp+rpLQcVxNvorC4BN5eHgj09Wn22lV5BUXIzMlFTl4BxCIhIsNCEOTve9d/tFVZJcel64morJTDz8cL/j5ecJE5W97xHsrMyUVqehbUGg26d+4IZyfHJu1fWSXHjeQUlJaVo6JKDolYBCcHB0SEBsPb0/0u9do6SbfSkFdQiJKycuj1ejg5OsDN1QUdoyOs+k6UX1iEpFtp6NcrjvGd+syFy1Cp1ejRuROEQsvXf0tey+T+RN/ACSH3XN3IipKSEhQVFaG6utp4Y7i5Ixvql3O5XFRUVEChUKCiogKFhYWoqamBRqNBZWUl5HK5sbyqqso46srScS31h8vlory8HKWlpSZDkAeZwWCAWq1GZWUlioqKjGHOnYxkqavncrmoqqpCSUkJFAqF8Rz5+fmorKxEaWmp8VooLi6GVCo1rod1J9dZHQ6Hg9LSUlRXV5td9+xBwuFwwOPxoFQqsWbNGlRXV7M+3NZ90WkY6pkrr1M3Eqpu9FJd6LVz507je7Au/OZwONiyZUuTjlt/nTRz/aobqeXv74+oqCjW6Cxym1QqRadOnWAwGPDhhx9i/fr14HK5eOihh5q9vpOrq6vJEbVNpdPpkJWVhcWLF6O4uBhPPvkkBg8ezDg2IaRlDOrFXu+BENJ6lVco4exk3Ro4FFKR+0lWTh7e+mgZjp85j5w85tpRjg726BgdibdfmYkhA+KbfOxt/+zD8lW/IuHyNSirq43lTo4OeHbCGLz50gy4urCDF71ej17Dx+P8lesAgNnTJuPzD982e56ComLM+2ApDh4/hfzCYlZ9lw4xmPWfCZjy1BMWPyfrdDr8snELfvhtE85evMKqFwrtMGxQP3z45iuIDg9l1b+z5HNs2b0XyakZjPKU9Ew4h3eDo4MUjw97GMs/epdRn19YhDc+/BSnEi4iLTObddwAXx88N3k8pk8aBzcXWaOPoaHzV67j7Y8+w6lzF1l1U195Gy++9QE6Rkfiv+++jt5xnRs91pZde7H4i29x+XqSsYzD4aB9VDhenTEFz4x/rNH99x05gcWff4tT5y+Z/Z4ZHhKIqRPGYM6MKYzvzM19bq1RUVmFRZ+vwIZtu0xeQ0DtWmgD+nTHR2/PRfsG66TVWf3bRrz63kdQqzUIDwlEwp4tWLflbyz89CvjcUUiIeK7d8WyhW+iXSTzOC15LZP7H8fwoN9JI4TcU+np6di1axc2bNiAoqIi47pD5v5X1HBkQ33mbjpzOByUlJSgpqYGIpHIuDaVXq9HSUkJ1Go1JBIJnJycwOVyWcdu7GZ2Y/2pG7lVN1Lr5ZdfxtChQ+Hr62vpaWnTqqqqsGPHDvz555+4du0avLy8GKFgfY09v43V6/V6VFdXo6ysDCKRyLhemE6nQ1VVFeRyOcRiMaRSqXFaQWvPa831UF5eDrFYjGHDhmH69Onw8PCw6pdybZFKpUJxcTFu3ryJqqoq6HQ6k+8za5lb36ux17DhtJFNOa6lOqD2WpBIJPDz84O/vz+cnO6veeLvJa1WC7lcjqNHj2Lz5s3gcDgYOnQoHnnkEcbab01xJ9NqGgwGVFdXIyEhAZs3b0ZCQgKmTZuGfv36wd/fHwKBoFl9IoQwxY5fafz7o7mN38AhhNz/yiqVOHAi0RhSAbUBFYVUbYs294atu9AsBoMBq9ZuwFuLl1m1hs+Up57ApwveMDlS5pV3FuPbn9cZt6PCQjDqkUFYuuKHRo8ZGhSAHWtXIjwkiFF+8txF9Bs90bjtYC9FwdUTJj+Trt+6E6+8uxilZRUWH8PQQf3w69efmF3Pqri0DBOffw0Hj52yeCwul4s5M6fg43dfN5ZdupaIbkOesLgvAJzctRHdOrUHABw7k4AJM+eYDUbq83R3w78bfmQFG40Z8+xsbP93v8V2wwb3w/Zfaz+rNHxNgdqg8Osf1zZ6jFXLFmHqhDGscmV1NWa89h42bNtldb8H9e2F3etWg8vlNvu5tcY/B47gudfeter5B2qXX9j1+yoMjO/Jqhv4+GQcO5Ng3J7/yix88s1qk2Gcv483Us/efl1a8lq+H/B9om3dhVaPRlARQu4pmUyG7t27QywWo7i4GDwejzVFH2B5ZEv9NubKGq5N1ZCp81o6rrn+1N+Hy+VCIpGgW7ducHRs2vDvtkgoFKJTp07g8XjIycm5o+fXXH397YZrU5lal6ylrrM6er0eDg4OiIqKgpOT0wM9qsbOzg7u7u5wcHAwjnRqzjpTlurq6ptz7dzJyL26Oj6fD6FQ+MAGkdaqW1OuV69eUCqVOHr0KPbv34+KigqMGjUKzs7OEAqFTQqbmhtM6XQ6lJSU4NSpUzhy5AiKioowceJE9OnTBz4+PhROEUIIIc1EI6lIa/bx16uw4JPlVrf/ef1fuHT1Bk7u2mjxe13irVQk3kq1eMyU9Ew8PuVFnN+3hfH94fT5S4x2VXIFbqVnskYsrd+6E5NfnGf1Y/jnwBE8NWsu/vnjB9ZnZ5VKjSHjp+LKjZtWHUuv1+Oz79ZAIhZjwWsvAgBCAv3hYC9FlVzR6L4CgQAu/w8WquQKDH1qGlQqNaONRCyGk6M98gqKGOUFRcUYNmE60s4esPr7dWz7aKsCqj7duzZabymcAoCZ8xYgJCgA/XvFMco/++6nJoVTAHDg6En88PsmzJj8ZLOeW2soq6vx1My5UCgth7R1tFotpr76NlJO72O8BgaDAWcuXma0XbJ8ZcPdjXLyCyBXKGAvlbbotUzaDgqoCCH3lL29PaKjoxEUFASNRmMclWJubmNz9XXTe9XVNXVqvuYe15r+crlc8Pl8SKXSZk9j1ZYIBAKEhITA29sbKpXK+Iua5j6/purrhyCm9q2rb+nrrH4dj8eDWCw2uebRg4TD4cDOzo6CG2LE5XLh5uaGAQMGgM/n49ixYzhx4gQqKirQt29fhIWFGUfT3g06nQ5KpRJZWVk4efIkEhISoNFo0LdvX4waNQpubm50vRJCCCF3iEIq0hqlZmSZvHHet0c3jBgyEIF+Pjh9/hK+WfM7NBqNsf7C1Rv4+sff8OqM/1h9rl7dYjFiyCDYCQTY8e9+HDl1jlGflJKGz1b+hLdfnmksMzVLh1qtYWzLFQq88eFSRhmXy8UbL07HlKeegL1UgsMnzmDu+x+joOj2yJgDR0/iu5//wAtTJzL2ff/Tr0yGU2NGPII+3bugoLAY/x46hotXmSPmFn2+AlOfegL+vt5wsJfi3J6/sHPfIXz81SoUFpcY29lLJXjt+WkICw5AXGwHhAT6A6gNGhqGU+u//wKjHhkEgUCAzJxcTJg5F2cu3A4+8gqKcDLhIuItBEp13nxpOrp2bIff/9qOjdt2M+oefXgABsX3RPvIcMT3sHw8DoeD3t0645FBfVFWXoGVv6xHdU2Nsd5gMGDNuj9ZAdXvm7cztiViMb5cNB/DH+oPZ0dH5BYUYvmqX7Dip98Z7f78+1/MmPxks55ba2zdvY8VTg1/qD8WvPYSosNDoFKpcfbiFTz/xkJk5uQa2+TkFbBeA71ez7pO63A4HESFhSAnvwCVVXIAwKQxI2Evlbb4tUzaDgqoCCH3FI/Hg1QqhURi3bzm96u6UTr0C4/a50IsFkMkEtm6K/cEveaEmObh4YHBgwfD2dkZe/fuxYEDB1BeXo4uXbogJiYG3t7ekEgk4PF4dzwKUa/XQ6fToaamBsXFxUhNTUVCQgKuXLkCBwcHDBo0CIMGDYKnp2cLPTpCCCGEUEhFWptX3v0INTUqRtmEx0fg129u3yQfM+IRPDbsYQyf+BzjBv7CT7/GuJFD4ett+fPisxPG4Ptli4zbr874D2a+/h7W/LGZ0e6XDVsYAZU1Pv5qFWt00cfvvo45M6cYt8ePHo4uHduh06BRjODg259+Z9zUr5Ir8M2a3xjH4nK5WPPlEkwaM8pYtvjtOZg1bwF+XPcno+3uA0cwY/KTAGpHUc2eNhkbtu5khCie7m54d87zrMexa/9hxraPlweeeHSI8ftzgK8P9m76CcMnPIfjZ88DqL1/FORn/ZIJdnZ2GP5Qf/D5PFZANX7UMEx8YqTVx1q1bBGmPHV7qr1B8T0xcvIsRpvj9aa4A2rX10pJvz3VqVQiwa51qxnrXQUH+OHLxe/g4PHTuH7zlrH8XL11wJr63Frj2GlmX8eOHIq13yw1/khQIhZjyIB4fPLe65gway6j7bmLV60KCX29PXF8x3rje+ZaUjIUymp079wRQMtey6RteXB/4k0Isam6qffa6n91j5HcZuvX5F6+9oQQNg6HA2dnZ/Tt2xczZ87EqFGjcOHCBaxZswYbNmzA+fPnkZubC7lcbhxha269uobq2un1euj1eiiVShQWFuL69evYtWsXVqxYgR07diAiIgLTpk3D448/TuEUIYQQchfUhVRdYgKMZSs3nTMGVYTcKzU1Kvx78CijLCI0CN8tXchq2zuuM156dhKjTKFUYs+hYxbPE9s+Gl999B6r/ItF8xEeEsgoS0nPZIQS1tix5yBj29/HG7OnPc1qFxYciGcbrImUlJLGCEy27t7HGsX01GOPMsKpOis+fh+PDXuIUdaUETsNCRvMWJCbX4hJL7zOGM0lEYuxfe1KLHlnLp6fMgH/rv8Rfj5ezT5nc00eN5oRTgG1ayEF+PowykrLmWsoeXm44+H+fYzbyxe/wwin6ovr3IGxXSVXMEbxtbQnH3vUOKV5ZGgw1nyxxOQMFj26dGKVlZSWWXWOZe+/yQh020WGG8MpoGWvZdK20AgqQgghhBByzwgEAvj7++Ppp59Gz5498c8//+DYsWM4ePAgIiIiMHDgQLRr1w7u7u4Qi8XGtb7MTQGo0+mg0WigUqlQXV0NpVKJ69ev49ixY7hw4QLUajViY2MxZ84chIeHw8XFhdabIoQQQu4iGklFWoPEW6msHzqNHDIIUjOzubz47NP45JvVjLIbySkWzzPh8REQCtnTRUvEYox4eCC++P5nRnlyajpiIsIsHheo/ZybnJbBKOvZLRY6nR46nZrVvlunDgDWM8rSM7MRGlQbGDcc8QMAc2dNNXluHo+Htd98ilff+whXE29i3KjheKhfb6v6bcrAPj1ZI7I2bd+NTdt3IzjAD4Pie2Jw31549OEBmPfC9GafpyVMHjvaZHlUeAhj+jtT60TtWrcaZy9egczJEWHBtwPKjOwcnE64hKOnz2H/0ZNITs1g7VtZpYCri3MLPAK2/r3ikH3hMK4mJSOuUweIxbUz3FRX1+D81es4efYCDh4/jaOn2D8mqJuqrzEcDgdDB/U1W9/S1zJpWyigIoQQQggh9wyHwwGfz4e9vT0iIiLg6OiI7t27IykpCUlJSdi+fTt27NgBDw8P+Pv7w9fXF+7u7nB0dGSs7Ve3tlRFRQVKSkqQn5+P7OxspKWlQafTwcXFBQ899BCioqIQHBwMPz8/2Nvbg8fj0WhHQggh5C6jkIrYmqmRSu2jws229/Z0h71UArni9jR/15Isj3bqFBNpti46PJRVVlBUYqKlaSnpWaxRNXWhjrXqr4+ck1/IqouJYPexjkgkxMpPP7T6XI0ZP3oYdh84wlqjCQDSMrPx47o/8eO6PyGVSDBmxBBMnzQevbrFtsi5myoowM9kuZuLzKr942I74EZyCpYsX4kzFy7j7IUrjKn6zLnbX1FcZM6I69QBf+89iEMnzuDshcu4kpgMrVZroV+WOxYREgR7qdRsfUtfy6RtoYCKEEIIIYTccxwOB1KpFMHBwfD29kZoaCjCw8ORkZGB/Px8VFRUIDMzEykpKdBoNODz+RAIBMbFpA0Gg/FLDpfLhUgkgkAgQHh4OLy9vREQEIDQ0FD4+vrC3t6evtAQQggh9xiFVMSW0rNyWGXmgoc6To4OjIDKmlAhPDTIbF2wifPVP74l2bl5Vrc1RebshE7toozbRQ0ej7vrvZtZgMPh4OevPsZD/Xph9vxFZp8HhVKJXzduxdpN2/Dfd17Da88/e0/6V59UIjZZzudbXic3N78Qz8x+A4dPnGnpbt2xVWs34K3Fy0yO/KpTt556U1mairGlr2XStlBARQghhBBCbIbL5UIsFiMgIAC+vr5Qq9UoKipCVlYWsrOzkZ+fj5KSEigUtfOy63Q6GAwG8Pl8SCQSSKVSODs7w93dHb6+vggODoaHh4cxsKJgihBCCLEdCqmIrfj7eLPKsnIav0ne8Ma9NSNmGgucik2s3RPgy+6XOX4mHoO9VIKO0eZHbQnsBPDz9kLHmEgMG9wPLrLbU8a5ubow2paWV0Cv11v8vFxSWg4XmVOLzELw9NjRGPXIYGzdvQ8btu3CoROnoVaz114yGAx4a/Ey6A36ez7lX8P1sqxVUVmF3iOeRE5eAavOw80VPbvGole3WMT36IYd/+7H0hU/MNrcze8tX3z/M974cCmrnMfjIbZdFHp2jUXvuM7o070rgroNbHK/JGLToV6dlr6WSdtCARUhhJA2y2AwQKNTQ68DwOGBz+OBz6OpvQhpbTgcDjgcDrhcLgQCAcRiMfz9/aHX66HX62EwGKDVao0BFVD7ZUogEIDP5zP2pyn8CCGEkNaFQipiCzGR7HWert64CTxuun1hcQlrrZ12Jo7RUGNrSiXeSmOVhTRhDZ2QQD8IBALG1Gh9e3bD9l9XWn2M+qLDQ/DvwaPGbY1Gg7RG1vXR6XR4csar2PbPfsREhGHj6uWIDAtu1rmB2oCwokqO9lHheGb8Y3hm/GOorq7B4ZNnsHPfIfy+eQcrJPxt03abr0llrY+/WsUKp/rEdcEnC+ahR5dOjPK1m7ay9r9bAVVxaRkWfvo1o0wkEuLdOS/gxakTGVPzFRQVm+iX5e9Wkv+vaWVOS1/LpG2hn5QSQghpszgcDkqripBTWIL8YiVUmqYPVSeE3Ht1QZNAIIBQKIRIJIK9vT1kMhnc3Nzg5uYGmUwGe3t7iEQiCIVC2NnZGcMqQgghhLQudSFVl5jbN8JXbjqH91YctGGvSFsWHR7CKtu57xBUKrXJ9t/9vI5VZirkaujrH38zWV5To8Lq3zawysOaEFDx+XxW+1MJlxqdgk2uUOD5N95H90fG4Osf1zLqokysibVize9mjzV/yefY9s9+ALVreq35409Wm4afvU2NhgKAr39ci9AeD6Hz4NF47rV3jeVisQhDB/XD10sWIPnkXta6Xddv3kJFZZXZPppi6vuAuX61pCOnzjK2eTwe/vj+C1Y4BQCnz19ilTUMSK19bi1JuHQVyupqRtnUp57Amy89x1o3ynS/zE8JWMdSuNbS1zJpWyigIoQQ0ubo9Fqk5Sfhj4Pf4dPN85H655s4/MvX+HzlPny56Rb2nitEXkkNNFq9rbtKCCGEEELIA6H+SKo6Ow4nYfuhJBv1iLRlErEYg/v2YpRdS7qFOQuWsNqev3IdX/3AvAEuEglZ+5ty+MQZ1s1znU6H1z/4BLn5hYzy3t06w9HB3tqHAAAYMqAPY7usvALv/vcLk23zC4vw8Lip+OH3Tbhw9QbeWrwMNTUqY318967g8ZjrKK36bQMOnzzb8FBY8dPv+HzlT4wyUwGJa4NpEEvLKxijZOr8+PsmYxjx8/q/8Nl3a0wcyxmjHhnEKrdr4jpZDfsEAPkmRga1tPxC5jn4fB7EIiGr3dpN23DpWiKrPOlWKmPb2ue2qf0CAHuJhFVWWlaOj774zmK/TBEILE/S1pLXMmlbaIo/QgghbUp6/k1cTjuN7OJ0SIT2GB43Hj5Zt8C7eBVSgQtUshgkZ8txNrEcro4CBHhKEORV+59YaHnRU0IIIYQQQkjzbP7nPGO7a4wPRg0wvwYJIXfi6yXvIXbwaEawsvq3jcjKzcfIIQPh6e6KS9eS8Om3P7Bufs9/ZRYC/XytOs/cBf/FnkPH8cjAeADA5h3/4tiZBFa7hfNmN/kxLHjtJWzYtosRMixd8QP4fD6mTxoHf19vaDQa/LFlJ97+6DMUFpcY2/l5e0FULyCJDAvG7GlP48tVvxjLVCo1HnnyWbz07CT06d4VpeUV2HfkBDb//S+rL3WPrz4fTw/GtkKpxAtvfYBB8T2RlpkNe6kEz04Yg26xHXAt6Zax3VuLl4HD4WDiEyPg5eEOjUaDY6cT8GuDqe/CggMhtjB9HLtP7qyy739dDzcXGTQaLXLzC/DowwPRsyt7ZNOd6B3XGZk5ucZtlUqNNxd9igWvvQRfb0+kZ+Xg141bsejzFSb3v3bzFgb3613vcVj33DYcBWWqXw39+MefGBjfE4Pie0Kj0eL0+UuYPX8RbiSnsNpeT06BwWC445kqWvJaJm0LBVSEEELue1qdFsUV+UjOuYrckgyUK0rh4uCOKP9YtA+Og8Y7C6LiHPhw8gFfNfJVjkjPV0JZo0VGgRJlVWpcTq2Ap0wELxchvF1EcJDywaWpwgghhBBCCGkRP2w8hrTs2zcmu8b44MeFo2zYI9LWhYcE4e2XZ+GDZcz1d/45cAT/HDhidr/o8FC8/vyzTTqXpWP26xWHgfE9m3RMAHB0sMfnH7yNic+/xihfsnwllixfCW9Pd5RVVJocXfLunBdYZe+//hL+/PtfZOfmG8t0Oh2Wr/4Vy1f/arYfT48dhaGD+rHKfbw8WWU/r/8LP6//y7hdUSnH5HGP4ZcNWxjt3lz0Kd5c9Cn8fLxQVl4JhVLJOtbL058x2ydzPNxcwefzodVqjWXZufl4/o33jdvfrPkN2RePmAxdzAUxDcsbbg8ZEI/1W3cyytb8sRk/rf8LErHY5OOr79DxM4zHa+1z+95c9utcX3hIEIL8fZGelWMsKy2rwPCJz8HOTgCDAY2OzCqvqMSla4mIbR9tto0162e19LVM2g6a4o8QQsh9S6fXolxegpvZV3D+1nGcu3kUlcoyxIb2xLC4JxEb2hN8Lg9ijyCI/aJhp1fCvfIKBsS6YcojARgd7412QY6QV+uQmCnHyWulOHmtFKdvlOJySgUy8pUoqVRDrdWjkamRCSGEEEIIIY2gcIrYyvxXZmLZwresHn0xcsgg/LthDQQmppVruIbTuFHDMHncaIvHHBjfExtXLbeuwyaMGzUMPy3/L1xkTqy6vIIi1g19Ho+HpQveMNk3e6kUezasQXz3rlaff+zIofj+0w9N1s2YPB6e7m6N7l9SWob+veLMBinZufkmw5sXp07CzGeetLqfdbhcLua/MqvRNgplNaprVOjUjjntaFhwoNlpGBu27dapPWP7qceG48nRw1n7GQwGxuNzlTnj6yXvsfZPy8xibFv73Frjp68+hpOjA6tcrdYwwqmRQwZh2cK3WO3SMrONf/N4PHTpEMOoj4vtYFU/WvJaJm0HBVSEEELuOwYYoNJUo0JeiospJ7H1+M/Yd2Erukf1x8RBL6JH1EA4SZnzNTtG9QJP4ojyKwcBgx6AAb5uYvTt6IoZI4Pw5oQIDO3uAY3WgE2HcrH8zxRsOpSDi8kVKKvSQFGjhVKlg4bCKkIIIYQQQqxG4RSxJS6Xi1eeewbn927B0EH94GDPng6Nx+MhJiIMa1d8ir9++gbeJqaIA4BHH+4Pe6nEuM+kJ0ZizZf/xfLF78LXmz3axd3VBa8/Pw27162Gq4szq97QhC+WT48djauHd2LSmFGQObNv7gOAQCDApDGjcO3ITsyZOcXsscJDgnDgr1+xfPG7CAsONDn6xc5OgEF9e+Ho9j/wx8rPYWdnZ/JYHm6u+OWrj80GKV06xODV//dlwWsv4fDW3/HowwPMPsdOjg6I794VB/76FV8ufseqkTmmzH9lJh4f/rDJ0VAikRAfzZ8DNxcZHu7fB85Ojsa6p8ea/3/T0IF9jdcPh8PB02OZoYlAIMDaFZ/iteefNRnAuLnIMPOZp3D96C7M+s8ErF/1BXy8bk/jN2PyU4z2TXluLYnv3hUHNv+Kzu2jWc9pXeC07Zfv8NdP3+CV557BW7NnGOuDA/xY60c9PW608bl1kTmZXDvMnJa8lknbwDE05f+GhBBCiI3VzX184OJ2HLr4N+wEIsS3H4Iu4fEQ8AQQ2onB5bA/xBo0KhQd34Syywfh9dBUOEb2AId3+1dxer0BGq0BWp0eegOQX1qD6+lVuJpaicuplQjykqBzuBM6hzsh3N+epv8jhBBCzIgdv9L490dzH7NhTwghtkbhVNuhzb1h6y60CIPBgIzsXFy5ngS5Uono8FBEh4dCKDQdwDRUWSXH7v1HEN+jKyOU0ul0uHD1BtIzs6HX6xHXuSOCA/waPdaiz7/Fh599wyi7eGAb2kWGW+xHYXEJbiSnIiMrB57urggPCUKgnw94vKavq6xSqZGUkobk1HQ42EsRERqMAF/vJoVDer0epxIuITUjE3q9AS7OTogIDUZEaJDZfUpKy3E18SYysnMR4OuNqPAQeHmYDq6aq7C4BEdPnUNllRz2Ugk83FzRvXNHxrpW1dU12LX/MDrGRCI8xHx/gdp1oHbtO4xusR0svr43U9KRcPkqZE6O6NQu2mQoV1Ojwt97D6JLx3YICfQ3eZzmPLeNqZIrcPbCZeQVFiEmIgwxEWEmr/8rN24iIysHjwyMNzmi8FZaBi5dS8SjhIk4IAAAIABJREFUDw24o/WhWvJabo34PuanRiS1KKAihBBy3yiXlyAp6zKOXv0XYqEEgZ7hCPWOgpfMD072riaDqfrkqRdRen43VMVZCJmyFDyR6aH7BgOg0uhQqdCiXK5BhVyD/DIVcourUVqpht4AhPhIEO5rj2AfKWT2AnC5FFgRQgghAAVUhJBaFE61LW0loGpNps2Zj183bmWUJZ/aiyB/Xxv1iBDS0iigsoxv6w4QQgghlsirK3Er9xqSsq6gqrocHs7eCPWJQbhvDFwdvcDjWvfLGrF3KKQB7VF+cS+qc5Mh8Y0EVyhhteNwAJEdDyI7HjxkQuj1BhSVq5FfKkZeSQ2KK9VQ1Ohw8VYFzidXQOYggL+HGIGeEni6CGl0FSGEEEIIeaBROEUIm0KpxLZ/9qO6RoXrSclY99ffrDaOJqYgJISQtowCKkIIIa2SwWCAWqtCTnE60vNvIr0wGVqtBh7O3ugd8zBcnTytDqbq8MQOEPuEQeQdhtILe8B3cIXIPcDiflwuB54uQni6CNEpzAlVSi1S8xRIypQjr6QGFXINSqvUyCqshqujHdyc7eDqaAcnqQD2YvqnlhBCCCGEPDgonCLEtNH/eQGHT5wxW+/t6Q4XGXutKkIIacvorhkhhJBWxQADNBo1KpRlKCjNwqnEg8gvzUKodzS6dxiO0DscHm3n7AW3no8je9tncIruA6GrLzhNDLocJHx0CnVCp1An1Kh1yC6qxpW0SlxLq0R+iQq+7iKE+kgR4iOFp0wEqYgHsZAHOwEXPJoKkBBCCCGEtFEUThFimk6nw7HTCY22WThv9j3qDSGEtB4UUBFCCGlVOOAgqygVexI24+iVf9C/06P4z5C58HcPbpHj8+1lcGzXF4Yty6DIvAqRRyCEbqYXI7WGyI6HMF97hPna4/F4H1QqNThzowzHLpdi8+E86GFArxgZ+nZ0Q/tgB4iF9E8vIYQQQghpeyicIsQ8Ho+HYYP74e89B03Wvzh1Ep6dMPYe94oQQmyP7pIRQghpNVJyr+PEtX3ILcmAr1sQ3pm4HB4yHzhKZDDAAA5aZvQRlyeA10NTUXXzNIQuPncUUDUkEfIRFyVD+2BHyKt1KC5XITVPgd2nC7DpUA68XIRoH+KEmEAH+LiKwKURVYQQQggh5D5H4RQhlm34/gts2bUPB46dRFFJGUQiIXrHdUZ8967oGBNp6+4RQohNcAwGg8HWnSCEEPLgUmlqUFxZgHM3jyC3OB0iOwmCPCMQ7BUJb9cA2AmELRZM1TEY9NBUFCFj3QJIAjvArcfoFg2p6mh1BihrdCiTq1FcoUZhmQpF5SpUKrVQa/SwF/MR7C1BqI8UHjIhrVdFCCGkTYgdv9L496BeUTbsCSGkuQY34b1L4dSDQZt7w9ZdIISQ+w7/DpepeBDQnTBCCCE2odPrUFiei9S8G0gvSEZZVRE8Zb6I9OuIUJ8YSEUOd+3cHA4Xds6ekAR1hKooE4rMa3cloOLzOHCU8uEo5SPQU4IatQ75pbUjqrILq1FVra39u6gaDmI+3JyF8HIRwlMmhKNUQOtVEUIIue8dOJlo6y4QQprB2oCKwilCCCGE3AkKqAghhNxTOr0OFYpSFJTl4FbONWQU3gJgQI+ogYgO6Ayp2KHFR0yZ49xhIAoPrkXVzdNwiu4Nntjxrp5PZMdDkJcEQV4S6PUG5JeqkJwtR1KWHNcz5OBnK+DrJoKvmwiuTnZwcbSDg4QPBzEfAj73rvaNEEIIIYSQpqBwihBCCCF3igIqQggh94TBYIBOr4WiRo6zSYdx+PIuiIX26NdhKHrHDIaAL7znfZL6x0Do5gd56gVUpV6Ec7t+9+zcXC4HPm4i+LiJ0D/WDYoaHZIyq3AuqRwbDuaAwwECvSToEu6MjiGOcHG0A5/HAY/LAYfDAYcGVxFCCGmlZo3rZusuEEKaYeWmc1a3pXCKEEIIIS2B1qAihBByT1QoSnE++QT2X9gKkZ0Y8e2HIiawM2T2buDzBODYKHGRp15A8emt0FaVImzG1zbpAwAYDIBWb4BWq4daq0ducTWup1fhSlolbmUr4OsuRmyYI7pEOCPYSwo7AY2oIoQQQgghLaf++nEfzX3MbDsKpx5MtAYVIYQ0Ha1BZRmNoCKEEHJXyasrcSHlJK5nJECtUaFXzGCE+baDp7MvHCRO4HFt+0+R2DcSUv/2KDq+Ecqs6xB5hoBrJ7rn/eBwAAGPAwGPB7GQBwGfC1cnITqGOqGsSoP80hoUlKnw56Fc8Hgc+LmJEBnggHA/ezhK+ODSelWEEEIIIeQuo3CKEEIIIS2JAipCCCF3RZWyAim513Ej8yLkNZVwlMgQ4BGKcN92cHfyBofTOkYA8YQSiP0iIfaNRMHBX+E3eq5NAqqGJEIeJEIePJyFMBiAonIV8kpqkFtSg9JKNbQ6PRKSypGQVA53ZyH83EXwcxfDy1UEHoVVhBBCCCGkhVE4RQghhJCWRgEVIYSQFqWoqUJ+aTayilKQlHUFaq0KEX4d0C6wC3xcA8Dl8mzdRRahizecYuKRvnY+3HqPAU/iBK7g3q+JZQ6HA3jIhPCQCdEh1BEVCi2yCpVIz6tGap4cafkK5JfWILOgGu4yIVwd7eDqaAdnewHEQh6tV0UIIYQQQu4IhVOEEEIIuRsooCKEEHLHDDBArVFBXl2JrKJUXE49jZTc6wjzbYcRsRPgKfMFnyewdTfNEji4QewTDq5YCkXaZQgcPSDyCLR1t0zicjiQ2Qsgs3dCxxAnaHUGpOUpcDWtEtfSKnHwQhFCfKQI9pEixEcKD2chhAIuxEIuRAIeeDxKqwghhBBCiPUonCKEEELI3UIBFSGEkDvGAQcFZTnYfPRHZBQkIyawC54b/hb83INt3TXrcDjgS2XwHfEKio5vgsg7tNUGVA3xeRyE+9kj3M8ej/f1QVW1Fqevl+LktTJsOJANHpeD7lEyxHd0RYcQR/B4rW8EGyGEEEIIaZ0onCLk3tJqtfhr5x78e+gYxo54BEMH9QPHxlNiaDQa7D18AsfPJKCwpBRyhRJODvYIDvTH2BGPIDQowKb9I81z7EwC1m7cik7tovH02FFwdLC/o+OdSriEMxcu4dGHBtA1QZqEYzAYDLbuBCGEkPuTRqtGVlEKjl/di+Tca4j064CYwK7wdw+Go1QGO37rmSbPEoNOC71GhZvfTIdT+/5w6zEadjJvW3erSQwAdDoDlCodqmt0qFBqUFCqQlqeAml5SihrtPB2FaFdsCOiAhzg5yEGjacihBBCCCEAEDt+pfHvj+Y+RuEUYdDm3rB1Fx4Iq9ZuwItvfWDc3r/5F/TrGWez/hSXlmHctJdx7EyCyXqJWIwL+7ciJND/HveM3IlrScmIHTTauD3h8RH49ZulzT7e+0u/wpLlt/8NObV7E7p2bHdHfWwr+D7Rtu5Cq0cjqAghhDSZAQak5ibiavo55BSnQSJ0wMBOIxDkFQEvmR/EQqmtu9hkHB4fPB4fTjHxqMlPRVXKebh2e9TW3WoSDmpHVDlK+HCU8OEus4O3iwh+7iJEBzqgqFwFRY0Wt7LluJJaCQGfi1AfKcJ8pfBzF0MiotFVhBBCCCGERk4RYisHjp1ibO89dNymAdWkF143G04BgLK6GonJqRRQ3Wf+OXCUsf3voaPQ6/XgcrnNOt6Kn35nbP+y4S8KqIjVKKAihBBiNZ1eh+LKfCRmXkJReS5Kq4rg4uCBdkFdEOUfCwHfztZdvGPOnR5C7q4VUKRfhmNkTwgcXG3dpWbjcjhwkPDhIOEj2FsKlVqPzEIlsouqkV1Ug5JKNTIKlCgoq4GDhA8Xx9pAy8dNBAeJAFwaXkUIIYQQ8kCicIoQ20jPzGZsFxaX2KgnQFl5BQ4dP80q93R3g6ODFPmFxejcPhqD4nvaoHfkTmRm5zK2S8sqoFBWw8G+eT82bh8ZjuNnzxu33V3v3/so5N6jgIoQQohFWp0GVdUVyC/NRnLOVaTk3oC92BGxYb0QE9gF9iJHW3exxUh8IiDyCIK6NBeKtEtw7jjI1l1qMUI7rnG9Ko1Wj7IqDRKz5EjMrMKN9CrYCbjw+n9A5SETwtleAEcJH1IRH3aC5v2SihBCCCGE3L8onCLkwZWUkg69Xs8oG9S3F3avW93skTak9Wr4WjfFwnmzMWfBEtxMTcfgvr3wwpSJLdgz0tZRQEUIIcQsg0EPjVaNquoKXEw5hYOX/oaipgpPDZiFDsHdIBHe2SKarRKHA6d2fVFybifKruz/f0BlANrYak0CPhceMiE8ZEL06+iKGpUOiVlynEksw297sqA3AO2CHNAtyhkxgQ5wcxJCpzdAwOeCx+XAxuv0EkIIIYSQu4zCKUIebGXl5ayyAb27UzhFWAb06YEL+7dBp9OBx6OlA0jTUEBFCCHEJIPBALVWjcOXduLE9b0QCEQY0nUMuoT1Bo/Lh1AgtHUX7xppYEco0q+g/OphKDKvQOwdAW4bfrwAYCfgIirAHqE+Ukwc7Iec4hokZVbh1LVSbDiQA6mYh+5RLugW6YxALwmENKKKEEIIIaTNonCK2FJ+YRGSbqWhX684cP7/yziDwYAzFy5DpVajR+dOEAotTy9fUlqOq4k3UVhcAm8vDwT6+sDf17vJ/amskuNGcgpKy8pRUSWHRCyCk4MDIkKD4e3p3uTjlZSW48qNJDg7OaJdZBgEAoHV+6pUapw4ex5+Pl4IDwmy2D43vxAXrlxHh5gIBPj6NKmfGq2WVRZq5VpTer0eyakZSE5Nh6eHG9pFhkEiFlt97pa6BhqSKxQ4fuY84nt0hVQiYdSVlpXj4tUb8PPxRkRokNljZGTn4FZqBtpHR8DT3a3JfUi6lYa8gkKUlJVDr9fDydEBbq4u6BgdAT6/abfqa2pUSMvKRnZuPsrKKxAU4IeosBA4OtzZj4kzc3KRmp4FtUaD7p07wtnJ8qw5hcUluHI9Cf17dzf7OK7fvIWi4lL06hYLOzvm65eVk4fL15MQFhyIyLDgZvW7pd7z5N6igIoQQgiDwWBAhaIUNzIv4nTiAYjspIiLHIBQnyh4yfxhL2470/mZwxUIIfGPQU1hBgoPr0PA2PlAGw+ouFwORHY8iP7/GTHQUwyZgwDRgQ4oqVSjuEKNgtIabDqcAzs+F37uIkT42SPc3wFSIQ88Hg2pIoQQQghpC7zdHSicIjaz+reNePW9j6BWaxAeEoiEPVuwbsvfWPjpV8gvrF0bTSQSIr57Vyxb+CbaRYYz9i8oKsa8D5bi4PFTxvb1dekQg1n/mYApTz1hDD7M2XfkBBZ//i1Onb8EnU5nsk14SCCmThiDOTOmNBouaDQavL/0K6zfugtZuXnGcqHQDp1iojB7+mQ89dijjfZHrlCg/2NP4/L1JADAO6/OwsJ5L5ttn5qRhbhHxqCySg6BQIB/1v+Afj3jGj2Hsroaz746H2cvXEFOfgGr/pnZb+LldxYjLDgAC+e9jIf69WbUr/njT3z/y3pcT05BTY3KWM7hcBAaFIAh/ftg0VuvNhqg3Ok1YM65S1fxyJPPorJKDgd7KQ7+tRYdYyLxyTersXbTVtxMSTe2dXOR4eH+ffDd0oWQSiSoqVFh3oefYPu/B5CbX2hsFxLoj6ED+2LZwjcbDRorKquw6PMV2LBtl8nrEgAkYjEG9OmOj96ei/ZRjT+mxFup+PqHtVi/dScqq+Ss+kA/H7z2/DRMnzS2SQHoll17sfiLb43XGFD72rWPCserM6bgmfGPmdzvlw1b8PybC6HRaODj5YHjO9bDz8eL0ea19/+Lr35YCwDo37s79m78CVVyBea+/1/sO3ICOXm3rzcPN1cM7NMDX330Llxkzo32uSXf88Q2OAaDwWDrThBCCGkdKpVlSMm9gZvZV1FVXQ57sSMC3EMR5tse7k5eD9Q/5lp5Gcou7Ufuzq8RMftHCN38wBWIbN0tm9DqDCgqVyGvpAb5ZSqUVqpRrdJBpzdApzfA20UELxchgryk8JAJwaewihBCCCHkvhM7fiWNnCJmaXNv3JPzDHx8Mo6dSTBuz39lFj75ZrXJgMjfxxupZ/cbt9dv3YlX3l2M0rIKi+cZOqgffv36E8icnVh1yupqzHjtPWzYtsvqfje2NlNaZjYmPf8azl680ugxZk+bjMMnzzDCgWcnjMH3yxYBALbu3odx028HUkKhHW6d2gsvD9OjuF54cyFW/7bRuD190jh8t/SDRvuw5o8/MfP1BY22qdOrWyyObFsHAEjPysHMeQtw4OhJi/v5enti5dIPMHRQP5P1d3INNOatxcvw2XdrjNvTJo5FaXkFtuzaa3affr3isPqzxZj84jycuXDZbLsnHh2Cdd99ZnJ6u38OHMFzr71rNphqiM/nY9fvqzAwvqfJ+nV/7cCsee+juqbG4rFCAv2x5ecViIkIY5S/8s5ifPvzOkbZ7GmT8fWPaxs93qplizB1whhWecPX7JP35mHurKnGbb1eD++OfRjvzb0bf8Lc9/+LKzdumj1f147tsGfjT2YDzZZ6z99NfJ/oe3q++xHNz0MIIQ84vUEPlaYaqXk3cDbpCC7cOoGSygK4O3ljQKcR6BE9EB7O3g9UOAUAfHsZJL6RELoFoPzqYWirSm3dJZvh8zjwdhWhS4QzhvfwxMjeXugeLYO7kxCVCg3S8hS4kVGFE9dKcOxyCa6mVSKnuAZKlQ70OxhCCCGEkPsDhVPE1gwGA85cZIYAS5avNDt6KSe/AHKFAkDtjerJL86z6kY1UBsaPDVrrsnvK59991OTwikAOHD0JH74fROrXK/XY9z0ly2GUwDw9Y9rGeFUQw5S5pR0KpUav2zcarKtQqnEH1v+ZpR5e3pY7IO1o5EAwNe7doRMWXkFej/6pFXhFADk5BVg5ORZ2Lp7H6vuTq4BSy5eZYasP677s9FwCgCOnDyLyN6PNBpOAcBfO/dg8RffssqV1dV4auZcq8MpANBqtZj66tsmH/OGrbvwn9lvWhVOAbWj6IZPfA7ZufkW21oKpwBg5rwFOHzyLKv8dIPnp+F1nFdQxHpvPjx+aqPhFAAkXL6GJ2e8arKuJd/zxLYooCKEkAeUwWCAWlODksoC3My+ir0JW3Dy+n6IhfYY0XMiRveeDC+ZH3jcB3c2WDsXb7j3GYuS01tRU5wFGPS27lKr4CQVoFOoE8b098HbkyIxsrc3XBztkJhRhS3HcrHlaC6OXS7GjfTaoKq0qnbEFX0QJIQQQghpvSicIram1+uhVmtM1nE4HESHhzJGUkwaMxL2UinkCgXe+HApoz2Xy8Vbs2cg8fg/yL54BL9/u4y1XtCBoyfx3c9/sM71++btjG2JWIxVyxYh++IRyFMv4ubJPXhx6iTWfn/+/a+JY+3ApWuJrPK42A74aP5cLF3wBvr26GbyMTfUt2c32DcIqTaaCdJ27DkIuULJKBs22PSIpfp6dOmEk7s24qP5c/H48IdZ9aOHDsbit+dg4+rl+O6ThQCA+Us+R1EJ8wedrjJnTJ80Duu++wzLF7+LuNgOrGO98u5iVMmZ4VJzrwFrmPs26u/jjRenTsKrM/7T6Jpi9lIJJo0ZhZnPPAVXE9POmQq7tu7eB4WS+ToMf6g/Tu3ehIpbCSi8dhI7f1/FWh8sJ68AJxMusspemv8h6xwBvj54a/YMfLn4HYwdORQO9lLWfh9+tsLs46qPw+GgT1wXfPjmK5gzcwrEIuYsMgaDAWvW/cnaT69n3ivRaE2/hqa4ucgwbeJYfDR/Lnp368yq33fkBDKycxhlLf2eJ7b14N51JISQBxyHw0FmYQr+TdiMAxe3Y1TPpzFlyKvwcw+xdddaDYGDK5w7DET6+g+gzLoOsWcwBE6Wf3X2oInwt0eEvz0e7+uD0io1LiRX4NS1Uvx5OBdiIQ+dw53QM9oFPdrJ8GCNwyOEEEIIIYTcKV9vTxzfsR6+3p4AgGtJyVAoq9G9c0cAwMdfrUJeQRFjn4/ffR1zZk4xbo8fPRxdOrZDp0GjGAHItz/9jhemTjRu5xcWISU907gtlUiwa91q9I67feM8OMAPXy5+BwePn8b1m7eM5edMjJJ6f+lXrLInHh2CDau+NG7PmTkFL771AVat3dDo82BnZ4fxo4ZhzR+bjWWXrych8VYqosKY3+Mbjp4KDwk0GRKZ0q1Te3Tr1B7b/93PCl0mPjESTzw6xLh9+vwl1sgxkUiIvZt+RofoCGPZ9EljMfXVt7Fx225jWW5+IRZ++hU+++Bti32ydA00l6+3J45s+924XtLUCWPQaSA7rBeLRPhn/Y/o0aUTgNrnof9jzJAyJSOLtd+x0wmM7bEjh2LtN0uN65VJxGIMGRCPT957HRNmzWW0PXfxKuK7dzVuf//repRXVDLa9OoWiy0/fQtXl9rA7MWpk3A1MRkDn5jMaLtr/+HGn4j/W7VsEaY89YRxe1B8T4ycPIvR5viZhIa7NVtMRBhO7doIsbg2CHv1uWcwaMwzOH2eOSLrxNkLCPTzNW635Hue2B6NoCKEkAdQYtYl/LznC/x1/Ce4OLhj8ZTVGNZ9PNydfSzv/CDhcAAOB14PT4ci4yrkaRct7/OAc5Dw0S3SGc8OD8R/Z8Rg5sggeLuIcORSMV5efhlfbb6FPecKkVmohFZHI9IIIYQQQgghjVv2/pvGYAKonYaufjCxY89BRnt/H2/MnvY06zhhwYF4tsH6OUkpaYxAysvDHQ/372PcXr74HUY4VV9cZ2bgUyVXQKO5fSM8r6AIWbl5jDaBfj5Y9f81per7ctF8qwKk6U+PZ5U1HEVVWlaOPYeOM8r+8+QTuBv+bvDcA7XPWf1wCqgN175a/B6kEuYIsL8sTLFXx9I10FzPThhrDKeA2sCkfRR7msN5L043hlMA0DuuM0IC/RltampUUKnUjLInH3sUAoEAABAZGow1XywxhlP11T92nZLSMsb2+q07GdtcLhc/fL7EGE7VaR8Vjm2/fAeRSGgsCw7whSWTx41mhFNA7bpNDUd3lZZbN6WeJSKREH98/7kxnAJqr5PJ4x5jta2orGJst+R7ntgejaAihJAHhEpTg+LKfJxLOoL8shwIeAL0iByIQM9w+LkHg8flP3DrTFmDayeCW/eRyNiwCNV5KZAGF8KORlGZJeBx4STlwkkqgF5vgIdMBB83EUJ9pSgsV6GkQo3EjCpcS62EvZiPQC8Jgrwl8HUTQSLk0TVICCGEEEIIMeJwOBg6qK/Zep1Oh+S0DEZZz26x0On00OnUrPbdOnUAsJ5Rlp6ZjdCgAOP2rnWrcfbiFcicHBEWHGgsz8jOwemESzh6+hz2Hz2J5FTmeQGgskphDAxSMtg3wUc8PBBOjg6scoFAgPGjh1tcqyoutgM6tYtiTBu4YdsuLHjtJeP25p17oNVqjdtcLheTx45u9LjNdS3pFqts/KhhJtu6ujhjwuOPMkZcZefmo0quYE1LV5+la+BODO7Xi1XmYmL6vn492dMwusqckdpg1FSNSgWh0M643b9XHLIvHMbVpGTEdepgDGOqq2tw/up1nDx7AQePn8bRU+dYx6+skhv/zs0vRFpmNqN+xMMDEREaZPJx9Y7rjM0/fo0lX65EgJ8P3pz9nMl29Zm7RqLCQ5CZk2vcbjgtY3P16tYZMRFhJs4XyiqrP13l3XjPE9uigIoQQto4laYGZfJipOUlIjnnKsoVpfBzD0FMQGf4uAbCUcL+8EVu4/AEELr5Q+IbCXVZHhRpl2AXy56Lm7BxuRw4SvhwlNQGUTqdAbklNcgurEZGgRIllWqk5iqQV1oDexEfni5CeLkI4eokhEwqAM0HSAghhBBCyIMtIiSo0TWGUtKzGKOWAGDT9t3YtH23mT3YuFz2BFNxsR1wIzkFS5avxJkLl3H2whUUFpdYPFb939ulprOnfGvfYGRRfbHtoqzq77SJ4/DyO7dHYd1MScfFqzcQ2z4aALC+wfR+jwyMh4/X3fmRZf0pDoHa6Q8be70iQoNNHsPUCCLjPhaugTvh48l+XqQSMavMy4O9NpW0wXpg5rjInBHXqQP+3nsQh06cwdkLl3ElMZkRIppS/8ebOfkFrPqOMeavJQAYMiAeQwbEW9VHAAgK8DNZ7uYis/oYTRFi5nym1vfS11vP+m6954ntUEBFCCFtlMFgQKm8CAWlObiVew0ZBbeg1+vQt8NQRPh1gL3Y0dZdvK84tR+AohObUHnzFJyi+4ArtO7DKKnF5XDA5XMQ6ClBoKcE3aNlKKlUIylTjsTMKqTnKXEziwtvVyF83ETwcRXD/v/hlr2YDzsBfYAkhBBCCCHkQVN/+jVTshtModdUMmcndGoQDOXmF+KZ2W/g8Ikzd3Ts4gZTtAGAp5ur2fYhQf5m6+qb+MQIvLnoU1TX1BjLNm7bhdj20cjOzceRBqNxptyl6f0AID0rh7FtLuSo4+RozyorLGo8+LN0DdwJF2cnVpmAL2CVSepNQ1fH2pBj1doNeGvxskZHHnE4HBjqhTANFRWXssq8TYRrd8JUMAcAfD6vRc9TRyI2fT6BiSkQ67sb73liWxRQEUJIG2OAATqdDhqtCieu7cPJ63shspNgUOxoxLcfYvkAxCSH0C6ouHIAiuwbUOYmwz7Y/C+8iGUCPhdeLiJ4uYjQP9YNNSodEjPlOH2jFFuP5iGvVIVukc7oEu6MdsEO8HIVgcfhgMOtDbsIIYQQQgghbZ+5m9h1/Hy8WWX2Ugk6Rkea3UdgJ4Cftxc6xkRi2OB+jCndKiqr0HvEk8jJY49Y8XBzRc+usejVLRbxPbphx7/7sXTFD4w29UOLAF923/KLis32Ky0zx2xdfU6ODhg/ehh+2bDFWLZx+24seec1bGwwisRV5oyRQwZaddzm8PfxYoRUWTmNhwdVciWrzM3VpdF9LF0Dd0IkFFpudAd9+OL7n/HGh0tZ5TweD7HtotCzayx6x3XTNzM5AAAgAElEQVRGn+5dEdSN+TrVv5bcXNmjmEwFoA1pNBooq2tMTivZkNDOzmKbliQSNu98Lf2eJ7ZHARUhhLQx5VUlSEg+ip2n/4CfewiGxo1HdEBnOEnvzrDsBwaHA4eo3tDWyFF8YhMFVC1MaMdDu2AHRPjbY9wAHYor1biaWolzN8ux/UQe7MV8dI5wRtcIJwR5SiG0oxFVhBBCCCGEtHWmRq7UFxLoB4FAwJjyq2/Pbtj+68pmne/jr1axwqk+cV3wyYJ5rGno1m7aytq/fqhgao2by9cTWWXW1DU0beI4RkCVkZ2LUwmXWNP7TRozEgIBe0RQS4mJDGMEVCnpmaiurjGutdRQcmo6+xgR7DWH6rN0DbRWxaVlWPjp14wykUiId+e8gBenTmRMW1hgIrjkcm//MDPaxLpMSbfSGj3/viMnMPnFeaiUy/H689PwwRsvN/Uh3FXNXX+6pd/zxPbo7g4hhLQRZfJiHLmyCxsOf48bWZcwpNtYPNpjAjqG9ICLgzvs+Nb9MoiYJw3sALF3OJTZiagpyoReU2N5J2IVDqd2VJVExIPMwQ5BnhL06eCK8QN98Z+hgejX0Q3Kai02HcrFF3/ewm97MnHqeimKK9mLoBJCCCGEEELaBkvTqPH5fIQ1CIJOJVxqdLo0uUKB5994H93/x959h8dVnYkf/957p3f1Xi1blnu3sQHTOySEkFCzBNI2dZP8IHWzsNlNsullk7BppGwCmwABAiR0bGNwb5ItW8VW73VmNH3u/f0xRvZYcgFsC5v38zw8+J4599x3zsxImnnnvOfyG/jJr/+QdtvaDZvTjjVN48H/+cGkeyRt3LZzQps/EBz/92QJqr+/sJZRf2BCu67rPP6PF44a85HOWbKA2dVVaW1f//5P2V5Xn9Z2x003nPCYb8bsGekxGIbBX//+3KR9R0b9PHhEAq2oIO+4q3vO1P2Ctu6sIxQOp7V98Kb38IVPfnjCnlqTP5cOlQR0u5wT9hF75Kln6O7tn/TaLe2d3P6JuxkYGiYWi/Nf//3LSZ93Z6KT/ZoXU+/MfIULIYQAUn/8BcKjbGlYx7NbH2Ff+y68zgwWV63inJqLmV40B58zE009NTWD32lMDg/2wulYMgrof+X/SIyNTnVIZyVFAYtZJS/DSnWJiyXVPlbMzmDpzAwWTvdSlGUjHNWpbw3w17Vd/OXlTjbuGaJ7MEI8oU91+EIIIYQQQoiTxGw+fvGnyy5YlXY8PDLKV7/5g0n79vT1c+mNH+RXf/wL2+vq+eJ/fJdIJHrY7ekrWUwmDbtt4pc9//CXx9m5e+KKp31N+8f/7XG7OH/FkrTbWzu6uP2Td6Pr6e9bvvyN77/hPa/uuvXGtONnX34l7XjR3FnMrZnxhsZ8o66epHzgJ754Hw3NLWltiUSCz9/7rbQEHsA1lx6//OCJPAfejo58LgG4HBP3sh4aHuE/f/DzCe2HP5cALli5PO04Go3xqS//e9rzF2BwaISrb/1wWgnAZDJJIpF8Q/G/nZ3M17yYepKgEkKIM5JBMOxnf3c9WxvWsb3pVQZHeynLm875c69i5exL8LmyJDF1CthySvHOWc3Aa48SH+lFT8gKnlPNpCnk+KzMr/Jy9Yp8rl1ZwMo5mfhcZsLRJPvag2zZN8K6XQO8WjfE7gN+OvvDjEUS6PrRv0UlhBBCCCGEOPN97fOfJD83O63t2z/9Ff/27R+P74kUj8f5/Z8fY/Gl72HLzrrxfsUF+dgOS0CtXLowbZxoNMYXvv6d8bJ/Le2d/Pv3fsqd//KlSWPZ3dCUdnzvJGXV/v7CWpZefgP3ffcnfOdnv+KSG+/gez//zRu4xym33XAd1mPs43OqV09BqvzhHTe9J60tOBbikhv/ibvv+y/++vRz/OEvj3P5++/k939OL4mYk5XJ17/wmVMe41Q58rkE8OsHH+a5NetJJpNEIlHWvLqJC66/nW21eyb03dPYnLYq6Jtf+TwuZ3qC6/F/vMB577qFn//2QZ589iW++s0fsOramyYkCJcumEtW5tmz79LJfM2LqXdmpqCFEOIdLBILMxYJ0N7fzPam19jTtp1l1eezavZl5GeUnLHL388UFl8+zrK5JKMhQl0NmL05WDImbtIpTh2f24zPbWZ2hYd4QqejP8LO5lG2N46wbtcQeRlWZpWn9rPKdFuwWlRsZhWbVUNT31ydayGEEEIIIcTpdyLvbz1uF9+/70vc8s+fT2v/xo/u5xs/up+CvByGR/2Trpr46mc/nnZ82QXn8tBjT6W1/ebBR3jgoUdx2O2MhULHjOXl9Zv49Ic+MH583vIlXHL+Sp5f+2pav1179rFrz77j3rdjyfB5ee81V/DHR56YcJvVauHm669+S+OrysS5n+zx+K+v/j+efPaltBU73b39/PAXv+OHv/jdUcf/7r1fIMPnPX4cJ+kzDm2ScdRJ3h9q2sR+k7ZNMt7h/aZXllNeUpS2R9fQ8ChX3fJhLBYzhkHaPkpHGhn1s3P3XhbMqQGgMD+X++75NJ//t2+l9dtRV8+nv/L1o47jdDi4/zv/ntY22f5PR9sT6sj2yfqpqkoyeWiF1pHPnckew6M9rpOOf0TbyXzNi6knn2IKIcQZprWvkd88811+++wPMQyDe973HW449y4Ks8okOXU6KAoWbw5FV32Cke3PEelrneqI3tHMJpWKAgfvPreA+z5Yw9fuqGblnEz2tQX56q/28OVf7uZ3/2ijdr+fWPzsKWkghBBCCCHE2UbTNBbNnZXWtnTB3BM698brruSBH32TzIyJCY/u3v4JH1Rrmsa3v3YPt9/4rrT2m959Fe9/11UTxjAMIy05lZXh4yff+FeWzJ+T1u9AW/uEcx+8//tcunrVhPYjffajd0yIZ1b19GOec9et7520/d1XXILP6znuNY9l8RH3DZhwfwEyM3ysffyPrFq66ITGzfB5+e2Pv8Ut77l2wm1v5TlwPKuWpce3ZP4czGbzhH7nLElf+VQzfdqkc3nkeHNrZkzYW+qBH39r0j22YrF4WnLq2ssu4rv3fnFCvwNtHWnHn7zzNr577xex22wT+k4mNzuLP//yh8ybVZ3WPn/2zLTjqooyPG7XpGMc2Xey58DSI9qWLZqXdlyQl0NRQV76OQsnf1ynlZdMeB1Pds2T9ZoXU08+yRRCiDNAJBamsbOOXz79X/z1ld9SnF3BHZf9C9edcys+ZyaKrAo5rTS7m+yVNxDz9xNq200iMDTVIYmDMlwWlszM4J+uKOXb/zyHO68qI8dn5cXt/fzbA3v54V+aeGZTL219IdmvSgghhBBCiLeZ22581/gKiswML9ddftGJn/ved1G35iluveG6o67MMZvN3HrDdexe+xSf/egdk97+h59+h8//852TfvCdnZnBRz9wE3vWPc3H/ulmHvrFDyjMzx2//SO33zThHJ/Xw9/+cD93f/wusjMzJtw+s6qS//vFD/n21+7hlvdcO540cTocx73/5y1fQllx4YT2I8vuvRkFeTlcefH548eXrl5FcWH+pH2nV5bz4qO/57v3fpFp5aWT9snM8PL+d11F7ct/49Ybrjvqdd/Kc+BY3nP1ZeOJJlVVuf3Gd0/a77rLLxrvpygKH3jf5P0OHy/V7/oJfc5dtpgXH/k9C+fUTPhC8evJuMd/93MefeC/+cyHP8AXP/WR8dsrSosn7LWkqiqf+fAH2PHi41x96QWTJr8g9djdd8+naXjtGS674NwJt1+6elVa0u229x798bjiwvNwu5zj9/O2905M8Nx247vG71+Gz8u7rrh4Qp873n9ofgrzc7nk/JWTXk/TND5w2GNTWVbCucsXT9r3ZLzmxdRTjMOLWQohhHhbSepJWnob2bV/I71DHdisdsrzZlCeP4P8jGJslokbbIrTp+3hb5IIDpO56Ap8807OH83i5BodizPkj9E3EmNgJMpwIEY4lmQsksRu0SjNc1BZ6KAi34nFLN/bEUIIIYQQYjKJrvrTdq2mA63s3L2Xqy+54C3tFdM3MEh9435a2zvJy8liemU5ZcWFaNqJ79Xc0NzC1l11ZHg9zJ9dQ0FezoQ+kUiUJ597iUXzZlNZVnLcMRv3t7BlZx0uh4MZ0yqYXpleDaW7t5+1r23myovPP+qqltcNDY9QvuQiwpHIeFtJYQFNG587aRVW1ry2GUWB81csPeFzxkIh6vY20tDcQn5uNnNmzph07o7mZD0HjhQKh3nquZdZtmgeZcVFx+z39PNrWDx/DhWlxccdb+nCeZSXHH08gEBwjM3bd9Hd18+sGVXMmlE16R5itfUNtLZ3cvmF5066wutI3b391Dc0MTg8QmV5KdOPsRrqcOFwhKdfWMO8WdVMryw/Zt+xUIinn1/DkgVzjzofLe2dbNlRy9WXXIDdPvkKr937Gtnf0s4VF5133Pu2a88+2jq6uPLi80/4NXsyXvMnm6mwZsqufaaQBJUQQpxm0XiEUCSIqqp4nZmT9jEMne6hdpq76mnv308kHsZqsrFw+kqmFdRgNZ/Ycm5xagUP7KDnhQewZZdScPlH0OyTf3tJvD3EEjpdA2GaO8fo6I8wOhbHrCnYbRpuu4kcn5W8TCtZHgtZHstRa3ALIYQQQgjxTnM6E1TixN37nR/znz+8P63tXz/3cb72+U9OUURCiMNJgur4TFMdgBBCvNO09zdzoGcfPmcWS2acn/YheCIZJxAapXekk71tO2jursfjyGDBtHOYW7FEVky9zTjL5mLNLCI63M1Y+x48M5ZPdUjiGCwmlfJ8J+X5ThJJg/6RKI0dQRrag2xtGMHtMJOfaaU0z06O14rPbcFtN+G0a5hNKieSrhryx0gkDdwOE3br1H1LSwghhBBCCHH2SSQSxOJxHHY7L6/fyA/+53dpt2uaxl233DhF0QkhxBsnCSohhDhtDEBhza6n2dKwlrnly5hbsQybxY6Bga7rDPn72dKwjme2PoLP6eOaFbcxp2IJdklMvS0pqoZ3zmoGNz3B8LZnJEF1BjFpCgVZNgqybJw/P5ukbrCzeZTN9cM8sqaL4UCcmlI3S6p9zJ3mJS/TiqYqKAqox1hZtWXfCL3DURZN9zK74q1tSiyEEEIIIYQQr/vZA3/ia9/+EaP+ADablUgkOqHPzddfTVFB3hREJ4QQb44kqIQQ4jSJxiNsbVjPvvadDPn7ae1r4rU9z3PhgmsJhEZ4pe5ZtjSsxWa2c/MFH6O6dB52iwOLScr5vZ05i2sY27+D0T3riPQewJpTiqLKypkzjarArDI3VYVOblhdyEggzt62IDv3+3lsfTcep5lF030sqPIyo8SFSZuYpNJ1g71tAV7bPczuA37uuqqMyiInmiqlAoUQQgghhBBvzU8f+F9G/QGASZNTBXk53HfPp093WEII8ZZo9957771THYQQQpzt4sk4w4FBHnzpZ3QMtBCLR4nFw4yMDWIxWfn7pj8TigaZWTqfc2ZdQmVBNV5nJmaT7IPzdqeazCQjQeKBQSLdzbinL0HRjr+RqXh7URQFk6ZitWjYrBpOu4lsr5VphU6qS9xkeSwMB+LU7vfzSu0AHf1hYgkDs6bgsGkoKBzoDrGudpDGjiDBcILO/gjFuXYcNhNm08nZoFgIIYQQQoipoAcGpjqEd7x1G7ZQ39A86W3Tykt5/uHfUV5SdJqjEkIci+rOmeoQ3vYUwzCMqQ5CCCHOdgOjvazZ9TR/2/C/hCJBDFI/eu1WJ1WFs6jMn0lFwUxmFM8hx1swxdGKNyo22sfwjufoffH3zPj4/VizilBMlqkOS5wkSd1gyB+joz9M50CEwdEoiaRBPGlg0hR8LjMV+U72tAZ4aXs/rb0hVEXBYlJZOSeTa87Jp7rUhc0iK+uEEEIIIcSZKdFVP9UhvOOFwmF+/aeH2bl7L0PDo1gsZubPnsnCOTWcs2QhXo97qkMUQhzBVFgz1SG87UmJPyGEOMXCsRDNXXt4aecTRGLh8eQUQDQepqGjluvOuZ2ZJfOxWexTGKl4syzeXBxFM1EtVkb3ridzwWWYvblTHZY4STRVIcdnJcdnZeF0CITi7O8K0dgRpL0/TO9wlCF/jPrWIEOB+Piqx1hCZ83OAWwWDUWBqmIXDqskqYQQQgghhBBvnMNu51N33T7VYQghxEklCSohhDilDNr79rO5YS09Q+0oSnqZL13XCcb8dPQfoCSnQhJUZzBrVhHZy9/N4KYncBbPwuzNAaQ849nI7TAzv8rL/CovsbhOe3+YV+uGaOsLMxZOpD3qhgHPb+lDNwysFo3KAidmkzwvhBBCCCGEEEIIIWRDBCGEOIUisTB1LZt5bc8LE5JTrzNpZp7b+ijtfftPc3TiZDJ7c8lacjWh9r2EuhpJhPxTHZI4DSxmlYJMG9OKnNitk7/GE7rBC1v7eeiFDnqGIqc5QiGEEEIIIYQQQoi3J0lQCSHEKfTqnhfY2vAKiWTs6J0U6Pd3s735Vdr7JUl1plJUBdXmIvf8mxhr3cnYgZ1THZI4TcaiSV6rGyIa04/aR9cN6g74+fVTLXT2h4knjt5XCCGEEEIIIYQQ4p1ASvwJIcQpMhIcZMjfh83qoKZ0ATarA4tmRVU1zCYLFpMVi8mC2WTFarZRlleFw+qc6rDFm6agWqxkL7+ejse/R6hzH46yOZhdmVMdmDiFdAMCYwm27hshFE0CoChg0lQcVg2nTcNp13DaTGR5LBRk2YjGkxjGcQYWQgghhBBCCCGEOMtJgkoIIU4RRVGoKVtIcU4FFrMVVVExaWZsFgdmzYLZZMakHfrPYrJi1sxTHbZ4CxTVhL14JpbsEqKDnYS7GjHPWD7VYYlTKJHUicaTZHktVBY5sFs0rGYVp82Ey2HC4zThcZhwO0zYzBoepwmvy4ymyT5UQgghhBBCCCGEeGeTBJUQQpwiXmcmXqesnnmnURQF39wLGdryFIF9G3BVLEA1W6c6LHGKGAZ4nWZuuqiIDLcZn8uM12XGbtVQFUlCCSGEEEIIIYQQQhyNJKiEEEKIk8w7cyWjdWsIHthJdKAde0HVVIckThGrWaUw20Zhtm2qQxFCCCGEEEK8Q4XCYf74yN/YvL2WW2+4ltUrl011SGekUX+Ah598huxMH1dedD4Wi2WqQxLirKcYhuyCIIQQQpxswzufY3j7c5jcWZTe8IWpDkcIIYQQQggh3rREV/1UhyCO4XNf+yY/+fUfxo/XPfEgKxbPn8KIzjyDQyPMWHkZ/kAQgKsvvYDHfvuzKY5KnOlMhTVTHcLbnjrVAQghhBBnI1fFQuxF1QSbNhMPDGIk41MdkhBCCCGEEEKIs9DTL6xJO37y2RenKJIz1x8ffWI8OQXw9PNr6O0fmMKIhHhnkASVEEIIcQqYPdk4imdi9uQwuOExkuHg8U8SQgghhBBCCCHeoJb2zrTjA+0dUxTJmWtO9fS0Y5fTgcNun6JohHjnkASVEEIIcYrYcsrwzj6fvrV/IjbcjaEnpzokId6UnpEI3cNhApHEVIcihBBCCCGEECfdheeu4J/efz0et4viwny+f9+XcLucUx2WEGc901QHIIQQQpytLBn5OMvnEulrJdzdjNmTg9mbM9VhCfGGvVDbQ0I3mF/mY0F5xlSHI4QQQgghhBAnlaIo/Or7/8kvvvt1FEVBUZSpDkmIdwRJUAkhhBCniKKZMHuyyb/sLoZ3PIcls0ASVOKM9PiWDsaiCaxmVRJUQgghhBBCnCUGh0ao29tA38AgBfm5lBUVUlJU8IbH8QeC1Dc2MzQ8wmggiMNuw+t2M2NaBQV5x38PbBgGa1/bPKH/wNAwm7btYk7NdEqLCiecF4lEefnVjdTMmEZZcVHabdFojK27dhOJRFi2aB4u5/FXQxmGwfrN2ygpLKC8pGjSPif7modLJBLUN+5nYHCIBXNqyPB5J/R5dfN28nOzqSwreUNjC/F2JQkqIYQQ4hQyOTPIW307Tb/8NOHOfTiKqtHs7qkOS4g3xDj4nzg9onGd2vYRXqnvozzXxarqbHI8tqkOSwghhBBCnAV6+we4+75v89L6DfT0DUy4fdHcWXzsn27mjpvec9xVRM+vfZX/+P7P2LBtJ8nk5CXtp1eW8cGbb+CzH7kDk2niR9GhcJhzrno/exqaMJlM/PDrX+bGa6/gpo99jjWvbkLXdQAqy0q485b3cs8nPoSiKOxvbWfFVe9jeGQUTdN46n//h4vPX8ljf3+e7/3812yr3UMsFgdA0zQWzZ3FJ+68lVtvuG7SOBOJBIsvfQ97GppQFIUvffqj3HfPp9P6nOxrvm577R4++6/fYMuuOqLR2Hh79bQKLjl/Jd/8yuex221c9v47eemVDWiaxr9+7uN85V/++ZjjCnEmkD2ohBBCiFNINVmwZhfjLJ1DuLuZsda6qQ5JiDfs9belhmSpTgvdMOgaCrGpeYh93X5CMdm/TgghhBBCvHUPPfYU8y68lgf/+uSkySmAbbV7+Mj/+1eu+8A/MzwyOmmfUDjMbR//f1x584dYv3nbUZNTAI37W/nyf36fq2/76Hiy6XAbt+5kT0MTkEoS/e/Dj3PbJ+7mpVc2pPXf39rOV7/5A/734ScAeOr5l8fjSyaTPPDQo/zk13/gfR/+DBu27hxPFL1+++YdtXzwM1/ij488MWmc6zZsGY/DMAx+/rsHicfjaX1O9jUBvn//A6y69mbWb96WlpwC2Nd8gJ8+8Ecue/8H2bR9Fy+9smF87GONKcSZRFZQCSGEEKfSwW+cZS6+gr51DxFo3oKzYj6a1THFgYm3KhJP0jsSYSAQZXqBm+7hCP3+CPGkjtNqItdrozTbgXrwOTAyFqNtIISigNtupt8fIRBOkOmyMKfEh0lTCEQS9I2G6fdHCceSaKqC22aiNNtJhssyPhZAUjfoHY3QPhgiFE1gGJDhslCa5SDDaUFVU3113aB9MESfP0IomkRRwGUzU5HrxGM3o6mHxowndFr6x+jzR4jGddx2MzML3SSOeCOpGwajoTit/WPYLRqFGXbcdvP47Q3dAQLhONluK2U5qbIWB/qCBCIJzJqCSVVpHRjDpKlUF7jJ99lJ6DoDgSjtAyFCsQSKouC1m8nz2ijIsI+PnUjqDI3FaOkbIxhJoCjgsZvJ89kozjz268owDPpGo3QNh/FH4iSTBi67idIsJzkeK7phMByMsb8vSJbbSoHPjst26M/lQDhO60AIMCjMsJPpsp7w/CZ1g31dfpK6gc9pIakbdA2HCUUTeB1myrKdZLmthGNJdrWOUNc+Sp8/QkvfGFubhxgIRMl2WcnPsGM1qYyG4nQNh+j3R4kmdBwHH4dcrw2nVf7EF0IIIYQQ6R567Clu/8TdJ9z/Hy+u5aaPfY5/PPirCSupvvfzB/i/x59+Q9d/cd1r/OqPf+Ejt78/rX3Dtp3px1vTj4/U0HwAgLr6hrT2/3v86ePGZBgGd/7LlyktLuS85UvSbtu4fVfa8fDIKO1dPWml9E72NZ9+fg1f+Pp3jnk+pOZk1TU3pbXtb+0gHo9jNpuPcpYQZwZ59yqEEEKcBq5pixnc8jTR/jYiPc04y+ZOdUjiLRodi7OhcZBX9vVz9cJCtrUM0T4QYiQUw+swM6vIy3VLiinw2bGaVTqHwjy2pYNwNEG+z86+7gA9I2Hml2VQXeghGEmyo3WEzc2DNPUEGIsmUYCCDBvLq7JZNi2Loiw7qqJgAE09Adbv62dX2yhDwSjxhEFBpo2LZ+exrCqLHI+NWEJnf2+QF3f3sq/LTyCcOJhAMXHZ/HwWVWSS77WDkkpO7Wod4fm6Hhq6/MSTBjleK+dW5xAIJ7CY1NfzrSSSBh2DIR7Z2E6e18Zl8wvSElSv7O2jtm2E5dOzxxNU6/f1s7fTj/tg0mbdvn6sJpWPXjwdj91M90iYTU2DbG8ZZjAYBSDbbWVOiY8rFhSQ4bRgUlW6hsO82jDA2vo+QtEkBpDptHDB7FyKMhwcrQqJYUBL/xiv7htge8sww2MxErpOhtPC4spMVs7IIdttpXVgjF+/tJ+aIg9XzC9gVvGhuu8N3QH+trUTu1Xj6oWFuGzmE5hfG4qiEE/qPLWtC384TlGmg0RSp659hP5AjFyPldWzcjl3Zg6JpMELdT3s6RglGEnQ3BskEu8iz2djaWUmF8/JIxpX2N4yxJo9fbT0h0joOnazxvKqLC6Zm48zR/7EF0IIIYQQhwTHxrjn37+d1qaqKvd84kPccdN7cDkdrHl1E5/7t2/R239oZdWL617j5799kI9/8Ja0c49cveOw2/nh17/MVZesxufx0NXbx49+8Tt++sAf0/o9/OQzExJUkUj0qHFnZfjI8HlpbmnDMAwcdju3vvfY5fIAVi5ZyBUXn8/g0DD/84f/S7uGruv89qFHJySLJlvdFY8njnutN3vNeDzO3f/+XxPG8Xk9fOqu25g1o4qmljZ+9b9/prWja0K/ZDKJPzBGVqbvhGMU4u1I3r0KIYQQp4FqtuKbfR4jtS8zvPN5SVCdBUKxBG0DY7y8p5eNjQPEEvr4Sp+GrgAbG4cYiya49dxySrOdjIRibGkeon1wjHjSwGHVAMhyWQlFk2zdP8jPn2ukuTdItttKrs/GcDDG7o5R1tb3c9PKMj5y8TQcVhOJpM4vnm9mTX0fZk0hz2vDpKms2dPHYCCK2aRyydx8QtEE9z5cy/7eIBkuCyWZDuJJnefrenitcYC7r6vh2kVFmDWVwUCU+x6ppXMojMOiYbdqNPT4eWZHN5qqUpJ1aBVTUjfo90ep6xhlNBRnaVVW2tw0dAfYsn+IPN+hcw70jfHKvgFGxmIYpFY9haIJRkIx2gdDPLW9iwde3o/XYaamyEM4luTVhgFebRhkKBDllnPL8TktrK3v5+fPNhJPGqysziIUS7K9dRhNU7hifgFWszbhsUqVJjT4zUv7eb62B5OmUpXnwmMzs/FgkrFrOMwdq1jqv0gAACAASURBVCuxmTVq20Zo7R9jWp4rLUG1oXGAVxsGqC50k0gaJzy/NrNGUjfY0+mnuSdAIJJAVRQcVg2XzcSGxiC7O/yEY0lWzsimazjMYDBGImkwGo5jDI0RT+pU5blI6AZNXX7+/Fobm5uHqMh1UpHjor5zlI1Ng5TmOMeTgkIIIYQQQgB868e/oLu3P73tq/+Pz370jvHj973rKhbNm838i65LK1X3swf+mJag6unrp7mlbfzY6XDw9J9+ycqlC8fbKkqL+eF/fIWX1m8cL5sHsGVH7QnHfN89n+YLn/wwmqbhDwTZurOOBXNqyPB5j3neAz/6Jre9913jx6vPWcZ77vxkWp9XN28/4ThOxJu55oN/fYqG5pa0tuLCfJ7/y2+ZVl463vahW2/kuts/xuY3MHdCnEkkQSWEEEKcJq7pywh3NeJv2ERstB+zKwNFk1/FZ6qkbmDSFDRVIanDPdfVsLQqC1VRWFffx4/+3sBztT1cODuP0mwnugGqqhBPGswq9nDH6krmlfkwqyqKYvC7tQcYDES5flkJ711eQr7PjqYq/PczDby6b4DdHaOs3zfAhbPzeHJrFy0DY1TmuXjXkiIunVuASVPYemCIYCRBrsfGQCDKi3W99I5GWF6VxfXLSphf5mM0FOf5um4e3tDBpsZBKrKdlGQ7ea62h6FgjLklXm48p5RFFZn0jIT572caqe8YRVUVdD21CVUq32NgUhUsJpWknr45laYqmLUjtzpVUEiVIVxSmcnd19UQjelkuCz8fUcXa+r7KPDZ+eYt8ynMsKMqsG5vP794oZk/b2jjioWFxBI6PSNhXHYTl87N5+ZV5VhMKv5wHE1VsEySnAIIx5Osre+loSdAVb6baxcXcdGcPEyawj+2d/HIpg4augJsOzDEqhk5zCj0sL83SNtAiKFAlAyXFUWBna0jROJJpuW58TnN/GNnNz2jEZZXZXP9suK0+X1kY2p+y7OdLJ2WhWGAqkDSMCjKtHPNokKuWVSE3aLxb3+pZUfrCG0DY1w8J48vXz+bf+zo4rHNncwt8/LuJcXMKPBgManYLRr7e8foG40yq9jLN26ah9WsEY3rqAqYTbLFrBBCCCGESPe3Z19KOy4pLOBTd902oV9VRRl33nwD9//uofG2fc0HaG5pG0+a5OfmcOnqVTy3Zj0AP/qPr6Qlpw63dOHctARVIDh2QmXp5sycPp6cAvC4XVx47orj3s/b3ntdWqII4JrLLiQvJzttZdioP3DcsU7Um73m9to9E8b61899PC05BZCdmcFPvvk1Vlx540mLWYi3E/lUTAgxJQyMgx9VCvHOYXZlYCuoItLfysBrj5K3+hY0u3uqwxJvkgLoBrhtZlbPymVxZSZl2U40VWEw4KUy18n+vjEGgzEi8VS5PoXUnlA3LCtlybRMctw2QrEE7QMhOofD5PvsLCjzMaPQg/VgouHy+QUc6AvSMxKmpX8M3TCo7/QzGopx7swclk/PItdrBWDptCxiiSQOi4ne0Qi1bSOMRRKUZDtw2jQC4TixRJJpuannXc9oag+tTJeVvZ1+QtEky6dnM680gwKfHY/dzOevmckX/rQjtQpJOXTfUVKJKt1gQlk94+B/hzcrB/tPz3Nz86pycj02dD21QqjPH8UfilOR68QwUiuTNFXBoqnkeqxs7h9jKBDFZTVhNqkEwnFq20dY2T9GTbGHkizHobgmoesGu1pHGA7GmFvqJddjJRCOoyiQ67XhsGgMBKJ0D4exWzWWTsukbyRC28AYLf1j2KwmuoZS+z0VZdopy3aQSBrUto0QiiQoybZPmF/DSM1vvz+9ZImmKpw3M4erFhamEnGqwuwSLwcO7qkVjiUpynTgtJnRNAWbWSPDaSHTZRkfw2pWMWkKg4EoL+/p48LZeWS5UyUQD6YPhRBCCCGEAFKl4BoPtKa1rViygGRSJ5mMTei/ZP5c4KG0tpa2jrTEydN/+iWbd9SS4fVQVVE23t7a0cnGrTtZt3ELL6x7jcb96dcFTqgs3RUXnj+enHoj7rp1YhJHURSqqyrSkkXBsdAbHvtkX3Pfwb20Xme1Wrj9iETX6xbPm82qpYtYv3nbSYhYiLcXSVAJIU679v79bG18BY8jg2XVq3HZPYxFAowEB0noCUpyKlGVs/cb4IP+PsYiAexWBznegqkOR5xmzrK5RIc6GVj/MNnL34VqscsqqjOUqioYhoHZpDK9wE2W24rlYFIpw2WhMNNOU28QfzhONK6PbyysqgrzynzkuG0oSmo/p67hMLG4Tp7XRnGWYzw5BVBT5CHDaWF/7xjDY6k3kJ1DoVQiI8NOUaZjvK/PYQZS30ZsG0jS2j+GbsCeDj+DgRjawZiTBgQiCVyRBNGETiyh0zkcJqkbTMtzkee1je+lNLvYi92sMRZNlaV7nYKCYaSSP9okGSrdMNISV6//M89rY16pb3wu/KE4w8EYoViSkbE4j2xsR1NTvYfHYvT7oyR0g2A0gdWsUV3ooTLXxb6uAH9Yd4DqQg/zSn3MKHRTfNhcHE5RoG0gNWct/WM8vaPrYDIHErpB10gEgFAsiVlTObc6m/V7+2nsDtDYE6Qs18n6hgFGw3HmlGQzo8BDLKGf4Pwmx2MAUBWF0mwnpdmHyvDlH9ynLBrXiSf08fkyDANdN1DV9PmtKfJQU+Rh3d5+Ht7QRlNPkJoiDwvLMyjNdmJ64+/lhRBCCCHEWaq5pZ14PJ7W9pcn/s5fnvj7CY+hqhM/o1m6YC71jc1840f3s2n7LjZvr6VvYPC4Yx1tz9jDLZxbc8KxHa6ytGTS9qyM9ISYbkzcb+rNerPXbDgiQVVaVHDMlWXVVRWSoBJnJflETAhx2rX1NfHout9QnF3B7LKFuOweOvoPsLlhLUk9wY3nfxi7xTH+Ye7Zpq5lM81d9ZTlTefC+ddO+ofe2SqRjDMWCaAqGg6bE0195/0asmQW4CyuoV9PEjywHc/05ZjcmVMdlngLFFKl2w6nKgpWk5ZaNaQfWtOikFpBYzGp42/MkrpBKJZAUcBm0SaUxrOZNTRVQTcMkrqBYaT2v9L1VHLMMqGUXopuGIRjSQwMgpEEZi2W9mZwfpmPokw7+T47ScMgEktgYIyvzjmRO64AOsYJr9lRFAWzKX38WEInntQxDINIIslAIHooEaZARa6TkiwHOR4bTpuJ+aU+gstKeKG2h+6RCHu7/OxqG+GaRYV451lw2yb+XFGASCx5cE50hoKx8WsoClTluch0W8b3bqop8lLgs7P1wBAN3X4WVWSwbm8fyaTOjAIPpdkOWgfGTmx+vfYJsRzJpCmphCcG+glMZkWui0vn5qMbqf2+XqjrYfuBIbqGw1y5oJDqQlmZKYQQQgghUjq6ut/S+Rk+L/Nnz0xr6+rp4wOfuoc1r256S2MfTXHhm/syr8Num7TdbD51nz282WuGwpEjxrEfpWdKdmbGGwtMiDPEO++TQSHElDMMA93Q0Y1Dn8K19TXx0o6/YTaZuWbZzdjMNhTl7PwK+Pam19jauI7lMy/k3DmXY1Un/2PmbBQIj1J3YDNOq5vpxXNwO469rP9spCgq1uxSspdfz8Brj2LNKpYE1RnMMA4lgpK6MV7WLpbQGQ3FMQywWzQshyVkjlxtpKkKLpsJA8ZLvB1ueCxGNK5jNau4rBoo4LCYUFWFUDRJOJbAZUt90+71JJZJVdBUFbfdTFI3uHFFCefOzMFuOfRz9fXY7BYTe7v82C0mFBQCkQTReBKH1ZT6OW2kfm4f7vVEG0A8oRNLHIo5GtdJ6PpRy+0dmX+xWzSsplSsy6ZlcecFlbjsprTzNUXFbU/dZ4fFzvVLi7lwVi5r6/t4YmsXtW0jAFTlu8dXZx15TbfDjKYqXDArl5tWluI6mMh6/a5ZTSo2S6rNZtaoLnRT3znKgb4xattH2H5gmKIMO+U5Dtx28wnPr+0o+2IdGeDr93f8/wf/oR98jh05Zyurc5hX5qOu3c/fd3Tyyt4Bnt7ehcOqSYJKCCGEEEKMmyzZ43I6mFdTfdRzzBYzxQX5zJtVzZUXn0/mYauBRv0BVl7zfjq7eyecl5udxYrFCzhnyQLOXb6Evz3zAt/+6a/S+pzIl3SPlvQ5HpvV+qbOeyve7DVzsjLpHxwaPx4e8R+zf33j/jd1HSHe7iRBJYSYEkfuP1VZUMM1K24BwOX0oqpnZ3IKYEXNhRRll1OSU4nV/M5JTgF0DbRy/5PfoDCrjI9d8+V3ZIIKwJKRR9by6+j42w/JXHw1tvxpaNbJS5OJtzdFgVA0yct7+rhkbv74T7ahYJRdbcPoukGmy4LDevQ/uaxmlcIMO0ndYG+Xn67hcNrtW/cP0Tsaweswk+uzoQA5XhsWk0pr/xhtAyFmFXuBVGKsayiMw5pK+uR5bVhNGt0jEZIGZLomf/NkMankeKxoqkJjd4B5JT7KckyEY0nW7+0nEEmMJ6RS9zuVVNM0heFQnKHgoZIhrzX00zUU5ug7QqXLcFnw2M0Ew3H2dvrJ8dhwWI/9O0BTFbLcVq5fVkKO18ZvX95PMJKgcyg0eYLKgDxPas4Gg1H84Xhaib3JzCn1se3AMHs6RvndmijhWJIFFZnkelM/t9/I/L4ZiqKgoBCOJQlHk5P2cdnMrJiexYrpWXzxTztZv69/wp5XQgghhBDina2yrBiz2ZxW5u+8FUt44vf3v6nxvvXjX0xITq1auoj/+trdLF80P639D395bML5J5agOvZqoqOZiko8b/aa+Xk57GloGj/u6O5heGSUDJ930v6btu96U9cR4u1OElRCiFMqqScIhv30DnfhsDrJ9uYddqsx/s31LE8uc8oXYxg6psPKviWScQJhPwMj3ST0BE6bm0x3Di67Z5JrJQlFg/SPdJNIxrFbHFgtdlRFRVVVMlw5JPUkgdAIST2Jx+lDRaVrqBXd0PE4MvA6M9EmSY4l9SRjkQADoz0YhoHH4cPrysRiSv8wUjd0IrEw/SPdRGJjOGxuMlzZafGW5FQdbEv/o+P1fbjGIgHMJgsFmSVYzLbj7sdlGAbxZIxBfx+hSBCr2UamJweH1TU+h8GIn1g8itfpw2K2jycIDUMnGo8wMNqL15mJw+pE00zohk704P2IJ2O47B68zixslvQ/EiOxEGORIKqq4nNl4R8bYSQ4gGGkHlO3wzs+f9FYmOHgALqRJJaIMhTox+fvxWKyjieqEnqCIX8f/tAIGAY+VzZZnhyUs2xPMkU1odmc5JxzA8GWnVizCnFPXzbVYYk36uCql2giyf7eIN/9217mlqSe8ztah0nqsKo6mzzvYYloBZJHrIaxmFRyPTYWV2RQ3+nn0U3ttA+GKMlyEIkn+du2TrpHwqyelcvy6dmoqsJFs3Opax9hY9MgsaTO8qosrCaV9Q0DOCwaF8/JZ0F5BlcuLOTVhgGe3dnNaCjGkmlZuG0mApE4tW2jVOa6WDE9mwynhXNn5rB2bz/P1/YQjiWozHXROxrl+doeBoMx8ry28Z/ZJk2hIteJ126mrn2Up7Z1EgjHCcWSvFTXQ+tACJtF5fC7+vrqMo64/w6LiekFbqblu2kfDPGNv9axsCKTbLeVYCRB2+AYbQNjfPbqmQwFY6zd08dgMMriykzcNjMv1vbSNpCar7S5PozZpHLFggJ2tY+wqWmQaDzJ6ll55PtsDAWj7O8dw2UzsbwqazzZV1OYKuW3o3WY7pHU/lxLKjLI9aSukeW2nuD8ZqWSYYfVeTxaFT+FVJk/TVXIdlkxm1QauwP8bWsnXcMh3HYziyoyeWRjO009AWYWeSnPcTLgj9DUG8Bu0cg6iUkyIYQQQghx5jOZTFSVl1Lf2DzetmHrTgzDOGpyJTg2xt33fZutO+u4/X3v5lN33T5+29oNm9P6aprGg//zAwryciaMs3Hbzglt/kAQj9t1zJi1o5QxP5ucv2IJL657bfxY13W+9/Pf8B9f+uyEvg/+9Ul6+wdOZ3hCnDaSoBJCnDJ9I13Ut+1gZ/MGxqIBNNVEUVYZw8EBXv9mvUFqk8jOgRY2N6whnohRmluFZjHR0tPArv0bae1rwh8aRtd1zCYrRdllzK1YyoJp54xfa9Dfx/7uvWxpXMtwIPVL22yyoCqpPWDsFie3Xvwp/KEhtja8wlCgj5KcabT0NjAU6COpJ/E5s6gpXcDymosOlhhUSSTj9Ax1sKP5NZq69hCJhQAFi9lKUVYZC6pWMq2gBpNmIhwL0drbyIb6F+kZaieRjGPSLFSXzGNe5TKmFaQ2+dzbvoOmrt2U51dTlF0OQF3LVmoPbKK9r5lYIoZJM5HpyeGqZTdRnF1x1DkORYJ0DBxg/e5nGQ4MEImHMakmsr35zK9cQXXJPCKxEFsa19HS08DMkvmsmn0ZZpMFgJHgIA0dtTy37TEunH8NNWULURWV5u56tjSsJRAaJZaIYjFZKc6uYF7lMmaVLRq/flNXPdub1pPUkxRll9PSs49Bfx+aaiLLk8uKWRdTmV/NcGCAupYtvLL7WXTDYHRsiL9v/jNOm5vq4nlcuOBaFBSe3/4YbX1NjI4NA+Bx+Fg15zKqCmfjOZtWWykKqsVG9qr30v2P+xlr24OjdI6sojrDJHQDVVUwayr5PhtNPQF6RsKED+51VF3o5sZzSsn3pRIahpFKPKiKklYyT1UUbBaNW84t55EN7TT2BPj7ji7cdjOJpE4sobN8ejaXzy+g0GdHUxTmlfm4aHYez9f2UNs2QsdgCEiVCFxZnY3bbsJpNTGnxMt1S4rY0DDApqZB6jv92CwakYMlCTOdFnTDwGUzsbgik4tn57GtZYh1e/upbRtFVRVKshzohoHbZhovNacqCl67hVXVOQyNxdjX5affHyWhG2S7LWS5LJgmeVNp1pQJb4IVBWYXe7lqYQFPb+ti64FhGnuC2CwaiaROIplahabrqfkbHouxft8Ade2jWM0aPcNhMpxmlk/PoiJ38je6Zk1hRqGHqxYU8o+dXdR3plaquW1mookk8aTB/FJfWim9DKeF0mwneV47PSNhCjPsVOa5cNtTfz6/kfkF0El9u1JT1QklE3XDwKQpmDQFAzBrKrNKvJRkOahrH2FNfR9tgyFWVGVRU+QlltBp7A5Q1z6Kz2lJlYaMJlgyLZOlVVIyVAghhBBCpLvsglVpCarhkVG++s0f8J9f/tyEvj19/Vx/xyfYsrMOgN0NTXz41vdhs1kP3p6eKDGZNOy2iV+S+sNfHmfn7r0T2vc17ae4MP+Y8ZpMZ29Vndddd/nF3Pudn6S1/fCXv2PGtAo+8L53j7c9+/Ir3PXZr5zu8IQ4bSRBJYQ4JcKxEPVtO3h26yN09B+gJKeSRDJO30gXQ/4+jvz+eP9oNzuaNxKNh7n5wo8DsK9jF1ubXiEUGSPTnYOiqXQM7E8lMIJDVObX4LS7URSFps7dPL/9MVr7GqnIS9VRTiVL+rFbHdSULgDDwD82TGNnHXvbd5LtrWUkOEiWO5dAeIQD3Q30DneQ4ytgeuEcLGYrPUMdrN/9LK/Vv0A4OkZ+ZgmqkoqjoaOWnuEO3n/BR8ny5NI71MH6umfZtG8NeRlFuGxuAuFRugZbKcwqG09Q7e+uZ1vjenRdx1hkkNQTvLzzSfZ316MqGrkZhcTiEfa0bmfV7Mspyq6YtFBWPBmjc7CFJzc+SGNnLZnuXFx2DyNjQxzoaeBAzz4+cOlnsJntDIz0sHnfGgb8vSyZcT4mzYyiKAwF+tm472U21L/AkhnnEYmF6Bxo4bltf6Wlt4HSnGmYNDNdA6209zUzMNpDYXYZTpsbs2ZhYLSb7U2vMTo2hMfhw2yyoqkqoegY+zp2MeDv4fZLPkM8GScUHSMUCcDBFV/BsJ/EwfZwLERDxy7W7HoaVVHxuTJRFY2e4U7a+/ZTkl0JZ1OCClBNVlzl8zB7sokOtBPu3IercuFUhyXeBLtFY1lVFrpuEE2kEirZbis1RR5WTM8e35co32fj4jn5zCmJ4LGb08awmjSWVGaiorCrbZjOoTDRhI7VpFKUaWfFjGym5bqwmFJJH5/DwhXzC8j12tjTMUoomkA3oCjTzsoZ2ZTnODFpCl6HmfedU0pZtpOmngD+cIKkbmAxKeT77CyuzCTDacZiSiXZ3r+yjIo8Fx2DY0CqZN3yqix2t49iANPyUgkgRUklW86vycFlM7GrbYRoPInTamJRZSaDgVRJvNLsQ0nXheUZOK0a0/Mn7o+Un2Hnkjn5eO1m9nT68YfixHUDi6am5rLYjd1iwqyprKrOQVEUBgJRDAPKsh3UFHmYX5ZBhtMy6WOkKgoOi8ZFc/LI9ljZ1TrCyFiMaELHZtbIcJlZUplJge/QKlFVVVhYnkEsodM7GiHbbaUgwz6eeDvx+U3FZNYUzpuZQ02Rh6q89Dkoz3GyuiYPn9NMlsuKSVPI81q5elEhhRk2BgMxPA4zlXkuzJrKkspMFAUO9I0RiSfJ89ooyrRzzvTsCWMLIYQQQgjxtc9/kv97/Om05NK3f/orTCYTH7r1RkqKCojH4zz416f40n9+j76BwfF+xQX548kpgJVLF9LW2TV+HI3G+MLXv8PXPv9JigryaGnv5Pd/foyvf/+nk8ayu6GJi89feQru5Zllbs0Mrrv8Yp545oXxtmg0xl2f/TI//tXvKS0qpL6xmaYDrVMYpRCnniSohBCnRM9QO7tbttDRv5/inEquXPY+knqCupathCJBYonUHhkKqQ/6VEXFMPS0JeYWk5WCzFIqCmZSXTwXXdd5dfdzbNz3Mq19TRzo2cvM0gUYhsHe9p00de1hWkENt1z0CRLJOC9sf4xN+9bgc2Vx+eL34rS70dRU+bqxSACbxc6SGecxu2wxPUPtbG5YS89wB1sb1lGZP5NkUmN361Y27H2JSCzEeXOvpKZ0AaqisLt1G5v2vszmfWuYU76YxTPOp2PgAE1duwG4evlNFGaVMeDvJRwdI8uTe9jsKOi6DkaqBOJIcJDGzlqSepKL5l/H8poLCYb91B7YRKY7G+VQYaw0o8Eh9rRuY9O+NcwomsMli95NWW4VHf0HeHnXU2yof5GFVStZMeticjMK0VSN1t4GRoKDWM02zCYLw8EBmrvqcdhcFOdUEomF2H5wtdiMojlctfwmPHYfu1u3sX73s9S1bmHn/o0snLYSs8OCqqjEE1GCET8+VxbLZq6mKKuM5u561u56mtf2vMAVS95Hef4MllavBgz+9NLPcNrcrJ53FSW508hwZWMYOhvqX2Qo0MfKWZdy3twrsJpttPU347S6J5QWPCsoCopmwjd7NSN1L+GvX4+zbA6KZj7+ueJtQVMVkrqBWVMpyXJw6bx8LFpq1abLlkqmHG5anovyHCcGYDqi7rqigM2ssbI6m3NmZBGOJRmLJvDYU8mjyUpvVOa5qMxzEVlUSDCcQFEVfA5z2l5RZk2lLNtJWbaTWEInFE0QS+g4rSbsVg31sHFNmsr8Mh/zSn2EY6mEl8OioaoKS6ZlgZFaMXa4ilwX5TkurlpYSCiawOe0oKoKum5MKBlyxYICLp+fP+l9MakKBRl2rltSzDWLDUZDceLJVPLIZTOlxXnuzByWVWXhD8dJ6gYeuxmbefI5OlJhhp3Cg8mwUCxBMJLAZTXhsJowaRPPn1vqY1axh0TSwGxS0+J4o/NrM2vcdm45BnBk2f15pT5mFXlRFMYTYKqicOncfM6tziESS6CqCl5HKtm1oDyDuQcfp0AkidOq4bSa0h57IYQQQgjxzmExmwknD+1beuRWAR63i+/f9yVu+efPp7V/40f3840f3U9BXg7Do34ikYn7mX71sx9PO77sgnN56LGn0tp+8+AjPPDQozjsdsZCoWPG+vL6TXz6Qx84FOskf8Meb6uD1HkT+xztLcGR4002/mTjHRnbyb7m9//9i7yyaQtDw6Np7Tt375109ZkQZyNJUAkhTonm7r209DaS7S3ghvPuZG7FMsyamTnlS3ne81cef/UPqX2QXv9F/vpv9MMWVl244FouXHBt2ri6kWTA38uBnr0MjPag60n84VFGQ0O47R4WTl9JWV4VACPBAToHWoglYmR58lCV1AeYuqHjtntZMG0lH7rybgBi8SjxZJynNz1E52Dq2ynBiJ/93fX0DnewYNo53HbxJ8fjKMquwGKy8vC6X7Np7xqmF80lkYwTi0exmK3k+orIyyiiJKdywtwoSmqPEYNUya9gNIBuGFjNdnzuLIqyywCF6pJ5x5zjrsFWag9sxqKZec+5dzCjeC4Oq4vcjCLMJisb6l+ksaOOuRXLKMgsozRvOntat7G3fScuuwebxU7vcCeD/j6qCmeT5yuk9sBmmjp3k+st4H2rP0xJbhVmzUyWJ49EMsajr/yWnU0bmVW6EPC9vowCm8XOBy75NNUl87FZ7BRkltI30kX3zicZDPRSnj+DwqxSphfNQVU0PHYfNaULqSyYCaRW0IWjIXRdx+PMIM9XRIY7m4r86hN7wv1/9u47yqrrPvj+95Tb6/TeG0PvIIoEAgOSEOqyZQkc23ESJ068Yqe9b/ws208SW0+e+HUSO07iqNhySWxZsmxJSBZCSAgQIIpoM0zvvd07t7dz3j+GuWKYAYQ0CAb2Zy0tmHv33Weffa9mhv3bv9+ewVxzbsVb8za+xqOkjQ5iSsm51kMSPqALv22pskyqfeoMnrH20pRBkKnaWc8FTT4Is0HBbLh8CQyjKmNULz6+96/PpGurssRUgfLx9hajkswUA84FSia2n+qxqciSdNFMqHFGdSyz6sNSFQmnxTApk20qiizzQUrgf5D5VS7y/kuShEGd+rkL5/b9cUnYzQbsZhHUFgRBEARBuNktmlvNgSPHk18vWzRvUpuHtt1BJBrlq994fFJApKdvYFJ7RVH49t9+le0P3TPh8U/deye/2/M2v/jNzgmP67o+ITiVluLmG3/56AHdWwAAIABJREFUp/z4F79OlgsEaGnvmPC6pQsmjjUnK+OyJQBh7B6f+Nmzya8XzJmF0Tj17+PLFs2bEFRbumDu5DYXPJaW4qa0qOCqXrMoP483f/1T7vz0F+js7p2yH4B//vu/pbG5je8/9dMJj98MZ3UJNz4RoBIE4aoIhEYJhP2kOjKoyJ2Dcm6nSKojg6yUvHNBmkvTdZ2h0T7qu07TPdROMDzKsG+A5p46QCehje0OshgsGBUT/rCP7sHW5Gu7htoY8PaS5szEaXW/v7te17FbXBRlVSSvZTSYsJpsyJJMOBoCJDz+IYJhPyn29GR5vnFuexr56SVoWoIR/yCxeJT8jFIKsyo4ULOLx//nKywoW8GyylupzJ+Py5Yy1R1iUA0UpJeQ4cqhuaeWX739JKda3mVp5a0sLl+N2Wi5aFZAMBLAG/QQiYV56eDPsZjsKLKCpiXwBkZQFJXRkIdoPEJ2ah6VeXM52XyIE82HmFUwn6HRPpp6arGZHayYtQ6z0cJocIRA2EcoGuTZt57AoJqQZZlEIk6fp4vEufvVNC05DlmSMBssFGVVYDKMLRgbVCNuexroEImGSWhxADRdO3fnOpr+/u4ui9HKsqrbaB9o4tV3n+Vs+3ssKl/F0sq15KQWfqDMiJlKNppwVK1Ar9nH0MEXyL3ji9d6SMIV0HQdTdPHolSX+6YmCIIgCIIgCIJwFTz20D0cPHYCTdNwOR1s27xh6nYP3sPm9Wv5y2/+Izt3v8WIxzupjcFg4OFtd/C/vvLHlBUXTvn8T/7t/5Kfm83T//PcpGBXemoKD2zdzP/+qz8jNcXNHRtu5dZ7HqW7tx+AP9j+qQntN6y9hbycLLp6+gD4zMP3faA1gLs2riM9NYXB4bEzrLc/dO9F296zZQP/8M//zvCIF1mWefTBbZPa3HrLMkoK82lp7wRgx8P3oigTN4pN9zUBqivKOLP3ZZ78+a/45W920tjSzqjfz9L5c1m9fDF3bLyNtSuW8gdf/V8TXmcxm3G7nBe9viDMFCJAJQjCVRGNhYnFI6iKis3sQDovlVlVDGPZU+gX3Uuv6zrvNb3DvtO/o3uoHU3XcFrdRGNhIrEQJoP5XA4SWEw2nDY3iUSM95oO8p8v/QNxLUFLz1g6dHF2JXara0L/iqxguGCnuywryVKDoBOOhYgmoqiKAavJNqGtQTFiMlrQgUQijo5Gfnopty+6G4fFwbHGA5xoOkRLTx3zSpazavYGyvPmTHGnEoqsct/qz3C0YR9nWo9wuvUonQMtHG/Yz50rP0VRZgWKPHn3fCwRIxILoSoGDKoJCZKBI7vFwfyS5cwqWIDLmkKKPYPirEpMqpm6jhN4AyMMeHvo6G/GZDCztPJWzEYrkViYaDyChITJaEaRVeKJOJIkkebIJNWRSWnOrAtK7klIyGNjOPc+S5KMqhjRdQ3tXOnGyW/y+3+1GG0sq7yVcDTIieaDdA+1s+vo85xqeZfbF25jTvESnDfYGVRJkoyjfCnRoS48p98ia/0OZIMJSRE/oq93GU4TG+dlU5JpZ0GRG7tFvGeCIAiCIAiCIHz8vvDYw3zittUcee8Ud268Davl4mXyM9JS+dG/Pg5A/+AQtQ3NtHV0kZWRRkVpMUX5uZMCMxeSJInHv/YXPP61v6C+qZWjJ0+T4nKyYE41OVkZE9oW5edRt/93vLRrD4vnz5mUlWQ2m2h45zV27n6L8pIi5lRV8EFkZaTT8u4bvPz6myyaN3tSvxeOofXdPbz8+pssXTiP4oK8SW0MBgO1+17hld17KSrIY1515VW/5ut7D/D8y69RVV7Kn3z20/zp57dftL+a+oYL+s+9aFtBmEnESoogCFeFoqgoskJCi+MP+7CbHUjS2C84YwGdi6caSJLEaHCE0y1HqOs4gcPqZu28zdjNTjz+IY41HsDjf/9gT39olGAkgEE1YjPb6RxsQ5Zl0pyZLMwsY3H5movUL544hrGSg+ef3WJEkVVi8SiBsH9C22g8TDDiR0LCYrIiSwpmo4XKvLnYzU6Ksipp6qnlvcZ3eLfuTewW50UCVGNmFy3GZnFQlFVOQ+dpTrYc5tDZPZTnzSHVnjGWjXQBVVYxKkbiWpy5xUtIcWRgUN4PukkSuGxppDjSURWVdFc2RVmVNHSdoqmnlu6hdvzhUQrSS8lLLx67Z9WIqhgwm83MK1mO2Wg5F/ySzp2NMpYZZTZaP9h8TtVEHwuknd9aUVRSHOmsnvMJctMKae45y9mOE5xqeRdFVshNL7pxA1SA0Z2NJbcC79l3GD72Kinz16PaU6/1sITLsJlUZue5KM2yYzOpk86VEgRBEARBEARB+LgUF+RNGQS5lMz0NDLT0+CWZR/6upVlxVSWFV+yjdls4sG7t1z0eYPBwD1bNl7xtc1mEw9s3fyB2los5kuOAcbKGm7dtP5juebZxmbueOT3k1/vO3SEX/7Xv0yZPdbbP8DRkzUTHiu6wvdaEK5XIkAlCMJV4bC4cNpSCIR91LQfY0HJClTFQJ+ni9a+93d9jCfW6LqeDIIAjPgGGRjtIZqIUZ43h60rPk1CS3Cq5TD1nafwBobR9bGTnIZ8/Yz4h1Bkldy0Yty2NHQ00pxZVOTNpTir8rzr6UjnzqK6MKtHR58QVHFa3TitKURiIdr6G/CHRsdK7iHRM9RBU3ctiqKSn1GKyWDGHxolEguTl15MSXYV1YULGQ2McKr1XToGmy86hmDETyQWJjetiOKsSkqyqwhHQ+w/8xoD3l5Gg54pA1RWs500Zxb9nm4sJhuzChaQ4khHQhoLDIZGMaomjAZz8j1ZWLaS5t5ajjceYNjXj6oYqDzvrCuXLRW3LY1oPILLlsLsosVYjDYkSSIajxCJhTEqxmT22YXv2/v3qCFJ8rlA5HlzLY0dKhqMBIjGI2ha4txZXBoD3h6c1hQWla+mMn8e6a5s6jpO0tRTS/CCAOENR5IwZ5XinLWKvj0/xl48H8XqQpoic064fkiShNmoYJ7ifCBBEARBEARBEARBuJij553LBfDCK6/z2S//P3z/2/8Lu+39Kj7dvf3c8cjvE4/HJ7S/fc3Kj2WcgnC1iQCVIAhXRXZqAVkp+dS0HWPPey+Sl1qEpmu8fepVDtS8jnxu4X08k0rXNcZiQ9K5wJOGLMlIQCgSwBcaxR/ycqL5EM29ZzEbrGOv1UkGP4IRPz1D7bjtqRgVE5FYhK7BVhRZIS+9GJvZcS44JI0FTy4MUOnjwZSxcbntaaS7slBVIw1dZzh0dg9VBQtQZJn3mg9yvPEATqubucVLcVrdNHafYcDbQ1nObNJdWciSjKqomI1WTKrp/CslA2HReJj2/kZ6RzrJSyshw51NLB7DZnGgKCpG1ThleT8YO8+rIKOU2o73eP3YCzgsbkpzZiHLMqFIgLrOU5RkVZLhysZmceKwuphbspRX3v0FZ1qPktDizCtdTnXhwmSfuWmFZKfmc7r1CL878hzpzmycNjeSJOPxD9Lv6SE3tZDMlFxMBsuk9+39O9STu34k3n+PFFnFanLg8Q8y4OnGbUvFYrQCEruO/ZpFZavIcOegKiqqYkBVDVhNduSbIDPFlJaPo2IZ7b/830SGOjG4MlBtN27WmCAIgiAIgiAIgiDcrFYvX4LBYCAWiyUf+9lzv+W3v9vNyiULUVWFmrpG2jq7J722tKiAL33u0Y9zuIJw1YgAlSAIV0VV/jza+xs503qEo/X7ONXyLrquocgKimLAqJpIaPFkoEaSZOKJOPFEHJDISS0kzZmFP+Tj7VOv8l7TIULRAIlEjFgiRnZKfrIkX1FmOdkpeRyJRWjsqaGtv4FYIjaWniWN9bVh0TbuXvkokiQRT8SIxaOT0qYlIKHF0PQEEhKqYmBJxRqGRvt57ehz/PuLf49y7lygeCKG0+pmaeWtLK5Yg9Vko67zFC8d/DmaliDDnYs3MIwv6KEsp5rqgveDQLquE09E0TSNWDxGx0ALz+97mkBoFJvZgYbG8Gg/ZqONkuxKMtw5U85xfnoJSyrXcqTxbRq7a/jOr/4ah9WNIqv4gx5GQ14e2/Albpt/FzaLE7PRSmFGGWmubIKRAJqukerIpDJ/brLPWQUL6RvporH7DMca93O69QguWxqJRAx/aBS71ckn1/0hTlsKJoNlwvt2/nzKkkI8ESOhJUjoiWSAyWKyUZhZRk27l/98+VvEE3FWVm/g7pWP8tqR53jl8C9xWN3Ikowv6CGWiLF+/t24bTd+uTtJNWBKyyXz1k/jObkHgzNdBKgEQRAEQRAEQRAE4QZUXJDHDx7/On/2t39PKBxOPu7zB9j11v5LvvY73/gbjEbjJdsIwkwhAlSCIFwVBtXEsqp1OK0pnO14D3/Ih9looTi7ErPRQtdgGwbFgNU0lracnZrPLdW3E0vEUBUVg2pk3YK7SXdm09B1mtGQh1R7BiU5swhFAvhDXgoyy1AVlTOtR2nprSMrJZ81czezuGI1sXgEj3+Iw2ff4r3mdzhY+wablz5Eij2NecXLCIR95KUVTRhzfkYJa+ZuHjuzSTUAkJtWxJalD1KcVUlTdw2+kAejasJtT6Moq5I5RYsxG8YOH11RtQ6zwUJrXwPBiI/89GIy3XlUFy6gMn9e8jpVBQtQFQNFWRVYTFaWVq4lFAnQOdiCP+RFVYw4bW4WlKyksmDehHOlzidJEsVZFXzp7q/zXtNB+j1d+MM+AKwmOxmuHFZWbyDlvPKABoOJO5c9TEPXGWRZZkHpSlTFMKHPReWrcdvTqGk7zrCvn2AkgISE0+qmMLOcJRVrsJkdU7xv7/djM9upyp/H+gVbKcooS85RljuXT677A/aeepVhXz+SJDG3eAnZKXn80db/l7MdJ/EGRkgk4titLgozSllSsYZUR+aH+yDOMKrNTfYnfp+WH/81tuJ5WHIrkM+VaBQEQRAEQRAEQRAE4cbxe5+6nw1rb+Evvvl/eP7l1y7bfs3yJXzjL/+U21Yt/xhGJwgfD0m/sMaVIAjCNNF0jWDYT+9IJ6FIAKvJTqojA0VR8QaGkYDMlDyMqglfyMuIbxBN1yjMKEWWFRJaHK9/hH5vN9F4BJvJTqozk3g8RiDsI8WRjsuWys5D/8NrR5/HZUvh0xv+hKr8sTOVvIERXj/2a148+DOsJhv/9w9+hqoYGBrtJxoLk+bKwmFxJcfrDQwz7BvAoBjJTi1ATWZLxfGHRxke7SMQ9iNLMg6rG7ctFactJfn66Lmg2LBvgGgsgkE14LC6SbGnJwM6AAOeHgJhH1aznUx3Lrqu0+/pZsQ/SDwRQ5EVrCY7Wan5mFQTknTp8nYJLU7/uT4jsRDoYDKYx67tSMeoGjlXhw9d1/EEhvH4h5BlGee58V0oGgvT5+nCHxolnoijyAoWkxWnNZU05/vBoqnet/G5GA2MMDjaR05qIXaLA0VW0XWdSCxM91AboWgQRZZJsWeQ4comFA3SN9JFIOxDkmQsRituexqpjowpDwm9IZ37kdz01FdQ7SmkLbsbe+miazwoQRAEQRAEQRBudvHu2ms9BEG4oTW3dVBT18jZxmbqmlroGxgkNyuTwvxcivJzmVVRxpL5c671MIUrpOZWX+shXPdEgEoQhBlv97Hf8MqRXxJPxFg3/y4Ks8rRtAS+oJejDfuo7zzNrIIF/PG2r2E2Wq/1cAXhskbee42hwy9hK55H1u2fQVZF6r4gCIIgCIIgCNeOCFAJgiBcORGgujxR4k8QhBmvOLuSqvz5nGg+yK5jL5DhzkKWFEaDHsLREGW51axbuBX1IqXyBOF645y1Cs+pNwn1NBIZ6MCSU3athyQIgiAIgiAIgiAIgiAI00oEqARBmPHKcqtJcaQzp2gxZ1qPMujrx6gayU0rojx3DrOLFpLpzrvWwxSED0wx23FWr2b0zF48J18XASpBEARBEARBEARBEAThhiMCVIIg3BCcVjcLylYyu2gxmq4lHzeqRkwGyzUcmSB8OI6yJYS7Gxg9e4CMtY+gWhxws5zFJQiCIAiCIAiCIAiCINzwRIBKuGo0TUOSJKQbYEF1/Ki2G+FeblSqYkBVDNd6GIIwbQzONEyZxYQH2hh+90XSVz2IbDBd62Fdt4LRBAPeMIO+CIFInFhCw2JUyHJZyHCacFrE94fL0YFYXKPPG2bIF8EbjKHpOiZVIdVuJMttxmE2oCriZ+Fl6dDrDdHnDZ/7POqosoTLaiDbbcFlMWBQ5Ws9SkEQBEEQBEEQBEG4pkSAagodHR3U1NQwODhIPB6f9HwikUCSJGR5bGHBYDBQWlpKdXU1Lpfr4x7utPJ4PDQ0NNDb28uCBQvIysrCZLqyBdHu7m5effVVuru7WbhwIUuWLCEnJ+cqjfjq+81vfkNjYyPp6eksX76c6mpxuJ0gCFefpBiwlywgOtLD4MEXSFm4CcmRiiQr13po141YQqPXE6amy0trf4AeT4gRf5RITENRJEyqjM2sku0yU5xppzLHQUW241oP+7oT13QGvGFqukZp6ffTMxJiOBAlFteIazoS4LIayHCaKUizUp7toCrXgc0kfo28kCcYpaHHR123j67hIJ5gDE8gSlwbC1DZzSoZrnPzmDU2jyJ4KgiCIAiCIAiCINysxMrCFF577TV27txJfX090Wh00vPjgSlNGysjZjKZWL9+PTt27GDJkiUf61ink6Zp1NTU8OSTT9Lc3Mz27dvZtGkT+fn5H7iPRCJBbW0tf/d3f0dnZydbtmzhz//8z2dkgErTNBKJBD/84Q/Zt28fRUVFfOELXxABKkEQPjam9AKsBbMZ2PcL/G2ncJQvQbXO7I0Q08UbjNHQ6+N46wi7T/XS3B8gHE0wlgc0nuGjowMmg0JRuo2VFWncVp1JebYDl9WAIotMoJFAlNaBAEeah9lXO0BNl5doXEPTdSTenx8dHVmSyHGbWVSSypqqDOYWush2WzCJTCCicY2u4SDHW0fYd3aAg41DBCMJEpo2aR4lSSLLZWZxSQqrKtNZUpJKltuMQRHzKAiCIAiCIAiCINxcRIBqCj/96U85ePAg4XAYGAtIjQelLsZut7N48eIZHaDy+XwcOnSIp556CgC/309xcfEVBaj8fj9tbW20trYCcPz4cdra2q7GcK+6WCxGd3c3DQ0NjI6O0t7ezrFjx671sARBuIlIioo5swj3gg0MHXoBU2quCFAB4WiCo83DPHe4g71nB5D0seO5xqqwnh90GgsNxOIaDT0+Wvv9vHG6jz+/q4qlpamk2Ew39bFe4WiCE20jPPtOB/vrBtD09485ky+YmPEgS7cnTM/xbt6u7edTq4u4Y2EuRRk21Js42JfQdDqGAjx7sJ1X3uth2B9Nzt/F5rHPG+aV4z0cbhzi7iV5PLiigNxU6009j4IgCIIgXN/UXLFZVxAEQZh+IkA1hZGRkWTmlMFgwO12k5KSkjx/SJKk5JlEACkpKWzevJlVq1Zdk/FOl2AwiN/vT37t9XqJRCJX1EcikZhQFjGRSCQzzWYaXdcnZNCNZ1QJgiB8nIwpOWSseoiax+8ndcmdmLPLkNWbuyTY70728KuD7ZzpHEWGiTGpi5CksVJ23SMhHn+hhi9trmTLwhysN3GZutdO9vDf+9uo6RoF+EDBuvEm/kicJ95oYtgf5eFbCpmV67x6A72OJTQdbzDKD37XwP76QULRxKSg1FTGY6kjgSg/eqsFfzjGp1cXUy5KUAqCIAiCIAiCIAg3kZt3VeYSdF1PBqCKi4v58pe/zLJly1DVsekaD7iMZ1WpqkpKSgrp6enXZsDT5MJA0kwNLE2ny2XOCYIgXG2yasSYko17/gYCLScwpeRgK1lwrYd1TcQ1nVNtHl59r4eGXj8JTb/8i6bgDcZ4/nAnBlXmzkW5M7rU3/jvK9IVpIKFogkae338+t1OWgcCH/K6Y3++WdOP22rAalQoTLd9qL6uF1c6l7oO/nCcf9lZz6kOL+HYlW9i0XVAhz1n+sl2WTAbFPLTrFfcz/Xkw3wmBUEQBEEQBEEQhJuTCFBdhsPhoKysjIULF2I0Gi/bPhgMcuDAAerq6iguLmbu3LkUFRVNaJNIJOjo6OC1114jHo+zadMmCgsLJ/Q/MDBAbW0ttbW1DA0NEYlEcDgcFBcXM2/ePPLz87HZJi4EDQwM0NTUxP79+yktLWX16tVEo1HOnDlDXV0dHo8Hu91OeXk5c+bMoaSkJBmA6enp4dSpU9TV1SX78/v97NmzB7PZTH5+PhUVFZe9f03TJixIyLI8ZdaRpml0dXVRU1NDXV0dXq8XWZZxOp0UFhYye/bsCddramri6NGjtLS0sGnTJsrKynA6J+7Wjkaj7Nu3j/r6enJzc9m4cSMWiyU5nkAgQE9PD0eOHKGnpwefz4eqqmRkZFBUVMT8+fNJS0vDYBjLTBhfYBmfIxGsEgThmpAkFJON9FUPMrD3vwm0n8acW4lislzrkX2sNF0nEI7z/OEOznaPEowkPnR5vrim09Lv51DjEKVZdubkz9yyicFgkDNnznDq1Cluv/12cnNzMZlMF22v6zDki/DM3laa+wMEox8tM3jIF+HtswNkuszkp1qRZ3Cwr6amhsbGRpxOJ0uWLMFms6EoykXbe4JRTrZ5ONg4xLA/iv7h4qVIEgz7o7xxpo8st5lstwVVmbnzePToUbq7u0lLS2Pp0qWoqnrJeRQEQRAEQRAEQRBuXiJA9QF88J20Opqm8f3vf5/a2lqKi4vZvn37pADV4OAgb775Jt/5zndIJBKYTCbuvPNOcnJyCAaDtLW18dprr7F7927q6+vxeDxomobNZiMnJ4dFixYlSwqen7XV0tLCT37yE377299SXFzM0NAQw8PD7Nmzh4aGBjweD1arlaKiIjZs2MC2bdtYvHgxAPv27eO5557j4MGDyf68Xi+//e1vOXHiBOvXr+dv/uZvrnju9ClWawKBAKdOneL111/nrbfeoqmpCZ/PhyzL2Gw2ioqKWLx4MXfffTdLlizB4XBw5MgR/vVf/5WBgQHa2tr48pe/PClAVV9fz89//nP27dtHdXU1BQUFzJs3D1VV6evr4+jRo+zZs4fdu3czNDREMBhEURRSU1MpKSlh3bp1bNq0ifLy8gnBv/F7mOpeBEEQPg6SouKsXM7QoRcI97cRGezAmld5rYf1sQpFEzT1+XinYRBvMPaRz47yR+LUdnk51DA4owNUkUiEkydP8r3vfY9Tp06xatUq5s+fT15eHg7H5HJx/kiMxl4fe870oU3Tz7XWgQBHmoZZXZVOQdrMzaJqb2/nhRdeoK+vj82bN7Nw4UKqqqombF453+BohJeOdTEaihJLfPSs88ZePyfaPCwpSZ3RWVQNDQ3s3LmTYDDIpk2bmDt3LhUVFaSlpYlAlSAIgiAIgiAIgjCBCFBN4fxARDAYpLOzk7q6uuTixHiW0Hjgymaz4Xa7sdlsRKNR3nnnHYaGhmhvbyc/P5977rlnwiJRQ0MDr732GvX19QA0Njbi9XpJT0+nq6uLH/zgBzz77LP09/djt9txu92YTCa8Xi9Hjhzh3Xff5fTp00SjUR544AFgLIjW1tbGwYMH6ezspLOzk9HRUbxeL8PDw1gsFsLhMCMjI3R0dNDS0oLf72fevHkYDAbq6+upr69ncHAwOc5oNMrAwACBQIDMzEx0Xb9ssE6W5UmBnAszj06fPs2///u/89JLL+H1erHZbKSlpaFpGiMjI7S3t3P06FEOHz7M448/zsKFC+nq6uLYsWPEYjF++tOfctddd1FSUjIh6+z1119PZq8ZDAZaWlqYM2cOwWCQt99+myeeeILdu3cTj8fJzMzEbrcTi8Xo7Ozk7NmzvPnmm/T19fHYY4+xYMGC5L2KAJUgCNecJCEbLTirV+NvPIqv4RCWnDIk+eZZ7PUEorxTP4gvFP/Qpf0u1D4Y5FSHl2hcw6DIHznodS3Isszo6CgnT57k5MmTvPLKK9x1111s3LiRyspKMjIycDgcyZ/FA6MRjrYME4zEMajTkxkcjiXoGA5yvHVkRgeoALq6uti1axd79uzh9ttvZ+vWrSxfvpycnByysrKQZRlJkkhoOv3eMPvrBglHp6ckciSeoKXfz5lO74wOUOm6TktLCwcOHODll19m69at3HXXXSxevJjs7GzS09NFoEoQBEEQBEEQBEEARIDqsjo6Ovjnf/5nXC5XMmBxfuBCkiRWrVrFfffdx4oVKzAYDJSUlBAMBvH7/TQ1NXH69GluueWWZJ9NTU3s27cv+XVRURGZmZl4PB4OHDjA97///eRzc+bMYevWrWRlZfHKK6+wd+9eBgcH2b9/P8FgkM2bN2OxWFBVFU3TiEQiydeePHkSm83GnDlzqKys5PTp05w9e5ZwOExXVxdvvfUWjY2NlJWVcd999yHLMs899xxHjx4FwGAwsGTJEtauXcstt9zyoc4SmCqo9W//9m/s3LmT4eFhHA4Hc+fOZceOHUQiEXbu3MnevXvx+Xzs27ePp59+mq985StkZmaSlZVFW1sbPp+PEydOMH/+fAoKCpL97t69m66uLmAsaDh79mxUVaWuro7//u//5ne/+11yLOPZWb29vbz66qscPnyYYDDID37wA9LS0qiqqkKSJBGUEgThupK6cBO++ncZrdlH2vJtqNaZm/lzpXyhOEeaRohPQ6bKuHA0QZ8nTEu/n/JsB8oMjFBd+DN2fMPJj370IzZv3swXv/hFVq1alQxQDfvGytJN57lbEhKDoxFOtnnZtiR/2vr9uJ2/ySYcDrNz50527tzJ/Pnz2bFjB3/4h3+YzLD2BKO0DgbwBKMYlOkJ9ElIdI+EqOv2sXlBzrT0eS0oipI8zzUSifDcc8/x/PPPs3z5cj772c/y6KOPYrfbr/UwBUEQBEEQBEEQhOuACFBdRigUoqGhYUIW0IWZNTU1NTgcDlasWIGu69x///0MDAyNWXGXAAAgAElEQVTg9/vp7u7mrbfeSgaoenp6OHv2LD09PSiKQnFxMbNnzyY1NZW3336bp59+Onmdbdu28bnPfY61a9ciSRL33HMPX/va13jhhRfo7++nr6+PX/7yl2zbto2MjIwJWV0AdrudP/7jP+b3fu/3cLlcaJrGI488wpEjRwiHwwSDQU6fPk1BQQFlZWXce++9eDyeZIAqKyuLz3zmM2zdunXK0jZXKhAIcPr0aY4ePYrX68VsNnPrrbfyve99j5SUFABWrlzJk08+yQ9/+EMAXnnlFbZt20ZlZSVr1qyhra0NgIMHD7Jq1SoKCgqIRCJ0dXXR3NyM3+8nJSWFOXPmJINMzz77LMeOHQMgOzubz3/+8zz22GNkZ2ejaRorV67kr//6r6mvrycSiXDs2DEOHz7MypUrP/I9C4IgTCfFYsdRsRRv7X6GDr9I1rrHrvWQPjahWIK2oQDTlDw1RhrL/mkfClGR47x8++uQoihTbiDx+Xy8/PLLHDp0iEWLFvHwww+z4fb1BGMyncOhD7Xp5GIkCfzhON0joWnr81oYz446nyRJ1NfX80//9E88/fTTPPDAAzzwwP2YUgvp9YSnN9AngTcYo9cbnrY+r4WLfbZOnjzJN7/5TZ588knuu+8+7r//fsrKylBV8c8RQRAEQRAEQRCEm5X4F+FlyLKM1WqdUKJvfGeopmkoisKsWbOorBw7C8RsNrNt2zZ+85vf0NzcTG9vLwcOHADGAlpnz57lzJkzxONxzGYzd9xxB7m5uQD09/dz8uTJ5HUWL17MggULkoedG41GFi1axJEjR+jv7ycYDHL06FE2bdo05djXrl3L+vXrKS8vTwaY5s6dS2NjI729vcTjcUZGRtB1HZPJhMvlmnD2kqqqpKam4nKN7dDXNI3u7m5aWlrw+XzEYjFkWUaWZUpKSigqKrrkglc4HObYsWP4/X7i8TiFhYUsXLiQrKysZKZVeXk58+fPx2KxEAqF6Ovro7u7m4ULF7JmzRp+9rOfAXD8+HGamppYt24dPp+Pl156idHRUTRNo6ysjNWrVyfHUltbS09PT/L9ue+++8jIyMBgMCBJEvn5+axevZqWlhYikQidnZ20trYmA1Tnl/ibzgW9m01CixOIjhKKB4hrUdBBlQ1YDDasRieq/NGDoDeDhBbHH/UQigdJJGIAqLKK1ejAanCh3EQl325KkoyjYhmRoU5Gjr1K5tpPgSwjSdOTwXE9i8U1RgLRac1slSWZzq4e/r/vvcH3B08h6wlg8kaU8T8lSUpuWNE0bcLjl3vd1eo3FovR3t4+6d7i8Tg+n49AIIDH46G5uZmjR97FUrKS0ZBz2ssZxhI6J2sbuP+eb4BivOh4L3c/13J+e3t7aWxsnHBfuq4TDofp7e2lv7+fZ555hsOHD1G94hPEc5YiT/NERhOw7+ARHvifr4FiuG4+Z1fSb2dnJ83NzRPuS9d1QqEQ4XCYoaEhhoaGeOedd3jsscfYsmXLpHNFBUEQBEEQBEEQhJuDCFBdRlZWFtu2baO6ujpZL1/TxsoLjQdnCgsLmTt3LjBWFq+yspK5c+dSV1fH0NAQTU1N1NfXU1BQwJEjRzhz5gySJGG1Wtm6dSsZGRnJ855GRkaS1z58+DChUGhC9lZjYyNDQ0PA2OKTx+O56GJddXU1hYWFE7KfsrOzsVrHzjXQdZ1YLHbJM5bGH4vH4/T39/Mv//IvtLS0EAwGMRjGFk5isRi5ubl85jOfoaqq6qJzGYlEqKmpSZYhDAQCHD16lG9961sT5vTs2bMkEmOLhIlEglAohNPpZPbs2VRWVtLa2kpvby91dXX09PQQCoV4/vnn8fl8qKpKVVVVMmPN5/MxNDSUvKbX6+WXv/zlhGwzn89HfX090WgUGMuaC4Vm9i7w64WmJwjGfAwH+/BFhgnG/ETiIRLaWGBFkVXMqg2rwY7TnIbbnIHd5L7Go77+JPQEgYiHkXA/o+ERQjEf0USYhBZHQsKgmDCpFiwGOy5zOi5zOjajWOy7UZnSC7DmV+NrPILn9Fs4q1agmG/8clk6kEjo0xpY0dFRDQZS09LJTalAZvrKB35cIpEIwWDwos+Pb0LJyMggOzubhMNBQpv++9R0HdVgpGrWLDRmZqBcVVW6u7sv+ryu6zgcDvLz8klNS2fgKmwI0DQds9lCVVUVCWZm4DmRSCQ3Bl1ofLOP2+0mPz+flJQUcR6VIAiCIAiCIAjCTUwEqC4jIyODe++9l3Xr1mE0Gi/bfjzwsWrVKk6cOMHQ0BAej4dXXnmFjRs3cuLECTo6OrBarVRWVrJo0SIcDgderxe/3z+hr71793Lw4EGACTtTJUkiKyuLnJwc5s2blwxAjT83zuVyTRqz0WhMtpckacIO2gszhM4/gykejzMwMMCPfvQjhoaGJgWzZFlm0aJFFBYWXrKPwcHBZPBpaGiIt99+m8OHD0/qazxrKzU1ldzcXCwWC7m5uWzdupUnnniC0dFR6urqOHjwIKmpqbzzzjtomkZ6ejpz586lvLwcTdPweDzJwBPA6OgoTzzxxJTjd7vdGAwGZs2aRW5ubnI+zt85LM6k+uAi8TCj4UH6/O30+lrxRoZIaPEp2xpkIw5zKtn2InKcJbjM6SiyioTIWAvHg3hDA/T62+jzt+MJDVy0rUEx4Tank+0oJstehNuSjiTJYh5vMJKsYMkuw1m5kr7dT2PJLkMx2Zj2lJjrjCJL2M0qwWic6fpWrOs6mZnpPHTbMu5clItRnXkBAY/Hw1NPPcWePXsmPG4ymUhPT6ekpISlS5dyxx13sGzJYo71JPhN4ylGg7Fp/cioikRZYT7f/qN/mr5OP2Y7d+7E6/UmywnD2O8HTqeT/Px8SkpKuOOOO9iw4XbChjR2nuhHr/dN7zzKMG/2LL61/ZHp6/Rj9vOf/5yRkZHkuaAwVn3A5XJRUFBAVVUVW7ZsYd26deTn509LGWlBEARBEARBEARhZhIBqilcLMByJVauXMmePXs4dOhQ8hwISZJoa2sjFouRn5/Pli1bsNvtyLKMqqqTgkkVFRUUFBRMKiunKApms5ny8nI++9nPJs9vujBAZTQaJ71W1/XkTtXxw8DH7+/C14+3H7+my+UiKysLRVFIJBLJtpqmYTAYcDqdU+6C1c7bqW2xWJJtXC4Xubm5lJeXT5rj8WusX7+e5cuXJ3d/b9u2jR//+MfIskxdXR2vvvoqlZWVySDUggULmDNnTjK7y2QyTTjbwGKxsGbNGmBiqRpd15P3sHXrVtauXXvR0jbC5cW1GIOBLhqHTtDnb7ts+5gWZTjYey6g1caC3NtwmlIxKia4aYMrOnEtTp+vncah9xgKTr0b/XyxRISBQBcjoQEGA10syL0Vi2pHVYwiSHWDMaXn46xaQfuz/0DOlj/C4Eq/4bOojIpMboqF5n4/iWncLGA2yGS7zTM2vnfh5hKTyYTVaiU7O5uNGzeyY8cOFi9enNzoYhoYINtlxhuMTtv3BV0Hu0kl3Wmalv6uFU3TkvMkyzI2mw2bzca8efN4+OGH2b59e/J3q46hIKkOI5quo0zTh0fXwWU1kGq//Iao61kikUj+ridJEk6nE7vdzuLFi3nkkUe4//77k+WrBUEQBEEQBEEQhJubCFBN4fxgyfg5U+eX2fsgKisrqa6uJjMzk4GBAd58802i0WjybIP09HQefPDB5D/QHQ5H8lykWGys/Nn27dvZsWMHDocjOSZZlolGo/T09NDX10daWlpyEUBV1QnBIE3TJh08rSgK8fhYFksikUiWKRx/7vzX67qefM5gMFBYWMgbb7xBbW0tHo8nOR5FUaiuriY7O5tQKDRh/s4PiJnNZsrKypI7ZTMyMnj00Uf5q7/6qwlZTpIkEY1GOXXqFC6Xi7S0NACcTieLFi2iqKiIcDhMY2Mj4XCYmpqa5GtXrlzJ0qVLk2PLzMxMljSEsRKHP/jBD3C5XBgMhuTCnqZphMNhzp49S3p6Og6HIzlPlyqBKEyty9tIw+BxhkN9V/S6uBZnONTPofZXWJK3gUx7AbJ0c5b+0XSdds9Z6geOMRoZvqLXxrUo/YFO3m55gVsK78JtyUC6SefxRiUbLRhTckhdthXPqTcxONOw5ldf62FdVQ6LSlWug7bBAAlt+r4fW0wqs/NdqFf4c/56IcvyhJ/dJSUlbN++nXvuuYfKyspJv8Ok2o2UZtmp7RpluirU6eg4LCoV2Y7LN76Onf87mCRJbNmyhe3bt7NmzRocDseE36kynWaK0m0kdB1l2jYA6KTYjJRmzuxgs8FgSJZWliSJ++67j8cee4xly5Zhs9lEST9BEARBEARBEAQhSQSoLiMajdLb20tra2tyYeLC3cpGozG5O3ScJEksXLiQFStW8OKLLxKPxzl+/DjhcBi3282cOXOoqKiYsGiUlpZGVVUVp0+fBuDgwYMsWrSIdevWJdv09/fz7LPP8utf/xpd1/nCF77AJz7xiWQQ56NQFGXCooHf758QOJJlmdTUVBYtWkQ8Hk/OgyzLWK1WDAbDJc9uslgsLF++nKeeegqA7u5ujh8/zsDAAOnp6cm5aGpq4vnnn+eZZ57B7Xbz9a9/nY0bNybL7d1///088cQTybOoxs/tKikpYf78+WRmZk64bllZGZmZmfT39+P3+/nJT37C5z73OTIyMgAIh8O0tLTw7W9/m4aGBpYtW8anP/1pFi1a9JHn9GYT12L4IiM0D5++4qDKOF3XCMX8NA6+h4RElr3opkuiiiWiDId6aRh8j0Bs9EP1kdASBGN+6gaOUpWxhFRr9jSPUrjWVHsKuZv/kI7nHsdeMh9LTgWScuP+WE+xGVlbncEbp/uIXr75B5LpNDMr14HTMnNLjOm6TkpKCrfeeiv33nsvS5cupaSkhLS0tClLp2W5zCwvS+PFI12gTNM3V/39fmcyRVEoKSnhc5/7HHfffTfl5eXk5eVNmSFuMshkuy3ML3TT3OcnGv/o53ppOhSkW5lfNLPPYjQYDFRXV7N06VK2bt1KQUEBeXl5yaoBgiAIgiAIgiAIgjDuxl3J+gjGAyG6rtPb28vTTz/Nyy+/PCHTSNf15I5lWZYpLy/n9ttvZ+3atcl+5s2bx/Lly9m5cyeJRILR0bGF5jlz5rBy5cpJJf3Kysq4//77kwGq/fv3Y7FY6OnpISMjg9HRUQ4fPsyuXbs4efIkGRkZE0r0nf/38a8v5cLnzWYzZrM5+bXX6+W5556jra0Nk8lEQUEBmzZtwul0XtF8jrNYLCxevJh58+YxPDyMx+Ph8OHD/OM//iNr167F5XLR09PD22+/zZtvvkltbS15eXkTDn83Go3cc889vPzyy7S2thKJRJK7dNeuXUtZWdmkRaQtW7ZQW1vL7t278Xg8PPPMMwDMnz8fo9FIU1MTe/fuZdeuXfT391NYWJicn/HzsqaaL2GySDxE/eBxPOEB4lrsQ/ej6RoDwW6c/nRsJjd2o2saR/nxikajjIyMkEgkyM3N/UCvCcZ8NA+fwhcZRtM/7KKnjq5r9PnbcVsyMBvsWA0zd1d+OBxOBqNzcnKu8WiuD7LRir1kAQZ3FqHuBkK5jVjzZ13rYV01drPKvAI3FTkO6nt8BKOJjxy7Lsuyc0tl+rSM71qxWCysXLmSzMxMFi9enCzFezEuq4HZ+U7mFLjoHA4SiiYu2vaDynSZmZPvoixr5n6PAaiqqmLHjh1YrVZmz56NxWK5ZPsct5m7Fufyn7saP3KASgey3Waq85wUpVsv2/56tnDhQjIyMnC73cyePXtSNr8gCIIgCIIgCIIgjBP/YpyC0+lMlicZGRnhjTfeuOxrSkpKUFV1QoAqJyeH+fPnU1paSkNDA/D+rtJVq1ZN6iM3N5dt27axd+9eTp48SVdXFy+++CKNjY2Ul5czMDDA4cOHGRwcJCUlhRUrVrBkyRJsNhswFrw5v5ydwWCYtEhls9mSCwWSJE0IktlsNjIyMnA4HPh8PiKRCLt27WLfvn3JMxhuu+22Sx5mbTQaJyzoqKo6oQRhWloan/zkJ4lEIhw4cIC2tjaeeuopTp48idvtprOzk4aGBoaGhrDb7WzdupXi4uIJ/c+ZM4e5c+dSV1fH8PBwMptt48aNUwYAVq1aRX19PQMDA9TW1lJTU8OTTz5JZWUlRqOR9vZ2jh8/jq7rVFZWsnz5coqLi5FlGYfDkRy/oihikeUS4loMf9RDl7eBhB7/yP3FEmEGAl2kWDJndIAqHA5z6NAhzpw5w+LFi6mqqiI3N3dSgHpcLBHFGx6gZ7QF7SMHRSWiiTB9/nZc5vQZHaAKBoPs2bOH/v5+5s+fz+zZs8nIyLipS0VJsgyykZSFn8B7ei++hnex5JTfsFlUqiKT6TRz95I8/udAO819/g/9/4gO5KdaWFqWyvzCmZ2tYjKZmD17NrNnz/5A7VVFJi/VykMrC/nZ/la6hoLEP0LJRJNBYXFJCquqMrCZZ/Znr6CggIKCgg/c3m0zclt1JntO91HX7cMfjn/os8xMqszy8jSWlqZiN8/cjD4Y23BVVlZ2rYchCIIgCIIgCIIgzADKN77xjW9c60Fcb06fPk0gEEgGKC78z+VyTXosLy+PuXPnsnr16mQ/kiSRSCQIBALU1tZisVgoKChg27Zt3HnnnZMWVg0GAw6Hg6qqKoaHh9E0jXg8Tl9fHzU1NfT09GAwGMjJyeHWW2/lM5/5DLfccksyYDQ0NERzczOtra3Y7XYeeOAB5s6dOyFgNDAwwOnTpxkcHKS0tJS77rqL0tLSZDArEAjQ3t7O6OgoRqORWCxGJBJJHnJ9/rlZU1EUhYGBAXbt2oXRaKSiooINGzYwa9b7u/rHrxcOhwmHw+i6TkdHB42NjQwPD2OxWCgsLGT58uV89atfZdasWROCapIkEQwG6e/vZ3h4GKfTSWVlJV/60pfIz8+fVD7GYrGQlZWF1WrF6/Wi6zrBYJDW1lZaWlrw+XykpaVRXFzMpz71Ke69916Ki4uRJAm73c4LL7yA3+8nMzOTVatWsX79+g//4bqBBWN+ur1N9PhakKalJp9ELBHBpFrIshdOKKs5kwQCAX7xi1/wne98h9ra2uS5b6qqYjAYJgU9/dERukebGfB3TNs9xxIxrEYH6bbcGTuPXq+XJ554gv/4j/+gqakpef6doihTzuPNxODKwnP6TRJBL9aC2ajWD5flOhPIkkRplo3u4RD9oxFC0QRXGqPSdXDbDKyfk8Xm+TkUpNmuzmCvY0ZVZlaek+Y+P8P+KMEPMY+SNNZPVY6Te5fls7oqHUWemd9fPixFljCrCnaLgZ6RECPB6BWfjyZJY/3MynXyyVsKWVqWdtPNoyAIgiAIgiAIgnDzunlX9C7hm9/8Jm+99Ra1tbXJ8nHnGy/1Nr7QazKZkuX8LjRr1iz+5E/+BJ/Px+DgIJs3b+auu+66aBaSw+HgtttuY+XKlbzxxhscOnSIlpYWuru7ycjIoKSkhPXr17Ny5coJZ14BzJ49mwcffJDh4WEqKytZvXo1qampE9qsWbOGM2fO4Ha7WbJkCRs3bpwwlmXLlvHd736XZ555hpMnTxIIBHC5XCxatIgHH3xwQobWVBRFoaqqii996Uvs37+fhx56iKVLl05oY7PZeOyxx/jEJz7BgQMHePfdd2lsbCQYDJKamsqsWbNYsWIFGzZsuOh17rvvPmRZJjMzE1VV+eIXv0hhYeFFsylKS0v58pe/zEMPPcRzzz1HS0sLra2tybJrFRUVPPjgg+Tn5yf7GA90bd++nd27d1NYWMgDDzxwyfu/mYVjAQaDPdMUnBoTTYTxR0YIRX3YTDM3i0qSJPx+P++88w7vvPMOZWVlPProozzyyCMTgrcAgegoI8F+PvQ2/ClEEyECUQ/hmB+rceYGLyRJwuPxsHv3bnbv3k11dTWf//zneeCBByZkWt5sDI5UHOVLCLSewntmL5m3PnKth3TVSBJYjCqfXV+KJMHzhzuvKGtF1yEST7Budh6fvKWQ8mzH1R3wdUqWJGRF4s/uqEJVZF4+1oUvdGXZP5IkkWI38oWNZaysSMeg3JxnC5mNClsW5BCKxglFE5zu9KJeQYBJkSSsJpUvba5kYXHKTTuPgiAIgiAIgiAIws1J0sXBOpPE4/FkibuppkfTtGQmD4wFMsxmM1ardVLgSdd1otEoAwMDwFg2j8PhuGh5r3GJRAK/308wGCSRSJBIJJAkCVVVsdvtWK3WSRkD8XicQCCAz+dDURRSU1MnZTslEgmGh4eJxWLJknvnB3U0TSMSieDxeIjH42ialsxQcDqdmM3my2ZgjJ+5E4lEsNvt2O32Ke83Go0SCAQIh8MTznpSFAWbzXbJs640TcPr9RIKhYCxsowWi+WS5b50XScSiTA6Opq8NxhbZFMUBZfLhclkmpSBNTw8TCgUwmAwJOdemKx7tJnjXW/ij3mmLUil6zoZ9jxmZy4n21EyLX1+3AYHB/nud7/Lt771reRjFosFp9NJdnY2a9asYceOHcybNw+LxULL8GnO9B8kGPVN6zhynaVUpi8m0/7By1ddT3p6evj617/Of/3XfyUfs1gsuFwu8vPzWb9+PZ/85CdZsGDBTZlNFe5vZWD/rwh11VH+R99HVgwg3bgL3bGExsBohEMNgzx/uJNjLcOol1nYV2WJ/DQrD68sZPWsDLLdZsyGm7dEJEA8odM/GuZw4xC/PdLFu03DyJf52Og6pDlMrJ2VwadWFZGXasFhUZFnaHbmdPEGY5zt8vLy8W52neolEE5cMuCn6TrpDhOrKtPZcWsJOSkW7GYxj4IgCIIgCIIgCMLNRQSoBEGYFh3eeg53/I6E9tHPn5ogbMQcSkXy2jk/7qXrOrFYDBgrj3lh4DSRSKBp2kWzFWOxWLJM3Pmmu9/R0VFeeOEFfvWrX01qazKZSEtLo7S0lJUrV7JhwwbsBdCvNxCf5nmUQxYskXQ078Sg9cc1Dx+lX0mSGB4e5qc//Sm7du2adC2z2UxmZiaVlZUsW7aMTZs2MXfuXNLT0y81JTcUPRFn4MBzjNbuJ3XxJlzz1qOYbuzSdQlNZyQQpbHXx5lOLyfbPLQPBhn0Rc6VrNMxGxXS7SbyUi3MLnCxsCiFqhwHKTYjBvXGDeBdifF5bO7zU9Pp5USbh46hAAO+CMFIgoSmocoyKXYjOW4LlTkO5he6qcx1UpBmxWxQpjPhc0bzh+N0DQep6RrlWMswrf0Bej1hvKEY8YSGIku4LAZyUs7NY5GbWblOyrLsKLIs5lEQBEEQBEEQBEG46dx828wFQbgqdF1Hm+7glATxRIzunm46T3kmJIQoioLDMVaey+fzTcjCg7HsGlVV8fmmzkRyOBzE4/FkFt7V6jcYDNLd3T1l20gkQnd3N93d3YyMjJCdnU21q4CEPTFl+48iEPLT3jjEcGt4wiLoxzUPH6VfSZLw+Xz09/dPea1wOEx7ezsdHR2MjIxQUFBAaWnpJWbjxiMpKvaSBUSHuuh/+xc4Z60Go+WGzqJSZIl0h4lUu5HKHAfzCtz/f3t3HqxXfd95/vM723Oe7T53X3R1F+1oBQlJbAqrbYwNBoxxHA9ZqEo541R6epyOp1I1mRTV09OZ7lSmPJnqdle3YzuO3YnT2NjE+wIGDBgkLJCEJJDQfqW7b8++nHPmjwdkBBIg6eq5Enq//kKPzvmd7/ndi1R1P/r+vjo+VdRkrlKfTaVInmOrJelqQUtcAx1J9bYkCAHe4q37uKavWceni5rM1QOqMIxk20bNCU+dGV8D7Qn1tiYU9y7v7rPTSfmOVixoUn97Ust70jo8ltdErqJcqaYgDGUbo3TcVWfG12BHQj3NcaXjpw/5AQAAAAC4HBBQAZgTxhhZljO3HVSR5NiOmjNNSq/sPmMHVVdX1xk7cXp7e0+79Hvp8JmLdXO5nPbv33/aaz3PU0tLi/r6+rRx40YNDg4qlUqoaGwF0dyGfYl4QvHeDnXFT/1jv1H7cD7rGmM0NTWl7du3n/ZZsVhM7e3tWrRokdavX6+BgQHF4/EzbcX7lt81qOTAGo396lsqDh9QYuEK2X7q3W+8xFnGqDUV0+alsXe/GGdkGaPmpKeNS1rf/WK8o7hna/XCjFYvvHRnJwIAAAAA0AgEVADmhG0cxey4CuGsNIczqNrb2rVm1bXqSg3OyZqNNj09rSNHjujRRx89+ZnjOPI8T729vbrxxhv1h3/4h7rqqqvk+74OTb2smeGjKs5xN1p31wKtXLtR7YmFc7puo4yNjenFF1/UE088cfIzx3EUi8U0ODio2267TQ888ICuvvrqt82Ru1xYri+/e7GaV9+kyW3/IieZUbxn6XyXBQAAAAAAAJwWARWAOeHZvlKxZuWrs3MUT0mRIrlWTEm3eY5WbLwwDPXWUX8DAwO6//779YlPfEJr1qyR4zgnQxXPjivhZlSo5mTmbCfrX5+4k56z9RotCIK37ePSpUv1wAMP6O6779ayZctO2cfLVax1gTq3/Lb2/M2n1bTqRvldi2Uu8z0BAAAAAADAxYmACsCc8J2E2hI9Gs4ekjFzM5vEs32lvGYlvEs3WJHqnWDJZFKrV6/W3Xffrc2bN2vp0qXq6OhQLHbqsWQJN62WeIfG88c0V8Ny6vuYUdy9tI97i6JImUxGV155pe6++25t2rRJg4ODam9vf9s+Xq6M68tr61Vq6SblD72kWOsCJRZeMd9lAQAAAAAAAG9DQAVgTsSchDqSvXKsmCK9vdvl7EVq8lvVluyRNUeB13ywbVsbN27U5z73Oa1fv17r1q1Tf3+/PM877fVJr0mtiR459m4FYXVOamjy29Tsd1zS++h5nm688UYtXrxYV155pTZs2KCurq63zbi63BnLkh1LqGPLJzXx/HcVa+9TYuEKzdWxmwAAAAAAAMBcMdH5/xQZABQpUr4yq53Dv+JxkN0AACAASURBVNRw9rCqQfm81nPtmJa0rtNgyyo1+a1zVGXjlUolZbNZFYtF9ff3v6d7potj2jXyrIazhxRGwXk937NjWt5xtfozK5SKXbpHJRaLRWWzWZXLZfX19c13ORe1KAwUlPI69PW/UKx9oTq2fFJ+5+B8lwUAAAAAAACcgg4qAHPCyMh34lrRvlGFSlbTpTEFYe2c1rKMpdZ4lzqSvUrHWua40sbyfV++75/VPUmvSUva1ilbnlShklUQncs+Ghlj1J7oVWey75IOpyQpHo8rHo/PdxmXBGPZchJNSi/bpMKxPZrd+4z8jn7JMIsKAAAAAAAAFw9+WgVgzjiWp9ZElxa3rVXKa1YYhee0jmvHtLx9vbrS/TJzNIfpUuLaMXWl+rWs/Sr5blLROeyjMUaeHdPKrs1qTXRdgCpxsWtZ/yGFlZKmdz6hsFaZ73IAAAAAAACAUxBQAZhzC5uW6oqOTepOD8i859k3kRzLVWu8W9f0ffj12VOX7x9RljEaaF6plR2b1J5ccBa7GMm1PHUl+7Rl8G41xVplLuN9vJy56TalFl8lKxbX5K9/NN/lAAAAAAAAAKfgiD8Ac861Y+pO9yvmxNUa79Rw7ohmShOnn6cUSY7tqinWqq5Uv7rTg2pJdMmxXOk9xzLvR0auHVNP02L5blIj2cMazR/VdGn89JdHkmt7avY71JXuV1eqXy3xThljnUVIiPcTYztKL9us6vSopl74odqvuUeKIuky7EoEAAAAAADAxYeACsAFEXMS6kwtVNJrUtpv00xpQqVqXuWgeHI2lW1s+U5CcTeljN+ulkSXmmKt81z5xSXuJuXZ/Up6aWXi7ZopTahYzasalBREgYyMXNuTZ/uKuyk1+x1qSXQq5V3aM6cwN/yuRYovWKrs/q2a3fe8kn2rZPup+S4LAAAAAAAAkImiKJrvIgC8v0WKVAuqyldnVarmVQvr83Acy1XcTSnhpuXasXmu8uIXRZGCsKpsZUalWl5BWJVU38eEm1bCa3q98wz4jfyRlzX+q0dUmTyuvvv+XH57H11UAAAAAAAAmHcEVAAAvI8FpZyyr27V3r/5Ha35yx8o0bdSlhef77IAAAAAAABwmbPmuwAAAHDh2H5SftciZVbfqNm9T6s8MTTfJQEAAAAAAAAEVAAAvL8ZuU3t6r79M5rZ/bSKw69JNE8DAAAAAABgnhFQAQDwPmfF4mq64npZnq/C0d0qjR6c75IAAAAAAABwmSOgAgDgfc5YtuxYQs1rb1Fp5JCy+18QIygBAAAAAAAwnwioAAC4TLRc9UEZSfnDO1XLTsx3OQAAAAAAALiMEVABAHCZ8FoXKLnoSgXFnGb2PD3f5QAAAAAAAOAyRkAFAMBlpGnlDXIzHZra/mMpDCXVj/orTw4pu3+bCsf2zG+BAAAAAAAAuCw4810AAABonFh7n2JtvSqPH9PUjsfkd/Qpu+95zez+pSwvrpar71Bi4cr5LhMAAAAAAADvcwRUAABcRuxYQvEFy5U7+JJGfv5luZlO5Q6+qOLQK4r3rlBm7S3zXSIAAAAAAAAuAwRUAABcJmr5GVWmTqg0clDVqWFNvvDD+m8YKQpDGcuWMWZ+iwQAAAAAAMBlgYAKAID3uyhSFNaU3b9V4898S9M7H1dl6oSMZc93ZQAAAAAAALhMEVABAPA+V8tPa2Lr9zT06BdUnjiqKAwIpwAAAAAAADCvrPkuAAAAXFh2PKXW9ber+7Y/ULz3CknvdIwfR/wBAAAAAADgwqODCgCA9zlju3KbO9V2zd1ymjo08fx3Nbv3GdUKMzLmLf9WhRlUAAAAAAAAaAACKgAALhOxjn61xNNymzvlpFo0+/JTqkyfUBSGkiRjOyf/GwAAAAAAALiQCKgAALiMuKkWNa+5WbH2hXJTrZrc/iNVpoYVlHKKokiGDioAAAAAAAA0ADOoAAC4zBjLUmLBcg0+8O808Km/lN85KIWhFATSW4/8AwAAAAAAAC4AOqgAALhcRZGa19wit6lDI4//g6qzExINVAAAAAAAAGgAAioAAC5XxsiOp5VadJUs11d58oRi7QvnuyoAAAAAAABcBkwURdF8FwEAAOZZFCmslhRWy3KSzfNdDQAAAAAAAN7nCKgAAAAAAAAAAADQUExCBwAAAAAAAAAAQEMRUAEAAAAAAAAAAKChCKgAAAAAAAAAAADQUARUAAAAAAAAAAAAaCgCKgAAAAAAAAAAADQUARUAAAAAAAAAAAAaioAKAAAAAAAAAAAADUVABQAAAAAAAAAAgIYioAIAAAAAAAAAAEBDEVABAAAAAAAAAACgoQioAAAAAAAAAAAA0FAEVAAAAAAAAAAAAGgoAioAAAAAAAAAAAA0FAEVAAAAAAAAAAAAGoqACgAAAAAAAAAAAA3lzHcBAABcrGrZSWX3b1NQyiuz6ga5mc6zur8yNaz84R2q5WfUsv5DchKZC1QpAAAAAAAAcGmhgwoAgDMojhzQ4X96SK996V+rMPTqWd9fOPKyDv/Tv9W+L35Wlcnj51RDFNQUFGYVlPKKwuCc1njPz4pChZWSavlpRbWqFEUX9HkNF0WKahUFhVmFldL77/0AAAAAAAAuIQRUAAC8gyiKFEXhud1s9Pq9Uf0X5yAo5ZQ7+KIKQ68oLBfOrY73KKpWVJk+odk9T6uanbzggVijRWGgam5SuYMvqjJ5XGGtOt8lAQAAAAAAXLYIqAAAuIgVj+/Tga/8mY596z+oNHbkgj6rMj2i8V99R6996V9rZveTquWnL+jzGi0oZjX16x/ryMP/XuO/ekTV6eH5LgkAAAAAAOCyxQwqAADOQVgpqTJ5XLVSTrafkJtul5NsPu21xlgytq1aYUbV2XEpiuQ2d8mJp8+4fhQGimoVVaaHVRw9JBlLlelheS09smxHdqLpTRdHquWnVc1NKgqq8lp65MSbJPP2rq2gXFAtN6mgmJVxPMXa+2TZrsKgqursuMpjR1UaO6ryxJAqM6MyjivL9WW5sXfcj6CUUy07paBSlJPMyMt0nnx+FIYKK0XVcpNyUq2yPF/Gsk9511puUjK2bD8ly/VOfl6dGVVQyss4rtxMp2wv/uZNUlgpqZqdlJvpkHFc1XLTquUm6+/W2iNju/VLg5pq+WmVJ4ZUGj2k0vhRVaZHZMXTsr24LC8mySisllWdHVctPy1ju4q198n2/NPuJQAAAAAAAM4dARUAAGehOjOq7Gu/Vv7ILlWnxxTVyjK2rVhbn5KL1yuz8gYZ+01/vUaRIknTO3+hsFxQaeyojGXkNXcrtXSjUouukpPMnOY5Y5p99TlNbH1UikLVcpMaf+Zhzb78lBL9q9W2+WOy3JjKY0eVfW2bCsf2qJadkhTJzXQoOXilUovXy2vuOrlm/vAOZfe/oOLxfQpKOVmOp1jHgNqvvUfliSFNbvu+8odekuV6mtn9lCqTQ/J7liq95Gqll258+2ZEkaKgppk9v1Th6G6Vx48prJbkJJrkdw6qaeUWxToHFFUrKhzbo7FnHlZ66dXKrNwir3VBfYlaRYXj+zS983G5qRY1rbhOsfaFKo0f1czup1QeP6pabkqW48pt6VF6yUYlB9fJSTQpKBVUGHpFY0//szIrb1BQyqk0cki13KSsWEJ+92I1r7lZfseASqOHNP7stzX7yrMKSwUVDu/SyBPfkN85oOTgOjWvuVnFE/uU3b9NxaFXFRRmJUl+1yK1Xn2H4guWz+W3EQAAAAAAwGWPgAoAgDN6c9dMJEkqDr+m4Z9+Sfmju+U1tcvYjqqzkwprZSUH1srvXqxYS/fJzh1FkRQGGnnsa1IUKgpqCmtl1bKTSq+4Vgvv/pzSS66WcbxTnhwUZ5U/9JJy+7bVO6QKM5rd+ytZbkxRUFPLVR+SMZYmtn5PY88+rOrMqNx0m2QslccOKzmwTl23/p5a198uy4tLijT61Dc1vePnCisluek2RUFVUy/9TKmlV6s8ckjZfVtVGj0sGVuFIy+rPHpYielReU3tpw2owkpR+SMv68SP/6tKw69JxpLl+QqKOQWlnLo/OK726++Tk8ioNHpYx7/3/ymz+rfkNXefDKjCalmTL/xQo098XU3LNives0yybI08/nVNvvB9SZIdSyqsllSdGVN6+TXqvfNfKb3kagWlrHIHtmvo0S8of2SXgty0wkpBsmwFhdl6UBhFatv8sXrgt/dZFYdeUVgtqzRWD7LKo/2ynJiall+jiece1cTWRxUUsnIznYpqFc3ufVp+z1ICKgAAAAAAgDlGQAUAwJlEoYwxMsbUgyZJtfysarlpdVx/vzKrb5STbNL0zl9o/OmHld37jHL7tspe/VtymzoUhaFkTP2ouukTarvmXqWXX6PS8H6d+OmXNf3Sz9S0/Bp5zV3yuxaf8mivtUcdN3xSluvr2Hf+Wk6yVV23/p4SvSvltfZIxlLhxH4N/+xLCmsVtW64Q+3X3acoCjX0nb9R7uB2Tb30U8Xa+5QcXKfa7ISmtv9EiiJ1/Nan1H7tvarO1GdOOcmM/FVbJEWafOFHGn/2YbVc9QGllmxUom+lYm0LT7s9lZlRHf32f1T+4ItKL9uk9hs+Ka99obKvPq9j3/4PGnvqnxRrXaCOG+5Xsm+lnERG+aO7VZ48rqhWkbFdRWGg2ZefUFDMKda1SE66VbkD23X0f/xfSi66Ul03P6Cm1TepNHJAJ370XzT16x8qNbhWblO7LC9ePz7RspV7dasSfVeo7dqPy2vp1tSLP9Hsnqc1veMxxbsWKzm4Tj23f0YTzz+q6Zd+puTiq9S85mYl+tco1tarWnZS0zseU1DMqW3zx9R58wMKillNbv0XuU3tiqKo/n0AAAAAAACAOUFABQDAu6hHU/VwonndLcqsvF5RGMg4rozlyFiugsKshn/6JVUmjyuslE69P6iq7/7/Xc1rb5Hb1KHK5JCM62vo0S8od/BFpZZseFtAZceSivcsVWrRlZKx5KZblV6yUellGyUZVWfHNfr41xQUs+q86dPq2PLbincvVVgrq/+3/0IHvvJ5lYYPqnB0t5L9a1QrzNTDNseT19ylePcS+R39Sg6sk+0nJWMp0bdS+SO7FAU1JfpWK7PqBsXa+0+ZF/WGsFpWefyYprb/SKklG9V16x+oee3NkjGKdw4qf2C7ZnY/pdLYEQWlvKxYQs3rP6TZ3U+qcHS3iovXy+8cVHV2Qvmje+Rm2hXvXqKgmNXU9h9LirTgI3+slnW3yWvrld/RJzeZ0Stf+H3lj+xSevk1J7uaoihQctFVWnjP59S04lpFtZpSS6/W7n9/jypTw6rmpuSkWpTsX63CkZc1u+eXinctVtOK65RYuFIyqneORaEsNyY306l492JFQaB4zzLZsTjhFAAAAAAAwBwjoAIA4EyMURRF9e6p1wMKy3ZUe/1ou/p8pElVpkeVP7xTkhSUC28kWpKxJGNkOTEl+9co1rZQxnbktfTUg6YoVHVmVGG5cJpnWzK29frxfPVf237i5K+jMFBh6FWFlZKy+7YpKBdlewlFYU1BpaDK1HHZsaRqhVkZ11Oss19upkul4f0afeIbKo8dUdOqLWpado1kbBnLkh1L1cOoKJLleLL9tCw3dtqtCcsFVadHFNaqqmUnNPbMw5p99bn6XKpaVcXjrygoZRUUswpKOdmJJrVt/Khyr72g/OGdKg2/JieZ0cyeXyoKqiePRwwrRZXHjsg4rqZ3PKbi0KuyvLiioKpqdkJhraLK9KjCSlHGWJJCKQyV7F+tWNtCOckWRVEo37JkLEdBMasoqNa/drGEjOPVu6FsV3aiSVYsLkWRYu0L5bX2vD6r6mFVpo4rs/ompZdukk4T0AEAAAAAAOD8EFABAHAWisMHNbnte5rZ/UtFUSDbSygo5VSePC5Jrx/rV7/2jaYbYzuyE031mUiSjOPKbeqQJIXViqIofA9PjvTWmVi17ISiMFBQmK2HOpYt48akKFSsrU9eS5e8lm4ZY8n2Euq69fc09dLPlHvt1xp/9lvKHXxJ2WXPqeOG++uzn4ypP8ayTn2B0whrFdXyMzKWJUWRqrPjCkq518O5SHa8SYmF9eMBLS8u208rs/pGuU3tKg69quKJ1xRr79fMzscV1apKLd4gv2uR8od2qpaflrFs1XKTMpatsFqRFCkKAyX6VinRu0JOquX1ALFep51oOhneGWPJcmOvh1HByeMZ37x3kaI3fYGMbD+l9hs+KctPKX9opya2fk/5QzvrRxde93El+ladtpMMAAAAAAAA54aACgCA9yiqVpQ78GuNPP411Qqzall3q2Lt/fXj/ixH5bEjp7kpqgdQQe03HwWBgmJWUiTL9WRs9z08PKyHKm9inPp98QVLlV62uR7IhDVJRsZ25GY6lehdcfL6ts0fk9fSrXjPUmX3b1P+4EvKHdguv2NAbqbzjQe9XnZ4mmDnTc82loxd77ZyUs1KDqxRvHvJm8I2I2NZJ48QtFyv/uwFy1UeO6Lisb1y023K7n9BdjKjxMIr5KbbJFOvPQpDJQevrAdcsYT0pnePtffL71qkU7bjNLUay5LRGUK28O3v13LVB+W1dGtm1xPKHdyu7KtbVRh6RV6mU15Lj9ym9jPuBwAAAAAAAM4OARUAAGfylg6iSnZcxeOvqjozqvQV12vhJ/5c8Y5BFU/s16ikmd1PnWaRSFFQU2nssLy2XtnxJgXFWeUP7ZCMJSfdJttPvXMNxigoFxTV6kfVRWEoY9nyuwZVPLFPftcitV59h2IdAydvC6vl+u22I0WRwmpZUVBV04prlVm5RbnDO3TsO3+jyW0/UOHYXqWXbX79WfXuqaBUOHk0nqLw5OdvsGJxuc1dkrFk+Sk1r7n59RlU1skao1q5PqPL+U0Al7niehUO71L+yC7VirOqzoyqZcOH5bV0S5LsWEJea4+ioKZk/xq1XPVBuZmO37xXpShZtizbVWV65PX6Ip0ph3rrXhrLkmQUVor1WWGvB4hRUFUU1JQaWKvUoqtUGjmgI9/6j5p+6acqDO1VeeIYARUAAAAAAMAcst79EgAAIElRtaSwXJQkGRnZji9Jyh/aoemXfloPP956LF4USVGow9/8d5re+bjCalGlkUMa/sl/U1DKKb5gxclw5nSMZctyYiqPHVVQziusVRTVKrLcmNJLN8lyY5rY9gNN7/zFKfcVT+xXYWivarlJhdWS8kd2aeblJ1WdHZeMkdfco0TvypNH9ElR/VleXDJG5dGDJ2djhbXK2+qyY0nF2vvkptuUe+3Xmtn95MljDiUpKGaV3f+CyuOndpWlV90gr22Bcq+9oInnH5Xl+Wq56kP17ilJTqpF8d4rJEkjv/iaCsf2nHJ/9sB2FYdfqx8n+JtdOuP+nbKXxpLlJ2WMpfLkkGrZiXr4V8qrcGyvZl5+UuXxYzKWLTfTqdTgOllOrD6HDAAAAAAAAHOKDioAAM7kZDBhJEXyWnoUX7hCxvY0u/cZ7f3C78kYo1puSpWZMclY9SujN47Jq4c+dqJJlcnjOvzNf6tjj/x1fX7T7ITiC5apec2NirX3nbEE208pOXilsq8+p0Nf+3MZ21Pzulu08O5/o44t92tm91PK7d+mY9/9fzT65H+Xk2xRLT+tWmFGzWtuUvv19ynes0ylkQMaeexrCko5ualWhdWyiiMHZMUSSi1eL6+5W2FQU6JvpWSMJrd9X7OvPKvk4Dq1brhD7dd9/NTCjJHX3KX+T/4fOvGj/6yxpx/W9I7H5aRbJWOpOjUsy/PV8+HPyu9ecvI2v71ffvcSOek2heWCLC+uzMrrZScykqRYW6/aNn5EU7/+oUojB/XaV/6NvEyXLNevv1duUp23/K7aNt0lO5as54FRKCOrfmzfWxlz8utYD/WulnFjyu3fptf+7n9VvHeF0iuuk9fSrfFf/rMq0yP1TqkoUmHoFUVRqNSiq+S/qTsNAAAAAAAA589+6KGHHprvIgAAuBiFpZwqM2PymrvUsu42xdp6ZXsJuakWBbWSjGXLSTQpOXilmlZcI8v1lFm1RYmepbL9pILirGqFGfmdi9S89hYZ21EYVOUkm5VacrW6P/SHSi/ZICfRdMYajO3Ka+5UWC1JkmzPV3JwnZpWXi/bT8lrXSA7kakHNWGgKAplub78zkFlVt2g5MBa2fEm2V5C1ekRRUFFYbkg47jyOwfVdfMDyqy5SW5TuyzHkeXFZWy3Pr/JshVr76sHNF2Db6vNsl3FOupdVJYXV1SrHyNoJDmpVjVdcZ0yK649pUPM2I7CalnGtuW19Khp5Ra1brhd9uudW8Z25STSivdeIcv1ZdmOwlpZUVCTFUsosfAKNa+9VfGeJTK2o6BcUHlySM1rb1G8d8XJvTSS8od3ye9apPSyTfI7ByXLkpPMKKxW6vOzJDnpVqWXbFByYI2CwqyCcl4KAxnLltfSpY4tv62WdbfJa+mWMTSeAwAAAAAAzBUTcW4NAACnVSvMKH9wh8JqUemlm+SkWhQFNVVzE8odeFFBflqWl1C8Z6ksP6ni0F7Fe5bJa+2R5fqqzo6reHxfPexoW6DS6GFVp0dkHE9ec5eSA2tlef47FxFFCiolZfdvUy07Xg+NOvqVGrzy5HGCpbEjKo8dVnV2QkEpe/L4Pb9r0Slzk/JHXlZ59JCCckHGduU2tSu56Mp6J5JVD1/CalnliSEVj7+qsJyX29yteM9Sec1d77hPxeP7VJ0eqa9t2fWj+rqXyG3ulOWe+o7VmVGVRg8rKGblZjqV6F0u43hvW7dwbK/K40cUFHOKwkC2n1KsdYH87sWy42mF1bIq08PKH9qhxMIrFGvtlRVL1N+jUtLsq8/Jclz5XYtPCcnK40dVGHpVQWFGdjytRN8qeS3dKg0fUGn0kKqzEzK2LTfdruSidXKSzTKW/c5fJwAAAAAAAJwVAioAAAAAAAAAAAA0FGfVAAAAAAAAAAAAoKEIqAAAAAAAAAAAANBQBFQAAAAAAAAAAABoKAIqAAAAAAAAAAAANBQBFQAAAAAAAAAAABqKgAoAAAAAAAAAAAANRUAFAAAAAAAAAACAhiKgAgAAAAAAAAAAQEMRUAEAAAAAAAAAAKChCKgAAAAAAAAAAADQUARUAAAAAAAAAAAAaCgCKgAAAAAAAAAAADQUARUAAAAAAAAAAAAaypnvAgAAeEdRpMLQXuWP7Fbh6G5Vpk6olpuWMZIVT8tr7lK8e6kS/auU7F8jy41Jxsx31Rel8sSQ8od3qnB0t8pjR1TNTigKarJiCXnNXfK7Fys5sFbpJVfLWDb7eBpRUFN58rjyh3aocHSPyuNHVctPKQqqsv203OZOJXqvUHJgjRL9a2Q5jiT2EQAAAAAA4K1MFEXRfBcBAMBbhdWySqOHNbPrF8odekml4/tVmRlVdXZMYSkvSTKeLyeRkdfSo3j3YiX6VyuzaosSC1fJjqfm+Q0uElGo6uyEZl5+Utl9W1U8sV+l0YOqzo4rKOYURYEsJyY70aRYW6/8zgElFq5SZu3NSvatkh1Pz/cbXBSiMFRl8rhmd/9Ss/u3qnh8n8rjR1TLTigoFRSFgSzPl5tuk5vpkN+9RKmBdcqsvVmJBctlxeLz/QoAAAAAAAAXFQIqAMBFp5afUf7wTk3++keaeO47qk6PKAqqkszbu3qiSFIkGUtOqkUt629X68aPqmnZJrmZzvko/6IR1aoqjR7W9M6fa+zJf1Jh6BWFlYJOu4+KpEiSsWS5MbVt/pjaNt+l9LLNcjMd81D9xSOsVlQ8sU/T23+i8ee+o8LR3e/+/WjZcpItatt8p1o33qn0kg1yUq3zUT4AAAAAAMBFyX7ooYcemu8iAAB4Q1guKLvveQ3/7MsafezvFZbz9R/6m9OFKjrl87BSUuHwTlUmj8uOp+V3LZbleA1+g4tDFIYqjR3V+LPf0tC//L8qjR5UFAZn3sc3hS1RWFPhyE5VpoblJDLyuwYv231UGKo4vF+jv/i6Tvz0SyqPHVE9EH0v349F5fa/oFpuUm5Th7zW3st3HwEAAAAAAN6CGVQAgItK7uBLGv75VzW59V9k7HP4a8oYze59RlG1LCuWVPs1d899kZeAsFLU2JPf0Imf/DcFxaxkrLNbwFia2fO0oiiS5flqu0z3MagUdOJH/0VjzzyssFw467lcxnY0vePnkmXL8uJqXnfrBaoUAAAAAADg0nKWP60CAODCqUyd0NjT/6zZvc+cfaDyJsayVDyxXyOPfVWVqWGF1fIcVtl453Ia7/jT/0PTu36hoJw/5+cay1LhyMsae+ZhlSeOKQpq57zWfIui6Jz2ceRnX1F231aFldK5P9xYyu17XuO/+rbK40fPfZ2LwLnuIwAAAAAAwFtxxB8A4KIx9uwjmnrhByqPHjrrTpVTGUW1iqJqRcbxFO9ZKjuemqsyG6pSqejEiRP66le/qiiKlEwmFY/Hz3h9FNRUy03qxE/+q3L7tymqliSd614aRdWyojCQ5cWU7F8jy42d41rzq1wu68CBA/riF7+oZDKpRCIh3/fPeH1YLauWn9bQo19Q4dgeRbWyzn0f68dPRlEkJ55WasmGc15nvhWLRe3Zs0f/8A//oHQ6rVQqJc/j2EIAAAAAAHD2CKgAAPMvCiVjdOyRv1bh6B6FleJ5BlSvLxsFKo8fVeuG2+WkWmXOoytrvpTLZQ0NDekv/uIv9Oqrr2p6elrValXxeFzJZPJt14eVomZefkoT276nyvjR8+pEkyQZKQoDVWfH1H7dx2W5vswcfG0arVQqac+ePfrTP/1TTU5OampqSsYY+b6vRCLxtutr+RlN73pSE89/V7XZ8bn5fgxrCqtldVz/iZPf85eaQqGgrVu36q/+6q80Ojqq6elpWZalWCz2jsEpAAAAAADAWxFQAQDmXRhUVZ0Z1fEffVGVyaE5WzcKAlWmTqht013ymrtlOZdep0etVtPY2Jj+Q9UYYQAADQdJREFU9m//Vtu3b9f27ds1NDSkWq0m13UlSZ7nybZtSVJULWv4sb9X4dAOBaXcHIQgRlGtqqBcUNumu+Qkm2Rs9zzXbLxqtaoDBw7o7/7u77Rjxw7t2rVLQ0NDCsNQjuPItm15nifLqgd6teyEhn/2FRWP7ZmbIyKNUVgtyxijto0flfF8Gcs+/3UbrFqtaufOnfrKV76i7du3a9euXRodHa3PKrMsua4rx3FO7iMAAAAAAMCZ8NMDAMC8iyolZfe/UJ/zM6fzbSIZSYVje1UrzMzhuvNnZGREjzzyiP7kT/5Ev//7v6/vf//7Gh4eVhAECsNQklH+4IuqFWfnsEMnkqJQ+UM7FBRzc7Tm/Dpy5Ij+8R//UZ/97Gf1x3/8x3rsscc0Pj5+ch+jWkW5116od/PNlShSWC0ru/8FRecz0+oicvDgQX35y1/WZz7zGX3+85/XM888o5mZGdVqNWZVAQAAAACAd+TMdwEAAIRBTeXxY1IYXIDVjWrZcVlB9QKsfeG90d3zVrVaTbt379af/dmf6YorrtBdd92le+65RyuXLlJl8oSiamVO6zDGUm1mRFZ0Ib5GF57ruqfdx3K5rOeff15/9Ed/pCuvvFL33nuvPvKRj6gvZVQeP6YorM1tIWGo6tRx2Zfe6X6SzryP+Xxejz/+uH71q19p06ZNuu+++3THHXdo4cKF81AlAAAAAAC4FJiIf94KAJhn1dlxjTz+NQ3/7MuqTB6f07VthXq+2qdtpU6NVGMyik7OUHrjr0DLsk7p9njzf79xVFm9O6nuzTOYjDEnf+9CrBuGofL5vJ577jnVaqcPSxKJhDo7O7Vhwwbde+cd6n/8L+WE5TntRitFjl6s9WpbsU2TNVfmDPW++T0upv0NgkBTU1Patm3bGd8xlUqpq6tL1113ne64do0WP/N/KzTWnO5jNorV97HQqlzoyLxedyP24XSzw8523SAINDIyoh07dpzxHZuamtTb26vrr79e999/v26//fZ32xYAAAAAAHAZooMKADD/jJHlxefwSLrfiKJI3Qv7tS65QlknM8dHCF5YxhhVq1WNj49r27ZtZwyoXNdVV1eXli1bpoULe2XH4jKlqqI57HZybFsLFgzqqsRSFezUJbePlUpFhw8ffseAyvM8LViwQMuXL1dPzwIZx5OCQNLcvavneerrW6LQX6yy8ed07QvNGKNyuay9e/e+Y0Dl+776+vq0dOlSdXZ2NrBCAAAAAABwKSGgAgDMO2PZirUtlGW7c752rVbTDR+8Ux/bdKdi7X1zvv6FVigUtHfvXn3zm99UuVw++bnjOEqn01q8eLHWr1+v2267Tddcc436u9u164kBFY+/qqhcmLM6fN/XLXd9Qi1XfkBec9ecrdsouVxOTz31lL7xjW+c8rnrumpubtaiRYu0adMmfeADH9D69evV6VW1+4luRTOjCqvlM6x69jItrVp736fVvPomOcnmOVu3UWZnZ/Xd735XjzzyyCmfe56ntrY2LVq0SNdee61uu+02bdiwQd3d3fNUKQAAAAAAuNgRUAEA5p3l+Ur0rZTlxiRjSVH47je9F8bIWLb8jgHZiaa5WXOeeZ6ndDqt1tZWrVmzRg8++KC2bNmilpYWSVJQmFVi4RWqTJ1QWMrPTVeaMZJlKd69VHYsef7rzZM3H1fn+75SqZTa29u1ceNGPfjgg9q0aZPS6bQkqTR6SMnBdZrd8/TcBVTGknFjSvSulOX6c7PmPPN9X+l0Wl1dXdqyZYsefPBBrV27VvF4fL5LAwAAAAAAFzkCKgDAvLNcX/GepXKa2mWNHVZYKc3Jusay5TRl5Pcuk3OJBlRvzAV643i/lpYW3XXXXXrwwQd1/fXXv/0Gy1Jm5Rbl9r+gytQJGZ1/QGVsV06yWYm+VbL9SzOgeuv8pY6ODt1333363d/9XW3YsOFt19uxpJrX3qrcay9IuWhOgj7LceWk2pToXX7ea80XY8wpQV9vb69+53d+R5/61Ke0evXqeawMAAAAAABcagioAAAXjdaNH1V1ZkyFob0yxjrP1SI5yWZ13vyAHD8lzUFQM18cx9Hq1at1880365ZbbtGyZcvOONvH8uJqXv9BjT/3HZVGDykKquf38CiS19yp9us/Uc9oLsCcsEaJx+NatGjRyXBv+fLlam1tPe21djKj1s13avTJ/67q7MQc7GOoWHufWq/+8PmtcxFIp9PasGGDPv3pT2vz5s1avHjxGfcRAAAAAADgTOyHHnroofkuAgAASXLTbSqNHFR59LCiWlnnHCpFkaxYXMn+1eq75/PymjtlLsB8q0YwxsjzPC1btky33nqr1q5dq87OTnmed4brLdleQkExq/L4EVWmh88j7ItkeXGll16t3jv/ldx0m4xln/vLzCNjjBKJhNauXatbb71Va9asUVtb25n30bJkx5KqZidUHj2sanb8vEJTK5ZQZvWN6r71D+SmL90wxxijdDqtlStX6uabb9bKlSvV0tIi1700//8CAAAAAADzhw4qAMBFw+9apLZNd6o6M6bZPb+sH/V3Dh07xnEV712h9mvvVXzBsgtQaeM4jqO2tjbdcccd7/0mY5RZe4tK40dUy02pMnn83PbRcpQYWK3WjXcq3rP0rO+/mLiuq56eHt19993v8Y76frVt/KjKY0cUFGdVnRk7h32MZGxP6aUb1brxo/K7l5zl/RcXz/M0MDCggYGB+S4FAAAAAABc4uigAgBcVLyWblluTNXpYdVy04rC2lndb8USivcsU/u196jzpgdkef4FqvTi5qZaZPspRbWyKlPDCkpZnU1HmuX6ivcuV8f196v9untlxxIXrtiLmJtuk+3FFVSKqs6MKSwXzuJuI8v1lexfrc6b/ie1rv+gbC9+wWoFAAAAAAC4lNBBBQC4qNjxtNqvuVteS7cOf/P/VOHQDoVBVYqid77RGBljye9arAV3/M9qv/ZeWZd5GJBetkluuk1WLKETP/hPkrEUhaGkd9hLY6QoUqyjX713/i9qvfojsv1kw2q+GGXW3CQn3SbL8zX6i69LUaQoDN75JmNkbEexzkEt/Pj/pszqGwmnAAAAAAAA3sRE0bv9xA8AgAaLQgXlomq5KU1u/7HGn/228odeUlDM1ucAvXHMWhQpikJZri+/e4nar/u4Wjd8WLGOPtl+WsY695lB7xdhraJadkLF4/s08vjXNP3yk6pOj5y6j4oUhZEs15PftUitmz+m9mvvVaytV7afumTnTs2lsFJSdXZchaMva/ixv9fs3mdVy03JGHPq96MiWW5M8d4r1H7NPWq/7h656XZZfvK8ZlgBAAAAAAC83xBQAQAuWlEYqjozqvLEkMpjh1UaPaTK5AlVcxMyMrL8lLyWbvmdA4p1DCjW1iuvufuyPdbvTKIwVFgpqDRySOWJoyoNH1R57IiquUlFQU12LCG3uVN+5yL5nYOKdfTJa+mR5cbmu/SLShTUVCvMqjx2ROWJYyqNHFR5/Khq+WlFtaqcZEZuc5f8jgH53Yvr348tPTI2DesAAAAAAABvRUAFALgkhNWyqrNjqs5OKKqWJBlZni872VyfE3SZH0P3XkVBTbX8tKqz4wqKOUVhIMuNyUlm5KRa5CSb57vES0IU1FTNTqqWHVdQKigKazKOKzfVIjfTxfcjAAAAAADAuyCgAgAAAAAAAAAAQEMxDAEAAAAAAAAAAAANRUAFAAAAAAAAAACAhiKgAgAAAAAAAAAAQEMRUAEAAAAAAAAAAKChCKgAAAAAAAAAAADQUARUAAAAAAAAAAAAaCgCKgAAAAAAAAAAADQUARUAAAAAAAAAAAAaioAKAAAAAAAAAAAADUVABQAAAAAAAAAAgIYioAIAAAAAAAAAAEBDEVABAAAAAAAAAACgoQioAAAAAAAAAAAA0FAEVAAAAAAAAAAAAGgoAioAAAAAAAAAAAA0FAEVAAAAAAAAAAAAGoqACgAAAAAAAAAAAA1FQAUAAAAAAAAAAICGIqACAAAAAAAAAABAQxFQAQAAAAAAAAAAoKEIqAAAAAAAAAAAANBQBFQAAAAAAAAAAABoKAIqAAAAAAAAAAAANBQBFQAAAAAAAAAAABqKgAoAAAAAAAAAAAANRUAFAAAAAAAAAACAhiKgAgAAAAAAAAAAQEMRUAEAAAAAAAAAAKChCKgAAAAAAAAAAADQUARUAAAAAAAAAAAAaCgCKgAAAAAAAAAAADQUARUAAAAAAAAAAAAaioAKAAAAAAAAAAAADUVABQAAAAAAAAAAgIYioAIAAAAAAAAAAEBDEVABAAAAAAAAAACgoQioAAAAAAAAAAAA0FAEVAAAAAAAAAAAAGgoAioAAAAAAAAAAAA0FAEVAAAAAAAAAAAAGoqACgAAAAAAAAAAAA1FQAUAAAAAAAAAAICGIqACAAAAAAAAAABAQxFQAQAAAAAAAAAAoKH+f1wcuy8c9hx1AAAAAElFTkSuQmCC)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Zpo_w5Esimar" + }, + "source": [ + "### **Example 1: Medical Coding**\n", + "\n", + "Here, we want to construct a simple medical coding task where effectively we want to translate every piece of text in a patient's history to a set of medical codes. Here, you'll notice a couple of things:\n", + "\n", + "1. The processing of data is functionally the same as exploring data with PyHealth datasets [here](https://colab.research.google.com/drive/1vI_oljc7rU5ocsC26ITM7HUgD5SGZkFE?usp=sharing)\n", + "2. We have defined a 'text' input for our models and 'icd_codes' as our output. These explicitly outline how pyhealth models will interact with the \"pyhealth.task\" and for interested users, what the task is attempting to accomplish.\n", + "3. Here we only need to define 3 specific things:\n", + "\n", + "\n", + "```\n", + "input_schema : {\"our feature identifier\" : \"processor_name\"}\n", + "output_schema : {\"our label\" : \"type of label\"}\n", + "```\n", + "\n", + "\n", + "\n", + "See the API for our processors [here](https://pyhealth.readthedocs.io/en/latest/api/processors.html). Each processor effectively defines how the processed data will be represented in tensor space. You can imagine it being equivalent to a tokenizer in LLMs, a image transform in medical imaging, and a static embedding model for various categorical models." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "DjySbbSKtyqc" + }, + "outputs": [], + "source": [ + "from pyhealth.data import Patient\n", + "from pyhealth.tasks.base_task import BaseTask\n", + "from typing import Dict, List\n", + "from pyhealth.processors import TextProcessor, MultiLabelProcessor\n", + "import polars as pl\n", + "class MIMIC3ICD9Coding(BaseTask):\n", + " \"\"\"Medical coding task for MIMIC-III using ICD-9 codes.\n", + "\n", + " This task uses clinical notes to predict ICD-9 codes for a patient.\n", + "\n", + " Args:\n", + " task_name: Name of the task\n", + " input_schema: Definition of the input data schema\n", + " output_schema: Definition of the output data schema\n", + " \"\"\"\n", + " task_name: str = \"mimic3_icd9_coding\"\n", + " input_schema: Dict[str, str] = {\"text\": TextProcessor}\n", + " output_schema: Dict[str, str] = {\"icd_codes\": MultiLabelProcessor}\n", + "\n", + " def pre_filter(self, df: pl.LazyFrame) -> pl.LazyFrame:\n", + " filtered_df = df.filter(\n", + " pl.col(\"patient_id\").is_in(\n", + " df.filter(pl.col(\"event_type\") == \"noteevents\")\n", + " .select(\"patient_id\")\n", + " .unique()\n", + " .collect()\n", + " .to_series()\n", + " )\n", + " )\n", + " return filtered_df\n", + "\n", + " def __call__(self, patient: Patient) -> List[Dict]:\n", + " \"\"\"Process a patient and extract the clinical notes and ICD-9 codes.\n", + "\n", + " Args:\n", + " patient: Patient object containing events\n", + "\n", + " Returns:\n", + " List of samples, each containing text and ICD codes\n", + " \"\"\"\n", + " samples = []\n", + " text = \"\"\n", + " icd_codes = set()\n", + "\n", + " diagnoses_icd = patient.get_events(\n", + " event_type=\"diagnoses_icd\",\n", + " )\n", + " procedures_icd = patient.get_events(\n", + " event_type=\"procedures_icd\",\n", + " )\n", + " noteevents = patient.get_events(\n", + " event_type=\"noteevents\",\n", + " )\n", + "\n", + "\n", + " for note in noteevents:\n", + " text += \" \" + note.text\n", + "\n", + " diagnoses_icd = [event.icd9_code for event in diagnoses_icd]\n", + " procedures_icd = [event.icd9_code for event in procedures_icd]\n", + " icd_codes = list(set(diagnoses_icd + procedures_icd))\n", + "\n", + " samples.append({\n", + " \"patient_id\": patient.patient_id,\n", + " \"text\": text,\n", + " \"icd_codes\": icd_codes\n", + " })\n", + "\n", + " return samples\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZDVuIMBNkV1e" + }, + "source": [ + "### Connecting the medical coding task to the MIMIC3 dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "6s-hW_4zmELB", + "outputId": "f48cc318-c453-43a6-a00f-9e7a440446a1" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "No config path provided, using default config\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.mimic3:No config path provided, using default config\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Initializing mimic3 dataset from https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III (dev mode: False)\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Initializing mimic3 dataset from https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III (dev mode: False)\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Setting task mimic3_icd9_coding for mimic3 base dataset...\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Setting task mimic3_icd9_coding for mimic3 base dataset...\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "No cache_dir provided. Using default cache dir: /root/.cache/pyhealth/54ab2756-e441-5796-8994-6521684b496b\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:No cache_dir provided. Using default cache dir: /root/.cache/pyhealth/54ab2756-e441-5796-8994-6521684b496b\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Applying task transformations on data with 1 workers...\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Applying task transformations on data with 1 workers...\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Detected Jupyter notebook environment, setting num_workers to 1\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Detected Jupyter notebook environment, setting num_workers to 1\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Single worker mode, processing sequentially\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Single worker mode, processing sequentially\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Worker 0 started processing 49993 patients. (Polars threads: 2)\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Worker 0 started processing 49993 patients. (Polars threads: 2)\n", + " 0%| | 0/49993 [00:00 0:\n", + " print(\"First sample:\")\n", + " print(f\" - Text length: {len(samples[0]['text'])} characters\")\n", + " print(f\" - Number of ICD codes: {len(samples[0]['icd_codes'])}\")\n", + " if len(samples[0]['icd_codes']) > 0:\n", + " print(f\" - Sample ICD codes: {samples[0]['icd_codes'][:5] if len(samples[0]['icd_codes']) > 5 else samples[0]['icd_codes']}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dFBpilGxmHj_" + }, + "source": [ + "#### Dataset Splitting for Training Purposes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "sKXwIMqumG5l" + }, + "outputs": [], + "source": [ + "from pyhealth.datasets import split_by_sample\n", + "\n", + "\n", + "train_dataset, val_dataset, test_dataset = split_by_sample(\n", + " dataset=samples,\n", + " ratios=[0.7, 0.1, 0.2]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Q2W0_EAHmN9I" + }, + "outputs": [], + "source": [ + "from pyhealth.datasets import get_dataloader\n", + "\n", + "\n", + "train_dataloader = get_dataloader(train_dataset, batch_size=32, shuffle=True)\n", + "val_dataloader = get_dataloader(val_dataset, batch_size=32, shuffle=False)\n", + "test_dataloader = get_dataloader(test_dataset, batch_size=32, shuffle=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000, + "referenced_widgets": [ + "7a9122563c9e4687bdee551f80e6a04c", + "76820c961c5546eea07dffaee4fc1afe", + "b95f052369ca4c9aa408202a79605ad5", + "0de857438cdd48be99a9976d55f17d90", + "0a79aae3c59841b7997121f1dfc9018b", + "63e183f44074438b9668cb550609ef6e", + "0c902e1ced3c4f55aa24fd89f489a868", + "24a2f7608ff140068dca3da173d800f2", + "0e04bc0fd737470eabcc2e8b096ebd0b", + "79670c3c4caa477c8d9f532a67713bd1", + "73ef445b199549f0b9b373c8c811086e", + "19ef55e52f564055bacaa051e3a5e741", + "847814036a204dd5aec948d54f680c73", + "4d4f9f0c7f2c4b5cb89477736a1be044", + "54fde265e53744949ba84c061daf8052", + "81cd3f0f3550491896178867b92998e9", + "7e4b3d65b6ea4a45a031d0b124935936", + "c3c29ecd8c164cf59b02b109b8b7fff2", + "1305abdd6b2343a992653d0b0ca00ac6", + "89e8167d12ad4cadb221a7ad251dbb0f", + "e61505c4a66849ad939b527cb8d987b3", + "f3de069a2d6941ef8340549574f79b7d", + "15b29ca96820408eb704ef4ffc6a31bf", + "695d2abbe9474d60922fa34a2eb37048", + "6a8edb7b0481458a810aaaf20c31ea2a", + "515d714490244e85ac709cc287e2c969", + "dd9f1067b55e4ba79031de14541d6241", + "1fd4ea84b4fc4b89b3d44c6dd6b33939", + "05f7b1c15e334899ba95697314d1f262", + "9759b8e4fdda4ec99ed55c206323b8bb", + "048fef6a53924f7287500af1aad452fc", + "6ab68fe5e80d4d338a3c90cc6610b8c0", + "d21606905a2848cb924a93729072b0a0", + "3d02a6191a2a4c78a6489e14a8b940c0", + "914494fbc0b94890b5d4ba2f586531dd", + "09e8ed1d23854e3386dd69427e844388", + "f817d9d70b64411ca6c8621aa167f81d", + "faea50f2c8374f00a2812cde1d160035", + "3241772b17c0457daacec72d22e48802", + "19c37da106fc4625818ef41b30b30457", + "6db1a2d475674bdb8da0855aa7cdaf2b", + "466ec22790b44cd0b7708469d5d70fbc", + "a48767e85091415592470d1a82d5017b", + "819dd68e15c4417b8a8660bc736c9b20", + "442bf25d617a4a2b86409415d58086d5", + "dbd0f3bdce2847b98793c0e31489e6e1", + "3b878bf8ee034863a6fc69b76ffa327c", + "379bf5b86d04463d99fa3f0a470fa7d0", + "103ed91a868f4da2b82a22ab7c815624", + "02012ea045354c0e8eb824877e803d29", + "f2cc19a0cc954c2a93896387ace72b52", + "05b026da265749c09e06ce39261b9881", + "9f7219f85d7e4f939538373d372259cf", + "afcb399895c343ba91770fae1f2b6f55", + "ac534e2be9b84087be55cbac903f6b3d" + ] + }, + "id": "aazMWCTGmSKU", + "outputId": "2ca8d1d5-da4b-48fa-a6df-f5fbd252fa69" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "/usr/local/lib/python3.12/dist-packages/huggingface_hub/utils/_auth.py:94: UserWarning: \n", + "The secret `HF_TOKEN` does not exist in your Colab secrets.\n", + "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n", + "You will be able to reuse this secret in all of your notebooks.\n", + "Please note that authentication is recommended but still optional to access public models or datasets.\n", + " warnings.warn(\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "config.json: 0%| | 0.00/385 [00:00\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.trainer:Optimizer: \n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Optimizer params: {'lr': 5e-05}\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.trainer:Optimizer params: {'lr': 5e-05}\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Weight decay: 0.0\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.trainer:Weight decay: 0.0\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Max grad norm: None\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.trainer:Max grad norm: None\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Val dataloader: \n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.trainer:Val dataloader: \n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Monitor: f1_micro\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.trainer:Monitor: f1_micro\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Monitor criterion: max\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.trainer:Monitor criterion: max\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Epochs: 1\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.trainer:Epochs: 1\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Patience: None\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.trainer:Patience: None\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.trainer:\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "Epoch 0 / 1: 0%| | 0/1094 [00:00 List[Dict[str, Any]]:\n", + " \"\"\"Process a patient's chest X-ray data to classify COVID-19 status.\n", + "\n", + " Args:\n", + " patient: A patient object containing chest X-ray data.\n", + "\n", + " Returns:\n", + " List[Dict[str, Any]]: A list containing a single dictionary with:\n", + " - \"image\": Path to the chest X-ray image\n", + " - \"disease\": The disease classification label\n", + "\n", + " Raises:\n", + " AssertionError: If the patient has more than one chest X-ray event.\n", + " \"\"\"\n", + " event = patient.get_events(event_type=\"covid19_cxr\")\n", + " # There should be only one event\n", + " assert len(event) == 1\n", + " event = event[0]\n", + " image = event.path\n", + " disease = event.label\n", + " return [{\"image\": image, \"disease\": disease}]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3w9KpgmjljMR" + }, + "source": [ + "#### We apply this task function on the COVID19CXR dataset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "YxuCmBsmuwNj", + "outputId": "9116441a-4c22-471e-b9f4-0e291b53c7fa" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "--2026-03-20 17:13:58-- https://storage.googleapis.com/pyhealth/covid19_cxr_data/archive.zip\n", + "Resolving storage.googleapis.com (storage.googleapis.com)... 142.250.4.207, 64.233.170.207, 142.251.10.207, ...\n", + "Connecting to storage.googleapis.com (storage.googleapis.com)|142.250.4.207|:443... connected.\n", + "HTTP request sent, awaiting response... 200 OK\n", + "Length: 816029038 (778M) [application/zip]\n", + "Saving to: ‘archive.zip’\n", + "\n", + "archive.zip 100%[===================>] 778.23M 19.7MB/s in 45s \n", + "\n", + "2026-03-20 17:14:44 (17.3 MB/s) - ‘archive.zip’ saved [816029038/816029038]\n", + "\n", + "COVID\n", + "COVID.metadata.xlsx\n", + "Lung_Opacity\n", + "Lung_Opacity.metadata.xlsx\n", + "Normal\n", + "Normal.metadata.xlsx\n", + "README.md.txt\n", + "'Viral Pneumonia'\n", + "'Viral Pneumonia.metadata.xlsx'\n" + ] + } + ], + "source": [ + "!wget -N https://storage.googleapis.com/pyhealth/covid19_cxr_data/archive.zip\n", + "!unzip -q -o archive.zip\n", + "!ls -1 COVID-19_Radiography_Dataset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "nwnHieQiXcZA", + "outputId": "111ab043-6edf-4480-ea5c-654957b53999" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "No config path provided, using default config\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.covid19_cxr:No config path provided, using default config\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Initializing covid19_cxr dataset from /content/COVID-19_Radiography_Dataset (dev mode: False)\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Initializing covid19_cxr dataset from /content/COVID-19_Radiography_Dataset (dev mode: False)\n" + ] + } + ], + "source": [ + "from pyhealth.datasets import COVID19CXRDataset\n", + "\n", + "\n", + "root = \"/content/COVID-19_Radiography_Dataset\"\n", + "cxr_dataset = COVID19CXRDataset(root)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 731 + }, + "id": "U5p4Eux-u9LV", + "outputId": "a1e171d9-6e2c-4bc9-e4f3-800f21275f8b" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Setting task COVID19CXRClassification for covid19_cxr base dataset...\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Setting task COVID19CXRClassification for covid19_cxr base dataset...\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "No cache_dir provided. Using default cache dir: /root/.cache/pyhealth/ebe82b39-fbfc-5f16-8c90-0a3824c58fab\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:No cache_dir provided. Using default cache dir: /root/.cache/pyhealth/ebe82b39-fbfc-5f16-8c90-0a3824c58fab\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Applying task transformations on data with 1 workers...\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Applying task transformations on data with 1 workers...\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Scanning table: covid19_cxr from /content/COVID-19_Radiography_Dataset/covid19_cxr-metadata-pyhealth.csv\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Scanning table: covid19_cxr from /content/COVID-19_Radiography_Dataset/covid19_cxr-metadata-pyhealth.csv\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Caching event dataframe to /root/.cache/pyhealth/ebe82b39-fbfc-5f16-8c90-0a3824c58fab/global_event_df.parquet...\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Caching event dataframe to /root/.cache/pyhealth/ebe82b39-fbfc-5f16-8c90-0a3824c58fab/global_event_df.parquet...\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Detected Jupyter notebook environment, setting num_workers to 1\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Detected Jupyter notebook environment, setting num_workers to 1\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Single worker mode, processing sequentially\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Single worker mode, processing sequentially\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Worker 0 started processing 21165 patients. (Polars threads: 2)\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:pyhealth.datasets.base_dataset:Worker 0 started processing 21165 patients. (Polars threads: 2)\n", + " 0%| | 0/21165 [00:00', '>='\nabnormal_labs = patient.get_events(\n event_type=\"lab\",\n filters=[(\"status\", \"!=\", \"high\")] # exclude 'high', keep 'abnormal' and 'critical'\n)\nprint(f\"Non-high abnormal labs: {len(abnormal_labs)}\")\nfor lab in abnormal_labs:\n print(f\" {lab['name']}: {lab['value']} {lab['unit']} — {lab['status']}\")" + "source": "# 3d. Attribute filters: only abnormal or critical lab results\n# Format: [(attribute_name, operator, value), ...]\n# Supported operators: '==', '!=', '<', '<=', '>', '>='\nabnormal_labs = patient.get_events(\n event_type=\"lab\",\n filters=[(\"status\", \"!=\", \"high\")] # exclude 'high', keep 'abnormal' and 'critical'\n)\nprint(f\"Non-high abnormal labs: {len(abnormal_labs)}\")\nfor lab in abnormal_labs:\n print(f\" {lab['name']}: {lab['value']} {lab['unit']} \u2014 {lab['status']}\")" }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], - "source": "# 3e. Multiple filters (AND logic)\ncritical_glucose = patient.get_events(\n event_type=\"lab\",\n filters=[\n (\"name\", \"==\", \"Blood Glucose\"),\n (\"value\", \">\", 200.0),\n ]\n)\nprint(f\"Critical glucose readings (>200 mg/dL): {len(critical_glucose)}\")\nfor lab in critical_glucose:\n print(f\" {lab.timestamp.date()} — {lab['value']} mg/dL\")" + "source": "# 3e. Multiple filters (AND logic)\ncritical_glucose = patient.get_events(\n event_type=\"lab\",\n filters=[\n (\"name\", \"==\", \"Blood Glucose\"),\n (\"value\", \">\", 200.0),\n ]\n)\nprint(f\"Critical glucose readings (>200 mg/dL): {len(critical_glucose)}\")\nfor lab in critical_glucose:\n print(f\" {lab.timestamp.date()} \u2014 {lab['value']} mg/dL\")" }, { "cell_type": "code", @@ -129,40 +136,40 @@ { "cell_type": "markdown", "metadata": {}, - "source": "---\n## Part 4: Realistic Longitudinal Example\n\nLet's build a richer patient record — a 63-year-old with Type 2 diabetes, chronic kidney disease, and polyneuropathy, tracked across three hospital admissions over 18 months." + "source": "---\n## Part 4: Realistic Longitudinal Example\n\nLet's build a richer patient record \u2014 a 63-year-old with Type 2 diabetes, chronic kidney disease, and polyneuropathy, tracked across three hospital admissions over 18 months." }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], - "source": "# A more complete patient record with three admissions\nfull_data = pl.DataFrame(\n {\n \"event_type\": [\n # Admission 1 (Jan 2023)\n \"admission\",\n \"diagnosis\", \"diagnosis\",\n \"lab\", \"lab\", \"lab\",\n \"note\",\n # Admission 2 (Jul 2023)\n \"admission\",\n \"diagnosis\", \"diagnosis\", \"diagnosis\",\n \"lab\", \"lab\",\n # Admission 3 (Jan 2024)\n \"admission\",\n \"diagnosis\",\n \"lab\",\n \"note\",\n ],\n \"timestamp\": [\n # Admission 1\n datetime(2023, 1, 10, 8, 0),\n datetime(2023, 1, 10, 9, 0),\n datetime(2023, 1, 10, 9, 5),\n datetime(2023, 1, 10, 10, 0),\n datetime(2023, 1, 10, 10, 5),\n datetime(2023, 1, 10, 10, 10),\n datetime(2023, 1, 12, 14, 0),\n # Admission 2\n datetime(2023, 7, 20, 8, 0),\n datetime(2023, 7, 20, 9, 0),\n datetime(2023, 7, 20, 9, 5),\n datetime(2023, 7, 20, 9, 10),\n datetime(2023, 7, 20, 10, 0),\n datetime(2023, 7, 20, 10, 5),\n # Admission 3\n datetime(2024, 1, 5, 8, 0),\n datetime(2024, 1, 5, 9, 0),\n datetime(2024, 1, 5, 10, 0),\n datetime(2024, 1, 7, 15, 0),\n ],\n # ── admission columns ──────────────────────────────────────────────────\n \"admission/hadm_id\": [\n \"ADM001\", None, None, None, None, None, None,\n \"ADM002\", None, None, None, None, None,\n \"ADM003\", None, None, None,\n ],\n \"admission/admit_type\": [\n \"EMERGENCY\", None, None, None, None, None, None,\n \"ELECTIVE\", None, None, None, None, None,\n \"URGENT\", None, None, None,\n ],\n # ── diagnosis columns ──────────────────────────────────────────────────\n \"diagnosis/icd_code\": [\n None, \"E11.9\", \"E11.65\", None, None, None, None,\n None, \"E11.22\", \"E11.42\", \"N18.3\", None, None,\n None, \"E11.65\", None, None,\n ],\n \"diagnosis/description\": [\n None,\n \"T2DM without complications\",\n \"T2DM with hyperglycemia\",\n None, None, None, None,\n None,\n \"T2DM with chronic kidney disease\",\n \"T2DM with polyneuropathy\",\n \"Chronic kidney disease, stage 3\",\n None, None,\n None,\n \"T2DM with hyperglycemia\",\n None, None,\n ],\n # ── lab columns ────────────────────────────────────────────────────────\n \"lab/name\": [\n None, None, None,\n \"HbA1c\", \"eGFR\", \"Creatinine\",\n None,\n None, None, None, None,\n \"HbA1c\", \"eGFR\",\n None, None,\n \"HbA1c\",\n None,\n ],\n \"lab/value\": [\n None, None, None,\n 8.5, 52.0, 1.8,\n None,\n None, None, None, None,\n 9.1, 44.0,\n None, None,\n 7.8,\n None,\n ],\n \"lab/unit\": [\n None, None, None,\n \"%\", \"mL/min/1.73m²\", \"mg/dL\",\n None,\n None, None, None, None,\n \"%\", \"mL/min/1.73m²\",\n None, None,\n \"%\",\n None,\n ],\n # ── note columns ───────────────────────────────────────────────────────\n \"note/note_type\": [\n None, None, None, None, None, None,\n \"Discharge Summary\",\n None, None, None, None, None, None,\n None, None, None,\n \"Progress Note\",\n ],\n \"note/text\": [\n None, None, None, None, None, None,\n \"Patient admitted for hyperglycemia management. HbA1c 8.5%. eGFR 52. Discharged on insulin glargine.\",\n None, None, None, None, None, None,\n None, None, None,\n \"HbA1c improved to 7.8%. eGFR stable. Continue current regimen.\",\n ],\n }\n)\n\nrichPatient = Patient(patient_id=\"P999\", data_source=full_data)\nprint(\"Patient P999 record:\")\nprint(f\" Total events: {len(richPatient.get_events())}\")\nprint(f\" Event types present: {list(richPatient.event_type_partitions.keys())}\")" + "source": "# A more complete patient record with three admissions\nfull_data = pl.DataFrame(\n {\n \"event_type\": [\n # Admission 1 (Jan 2023)\n \"admission\",\n \"diagnosis\", \"diagnosis\",\n \"lab\", \"lab\", \"lab\",\n \"note\",\n # Admission 2 (Jul 2023)\n \"admission\",\n \"diagnosis\", \"diagnosis\", \"diagnosis\",\n \"lab\", \"lab\",\n # Admission 3 (Jan 2024)\n \"admission\",\n \"diagnosis\",\n \"lab\",\n \"note\",\n ],\n \"timestamp\": [\n # Admission 1\n datetime(2023, 1, 10, 8, 0),\n datetime(2023, 1, 10, 9, 0),\n datetime(2023, 1, 10, 9, 5),\n datetime(2023, 1, 10, 10, 0),\n datetime(2023, 1, 10, 10, 5),\n datetime(2023, 1, 10, 10, 10),\n datetime(2023, 1, 12, 14, 0),\n # Admission 2\n datetime(2023, 7, 20, 8, 0),\n datetime(2023, 7, 20, 9, 0),\n datetime(2023, 7, 20, 9, 5),\n datetime(2023, 7, 20, 9, 10),\n datetime(2023, 7, 20, 10, 0),\n datetime(2023, 7, 20, 10, 5),\n # Admission 3\n datetime(2024, 1, 5, 8, 0),\n datetime(2024, 1, 5, 9, 0),\n datetime(2024, 1, 5, 10, 0),\n datetime(2024, 1, 7, 15, 0),\n ],\n # \u2500\u2500 admission columns \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n \"admission/hadm_id\": [\n \"ADM001\", None, None, None, None, None, None,\n \"ADM002\", None, None, None, None, None,\n \"ADM003\", None, None, None,\n ],\n \"admission/admit_type\": [\n \"EMERGENCY\", None, None, None, None, None, None,\n \"ELECTIVE\", None, None, None, None, None,\n \"URGENT\", None, None, None,\n ],\n # \u2500\u2500 diagnosis columns \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n \"diagnosis/icd_code\": [\n None, \"E11.9\", \"E11.65\", None, None, None, None,\n None, \"E11.22\", \"E11.42\", \"N18.3\", None, None,\n None, \"E11.65\", None, None,\n ],\n \"diagnosis/description\": [\n None,\n \"T2DM without complications\",\n \"T2DM with hyperglycemia\",\n None, None, None, None,\n None,\n \"T2DM with chronic kidney disease\",\n \"T2DM with polyneuropathy\",\n \"Chronic kidney disease, stage 3\",\n None, None,\n None,\n \"T2DM with hyperglycemia\",\n None, None,\n ],\n # \u2500\u2500 lab columns \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n \"lab/name\": [\n None, None, None,\n \"HbA1c\", \"eGFR\", \"Creatinine\",\n None,\n None, None, None, None,\n \"HbA1c\", \"eGFR\",\n None, None,\n \"HbA1c\",\n None,\n ],\n \"lab/value\": [\n None, None, None,\n 8.5, 52.0, 1.8,\n None,\n None, None, None, None,\n 9.1, 44.0,\n None, None,\n 7.8,\n None,\n ],\n \"lab/unit\": [\n None, None, None,\n \"%\", \"mL/min/1.73m\u00b2\", \"mg/dL\",\n None,\n None, None, None, None,\n \"%\", \"mL/min/1.73m\u00b2\",\n None, None,\n \"%\",\n None,\n ],\n # \u2500\u2500 note columns \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n \"note/note_type\": [\n None, None, None, None, None, None,\n \"Discharge Summary\",\n None, None, None, None, None, None,\n None, None, None,\n \"Progress Note\",\n ],\n \"note/text\": [\n None, None, None, None, None, None,\n \"Patient admitted for hyperglycemia management. HbA1c 8.5%. eGFR 52. Discharged on insulin glargine.\",\n None, None, None, None, None, None,\n None, None, None,\n \"HbA1c improved to 7.8%. eGFR stable. Continue current regimen.\",\n ],\n }\n)\n\nrichPatient = Patient(patient_id=\"P999\", data_source=full_data)\nprint(\"Patient P999 record:\")\nprint(f\" Total events: {len(richPatient.get_events())}\")\nprint(f\" Event types present: {list(richPatient.event_type_partitions.keys())}\")" }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], - "source": "# Query: track HbA1c trend across admissions\nhba1c_labs = richPatient.get_events(\n event_type=\"lab\",\n filters=[(\"name\", \"==\", \"HbA1c\")]\n)\nprint(\"HbA1c Trend:\")\nfor lab in hba1c_labs:\n print(f\" {lab.timestamp.date()} — HbA1c: {lab['value']} {lab['unit']}\")" + "source": "# Query: track HbA1c trend across admissions\nhba1c_labs = richPatient.get_events(\n event_type=\"lab\",\n filters=[(\"name\", \"==\", \"HbA1c\")]\n)\nprint(\"HbA1c Trend:\")\nfor lab in hba1c_labs:\n print(f\" {lab.timestamp.date()} \u2014 HbA1c: {lab['value']} {lab['unit']}\")" }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], - "source": "# Query: get all diagnoses within a specific year (2023)\ndiagnoses_2023 = richPatient.get_events(\n event_type=\"diagnosis\",\n start=datetime(2023, 1, 1),\n end=datetime(2023, 12, 31, 23, 59, 59),\n)\nprint(\"Diagnoses in 2023:\")\nfor dx in diagnoses_2023:\n print(f\" {dx.timestamp.date()} — {dx['icd_code']}: {dx['description']}\")" + "source": "# Query: get all diagnoses within a specific year (2023)\ndiagnoses_2023 = richPatient.get_events(\n event_type=\"diagnosis\",\n start=datetime(2023, 1, 1),\n end=datetime(2023, 12, 31, 23, 59, 59),\n)\nprint(\"Diagnoses in 2023:\")\nfor dx in diagnoses_2023:\n print(f\" {dx.timestamp.date()} \u2014 {dx['icd_code']}: {dx['description']}\")" }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], - "source": "# Query: get declining kidney function (eGFR < 50)\nlow_egfr = richPatient.get_events(\n event_type=\"lab\",\n filters=[\n (\"name\", \"==\", \"eGFR\"),\n (\"value\", \"<\", 50.0),\n ]\n)\nprint(\"eGFR readings below 50 (worrisome kidney function):\")\nfor lab in low_egfr:\n print(f\" {lab.timestamp.date()} — eGFR: {lab['value']} {lab['unit']}\")" + "source": "# Query: get declining kidney function (eGFR < 50)\nlow_egfr = richPatient.get_events(\n event_type=\"lab\",\n filters=[\n (\"name\", \"==\", \"eGFR\"),\n (\"value\", \"<\", 50.0),\n ]\n)\nprint(\"eGFR readings below 50 (worrisome kidney function):\")\nfor lab in low_egfr:\n print(f\" {lab.timestamp.date()} \u2014 eGFR: {lab['value']} {lab['unit']}\")" }, { "cell_type": "markdown", "id": "26be2c24", - "source": "---\n## Summary\n\n| Concept | Key API |\n|---------|----------|\n| Create an event | `Event(event_type, timestamp, **kwargs)` |\n| Access event attribute | `event[\"key\"]`, `event.key`, `\"key\" in event` |\n| Build from raw dict | `Event.from_dict(dict)` |\n| Create a patient | `Patient(patient_id, data_source=pl.DataFrame(...))` |\n| Get all events | `patient.get_events()` |\n| Filter by type | `patient.get_events(event_type=\"diagnoses_icd\")` |\n| Filter by time | `patient.get_events(start=..., end=...)` |\n| Filter by attribute | `patient.get_events(event_type=\"prescriptions\", filters=[(\"route\", \"==\", \"IV\")])` |\n| Return as DataFrame | `patient.get_events(return_df=True)` |\n| Load from MIMIC-III | `MIMIC3Dataset(root=..., tables=[\"diagnoses_icd\", ...])` |\n\n### Table name = event type\n\nWhen using a dataset loader like `MIMIC3Dataset`, the `event_type` on every event equals the **table name** from `mimic3.yaml` — not a generic category like `\"diagnosis\"`. The available attributes on each event are exactly the columns listed under that table's `attributes` key in the YAML.\n\n```\nTable name → event_type → example attributes\n─────────────────────────────────────────────────────────────────\ndiagnoses_icd → \"diagnoses_icd\" → icd9_code, hadm_id, seq_num\nprescriptions → \"prescriptions\" → drug, ndc, dose_val_rx, route\nnoteevents → \"noteevents\" → text, category, description\nadmissions → \"admissions\" → hadm_id, admission_type, hospital_expire_flag\nicustays → \"icustays\" → icustay_id, first_careunit, outtime\n```\n\nWhen writing a custom task, always check the relevant YAML config to know the exact attribute names available on events from each table.", + "source": "---\n## Part 5: Bridging to a Real Dataset\n\n| Concept | Key API |\n|---------|----------|\n| Create an event | `Event(event_type, timestamp, **kwargs)` |\n| Access event attribute | `event[\"key\"]`, `event.key`, `\"key\" in event` |\n| Build from raw dict | `Event.from_dict(dict)` |\n| Create a patient | `Patient(patient_id, data_source=pl.DataFrame(...))` |\n| Get all events | `patient.get_events()` |\n| Filter by type | `patient.get_events(event_type=\"diagnoses_icd\")` |\n| Filter by time | `patient.get_events(start=..., end=...)` |\n| Filter by attribute | `patient.get_events(event_type=\"prescriptions\", filters=[(\"route\", \"==\", \"IV\")])` |\n| Return as DataFrame | `patient.get_events(return_df=True)` |\n| Load from MIMIC-III | `MIMIC3Dataset(root=..., tables=[\"diagnoses_icd\", ...])` |\n\n### Table name = event type\n\nWhen using a dataset loader like `MIMIC3Dataset`, the `event_type` on every event equals the **table name** from `mimic3.yaml` \u2014 not a generic category like `\"diagnosis\"`. The available attributes on each event are exactly the columns listed under that table's `attributes` key in the YAML.\n\n```\nTable name \u2192 event_type \u2192 example attributes\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\ndiagnoses_icd \u2192 \"diagnoses_icd\" \u2192 icd9_code, hadm_id, seq_num\nprescriptions \u2192 \"prescriptions\" \u2192 drug, ndc, dose_val_rx, route\nnoteevents \u2192 \"noteevents\" \u2192 text, category, description\nadmissions \u2192 \"admissions\" \u2192 hadm_id, admission_type, hospital_expire_flag\nicustays \u2192 \"icustays\" \u2192 icustay_id, first_careunit, outtime\n```\n\nWhen writing a custom task, always check the relevant YAML config to know the exact attribute names available on events from each table.", "metadata": {} }, { @@ -176,7 +183,7 @@ { "cell_type": "code", "id": "65a34af1", - "source": "# Pull diagnosis events — event_type matches the table name exactly: \"diagnoses_icd\"\ndiagnoses = patient.get_events(\"diagnoses_icd\")\nprint(f\"Diagnosis events: {len(diagnoses)}\")\nprint()\n\n# Each event's attributes come from the 'attributes' list in mimic3.yaml\n# hadm_id, icd9_code, seq_num\nfor dx in diagnoses[:5]:\n print(f\" [{dx.timestamp.date()}] hadm={dx['hadm_id']} \"\n f\"ICD-9={dx['icd9_code']} seq={dx['seq_num']}\")", + "source": "# Pull diagnosis events \u2014 event_type matches the table name exactly: \"diagnoses_icd\"\ndiagnoses = patient.get_events(\"diagnoses_icd\")\nprint(f\"Diagnosis events: {len(diagnoses)}\")\nprint()\n\n# Each event's attributes come from the 'attributes' list in mimic3.yaml\n# hadm_id, icd9_code, seq_num\nfor dx in diagnoses[:5]:\n print(f\" [{dx.timestamp.date()}] hadm={dx['hadm_id']} \"\n f\"ICD-9={dx['icd9_code']} seq={dx['seq_num']}\")", "metadata": {}, "execution_count": null, "outputs": [] @@ -192,7 +199,7 @@ { "cell_type": "code", "id": "8892105c", - "source": "# Clinical note events — attribute 'text' holds the full note body\n# Attributes available: text, category, description, hadm_id, storetime (from mimic3.yaml)\nnotes = patient.get_events(\"noteevents\")\nprint(f\"Note events: {len(notes)}\")\n\ndischarge_notes = patient.get_events(\n event_type=\"noteevents\",\n filters=[(\"category\", \"==\", \"Discharge summary\")],\n)\nprint(f\"Discharge summaries: {len(discharge_notes)}\")\nif discharge_notes:\n first_note = discharge_notes[0]\n print(f\"\\n Date: {first_note.timestamp.date()}\")\n print(f\" Category: {first_note['category']}\")\n print(f\" Text preview: {str(first_note['text'])[:200]}...\")", + "source": "# Clinical note events \u2014 attribute 'text' holds the full note body\n# Attributes available: text, category, description, hadm_id, storetime (from mimic3.yaml)\nnotes = patient.get_events(\"noteevents\")\nprint(f\"Note events: {len(notes)}\")\n\ndischarge_notes = patient.get_events(\n event_type=\"noteevents\",\n filters=[(\"category\", \"==\", \"Discharge summary\")],\n)\nprint(f\"Discharge summaries: {len(discharge_notes)}\")\nif discharge_notes:\n first_note = discharge_notes[0]\n print(f\"\\n Date: {first_note.timestamp.date()}\")\n print(f\" Category: {first_note['category']}\")\n print(f\" Text preview: {str(first_note['text'])[:200]}...\")", "metadata": {}, "execution_count": null, "outputs": [] @@ -200,7 +207,7 @@ { "cell_type": "markdown", "metadata": {}, - "source": "---\n## Summary\n\n| Concept | Key API |\n|---------|----------|\n| Create an event | `Event(event_type, timestamp, **kwargs)` |\n| Access event attribute | `event[\"key\"]`, `event.key`, `\"key\" in event` |\n| Build from raw dict | `Event.from_dict(dict)` |\n| Create a patient | `Patient(patient_id, data_source=pl.DataFrame(...))` |\n| Get all events | `patient.get_events()` |\n| Filter by type | `patient.get_events(event_type=\"lab\")` |\n| Filter by time | `patient.get_events(start=..., end=...)` |\n| Filter by attribute | `patient.get_events(event_type=\"lab\", filters=[(\"value\", \">\", 7.0)])` |\n| Return as DataFrame | `patient.get_events(return_df=True)` |\n\nIn practice you won't create `Patient` objects manually — `MIMIC3Dataset` and other dataset loaders build them from raw EHR tables and expose them through the `set_task()` pipeline. But understanding the underlying API helps when writing custom tasks or debugging." + "source": "---\n## Summary\n\n| Concept | Key API |\n|---------|----------|\n| Create an event | `Event(event_type, timestamp, **kwargs)` |\n| Access event attribute | `event[\"key\"]`, `event.key`, `\"key\" in event` |\n| Build from raw dict | `Event.from_dict(dict)` |\n| Create a patient | `Patient(patient_id, data_source=pl.DataFrame(...))` |\n| Get all events | `patient.get_events()` |\n| Filter by type | `patient.get_events(event_type=\"lab\")` |\n| Filter by time | `patient.get_events(start=..., end=...)` |\n| Filter by attribute | `patient.get_events(event_type=\"lab\", filters=[(\"value\", \">\", 7.0)])` |\n| Return as DataFrame | `patient.get_events(return_df=True)` |\n\nIn practice you won't create `Patient` objects manually \u2014 `MIMIC3Dataset` and other dataset loaders build them from raw EHR tables and expose them through the `set_task()` pipeline. But understanding the underlying API helps when writing custom tasks or debugging." } ], "metadata": { diff --git a/examples/tutorials/tutorial_pyhealth_medcode.ipynb b/examples/tutorials/tutorial_pyhealth_medcode.ipynb index 04c545f10..07a8cad8b 100644 --- a/examples/tutorials/tutorial_pyhealth_medcode.ipynb +++ b/examples/tutorials/tutorial_pyhealth_medcode.ipynb @@ -3,7 +3,14 @@ { "cell_type": "markdown", "metadata": {}, - "source": "# PyHealth Medical Code Ontology Tutorial\n\nThis notebook covers **`pyhealth.medcode`** — a medical code ontology library for looking up codes, exploring hierarchies, and translating between code systems.\n\nYou will learn:\n- How to load a medical code system using **`InnerMap`**\n- How to look up diabetes codes in **ICD-10-CM** with detailed explanations\n- How to traverse the **code hierarchy** (ancestors and descendants)\n- How to **translate codes** between systems using **`CrossMap`** (e.g., ICD-9 → ICD-10, ICD-10 → CCS)\n- Practical patterns for preprocessing EHR datasets\n\n---" + "source": "# PyHealth Medical Code Ontology Tutorial\n\nThis notebook covers **`pyhealth.medcode`** \u2014 a medical code ontology library for looking up codes, exploring hierarchies, and translating between code systems.\n\nYou will learn:\n- How to load a medical code system using **`InnerMap`**\n- How to look up diabetes codes in **ICD-10-CM** with detailed explanations\n- How to traverse the **code hierarchy** (ancestors and descendants)\n- How to **translate codes** between systems using **`CrossMap`** (e.g., ICD-9 \u2192 ICD-10, ICD-10 \u2192 CCS)\n- Practical patterns for preprocessing EHR datasets\n\n---" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "!pip install pyhealth" }, { "cell_type": "code", @@ -15,7 +22,7 @@ { "cell_type": "markdown", "metadata": {}, - "source": "---\n## Background: Medical Code Systems\n\nEHR data uses several overlapping code systems:\n\n| System | Domain | Example |\n|--------|--------|----------|\n| **ICD-10-CM** | Diagnoses (current US standard) | `E11.9` = Type 2 DM |\n| **ICD-9-CM** | Diagnoses (legacy, pre-2015) | `250.00` = Type 2 DM |\n| **ICD-10-PCS / ICD-9-PROC** | Procedures | |\n| **ATC** | Drug classification hierarchy | `A10BA02` = Metformin |\n| **RxNorm** | Drug concepts (US) | `860975` = Metformin 500mg tablet |\n| **NDC** | Drug product codes (package level) | |\n| **CCSCM** | Clinical Classifications Software — Diagnoses | `49` = Diabetes mellitus |\n| **CCSPROC** | CCS — Procedures | |\n\n**Why code mapping matters in ML:**\n- MIMIC-III uses ICD-9 codes (pre-2015 data), while MIMIC-IV uses ICD-10\n- Vocabularies differ in specificity: ICD-10 has ~70,000 codes vs ICD-9's ~14,000\n- ML models trained on ICD-9 codes cannot directly generalize to ICD-10 datasets without mapping\n- CCS groups 70,000 ICD-10 codes into ~300 categories — much better for small datasets" + "source": "---\n## Background: Medical Code Systems\n\nEHR data uses several overlapping code systems:\n\n| System | Domain | Example |\n|--------|--------|----------|\n| **ICD-10-CM** | Diagnoses (current US standard) | `E11.9` = Type 2 DM |\n| **ICD-9-CM** | Diagnoses (legacy, pre-2015) | `250.00` = Type 2 DM |\n| **ICD-10-PCS / ICD-9-PROC** | Procedures | |\n| **ATC** | Drug classification hierarchy | `A10BA02` = Metformin |\n| **RxNorm** | Drug concepts (US) | `860975` = Metformin 500mg tablet |\n| **NDC** | Drug product codes (package level) | |\n| **CCSCM** | Clinical Classifications Software \u2014 Diagnoses | `49` = Diabetes mellitus |\n| **CCSPROC** | CCS \u2014 Procedures | |\n\n**Why code mapping matters in ML:**\n- MIMIC-III uses ICD-9 codes (pre-2015 data), while MIMIC-IV uses ICD-10\n- Vocabularies differ in specificity: ICD-10 has ~70,000 codes vs ICD-9's ~14,000\n- ML models trained on ICD-9 codes cannot directly generalize to ICD-10 datasets without mapping\n- CCS groups 70,000 ICD-10 codes into ~300 categories \u2014 much better for small datasets" }, { "cell_type": "markdown", @@ -27,7 +34,7 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": "# Load ICD-10-CM (Clinical Modification) — the US standard for diagnosis codes\nicd10cm = InnerMap.load(\"ICD10CM\")\n\n# Print statistics\nicd10cm.stat()" + "source": "# Load ICD-10-CM (Clinical Modification) \u2014 the US standard for diagnosis codes\nicd10cm = InnerMap.load(\"ICD10CM\")\n\n# Print statistics\nicd10cm.stat()" }, { "cell_type": "code", @@ -46,7 +53,7 @@ { "cell_type": "markdown", "metadata": {}, - "source": "---\n## Part 2: Diabetes Code Deep Dive\n\nDiabetes mellitus is encoded in ICD-10-CM chapter E08–E13. Here's the full taxonomy:\n\n```\nE08 Diabetes mellitus due to underlying condition\nE09 Drug or chemical induced diabetes mellitus\nE10 Type 1 diabetes mellitus\nE11 Type 2 diabetes mellitus\nE13 Other specified diabetes mellitus\n```\n\nEach category expands into complication subtypes. The E11 (Type 2) branch has **87 billable codes** in 2026 ICD-10-CM — a testament to how granular modern clinical coding is:\n\n```\nE11 Type 2 DM\n├─ E11.00 / E11.01 with hyperosmolarity (without / with coma)\n├─ E11.10 / E11.11 with ketoacidosis (without / with coma)\n├─ E11.2 with kidney complications\n│ ├─ E11.21 with diabetic nephropathy\n│ └─ E11.22 with diabetic chronic kidney disease\n├─ E11.3 with ophthalmic complications\n│ ├─ E11.311 / E11.319 unspecified retinopathy (with / without macular edema)\n│ ├─ E11.321x–E11.359x nonproliferative/proliferative retinopathy\n│ │ (each further split: right eye / left eye / bilateral / unspecified)\n│ └─ E11.36 with diabetic cataract\n├─ E11.4 with neurological complications\n│ ├─ E11.40 with diabetic neuropathy, unspecified\n│ └─ E11.42 with diabetic polyneuropathy\n├─ E11.5 with circulatory complications\n│ ├─ E11.51 with peripheral angiopathy without gangrene\n│ └─ E11.52 with peripheral angiopathy with gangrene\n├─ E11.6 with other specified complications\n│ ├─ E11.65 with hyperglycemia\n│ └─ E11.69 with other specified complication\n├─ E11.8 with unspecified complications\n├─ E11.9 without complications (most common at initial diagnosis)\n└─ E11.A without complications, in remission ← NEW in FY2026\n```\n\n> **2026 addition — `E11.A`:** *\"Type 2 diabetes mellitus without complications in remission\"* — confirmed valid for HIPAA transactions in the FY2026 ICD-10-CM release. This distinguishes patients who have achieved sustained normoglycemia (remission — e.g., post-bariatric surgery or sustained lifestyle intervention) from those still actively managed (E11.9).\n\n> **Note on retinopathy codes:** The E11.3x subcategory is highly specific. Real-world MIMIC data often uses the unspecified form (`E11.319`) because the laterality (left/right/bilateral) was not recorded at the time of billing. ML models typically collapse these to the 3–4 character level (e.g., using CCS or grouping all `E11.3xx` together)." + "source": "---\n## Part 2: Diabetes Code Deep Dive\n\nDiabetes mellitus is encoded in ICD-10-CM chapter E08\u2013E13. Here's the full taxonomy:\n\n```\nE08 Diabetes mellitus due to underlying condition\nE09 Drug or chemical induced diabetes mellitus\nE10 Type 1 diabetes mellitus\nE11 Type 2 diabetes mellitus\nE13 Other specified diabetes mellitus\n```\n\nEach category expands into complication subtypes. The E11 (Type 2) branch has **87 billable codes** in 2026 ICD-10-CM \u2014 a testament to how granular modern clinical coding is:\n\n```\nE11 Type 2 DM\n\u251c\u2500 E11.00 / E11.01 with hyperosmolarity (without / with coma)\n\u251c\u2500 E11.10 / E11.11 with ketoacidosis (without / with coma)\n\u251c\u2500 E11.2 with kidney complications\n\u2502 \u251c\u2500 E11.21 with diabetic nephropathy\n\u2502 \u2514\u2500 E11.22 with diabetic chronic kidney disease\n\u251c\u2500 E11.3 with ophthalmic complications\n\u2502 \u251c\u2500 E11.311 / E11.319 unspecified retinopathy (with / without macular edema)\n\u2502 \u251c\u2500 E11.321x\u2013E11.359x nonproliferative/proliferative retinopathy\n\u2502 \u2502 (each further split: right eye / left eye / bilateral / unspecified)\n\u2502 \u2514\u2500 E11.36 with diabetic cataract\n\u251c\u2500 E11.4 with neurological complications\n\u2502 \u251c\u2500 E11.40 with diabetic neuropathy, unspecified\n\u2502 \u2514\u2500 E11.42 with diabetic polyneuropathy\n\u251c\u2500 E11.5 with circulatory complications\n\u2502 \u251c\u2500 E11.51 with peripheral angiopathy without gangrene\n\u2502 \u2514\u2500 E11.52 with peripheral angiopathy with gangrene\n\u251c\u2500 E11.6 with other specified complications\n\u2502 \u251c\u2500 E11.65 with hyperglycemia\n\u2502 \u2514\u2500 E11.69 with other specified complication\n\u251c\u2500 E11.8 with unspecified complications\n\u251c\u2500 E11.9 without complications (most common at initial diagnosis)\n\u2514\u2500 E11.A without complications, in remission \u2190 NEW in FY2026\n```\n\n> **2026 addition \u2014 `E11.A`:** *\"Type 2 diabetes mellitus without complications in remission\"* \u2014 confirmed valid for HIPAA transactions in the FY2026 ICD-10-CM release. This distinguishes patients who have achieved sustained normoglycemia (remission \u2014 e.g., post-bariatric surgery or sustained lifestyle intervention) from those still actively managed (E11.9).\n\n> **Note on retinopathy codes:** The E11.3x subcategory is highly specific. Real-world MIMIC data often uses the unspecified form (`E11.319`) because the laterality (left/right/bilateral) was not recorded at the time of billing. ML models typically collapse these to the 3\u20134 character level (e.g., using CCS or grouping all `E11.3xx` together)." }, { "cell_type": "code", @@ -74,26 +81,26 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": "# --- 2026 New Code: E11.A ---\n# Confirmed valid for HIPAA transactions in FY2026 ICD-10-CM (verified against live 2026 dataset).\n# Official description: \"Type 2 diabetes mellitus without complications in remission\"\n#\n# Clinical context:\n# E11.9 = T2DM without complications (still being managed / monitored)\n# E11.A = T2DM without complications IN REMISSION (achieved normoglycemia)\n#\n# Remission criteria (ADA 2021 Consensus): HbA1c < 6.5% for at least 3 months\n# without the use of glucose-lowering pharmacotherapy.\n# Common after bariatric surgery or significant sustained lifestyle intervention.\n\nnew_2026_code = \"E11.A\"\nif new_2026_code in icd10cm:\n name = icd10cm.lookup(new_2026_code)\n print(f\"[{new_2026_code}] {name}\")\n print()\n print(\"Contrast with E11.9:\")\n print(f\" [E11.9 ] {icd10cm.lookup('E11.9')}\")\n print(f\" [E11.A ] {name}\")\n print()\n print(\"Clinical significance:\")\n print(\" E11.9 → patient still has diabetes, actively managed\")\n print(\" E11.A → patient has achieved remission (ADA criteria: HbA1c < 6.5% for ≥3 months,\")\n print(\" no glucose-lowering medication)\")\nelse:\n # Fallback if the PyHealth cache predates FY2026\n print(\"E11.A not yet in local ICD10CM cache.\")\n print(\"Official 2026 description: 'Type 2 diabetes mellitus without complications in remission'\")\n print(\"Update the cache with: InnerMap.load('ICD10CM', refresh_cache=True)\")" + "source": "# --- 2026 New Code: E11.A ---\n# Confirmed valid for HIPAA transactions in FY2026 ICD-10-CM (verified against live 2026 dataset).\n# Official description: \"Type 2 diabetes mellitus without complications in remission\"\n#\n# Clinical context:\n# E11.9 = T2DM without complications (still being managed / monitored)\n# E11.A = T2DM without complications IN REMISSION (achieved normoglycemia)\n#\n# Remission criteria (ADA 2021 Consensus): HbA1c < 6.5% for at least 3 months\n# without the use of glucose-lowering pharmacotherapy.\n# Common after bariatric surgery or significant sustained lifestyle intervention.\n\nnew_2026_code = \"E11.A\"\nif new_2026_code in icd10cm:\n name = icd10cm.lookup(new_2026_code)\n print(f\"[{new_2026_code}] {name}\")\n print()\n print(\"Contrast with E11.9:\")\n print(f\" [E11.9 ] {icd10cm.lookup('E11.9')}\")\n print(f\" [E11.A ] {name}\")\n print()\n print(\"Clinical significance:\")\n print(\" E11.9 \u2192 patient still has diabetes, actively managed\")\n print(\" E11.A \u2192 patient has achieved remission (ADA criteria: HbA1c < 6.5% for \u22653 months,\")\n print(\" no glucose-lowering medication)\")\nelse:\n # Fallback if the PyHealth cache predates FY2026\n print(\"E11.A not yet in local ICD10CM cache.\")\n print(\"Official 2026 description: 'Type 2 diabetes mellitus without complications in remission'\")\n print(\"Update the cache with: InnerMap.load('ICD10CM', refresh_cache=True)\")" }, { "cell_type": "markdown", "metadata": {}, - "source": "---\n## Part 3: Hierarchy Exploration\n\nICD codes form a tree where more specific codes are children of broader parent codes. PyHealth exposes this hierarchy via:\n- `get_ancestors(code)` — returns parent codes, ordered from closest to farthest\n- `get_descendants(code)` — returns child codes, ordered from closest to farthest\n\nThis hierarchy is stored internally as a **directed graph** using NetworkX." + "source": "---\n## Part 3: Hierarchy Exploration\n\nICD codes form a tree where more specific codes are children of broader parent codes. PyHealth exposes this hierarchy via:\n- `get_ancestors(code)` \u2014 returns parent codes, ordered from closest to farthest\n- `get_descendants(code)` \u2014 returns child codes, ordered from closest to farthest\n\nThis hierarchy is stored internally as a **directed graph** using NetworkX." }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], - "source": "# --- Ancestor lookup: trace E11.22 up the hierarchy ---\n# E11.22 = Type 2 diabetes mellitus with diabetic chronic kidney disease\ncode = \"E11.22\"\nancestors = icd10cm.get_ancestors(code)\n\nprint(f\"Ancestors of {code} ({icd10cm.lookup(code) if code in icd10cm else 'T2DM with CKD'}):\")\nprint(f\" [{code}] (starting code)\")\nfor anc in ancestors:\n if anc in icd10cm:\n name = icd10cm.lookup(anc)\n print(f\" ↑ [{anc}] {name}\")\n else:\n print(f\" ↑ [{anc}] (category node)\")" + "source": "# --- Ancestor lookup: trace E11.22 up the hierarchy ---\n# E11.22 = Type 2 diabetes mellitus with diabetic chronic kidney disease\ncode = \"E11.22\"\nancestors = icd10cm.get_ancestors(code)\n\nprint(f\"Ancestors of {code} ({icd10cm.lookup(code) if code in icd10cm else 'T2DM with CKD'}):\")\nprint(f\" [{code}] (starting code)\")\nfor anc in ancestors:\n if anc in icd10cm:\n name = icd10cm.lookup(anc)\n print(f\" \u2191 [{anc}] {name}\")\n else:\n print(f\" \u2191 [{anc}] (category node)\")" }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], - "source": "# --- Descendant lookup: find all Type 2 DM subtypes ---\n# The live 2026 ICD-10-CM dataset has 87 billable E11 codes —\n# reflecting the high granularity of modern diabetes coding (retinopathy\n# laterality, ketoacidosis severity, complication type, etc.)\n\nparent_code = \"E11\"\ndescendants = icd10cm.get_descendants(parent_code)\n\nprint(f\"Descendants of {parent_code} (Type 2 Diabetes Mellitus):\")\nprint(f\" Total subtypes in this ICD10CM version: {len(descendants)}\")\nprint(f\" (FY2026 live dataset has 87 billable codes)\")\nprint()\n\n# Show the first 20 for readability — the full list is much longer\nprint(\"First 20 (sorted by code):\")\nfor desc in sorted(descendants)[:20]:\n if desc in icd10cm:\n name = icd10cm.lookup(desc)\n print(f\" [{desc}] {name}\")" + "source": "# --- Descendant lookup: find all Type 2 DM subtypes ---\n# The live 2026 ICD-10-CM dataset has 87 billable E11 codes \u2014\n# reflecting the high granularity of modern diabetes coding (retinopathy\n# laterality, ketoacidosis severity, complication type, etc.)\n\nparent_code = \"E11\"\ndescendants = icd10cm.get_descendants(parent_code)\n\nprint(f\"Descendants of {parent_code} (Type 2 Diabetes Mellitus):\")\nprint(f\" Total subtypes in this ICD10CM version: {len(descendants)}\")\nprint(f\" (FY2026 live dataset has 87 billable codes)\")\nprint()\n\n# Show the first 20 for readability \u2014 the full list is much longer\nprint(\"First 20 (sorted by code):\")\nfor desc in sorted(descendants)[:20]:\n if desc in icd10cm:\n name = icd10cm.lookup(desc)\n print(f\" [{desc}] {name}\")" }, { "cell_type": "code", @@ -119,21 +126,21 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": "# --- ICD-9-CM → ICD-10-CM ---\n# MIMIC-III uses ICD-9; MIMIC-IV uses ICD-10.\n# The GEM (General Equivalence Mappings) crosswalk handles this translation.\n\ncm_9to10 = CrossMap.load(\"ICD9CM\", \"ICD10CM\")\n\n# ICD-9 diabetes codes (legacy MIMIC-III style)\nicd9_diabetes = {\n \"250.00\": \"Diabetes mellitus without mention of complication, type II\",\n \"250.10\": \"Diabetes with ketoacidosis, type II\",\n \"250.40\": \"Diabetes with renal manifestations, type II\",\n \"250.60\": \"Diabetes with neurological manifestations, type II\",\n}\n\nprint(\"ICD-9-CM → ICD-10-CM Diabetes Code Mapping:\")\nprint()\nfor icd9_code, icd9_name in icd9_diabetes.items():\n icd10_codes = cm_9to10.map(icd9_code)\n print(f\" ICD-9 {icd9_code} — {icd9_name}\")\n print(f\" → ICD-10: {icd10_codes}\")\n for c in icd10_codes:\n if c in icd10cm:\n print(f\" [{c}] {icd10cm.lookup(c)}\")\n print()" + "source": "# --- ICD-9-CM \u2192 ICD-10-CM ---\n# MIMIC-III uses ICD-9; MIMIC-IV uses ICD-10.\n# The GEM (General Equivalence Mappings) crosswalk handles this translation.\n\ncm_9to10 = CrossMap.load(\"ICD9CM\", \"ICD10CM\")\n\n# ICD-9 diabetes codes (legacy MIMIC-III style)\nicd9_diabetes = {\n \"250.00\": \"Diabetes mellitus without mention of complication, type II\",\n \"250.10\": \"Diabetes with ketoacidosis, type II\",\n \"250.40\": \"Diabetes with renal manifestations, type II\",\n \"250.60\": \"Diabetes with neurological manifestations, type II\",\n}\n\nprint(\"ICD-9-CM \u2192 ICD-10-CM Diabetes Code Mapping:\")\nprint()\nfor icd9_code, icd9_name in icd9_diabetes.items():\n icd10_codes = cm_9to10.map(icd9_code)\n print(f\" ICD-9 {icd9_code} \u2014 {icd9_name}\")\n print(f\" \u2192 ICD-10: {icd10_codes}\")\n for c in icd10_codes:\n if c in icd10cm:\n print(f\" [{c}] {icd10cm.lookup(c)}\")\n print()" }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], - "source": "# --- ICD-10-CM → CCS (Clinical Classifications Software) ---\n# CCS groups ~70,000 ICD-10 codes into ~300 clinically meaningful categories.\n# CCS category 49 = \"Diabetes mellitus without complication\"\n# CCS category 50 = \"Diabetes mellitus with complications\"\n\ncm_10toCCS = CrossMap.load(\"ICD10CM\", \"CCSCM\")\n\nprint(\"ICD-10-CM → CCS Category Mapping:\")\nprint()\ntype2_sample = [\"E11.9\", \"E11.65\", \"E11.22\", \"E11.42\", \"E10.9\"]\nfor code in type2_sample:\n ccs_cats = cm_10toCCS.map(code)\n icd_name = icd10cm.lookup(code) if code in icd10cm else \"(not found)\"\n print(f\" [{code}] {icd_name}\")\n print(f\" → CCS: {ccs_cats}\")\n print()" + "source": "# --- ICD-10-CM \u2192 CCS (Clinical Classifications Software) ---\n# CCS groups ~70,000 ICD-10 codes into ~300 clinically meaningful categories.\n# CCS category 49 = \"Diabetes mellitus without complication\"\n# CCS category 50 = \"Diabetes mellitus with complications\"\n\ncm_10toCCS = CrossMap.load(\"ICD10CM\", \"CCSCM\")\n\nprint(\"ICD-10-CM \u2192 CCS Category Mapping:\")\nprint()\ntype2_sample = [\"E11.9\", \"E11.65\", \"E11.22\", \"E11.42\", \"E10.9\"]\nfor code in type2_sample:\n ccs_cats = cm_10toCCS.map(code)\n icd_name = icd10cm.lookup(code) if code in icd10cm else \"(not found)\"\n print(f\" [{code}] {icd_name}\")\n print(f\" \u2192 CCS: {ccs_cats}\")\n print()" }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], - "source": "# --- ICD-9-CM → CCS (direct, for MIMIC-III) ---\ncm_9toCCS = CrossMap.load(\"ICD9CM\", \"CCSCM\")\n\nprint(\"ICD-9-CM → CCS (useful for MIMIC-III preprocessing):\")\nprint()\nfor icd9_code in [\"250.00\", \"250.10\", \"250.40\", \"250.60\"]:\n ccs_cats = cm_9toCCS.map(icd9_code)\n print(f\" {icd9_code} → CCS: {ccs_cats}\")" + "source": "# --- ICD-9-CM \u2192 CCS (direct, for MIMIC-III) ---\ncm_9toCCS = CrossMap.load(\"ICD9CM\", \"CCSCM\")\n\nprint(\"ICD-9-CM \u2192 CCS (useful for MIMIC-III preprocessing):\")\nprint()\nfor icd9_code in [\"250.00\", \"250.10\", \"250.40\", \"250.60\"]:\n ccs_cats = cm_9toCCS.map(icd9_code)\n print(f\" {icd9_code} \u2192 CCS: {ccs_cats}\")" }, { "cell_type": "markdown", @@ -152,7 +159,7 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": "# --- ATC (Anatomical Therapeutic Chemical) for drug classification ---\n# ATC hierarchy: Level1 (organ system) → Level2 (main group) → Level3 → Level4 → Level5 (substance)\n# A10BA02 = Metformin\natc = InnerMap.load(\"ATC\")\natc.stat()\n\nmetformin_code = \"A10BA02\"\nif metformin_code in atc:\n print(f\"ATC code {metformin_code}: {atc.lookup(metformin_code)}\")\n ancestors_atc = atc.get_ancestors(metformin_code)\n print(\"Ancestors (drug class hierarchy):\")\n for anc in ancestors_atc:\n if anc in atc:\n print(f\" ↑ [{anc}] {atc.lookup(anc)}\")" + "source": "# --- ATC (Anatomical Therapeutic Chemical) for drug classification ---\n# ATC hierarchy: Level1 (organ system) \u2192 Level2 (main group) \u2192 Level3 \u2192 Level4 \u2192 Level5 (substance)\n# A10BA02 = Metformin\natc = InnerMap.load(\"ATC\")\natc.stat()\n\nmetformin_code = \"A10BA02\"\nif metformin_code in atc:\n print(f\"ATC code {metformin_code}: {atc.lookup(metformin_code)}\")\n ancestors_atc = atc.get_ancestors(metformin_code)\n print(\"Ancestors (drug class hierarchy):\")\n for anc in ancestors_atc:\n if anc in atc:\n print(f\" \u2191 [{anc}] {atc.lookup(anc)}\")" }, { "cell_type": "markdown", @@ -171,14 +178,14 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": "# Pattern 2: Translate a patient's ICD-9 code list to CCS before modeling\n# This dramatically reduces vocabulary size.\n\ndef translate_codes_to_ccs(icd9_codes, crossmap):\n \"\"\"Map a list of ICD-9 codes to CCS categories.\"\"\"\n ccs_codes = []\n for code in icd9_codes:\n mapped = crossmap.map(code)\n ccs_codes.extend(mapped)\n return list(set(ccs_codes)) # deduplicate\n\n# Example patient from MIMIC-III\npatient_icd9_codes = [\"250.00\", \"401.9\", \"428.0\", \"585.3\"]\n# = Type 2 DM, Essential hypertension, Heart failure, CKD stage 3\n\npatient_ccs_codes = translate_codes_to_ccs(patient_icd9_codes, cm_9toCCS)\nprint(\"ICD-9 codes:\", patient_icd9_codes)\nprint(\"CCS codes: \", patient_ccs_codes)\nprint(f\"Vocabulary reduction: {len(icd9cm.graph.nodes)} ICD-9 codes → ~300 CCS categories\")" + "source": "# Pattern 2: Translate a patient's ICD-9 code list to CCS before modeling\n# This dramatically reduces vocabulary size.\n\ndef translate_codes_to_ccs(icd9_codes, crossmap):\n \"\"\"Map a list of ICD-9 codes to CCS categories.\"\"\"\n ccs_codes = []\n for code in icd9_codes:\n mapped = crossmap.map(code)\n ccs_codes.extend(mapped)\n return list(set(ccs_codes)) # deduplicate\n\n# Example patient from MIMIC-III\npatient_icd9_codes = [\"250.00\", \"401.9\", \"428.0\", \"585.3\"]\n# = Type 2 DM, Essential hypertension, Heart failure, CKD stage 3\n\npatient_ccs_codes = translate_codes_to_ccs(patient_icd9_codes, cm_9toCCS)\nprint(\"ICD-9 codes:\", patient_icd9_codes)\nprint(\"CCS codes: \", patient_ccs_codes)\nprint(f\"Vocabulary reduction: {len(icd9cm.graph.nodes)} ICD-9 codes \u2192 ~300 CCS categories\")" }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], - "source": "# Pattern 3: Code validation — remove invalid/unknown codes before training\n# Real EHR data often contains typos, deprecated codes, and free-text entries.\n\nraw_codes_from_ehr = [\"E11.9\", \"E11.99\", \"DIAB\", \"E11.65\", \"999.999\", \"E10.9\"]\nvalid_codes = [c for c in raw_codes_from_ehr if c in icd10cm]\ninvalid_codes = [c for c in raw_codes_from_ehr if c not in icd10cm]\n\nprint(\"Raw codes from EHR:\", raw_codes_from_ehr)\nprint(\"Valid ICD-10-CM codes:\", valid_codes)\nprint(\"Invalid / unknown codes:\", invalid_codes)" + "source": "# Pattern 3: Code validation \u2014 remove invalid/unknown codes before training\n# Real EHR data often contains typos, deprecated codes, and free-text entries.\n\nraw_codes_from_ehr = [\"E11.9\", \"E11.99\", \"DIAB\", \"E11.65\", \"999.999\", \"E10.9\"]\nvalid_codes = [c for c in raw_codes_from_ehr if c in icd10cm]\ninvalid_codes = [c for c in raw_codes_from_ehr if c not in icd10cm]\n\nprint(\"Raw codes from EHR:\", raw_codes_from_ehr)\nprint(\"Valid ICD-10-CM codes:\", valid_codes)\nprint(\"Invalid / unknown codes:\", invalid_codes)" }, { "cell_type": "markdown", diff --git a/examples/tutorials/tutorial_pyhealth_metrics.ipynb b/examples/tutorials/tutorial_pyhealth_metrics.ipynb index bbb5ba657..263d3bf34 100644 --- a/examples/tutorials/tutorial_pyhealth_metrics.ipynb +++ b/examples/tutorials/tutorial_pyhealth_metrics.ipynb @@ -3,7 +3,14 @@ { "cell_type": "markdown", "metadata": {}, - "source": "# PyHealth Metrics Tutorial\n\nThis notebook covers **`pyhealth.metrics`** — a collection of evaluation functions for clinical prediction tasks.\n\nYou will learn:\n- **Binary classification** metrics: AUC-ROC, AUC-PR, F1, ECE, and more\n- **Multiclass classification** metrics: accuracy, macro/micro F1\n- **Multilabel classification** metrics: hamming loss, sample-level AUC\n- **Fairness metrics**: disparate impact and statistical parity difference\n- How to call `trainer.inference()` to get raw predictions for custom evaluation\n\n> **Design note:** All metric functions in PyHealth accept raw numpy arrays, so they can be used independently of the Trainer or with any ML framework.\n\n---" + "source": "# PyHealth Metrics Tutorial\n\nThis notebook covers **`pyhealth.metrics`** \u2014 a collection of evaluation functions for clinical prediction tasks.\n\nYou will learn:\n- **Binary classification** metrics: AUC-ROC, AUC-PR, F1, ECE, and more\n- **Multiclass classification** metrics: accuracy, macro/micro F1\n- **Multilabel classification** metrics: hamming loss, sample-level AUC\n- **Fairness metrics**: disparate impact and statistical parity difference\n- How to call `trainer.inference()` to get raw predictions for custom evaluation\n\n> **Design note:** All metric functions in PyHealth accept raw numpy arrays, so they can be used independently of the Trainer or with any ML framework.\n\n---" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "!pip install pyhealth" }, { "cell_type": "code", @@ -15,7 +22,7 @@ { "cell_type": "markdown", "metadata": {}, - "source": "---\n## Part 1: Binary Classification Metrics\n\nBinary classification is the most common setup in clinical prediction:\n- In-hospital mortality (alive / deceased)\n- Readmission within 30 days (yes / no)\n- Disease onset (positive / negative)\n\n```python\nbinary_metrics_fn(\n y_true: np.ndarray, # shape (n_samples,), values in {0, 1}\n y_prob: np.ndarray, # shape (n_samples,), values in [0, 1]\n metrics: Optional[List[str]] = None, # default: [\"pr_auc\", \"roc_auc\", \"f1\"]\n threshold: float = 0.5, # decision boundary for accuracy/F1/etc.\n)\n```\n\n### Supported metrics\n\n| Metric | Description |\n|--------|-------------|\n| `roc_auc` | Area under ROC curve — threshold-free measure of discrimination |\n| `pr_auc` | Area under Precision-Recall curve — better for imbalanced datasets |\n| `f1` | Harmonic mean of precision and recall |\n| `accuracy` | Fraction of correct predictions |\n| `balanced_accuracy` | Accuracy adjusted for class imbalance |\n| `precision` | TP / (TP + FP) |\n| `recall` | TP / (TP + FN) |\n| `cohen_kappa` | Agreement beyond chance |\n| `jaccard` | Intersection over union for positive class |\n| `ECE` | Expected Calibration Error (calibration quality) |\n| `ECE_adapt` | Adaptive ECE (equal-mass bins) |" + "source": "---\n## Part 1: Binary Classification Metrics\n\nBinary classification is the most common setup in clinical prediction:\n- In-hospital mortality (alive / deceased)\n- Readmission within 30 days (yes / no)\n- Disease onset (positive / negative)\n\n```python\nbinary_metrics_fn(\n y_true: np.ndarray, # shape (n_samples,), values in {0, 1}\n y_prob: np.ndarray, # shape (n_samples,), values in [0, 1]\n metrics: Optional[List[str]] = None, # default: [\"pr_auc\", \"roc_auc\", \"f1\"]\n threshold: float = 0.5, # decision boundary for accuracy/F1/etc.\n)\n```\n\n### Supported metrics\n\n| Metric | Description |\n|--------|-------------|\n| `roc_auc` | Area under ROC curve \u2014 threshold-free measure of discrimination |\n| `pr_auc` | Area under Precision-Recall curve \u2014 better for imbalanced datasets |\n| `f1` | Harmonic mean of precision and recall |\n| `accuracy` | Fraction of correct predictions |\n| `balanced_accuracy` | Accuracy adjusted for class imbalance |\n| `precision` | TP / (TP + FP) |\n| `recall` | TP / (TP + FN) |\n| `cohen_kappa` | Agreement beyond chance |\n| `jaccard` | Intersection over union for positive class |\n| `ECE` | Expected Calibration Error (calibration quality) |\n| `ECE_adapt` | Adaptive ECE (equal-mass bins) |" }, { "cell_type": "code", @@ -53,7 +60,7 @@ { "cell_type": "markdown", "metadata": {}, - "source": "---\n## Part 2: Multiclass Classification Metrics\n\nMulticlass is used when predicting among 3+ mutually exclusive outcomes, for example:\n- Primary discharge diagnosis category (e.g., CCS groups)\n- Length-of-stay bucket (short / medium / long)\n- Triage acuity level (1–5)\n\n```python\nmulticlass_metrics_fn(\n y_true: np.ndarray, # shape (n_samples,), integer class indices\n y_prob: np.ndarray, # shape (n_samples, num_classes), sum to 1\n metrics: Optional[List[str]] = None, # default: [\"accuracy\", \"f1_macro\", \"f1_micro\"]\n)\n```\n\n### Macro vs Micro averaging\n\n| Averaging | Computes | Best for |\n|-----------|----------|----------|\n| `macro` | Mean of per-class metric | When all classes are equally important |\n| `micro` | Global TP/FP/FN counts | When overall performance matters more than per-class balance |\n| `weighted` | Weighted by class support | When you want to account for class frequency |" + "source": "---\n## Part 2: Multiclass Classification Metrics\n\nMulticlass is used when predicting among 3+ mutually exclusive outcomes, for example:\n- Primary discharge diagnosis category (e.g., CCS groups)\n- Length-of-stay bucket (short / medium / long)\n- Triage acuity level (1\u20135)\n\n```python\nmulticlass_metrics_fn(\n y_true: np.ndarray, # shape (n_samples,), integer class indices\n y_prob: np.ndarray, # shape (n_samples, num_classes), sum to 1\n metrics: Optional[List[str]] = None, # default: [\"accuracy\", \"f1_macro\", \"f1_micro\"]\n)\n```\n\n### Macro vs Micro averaging\n\n| Averaging | Computes | Best for |\n|-----------|----------|----------|\n| `macro` | Mean of per-class metric | When all classes are equally important |\n| `micro` | Global TP/FP/FN counts | When overall performance matters more than per-class balance |\n| `weighted` | Weighted by class support | When you want to account for class frequency |" }, { "cell_type": "code", @@ -105,19 +112,19 @@ { "cell_type": "markdown", "metadata": {}, - "source": "### Interpreting multilabel metrics\n\n| Metric | Clinical interpretation |\n|--------|------------------------|\n| `pr_auc_samples` | How well the model ranks drugs for each patient — the primary metric for drug recommendation |\n| `hamming_loss` | Fraction of all (patient, drug) pairs incorrectly classified — penalizes false positives equally |\n| `accuracy` (exact match) | Very strict — 1 only if all drugs are exactly correct |\n| `f1_macro` | Per-drug average F1 — gives equal weight to rare and common drugs |" + "source": "### Interpreting multilabel metrics\n\n| Metric | Clinical interpretation |\n|--------|------------------------|\n| `pr_auc_samples` | How well the model ranks drugs for each patient \u2014 the primary metric for drug recommendation |\n| `hamming_loss` | Fraction of all (patient, drug) pairs incorrectly classified \u2014 penalizes false positives equally |\n| `accuracy` (exact match) | Very strict \u2014 1 only if all drugs are exactly correct |\n| `f1_macro` | Per-drug average F1 \u2014 gives equal weight to rare and common drugs |" }, { "cell_type": "markdown", "metadata": {}, - "source": "---\n## Part 4: Fairness Metrics\n\nFairness metrics assess whether a model's performance is **equitable across subgroups** defined by sensitive attributes (e.g., race, sex, age group). This is crucial in clinical AI to avoid perpetuating historical health disparities.\n\n```python\nfairness_metrics_fn(\n y_true: np.ndarray, # (n_samples,) true labels\n y_prob: np.ndarray, # (n_samples,) predicted probabilities\n sensitive_attributes: np.ndarray, # (n_samples,) 1=protected group, 0=unprotected\n favorable_outcome: int = 1, # which label value is considered positive\n metrics: Optional[List[str]] = None, # default: both below\n threshold: float = 0.5,\n)\n```\n\n### Supported fairness metrics\n\n| Metric | Formula | Interpretation |\n|--------|---------|----------------|\n| `disparate_impact` | P(ŷ=1 | protected) / P(ŷ=1 | unprotected) | Should be ≥ 0.8 (80% rule). 1.0 = perfect parity |\n| `statistical_parity_difference` | P(ŷ=1 | protected) − P(ŷ=1 | unprotected) | Should be close to 0. Negative = protected group predicted positive less often |" + "source": "---\n## Part 4: Fairness Metrics\n\nFairness metrics assess whether a model's performance is **equitable across subgroups** defined by sensitive attributes (e.g., race, sex, age group). This is crucial in clinical AI to avoid perpetuating historical health disparities.\n\n```python\nfairness_metrics_fn(\n y_true: np.ndarray, # (n_samples,) true labels\n y_prob: np.ndarray, # (n_samples,) predicted probabilities\n sensitive_attributes: np.ndarray, # (n_samples,) 1=protected group, 0=unprotected\n favorable_outcome: int = 1, # which label value is considered positive\n metrics: Optional[List[str]] = None, # default: both below\n threshold: float = 0.5,\n)\n```\n\n### Supported fairness metrics\n\n| Metric | Formula | Interpretation |\n|--------|---------|----------------|\n| `disparate_impact` | P(\u0177=1 | protected) / P(\u0177=1 | unprotected) | Should be \u2265 0.8 (80% rule). 1.0 = perfect parity |\n| `statistical_parity_difference` | P(\u0177=1 | protected) \u2212 P(\u0177=1 | unprotected) | Should be close to 0. Negative = protected group predicted positive less often |" }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], - "source": "# --- Simulate a biased binary classifier ---\n# Scenario: predicting ICU readmission\n# Protected group (attr=1): elderly patients (age >= 65)\n# Unprotected group (attr=0): younger patients (age < 65)\n\nn_fair = 400\nnp.random.seed(99)\n\n# 40% of patients are elderly\nsensitive = np.random.binomial(1, 0.40, size=n_fair) # 1 = elderly\n\n# True outcomes — elderly have slightly higher readmission rate\ny_true_fair = np.where(\n sensitive == 1,\n np.random.binomial(1, 0.35, size=n_fair), # elderly: 35% readmission\n np.random.binomial(1, 0.25, size=n_fair), # younger: 25% readmission\n).astype(np.float32)\n\n# Biased model: under-predicts readmission for elderly\n# (e.g., trained on historical data that under-tested elderly patients)\ny_prob_fair = np.where(\n sensitive == 1,\n np.random.beta(2, 5, size=n_fair), # lower probs for elderly (biased)\n np.random.beta(3, 4, size=n_fair), # higher probs for younger\n).astype(np.float32)\n\nprint(f\"Patients: {n_fair}\")\nprint(f\"Protected (elderly) group: {sensitive.sum()} ({sensitive.mean():.1%})\")\nprint(f\"True readmission rate — elderly: {y_true_fair[sensitive==1].mean():.1%}\")\nprint(f\"True readmission rate — younger: {y_true_fair[sensitive==0].mean():.1%}\")\nprint(f\"Mean predicted prob — elderly: {y_prob_fair[sensitive==1].mean():.3f}\")\nprint(f\"Mean predicted prob — younger: {y_prob_fair[sensitive==0].mean():.3f}\")" + "source": "# --- Simulate a biased binary classifier ---\n# Scenario: predicting ICU readmission\n# Protected group (attr=1): elderly patients (age >= 65)\n# Unprotected group (attr=0): younger patients (age < 65)\n\nn_fair = 400\nnp.random.seed(99)\n\n# 40% of patients are elderly\nsensitive = np.random.binomial(1, 0.40, size=n_fair) # 1 = elderly\n\n# True outcomes \u2014 elderly have slightly higher readmission rate\ny_true_fair = np.where(\n sensitive == 1,\n np.random.binomial(1, 0.35, size=n_fair), # elderly: 35% readmission\n np.random.binomial(1, 0.25, size=n_fair), # younger: 25% readmission\n).astype(np.float32)\n\n# Biased model: under-predicts readmission for elderly\n# (e.g., trained on historical data that under-tested elderly patients)\ny_prob_fair = np.where(\n sensitive == 1,\n np.random.beta(2, 5, size=n_fair), # lower probs for elderly (biased)\n np.random.beta(3, 4, size=n_fair), # higher probs for younger\n).astype(np.float32)\n\nprint(f\"Patients: {n_fair}\")\nprint(f\"Protected (elderly) group: {sensitive.sum()} ({sensitive.mean():.1%})\")\nprint(f\"True readmission rate \u2014 elderly: {y_true_fair[sensitive==1].mean():.1%}\")\nprint(f\"True readmission rate \u2014 younger: {y_true_fair[sensitive==0].mean():.1%}\")\nprint(f\"Mean predicted prob \u2014 elderly: {y_prob_fair[sensitive==1].mean():.3f}\")\nprint(f\"Mean predicted prob \u2014 younger: {y_prob_fair[sensitive==0].mean():.3f}\")" }, { "cell_type": "code", @@ -131,7 +138,7 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": "# --- Interpret the results ---\ndi = fairness_results[\"disparate_impact\"]\nspd = fairness_results[\"statistical_parity_difference\"]\n\nprint(\"Interpretation:\")\nprint()\nprint(f\" Disparate Impact = {di:.4f}\")\nif di >= 0.8:\n print(\" ✓ Above 0.8 threshold — model passes the 80% rule\")\nelse:\n print(\" ✗ Below 0.8 threshold — model fails the 80% rule (legally significant in US)\")\nprint()\nprint(f\" Statistical Parity Difference = {spd:.4f}\")\nif abs(spd) < 0.05:\n print(\" ✓ Close to 0 — model predictions are roughly equally distributed across groups\")\nelif spd < 0:\n print(f\" ✗ Negative ({spd:.4f}): protected group is predicted positive {abs(spd):.1%} less often\")\nelse:\n print(f\" Protected group is predicted positive {spd:.1%} more often\")" + "source": "# --- Interpret the results ---\ndi = fairness_results[\"disparate_impact\"]\nspd = fairness_results[\"statistical_parity_difference\"]\n\nprint(\"Interpretation:\")\nprint()\nprint(f\" Disparate Impact = {di:.4f}\")\nif di >= 0.8:\n print(\" \u2713 Above 0.8 threshold \u2014 model passes the 80% rule\")\nelse:\n print(\" \u2717 Below 0.8 threshold \u2014 model fails the 80% rule (legally significant in US)\")\nprint()\nprint(f\" Statistical Parity Difference = {spd:.4f}\")\nif abs(spd) < 0.05:\n print(\" \u2713 Close to 0 \u2014 model predictions are roughly equally distributed across groups\")\nelif spd < 0:\n print(f\" \u2717 Negative ({spd:.4f}): protected group is predicted positive {abs(spd):.1%} less often\")\nelse:\n print(f\" Protected group is predicted positive {spd:.1%} more often\")" }, { "cell_type": "code", @@ -143,12 +150,12 @@ { "cell_type": "markdown", "metadata": {}, - "source": "### Clinical context for fairness metrics\n\nIn the healthcare domain, fairness is particularly important because:\n\n1. **Historical bias in training data:** If a hospital historically provided less aggressive treatment to a subgroup, the training labels may reflect these disparities — and the model will learn to replicate them.\n\n2. **Feature proxies:** Features like ZIP code, insurance type, or language can serve as proxies for race/ethnicity. A model may be technically race-unaware yet still exhibit disparate impact.\n\n3. **Regulatory considerations:** The US 2021 Algorithmic Accountability Act and emerging EU AI Act both require documentation of bias audits for high-risk AI (which includes clinical decision support).\n\n**PyHealth's approach:** Report fairness metrics alongside clinical performance metrics. A model with excellent AUC-ROC but poor disparate impact is not ready for deployment." + "source": "### Clinical context for fairness metrics\n\nIn the healthcare domain, fairness is particularly important because:\n\n1. **Historical bias in training data:** If a hospital historically provided less aggressive treatment to a subgroup, the training labels may reflect these disparities \u2014 and the model will learn to replicate them.\n\n2. **Feature proxies:** Features like ZIP code, insurance type, or language can serve as proxies for race/ethnicity. A model may be technically race-unaware yet still exhibit disparate impact.\n\n3. **Regulatory considerations:** The US 2021 Algorithmic Accountability Act and emerging EU AI Act both require documentation of bias audits for high-risk AI (which includes clinical decision support).\n\n**PyHealth's approach:** Report fairness metrics alongside clinical performance metrics. A model with excellent AUC-ROC but poor disparate impact is not ready for deployment." }, { "cell_type": "markdown", "metadata": {}, - "source": "---\n## Part 5: Integration with Trainer — Getting Raw Predictions\n\nIn practice you'll get predictions from a trained model and then compute any of the above metrics. The Trainer provides two ways:\n\n**Option A — `trainer.evaluate(loader)`:** Returns a dict of metrics using the model's default metric function. Convenient, but limited to the default metrics.\n\n**Option B — `trainer.inference(loader)`:** Returns raw numpy arrays `(y_true, y_prob, loss)`. Use this when you want to compute custom metrics, fairness analysis, or calibration plots.\n\n```python\n# After training (see tutorial_pyhealth_trainer.ipynb)\ny_true, y_prob, loss = trainer.inference(test_loader)\n\n# Now compute any metric combination you want\nbinary_metrics_fn(y_true, y_prob, metrics=[\"roc_auc\", \"pr_auc\", \"ECE\"])\nfairness_metrics_fn(y_true, y_prob, sensitive_attributes=race_labels)\n```\n\nThe decoupling of **inference** from **metric computation** means you can:\n- Cache the raw predictions and recompute metrics without re-running the model\n- Apply post-hoc calibration (Platt scaling, temperature scaling) and re-evaluate\n- Run bootstrap confidence intervals on any metric" + "source": "---\n## Part 5: Integration with Trainer \u2014 Getting Raw Predictions\n\nIn practice you'll get predictions from a trained model and then compute any of the above metrics. The Trainer provides two ways:\n\n**Option A \u2014 `trainer.evaluate(loader)`:** Returns a dict of metrics using the model's default metric function. Convenient, but limited to the default metrics.\n\n**Option B \u2014 `trainer.inference(loader)`:** Returns raw numpy arrays `(y_true, y_prob, loss)`. Use this when you want to compute custom metrics, fairness analysis, or calibration plots.\n\n```python\n# After training (see tutorial_pyhealth_trainer.ipynb)\ny_true, y_prob, loss = trainer.inference(test_loader)\n\n# Now compute any metric combination you want\nbinary_metrics_fn(y_true, y_prob, metrics=[\"roc_auc\", \"pr_auc\", \"ECE\"])\nfairness_metrics_fn(y_true, y_prob, sensitive_attributes=race_labels)\n```\n\nThe decoupling of **inference** from **metric computation** means you can:\n- Cache the raw predictions and recompute metrics without re-running the model\n- Apply post-hoc calibration (Platt scaling, temperature scaling) and re-evaluate\n- Run bootstrap confidence intervals on any metric" }, { "cell_type": "markdown", @@ -169,4 +176,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} +} \ No newline at end of file diff --git a/examples/tutorials/tutorial_pyhealth_model.ipynb b/examples/tutorials/tutorial_pyhealth_model.ipynb index 71a5affd2..8e4e0ba7c 100644 --- a/examples/tutorials/tutorial_pyhealth_model.ipynb +++ b/examples/tutorials/tutorial_pyhealth_model.ipynb @@ -3,7 +3,14 @@ { "cell_type": "markdown", "metadata": {}, - "source": "# PyHealth Models Tutorial — RNN Deep Dive\n\nThis notebook walks through **`pyhealth.models`** from first principles.\n\nYou will learn:\n- The **`BaseModel`** contract every PyHealth model must satisfy\n- How to create synthetic test data with **`create_sample_dataset()`**\n- The internal architecture of **`RNNLayer`** and **`RNN`** — reading the source code line by line\n- How to run forward and backward passes\n- How **`MultimodalRNN`** handles mixed input types\n\n---" + "source": "# PyHealth Models Tutorial \u2014 RNN Deep Dive\n\nThis notebook walks through **`pyhealth.models`** from first principles.\n\nYou will learn:\n- The **`BaseModel`** contract every PyHealth model must satisfy\n- How to create synthetic test data with **`create_sample_dataset()`**\n- The internal architecture of **`RNNLayer`** and **`RNN`** \u2014 reading the source code line by line\n- How to run forward and backward passes\n- How **`MultimodalRNN`** handles mixed input types\n\n---" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "!pip install pyhealth" }, { "cell_type": "code", @@ -15,19 +22,19 @@ { "cell_type": "markdown", "metadata": {}, - "source": "---\n## Part 1: The `BaseModel` Contract\n\nEvery PyHealth model inherits from `BaseModel`, which itself inherits from both `ABC` (abstract base class) and `nn.Module` (PyTorch module).\n\n```python\nclass BaseModel(ABC, nn.Module):\n def __init__(self, dataset: SampleDataset):\n ...\n self.feature_keys = list(dataset.input_schema.keys())\n self.label_keys = list(dataset.output_schema.keys())\n\n def forward(self, **kwargs) -> dict[str, torch.Tensor]:\n # Subclasses implement this\n raise NotImplementedError\n```\n\n### What `BaseModel` provides\n\n| Method | Description |\n|--------|-------------|\n| `device` property | Returns the device the model lives on (CPU / CUDA) |\n| `get_output_size()` | Returns the FC head size (1 for binary, num_classes for multiclass) |\n| `get_loss_function()` | Returns the appropriate loss: BCE for binary/multilabel, CrossEntropy for multiclass |\n| `prepare_y_prob(logits)` | Applies sigmoid (binary/multilabel) or softmax (multiclass) to produce probabilities |\n\n### The `forward()` output contract\n\nEvery `forward(**kwargs)` call returns a dict:\n```python\n{\n \"loss\": torch.Tensor, # scalar — backpropagatable\n \"y_prob\": torch.Tensor, # predicted probabilities\n \"y_true\": torch.Tensor, # ground truth labels\n \"logit\": torch.Tensor, # raw logits before activation\n \"embed\": torch.Tensor, # (optional) patient embeddings, only if embed=True in kwargs\n}\n```\n\nThis uniform interface means any model can plug into the `Trainer` without modification." + "source": "---\n## Part 1: The `BaseModel` Contract\n\nEvery PyHealth model inherits from `BaseModel`, which itself inherits from both `ABC` (abstract base class) and `nn.Module` (PyTorch module).\n\n```python\nclass BaseModel(ABC, nn.Module):\n def __init__(self, dataset: SampleDataset):\n ...\n self.feature_keys = list(dataset.input_schema.keys())\n self.label_keys = list(dataset.output_schema.keys())\n\n def forward(self, **kwargs) -> dict[str, torch.Tensor]:\n # Subclasses implement this\n raise NotImplementedError\n```\n\n### What `BaseModel` provides\n\n| Method | Description |\n|--------|-------------|\n| `device` property | Returns the device the model lives on (CPU / CUDA) |\n| `get_output_size()` | Returns the FC head size (1 for binary, num_classes for multiclass) |\n| `get_loss_function()` | Returns the appropriate loss: BCE for binary/multilabel, CrossEntropy for multiclass |\n| `prepare_y_prob(logits)` | Applies sigmoid (binary/multilabel) or softmax (multiclass) to produce probabilities |\n\n### The `forward()` output contract\n\nEvery `forward(**kwargs)` call returns a dict:\n```python\n{\n \"loss\": torch.Tensor, # scalar \u2014 backpropagatable\n \"y_prob\": torch.Tensor, # predicted probabilities\n \"y_true\": torch.Tensor, # ground truth labels\n \"logit\": torch.Tensor, # raw logits before activation\n \"embed\": torch.Tensor, # (optional) patient embeddings, only if embed=True in kwargs\n}\n```\n\nThis uniform interface means any model can plug into the `Trainer` without modification." }, { "cell_type": "markdown", "metadata": {}, - "source": "---\n## Part 2: Creating Test Data with `create_sample_dataset()`\n\n`create_sample_dataset()` is a convenience helper that:\n1. Accepts a list of raw sample dicts\n2. Fits tokenizers / processors based on the provided schema\n3. Returns an `InMemorySampleDataset` (no disk I/O) ready for model instantiation\n\nThis is exactly how PyHealth's own unit tests create datasets — no MIMIC or real EHR data required." + "source": "---\n## Part 2: Creating Test Data with `create_sample_dataset()`\n\n`create_sample_dataset()` is a convenience helper that:\n1. Accepts a list of raw sample dicts\n2. Fits tokenizers / processors based on the provided schema\n3. Returns an `InMemorySampleDataset` (no disk I/O) ready for model instantiation\n\nThis is exactly how PyHealth's own unit tests create datasets \u2014 no MIMIC or real EHR data required." }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], - "source": "# --- Define raw samples ---\n# Each sample is a dict. Keys must match the schemas below.\nsamples = [\n {\n \"patient_id\": \"patient-0\",\n \"visit_id\": \"visit-0\",\n \"conditions\": [\"E11.9\", \"E11.65\", \"I10\"], # Type 2 DM, hypertension\n \"procedures\": [\"99213\", \"36415\"], # Office visit, blood draw\n \"label\": 1,\n },\n {\n \"patient_id\": \"patient-1\",\n \"visit_id\": \"visit-1\",\n \"conditions\": [\"E11.9\"],\n \"procedures\": [\"99213\"],\n \"label\": 0,\n },\n {\n \"patient_id\": \"patient-2\",\n \"visit_id\": \"visit-2\",\n \"conditions\": [\"E11.22\", \"N18.3\", \"E11.42\"], # DM with CKD and neuropathy\n \"procedures\": [\"99213\", \"86900\", \"81001\"],\n \"label\": 1,\n },\n {\n \"patient_id\": \"patient-3\",\n \"visit_id\": \"visit-3\",\n \"conditions\": [\"E11.65\", \"E11.9\"],\n \"procedures\": [\"36415\"],\n \"label\": 0,\n },\n]\n\n# --- Define schemas ---\n# Processor aliases:\n# 'sequence' → SequenceProcessor (tokenizes a list of codes to integer IDs)\n# 'multi_hot' → MultiHotProcessor (binary vector over vocabulary)\n# 'timeseries' → TimeseriesProcessor (continuous time series)\n# 'tensor' → TensorProcessor (fixed-size dense vector)\n# 'binary' → BinaryLabelProcessor (0 or 1)\ninput_schema = {\"conditions\": \"sequence\", \"procedures\": \"sequence\"}\noutput_schema = {\"label\": \"binary\"}\n\n# --- Create the dataset ---\ndataset = create_sample_dataset(\n samples=samples,\n input_schema=input_schema,\n output_schema=output_schema,\n dataset_name=\"diabetes_demo\",\n task_name=\"mortality\",\n)\n\nprint(\"Dataset type: \", type(dataset).__name__)\nprint(\"Input schema: \", dataset.input_schema)\nprint(\"Output schema: \", dataset.output_schema)\nprint(\"Num samples: \", len(dataset))" + "source": "# --- Define raw samples ---\n# Each sample is a dict. Keys must match the schemas below.\nsamples = [\n {\n \"patient_id\": \"patient-0\",\n \"visit_id\": \"visit-0\",\n \"conditions\": [\"E11.9\", \"E11.65\", \"I10\"], # Type 2 DM, hypertension\n \"procedures\": [\"99213\", \"36415\"], # Office visit, blood draw\n \"label\": 1,\n },\n {\n \"patient_id\": \"patient-1\",\n \"visit_id\": \"visit-1\",\n \"conditions\": [\"E11.9\"],\n \"procedures\": [\"99213\"],\n \"label\": 0,\n },\n {\n \"patient_id\": \"patient-2\",\n \"visit_id\": \"visit-2\",\n \"conditions\": [\"E11.22\", \"N18.3\", \"E11.42\"], # DM with CKD and neuropathy\n \"procedures\": [\"99213\", \"86900\", \"81001\"],\n \"label\": 1,\n },\n {\n \"patient_id\": \"patient-3\",\n \"visit_id\": \"visit-3\",\n \"conditions\": [\"E11.65\", \"E11.9\"],\n \"procedures\": [\"36415\"],\n \"label\": 0,\n },\n]\n\n# --- Define schemas ---\n# Processor aliases:\n# 'sequence' \u2192 SequenceProcessor (tokenizes a list of codes to integer IDs)\n# 'multi_hot' \u2192 MultiHotProcessor (binary vector over vocabulary)\n# 'timeseries' \u2192 TimeseriesProcessor (continuous time series)\n# 'tensor' \u2192 TensorProcessor (fixed-size dense vector)\n# 'binary' \u2192 BinaryLabelProcessor (0 or 1)\ninput_schema = {\"conditions\": \"sequence\", \"procedures\": \"sequence\"}\noutput_schema = {\"label\": \"binary\"}\n\n# --- Create the dataset ---\ndataset = create_sample_dataset(\n samples=samples,\n input_schema=input_schema,\n output_schema=output_schema,\n dataset_name=\"diabetes_demo\",\n task_name=\"mortality\",\n)\n\nprint(\"Dataset type: \", type(dataset).__name__)\nprint(\"Input schema: \", dataset.input_schema)\nprint(\"Output schema: \", dataset.output_schema)\nprint(\"Num samples: \", len(dataset))" }, { "cell_type": "code", @@ -39,7 +46,7 @@ { "cell_type": "markdown", "metadata": {}, - "source": "---\n## Part 3: `RNNLayer` Architecture\n\n`RNNLayer` is a low-level building block that wraps PyTorch's native RNN/LSTM/GRU with:\n- **Dropout** before the recurrent computation\n- **Variable-length sequence support** via pack/pad operations\n- **Bidirectional support** with a down-projection to maintain hidden_size\n\n```python\nclass RNNLayer(nn.Module):\n\n def __init__(\n self,\n input_size: int,\n hidden_size: int,\n rnn_type: str = \"GRU\", # one of \"RNN\", \"LSTM\", \"GRU\"\n num_layers: int = 1,\n dropout: float = 0.5,\n bidirectional: bool = False,\n ):\n ...\n self.dropout_layer = nn.Dropout(dropout)\n rnn_module = getattr(nn, rnn_type) # nn.GRU, nn.LSTM, or nn.RNN\n self.rnn = rnn_module(\n input_size, hidden_size,\n num_layers=num_layers,\n dropout=dropout if num_layers > 1 else 0,\n bidirectional=bidirectional,\n batch_first=True,\n )\n if bidirectional:\n self.down_projection = nn.Linear(hidden_size * 2, hidden_size)\n\n def forward(\n self,\n x: torch.Tensor, # shape: (batch, seq_len, input_size)\n mask: Optional[torch.Tensor] = None, # shape: (batch, seq_len), 1=valid\n ) -> Tuple[torch.Tensor, torch.Tensor]:\n x = self.dropout_layer(x)\n\n # Compute actual sequence lengths from mask (or assume full sequences)\n lengths = torch.sum(mask.int(), dim=-1).cpu() if mask is not None else ...\n lengths = torch.clamp(lengths, min=1) # avoid zero-length sequences\n\n # Pack → RNN → Unpack (cuDNN optimization for variable-length batches)\n x = rnn_utils.pack_padded_sequence(x, lengths, batch_first=True, enforce_sorted=False)\n outputs, _ = self.rnn(x)\n outputs, _ = rnn_utils.pad_packed_sequence(outputs, batch_first=True)\n\n # Extract final hidden state at each sample's actual last position\n last_outputs = outputs[torch.arange(batch_size), (lengths - 1), :]\n\n if self.bidirectional:\n # Concatenate forward/backward final states, then project back to hidden_size\n last_outputs = self.down_projection(last_outputs)\n\n return outputs, last_outputs\n # outputs: (batch, seq_len, hidden_size) — all time steps\n # last_outputs: (batch, hidden_size) — final hidden state\n```\n\n### Key design decisions in `RNNLayer`\n\n1. **`pack_padded_sequence`** — tells cuDNN to skip padding positions, which is both faster and numerically correct. Without this, the RNN would process padding tokens and corrupt the final hidden state.\n\n2. **`lengths = clamp(lengths, min=1)`** — `pack_padded_sequence` raises an error for length-0 sequences (empty visits). Clamping to 1 is a safe fallback.\n\n3. **Bidirectional down-projection** — bidirectional outputs have `2 × hidden_size` channels; the linear layer projects back to `hidden_size` so downstream code sees a consistent dimension regardless of directionality." + "source": "---\n## Part 3: `RNNLayer` Architecture\n\n`RNNLayer` is a low-level building block that wraps PyTorch's native RNN/LSTM/GRU with:\n- **Dropout** before the recurrent computation\n- **Variable-length sequence support** via pack/pad operations\n- **Bidirectional support** with a down-projection to maintain hidden_size\n\n```python\nclass RNNLayer(nn.Module):\n\n def __init__(\n self,\n input_size: int,\n hidden_size: int,\n rnn_type: str = \"GRU\", # one of \"RNN\", \"LSTM\", \"GRU\"\n num_layers: int = 1,\n dropout: float = 0.5,\n bidirectional: bool = False,\n ):\n ...\n self.dropout_layer = nn.Dropout(dropout)\n rnn_module = getattr(nn, rnn_type) # nn.GRU, nn.LSTM, or nn.RNN\n self.rnn = rnn_module(\n input_size, hidden_size,\n num_layers=num_layers,\n dropout=dropout if num_layers > 1 else 0,\n bidirectional=bidirectional,\n batch_first=True,\n )\n if bidirectional:\n self.down_projection = nn.Linear(hidden_size * 2, hidden_size)\n\n def forward(\n self,\n x: torch.Tensor, # shape: (batch, seq_len, input_size)\n mask: Optional[torch.Tensor] = None, # shape: (batch, seq_len), 1=valid\n ) -> Tuple[torch.Tensor, torch.Tensor]:\n x = self.dropout_layer(x)\n\n # Compute actual sequence lengths from mask (or assume full sequences)\n lengths = torch.sum(mask.int(), dim=-1).cpu() if mask is not None else ...\n lengths = torch.clamp(lengths, min=1) # avoid zero-length sequences\n\n # Pack \u2192 RNN \u2192 Unpack (cuDNN optimization for variable-length batches)\n x = rnn_utils.pack_padded_sequence(x, lengths, batch_first=True, enforce_sorted=False)\n outputs, _ = self.rnn(x)\n outputs, _ = rnn_utils.pad_packed_sequence(outputs, batch_first=True)\n\n # Extract final hidden state at each sample's actual last position\n last_outputs = outputs[torch.arange(batch_size), (lengths - 1), :]\n\n if self.bidirectional:\n # Concatenate forward/backward final states, then project back to hidden_size\n last_outputs = self.down_projection(last_outputs)\n\n return outputs, last_outputs\n # outputs: (batch, seq_len, hidden_size) \u2014 all time steps\n # last_outputs: (batch, hidden_size) \u2014 final hidden state\n```\n\n### Key design decisions in `RNNLayer`\n\n1. **`pack_padded_sequence`** \u2014 tells cuDNN to skip padding positions, which is both faster and numerically correct. Without this, the RNN would process padding tokens and corrupt the final hidden state.\n\n2. **`lengths = clamp(lengths, min=1)`** \u2014 `pack_padded_sequence` raises an error for length-0 sequences (empty visits). Clamping to 1 is a safe fallback.\n\n3. **Bidirectional down-projection** \u2014 bidirectional outputs have `2 \u00d7 hidden_size` channels; the linear layer projects back to `hidden_size` so downstream code sees a consistent dimension regardless of directionality." }, { "cell_type": "code", @@ -58,7 +65,7 @@ { "cell_type": "markdown", "metadata": {}, - "source": "---\n## Part 4: `RNN` Model Architecture\n\nThe `RNN` class sits one level above `RNNLayer`. It applies **separate** embedding and RNN layers for each input feature, then concatenates the final hidden states and passes them through a shared fully-connected head.\n\n```python\nclass RNN(BaseModel):\n\n def __init__(\n self,\n dataset: SampleDataset,\n embedding_dim: int = 128,\n hidden_dim: int = 128,\n **kwargs # forwarded to RNNLayer (rnn_type, num_layers, dropout, ...)\n ):\n super().__init__(dataset=dataset)\n\n # One embedding model shared across all features\n self.embedding_model = EmbeddingModel(dataset, embedding_dim)\n\n # One independent RNN layer per feature key\n self.rnn = nn.ModuleDict()\n for feature_key in self.feature_keys:\n self.rnn[feature_key] = RNNLayer(\n input_size=embedding_dim, hidden_size=hidden_dim, **kwargs\n )\n\n # Final FC: concatenation of all hidden states → output\n output_size = self.get_output_size() # 1 for binary\n self.fc = nn.Linear(len(self.feature_keys) * hidden_dim, output_size)\n```\n\n### `RNN.forward()` step by step\n\n```python\n def forward(self, **kwargs):\n patient_emb = []\n\n # 1. Extract value tensors and masks from each feature\n for feature_key in self.feature_keys:\n # Feature is a tuple: (value_tensor, mask_tensor, ...)\n # Schema tells us which tuple index is 'value' and which is 'mask'\n inputs[feature_key] = value\n masks[feature_key] = mask\n\n # 2. Embed all features (tokenized codes → dense vectors)\n embedded = self.embedding_model(inputs, masks=masks)\n # embedded[key] shape:\n # SequenceProcessor → (B, seq_len, D)\n # NestedSequenceProcessor → (B, num_visits, num_codes, D)\n # TimeseriesProcessor → (B, T, D)\n\n # 3. Handle dimensionality:\n for feature_key in self.feature_keys:\n x = embedded[feature_key]\n if x.dim() == 4: # nested: (B, V, C, D) → sum-pool codes → (B, V, D)\n x = x.sum(dim=2)\n elif x.dim() == 2: # static single value: (B, D) → (B, 1, D)\n x = x.unsqueeze(1)\n # Now x is always (B, T, D)\n\n # 4. Run per-feature RNN, take final hidden state\n _, x = self.rnn[feature_key](x, mask)\n patient_emb.append(x) # each x: (B, hidden_dim)\n\n # 5. Concatenate all features' hidden states\n patient_emb = torch.cat(patient_emb, dim=1) # (B, num_features * hidden_dim)\n\n # 6. Project to label space\n logits = self.fc(patient_emb)\n\n # 7. Compute loss, probabilities\n y_true = kwargs[self.label_key]\n loss = self.get_loss_function()(logits, y_true)\n y_prob = self.prepare_y_prob(logits)\n\n return {\"loss\": loss, \"y_prob\": y_prob, \"y_true\": y_true, \"logit\": logits}\n```\n\n**Why separate RNNs per feature?** Different clinical features have different sequence semantics. Diagnosis codes have a different distributional structure than procedure codes. Separate RNNs let each feature develop its own specialized temporal representation before combining." + "source": "---\n## Part 4: `RNN` Model Architecture\n\nThe `RNN` class sits one level above `RNNLayer`. It applies **separate** embedding and RNN layers for each input feature, then concatenates the final hidden states and passes them through a shared fully-connected head.\n\n```python\nclass RNN(BaseModel):\n\n def __init__(\n self,\n dataset: SampleDataset,\n embedding_dim: int = 128,\n hidden_dim: int = 128,\n **kwargs # forwarded to RNNLayer (rnn_type, num_layers, dropout, ...)\n ):\n super().__init__(dataset=dataset)\n\n # One embedding model shared across all features\n self.embedding_model = EmbeddingModel(dataset, embedding_dim)\n\n # One independent RNN layer per feature key\n self.rnn = nn.ModuleDict()\n for feature_key in self.feature_keys:\n self.rnn[feature_key] = RNNLayer(\n input_size=embedding_dim, hidden_size=hidden_dim, **kwargs\n )\n\n # Final FC: concatenation of all hidden states \u2192 output\n output_size = self.get_output_size() # 1 for binary\n self.fc = nn.Linear(len(self.feature_keys) * hidden_dim, output_size)\n```\n\n### `RNN.forward()` step by step\n\n```python\n def forward(self, **kwargs):\n patient_emb = []\n\n # 1. Extract value tensors and masks from each feature\n for feature_key in self.feature_keys:\n # Feature is a tuple: (value_tensor, mask_tensor, ...)\n # Schema tells us which tuple index is 'value' and which is 'mask'\n inputs[feature_key] = value\n masks[feature_key] = mask\n\n # 2. Embed all features (tokenized codes \u2192 dense vectors)\n embedded = self.embedding_model(inputs, masks=masks)\n # embedded[key] shape:\n # SequenceProcessor \u2192 (B, seq_len, D)\n # NestedSequenceProcessor \u2192 (B, num_visits, num_codes, D)\n # TimeseriesProcessor \u2192 (B, T, D)\n\n # 3. Handle dimensionality:\n for feature_key in self.feature_keys:\n x = embedded[feature_key]\n if x.dim() == 4: # nested: (B, V, C, D) \u2192 sum-pool codes \u2192 (B, V, D)\n x = x.sum(dim=2)\n elif x.dim() == 2: # static single value: (B, D) \u2192 (B, 1, D)\n x = x.unsqueeze(1)\n # Now x is always (B, T, D)\n\n # 4. Run per-feature RNN, take final hidden state\n _, x = self.rnn[feature_key](x, mask)\n patient_emb.append(x) # each x: (B, hidden_dim)\n\n # 5. Concatenate all features' hidden states\n patient_emb = torch.cat(patient_emb, dim=1) # (B, num_features * hidden_dim)\n\n # 6. Project to label space\n logits = self.fc(patient_emb)\n\n # 7. Compute loss, probabilities\n y_true = kwargs[self.label_key]\n loss = self.get_loss_function()(logits, y_true)\n y_prob = self.prepare_y_prob(logits)\n\n return {\"loss\": loss, \"y_prob\": y_prob, \"y_true\": y_true, \"logit\": logits}\n```\n\n**Why separate RNNs per feature?** Different clinical features have different sequence semantics. Diagnosis codes have a different distributional structure than procedure codes. Separate RNNs let each feature develop its own specialized temporal representation before combining." }, { "cell_type": "markdown", @@ -91,7 +98,7 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": "# --- Requesting patient embeddings ---\n# Pass embed=True to get the concatenated hidden state before the FC layer\nwith torch.no_grad():\n output_with_embed = model(**data_batch, embed=True)\n\nprint(\"With embed=True:\")\nprint(f\" embed shape: {output_with_embed['embed'].shape}\")\nprint(f\" Expected: (batch_size=2, num_features={len(model.feature_keys)} × hidden_dim={model.hidden_dim} = {len(model.feature_keys) * model.hidden_dim})\")\nprint()\nprint(\"These embeddings can be used for:\")\nprint(\" - Patient similarity search\")\nprint(\" - Visualization with UMAP / t-SNE\")\nprint(\" - Downstream tasks (transfer learning)\")" + "source": "# --- Requesting patient embeddings ---\n# Pass embed=True to get the concatenated hidden state before the FC layer\nwith torch.no_grad():\n output_with_embed = model(**data_batch, embed=True)\n\nprint(\"With embed=True:\")\nprint(f\" embed shape: {output_with_embed['embed'].shape}\")\nprint(f\" Expected: (batch_size=2, num_features={len(model.feature_keys)} \u00d7 hidden_dim={model.hidden_dim} = {len(model.feature_keys) * model.hidden_dim})\")\nprint()\nprint(\"These embeddings can be used for:\")\nprint(\" - Patient similarity search\")\nprint(\" - Visualization with UMAP / t-SNE\")\nprint(\" - Downstream tasks (transfer learning)\")" }, { "cell_type": "code", @@ -103,7 +110,7 @@ { "cell_type": "markdown", "metadata": {}, - "source": "---\n## Part 6: `MultimodalRNN` — Mixed Input Modalities\n\n`MultimodalRNN` extends `RNN` to handle **heterogeneous inputs**. It automatically classifies each feature into:\n\n- **Sequential** (gets its own `RNNLayer`): `SequenceProcessor`, `NestedSequenceProcessor`, `TimeseriesProcessor`, ...\n- **Non-sequential** (embeddings only, no RNN): `MultiHotProcessor`, `TensorProcessor`\n\nThe architecture:\n```\nconditions (seq) → Embed → RNNLayer → hidden_cond ┐\nvitals (tensor) → Linear → embed_vitals ├→ Concat → FC → logit\nrace (multi_hot) → Linear → embed_race ┘\n```\n\nThe final FC input size is `(num_sequential × hidden_dim) + (num_non_sequential × embedding_dim)`." + "source": "---\n## Part 6: `MultimodalRNN` \u2014 Mixed Input Modalities\n\n`MultimodalRNN` extends `RNN` to handle **heterogeneous inputs**. It automatically classifies each feature into:\n\n- **Sequential** (gets its own `RNNLayer`): `SequenceProcessor`, `NestedSequenceProcessor`, `TimeseriesProcessor`, ...\n- **Non-sequential** (embeddings only, no RNN): `MultiHotProcessor`, `TensorProcessor`\n\nThe architecture:\n```\nconditions (seq) \u2192 Embed \u2192 RNNLayer \u2192 hidden_cond \u2510\nvitals (tensor) \u2192 Linear \u2192 embed_vitals \u251c\u2192 Concat \u2192 FC \u2192 logit\nrace (multi_hot) \u2192 Linear \u2192 embed_race \u2518\n```\n\nThe final FC input size is `(num_sequential \u00d7 hidden_dim) + (num_non_sequential \u00d7 embedding_dim)`." }, { "cell_type": "code", @@ -129,7 +136,7 @@ { "cell_type": "markdown", "metadata": {}, - "source": "---\n## Summary\n\n| Concept | Key API |\n|---------|----------|\n| Create synthetic dataset | `create_sample_dataset(samples, input_schema, output_schema)` |\n| Batch iteration | `get_dataloader(dataset, batch_size=32, shuffle=True)` |\n| Instantiate RNN | `RNN(dataset, embedding_dim=128, hidden_dim=128, rnn_type=\"GRU\")` |\n| Forward pass | `model(**batch)` → `{loss, y_prob, y_true, logit}` |\n| Request embeddings | `model(**batch, embed=True)` → adds `embed` key |\n| Backward pass | `output[\"loss\"].backward()` |\n| Mixed modalities | `MultimodalRNN(dataset, embedding_dim=128, hidden_dim=128)` |\n\n### Choosing hyperparameters\n\n| Hyperparameter | Guidance |\n|----------------|----------|\n| `rnn_type` | GRU is a good default; LSTM has more parameters but can model longer dependencies |\n| `embedding_dim` | 64–256 depending on vocabulary size |\n| `hidden_dim` | Usually equal to `embedding_dim`; increase for more complex patterns |\n| `num_layers` | 1–2; deeper RNNs need dropout > 0 between layers |\n| `dropout` | 0.3–0.5; reduces overfitting on small datasets |\n| `bidirectional` | Only meaningful when the full sequence is available at inference time |" + "source": "---\n## Summary\n\n| Concept | Key API |\n|---------|----------|\n| Create synthetic dataset | `create_sample_dataset(samples, input_schema, output_schema)` |\n| Batch iteration | `get_dataloader(dataset, batch_size=32, shuffle=True)` |\n| Instantiate RNN | `RNN(dataset, embedding_dim=128, hidden_dim=128, rnn_type=\"GRU\")` |\n| Forward pass | `model(**batch)` \u2192 `{loss, y_prob, y_true, logit}` |\n| Request embeddings | `model(**batch, embed=True)` \u2192 adds `embed` key |\n| Backward pass | `output[\"loss\"].backward()` |\n| Mixed modalities | `MultimodalRNN(dataset, embedding_dim=128, hidden_dim=128)` |\n\n### Choosing hyperparameters\n\n| Hyperparameter | Guidance |\n|----------------|----------|\n| `rnn_type` | GRU is a good default; LSTM has more parameters but can model longer dependencies |\n| `embedding_dim` | 64\u2013256 depending on vocabulary size |\n| `hidden_dim` | Usually equal to `embedding_dim`; increase for more complex patterns |\n| `num_layers` | 1\u20132; deeper RNNs need dropout > 0 between layers |\n| `dropout` | 0.3\u20130.5; reduces overfitting on small datasets |\n| `bidirectional` | Only meaningful when the full sequence is available at inference time |" } ], "metadata": { @@ -145,4 +152,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} +} \ No newline at end of file diff --git a/examples/tutorials/tutorial_pyhealth_tokenizer.ipynb b/examples/tutorials/tutorial_pyhealth_tokenizer.ipynb new file mode 100644 index 000000000..f325c4943 --- /dev/null +++ b/examples/tutorials/tutorial_pyhealth_tokenizer.ipynb @@ -0,0 +1,210 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": "# PyHealth Tokenizer Tutorial\n\nThis notebook covers **`pyhealth.tokenizer`** \u2014 a lightweight utility for converting medical codes (or any string tokens) into integer indices for use with ML models.\n\nYou will learn:\n- How to build a **`Vocabulary`** and use special tokens (``, ``)\n- How to wrap it in a **`Tokenizer`** for batch encoding/decoding\n- How to encode flat lists of codes (**2D**) with padding and truncation\n- How to encode visit-level patient histories (**3D**) for hierarchical models\n- How to build a tokenizer from a real dataset's code vocabulary and feed it to a PyTorch model\n\n> **Why a custom tokenizer?** Clinical codes (ICD, ATC, NDC, custom labels) are categorical and very sparse \u2014 a typical hospital uses only a few thousand of the ~70,000 valid ICD-10 codes. PyHealth's tokenizer is intentionally simple (no subword splitting, no BPE) because each medical code already *is* an atomic token. For free-text clinical notes, use a HuggingFace tokenizer instead.\n\n---" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "!pip install pyhealth" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "from pyhealth.tokenizer import Vocabulary, Tokenizer" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Part 1: The `Vocabulary` Class\n\n`Vocabulary` is the underlying token \u2194 index mapping. You will rarely use it directly \u2014 `Tokenizer` wraps it \u2014 but it is useful to understand how special tokens are handled.\n\n```python\nVocabulary(\n tokens: List[str], # the real tokens (ATC codes, ICD codes, drug names\u2026)\n special_tokens: Optional[List[str]] = None, # e.g. ['', '']\n)\n```\n\n### Special token conventions\n\n| Token | Purpose | When required |\n|-------|---------|---------------|\n| `` | Padding token for batch alignment | Required when `padding=True` in batch encoders |\n| `` | Catch-all for unknown tokens | Required if your input may contain tokens outside the vocab; otherwise an unknown token raises `ValueError` |\n\n**Order matters.** Special tokens are inserted *before* the real tokens, so `` will reliably get index `0` and `` will get index `1` when you pass them in that order. This convention is used throughout PyHealth's built-in models (e.g. embedding layers initialize the pad row to zero)." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# Build a vocabulary over the first 8 ATC level-3 codes\natc_codes = ['A01A', 'A02A', 'A02B', 'A02X', 'A03A', 'A03B', 'A03C', 'A03D']\nvocab = Vocabulary(tokens=atc_codes, special_tokens=['', ''])\n\nprint(f'Vocab size: {len(vocab)}')\nprint(f' \u2192 {vocab(\"\")}')\nprint(f' \u2192 {vocab(\"\")}')\nprint(f'A01A \u2192 {vocab(\"A01A\")}')\nprint(f'A03D \u2192 {vocab(\"A03D\")}')" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# Containment check \u2014 useful when sanity-checking external inputs\nprint('\"A03A\" in vocab:', 'A03A' in vocab)\nprint('\"Z99Z\" in vocab:', 'Z99Z' in vocab)\n\n# Calling the vocab with an unknown token falls back to \nprint('vocab(\"Z99Z\") \u2192', vocab('Z99Z'), ' (== index of )')" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "### What happens without ``?\n\nIf you build a vocab without `` and then query a token that is not in it, PyHealth raises an exception rather than silently mapping it to a default index. This is the safer default for supervised tasks where every input code should be known." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "strict_vocab = Vocabulary(tokens=atc_codes, special_tokens=[''])\ntry:\n strict_vocab('Z99Z')\nexcept ValueError as e:\n print(f'ValueError: {e}')" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Part 2: The `Tokenizer` Class\n\n`Tokenizer` wraps a `Vocabulary` and adds:\n- single-list encode/decode\n- 2D batch encode/decode (with padding + truncation)\n- 3D batch encode/decode (visit-level, for hierarchical models)\n\n```python\nTokenizer(\n tokens: List[str],\n special_tokens: Optional[List[str]] = None,\n)\n```" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# A more realistic ATC level-3 vocabulary (42 codes, covering ATC 'A' \u2014 alimentary tract)\ntoken_space = [\n 'A01A', 'A02A', 'A02B', 'A02X', 'A03A', 'A03B', 'A03C', 'A03D', 'A03E', 'A03F',\n 'A04A', 'A05A', 'A05B', 'A05C', 'A06A', 'A07A', 'A07B', 'A07C', 'A07D', 'A07E',\n 'A07F', 'A07X', 'A08A', 'A09A', 'A10A', 'A10B', 'A10X', 'A11A', 'A11B', 'A11C',\n 'A11D', 'A11E', 'A11G', 'A11H', 'A11J', 'A12A', 'A12B', 'A12C', 'A13A', 'A14A',\n 'A14B', 'A16A',\n]\n\ntokenizer = Tokenizer(tokens=token_space, special_tokens=['', ''])\n\nprint(f'Vocabulary size: {tokenizer.get_vocabulary_size()}') # 2 specials + 42 codes = 44\nprint(f'Padding index : {tokenizer.get_padding_index()}') # always 0 with this ordering" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Part 3: Converting Tokens \u2194 Indices\n\nFor a single (1D) list of codes \u2014 e.g. one patient's diagnoses on one visit:" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "tokens = ['A03C', 'A03D', 'A03E', 'A03F', 'A04A', 'A05A', 'A05B', 'B035', 'C129']\n# \u2191 \u2191\n# not in vocab \u2014 will map to (= 1)\n\nindices = tokenizer.convert_tokens_to_indices(tokens)\nprint('tokens \u2192 indices:', indices)\n\ntokens_back = tokenizer.convert_indices_to_tokens(indices)\nprint('indices \u2192 tokens :', tokens_back)\nprint()\nprint('Note: \"B035\" and \"C129\" round-trip to \"\" \u2014 the original surface form is lost.')" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Part 4: 2D Batch Encoding (Padding + Truncation)\n\nIn practice you feed *batches* into a model. `batch_encode_2d` handles padding and truncation in one call.\n\n```python\ntokenizer.batch_encode_2d(\n batch: List[List[str]], # one inner list per sample\n padding: bool = True, # pad shorter lists to the batch maximum\n truncation: bool = True, # truncate longer lists to max_length (keeps the LATEST tokens)\n max_length: int = 512,\n)\n```\n\n**Truncation note:** PyHealth keeps the *most recent* `max_length` tokens (`tokens[-max_length:]`). For longitudinal EHR sequences ordered oldest \u2192 newest, this preserves the most relevant recent history." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "batch = [\n ['A03C', 'A03D', 'A03E', 'A03F'], # 4 codes\n ['A04A', 'B035', 'C129'], # 3 codes (with two s)\n]\n\n# Default: pad to the batch maximum (4), no truncation needed\nout1 = tokenizer.batch_encode_2d(batch)\nprint('default (padding=True, truncation=True):')\nprint(' ', out1)\nprint(' shape: 2 rows \u00d7 4 cols \u2014 second row padded with index 0 ()')" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# padding=False: rows keep their natural lengths (jagged \u2014 not directly usable as a tensor)\nout2 = tokenizer.batch_encode_2d(batch, padding=False)\nprint('padding=False:')\nprint(' ', out2)" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# max_length=3 \u2014 forces truncation of the first row (which has 4 codes)\nout3 = tokenizer.batch_encode_2d(batch, max_length=3)\nprint('max_length=3 (drops oldest token of row 0):')\nprint(' ', out3)\nprint()\nprint('Row 0 truncated from [A03C, A03D, A03E, A03F] \u2192 [A03D, A03E, A03F]')\nprint('Then row 1 has length 3 already \u2014 no padding needed, batch max is 3.')" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "### Decoding back to tokens\n\n`batch_decode_2d` reverses the encode step. By default it strips `` tokens so you see only the real content. Pass `padding=True` to keep them visible (useful for debugging shape issues)." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "indices = [\n [8, 9, 10, 11],\n [12, 1, 1, 0], # \u2190 the trailing 0 is \n]\n\nprint('padding=False (default \u2014 drops ):')\nprint(' ', tokenizer.batch_decode_2d(indices))\nprint()\nprint('padding=True (keep visible):')\nprint(' ', tokenizer.batch_decode_2d(indices, padding=True))" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Part 5: 3D Batch Encoding for Visit-Level Histories\n\nEHR data is naturally **hierarchical**: a patient has multiple *visits*, each visit has multiple *codes*. Models like RETAIN, GAMENet, and SafeDrug expect input shaped as `[batch, visit, code]`.\n\n`batch_encode_3d` pads and truncates along *both* axes:\n\n```python\ntokenizer.batch_encode_3d(\n batch: List[List[List[str]]],\n padding: Tuple[bool, bool] = (True, True), # (visits, codes)\n truncation: Tuple[bool, bool] = (True, True),\n max_length: Tuple[int, int] = (10, 512), # (max visits, max codes per visit)\n)\n```\n\n| Argument | Axis 0 (visits) | Axis 1 (codes within a visit) |\n|----------|-----------------|-------------------------------|\n| `padding` | Pad batch to longest patient (visit count) | Pad to longest visit (code count) |\n| `truncation` | Keep the most recent `max_length[0]` visits | Keep the most recent `max_length[1]` codes per visit |\n\nThe two axes are controlled independently \u2014 you can pad visits but not codes, etc." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# Two patients with different visit counts and visit-level code counts\npatients = [\n # Patient 0 \u2014 two visits\n [\n ['A03C', 'A03D', 'A03E', 'A03F'], # visit 1 \u2014 4 codes\n ['A08A', 'A09A'], # visit 2 \u2014 2 codes\n ],\n # Patient 1 \u2014 one visit, with two unknown codes\n [\n ['A04A', 'B035', 'C129'],\n ],\n]\n\nout = tokenizer.batch_encode_3d(patients)\nprint('Default (pad both axes):')\nfor i, p in enumerate(out):\n print(f' patient {i}: {p}')" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# pad visits but NOT codes within a visit \u2014 batch becomes jagged on the inner axis\nout = tokenizer.batch_encode_3d(patients, padding=(True, False))\nprint('padding=(visits=True, codes=False):')\nfor i, p in enumerate(out):\n print(f' patient {i}: {p}')" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# Truncate aggressively \u2014 max 2 visits, max 2 codes/visit\nout = tokenizer.batch_encode_3d(patients, max_length=(2, 2))\nprint('max_length=(2 visits, 2 codes per visit):')\nfor i, p in enumerate(out):\n print(f' patient {i}: {p}')\nprint()\nprint('Patient 0 visit 0 truncated from [A03C, A03D, A03E, A03F] \u2192 last 2 = [A03E, A03F]')\nprint('Patient 1 has only 1 visit, so the second visit slot is fully padded.')" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "### Decoding 3D batches\n\n`batch_decode_3d` strips fully-padded visits by default. Pass `padding=True` to preserve the full rectangular shape \u2014 useful when you need the indices to align with attention masks or padding masks." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "indices_3d = [\n [\n [ 8, 9, 10, 11],\n [24, 25, 0, 0], # \u2190 two trailing pads\n ],\n [\n [12, 1, 1, 0], # \u2190 one trailing pad\n [ 0, 0, 0, 0], # \u2190 fully-padded visit (dropped by default)\n ],\n]\n\nprint('padding=False (default \u2014 drops empty visits):')\nfor i, p in enumerate(tokenizer.batch_decode_3d(indices_3d)):\n print(f' patient {i}: {p}')\n\nprint()\nprint('padding=True (full rectangular shape):')\nfor i, p in enumerate(tokenizer.batch_decode_3d(indices_3d, padding=True)):\n print(f' patient {i}: {p}')" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Part 6: Practical Pattern \u2014 Build a Tokenizer from a Real Dataset\n\nIn a real training pipeline you don't hard-code the vocabulary \u2014 you build it from the codes that actually appear in your training set. The standard recipe:\n\n1. Iterate over all training samples and collect the unique codes.\n2. Sort them for reproducibility.\n3. Build the tokenizer with `` and `` as special tokens.\n4. Use the tokenizer's `get_vocabulary_size()` to size your embedding layer.\n\nBelow is a self-contained example you can run without downloading any data." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# Simulate a small training set of patient visit histories\nimport random\nrandom.seed(0)\n\n# Pretend these are the ATC codes seen across the training corpus\ncorpus_codes = ['A01A', 'A02A', 'A02B', 'A03A', 'A03B', 'A04A', 'A05A',\n 'A10A', 'A10B', 'A11A', 'A12A', 'B01A', 'B02B', 'C09A']\n\n# 5 patients, each with 1\u20133 visits, each visit with 1\u20135 codes\ntrain_samples = []\nfor _ in range(5):\n n_visits = random.randint(1, 3)\n visits = [random.sample(corpus_codes, k=random.randint(1, 5))\n for _ in range(n_visits)]\n train_samples.append(visits)\n\nfor i, p in enumerate(train_samples):\n print(f'patient {i}: {p}')" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# Step 1: collect the unique codes that actually appear in the training data\nvocab_tokens = sorted({code for patient in train_samples\n for visit in patient\n for code in visit})\nprint(f'Unique codes in training data: {len(vocab_tokens)}')\nprint(vocab_tokens)\n\n# Step 2: build the tokenizer\ntokenizer = Tokenizer(tokens=vocab_tokens, special_tokens=['', ''])\nprint(f'\\nFinal vocab size (incl. specials): {tokenizer.get_vocabulary_size()}')" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# Step 3: encode the full training set in one shot\nencoded = tokenizer.batch_encode_3d(\n train_samples,\n padding=(True, True),\n truncation=(True, True),\n max_length=(5, 10), # max 5 visits, max 10 codes per visit\n)\n\nprint(f'Encoded batch \u2014 outer length: {len(encoded)} patients')\nprint(f' visit count per patient: {[len(p) for p in encoded]}')\nprint(f' codes per visit (patient 0): {[len(v) for v in encoded[0]]}')" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# Step 4 (sketch): use the tokenizer's vocab size as the embedding input dim\n# This is what PyHealth's built-in models do internally:\n#\n# import torch.nn as nn\n# embedding = nn.Embedding(\n# num_embeddings = tokenizer.get_vocabulary_size(),\n# embedding_dim = 128,\n# padding_idx = tokenizer.get_padding_index(), # zero-row for \n# )\n#\n# Passing `padding_idx` ensures gradients don't flow into the pad row \u2014\n# critical for training stability when batches are heavily padded.\n\nprint(f'num_embeddings = {tokenizer.get_vocabulary_size()}')\nprint(f'padding_idx = {tokenizer.get_padding_index()}')" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Part 7: Where the Tokenizer Fits in PyHealth\n\nMost users never instantiate `Tokenizer` directly. It is used *internally* by:\n\n| Component | Role of the tokenizer |\n|-----------|----------------------|\n| **`pyhealth.processors`** (`SequenceProcessor`, etc.) | Builds a tokenizer per feature key (e.g. one for diagnoses, one for procedures) when the dataset task is registered |\n| **Built-in models** (`RNN`, `Transformer`, `RETAIN`, `GAMENet`, \u2026) | Embeds tokenized inputs and uses `padding_idx` to mask the pad row |\n| **`set_task()`** pipeline | Wires processors + tokenizers into the dataloader so models receive ready-to-embed integer tensors |\n\nYou should reach for `Tokenizer` directly when:\n- You are writing a **custom model** that takes raw code lists instead of pre-processed tensors\n- You are running **inference** on data that was not produced by PyHealth's pipeline (e.g. a CSV of codes)\n- You need to **share a single vocabulary** across multiple datasets or tasks\n\nFor free-text inputs (clinical notes, discharge summaries), don't use this tokenizer \u2014 use a HuggingFace tokenizer inside one of PyHealth's text processors (see the `smart_processor_clinical_text_tutorial.ipynb` example for the canonical pattern)." + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## Summary\n\n| Task | API |\n|------|-----|\n| Build a vocabulary | `Vocabulary(tokens, special_tokens=['', ''])` |\n| Build a tokenizer | `Tokenizer(tokens, special_tokens=['', ''])` |\n| Get vocab size | `tokenizer.get_vocabulary_size()` |\n| Get pad index | `tokenizer.get_padding_index()` |\n| Encode one list | `tokenizer.convert_tokens_to_indices(tokens)` |\n| Decode one list | `tokenizer.convert_indices_to_tokens(indices)` |\n| Encode 2D batch | `tokenizer.batch_encode_2d(batch, padding=True, truncation=True, max_length=512)` |\n| Decode 2D batch | `tokenizer.batch_decode_2d(batch, padding=False)` |\n| Encode 3D batch | `tokenizer.batch_encode_3d(batch, padding=(True, True), truncation=(True, True), max_length=(10, 512))` |\n| Decode 3D batch | `tokenizer.batch_decode_3d(batch, padding=False)` |\n\n### Quick design checklist\n\n- Always include `` if you will batch with `padding=True`.\n- Include `` if you cannot guarantee a closed vocabulary at inference time.\n- Order specials as `['', '']` so the pad index is `0` (matches PyHealth's model defaults).\n- Build the vocab from the **training set only** \u2014 unseen codes at validation/test time should map to ``.\n- For hierarchical EHR models, use `batch_encode_3d` with explicit `max_length` to cap GPU memory." + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.9.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/examples/tutorials/tutorial_pyhealth_trainer.ipynb b/examples/tutorials/tutorial_pyhealth_trainer.ipynb index c5ee70f56..f32b3cf9d 100644 --- a/examples/tutorials/tutorial_pyhealth_trainer.ipynb +++ b/examples/tutorials/tutorial_pyhealth_trainer.ipynb @@ -3,7 +3,14 @@ { "cell_type": "markdown", "metadata": {}, - "source": "# PyHealth Trainer Tutorial — End-to-End Training with MIMIC-III\n\nThis notebook covers **`pyhealth.trainer`** — the training loop that ties datasets, models, and metrics together.\n\nYou will learn:\n- How to load a public **synthetic MIMIC-III** dataset (no credentials required)\n- How to apply a **mortality prediction task** to generate model-ready samples\n- How to split, batch, and feed data to an **RNN model**\n- How to use **`Trainer`** for training with validation, early stopping, and checkpointing\n- How to **evaluate** a trained model on a held-out test set\n\n> **Dataset Note:** The dataset used here is a fully synthetic MIMIC-III replica hosted by Google Cloud Storage. No PhysioNet account or data use agreement is needed.\n\n---" + "source": "# PyHealth Trainer Tutorial \u2014 End-to-End Training with MIMIC-III\n\nThis notebook covers **`pyhealth.trainer`** \u2014 the training loop that ties datasets, models, and metrics together.\n\nYou will learn:\n- How to load a public **synthetic MIMIC-III** dataset (no credentials required)\n- How to apply a **mortality prediction task** to generate model-ready samples\n- How to split, batch, and feed data to an **RNN model**\n- How to use **`Trainer`** for training with validation, early stopping, and checkpointing\n- How to **evaluate** a trained model on a held-out test set\n\n> **Dataset Note:** The dataset used here is a fully synthetic MIMIC-III replica hosted by Google Cloud Storage. No PhysioNet account or data use agreement is needed.\n\n---" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "!pip install pyhealth" }, { "cell_type": "code", @@ -15,7 +22,7 @@ { "cell_type": "markdown", "metadata": {}, - "source": "---\n## Step 1: Load the Synthetic MIMIC-III Dataset\n\n`MIMIC3Dataset` loads structured EHR tables from a root directory (local path or URL). The Google Cloud Storage path below hosts a synthetic copy — the schema is identical to real MIMIC-III but no real patient data is present.\n\nDefault tables always loaded: `[\"patients\", \"admissions\", \"icustays\"]`\nAdditional tables you can specify:\n- `\"diagnoses_icd\"` — ICD-9 diagnosis codes per admission\n- `\"procedures_icd\"` — ICD-9 procedure codes per admission\n- `\"prescriptions\"` — Medication orders (NDC codes)\n- `\"labevents\"` — Lab measurements\n- `\"noteevents\"` — Clinical notes (discharge summaries, radiology reports, etc.)" + "source": "---\n## Step 1: Load the Synthetic MIMIC-III Dataset\n\n`MIMIC3Dataset` loads structured EHR tables from a root directory (local path or URL). The Google Cloud Storage path below hosts a synthetic copy \u2014 the schema is identical to real MIMIC-III but no real patient data is present.\n\nDefault tables always loaded: `[\"patients\", \"admissions\", \"icustays\"]`\nAdditional tables you can specify:\n- `\"diagnoses_icd\"` \u2014 ICD-9 diagnosis codes per admission\n- `\"procedures_icd\"` \u2014 ICD-9 procedure codes per admission\n- `\"prescriptions\"` \u2014 Medication orders (NDC codes)\n- `\"labevents\"` \u2014 Lab measurements\n- `\"noteevents\"` \u2014 Clinical notes (discharge summaries, radiology reports, etc.)" }, { "cell_type": "code", @@ -46,7 +53,7 @@ { "cell_type": "markdown", "metadata": {}, - "source": "---\n## Step 3: Split the Dataset\n\n`split_by_patient` partitions samples so that **no patient appears in more than one split** — this is the correct way to split clinical data. Splitting by sample (the naive approach) would allow data leakage: a model could see one visit from patient X in training and another visit from the same patient in test.\n\nReturns three `SampleDataset` objects sharing the same fitted processors." + "source": "---\n## Step 3: Split the Dataset\n\n`split_by_patient` partitions samples so that **no patient appears in more than one split** \u2014 this is the correct way to split clinical data. Splitting by sample (the naive approach) would allow data leakage: a model could see one visit from patient X in training and another visit from the same patient in test.\n\nReturns three `SampleDataset` objects sharing the same fitted processors." }, { "cell_type": "code", @@ -154,7 +161,7 @@ { "cell_type": "markdown", "id": "b8313bc0", - "source": "---\n## API Reference: Available Metric Strings\n\nThe `metrics` argument to `Trainer.__init__` and the `monitor` argument to `trainer.train()` are plain strings drawn from a fixed list. The exact list depends on **`model.mode`**, which is set automatically from the task's output schema:\n\n```python\nprint(model.mode) # → \"binary\" | \"multiclass\" | \"multilabel\" | \"regression\"\n```\n\n`Trainer` uses `model.mode` to select the right metrics function, then passes your `metrics` list to it. Any string you pass to `monitor` must appear in that same list — otherwise evaluation will raise a `KeyError`.\n\nTo compute a non-default set of metrics and track a specific one:\n```python\ntrainer = Trainer(\n model=model,\n metrics=[\"roc_auc\", \"pr_auc\", \"balanced_accuracy\", \"ECE\"], # computed every eval epoch\n)\ntrainer.train(..., monitor=\"pr_auc\", monitor_criterion=\"max\")\n```\n\n---\n\n### Binary classification — `mode = \"binary\"`\n**Source:** `pyhealth.metrics.binary_metrics_fn` \n**Defaults when `metrics=None`:** `[\"pr_auc\", \"roc_auc\", \"f1\"]`\n\n| Metric string | Description | `monitor_criterion` |\n|---|---|---|\n| `\"pr_auc\"` | Area under the Precision-Recall curve | `\"max\"` |\n| `\"roc_auc\"` | Area under the ROC curve | `\"max\"` |\n| `\"f1\"` | F1 score at `threshold` (default 0.5) | `\"max\"` |\n| `\"accuracy\"` | Fraction of correct predictions | `\"max\"` |\n| `\"balanced_accuracy\"` | Accuracy adjusted for class imbalance | `\"max\"` |\n| `\"precision\"` | Precision at `threshold` | `\"max\"` |\n| `\"recall\"` | Recall at `threshold` | `\"max\"` |\n| `\"cohen_kappa\"` | Cohen's kappa (agreement beyond chance) | `\"max\"` |\n| `\"jaccard\"` | Jaccard similarity coefficient | `\"max\"` |\n| `\"ECE\"` | Expected Calibration Error (20 equal-width bins) | `\"min\"` |\n| `\"ECE_adapt\"` | Adaptive ECE (20 equal-size bins) | `\"min\"` |\n\n---\n\n### Multiclass classification — `mode = \"multiclass\"`\n**Source:** `pyhealth.metrics.multiclass_metrics_fn` \n**Defaults when `metrics=None`:** `[\"accuracy\", \"f1_macro\", \"f1_micro\"]`\n\n| Metric string | Description | `monitor_criterion` |\n|---|---|---|\n| `\"accuracy\"` | Overall accuracy | `\"max\"` |\n| `\"balanced_accuracy\"` | Accuracy adjusted for class imbalance | `\"max\"` |\n| `\"f1_macro\"` | F1, macro-averaged across classes | `\"max\"` |\n| `\"f1_micro\"` | F1, micro-averaged across classes | `\"max\"` |\n| `\"f1_weighted\"` | F1, weighted by class support | `\"max\"` |\n| `\"roc_auc_macro_ovo\"` | ROC-AUC, macro, one-vs-one | `\"max\"` |\n| `\"roc_auc_macro_ovr\"` | ROC-AUC, macro, one-vs-rest | `\"max\"` |\n| `\"roc_auc_weighted_ovo\"` | ROC-AUC, weighted, one-vs-one | `\"max\"` |\n| `\"roc_auc_weighted_ovr\"` | ROC-AUC, weighted, one-vs-rest | `\"max\"` |\n| `\"jaccard_micro\"` | Jaccard, micro-averaged | `\"max\"` |\n| `\"jaccard_macro\"` | Jaccard, macro-averaged | `\"max\"` |\n| `\"jaccard_weighted\"` | Jaccard, weighted | `\"max\"` |\n| `\"cohen_kappa\"` | Cohen's kappa | `\"max\"` |\n| `\"brier_top1\"` | Brier score for the top predicted class | `\"min\"` |\n| `\"ECE\"` | Expected Calibration Error (20 equal-width bins) | `\"min\"` |\n| `\"ECE_adapt\"` | Adaptive ECE (20 equal-size bins) | `\"min\"` |\n| `\"cwECEt\"` | Classwise ECE with threshold = min(0.01, 1/K) | `\"min\"` |\n| `\"cwECEt_adapt\"` | Classwise adaptive ECE | `\"min\"` |\n| `\"hits@n\"` | HITS@1 / HITS@5 / HITS@10 (produces 3 dict keys) | `\"max\"` |\n| `\"mean_rank\"` | Mean rank + mean reciprocal rank | `\"min\"` |\n\n---\n\n### Multilabel classification — `mode = \"multilabel\"`\n**Source:** `pyhealth.metrics.multilabel_metrics_fn` \n**Defaults when `metrics=None`:** `[\"pr_auc_samples\"]` \n**Note:** threshold defaults to `0.3` (not `0.5`) — lower thresholds are common in drug recommendation tasks.\n\n| Metric string | Description | `monitor_criterion` |\n|---|---|---|\n| `\"pr_auc_samples\"` | PR-AUC, averaged across samples | `\"max\"` |\n| `\"pr_auc_micro\"` | PR-AUC, micro-averaged | `\"max\"` |\n| `\"pr_auc_macro\"` | PR-AUC, macro-averaged | `\"max\"` |\n| `\"pr_auc_weighted\"` | PR-AUC, weighted | `\"max\"` |\n| `\"roc_auc_samples\"` | ROC-AUC, samples-averaged | `\"max\"` |\n| `\"roc_auc_micro\"` | ROC-AUC, micro-averaged | `\"max\"` |\n| `\"roc_auc_macro\"` | ROC-AUC, macro-averaged | `\"max\"` |\n| `\"roc_auc_weighted\"` | ROC-AUC, weighted | `\"max\"` |\n| `\"f1_samples\"` | F1, samples-averaged | `\"max\"` |\n| `\"f1_micro\"` | F1, micro-averaged | `\"max\"` |\n| `\"f1_macro\"` | F1, macro-averaged | `\"max\"` |\n| `\"f1_weighted\"` | F1, weighted | `\"max\"` |\n| `\"precision_micro\"` / `\"_macro\"` / `\"_weighted\"` / `\"_samples\"` | Precision variants | `\"max\"` |\n| `\"recall_micro\"` / `\"_macro\"` / `\"_weighted\"` / `\"_samples\"` | Recall variants | `\"max\"` |\n| `\"jaccard_micro\"` / `\"_macro\"` / `\"_weighted\"` / `\"_samples\"` | Jaccard variants | `\"max\"` |\n| `\"accuracy\"` | Element-wise accuracy | `\"max\"` |\n| `\"hamming_loss\"` | Hamming loss | `\"min\"` |\n| `\"ddi\"` | Drug-drug interaction rate (drug recommendation only) | `\"min\"` |\n| `\"cwECE\"` | Classwise ECE (20 equal-width bins) | `\"min\"` |\n| `\"cwECE_adapt\"` | Classwise adaptive ECE | `\"min\"` |\n\n---\n\n### Regression — `mode = \"regression\"`\n**Source:** `pyhealth.metrics.regression_metrics_fn` \n**Defaults when `metrics=None`:** `[\"kl_divergence\", \"mse\", \"mae\"]`\n\n| Metric string | Description | `monitor_criterion` |\n|---|---|---|\n| `\"mae\"` | Mean Absolute Error | `\"min\"` |\n| `\"mse\"` | Mean Squared Error | `\"min\"` |\n| `\"kl_divergence\"` | KL divergence between true and reconstructed distributions | `\"min\"` |", + "source": "---\n## API Reference: Available Metric Strings\n\nThe `metrics` argument to `Trainer.__init__` and the `monitor` argument to `trainer.train()` are plain strings drawn from a fixed list. The exact list depends on **`model.mode`**, which is set automatically from the task's output schema:\n\n```python\nprint(model.mode) # \u2192 \"binary\" | \"multiclass\" | \"multilabel\" | \"regression\"\n```\n\n`Trainer` uses `model.mode` to select the right metrics function, then passes your `metrics` list to it. Any string you pass to `monitor` must appear in that same list \u2014 otherwise evaluation will raise a `KeyError`.\n\nTo compute a non-default set of metrics and track a specific one:\n```python\ntrainer = Trainer(\n model=model,\n metrics=[\"roc_auc\", \"pr_auc\", \"balanced_accuracy\", \"ECE\"], # computed every eval epoch\n)\ntrainer.train(..., monitor=\"pr_auc\", monitor_criterion=\"max\")\n```\n\n---\n\n### Binary classification \u2014 `mode = \"binary\"`\n**Source:** `pyhealth.metrics.binary_metrics_fn` \n**Defaults when `metrics=None`:** `[\"pr_auc\", \"roc_auc\", \"f1\"]`\n\n| Metric string | Description | `monitor_criterion` |\n|---|---|---|\n| `\"pr_auc\"` | Area under the Precision-Recall curve | `\"max\"` |\n| `\"roc_auc\"` | Area under the ROC curve | `\"max\"` |\n| `\"f1\"` | F1 score at `threshold` (default 0.5) | `\"max\"` |\n| `\"accuracy\"` | Fraction of correct predictions | `\"max\"` |\n| `\"balanced_accuracy\"` | Accuracy adjusted for class imbalance | `\"max\"` |\n| `\"precision\"` | Precision at `threshold` | `\"max\"` |\n| `\"recall\"` | Recall at `threshold` | `\"max\"` |\n| `\"cohen_kappa\"` | Cohen's kappa (agreement beyond chance) | `\"max\"` |\n| `\"jaccard\"` | Jaccard similarity coefficient | `\"max\"` |\n| `\"ECE\"` | Expected Calibration Error (20 equal-width bins) | `\"min\"` |\n| `\"ECE_adapt\"` | Adaptive ECE (20 equal-size bins) | `\"min\"` |\n\n---\n\n### Multiclass classification \u2014 `mode = \"multiclass\"`\n**Source:** `pyhealth.metrics.multiclass_metrics_fn` \n**Defaults when `metrics=None`:** `[\"accuracy\", \"f1_macro\", \"f1_micro\"]`\n\n| Metric string | Description | `monitor_criterion` |\n|---|---|---|\n| `\"accuracy\"` | Overall accuracy | `\"max\"` |\n| `\"balanced_accuracy\"` | Accuracy adjusted for class imbalance | `\"max\"` |\n| `\"f1_macro\"` | F1, macro-averaged across classes | `\"max\"` |\n| `\"f1_micro\"` | F1, micro-averaged across classes | `\"max\"` |\n| `\"f1_weighted\"` | F1, weighted by class support | `\"max\"` |\n| `\"roc_auc_macro_ovo\"` | ROC-AUC, macro, one-vs-one | `\"max\"` |\n| `\"roc_auc_macro_ovr\"` | ROC-AUC, macro, one-vs-rest | `\"max\"` |\n| `\"roc_auc_weighted_ovo\"` | ROC-AUC, weighted, one-vs-one | `\"max\"` |\n| `\"roc_auc_weighted_ovr\"` | ROC-AUC, weighted, one-vs-rest | `\"max\"` |\n| `\"jaccard_micro\"` | Jaccard, micro-averaged | `\"max\"` |\n| `\"jaccard_macro\"` | Jaccard, macro-averaged | `\"max\"` |\n| `\"jaccard_weighted\"` | Jaccard, weighted | `\"max\"` |\n| `\"cohen_kappa\"` | Cohen's kappa | `\"max\"` |\n| `\"brier_top1\"` | Brier score for the top predicted class | `\"min\"` |\n| `\"ECE\"` | Expected Calibration Error (20 equal-width bins) | `\"min\"` |\n| `\"ECE_adapt\"` | Adaptive ECE (20 equal-size bins) | `\"min\"` |\n| `\"cwECEt\"` | Classwise ECE with threshold = min(0.01, 1/K) | `\"min\"` |\n| `\"cwECEt_adapt\"` | Classwise adaptive ECE | `\"min\"` |\n| `\"hits@n\"` | HITS@1 / HITS@5 / HITS@10 (produces 3 dict keys) | `\"max\"` |\n| `\"mean_rank\"` | Mean rank + mean reciprocal rank | `\"min\"` |\n\n---\n\n### Multilabel classification \u2014 `mode = \"multilabel\"`\n**Source:** `pyhealth.metrics.multilabel_metrics_fn` \n**Defaults when `metrics=None`:** `[\"pr_auc_samples\"]` \n**Note:** threshold defaults to `0.3` (not `0.5`) \u2014 lower thresholds are common in drug recommendation tasks.\n\n| Metric string | Description | `monitor_criterion` |\n|---|---|---|\n| `\"pr_auc_samples\"` | PR-AUC, averaged across samples | `\"max\"` |\n| `\"pr_auc_micro\"` | PR-AUC, micro-averaged | `\"max\"` |\n| `\"pr_auc_macro\"` | PR-AUC, macro-averaged | `\"max\"` |\n| `\"pr_auc_weighted\"` | PR-AUC, weighted | `\"max\"` |\n| `\"roc_auc_samples\"` | ROC-AUC, samples-averaged | `\"max\"` |\n| `\"roc_auc_micro\"` | ROC-AUC, micro-averaged | `\"max\"` |\n| `\"roc_auc_macro\"` | ROC-AUC, macro-averaged | `\"max\"` |\n| `\"roc_auc_weighted\"` | ROC-AUC, weighted | `\"max\"` |\n| `\"f1_samples\"` | F1, samples-averaged | `\"max\"` |\n| `\"f1_micro\"` | F1, micro-averaged | `\"max\"` |\n| `\"f1_macro\"` | F1, macro-averaged | `\"max\"` |\n| `\"f1_weighted\"` | F1, weighted | `\"max\"` |\n| `\"precision_micro\"` / `\"_macro\"` / `\"_weighted\"` / `\"_samples\"` | Precision variants | `\"max\"` |\n| `\"recall_micro\"` / `\"_macro\"` / `\"_weighted\"` / `\"_samples\"` | Recall variants | `\"max\"` |\n| `\"jaccard_micro\"` / `\"_macro\"` / `\"_weighted\"` / `\"_samples\"` | Jaccard variants | `\"max\"` |\n| `\"accuracy\"` | Element-wise accuracy | `\"max\"` |\n| `\"hamming_loss\"` | Hamming loss | `\"min\"` |\n| `\"ddi\"` | Drug-drug interaction rate (drug recommendation only) | `\"min\"` |\n| `\"cwECE\"` | Classwise ECE (20 equal-width bins) | `\"min\"` |\n| `\"cwECE_adapt\"` | Classwise adaptive ECE | `\"min\"` |\n\n---\n\n### Regression \u2014 `mode = \"regression\"`\n**Source:** `pyhealth.metrics.regression_metrics_fn` \n**Defaults when `metrics=None`:** `[\"kl_divergence\", \"mse\", \"mae\"]`\n\n| Metric string | Description | `monitor_criterion` |\n|---|---|---|\n| `\"mae\"` | Mean Absolute Error | `\"min\"` |\n| `\"mse\"` | Mean Squared Error | `\"min\"` |\n| `\"kl_divergence\"` | KL divergence between true and reconstructed distributions | `\"min\"` |", "metadata": {} } ], From 53fa513454afdb12867d9e347b66ac303e0707ba Mon Sep 17 00:00:00 2001 From: Chufan Gao Date: Sun, 7 Jun 2026 10:57:06 -0700 Subject: [PATCH 12/61] Add synthetic-EHR generative evaluation metrics (#1148) * Add synthetic-EHR generative evaluation metrics Adds pyhealth/metrics/generative/, a subpackage for evaluating synthetic EHR data along privacy, utility, and statistical-fidelity axes: - privacy.py: NNAAR, membership inference attack, discriminator privacy - utility.py: machine learning efficacy (TRTR vs TSTR), code-prevalence similarity (R2, Pearson, RMSE) - utils.py: shared data prep, an LSTM classifier, and a random-forest baseline - evaluate_synthetic_ehr(): convenience orchestrator for the full suite These functions are ported from a standalone evaluation script. The MIMIC-specific data-loading/CLI glue is dropped; the metrics work on any flat EHR dataframe. Public functions are re-exported from pyhealth.metrics. Adds unit tests in tests/core/test_generative_metrics.py and Sphinx docs. Co-Authored-By: Claude Opus 4.7 (1M context) * Add synthetic-EHR generative evaluation metrics Adds pyhealth/metrics/generative/, a subpackage for evaluating synthetic EHR data along privacy, utility, and statistical-fidelity axes: - privacy.py: NNAAR, membership inference attack, discriminator privacy - utility.py: machine learning efficacy (TRTR vs TSTR), code-prevalence similarity (R2, Pearson, RMSE) - utils.py: shared data prep, an LSTM classifier, and a random-forest baseline - evaluate_synthetic_ehr(): convenience orchestrator for the full suite These functions are ported from a standalone evaluation script. The MIMIC-specific data-loading/CLI glue is dropped; the metrics work on any flat EHR dataframe. Public functions are re-exported from pyhealth.metrics. Adds unit tests in tests/core/test_generative_metrics.py and Sphinx docs. Co-Authored-By: Claude Opus 4.7 (1M context) * add baselines * removed halo save file and updated promptehr to be more paper accurate * update docs * update docs * Update pyhealth.models.HALO.rst --------- Co-authored-by: Claude Opus 4.7 (1M context) --- docs/api/metrics.rst | 3 + .../metrics/pyhealth.metrics.generative.rst | 25 + docs/api/models.rst | 5 + docs/api/models/pyhealth.models.CorGAN.rst | 21 + docs/api/models/pyhealth.models.GPT2.rst | 17 + docs/api/models/pyhealth.models.HALO.rst | 19 + docs/api/models/pyhealth.models.MedGAN.rst | 21 + docs/api/models/pyhealth.models.PromptEHR.rst | 20 + docs/api/tasks.rst | 1 + .../api/tasks/pyhealth.tasks.generate_ehr.rst | 32 + examples/halo_mimic3.py | 132 ++++ pyhealth/metrics/__init__.py | 14 + pyhealth/metrics/generative/__init__.py | 276 +++++++ pyhealth/metrics/generative/privacy.py | 386 ++++++++++ pyhealth/metrics/generative/utility.py | 285 +++++++ pyhealth/metrics/generative/utils.py | 604 +++++++++++++++ pyhealth/models/__init__.py | 7 +- pyhealth/models/generators/__init__.py | 7 + pyhealth/models/generators/corgan.py | 691 +++++++++++++++++ pyhealth/models/generators/gpt2.py | 366 +++++++++ pyhealth/models/generators/halo.py | 724 ++++++++++++++++++ pyhealth/models/generators/medgan.py | 517 +++++++++++++ pyhealth/models/generators/promptehr.py | 517 +++++++++++++ pyhealth/tasks/__init__.py | 7 + pyhealth/tasks/generate_ehr.py | 253 ++++++ tests/core/test_corgan.py | 228 ++++++ tests/core/test_generative_metrics.py | 472 ++++++++++++ tests/core/test_gpt2.py | 151 ++++ tests/core/test_halo.py | 152 ++++ tests/core/test_medgan.py | 197 +++++ tests/core/test_promptehr.py | 161 ++++ 31 files changed, 6310 insertions(+), 1 deletion(-) create mode 100644 docs/api/metrics/pyhealth.metrics.generative.rst create mode 100644 docs/api/models/pyhealth.models.CorGAN.rst create mode 100644 docs/api/models/pyhealth.models.GPT2.rst create mode 100644 docs/api/models/pyhealth.models.HALO.rst create mode 100644 docs/api/models/pyhealth.models.MedGAN.rst create mode 100644 docs/api/models/pyhealth.models.PromptEHR.rst create mode 100644 docs/api/tasks/pyhealth.tasks.generate_ehr.rst create mode 100644 examples/halo_mimic3.py create mode 100644 pyhealth/metrics/generative/__init__.py create mode 100644 pyhealth/metrics/generative/privacy.py create mode 100644 pyhealth/metrics/generative/utility.py create mode 100644 pyhealth/metrics/generative/utils.py create mode 100644 pyhealth/models/generators/__init__.py create mode 100644 pyhealth/models/generators/corgan.py create mode 100644 pyhealth/models/generators/gpt2.py create mode 100644 pyhealth/models/generators/halo.py create mode 100644 pyhealth/models/generators/medgan.py create mode 100644 pyhealth/models/generators/promptehr.py create mode 100644 pyhealth/tasks/generate_ehr.py create mode 100644 tests/core/test_corgan.py create mode 100644 tests/core/test_generative_metrics.py create mode 100644 tests/core/test_gpt2.py create mode 100644 tests/core/test_halo.py create mode 100644 tests/core/test_medgan.py create mode 100644 tests/core/test_promptehr.py diff --git a/docs/api/metrics.rst b/docs/api/metrics.rst index 1767e0026..9e6bc160a 100644 --- a/docs/api/metrics.rst +++ b/docs/api/metrics.rst @@ -7,6 +7,8 @@ For applicable tasks, we provide the relevant metrics for model calibration, as Among these we also provide metrics related to uncertainty quantification, for model calibration, as well as metrics that measure the quality of prediction sets We also provide other metrics specically for healthcare tasks, such as drug drug interaction (DDI) rate. +For synthetic (generative) EHR data, we provide privacy, utility, and statistical +fidelity metrics. .. toctree:: @@ -19,3 +21,4 @@ tasks, such as drug drug interaction (DDI) rate. metrics/pyhealth.metrics.prediction_set metrics/pyhealth.metrics.fairness metrics/pyhealth.metrics.interpretability + metrics/pyhealth.metrics.generative diff --git a/docs/api/metrics/pyhealth.metrics.generative.rst b/docs/api/metrics/pyhealth.metrics.generative.rst new file mode 100644 index 000000000..85e448a52 --- /dev/null +++ b/docs/api/metrics/pyhealth.metrics.generative.rst @@ -0,0 +1,25 @@ +pyhealth.metrics.generative +=================================== + +Evaluation metrics for synthetic (generative) EHR data, covering privacy, +utility, and statistical fidelity. + +.. currentmodule:: pyhealth.metrics.generative + +.. autofunction:: evaluate_synthetic_ehr + +Privacy metrics +------------------------------------- + +.. autofunction:: calc_nnaar + +.. autofunction:: calc_membership_inference + +.. autofunction:: compute_discriminator_privacy + +Utility and fidelity metrics +------------------------------------- + +.. autofunction:: compute_mle + +.. autofunction:: compute_prevalence_metrics diff --git a/docs/api/models.rst b/docs/api/models.rst index 7c3ac7c4b..e98f74f5c 100644 --- a/docs/api/models.rst +++ b/docs/api/models.rst @@ -200,6 +200,11 @@ API Reference models/pyhealth.models.TFMTokenizer models/pyhealth.models.GAN models/pyhealth.models.VAE + models/pyhealth.models.HALO + models/pyhealth.models.GPT2 + models/pyhealth.models.PromptEHR + models/pyhealth.models.MedGAN + models/pyhealth.models.CorGAN models/pyhealth.models.SDOH models/pyhealth.models.VisionEmbeddingModel models/pyhealth.models.TextEmbedding diff --git a/docs/api/models/pyhealth.models.CorGAN.rst b/docs/api/models/pyhealth.models.CorGAN.rst new file mode 100644 index 000000000..783b3cafd --- /dev/null +++ b/docs/api/models/pyhealth.models.CorGAN.rst @@ -0,0 +1,21 @@ +pyhealth.models.CorGAN +=================================== + +CorGAN: a Correlation-capturing Convolutional GAN for synthetic EHR generation. +A 1D-CNN (or linear) autoencoder captures local code correlations, and a WGAN +generator/critic are trained in the autoencoder's latent space. Ported from the +reference implementation +(`cor-gan `_) and wrapped as a PyHealth +:class:`~pyhealth.models.BaseModel`. + +Reference: + Torfi, A., & Fox, E. A. (2020). + *CorGAN: Correlation-Capturing Convolutional Generative Adversarial + Networks for Generating Synthetic Healthcare Records.* + In Proceedings of the 33rd International FLAIRS Conference. + https://arxiv.org/abs/2001.09346 + +.. autoclass:: pyhealth.models.CorGAN + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/models/pyhealth.models.GPT2.rst b/docs/api/models/pyhealth.models.GPT2.rst new file mode 100644 index 000000000..9e13c10d4 --- /dev/null +++ b/docs/api/models/pyhealth.models.GPT2.rst @@ -0,0 +1,17 @@ +pyhealth.models.GPT2 +=================================== + +A decoder-only GPT-2 baseline for unconditional synthetic EHR generation, +wrapped as a PyHealth :class:`~pyhealth.models.BaseModel`. Patient visit-code +sequences are serialized into causal-LM token streams and modeled +autoregressively. + +Reference: + Radford, A., Wu, J., Child, R., Luan, D., Amodei, D., & Sutskever, I. (2019). + *Language Models are Unsupervised Multitask Learners.* OpenAI. + https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf + +.. autoclass:: pyhealth.models.GPT2 + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/models/pyhealth.models.HALO.rst b/docs/api/models/pyhealth.models.HALO.rst new file mode 100644 index 000000000..991fe3467 --- /dev/null +++ b/docs/api/models/pyhealth.models.HALO.rst @@ -0,0 +1,19 @@ +pyhealth.models.HALO +=================================== + +HALO (Hierarchical Autoregressive Language model) for synthetic EHR generation. +A faithful port of the reference implementation +(`HALO_Inpatient `_), +wrapped as a PyHealth :class:`~pyhealth.models.BaseModel`. + +Reference: + Theodorou, B., Xiao, C., & Sun, J. (2023). + *Synthesize high-dimensional longitudinal electronic health records via + hierarchical autoregressive language model.* + Nature Communications, 14, 5305. + https://www.nature.com/articles/s41467-023-41093-0 + +.. autoclass:: pyhealth.models.HALO + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/models/pyhealth.models.MedGAN.rst b/docs/api/models/pyhealth.models.MedGAN.rst new file mode 100644 index 000000000..cd328f651 --- /dev/null +++ b/docs/api/models/pyhealth.models.MedGAN.rst @@ -0,0 +1,21 @@ +pyhealth.models.MedGAN +=================================== + +MedGAN: a bag-of-codes Generative Adversarial Network for synthetic EHR +generation. An autoencoder is pre-trained on multi-hot patient records, then a +GAN with residual generator and minibatch-averaging discriminator is trained in +the autoencoder's latent space. Ported from the reference implementations +(`medgan `_ and its PyTorch reimplementation) +and wrapped as a PyHealth :class:`~pyhealth.models.BaseModel`. + +Reference: + Choi, E., Biswal, S., Malin, B., Duke, J., Stewart, W. F., & Sun, J. (2017). + *Generating Multi-label Discrete Patient Records using Generative + Adversarial Networks.* + In Proceedings of Machine Learning for Healthcare (MLHC) 2017. + https://arxiv.org/abs/1703.06490 + +.. autoclass:: pyhealth.models.MedGAN + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/models/pyhealth.models.PromptEHR.rst b/docs/api/models/pyhealth.models.PromptEHR.rst new file mode 100644 index 000000000..57ed8389f --- /dev/null +++ b/docs/api/models/pyhealth.models.PromptEHR.rst @@ -0,0 +1,20 @@ +pyhealth.models.PromptEHR +=================================== + +PromptEHR: prompt-learning BART for synthetic EHR generation. A port of the +reference implementation +(`PromptEHR `_) that consumes the +standard PyHealth interface and learns via a span-infilling objective with a +reparameterized soft prompt. + +Reference: + Wang, Z., & Sun, J. (2022). + *PromptEHR: Conditional Electronic Healthcare Records Generation with + Prompt Learning.* + In Proceedings of EMNLP 2022. + https://aclanthology.org/2022.emnlp-main.185/ + +.. autoclass:: pyhealth.models.PromptEHR + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/tasks.rst b/docs/api/tasks.rst index 69e5aa592..0ff286bfc 100644 --- a/docs/api/tasks.rst +++ b/docs/api/tasks.rst @@ -212,6 +212,7 @@ Available Tasks COVID-19 CXR Classification DKA Prediction (MIMIC-IV) Drug Recommendation + EHR Generation Length of Stay Prediction Medical Transcriptions Classification Mortality Prediction (Next Visit) diff --git a/docs/api/tasks/pyhealth.tasks.generate_ehr.rst b/docs/api/tasks/pyhealth.tasks.generate_ehr.rst new file mode 100644 index 000000000..77c332f33 --- /dev/null +++ b/docs/api/tasks/pyhealth.tasks.generate_ehr.rst @@ -0,0 +1,32 @@ +pyhealth.tasks.generate_ehr +=========================================== + +Task that turns a longitudinal EHR dataset into per-patient, per-visit code +sequences for training unconditional synthetic-EHR generators (HALO, GPT2, +PromptEHR, MedGAN, CorGAN), plus helpers to flatten generated output into the +long-form dataframe consumed by :mod:`pyhealth.metrics.generative`. + +Task Classes +------------ + +.. autoclass:: pyhealth.tasks.generate_ehr.EHRGeneration + :members: + :undoc-members: + :show-inheritance: + +.. autoclass:: pyhealth.tasks.generate_ehr.EHRGenerationMIMIC3 + :members: + :undoc-members: + :show-inheritance: + +.. autoclass:: pyhealth.tasks.generate_ehr.EHRGenerationMIMIC4 + :members: + :undoc-members: + :show-inheritance: + +Helper Functions +---------------- + +.. autofunction:: pyhealth.tasks.generate_ehr.decode_dataset + +.. autofunction:: pyhealth.tasks.generate_ehr.to_evaluation_dataframe diff --git a/examples/halo_mimic3.py b/examples/halo_mimic3.py new file mode 100644 index 000000000..3ea0a71be --- /dev/null +++ b/examples/halo_mimic3.py @@ -0,0 +1,132 @@ +"""Example: train HALO on MIMIC-III and generate synthetic patients. + +This example demonstrates: +1. Loading MIMIC-III data +2. Applying the EHRGenerationMIMIC3 task (per-visit ICD-9 code sequences) +3. Creating a SampleDataset with a NestedSequenceProcessor +4. Training the HALO generator with its custom training loop +5. Generating synthetic patients +6. Evaluating the synthetic data with the generative metrics suite +""" + +import pandas as pd + +from pyhealth.datasets import MIMIC3Dataset, split_by_patient +from pyhealth.metrics.generative import evaluate_synthetic_ehr +from pyhealth.models import HALO +from pyhealth.tasks import EHRGenerationMIMIC3 + +if __name__ == "__main__": + # STEP 1: Load MIMIC-III base dataset + base_dataset = MIMIC3Dataset( + root="/srv/local/data/MIMIC-III/mimic-iii-clinical-database-1.4", + tables=["diagnoses_icd"], + dev=True, + ) + + # STEP 2: Apply the EHR generation task (unconditional, no labels). + # This task is shared by all generators in pyhealth.models.generators. + sample_dataset = base_dataset.set_task(EHRGenerationMIMIC3()) + print(f"Total samples: {len(sample_dataset)}") + print(f"Input schema: {sample_dataset.input_schema}") + print(f"Output schema: {sample_dataset.output_schema}") + + sample = sample_dataset[0] + print("\nSample structure:") + print(f" Patient ID: {sample['patient_id']}") + print(f" Visits tensor shape: {tuple(sample['visits'].shape)}") + + # STEP 3: Split dataset by patient + train_dataset, val_dataset, test_dataset = split_by_patient( + sample_dataset, [0.8, 0.1, 0.1] + ) + + # STEP 4: Initialize HALO (small config for the dev subset) + model = HALO( + dataset=sample_dataset, + embed_dim=128, + n_heads=4, + n_layers=4, + n_ctx=48, + batch_size=16, + epochs=5, + lr=1e-4, + save_dir="./halo_save", + ) + num_params = sum(p.numel() for p in model.parameters()) + print(f"\nModel initialized with {num_params} parameters") + + # STEP 5: Train with HALO's custom loop (saves best checkpoint to save_dir) + model.train_model(train_dataset, val_dataset=val_dataset) + + # STEP 6: Generate synthetic patients (one per real training patient). + synthetic = model.generate(num_samples=len(train_dataset), random_sampling=True) + print("\nGenerated synthetic patients (first 3):") + for patient in synthetic[:3]: + print(f" {patient['patient_id']}: {len(patient['visits'])} visits") + print(f" {patient['visits']}") + + # STEP 7: Evaluate the synthetic data with the generative metrics suite. + # evaluate_synthetic_ehr (and every metric it calls) expects flat / + # long-format dataframes -- ONE ROW PER (patient, visit, code) event -- + # with four columns: + # - id patient identifier (any hashable; str here) + # - time visit index / timestep (sortable; int here) + # - visit_codes a SINGLE medical code (str or int; one per row, + # NOT a list/array -- a + # visit with k codes spans + # k rows) + # - labels per-patient binary label (0/1, int) + # train_df, test_df and syn_df below all share this exact schema. `labels` + # is a placeholder here: privacy metrics ignore it and the utility metric + # overwrites it with the next-visit prediction target. + index_to_code = { + v: k for k, v in sample_dataset.input_processors["visits"].code_vocab.items() + } + + def real_subset_to_records(subset): + for sample in subset: + pid = str(sample["patient_id"]) + visits_tensor = sample["visits"] + for t, visit in enumerate(visits_tensor.tolist()): + for idx in visit: + code = index_to_code.get(int(idx)) + if code in (None, "", ""): + continue + yield {"id": pid, "time": t, "visit_codes": code, "labels": 0} + + def synthetic_to_records(patients): + for p in patients: + pid = str(p["patient_id"]) + for t, visit in enumerate(p["visits"]): + for code in visit: + yield {"id": pid, "time": t, "visit_codes": code, "labels": 0} + + schema = {"visit_codes": str, "labels": int, "time": int, "id": str} + train_df = pd.DataFrame(real_subset_to_records(train_dataset)).astype(schema) + test_df = pd.DataFrame(real_subset_to_records(test_dataset)).astype(schema) + syn_df = pd.DataFrame(synthetic_to_records(synthetic)).astype(schema) + print( + f"\nEval rows -- train: {len(train_df)}, test: {len(test_df)}, " + f"synthetic: {len(syn_df)}" + ) + # Show the flat schema: one row per (patient, visit, code) event. + print("\ntrain_df schema (one row per (patient, visit, code)):") + print(train_df.head()) + + # sample_size / n_bootstraps / n_runs are kept small for the dev subset; + # raise them when running on the full MIMIC-III cohort. + results = evaluate_synthetic_ehr( + train_ehr=train_df, + test_ehr=test_df, + syn_ehr=syn_df, + sample_size=min(30, len(train_dataset), len(test_dataset)), + mode="lstm", + metrics="all", + lstm_params={"embed_dim": 16, "hidden_dim": 16, "batch_size": 16, "epochs": 3}, + n_bootstraps=5, + n_runs=3, + ) + print("\nGenerative metrics (mean +/- std):") + for name, (mean, std) in results.items(): + print(f" {name:30s} {mean:.4f} +/- {std:.4f}") diff --git a/pyhealth/metrics/__init__.py b/pyhealth/metrics/__init__.py index da8da0f5b..f04b6ba6a 100644 --- a/pyhealth/metrics/__init__.py +++ b/pyhealth/metrics/__init__.py @@ -1,5 +1,13 @@ from .binary import binary_metrics_fn from .drug_recommendation import ddi_rate_score +from .generative import ( + calc_membership_inference, + calc_nnaar, + compute_discriminator_privacy, + compute_mle, + compute_prevalence_metrics, + evaluate_synthetic_ehr, +) from .interpretability import ( ComprehensivenessMetric, Evaluator, @@ -17,6 +25,12 @@ __all__ = [ "binary_metrics_fn", "ddi_rate_score", + "calc_nnaar", + "calc_membership_inference", + "compute_discriminator_privacy", + "compute_mle", + "compute_prevalence_metrics", + "evaluate_synthetic_ehr", "ComprehensivenessMetric", "SufficiencyMetric", "RemovalBasedMetric", diff --git a/pyhealth/metrics/generative/__init__.py b/pyhealth/metrics/generative/__init__.py new file mode 100644 index 000000000..bf710dbf0 --- /dev/null +++ b/pyhealth/metrics/generative/__init__.py @@ -0,0 +1,276 @@ +"""Evaluation metrics for synthetic (generative) EHR data. + +This subpackage provides metrics for assessing synthetic electronic health +record (EHR) data along three axes: + + - **Privacy** (:mod:`pyhealth.metrics.generative.privacy`): NNAAR, + membership inference, and discriminator-based adversarial accuracy. + - **Utility / fidelity** (:mod:`pyhealth.metrics.generative.utility`): + machine learning efficacy (TRTR vs TSTR) and code-prevalence similarity. + +The convenience function :func:`evaluate_synthetic_ehr` runs the full suite +and returns a single merged dictionary of ``{metric_name: (mean, std)}``. + +Input format: + Every metric consumes plain pandas dataframes in *flat / long* format -- + **one row per (patient, visit, code) event** -- so the logic stays easy to + inspect. By default each dataframe has four columns + ``[id, time, visit_codes, labels]`` (override the names via the + ``subject_col`` / ``visit_col`` / ``code_col`` / ``label_col`` arguments): + + - ``id`` (``subject_col``): patient identifier. Any hashable value; + commonly ``str`` or ``int``. + - ``time`` (``visit_col``): visit index / timestep. Sortable, usually + ``int``; visits are ordered per patient by this column. + - ``visit_codes`` (``code_col``): a **single** medical code for this + row (``str`` or ``int``). One code per row -- a visit containing *k* + codes spans *k* rows. Cells are scalars, **not** lists or arrays. + - ``labels`` (``label_col``): per-patient binary label (0/1, ``int``). + + The real (``train_ehr``, ``test_ehr``) and synthetic (``syn_ehr``) + dataframes must all share this same schema. ``labels`` is ignored by the + privacy metrics and is overwritten internally by the utility metrics, but + is required so every dataframe has a uniform schema. + +Why dataframes (and not plain ``List[...]``)? + The flat dataframe is purely the *interchange* format -- a single, uniform + interface shared by every metric and produced once by + :func:`pyhealth.tasks.to_evaluation_dataframe`. Internally each family uses + whatever representation is most natural: + + - **Privacy** metrics immediately reduce the frame to a nested + ``List[List[set]]`` (sequence of per-visit code sets) via + ``convert_visits_to_sets`` and do all distance work on plain Python + lists -- no pandas in the hot loop. + - **Utility / fidelity** metrics genuinely benefit from the dataframe: + code-prevalence uses ``groupby(...).nunique()``, MLE builds the + next-visit-prediction supervision with grouped per-patient label + assignment, and the discriminator metric concatenates / filters / + relabels real-vs-synthetic rows. Re-implementing these on raw lists + would be more code for no gain. + + So the long-form frame keeps the public API consistent and the heavy + transforms readable, while the per-metric internals are free to drop down + to lists where that is simpler. + +Note: + The MLE (utility) component is currently hard-coded to next-visit + prediction and is therefore only meaningful for sequential generators + (HALO, GPT2, PromptEHR). It will be expanded to support pluggable + downstream tasks so that bag-of-codes generators (MedGAN, CorGAN) can + be evaluated with a static-label task (e.g. mortality, readmission). + Until then, prefer the privacy and prevalence metrics when evaluating + MedGAN/CorGAN output. +""" + +import logging +from typing import Dict, Optional, Tuple + +import pandas as pd + +from .privacy import ( + calc_membership_inference, + calc_nnaar, + compute_discriminator_privacy, +) +from .utility import compute_mle, compute_prevalence_metrics +from .utils import train_lstm_model, train_sklearn_model + +logger = logging.getLogger(__name__) + +__all__ = [ + "calc_nnaar", + "calc_membership_inference", + "compute_discriminator_privacy", + "compute_mle", + "compute_prevalence_metrics", + "evaluate_synthetic_ehr", +] + + +def evaluate_synthetic_ehr( + train_ehr: pd.DataFrame, + test_ehr: pd.DataFrame, + syn_ehr: pd.DataFrame, + subject_col: str = "id", + visit_col: str = "time", + code_col: str = "visit_codes", + label_col: str = "labels", + sample_size: int = 1000, + mode: str = "lstm", + metrics: str = "all", + lstm_params: Optional[Dict] = None, + sklearn_params: Optional[Dict] = None, + n_bootstraps: int = 100, + n_runs: int = 5, +) -> Dict[str, Tuple[float, float]]: + """Runs the full synthetic-EHR evaluation suite. + + Computes privacy and/or utility metrics comparing synthetic EHR data + against real train/test data, and returns a single merged dictionary. + + All three dataframes are flat / long-format (one row per + ``(patient, visit, code)`` event) and must share the same schema. See the + module docstring (:mod:`pyhealth.metrics.generative`) for the full column + contract, and the example below for how to build them. + + Args: + train_ehr: Real training EHR dataframe, flat + ``[id, time, visit_codes, labels]`` format. + test_ehr: Real held-out test EHR dataframe; same schema as ``train_ehr``. + syn_ehr: Synthetic EHR dataframe; same schema as ``train_ehr``. + subject_col: Column name for patient/subject identifiers. + visit_col: Column name for visit/timestep identifiers. + code_col: Column name for the medical codes (one code per row). + label_col: Column name for the per-patient binary label. + sample_size: Number of patients sampled per dataset for the + privacy metrics. + mode: Predictive backbone for the utility metrics; ``"lstm"`` uses the + built-in LSTM classifier, ``"rf"`` uses a random forest. + metrics: Which metric group to compute: ``"all"``, ``"privacy"`` or + ``"utility"``. + lstm_params: Optional overrides for the LSTM (``embed_dim``, + ``hidden_dim``, ``batch_size``, ``epochs``). + sklearn_params: Optional overrides for the sklearn model (``model``). + n_bootstraps: Number of bootstrap resamples for the utility metrics. + n_runs: Number of sampling runs for the privacy metrics. + + Returns: + Dictionary mapping each metric name to a ``(mean, std)`` tuple. + + Raises: + ValueError: If ``metrics`` or ``mode`` is not a recognized value. + + Examples: + The inputs are flat / long-format dataframes -- one row per + ``(patient, visit, code)`` event -- with four columns by default: + + - ``id``: patient identifier (any hashable; ``str`` or ``int``). + - ``time``: visit index / timestep (sortable; usually ``int``). + - ``visit_codes``: a single medical code for this row (``str`` or + ``int``). One code per row -- a visit with *k* codes spans *k* + rows; cells are scalars, not lists/arrays. + - ``labels``: per-patient binary label (0/1, ``int``). + + ``train_ehr``, ``test_ehr`` and ``syn_ehr`` must all share this schema. + + >>> import pandas as pd + >>> from pyhealth.metrics.generative import evaluate_synthetic_ehr + >>> + >>> # One row per (patient, visit, code). Patient "p0" has two visits + >>> # (time 0 with two codes, time 1 with one code); "p1" has one visit. + >>> rows = [ + ... {"id": "p0", "time": 0, "visit_codes": "428.0", "labels": 0}, + ... {"id": "p0", "time": 0, "visit_codes": "250.00", "labels": 0}, + ... {"id": "p0", "time": 1, "visit_codes": "401.9", "labels": 0}, + ... {"id": "p1", "time": 0, "visit_codes": "428.0", "labels": 0}, + ... ] + >>> train_ehr = pd.DataFrame(rows) + >>> test_ehr = train_ehr.copy() # same schema; real held-out patients + >>> syn_ehr = train_ehr.copy() # same schema; generator output + >>> + >>> results = evaluate_synthetic_ehr( + ... train_ehr, test_ehr, syn_ehr, metrics="privacy", sample_size=2 + ... ) + >>> nnaar_mean, nnaar_std = results["nnaar"] + >>> + >>> # Custom column names: pass *_col to match your dataframe. + >>> results = evaluate_synthetic_ehr( + ... train_ehr, test_ehr, syn_ehr, + ... subject_col="id", visit_col="time", + ... code_col="visit_codes", label_col="labels", + ... ) + """ + if metrics not in ("all", "privacy", "utility"): + raise ValueError( + f"Unknown metrics group: {metrics!r}. " + "Expected 'all', 'privacy' or 'utility'." + ) + if mode not in ("lstm", "rf"): + raise ValueError(f"Unknown mode: {mode!r}. Expected 'lstm' or 'rf'.") + + lstm_params = lstm_params or {} + sklearn_params = sklearn_params or {} + final_output: Dict[str, Tuple[float, float]] = {} + + if metrics in ("all", "privacy"): + final_output.update( + calc_nnaar( + train_ehr, + test_ehr, + syn_ehr, + subject_col=subject_col, + visit_col=visit_col, + code_col=code_col, + label_col=label_col, + sample_size=sample_size, + n_runs=n_runs, + ) + ) + final_output.update( + calc_membership_inference( + train_ehr, + test_ehr, + syn_ehr, + subject_col=subject_col, + visit_col=visit_col, + code_col=code_col, + label_col=label_col, + num_attack_samples=sample_size, + n_runs=n_runs, + ) + ) + + if metrics in ("all", "utility"): + if mode == "lstm": + train_fn = train_lstm_model + train_kwargs = { + "embed_dim": lstm_params.get("embed_dim", 32), + "hidden_dim": lstm_params.get("hidden_dim", 32), + "batch_size": lstm_params.get("batch_size", 32), + "epochs": lstm_params.get("epochs", 5), + "verbose": False, + } + else: + train_fn = train_sklearn_model + train_kwargs = {"model": sklearn_params.get("model", "rf")} + + final_output.update( + compute_mle( + train_fn=train_fn, + train_ehr=train_ehr, + test_ehr=test_ehr, + syn_ehr=syn_ehr, + subject_col=subject_col, + visit_col=visit_col, + code_col=code_col, + label_col=label_col, + n_bootstraps=n_bootstraps, + **train_kwargs, + ) + ) + final_output.update( + compute_discriminator_privacy( + train_fn=train_fn, + train_ehr=train_ehr, + test_ehr=test_ehr, + syn_ehr=syn_ehr, + subject_col=subject_col, + visit_col=visit_col, + code_col=code_col, + label_col=label_col, + n_bootstraps=n_bootstraps, + **train_kwargs, + ) + ) + final_output.update( + compute_prevalence_metrics( + train_ehr, + syn_ehr, + subject_col=subject_col, + code_col=code_col, + n_bootstraps=n_bootstraps, + ) + ) + + return final_output diff --git a/pyhealth/metrics/generative/privacy.py b/pyhealth/metrics/generative/privacy.py new file mode 100644 index 000000000..70edc4957 --- /dev/null +++ b/pyhealth/metrics/generative/privacy.py @@ -0,0 +1,386 @@ +"""Privacy metrics for synthetic EHR data. + +These metrics quantify how much a synthetic EHR dataset leaks about the real +records it was trained on. They include: + + - Nearest Neighbor Adversarial Accuracy Risk (NNAAR) + - Membership Inference Attack (MIA) metrics + - A discriminator-based adversarial-accuracy privacy score + +All functions take flat / long-format EHR dataframes -- one row per +``(patient, visit, code)`` event, with default columns +``[id, time, visit_codes, labels]`` (see :mod:`pyhealth.metrics.generative` +for the full column contract) -- and return ``{metric_name: (mean, std)}`` +summaries computed over multiple runs or bootstrap resamples. The real +(``train_ehr``, ``test_ehr``) and synthetic (``syn_ehr``) dataframes must share +the same schema. +""" + +import copy +import logging +from typing import Callable, Dict, Tuple + +import numpy as np +import pandas as pd +from sklearn import metrics as sklearn_metrics +from sklearn.model_selection import train_test_split +from tqdm import tqdm + +from .utils import ( + convert_visits_to_sets, + find_nearest_neighbor_dist, + summarize_metric_runs, +) + +logger = logging.getLogger(__name__) + +__all__ = [ + "calc_nnaar", + "calc_membership_inference", + "compute_discriminator_privacy", +] + + +def calc_nnaar( + train_ehr: pd.DataFrame, + test_ehr: pd.DataFrame, + syn_ehr: pd.DataFrame, + subject_col: str = "id", + visit_col: str = "time", + code_col: str = "visit_codes", + label_col: str = "labels", + sample_size: int = 1000, + n_runs: int = 5, + verbose: bool = False, +) -> Dict[str, Tuple[float, float]]: + """Computes the Nearest Neighbor Adversarial Accuracy Risk (NNAAR). + + NNAAR measures whether the synthetic data sits closer to the real training + data than to held-out test data, which would indicate memorization:: + + NNAAR = AA_ES - AA_TS + + where ``AA_ES`` is the adversarial accuracy between test and synthetic data + and ``AA_TS`` is the adversarial accuracy between train and synthetic data. + Values near 0 indicate low privacy risk. + + All three dataframes are flat ``[id, time, visit_codes, labels]`` frames + sharing the same schema (see :mod:`pyhealth.metrics.generative`). + + Args: + train_ehr: Real training EHR dataframe, flat + ``[id, time, visit_codes, labels]`` format. + test_ehr: Real held-out test EHR dataframe; same schema as ``train_ehr``. + syn_ehr: Synthetic EHR dataframe; same schema as ``train_ehr``. + subject_col: Column name for patient/subject identifiers. + visit_col: Column name for visit/timestep identifiers. + code_col: Column name for the medical codes (one code per row). + label_col: Column name for the label (unused, kept for a uniform API). + sample_size: Number of patients to sample per dataset per run. + n_runs: Number of independent sampling runs. + verbose: Whether to show per-run progress bars. + + Returns: + Dictionary mapping ``"nnaar"``, ``"aa_es"`` and ``"aa_ts"`` to their + ``(mean, std)`` across runs. + + Examples: + >>> from pyhealth.metrics.generative import calc_nnaar + >>> # train_ehr, test_ehr, syn_ehr are flat + >>> # [id, time, visit_codes, labels] dataframes sharing one schema -- + >>> # see evaluate_synthetic_ehr for how to build them. + >>> result = calc_nnaar(train_ehr, test_ehr, syn_ehr) + >>> nnaar_mean, nnaar_std = result["nnaar"] + """ + logger.info( + "Calculating NNAAR (sample_size=%d, n_runs=%d)", sample_size, n_runs + ) + train = convert_visits_to_sets(train_ehr, subject_col, visit_col, code_col) + test = convert_visits_to_sets(test_ehr, subject_col, visit_col, code_col) + synthetic = convert_visits_to_sets(syn_ehr, subject_col, visit_col, code_col) + + metrics_runs = [] + n = min(sample_size, len(train), len(test), len(synthetic)) + + for _ in range(n_runs): + if len(train) > n: + inds = np.random.choice(len(train), n, replace=False) + s_train = [train[i] for i in inds] + else: + s_train = list(train) + if len(test) > n: + inds = np.random.choice(len(test), n, replace=False) + s_test = [test[i] for i in inds] + else: + s_test = list(test) + if len(synthetic) > n: + inds = np.random.choice(len(synthetic), n, replace=False) + s_syn = [synthetic[i] for i in inds] + else: + s_syn = list(synthetic) + + # AA_ES (test vs synthetic). The within-set term searches the set the + # query came from, so we pass ``skip_index`` to exclude the query + # itself (otherwise every within-set distance is a trivial 0 self-match). + val1 = sum( + 1 + for i, p in enumerate( + tqdm(s_test, desc="Test vs Syn", disable=not verbose) + ) + if find_nearest_neighbor_dist(p, s_syn) + > find_nearest_neighbor_dist(p, s_test, skip_index=i) + ) + val2 = sum( + 1 + for i, p in enumerate( + tqdm(s_syn, desc="Syn vs Test", disable=not verbose) + ) + if find_nearest_neighbor_dist(p, s_test) + > find_nearest_neighbor_dist(p, s_syn, skip_index=i) + ) + # AA_TS (train vs synthetic). + val3 = sum( + 1 + for i, p in enumerate( + tqdm(s_train, desc="Train vs Syn", disable=not verbose) + ) + if find_nearest_neighbor_dist(p, s_syn) + > find_nearest_neighbor_dist(p, s_train, skip_index=i) + ) + val4 = sum( + 1 + for i, p in enumerate( + tqdm(s_syn, desc="Syn vs Train", disable=not verbose) + ) + if find_nearest_neighbor_dist(p, s_train) + > find_nearest_neighbor_dist(p, s_syn, skip_index=i) + ) + + aa_es = 0.5 * (val1 / n + val2 / n) + aa_ts = 0.5 * (val3 / n + val4 / n) + metrics_runs.append( + {"nnaar": aa_es - aa_ts, "aa_es": aa_es, "aa_ts": aa_ts} + ) + + return summarize_metric_runs(metrics_runs) + + +def calc_membership_inference( + train_ehr: pd.DataFrame, + test_ehr: pd.DataFrame, + syn_ehr: pd.DataFrame, + subject_col: str = "id", + visit_col: str = "time", + code_col: str = "visit_codes", + label_col: str = "labels", + num_attack_samples: int = 1000, + n_runs: int = 5, + verbose: bool = False, +) -> Dict[str, Tuple[float, float]]: + """Computes Membership Inference Attack (MIA) metrics. + + An attacker tries to tell members (training patients) from non-members + (test patients) using proximity to the synthetic data: members are expected + to be closer to synthetic records. Predictions are made by thresholding the + nearest-neighbor distance at its median; F1, precision, recall and accuracy + near 0.5 indicate low membership-inference risk. + + All three dataframes are flat ``[id, time, visit_codes, labels]`` frames + sharing the same schema (see :mod:`pyhealth.metrics.generative`). + + Args: + train_ehr: Real training EHR dataframe (members), flat + ``[id, time, visit_codes, labels]`` format. + test_ehr: Real held-out test EHR dataframe (non-members); same schema as + ``train_ehr``. + syn_ehr: Synthetic EHR dataframe; same schema as ``train_ehr``. + subject_col: Column name for patient/subject identifiers. + visit_col: Column name for visit/timestep identifiers. + code_col: Column name for the medical codes (one code per row). + label_col: Column name for the label (unused, kept for a uniform API). + num_attack_samples: Total attack-set size (half members, half not). + n_runs: Number of independent sampling runs. + verbose: Whether to show per-run progress bars. + + Returns: + Dictionary mapping ``"MIA_F1"``, ``"MIA_Precision"``, ``"MIA_Recall"`` + and ``"MIA_Accuracy"`` to their ``(mean, std)`` across runs. + + Examples: + >>> from pyhealth.metrics.generative import calc_membership_inference + >>> # train_ehr, test_ehr, syn_ehr are flat + >>> # [id, time, visit_codes, labels] dataframes sharing one schema -- + >>> # see evaluate_synthetic_ehr for how to build them. + >>> result = calc_membership_inference(train_ehr, test_ehr, syn_ehr) + >>> f1_mean, f1_std = result["MIA_F1"] + """ + logger.info( + "Calculating Membership Inference (attack_size=%d, n_runs=%d)", + num_attack_samples, + n_runs, + ) + train = convert_visits_to_sets(train_ehr, subject_col, visit_col, code_col) + test = convert_visits_to_sets(test_ehr, subject_col, visit_col, code_col) + synthetic = convert_visits_to_sets(syn_ehr, subject_col, visit_col, code_col) + + metrics_runs = [] + for _ in range(n_runs): + # Build a balanced attack set: 50% members, 50% non-members. + n_half = min(len(train), len(test), num_attack_samples) // 2 + if n_half == 0: + continue + + pos_inds = np.random.choice(len(train), n_half, replace=False) + pos_samples = [train[i] for i in pos_inds] + neg_inds = np.random.choice(len(test), n_half, replace=False) + neg_samples = [test[i] for i in neg_inds] + + attack_data = pos_samples + neg_samples + attack_labels = [1] * len(pos_samples) + [0] * len(neg_samples) + + distances = [ + find_nearest_neighbor_dist(record, synthetic) + for record in tqdm( + attack_data, desc="Calculating Distances", disable=not verbose + ) + ] + if len(distances) == 0: + continue + + # Members are expected to be closer (smaller distance) to synthetic. + median_dist = np.median(distances) + predictions = [1 if d < median_dist else 0 for d in distances] + + metrics_runs.append( + { + "MIA_F1": sklearn_metrics.f1_score(attack_labels, predictions), + "MIA_Precision": sklearn_metrics.precision_score( + attack_labels, predictions, zero_division=0 + ), + "MIA_Recall": sklearn_metrics.recall_score( + attack_labels, predictions, zero_division=0 + ), + "MIA_Accuracy": sklearn_metrics.accuracy_score( + attack_labels, predictions + ), + } + ) + + summary = summarize_metric_runs(metrics_runs) + logger.info("MIA results: %s", summary) + return summary + + +def compute_discriminator_privacy( + train_fn: Callable, + train_ehr: pd.DataFrame, + test_ehr: pd.DataFrame, + syn_ehr: pd.DataFrame, + subject_col: str = "id", + visit_col: str = "time", + code_col: str = "visit_codes", + label_col: str = "labels", + n_bootstraps: int = 5, + seed: int = 4, + **kwargs, +) -> Dict[str, Tuple[float, float]]: + """Computes a discriminator-based adversarial-accuracy privacy score. + + A classifier is trained to predict whether a record is real (1) or + synthetic (0). An accuracy near 0.5 means real and synthetic data are + indistinguishable (good privacy); accuracy well above 0.5 means the + synthetic data is easy to tell apart (poor privacy). The ``Privacy_Score`` + rescales accuracy so 1.0 is perfect privacy and 0.0 is none. + + Args: + train_fn: A training function such as + :func:`pyhealth.metrics.generative.utils.train_lstm_model` or + ``train_sklearn_model``. It must accept ``train_ehr``, ``test_ehr``, + the four column-name arguments and return ``(model, y_true, + y_pred)``. + train_ehr: Real training EHR dataframe, flat + ``[id, time, visit_codes, labels]`` format. + test_ehr: Real held-out test EHR dataframe (unused; kept for a uniform + API with the other metrics); same schema as ``train_ehr``. + syn_ehr: Synthetic EHR dataframe; same schema as ``train_ehr``. + subject_col: Column name for patient/subject identifiers. + visit_col: Column name for visit/timestep identifiers. + code_col: Column name for the medical codes (one code per row). + label_col: Column name for the original label (unused; the + discriminator target replaces it). + n_bootstraps: Number of bootstrap resamples of the predictions. + seed: Random seed for the patient-level train/test split. + **kwargs: Extra keyword arguments forwarded to ``train_fn``. + + Returns: + Dictionary mapping ``"Privacy_Discriminator_Accuracy"`` and + ``"Privacy_Score"`` to their ``(mean, std)`` across bootstraps. + + Examples: + >>> from pyhealth.metrics.generative import compute_discriminator_privacy + >>> from pyhealth.metrics.generative.utils import train_lstm_model + >>> # train_ehr, test_ehr, syn_ehr are flat + >>> # [id, time, visit_codes, labels] dataframes sharing one schema -- + >>> # see evaluate_synthetic_ehr for how to build them. + >>> result = compute_discriminator_privacy( + ... train_lstm_model, train_ehr, test_ehr, syn_ehr + ... ) + >>> score_mean, score_std = result["Privacy_Score"] + """ + logger.info("Computing discriminator privacy") + + # Label data: real = 1, synthetic = 0. + real_df = copy.deepcopy(train_ehr) + syn_df = copy.deepcopy(syn_ehr) + disc_label = "is_real" + real_df[disc_label] = 1 + syn_df[disc_label] = 0 + + # Disambiguate subject IDs so real/synthetic patients never collide. + real_df[subject_col] = real_df[subject_col].astype(str) + "_real" + syn_df[subject_col] = syn_df[subject_col].astype(str) + "_syn" + + combined_df = pd.concat([real_df, syn_df]) + unique_patients = combined_df[subject_col].unique() + train_ids, test_ids = train_test_split( + unique_patients, test_size=0.2, random_state=seed + ) + disc_train = combined_df[combined_df[subject_col].isin(train_ids)] + disc_test = combined_df[combined_df[subject_col].isin(test_ids)] + + logger.info( + "Discriminator train size=%d, test size=%d", + len(disc_train), + len(disc_test), + ) + _, y_true, y_pred = train_fn( + train_ehr=disc_train, + test_ehr=disc_test, + subject_col=subject_col, + visit_col=visit_col, + code_col=code_col, + label_col=disc_label, + **kwargs, + ) + + metrics_runs = [] + n_samples = len(y_true) + for _ in range(n_bootstraps): + if n_samples > 0: + indices = np.random.choice(n_samples, n_samples, replace=True) + y_t, y_p = y_true[indices], y_pred[indices] + else: + y_t, y_p = y_true, y_pred + + acc = sklearn_metrics.accuracy_score(y_t, y_p) if len(y_t) > 0 else 0.0 + metrics_runs.append( + { + "Privacy_Discriminator_Accuracy": acc, + # 1.0 = perfect privacy (acc 0.5); 0.0 = no privacy. + "Privacy_Score": 1.0 - 2 * abs(0.5 - acc), + } + ) + + summary = summarize_metric_runs(metrics_runs) + logger.info("Discriminator privacy results: %s", summary) + return summary diff --git a/pyhealth/metrics/generative/utility.py b/pyhealth/metrics/generative/utility.py new file mode 100644 index 000000000..d36ffcacd --- /dev/null +++ b/pyhealth/metrics/generative/utility.py @@ -0,0 +1,285 @@ +"""Utility and statistical-fidelity metrics for synthetic EHR data. + +These metrics quantify how *useful* synthetic EHR data is as a stand-in for +real data: + + - Machine Learning Efficacy (MLE): compares a model trained on real data + against one trained on synthetic data, both evaluated on real data. + - Code-prevalence similarity: compares per-code patient-level prevalence + between real and synthetic data (R-squared, Pearson correlation, RMSE). + +All functions take flat / long-format EHR dataframes -- one row per +``(patient, visit, code)`` event, with default columns +``[id, time, visit_codes, labels]`` (see :mod:`pyhealth.metrics.generative` +for the full column contract) -- and return ``{metric_name: (mean, std)}`` +summaries over bootstrap resamples. The real (``train_ehr``, ``test_ehr``) and +synthetic (``syn_ehr``) dataframes must share the same schema. +""" + +import copy +import logging +from typing import Callable, Dict, Tuple + +import numpy as np +import pandas as pd +from sklearn import metrics as sklearn_metrics + +from .utils import build_next_visit_prediction_dataset, summarize_metric_runs + +logger = logging.getLogger(__name__) + +__all__ = [ + "compute_mle", + "compute_prevalence_metrics", +] + + +def compute_mle( + train_fn: Callable, + train_ehr: pd.DataFrame, + test_ehr: pd.DataFrame, + syn_ehr: pd.DataFrame, + subject_col: str = "id", + visit_col: str = "time", + code_col: str = "visit_codes", + label_col: str = "labels", + n_bootstraps: int = 5, + **kwargs, +) -> Dict[str, Tuple[float, float]]: + """Computes Machine Learning Efficacy (utility) for synthetic data. + + Two classifiers are trained on a next-visit prediction task: one on real + training data (Train-Real-Test-Real, TRTR) and one on synthetic data + (Train-Synthetic-Test-Real, TSTR). Both are evaluated on the same real test + set. Synthetic accuracy/F1 close to real accuracy/F1 indicates high utility. + + Note: + The current implementation hard-codes the downstream task to + next-visit prediction (built via + :func:`build_next_visit_prediction_dataset`). This is degenerate for + bag-of-codes generators such as MedGAN and CorGAN, which emit a + single aggregate visit per patient and so always get label=0. A + future revision will let callers plug in static-label tasks + (mortality, readmission, "ever diagnosed with X", ...) so MLE is + meaningful for both sequential (HALO, GPT2, PromptEHR) and + bag-of-codes (MedGAN, CorGAN) generators. + + Args: + train_fn: A training function such as + :func:`pyhealth.metrics.generative.utils.train_lstm_model` or + ``train_sklearn_model``, returning ``(model, y_true, y_pred)``. + train_ehr: Real training EHR dataframe, flat + ``[id, time, visit_codes, labels]`` format. + test_ehr: Real held-out test EHR dataframe; same schema as ``train_ehr``. + syn_ehr: Synthetic EHR dataframe; same schema as ``train_ehr``. + subject_col: Column name for patient/subject identifiers. + visit_col: Column name for visit/timestep identifiers. + code_col: Column name for the medical codes (one code per row). + label_col: Column name for the label (overwritten by the next-visit + prediction label). + n_bootstraps: Number of bootstrap resamples of the predictions. + **kwargs: Extra keyword arguments forwarded to ``train_fn``. + + Returns: + Dictionary mapping the MLE metrics (real/synthetic accuracy and F1, + their difference and ratio) to their ``(mean, std)`` across + bootstraps. + + Examples: + >>> from pyhealth.metrics.generative.utility import compute_mle + >>> from pyhealth.metrics.generative.utils import train_lstm_model + >>> # train_ehr, test_ehr, syn_ehr are flat + >>> # [id, time, visit_codes, labels] dataframes sharing one schema -- + >>> # see evaluate_synthetic_ehr for how to build them. + >>> result = compute_mle(train_lstm_model, train_ehr, test_ehr, syn_ehr) + >>> synth_acc_mean, synth_acc_std = result["MLE_Synth_Accuracy"] + """ + logger.info("Computing MLE (utility)") + + train_task = build_next_visit_prediction_dataset( + train_ehr, subject_col, visit_col, label_col + ) + test_task = build_next_visit_prediction_dataset( + test_ehr, subject_col, visit_col, label_col + ) + syn_task = build_next_visit_prediction_dataset( + syn_ehr, subject_col, visit_col, label_col + ) + + # Train on Real, test on Real (TRTR). + _, real_y_true, real_y_pred = train_fn( + copy.deepcopy(train_task), + copy.deepcopy(test_task), + subject_col=subject_col, + visit_col=visit_col, + code_col=code_col, + label_col=label_col, + **kwargs, + ) + # Train on Synthetic, test on Real (TSTR). + _, syn_y_true, syn_y_pred = train_fn( + copy.deepcopy(syn_task), + copy.deepcopy(test_task), + subject_col=subject_col, + visit_col=visit_col, + code_col=code_col, + label_col=label_col, + **kwargs, + ) + + metrics_runs = [] + n_samples = len(real_y_true) + for _ in range(n_bootstraps): + if n_samples > 0: + indices = np.random.choice(n_samples, n_samples, replace=True) + r_true, r_pred = real_y_true[indices], real_y_pred[indices] + s_true, s_pred = syn_y_true[indices], syn_y_pred[indices] + else: + r_true, r_pred = real_y_true, real_y_pred + s_true, s_pred = syn_y_true, syn_y_pred + + real_acc = ( + sklearn_metrics.accuracy_score(r_true, r_pred) + if len(r_true) > 0 + else 0.0 + ) + syn_acc = ( + sklearn_metrics.accuracy_score(s_true, s_pred) + if len(s_true) > 0 + else 0.0 + ) + real_f1 = ( + sklearn_metrics.f1_score(r_true, r_pred, average="macro") + if len(r_true) > 0 + else 0.0 + ) + syn_f1 = ( + sklearn_metrics.f1_score(s_true, s_pred, average="macro") + if len(s_true) > 0 + else 0.0 + ) + + metrics_runs.append( + { + "MLE_Real_Accuracy": real_acc, + "MLE_Synth_Accuracy": syn_acc, + "MLE_Difference": real_acc - syn_acc, + "MLE_Ratio": syn_acc / real_acc if real_acc > 0 else 0.0, + "MLE_Real_F1": real_f1, + "MLE_Synth_F1": syn_f1, + } + ) + + summary = summarize_metric_runs(metrics_runs) + logger.info("MLE results: %s", summary) + return summary + + +def compute_prevalence_metrics( + train_ehr: pd.DataFrame, + syn_ehr: pd.DataFrame, + subject_col: str = "id", + code_col: str = "visit_codes", + n_bootstraps: int = 5, +) -> Dict[str, Tuple[float, float]]: + """Compares per-code patient-level prevalence of real vs synthetic data. + + For every code, prevalence is the fraction of unique patients who have that + code at least once. The real and synthetic prevalence vectors are compared + with R-squared, Pearson correlation and RMSE; bootstrap resampling is over + codes. + + This metric only reads ``subject_col`` and ``code_col``, but ``train_ehr`` + and ``syn_ehr`` are expected to be the same flat + ``[id, time, visit_codes, labels]`` frames used by the other metrics. + + Args: + train_ehr: Real training EHR dataframe, flat + ``[id, time, visit_codes, labels]`` format. + syn_ehr: Synthetic EHR dataframe; same schema as ``train_ehr``. + subject_col: Column name for patient/subject identifiers. + code_col: Column name for the medical codes (one code per row). + n_bootstraps: Number of bootstrap resamples over codes. + + Returns: + Dictionary mapping ``"Prevalence_R2"``, ``"Prevalence_Pearson"`` and + ``"Prevalence_RMSE"`` to their ``(mean, std)`` across bootstraps. + + Examples: + >>> from pyhealth.metrics.generative.utility import ( + ... compute_prevalence_metrics, + ... ) + >>> # train_ehr and syn_ehr are flat [id, time, visit_codes, labels] + >>> # dataframes sharing one schema -- see evaluate_synthetic_ehr for + >>> # how to build them. + >>> result = compute_prevalence_metrics(train_ehr, syn_ehr) + >>> r2_mean, r2_std = result["Prevalence_R2"] + """ + logger.info("Computing prevalence metrics") + + all_codes = set() + all_codes.update(train_ehr[code_col].unique().tolist()) + all_codes.update(syn_ehr[code_col].unique().tolist()) + + n_train = train_ehr[subject_col].nunique() + n_syn = syn_ehr[subject_col].nunique() + if n_train == 0 or n_syn == 0: + return { + "Prevalence_R2": (0.0, 0.0), + "Prevalence_Pearson": (0.0, 0.0), + "Prevalence_RMSE": (0.0, 0.0), + } + + # Count unique patients per code. + train_counts = train_ehr.groupby(code_col)[subject_col].nunique() + syn_counts = syn_ehr.groupby(code_col)[subject_col].nunique() + for code in all_codes: + if code not in train_counts.index: + train_counts.loc[code] = 0 + if code not in syn_counts.index: + syn_counts.loc[code] = 0 + + train_probs = train_counts / n_train + syn_probs = syn_counts / n_syn + df_compare = pd.DataFrame( + {"real": train_probs, "syn": syn_probs} + ).fillna(0) + + metrics_runs = [] + n_samples = len(df_compare) + for _ in range(n_bootstraps): + if n_samples > 0: + df_sampled = df_compare.sample(n=n_samples, replace=True) + real_vec = df_sampled["real"].values + syn_vec = df_sampled["syn"].values + else: + real_vec = df_compare["real"].values + syn_vec = df_compare["syn"].values + + r2 = ( + sklearn_metrics.r2_score(real_vec, syn_vec) + if n_samples > 1 + else 0.0 + ) + # Pearson correlation via numpy (avoids a hard scipy dependency). + if len(np.unique(real_vec)) > 1 and len(np.unique(syn_vec)) > 1: + rho = float(np.corrcoef(real_vec, syn_vec)[0, 1]) + else: + rho = 0.0 + rmse = ( + float(np.sqrt(sklearn_metrics.mean_squared_error(real_vec, syn_vec))) + if n_samples > 0 + else 0.0 + ) + + metrics_runs.append( + { + "Prevalence_R2": r2, + "Prevalence_Pearson": rho, + "Prevalence_RMSE": rmse, + } + ) + + summary = summarize_metric_runs(metrics_runs) + logger.info("Prevalence results: %s", summary) + return summary diff --git a/pyhealth/metrics/generative/utils.py b/pyhealth/metrics/generative/utils.py new file mode 100644 index 000000000..d5f89d216 --- /dev/null +++ b/pyhealth/metrics/generative/utils.py @@ -0,0 +1,604 @@ +"""Shared utilities for synthetic-EHR generative evaluation metrics. + +This module contains the data-preparation helpers, distance functions, and +the lightweight predictive models (an LSTM classifier and a random-forest +baseline) that the privacy and utility metrics build on. It is not intended +to be used directly; see :mod:`pyhealth.metrics.generative.privacy` and +:mod:`pyhealth.metrics.generative.utility` for the public metric functions. +""" + +from typing import Dict, List, Optional, Tuple + +import numpy as np +import pandas as pd +import torch +import torch.nn as nn + +__all__ = [ + "summarize_metric_runs", + "convert_visits_to_sets", + "calculate_hamming_distance_cutoff", + "find_nearest_neighbor_dist", + "process_patient_data_for_lstm", + "collate_fn", + "EHRDataset", + "EHR_LSTM_Classifier", + "train_lstm_model", + "aggregate_patient_visits", + "train_sklearn_model", + "build_next_visit_prediction_dataset", + "convert_cols_to_multihot", +] + + +def summarize_metric_runs( + metrics_list: List[Dict[str, float]] +) -> Dict[str, Tuple[float, float]]: + """Summarizes a list of per-run metric dicts into (mean, std) tuples. + + Args: + metrics_list: List of dicts, one per run, mapping metric name to value. + + Returns: + Dictionary mapping each metric name to a ``(mean, std)`` tuple computed + across the runs. Returns an empty dict if ``metrics_list`` is empty. + """ + if not metrics_list: + return {} + summary: Dict[str, Tuple[float, float]] = {} + for key in metrics_list[0].keys(): + values = [run[key] for run in metrics_list if key in run] + summary[key] = (float(np.mean(values)), float(np.std(values))) + return summary + + +# --- Privacy distance helpers --------------------------------------------- + + +def convert_visits_to_sets( + df: pd.DataFrame, + subject_col: str = "id", + visit_col: str = "time", + code_col: str = "visit_codes", +) -> List[List[set]]: + """Converts a flat EHR dataframe into per-patient lists of code sets. + + Each patient becomes a list of visits, and each visit is a ``set`` of the + codes recorded at that timestep. + + Args: + df: Input dataframe with one row per (patient, visit, code) event. + subject_col: Column name for patient/subject identifiers. + visit_col: Column name for visit/timestep identifiers. + code_col: Column name for the medical codes. + + Returns: + List of patients, where each patient is a list of code sets. + """ + records = ( + df.groupby(subject_col)[[visit_col, code_col]] + .apply(lambda x: x.groupby(visit_col)[code_col].apply(set).tolist()) + .tolist() + ) + return records + + +def calculate_hamming_distance_cutoff( + v1: List[set], v2: List[set], cutoff: float +) -> float: + """Computes a set-based Hamming distance between two patients, with cutoff. + + The distance accumulates the symmetric-difference size of aligned visits + plus a penalty for differing sequence lengths. Computation stops early once + the running distance reaches ``cutoff``. + + Args: + v1: First patient as a list of code sets. + v2: Second patient as a list of code sets. + cutoff: Distance value at which to stop early. + + Returns: + The distance between ``v1`` and ``v2``, capped at ``cutoff``. + """ + len1, len2 = len(v1), len(v2) + dist = 0 if len1 == len2 else 1 + if dist >= cutoff: + return cutoff + + min_len = min(len1, len2) + for i in range(min_len): + dist += len(v1[i] ^ v2[i]) + if dist >= cutoff: + return cutoff + + if len1 > min_len: + dist += sum(len(v) for v in v1[min_len:]) + elif len2 > min_len: + dist += sum(len(v) for v in v2[min_len:]) + return dist + + +def find_nearest_neighbor_dist( + query: List[set], + reference_dataset: List[List[set]], + skip_index: Optional[int] = None, +) -> float: + """Finds the distance from a query patient to its nearest neighbor. + + Args: + query: Query patient as a list of code sets. + reference_dataset: Patients to search over. + skip_index: Optional index in ``reference_dataset`` to skip. Use this + when ``query`` is itself a member of ``reference_dataset`` (i.e. a + within-set nearest-neighbor search) so the patient does not match + itself at distance 0. Genuine duplicates at other indices can still + legitimately produce a distance of 0. + + Returns: + The smallest :func:`calculate_hamming_distance_cutoff` distance between + ``query`` and any patient in ``reference_dataset`` (excluding + ``skip_index`` when provided). + """ + best = float("inf") + for i, ref in enumerate(reference_dataset): + if i == skip_index: + continue + d = calculate_hamming_distance_cutoff(query, ref, best) + if d == 0: + return 0 + if d < best: + best = d + return best + + +# --- LSTM classifier ------------------------------------------------------- + + +def process_patient_data_for_lstm( + df: pd.DataFrame, + subject_col: str = "id", + visit_col: str = "time", + code_col: str = "visit_codes", + label_col: str = "labels", + code_to_idx: Optional[Dict] = None, +) -> Tuple[List[Tuple[torch.Tensor, int]], Dict]: + """Transforms a flat EHR dataframe into multi-hot visit sequences. + + Each patient is converted into a ``(seq_len, vocab_size)`` tensor of + multi-hot visit vectors, paired with a single static label (the per-patient + max of ``label_col``). + + Args: + df: Input dataframe with one row per (patient, visit, code) event. + subject_col: Column name for patient/subject identifiers. + visit_col: Column name for visit/timestep identifiers. + code_col: Column name for the medical codes. + label_col: Column name for the binary label. + code_to_idx: Optional precomputed mapping from code to integer index. + If ``None``, one is built from ``df``. + + Returns: + A tuple ``(patients, code_to_idx)`` where ``patients`` is a list of + ``(sequence_tensor, label)`` tuples. + """ + assert label_col in df.columns, f"Label column '{label_col}' not found." + assert subject_col in df.columns, f"Subject column '{subject_col}' not found." + assert visit_col in df.columns, f"Visit column '{visit_col}' not found." + + df = df.copy() + if code_to_idx is None: + vocab_size = df[code_col].nunique() + 1 + code_to_idx = { + code: idx for idx, code in enumerate(df[code_col].unique(), start=0) + } + else: + vocab_size = len(code_to_idx) + 1 + df[code_col] = df[code_col].map(code_to_idx) + + patients = [] + for _, group in df.groupby(subject_col): + # Static per-patient label: the max over visits (e.g. "ever diagnosed"). + label = group[label_col].max() + visits = group.sort_values(visit_col).groupby(visit_col) + patient_seq = [] + for _, visit_data in visits: + multi_hot = torch.zeros(vocab_size) + codes = visit_data[code_col].values + multi_hot[codes] = 1.0 + patient_seq.append(multi_hot) + patient_seq_tensor = torch.stack(patient_seq) + patients.append((patient_seq_tensor, label)) + + return patients, code_to_idx + + +def collate_fn(batch): + """Pads variable-length visit sequences for batched LSTM training. + + Args: + batch: List of ``(sequence_tensor, label)`` tuples. + + Returns: + A tuple ``(padded_seqs, lengths, labels)``. + """ + sequences, labels = zip(*batch) + lengths = torch.tensor([len(seq) for seq in sequences]) + padded_seqs = torch.nn.utils.rnn.pad_sequence( + sequences, batch_first=True, padding_value=0 + ) + labels = torch.tensor(labels, dtype=torch.float32) + return padded_seqs, lengths, labels + + +class EHRDataset(torch.utils.data.Dataset): + """A minimal :class:`torch.utils.data.Dataset` wrapper over a list.""" + + def __init__(self, data): + self.data = data + + def __len__(self): + return len(self.data) + + def __getitem__(self, idx): + return self.data[idx] + + +class EHR_LSTM_Classifier(nn.Module): + """A simple LSTM classifier over multi-hot EHR visit sequences. + + The model embeds each multi-hot visit vector, encodes the sequence with an + LSTM, and classifies using the final hidden state. + + Args: + vocab_size: Size of the code vocabulary (input dimension per visit). + embed_dim: Dimension of the dense visit embedding. + hidden_dim: Hidden dimension of the LSTM. + num_layers: Number of stacked LSTM layers. + """ + + def __init__( + self, + vocab_size: int, + embed_dim: int, + hidden_dim: int, + num_layers: int = 1, + ): + super().__init__() + self.embedding = nn.Linear(vocab_size, embed_dim) + self.lstm = nn.LSTM( + input_size=embed_dim, + hidden_size=hidden_dim, + num_layers=num_layers, + batch_first=True, + ) + self.fc = nn.Linear(hidden_dim, 1) + self.sigmoid = nn.Sigmoid() + + def forward(self, x: torch.Tensor, lengths: torch.Tensor) -> torch.Tensor: + x = self.embedding(x) + packed_x = torch.nn.utils.rnn.pack_padded_sequence( + x, lengths.cpu(), batch_first=True, enforce_sorted=False + ) + _, (h_n, _) = self.lstm(packed_x) + final_encoding = h_n[-1] + logits = self.fc(final_encoding) + probs = self.sigmoid(logits) + return probs.squeeze(-1) + + +def train_lstm_model( + train_ehr: pd.DataFrame, + test_ehr: pd.DataFrame, + subject_col: str, + visit_col: str, + code_col: str, + label_col: str, + embed_dim: int = 32, + hidden_dim: int = 32, + batch_size: int = 32, + epochs: int = 5, + verbose: bool = True, + seed: int = 4, +) -> Tuple[nn.Module, np.ndarray, np.ndarray]: + """Trains :class:`EHR_LSTM_Classifier` and evaluates it on a test set. + + Args: + train_ehr: Training EHR dataframe. + test_ehr: Test EHR dataframe. + subject_col: Column name for patient/subject identifiers. + visit_col: Column name for visit/timestep identifiers. + code_col: Column name for the medical codes. + label_col: Column name for the binary label. + embed_dim: Visit embedding dimension. + hidden_dim: LSTM hidden dimension. + batch_size: Training/eval batch size. + epochs: Number of training epochs. + verbose: Whether to print per-epoch loss. + seed: Random seed for reproducibility. + + Returns: + A tuple ``(model, y_true, y_pred)`` where ``y_true`` and ``y_pred`` are + numpy arrays of test labels and binary predictions. + """ + torch.manual_seed(seed) + all_codes = set() + all_codes.update(train_ehr[code_col].unique().tolist()) + all_codes.update(test_ehr[code_col].unique().tolist()) + # Sort before enumerating: a Python set has non-deterministic iteration + # order across processes, which would make the feature mapping (and thus + # the trained model / reported metrics) irreproducible even with a fixed + # seed. Start indices at 1 to reserve 0 for padding. + code_to_idx = { + code: idx for idx, code in enumerate(sorted(all_codes), start=1) + } + + train_data, _ = process_patient_data_for_lstm( + train_ehr, subject_col, visit_col, code_col, label_col, code_to_idx + ) + test_data, _ = process_patient_data_for_lstm( + test_ehr, subject_col, visit_col, code_col, label_col, code_to_idx + ) + train_dataloader = torch.utils.data.DataLoader( + dataset=EHRDataset(train_data), + batch_size=batch_size, + collate_fn=collate_fn, + shuffle=True, + ) + test_dataloader = torch.utils.data.DataLoader( + dataset=EHRDataset(test_data), + batch_size=batch_size, + collate_fn=collate_fn, + shuffle=False, + ) + + model = EHR_LSTM_Classifier( + vocab_size=len(code_to_idx) + 1, + embed_dim=embed_dim, + hidden_dim=hidden_dim, + ) + criterion = nn.BCELoss() + optimizer = torch.optim.Adam(model.parameters(), lr=0.001) + + use_cuda = torch.cuda.is_available() + if use_cuda: + model = model.cuda() + + model.train() + for epoch in range(epochs): + total_loss = 0.0 + for batch_x, batch_lens, batch_y in train_dataloader: + optimizer.zero_grad() + if use_cuda: + batch_x, batch_y = batch_x.cuda(), batch_y.cuda() + predictions = model(batch_x, batch_lens) + loss = criterion(predictions, batch_y) + loss.backward() + optimizer.step() + total_loss += loss.item() + if verbose: + avg_loss = total_loss / max(len(train_dataloader), 1) + print(f"Epoch {epoch + 1}/{epochs}, Loss: {avg_loss:.4f}") + + model.eval() + all_preds: List[float] = [] + all_labels: List[float] = [] + with torch.no_grad(): + for batch_x, batch_lens, batch_y in test_dataloader: + if use_cuda: + batch_x, batch_y = batch_x.cuda(), batch_y.cuda() + predictions = model(batch_x, batch_lens) + all_preds.extend(predictions.cpu().numpy()) + all_labels.extend(batch_y.cpu().numpy()) + + y_true = np.array(all_labels) + y_pred = np.array([1 if p >= 0.5 else 0 for p in all_preds]) + return model, y_true, y_pred + + +# --- Random-forest baseline ------------------------------------------------ + + +def aggregate_patient_visits( + df: pd.DataFrame, + subject_col: str, + code_col: str, + label_col: str, + code_to_idx: Dict, +) -> Tuple[np.ndarray, np.ndarray]: + """Aggregates each patient's visits into a single multi-hot vector. + + Args: + df: Input dataframe with integer-encoded codes in ``code_col``. + subject_col: Column name for patient/subject identifiers. + code_col: Column name for the (integer-encoded) medical codes. + label_col: Column name for the binary label. + code_to_idx: Mapping from code to index (used to size the vector). + + Returns: + A tuple ``(patient_vectors, patient_labels)`` of numpy arrays. + """ + patient_vectors = [] + patient_labels = [] + for _, group in df.groupby(subject_col): + codes = group[code_col].unique() + multi_hot = np.zeros(len(code_to_idx) + 1) + multi_hot[codes] = 1 + patient_vectors.append(multi_hot) + patient_labels.append(group[label_col].max()) + return np.array(patient_vectors), np.array(patient_labels) + + +def train_sklearn_model( + train_ehr: pd.DataFrame, + test_ehr: pd.DataFrame, + subject_col: str, + visit_col: str, + code_col: str, + label_col: str, + model: str = "rf", + seed: int = 4, +) -> Tuple[object, np.ndarray, np.ndarray]: + """Trains an sklearn classifier on aggregated patient-level multi-hot data. + + Args: + train_ehr: Training EHR dataframe. + test_ehr: Test EHR dataframe. + subject_col: Column name for patient/subject identifiers. + visit_col: Column name for visit/timestep identifiers (unused, kept for + a uniform signature with :func:`train_lstm_model`). + code_col: Column name for the medical codes. + label_col: Column name for the binary label. + model: Which model to train. Only ``"rf"`` (random forest) is supported. + seed: Random seed for reproducibility. + + Returns: + A tuple ``(model, y_true, y_pred)``. + """ + train_ehr = train_ehr.copy() + test_ehr = test_ehr.copy() + + all_codes = set() + all_codes.update(train_ehr[code_col].unique().tolist()) + all_codes.update(test_ehr[code_col].unique().tolist()) + # Sort before enumerating: a Python set has non-deterministic iteration + # order across processes, which would make the feature mapping (and thus + # the trained model / reported metrics) irreproducible even with a fixed + # seed. Start indices at 1 to reserve 0 for padding. + code_to_idx = { + code: idx for idx, code in enumerate(sorted(all_codes), start=1) + } + train_ehr[code_col] = train_ehr[code_col].map(code_to_idx) + test_ehr[code_col] = test_ehr[code_col].map(code_to_idx) + + X_train, y_train = aggregate_patient_visits( + train_ehr, subject_col, code_col, label_col, code_to_idx + ) + X_test, y_test = aggregate_patient_visits( + test_ehr, subject_col, code_col, label_col, code_to_idx + ) + + if model == "rf": + from sklearn.ensemble import RandomForestClassifier + + clf = RandomForestClassifier(n_estimators=100, random_state=seed) + else: + raise NotImplementedError(f"Model '{model}' not implemented.") + clf.fit(X_train, y_train) + + y_pred = clf.predict(X_test) + return clf, y_test, y_pred + + +# --- Task / feature construction ------------------------------------------ + + +def build_next_visit_prediction_dataset( + df: pd.DataFrame, + subject_col: str, + visit_col: str, + label_col: str, + multi_visit_sample_frac: float = 0.5, + seed: int = 4, +) -> pd.DataFrame: + """Builds a next-visit prediction task from an EHR dataframe. + + For patients with multiple visits, a fraction is sampled and their last + visit is dropped; these patients are labeled 1 (has a next visit). The + remaining multi-visit patients are kept intact and labeled 0. Single-visit + patients are labeled 0 by definition. + + Args: + df: Input EHR dataframe. + subject_col: Column name for patient/subject identifiers. + visit_col: Column name for visit/timestep identifiers. + label_col: Column name to overwrite with the next-visit label. + multi_visit_sample_frac: Fraction of multi-visit patients to truncate. + seed: Random seed for reproducibility. + + Returns: + A new dataframe with ``label_col`` set to the next-visit label. + """ + assert 0.0 <= multi_visit_sample_frac <= 1.0, ( + "multi_visit_sample_frac must be in [0, 1]." + ) + + rng = np.random.default_rng(seed) + transformed_groups = [] + + for _, group in df.groupby(subject_col): + group_sorted = group.sort_values(visit_col) + unique_visits = np.sort(group_sorted[visit_col].unique()) + n_visits = len(unique_visits) + + if n_visits <= 1: + g = group_sorted.copy() + g[label_col] = 0 + transformed_groups.append(g) + continue + + should_truncate = rng.random() < multi_visit_sample_frac + if should_truncate: + last_visit = unique_visits[-1] + g = group_sorted[group_sorted[visit_col] != last_visit].copy() + if g.empty: + # Defensive fallback for unexpected edge cases. + g = group_sorted.copy() + g[label_col] = 0 + else: + g[label_col] = 1 + else: + g = group_sorted.copy() + g[label_col] = 0 + transformed_groups.append(g) + + if len(transformed_groups) == 0: + return df.copy() + return pd.concat(transformed_groups, ignore_index=True) + + +def convert_cols_to_multihot( + df: pd.DataFrame, + code_col: str, + visit_col: str, + cat_cols: List[str], + num_cols: List[str], + bins_per_num: int = 5, +) -> pd.DataFrame: + """Folds categorical and numeric columns into per-visit multi-hot codes. + + Categorical columns are prefixed with their column name; numeric columns + are quantile-binned and likewise prefixed. All values are combined with the + original code into a single comma-separated ``combined_codes`` column. + + Args: + df: Input dataframe. + code_col: Column name for the existing medical codes. + visit_col: Column name for visit/timestep identifiers (kept for a + uniform signature; not modified). + cat_cols: Categorical column names to fold in. + num_cols: Numeric column names to bin and fold in. + bins_per_num: Number of quantile bins per numeric column. + + Returns: + A copy of ``df`` with an added ``combined_codes`` column. + """ + df = df.copy() + for col in cat_cols: + df[col] = col + "_" + df[col].astype(str) + + for col in num_cols: + df[col + "_binned"] = pd.qcut( + df[col], q=bins_per_num, duplicates="drop" + ).astype(str) + df[col + "_binned"] = col + "_" + df[col + "_binned"] + + def combine_codes(row): + codes = [str(row[code_col])] + for col in cat_cols: + codes.append(str(row[col])) + for col in num_cols: + codes.append(str(row[col + "_binned"])) + return ",".join(codes) + + df["combined_codes"] = df.apply(combine_codes, axis=1) + return df diff --git a/pyhealth/models/__init__.py b/pyhealth/models/__init__.py index 4c168d3e3..18500b9c0 100644 --- a/pyhealth/models/__init__.py +++ b/pyhealth/models/__init__.py @@ -45,4 +45,9 @@ from .sdoh import SdohClassifier from .medlink import MedLink from .unified_embedding import UnifiedMultimodalEmbeddingModel, SinusoidalTimeEmbedding -from .califorest import CaliForest \ No newline at end of file +from .califorest import CaliForest +from .generators.halo import HALO +from .generators.gpt2 import GPT2 +from .generators.promptehr import PromptEHR +from .generators.medgan import MedGAN +from .generators.corgan import CorGAN \ No newline at end of file diff --git a/pyhealth/models/generators/__init__.py b/pyhealth/models/generators/__init__.py new file mode 100644 index 000000000..a65bbd975 --- /dev/null +++ b/pyhealth/models/generators/__init__.py @@ -0,0 +1,7 @@ +from .halo import HALO +from .gpt2 import GPT2 +from .promptehr import PromptEHR +from .medgan import MedGAN +from .corgan import CorGAN + +__all__ = ["HALO", "GPT2", "PromptEHR", "MedGAN", "CorGAN"] diff --git a/pyhealth/models/generators/corgan.py b/pyhealth/models/generators/corgan.py new file mode 100644 index 000000000..807ed8066 --- /dev/null +++ b/pyhealth/models/generators/corgan.py @@ -0,0 +1,691 @@ +"""CorGAN: Correlation-capturing GAN for synthetic EHR generation. + +This is a port of the reference implementation +(https://github.com/astorfi/cor-gan, specifically +``reference/cor-gan/Generative/corGAN/pytorch/CNN/MIMIC/wgancnnmimic.py``) +wrapped as a PyHealth ``BaseModel`` so it consumes the standard +``dataset -> SampleDataset -> model`` pipeline. + +CorGAN treats each patient as a flat bag-of-codes (no visit structure), so it +expects an input feature named ``visits`` backed by a ``MultiHotProcessor``. +Training has two phases (mirroring the reference): + +* a **convolutional autoencoder** is pre-trained with a sparse-friendly BCE + reconstruction loss, then +* a **WGAN** adversarial phase runs the generator + decoder against a + Lipschitz-clipped critic (no sigmoid; weight clipping in + ``[clamp_lower, clamp_upper]``). + +For tiny vocabularies that can't survive the 6-layer convolutional chain we +automatically fall back to the linear-autoencoder variant noted in the +reference's commented-out alternative. The public ``CorGAN`` class follows the +same API style as :class:`pyhealth.models.generators.HALO` +(``train_model`` / ``generate`` / ``save_model`` / ``load_model``). +""" + +import os +from typing import Dict, List, Optional + +import numpy as np +import torch +import torch.nn as nn +from torch.utils.data import DataLoader, Dataset, RandomSampler +from tqdm import tqdm + +from pyhealth.models import BaseModel + + +# ---------------------------------------------------------------------------- +# Building blocks (ported from reference wgancnnmimic.py) +# ---------------------------------------------------------------------------- +class _MultiHotDataset(Dataset): + """Tiny ``torch.utils.data.Dataset`` over a multi-hot numpy matrix.""" + + def __init__(self, data: np.ndarray): + self.data = np.clip(data.astype(np.float32), 0.0, 1.0) + + def __len__(self) -> int: + return len(self.data) + + def __getitem__(self, idx): + return torch.from_numpy(self.data[idx]) + + +class CorGANCNNAutoencoder(nn.Module): + """1D-CNN autoencoder from the reference CorGAN paper. + + Six 1D conv layers compress the multi-hot vector down to a tiny latent; + six transposed-conv layers project back. When ``use_adaptive_pooling`` is + True we tack on an ``AdaptiveAvgPool1d`` so the decoder hits the exact + vocabulary size for any input dim (the original CNN was hard-coded for + MIMIC's vocabulary). + + Args: + feature_size: Vocabulary size. + use_adaptive_pooling: If True, force decoder output to ``feature_size`` + via adaptive average pooling. Default: True. + """ + + def __init__(self, feature_size: int, use_adaptive_pooling: bool = True): + super().__init__() + self.feature_size = feature_size + self.use_adaptive_pooling = use_adaptive_pooling + c = 4 # n_channels_base, per reference + + # Encoder: kernels (5,5,5,5,5,8), strides (2,2,3,3,3,1). + self.encoder = nn.Sequential( + nn.Conv1d(1, c, kernel_size=5, stride=2), + nn.LeakyReLU(0.2, inplace=True), + nn.Conv1d(c, 2 * c, kernel_size=5, stride=2), + nn.BatchNorm1d(2 * c), + nn.LeakyReLU(0.2, inplace=True), + nn.Conv1d(2 * c, 4 * c, kernel_size=5, stride=3), + nn.BatchNorm1d(4 * c), + nn.LeakyReLU(0.2, inplace=True), + nn.Conv1d(4 * c, 8 * c, kernel_size=5, stride=3), + nn.BatchNorm1d(8 * c), + nn.LeakyReLU(0.2, inplace=True), + nn.Conv1d(8 * c, 16 * c, kernel_size=5, stride=3), + nn.BatchNorm1d(16 * c), + nn.LeakyReLU(0.2, inplace=True), + nn.Conv1d(16 * c, 32 * c, kernel_size=8, stride=1), + nn.Tanh(), + ) + + # Decoder: kernels (5,5,7,7,7,3), strides (1,4,4,3,2,2). First layer + # has NO BatchNorm, matching the reference. The original was hard-coded + # for MIMIC's vocabulary; adaptive pooling rescales the output to any + # feature_size for downstream Sigmoid binarisation. + decoder_layers = [ + nn.ConvTranspose1d(32 * c, 16 * c, kernel_size=5, stride=1), + nn.ReLU(), + nn.ConvTranspose1d(16 * c, 8 * c, kernel_size=5, stride=4), + nn.BatchNorm1d(8 * c), + nn.ReLU(), + nn.ConvTranspose1d(8 * c, 4 * c, kernel_size=7, stride=4), + nn.BatchNorm1d(4 * c), + nn.ReLU(), + nn.ConvTranspose1d(4 * c, 2 * c, kernel_size=7, stride=3), + nn.BatchNorm1d(2 * c), + nn.ReLU(), + nn.ConvTranspose1d(2 * c, c, kernel_size=7, stride=2), + nn.BatchNorm1d(c), + nn.ReLU(), + nn.ConvTranspose1d(c, 1, kernel_size=3, stride=2), + ] + if use_adaptive_pooling: + decoder_layers.append(nn.AdaptiveAvgPool1d(output_size=feature_size)) + decoder_layers.append(nn.Sigmoid()) + self.decoder = nn.Sequential(*decoder_layers) + + def forward(self, x): + # Allow either (B, F) or (B, 1, F) input. + if x.dim() == 2: + x = x.unsqueeze(1) + decoded = self.decoder(self.encoder(x)) + if decoded.dim() == 3 and decoded.shape[1] == 1: + decoded = decoded.squeeze(1) + return decoded + + def decode(self, latent): + """Decode a latent emitted by the generator. + + The generator outputs ``(B, hidden_dim)``; we add a length-1 spatial + axis so the transposed-conv stack accepts it. + """ + if latent.dim() == 2: + latent = latent.unsqueeze(2) + decoded = self.decoder(latent) + if decoded.dim() == 3 and decoded.shape[1] == 1: + decoded = decoded.squeeze(1) + return decoded + + +class CorGANLinearAutoencoder(nn.Module): + """Linear autoencoder, the reference's commented-out alternative. + + Used as a fallback for small vocabularies where the 6-layer CNN can't + physically compress the input (its smallest viable input is ~500 + features). For unordered code spaces this is often a stronger baseline + anyway, since 1D conv assumes spatial locality. + """ + + def __init__(self, feature_size: int, latent_dim: int = 128): + super().__init__() + self.feature_size = feature_size + self.encoder = nn.Sequential( + nn.Linear(feature_size, latent_dim), + nn.ReLU(), + nn.BatchNorm1d(latent_dim), + ) + self.decoder = nn.Sequential( + nn.Linear(latent_dim, feature_size), + nn.Sigmoid(), + ) + + def forward(self, x): + return self.decoder(self.encoder(x)) + + def decode(self, latent): + return self.decoder(latent) + + +class CorGANGenerator(nn.Module): + """Two-layer MLP generator with residual connections (per reference).""" + + def __init__(self, latent_dim: int = 128, hidden_dim: int = 128): + super().__init__() + self.linear1 = nn.Linear(latent_dim, hidden_dim) + self.bn1 = nn.BatchNorm1d(hidden_dim, eps=0.001, momentum=0.01) + self.act1 = nn.ReLU() + + self.linear2 = nn.Linear(hidden_dim, hidden_dim) + self.bn2 = nn.BatchNorm1d(hidden_dim, eps=0.001, momentum=0.01) + self.act2 = nn.Tanh() + + def forward(self, x): + residual = x + out = self.act1(self.bn1(self.linear1(x))) + residual + + residual = out + out = self.act2(self.bn2(self.linear2(out))) + residual + return out + + +class CorGANCritic(nn.Module): + """4-layer MLP critic with optional minibatch averaging. + + No sigmoid at the output: this is a Wasserstein critic (not a + classifier), per the reference WGAN training loop. + """ + + def __init__( + self, + input_dim: int, + hidden_dim: int = 256, + minibatch_averaging: bool = True, + ): + super().__init__() + self.minibatch_averaging = minibatch_averaging + model_input_dim = input_dim * 2 if minibatch_averaging else input_dim + + self.model = nn.Sequential( + nn.Linear(model_input_dim, hidden_dim), + nn.ReLU(True), + nn.Linear(hidden_dim, hidden_dim), + nn.ReLU(True), + nn.Linear(hidden_dim, hidden_dim), + nn.ReLU(True), + nn.Linear(hidden_dim, 1), + ) + + def forward(self, x): + if self.minibatch_averaging: + x_mean = torch.mean(x, dim=0).repeat(x.shape[0], 1) + x = torch.cat((x, x_mean), dim=1) + return self.model(x) + + +def _weights_init(m): + """Reference initialization scheme (wgancnnmimic.py).""" + name = m.__class__.__name__ + if "Conv" in name: + nn.init.normal_(m.weight.data, 0.0, 0.02) + elif "BatchNorm" in name: + nn.init.normal_(m.weight.data, 1.0, 0.02) + nn.init.constant_(m.bias.data, 0) + if isinstance(m, nn.Linear): + nn.init.xavier_uniform_(m.weight) + m.bias.data.fill_(0.01) + + +def _autoencoder_loss(x_output, y_target): + """Sparse-friendly BCE used by the reference CorGAN autoencoder. + + Sum over features, then mean over batch -- equivalent to + ``BCELoss(reduction='sum') / batch_size``. ``BCELoss(reduction='mean')`` + additionally means over features and dilutes the signal for sparse + multi-hot targets. + """ + epsilon = 1e-12 + term = y_target * torch.log(x_output + epsilon) + ( + 1.0 - y_target + ) * torch.log(1.0 - x_output + epsilon) + return torch.mean(-torch.sum(term, dim=1), dim=0) + + +# Minimum feature size the 6-layer CNN encoder can survive without producing a +# non-positive spatial dimension. The reference targets MIMIC's ~7k vocabulary; +# the conv chain (kernels 5,5,5,5,5,8 / strides 2,2,3,3,3,1) only produces a +# positive output for inputs >= ~1000. We pick a slightly conservative floor. +_CNN_MIN_FEATURES = 1000 + +# The reference CNN autoencoder's bottleneck has 32 * n_channels_base (= 32*4) +# output channels. The generator output is fed in as those channels, so this +# is fixed by architecture. +_CNN_BOTTLENECK_DIM = 32 * 4 + + +# ---------------------------------------------------------------------------- +# PyHealth BaseModel wrapper +# ---------------------------------------------------------------------------- +class CorGAN(BaseModel): + """CorGAN synthetic-EHR generator, wrapped as a PyHealth ``BaseModel``. + + Trains a 1D-convolutional autoencoder + WGAN generator/critic on multi-hot + patient vectors and generates new synthetic patients by sampling noise, + pushing it through the generator, and decoding back with the (jointly + trained) decoder. + + Generation is **unconditional**: each synthetic patient is a flat bag of + codes (no visit structure), matching the ``multi_hot`` input schema. + + Args: + dataset: A fitted ``SampleDataset`` whose ``input_schema`` contains + ``{"visits": "multi_hot"}`` and whose ``output_schema`` is empty. + latent_dim: Generator noise dimensionality. Default: 128. + hidden_dim: Generator hidden width. Default: 128. + discriminator_hidden_dim: Critic hidden width. Default: 256. + minibatch_averaging: Concatenate per-batch mean to each critic input. + Default: True. + autoencoder_type: One of ``"cnn"`` (the reference) or ``"linear"`` + (the reference's commented-out alternative). For vocabularies + smaller than ~500 the CNN cannot compress the input, so we + silently fall back to ``"linear"``. Default: ``"cnn"``. + use_adaptive_pooling: When using the CNN autoencoder, add an + ``AdaptiveAvgPool1d`` so the decoder matches any vocabulary size. + Ignored for the linear variant. Default: True. + batch_size: Training batch size. Default: 512. + ae_epochs: Autoencoder pre-training epochs. Default: 100. + gan_epochs: Adversarial training epochs. Default: 200. + lr: Learning rate for all optimizers. Default: 1e-3. + weight_decay: L2 regularisation for all Adam optimizers. Default: 1e-4. + b1: Adam beta1. Default: 0.9. + b2: Adam beta2. Default: 0.999. + n_iter_D: Critic updates per generator update (reference: 5). + clamp_lower: WGAN critic weight-clip lower bound. Default: -0.01. + clamp_upper: WGAN critic weight-clip upper bound. Default: 0.01. + save_dir: Checkpoint directory used by ``train_model``. + Default: ``"./save/corgan/"``. + + Examples: + >>> from pyhealth.datasets import create_sample_dataset + >>> samples = [ + ... {"patient_id": "p1", "visits": ["A", "B", "C"]}, + ... {"patient_id": "p2", "visits": ["A", "C", "D"]}, + ... ] + >>> dataset = create_sample_dataset( + ... samples=samples, + ... input_schema={"visits": "multi_hot"}, + ... output_schema={}, + ... ) + >>> model = CorGAN(dataset, latent_dim=16, hidden_dim=16, batch_size=2) + >>> isinstance(model, CorGAN) + True + """ + + def __init__( + self, + dataset, + latent_dim: int = 128, + hidden_dim: int = 128, + discriminator_hidden_dim: int = 256, + minibatch_averaging: bool = True, + autoencoder_type: str = "cnn", + use_adaptive_pooling: bool = True, + batch_size: int = 512, + ae_epochs: int = 100, + gan_epochs: int = 200, + lr: float = 1e-3, + weight_decay: float = 1e-4, + b1: float = 0.9, + b2: float = 0.999, + n_iter_D: int = 5, + clamp_lower: float = -0.01, + clamp_upper: float = 0.01, + save_dir: str = "./save/corgan/", + ) -> None: + super().__init__(dataset) + + if "visits" not in dataset.input_processors: + raise ValueError( + "CorGAN expects an input feature named 'visits' backed by a " + "MultiHotProcessor." + ) + + self._batch_size = batch_size + self._ae_epochs = ae_epochs + self._gan_epochs = gan_epochs + self._lr = lr + self._weight_decay = weight_decay + self._betas = (b1, b2) + self._n_iter_D = n_iter_D + self._clamp_lower = clamp_lower + self._clamp_upper = clamp_upper + self.save_dir = save_dir + + # Code vocab from the MultiHotProcessor's label_vocab. + self.visits_processor = dataset.input_processors["visits"] + self.input_dim = self.visits_processor.size() + self._idx_to_code: List[Optional[str]] = [None] * self.input_dim + for code, idx in self.visits_processor.label_vocab.items(): + self._idx_to_code[idx] = code + + # CNN can't compress small vocabularies; fall back to linear. + if autoencoder_type == "cnn" and self.input_dim < _CNN_MIN_FEATURES: + autoencoder_type = "linear" + self.autoencoder_type = autoencoder_type + + if autoencoder_type == "linear": + # Linear AE: bottleneck = generator hidden dim (user-controlled). + self.autoencoder = CorGANLinearAutoencoder( + feature_size=self.input_dim, latent_dim=hidden_dim + ) + elif autoencoder_type == "cnn": + # CNN AE: bottleneck is fixed at 128 by the conv-channel ladder. + # The generator must emit that many features so the transposed-conv + # decoder accepts its output. We silently align hidden_dim to the + # CNN bottleneck (the reference always uses 128). + if hidden_dim != _CNN_BOTTLENECK_DIM: + hidden_dim = _CNN_BOTTLENECK_DIM + self.autoencoder = CorGANCNNAutoencoder( + feature_size=self.input_dim, + use_adaptive_pooling=use_adaptive_pooling, + ) + else: + raise ValueError( + f"Unknown autoencoder_type={autoencoder_type!r}; " + "expected 'cnn' or 'linear'." + ) + + # The generator's residual connection requires latent_dim == hidden_dim + # (per the reference). Align silently if the user mismatched. + if latent_dim != hidden_dim: + latent_dim = hidden_dim + self.latent_dim = latent_dim + self.hidden_dim = hidden_dim + + self.generator = CorGANGenerator( + latent_dim=latent_dim, hidden_dim=hidden_dim + ) + self.critic = CorGANCritic( + input_dim=self.input_dim, + hidden_dim=discriminator_hidden_dim, + minibatch_averaging=minibatch_averaging, + ) + + self.autoencoder.apply(_weights_init) + self.generator.apply(_weights_init) + self.critic.apply(_weights_init) + + # ------------------------------------------------------------------ + # forward -- required by BaseModel + # ------------------------------------------------------------------ + def forward(self, **kwargs) -> Dict[str, torch.Tensor]: + """CorGAN does not have a single supervised forward pass. + + Use :meth:`train_model` for training and :meth:`generate` for + synthesis. ``forward`` is implemented only to satisfy the + ``BaseModel`` abstract contract. + """ + raise NotImplementedError( + "CorGAN is a GAN: use train_model() and generate() instead of " + "forward()." + ) + + # ------------------------------------------------------------------ + # Custom training loop + # ------------------------------------------------------------------ + @staticmethod + def _resolve_device(device=None) -> torch.device: + """Resolve a user-supplied device, defaulting to CUDA when available.""" + if device is None: + return torch.device("cuda" if torch.cuda.is_available() else "cpu") + return torch.device(device) + + def _build_dataloader(self, dataset) -> DataLoader: + """Stack the multi-hot tensors of ``dataset`` into a DataLoader.""" + tensors = [dataset[i]["visits"] for i in range(len(dataset))] + matrix = torch.stack(tensors).numpy() + wrapped = _MultiHotDataset(matrix) + sampler = RandomSampler(wrapped, replacement=True) + return DataLoader( + wrapped, + batch_size=self._batch_size, + shuffle=False, + num_workers=0, + drop_last=True, + sampler=sampler, + ) + + def train_model(self, train_dataset, val_dataset=None, device=None) -> Dict: + """Train CorGAN with a custom two-phase loop. + + Named ``train_model`` (not ``train``) to avoid shadowing + ``nn.Module.train()``. Phase 1 pre-trains the autoencoder with + sparse BCE reconstruction loss; phase 2 runs WGAN adversarial + training (weight-clipped critic, joint generator + decoder). + + Args: + train_dataset: ``SampleDataset`` for training. + val_dataset: Unused; accepted for API symmetry. + device: Device to train on. If ``None``, uses CUDA when available. + + Returns: + Dict with keys ``"autoencoder_loss"``, ``"critic_loss"``, + ``"generator_loss"`` -- one float per epoch in each list. + """ + device = self._resolve_device(device) + self.to(device) + print(f"Training CorGAN on: {device}") + + os.makedirs(self.save_dir, exist_ok=True) + dataloader = self._build_dataloader(train_dataset) + history: Dict[str, List[float]] = { + "autoencoder_loss": [], + "critic_loss": [], + "generator_loss": [], + } + + # ---- Phase 1: Autoencoder pretraining ---- + optimizer_ae = torch.optim.Adam( + self.autoencoder.parameters(), + lr=self._lr, + betas=self._betas, + weight_decay=self._weight_decay, + ) + for epoch in tqdm(range(self._ae_epochs), desc="AE pretrain"): + self.autoencoder.train() + last_loss = 0.0 + for batch in dataloader: + real = batch.to(self.device) + recon = self.autoencoder(real) + loss = _autoencoder_loss(recon, real) + + optimizer_ae.zero_grad() + loss.backward() + optimizer_ae.step() + last_loss = loss.item() + history["autoencoder_loss"].append(last_loss) + + # ---- Phase 2: WGAN adversarial training ---- + # The reference jointly optimises the generator and the autoencoder's + # decoder, with a smaller LR on the decoder. We reuse that scheme. + g_params = [ + {"params": self.generator.parameters()}, + {"params": self.autoencoder.decoder.parameters(), "lr": 1e-4}, + ] + optimizer_g = torch.optim.Adam( + g_params, + lr=self._lr, + betas=self._betas, + weight_decay=self._weight_decay, + ) + optimizer_d = torch.optim.Adam( + self.critic.parameters(), + lr=self._lr, + betas=self._betas, + weight_decay=self._weight_decay, + ) + # WGAN sign convention under Adam (gradient *descent*): the critic + # maximises E[D(real)] - E[D(fake)] and the generator maximises + # E[D(fake)]. Maximising a term therefore means seeding ``backward`` + # with ``mone`` (so the descent step ascends that term); ``one`` is + # used for the term we want to push down. + one = torch.tensor(1.0, device=self.device) + mone = torch.tensor(-1.0, device=self.device) + gen_iters = 0 + + for epoch in tqdm(range(self._gan_epochs), desc="GAN train"): + self.generator.train() + self.critic.train() + self.autoencoder.eval() + self.autoencoder.decoder.train() + + last_d, last_g = 0.0, 0.0 + for real in dataloader: + real = real.to(self.device) + bs = real.size(0) + + # --- Train critic --- + for p in self.critic.parameters(): + p.requires_grad = True + # Reference: ramp up critic iterations at the start and at + # periodic intervals to keep the Wasserstein estimate tight. + n_iter_D = ( + 100 if (gen_iters < 25 or gen_iters % 500 == 0) + else self._n_iter_D + ) + for _ in range(n_iter_D): + for p in self.critic.parameters(): + p.data.clamp_(self._clamp_lower, self._clamp_upper) + + optimizer_d.zero_grad() + errD_real = torch.mean(self.critic(real)).squeeze() + # Maximise E[D(real)] -> ascend errD_real (seed with mone). + errD_real.backward(mone) + + z = torch.randn(bs, self.latent_dim, device=self.device) + fake = self.autoencoder.decode(self.generator(z)) + errD_fake = torch.mean(self.critic(fake.detach())).squeeze() + # Minimise E[D(fake)] -> descend errD_fake (seed with one). + errD_fake.backward(one) + last_d = (errD_real - errD_fake).item() + + optimizer_d.step() + + # --- Train generator --- + for p in self.critic.parameters(): + p.requires_grad = False + optimizer_g.zero_grad() + z = torch.randn(bs, self.latent_dim, device=self.device) + fake = self.autoencoder.decode(self.generator(z)) + errG = torch.mean(self.critic(fake)).squeeze() + # Maximise E[D(fake)] -> ascend errG (seed with mone). + errG.backward(mone) + optimizer_g.step() + last_g = errG.item() + gen_iters += 1 + + history["critic_loss"].append(last_d) + history["generator_loss"].append(last_g) + + self.save_model(os.path.join(self.save_dir, "final.pt")) + return history + + # ------------------------------------------------------------------ + # Synthesis + # ------------------------------------------------------------------ + def generate( + self, + num_samples: int, + random_sampling: bool = False, + device=None, + ) -> List[Dict]: + """Generate synthetic patient records. + + Each synthetic patient is decoded from a generated multi-hot vector + by thresholding (or, optionally, Bernoulli sampling) at 0.5 and + mapping the indices back to code strings. + + Args: + num_samples: Number of synthetic patients to generate. + random_sampling: If True, Bernoulli-sample the decoder output; + otherwise threshold at 0.5 (the reference's behaviour). + Default: False. + device: Device to generate on. If ``None``, uses CUDA when + available. + + Returns: + List of dicts + ``{"patient_id": "synthetic_i", "visits": [[code, ...]]}``. + ``visits`` is a list containing a **single** visit (matching + HALO's nested-list output structure). CorGAN is a bag-of-codes + model -- following the reference preprocessing, each patient is + represented by the union of codes across all of their + historical visits -- so the single inner list is that aggregate + bag. The inner list may be empty if the generator produced an + all-zero vector. + """ + device = self._resolve_device(device) + self.to(device) + + self.generator.eval() + self.autoencoder.eval() + + bs = min(self._batch_size, max(num_samples, 1)) + rows = np.zeros((num_samples, self.input_dim), dtype=np.float32) + pbar = tqdm(total=num_samples, desc="Generating patients") + with torch.no_grad(): + i = 0 + while i < num_samples: + cur = min(bs, num_samples - i) + z = torch.randn(cur, self.latent_dim, device=self.device) + probs = self.autoencoder.decode(self.generator(z)) + if random_sampling: + sample = torch.bernoulli(probs) + else: + sample = (probs >= 0.5).float() + rows[i : i + cur] = sample.cpu().numpy() + i += cur + pbar.update(cur) + pbar.close() + + results: List[Dict] = [] + for i in range(num_samples): + codes = [ + self._idx_to_code[idx] + for idx in np.nonzero(rows[i])[0] + if self._idx_to_code[idx] not in (None, "", "") + ] + # Wrap in a single-visit list to mirror HALO's nested output. + # CorGAN models the patient as one aggregate bag of codes. + results.append({"patient_id": f"synthetic_{i}", "visits": [codes]}) + return results + + # ------------------------------------------------------------------ + # Checkpoint I/O + # ------------------------------------------------------------------ + def save_model(self, path: str) -> None: + """Save weights, vocabulary, and architecture metadata.""" + torch.save( + { + "autoencoder": self.autoencoder.state_dict(), + "generator": self.generator.state_dict(), + "critic": self.critic.state_dict(), + "autoencoder_type": self.autoencoder_type, + "input_dim": self.input_dim, + "latent_dim": self.latent_dim, + "idx_to_code": self._idx_to_code, + }, + path, + ) + + def load_model(self, path: str) -> None: + """Load weights and vocabulary from a checkpoint.""" + ckpt = torch.load(path, map_location=self.device) + self.autoencoder.load_state_dict(ckpt["autoencoder"]) + self.generator.load_state_dict(ckpt["generator"]) + self.critic.load_state_dict(ckpt["critic"]) + if "idx_to_code" in ckpt: + self._idx_to_code = ckpt["idx_to_code"] diff --git a/pyhealth/models/generators/gpt2.py b/pyhealth/models/generators/gpt2.py new file mode 100644 index 000000000..99b3bf92a --- /dev/null +++ b/pyhealth/models/generators/gpt2.py @@ -0,0 +1,366 @@ +"""GPT-2 baseline for unconditional synthetic EHR generation. + +A simple decoder-only baseline that mirrors the standalone reference script +``generate_synthetic_mimic3_gpt2.py`` (``--mode transformer_baseline``) but +plugged into the standard PyHealth ``dataset -> set_task -> SampleDataset -> +model`` pipeline. It consumes the same :class:`~pyhealth.tasks.EHRGeneration` +task as :class:`~pyhealth.models.HALO`. + +Each patient's visits are flattened into a single token stream:: + + [BOS] [VISIT_DELIM] ... [EOS] + +A small :class:`transformers.GPT2LMHeadModel` is trained on these streams with +causal language modeling. Generation autoregressively samples a token stream +(``do_sample`` + top-k/top-p) and decodes it back into per-visit code lists, +splitting on the ``[VISIT_DELIM]`` token. + +The code vocabulary is taken from the dataset's ``NestedSequenceProcessor`` +(which already reserves index 0 for ```` and index 1 for ````); three +special tokens (BOS, EOS, VISIT_DELIM) are appended, and ```` (index 0) is +reused as the padding token. +""" + +import os +from typing import Dict, List, Optional + +import numpy as np +import torch +import torch.nn.functional as F +from tqdm import tqdm +from transformers import GPT2Config, GPT2LMHeadModel + +from pyhealth.datasets import get_dataloader +from pyhealth.models import BaseModel + + +class GPT2(BaseModel): + """GPT-2 baseline synthetic-EHR generator, wrapped as a PyHealth ``BaseModel``. + + Args: + dataset: A fitted ``SampleDataset`` whose ``input_schema`` contains + ``{"visits": NestedSequenceProcessor}`` and whose ``output_schema`` + is empty. + embed_dim: GPT-2 embedding dimension (``n_embd``). Must be divisible by + ``n_heads``. Default: 512. + n_heads: Number of attention heads. Default: 8. + n_layers: Number of transformer layers. Default: 8. + max_len: Maximum token-stream length (``n_positions``); streams are + truncated to this length. Default: 512. + batch_size: Training batch size. Default: 64. + epochs: Number of training epochs. Default: 50. + lr: Learning rate for the Adam optimizer. Default: 1e-4. + save_dir: Directory for checkpoints written by ``train_model``. + Default: ``"./save/"``. + + Examples: + >>> from pyhealth.datasets import create_sample_dataset + >>> samples = [ + ... {"patient_id": "p1", "visits": [["A", "B"], ["C"]]}, + ... {"patient_id": "p2", "visits": [["A"], ["B", "C"]]}, + ... ] + >>> dataset = create_sample_dataset( + ... samples=samples, + ... input_schema={"visits": "nested_sequence"}, + ... output_schema={}, + ... ) + >>> model = GPT2(dataset, embed_dim=16, n_heads=2, n_layers=2, max_len=64) + >>> isinstance(model, GPT2) + True + """ + + def __init__( + self, + dataset, + embed_dim: int = 512, + n_heads: int = 8, + n_layers: int = 8, + max_len: int = 512, + batch_size: int = 64, + epochs: int = 50, + lr: float = 1e-4, + save_dir: str = "./save/", + ) -> None: + super(GPT2, self).__init__(dataset) + + if "visits" not in dataset.input_processors: + raise ValueError( + "GPT2 expects an input feature named 'visits' backed by a " + "NestedSequenceProcessor." + ) + + self.save_dir = save_dir + self._batch_size = batch_size + self._epochs = epochs + self._lr = lr + self.max_len = max_len + + # Code vocab from the NestedSequenceProcessor (includes =0, =1). + self.visits_processor = dataset.input_processors["visits"] + self.code_vocab_size = self.visits_processor.vocab_size() + # Append three special tokens after the code vocab; reuse =0 as PAD. + self.bos_id = self.code_vocab_size + self.eos_id = self.code_vocab_size + 1 + self.delim_id = self.code_vocab_size + 2 + self.pad_id = 0 + total_vocab_size = self.code_vocab_size + 3 + + config = GPT2Config( + vocab_size=total_vocab_size, + n_positions=max_len, + n_embd=embed_dim, + n_layer=n_layers, + n_head=n_heads, + bos_token_id=self.bos_id, + eos_token_id=self.eos_id, + ) + # Registered as a sub-module so .parameters()/.to() work. + self.gpt2 = GPT2LMHeadModel(config) + + # ------------------------------------------------------------------ + @staticmethod + def _resolve_device(device=None) -> torch.device: + """Resolve a user-supplied device, defaulting to CUDA when available.""" + if device is None: + return torch.device("cuda" if torch.cuda.is_available() else "cpu") + return torch.device(device) + + # ------------------------------------------------------------------ + # Visit index tensor -> flat causal-LM token stream + # ------------------------------------------------------------------ + def _encode_visits(self, visits: torch.Tensor): + """Flatten the padded visit-index tensor into causal-LM token streams. + + Args: + visits: LongTensor ``(batch, max_visits, max_codes_per_visit)`` from + the ``NestedSequenceProcessor``. Index 0 is ```` and is + skipped. + + Returns: + input_ids: LongTensor ``(batch, L)`` token streams, right-padded. + attention_mask: LongTensor ``(batch, L)`` (1 for real tokens). + labels: ``input_ids`` with padding positions set to ``-100`` so they + are ignored by the cross-entropy loss. + """ + batch_seqs: List[List[int]] = [] + for i in range(visits.shape[0]): + n_visits = int((visits[i].sum(dim=-1) > 0).sum().item()) + seq: List[int] = [self.bos_id] + for j in range(n_visits): + codes = [int(c) for c in visits[i, j].tolist() if c > 0] + seq.extend(codes) + if j < n_visits - 1: + seq.append(self.delim_id) + seq.append(self.eos_id) + batch_seqs.append(seq[: self.max_len]) + + length = max(len(s) for s in batch_seqs) + input_ids = torch.full( + (len(batch_seqs), length), self.pad_id, dtype=torch.long, device=self.device + ) + attention_mask = torch.zeros( + (len(batch_seqs), length), dtype=torch.long, device=self.device + ) + for i, seq in enumerate(batch_seqs): + input_ids[i, : len(seq)] = torch.tensor(seq, device=self.device) + attention_mask[i, : len(seq)] = 1 + + labels = input_ids.clone() + labels[attention_mask == 0] = -100 + return input_ids, attention_mask, labels + + # ------------------------------------------------------------------ + # forward -- required by BaseModel + # ------------------------------------------------------------------ + def forward(self, visits: torch.Tensor, **kwargs) -> Dict[str, torch.Tensor]: + """Forward pass. + + Args: + visits: LongTensor ``(batch, max_visits, max_codes_per_visit)`` from + the ``NestedSequenceProcessor``. + **kwargs: Any other batch keys are ignored. + + Returns: + Dict with ``loss`` (scalar causal-LM cross-entropy) and ``y_prob`` + (next-token probabilities, shape ``(batch, L, vocab_size)``). + """ + visits = visits.to(self.device) + input_ids, attention_mask, labels = self._encode_visits(visits) + out = self.gpt2( + input_ids=input_ids, attention_mask=attention_mask, labels=labels + ) + return {"loss": out.loss, "y_prob": F.softmax(out.logits, dim=-1)} + + # ------------------------------------------------------------------ + # Custom training loop + # ------------------------------------------------------------------ + def train_model(self, train_dataset, val_dataset=None, device=None) -> None: + """Train the GPT-2 baseline with a custom loop. + + Named ``train_model`` (not ``train``) to avoid shadowing + ``nn.Module.train()``. Uses the standard ``get_dataloader``, an Adam + optimizer, and causal-LM loss. When ``val_dataset`` is given, validation + loss is computed after each epoch and the best checkpoint is saved to + ``self.save_dir``. + + Args: + train_dataset: ``SampleDataset`` for training. + val_dataset: Optional ``SampleDataset`` for validation. + device: Device to train on, e.g. ``"cuda"``, ``"cuda:1"``, or + ``"cpu"``. If ``None`` (default), uses CUDA when available and + falls back to CPU. + """ + device = self._resolve_device(device) + self.to(device) + print(f"Training on: {device}") + + os.makedirs(self.save_dir, exist_ok=True) + optimizer = torch.optim.Adam(self.gpt2.parameters(), lr=self._lr) + + checkpoint_path = os.path.join(self.save_dir, "gpt2_model") + if os.path.exists(checkpoint_path): + checkpoint = torch.load(checkpoint_path, map_location=self.device) + self.gpt2.load_state_dict(checkpoint["model"]) + optimizer.load_state_dict(checkpoint["optimizer"]) + + train_loader = get_dataloader( + train_dataset, batch_size=self._batch_size, shuffle=True + ) + + global_loss = 1e10 + for epoch in tqdm(range(self._epochs), desc="Epochs"): + self.gpt2.train() + batch_iter = tqdm(train_loader, desc=f"Epoch {epoch}", leave=False) + for batch in batch_iter: + visits = batch["visits"].to(self.device) + input_ids, attention_mask, labels = self._encode_visits(visits) + + optimizer.zero_grad() + out = self.gpt2( + input_ids=input_ids, + attention_mask=attention_mask, + labels=labels, + ) + out.loss.backward() + optimizer.step() + batch_iter.set_postfix(loss=f"{out.loss.item():.4f}") + + if val_dataset is not None: + self.gpt2.eval() + val_loader = get_dataloader( + val_dataset, batch_size=self._batch_size, shuffle=False + ) + val_losses = [] + with torch.no_grad(): + for val_batch in val_loader: + visits = val_batch["visits"].to(self.device) + input_ids, attention_mask, labels = self._encode_visits(visits) + out = self.gpt2( + input_ids=input_ids, + attention_mask=attention_mask, + labels=labels, + ) + val_losses.append(out.loss.item()) + + cur_val_loss = float(np.mean(val_losses)) + print(f"Epoch {epoch} Validation Loss: {cur_val_loss:.7f}") + if cur_val_loss < global_loss: + global_loss = cur_val_loss + state = { + "model": self.gpt2.state_dict(), + "optimizer": optimizer.state_dict(), + "epoch": epoch, + } + torch.save(state, checkpoint_path) + print("------------ Save best model ------------") + + # ------------------------------------------------------------------ + # Synthesis + # ------------------------------------------------------------------ + def _decode_ids(self, ids: List[int], index_to_code: Dict[int, str]) -> List[List[str]]: + """Decode a generated token stream into per-visit code lists.""" + visits_out: List[List[str]] = [] + current: List[str] = [] + for tid in ids: + if tid in (self.bos_id, self.pad_id): + continue + if tid == self.eos_id: + break + if tid == self.delim_id: + if current: + visits_out.append(current) + current = [] + continue + if tid < self.code_vocab_size: + code = index_to_code.get(int(tid)) + if code not in (None, "", ""): + current.append(code) + if current: + visits_out.append(current) + return visits_out + + def generate( + self, + num_samples: int, + device=None, + top_k: int = 50, + top_p: float = 0.95, + ) -> List[Dict]: + """Generate synthetic patients with the trained GPT-2 baseline. + + Feeds a ``[BOS]`` token and autoregressively samples a token stream with + ``top_k``/``top_p`` sampling, then decodes it into per-visit code lists. + + Args: + num_samples: Number of synthetic patients to generate. + device: Device to generate on, e.g. ``"cuda"``, ``"cuda:1"``, or + ``"cpu"``. If ``None`` (default), uses CUDA when available and + falls back to CPU. + top_k: Top-k sampling cutoff. Default: 50. + top_p: Nucleus (top-p) sampling cutoff. Default: 0.95. + + Returns: + List of dicts, each ``{"patient_id": "synthetic_i", + "visits": [[code, ...], ...]}`` with decoded code strings. + """ + device = self._resolve_device(device) + self.to(device) + + index_to_code = {v: k for k, v in self.visits_processor.code_vocab.items()} + + self.gpt2.eval() + synthetic_dataset: List[Dict] = [] + sample_batch_size = min(num_samples, 256) + generated = 0 + pbar = tqdm(total=num_samples, desc="Generating patients") + + with torch.no_grad(): + while generated < num_samples: + bs = min(sample_batch_size, num_samples - generated) + input_ids = torch.full( + (bs, 1), self.bos_id, dtype=torch.long, device=self.device + ) + out_ids = self.gpt2.generate( + input_ids, + max_length=self.max_len, + do_sample=True, + top_k=top_k, + top_p=top_p, + pad_token_id=self.pad_id, + eos_token_id=self.eos_id, + ) + for i in range(bs): + visits_out = self._decode_ids( + out_ids[i].tolist(), index_to_code + ) + synthetic_dataset.append( + { + "patient_id": f"synthetic_{generated + i}", + "visits": visits_out, + } + ) + generated += bs + pbar.update(bs) + pbar.close() + + return synthetic_dataset diff --git a/pyhealth/models/generators/halo.py b/pyhealth/models/generators/halo.py new file mode 100644 index 000000000..374c14000 --- /dev/null +++ b/pyhealth/models/generators/halo.py @@ -0,0 +1,724 @@ +"""HALO: Hierarchical Autoregressive Language mOdel for synthetic EHR generation. + +This is a faithful port of the reference implementation +(https://github.com/Brandon-Theodorou/HALO_Inpatient) wrapped as a PyHealth +``BaseModel`` so it consumes the standard +``dataset -> set_task -> SampleDataset -> model`` pipeline. + +HALO is a two-level model: + +* a GPT-2-style **coarse** transformer operates over visit-level multi-hot + vectors, and +* a **fine** autoregressive head predicts the (multi-label) set of codes within + each visit. + +The transformer/head classes below (``LayerNorm``, ``Conv1D``, ``Attention``, +``MLP``, ``Block``, ``CoarseTransformerModel``, ``AutoregressiveLinear``, +``FineAutoregressiveHead``, ``HALOModel``) are ported verbatim from the +reference ``model.py``. The only behavioural change is that PyHealth's HALO is +**unconditional** (``label_vocab_size = 0``): it generates visit-code sequences +without conditioning on CCS labels. +""" + +import copy +import math +import os +from typing import Dict, List, Optional + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from tqdm import tqdm + +from pyhealth.datasets import get_dataloader +from pyhealth.models import BaseModel + + +# ---------------------------------------------------------------------------- +# Configuration (plain class, not a dataclass; mirrors reference config.py) +# ---------------------------------------------------------------------------- +class HALOConfig: + """Hyperparameter container for the HALO transformer. + + Kept as a plain class with explicit ``__init__`` assignments (matching the + reference ``config.py``) so the low-level modules can read attributes such + as ``config.n_embd``. + """ + + def __init__( + self, + total_vocab_size: int, + code_vocab_size: int, + label_vocab_size: int = 0, + special_vocab_size: int = 3, + n_positions: int = 56, + n_ctx: int = 48, + n_embd: int = 768, + n_layer: int = 12, + n_head: int = 12, + layer_norm_epsilon: float = 1e-5, + initializer_range: float = 0.02, + batch_size: int = 48, + epoch: int = 50, + pos_loss_weight: Optional[float] = None, + lr: float = 1e-4, + ) -> None: + self.total_vocab_size = total_vocab_size + self.code_vocab_size = code_vocab_size + self.label_vocab_size = label_vocab_size + self.special_vocab_size = special_vocab_size + self.n_positions = n_positions + self.n_ctx = n_ctx + self.n_embd = n_embd + self.n_layer = n_layer + self.n_head = n_head + self.layer_norm_epsilon = layer_norm_epsilon + self.initializer_range = initializer_range + self.batch_size = batch_size + self.epoch = epoch + self.pos_loss_weight = pos_loss_weight + self.lr = lr + + +# ---------------------------------------------------------------------------- +# Transformer building blocks (ported verbatim from reference model.py) +# ---------------------------------------------------------------------------- +class LayerNorm(nn.Module): + def __init__(self, hidden_size, eps=1e-12): + """Construct a layernorm module in the TF style (epsilon inside sqrt).""" + super(LayerNorm, self).__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.bias = nn.Parameter(torch.zeros(hidden_size)) + self.variance_epsilon = eps + + def forward(self, x): + u = x.mean(-1, keepdim=True) + s = (x - u).pow(2).mean(-1, keepdim=True) + x = (x - u) / torch.sqrt(s + self.variance_epsilon) + return self.weight * x + self.bias + + +class Conv1D(nn.Module): + def __init__(self, nf, nx): + super(Conv1D, self).__init__() + self.nf = nf + w = torch.empty(nx, nf) + nn.init.normal_(w, std=0.02) + self.weight = nn.Parameter(w) + self.bias = nn.Parameter(torch.zeros(nf)) + + def forward(self, x): + size_out = x.size()[:-1] + (self.nf,) + x = torch.addmm(self.bias, x.view(-1, x.size(-1)), self.weight) + x = x.view(*size_out) + return x + + +class Attention(nn.Module): + def __init__(self, nx, n_ctx, config, scale=False): + super(Attention, self).__init__() + n_state = nx # in Attention: n_state=n_embd (nx=n_embd) + assert n_state % config.n_head == 0 + self.register_buffer( + "bias", torch.tril(torch.ones(n_ctx, n_ctx)).view(1, 1, n_ctx, n_ctx) + ) + self.n_head = config.n_head + self.split_size = n_state + self.scale = scale + self.c_attn = Conv1D(n_state * 3, nx) + self.c_proj = Conv1D(n_state, nx) + + def _attn(self, q, k, v): + w = torch.matmul(q, k) + if self.scale: + w = w / math.sqrt(v.size(-1)) + nd, ns = w.size(-2), w.size(-1) + b = self.bias[:, :, ns - nd:ns, :ns] + w = w * b - 1e10 * (1 - b) + w = nn.Softmax(dim=-1)(w) + return torch.matmul(w, v) + + def merge_heads(self, x): + x = x.permute(0, 2, 1, 3).contiguous() + new_x_shape = x.size()[:-2] + (x.size(-2) * x.size(-1),) + return x.view(*new_x_shape) + + def split_heads(self, x, k=False): + new_x_shape = x.size()[:-1] + (self.n_head, x.size(-1) // self.n_head) + x = x.view(*new_x_shape) + if k: + return x.permute(0, 2, 3, 1) # (batch, head, head_features, seq_length) + else: + return x.permute(0, 2, 1, 3) # (batch, head, seq_length, head_features) + + def forward(self, x, layer_past=None): + x = self.c_attn(x) + query, key, value = x.split(self.split_size, dim=2) + query = self.split_heads(query) + key = self.split_heads(key, k=True) + value = self.split_heads(value) + if layer_past is not None: + past_key, past_value = layer_past[0].transpose(-2, -1), layer_past[1] + key = torch.cat((past_key, key), dim=-1) + value = torch.cat((past_value, value), dim=-2) + present = torch.stack((key.transpose(-2, -1), value)) + a = self._attn(query, key, value) + a = self.merge_heads(a) + a = self.c_proj(a) + return a, present + + +class MLP(nn.Module): + def __init__(self, n_state, config): # in MLP: n_state=4 * n_embd + super(MLP, self).__init__() + nx = config.n_embd + self.c_fc = Conv1D(n_state, nx) + self.c_proj = Conv1D(nx, n_state) + + def forward(self, x): + # tanh-approximate GELU, matching the reference HALO implementation. + h = F.gelu(self.c_fc(x), approximate="tanh") + h2 = self.c_proj(h) + return h2 + + +class Block(nn.Module): + def __init__(self, n_ctx, config, scale=False): + super(Block, self).__init__() + nx = config.n_embd + self.ln_1 = LayerNorm(nx, eps=config.layer_norm_epsilon) + self.attn = Attention(nx, n_ctx, config, scale) + self.ln_2 = LayerNorm(nx, eps=config.layer_norm_epsilon) + self.mlp = MLP(4 * nx, config) + + def forward(self, x, layer_past=None): + a, present = self.attn(self.ln_1(x), layer_past=layer_past) + x = x + a + m = self.mlp(self.ln_2(x)) + x = x + m + return x, present + + +class CoarseTransformerModel(nn.Module): + def __init__(self, config): + super(CoarseTransformerModel, self).__init__() + self.n_layer = config.n_layer + self.n_embd = config.n_embd + self.n_vocab = config.total_vocab_size + + self.vis_embed_mat = nn.Linear( + config.total_vocab_size, config.n_embd, bias=False + ) + self.pos_embed_mat = nn.Embedding(config.n_positions, config.n_embd) + block = Block(config.n_ctx, config, scale=True) + self.h = nn.ModuleList( + [copy.deepcopy(block) for _ in range(config.n_layer)] + ) + self.ln_f = LayerNorm(config.n_embd, eps=config.layer_norm_epsilon) + + def forward(self, input_visits, position_ids=None, past=None): + if past is None: + past_length = 0 + past = [None] * len(self.h) + else: + past_length = past[0][0].size(-2) + if position_ids is None: + position_ids = torch.arange( + past_length, + input_visits.size(1) + past_length, + dtype=torch.long, + device=input_visits.device, + ) + position_ids = position_ids.unsqueeze(0).expand( + input_visits.size(0), input_visits.size(1) + ) + + inputs_embeds = self.vis_embed_mat(input_visits) + position_embeds = self.pos_embed_mat(position_ids) + hidden_states = inputs_embeds + position_embeds + for block, layer_past in zip(self.h, past): + hidden_states, _ = block(hidden_states, layer_past) + hidden_states = self.ln_f(hidden_states) + return hidden_states + + +class AutoregressiveLinear(nn.Linear): + """Same as Linear except it has a configurable mask on the weights.""" + + def __init__(self, in_features, out_features, bias=True): + super().__init__(in_features, out_features, bias) + self.register_buffer( + "mask", torch.tril(torch.ones(in_features, out_features)).int() + ) + + def forward(self, input): + return F.linear(input, self.mask * self.weight, self.bias) + + +class FineAutoregressiveHead(nn.Module): + def __init__(self, config): + super(FineAutoregressiveHead, self).__init__() + self.auto1 = AutoregressiveLinear( + config.n_embd + config.total_vocab_size, + config.n_embd + config.total_vocab_size, + ) + self.auto2 = AutoregressiveLinear( + config.n_embd + config.total_vocab_size, + config.n_embd + config.total_vocab_size, + ) + self.n_embd = config.n_embd + self.tot_vocab = config.total_vocab_size + + def forward(self, history, input_visits): + history = history[:, :-1, :] + input_visits = input_visits[:, 1:, :] + code_logits = self.auto2( + torch.relu(self.auto1(torch.cat((history, input_visits), dim=2))) + )[:, :, self.n_embd - 1:-1] + return code_logits + + def sample(self, history, input_visits): + history = history[:, :-1, :] + input_visits = input_visits[:, 1:, :] + currVisit = torch.cat((history, input_visits), dim=2)[:, -1, :].unsqueeze(1) + code_logits = self.auto2(torch.relu(self.auto1(currVisit)))[ + :, :, self.n_embd - 1:-1 + ] + return code_logits + + +class HALOModel(nn.Module): + """Low-level HALO transformer + autoregressive head (ported verbatim).""" + + def __init__(self, config): + super(HALOModel, self).__init__() + self.transformer = CoarseTransformerModel(config) + self.ehr_head = FineAutoregressiveHead(config) + + def forward( + self, + input_visits, + position_ids=None, + ehr_labels=None, + ehr_masks=None, + past=None, + pos_loss_weight=None, + ): + hidden_states = self.transformer(input_visits, position_ids, past) + code_logits = self.ehr_head(hidden_states, input_visits) + sig = nn.Sigmoid() + code_probs = sig(code_logits) + if ehr_labels is not None: + shift_labels = ehr_labels[..., 1:, :].contiguous() + loss_weights = None + if pos_loss_weight is not None: + loss_weights = torch.ones( + code_probs.shape, device=code_probs.device + ) + loss_weights = loss_weights + (pos_loss_weight - 1) * shift_labels + if ehr_masks is not None: + code_probs = code_probs * ehr_masks + shift_labels = shift_labels * ehr_masks + if pos_loss_weight is not None: + loss_weights = loss_weights * ehr_masks + + bce = nn.BCELoss(weight=loss_weights) + loss = bce(code_probs, shift_labels) + return loss, code_probs, shift_labels + + return code_probs + + def sample(self, input_visits, random=True): + sig = nn.Sigmoid() + hidden_states = self.transformer(input_visits) + i = 0 + while i < self.ehr_head.tot_vocab: + next_logits = self.ehr_head.sample(hidden_states, input_visits) + next_probs = sig(next_logits) + if random: + visit = torch.bernoulli(next_probs) + else: + visit = torch.round(next_probs) + + remaining_visit = visit[:, 0, i:] + nonzero = torch.nonzero(remaining_visit, as_tuple=True)[1] + if nonzero.numel() == 0: + break + + first_nonzero = nonzero.min() + input_visits[:, -1, i + first_nonzero] = visit[:, 0, i + first_nonzero] + i = i + first_nonzero + 1 + + return input_visits + + +# ---------------------------------------------------------------------------- +# PyHealth BaseModel wrapper +# ---------------------------------------------------------------------------- +class HALO(BaseModel): + """HALO synthetic-EHR generator, wrapped as a PyHealth ``BaseModel``. + + Trains a GPT-2-style transformer on patient visit-code sequences and + generates synthetic patients by autoregressive sampling. Generation is + **unconditional** (no label conditioning). + + The model infers its code vocabulary from the fitted ``SampleDataset``: + ``code_vocab_size = dataset.input_processors["visits"].vocab_size()`` + (the ``NestedSequenceProcessor`` vocab, which already reserves index 0 for + ```` and index 1 for ````). Three special tokens are appended for + start-of-sequence, end-of-sequence, and pad-visit. + + Args: + dataset: A fitted ``SampleDataset`` whose ``input_schema`` contains + ``{"visits": NestedSequenceProcessor}`` and whose ``output_schema`` + is empty. + embed_dim: Transformer embedding dimension (``n_embd``). Default: 768. + n_heads: Number of attention heads. Must divide ``embed_dim``. + Default: 12. + n_layers: Number of transformer layers. Default: 12. + n_ctx: Maximum number of visit positions (context length). Default: 48. + batch_size: Training batch size. Default: 48. + epochs: Number of training epochs. Default: 50. + pos_loss_weight: Positive-class weight for the BCE loss. ``None`` means + no weighting. Default: None. + lr: Learning rate for the Adam optimizer. Default: 1e-4. + save_dir: Directory for checkpoints written by ``train_model``. + Default: ``"./save/"``. + + Examples: + >>> from pyhealth.datasets import create_sample_dataset + >>> samples = [ + ... {"patient_id": "p1", "visits": [["A", "B"], ["C"]]}, + ... {"patient_id": "p2", "visits": [["A"], ["B", "C"]]}, + ... ] + >>> dataset = create_sample_dataset( + ... samples=samples, + ... input_schema={"visits": "nested_sequence"}, + ... output_schema={}, + ... ) + >>> model = HALO(dataset, embed_dim=16, n_heads=2, n_layers=2, n_ctx=8) + >>> isinstance(model, HALO) + True + """ + + def __init__( + self, + dataset, + embed_dim: int = 768, + n_heads: int = 12, + n_layers: int = 12, + n_ctx: int = 48, + batch_size: int = 48, + epochs: int = 50, + pos_loss_weight: Optional[float] = None, + lr: float = 1e-4, + save_dir: str = "./save/", + ) -> None: + super(HALO, self).__init__(dataset) + + if "visits" not in dataset.input_processors: + raise ValueError( + "HALO expects an input feature named 'visits' backed by a " + "NestedSequenceProcessor." + ) + + self.save_dir = save_dir + self._batch_size = batch_size + self._epochs = epochs + self._lr = lr + + # Code vocab from the NestedSequenceProcessor (includes , ). + self.visits_processor = dataset.input_processors["visits"] + code_vocab_size = self.visits_processor.vocab_size() + label_vocab_size = 0 # unconditional generation -- no output labels + # +3 special tokens: start-of-sequence, end-of-sequence, pad-visit. + total_vocab_size = code_vocab_size + label_vocab_size + 3 + + self.config = HALOConfig( + total_vocab_size=total_vocab_size, + code_vocab_size=code_vocab_size, + label_vocab_size=label_vocab_size, + special_vocab_size=3, + n_positions=n_ctx + 8, # position table needs a little slack + n_ctx=n_ctx, + n_embd=embed_dim, + n_layer=n_layers, + n_head=n_heads, + batch_size=batch_size, + epoch=epochs, + pos_loss_weight=pos_loss_weight, + lr=lr, + ) + + # Registered as a sub-module so .parameters()/.to() work. + self.halo_model = HALOModel(self.config) + + # ------------------------------------------------------------------ + # Multi-hot encoding helper + # ------------------------------------------------------------------ + def _encode_visits(self, visits: torch.Tensor): + """Convert a padded index tensor to HALO multi-hot format. + + ``NestedSequenceProcessor`` returns code indices; the transformer + expects multi-hot vectors of shape ``(batch, n_ctx, total_vocab_size)`` + with special tokens. Layout (mirrors the reference): position 0 is the + start token, visits occupy positions 2+, the end token is placed on the + last visit's row, and the pad token fills the remaining positions. + + Args: + visits: LongTensor ``(batch, max_visits, max_codes_per_visit)``. + Index 0 is ```` and is skipped. + + Returns: + batch_ehr: FloatTensor ``(batch, n_ctx, total_vocab_size)``. + batch_mask: FloatTensor ``(batch, n_ctx - 1, 1)``, shifted to align + with the autoregressive prediction targets. + """ + cfg = self.config + batch_size = visits.shape[0] + + batch_ehr = torch.zeros( + batch_size, cfg.n_ctx, cfg.total_vocab_size, device=self.device + ) + batch_mask = torch.zeros(batch_size, cfg.n_ctx, 1, device=self.device) + + start_idx = cfg.code_vocab_size + cfg.label_vocab_size + end_idx = start_idx + 1 + pad_idx = start_idx + 2 + + for i in range(batch_size): + # Count actual (non-padding) visits for this patient. + n_visits = int((visits[i].sum(dim=-1) > 0).sum().item()) + n_visits = min(n_visits, cfg.n_ctx - 2) + for j in range(n_visits): + for code_idx in visits[i, j]: + if code_idx > 0: # skip (index 0) + batch_ehr[i, j + 2, code_idx] = 1 + batch_mask[i, j + 2] = 1 + + batch_ehr[i, 0, start_idx] = 1 # start token + batch_ehr[i, n_visits + 1, end_idx] = 1 # end token (on last visit) + batch_ehr[i, n_visits + 2:, pad_idx] = 1 # pad visits + + batch_mask = batch_mask[:, 1:, :] # shift to align with shifted targets + return batch_ehr, batch_mask + + # ------------------------------------------------------------------ + # forward -- required by BaseModel + # ------------------------------------------------------------------ + def forward(self, visits: torch.Tensor, **kwargs) -> Dict[str, torch.Tensor]: + """Forward pass. + + Args: + visits: LongTensor ``(batch, max_visits, max_codes_per_visit)`` from + the ``NestedSequenceProcessor``. + **kwargs: Any other batch keys are ignored. + + Returns: + Dict with ``loss`` (scalar BCE) and ``y_prob`` (code probabilities, + shape ``(batch, n_ctx - 1, total_vocab_size)``). + """ + visits = visits.to(self.device) + batch_ehr, batch_mask = self._encode_visits(visits) + + loss, code_probs, _ = self.halo_model( + batch_ehr, + position_ids=None, + ehr_labels=batch_ehr, + ehr_masks=batch_mask, + pos_loss_weight=self.config.pos_loss_weight, + ) + return {"loss": loss, "y_prob": code_probs} + + # ------------------------------------------------------------------ + # Custom training loop + # ------------------------------------------------------------------ + @staticmethod + def _resolve_device(device=None) -> torch.device: + """Resolve a user-supplied device, defaulting to CUDA when available. + + Args: + device: ``None``, a device string (e.g. ``"cuda"``, ``"cuda:1"``, + ``"cpu"``), or a ``torch.device``. When ``None``, CUDA is used + if available, otherwise CPU. + """ + if device is None: + return torch.device("cuda" if torch.cuda.is_available() else "cpu") + return torch.device(device) + + def train_model(self, train_dataset, val_dataset=None, device=None) -> None: + """Train the HALO model with a custom loop. + + Named ``train_model`` (not ``train``) to avoid shadowing + ``nn.Module.train()``. Uses the standard ``get_dataloader`` (which pads + the variable visit dimension for us), an Adam optimizer, and BCE loss. + When ``val_dataset`` is given, validation loss is computed after each + epoch and the best checkpoint is saved to ``self.save_dir``. + + Args: + train_dataset: ``SampleDataset`` for training. + val_dataset: Optional ``SampleDataset`` for validation. + device: Device to train on, e.g. ``"cuda"``, ``"cuda:1"``, or + ``"cpu"``. If ``None`` (default), uses CUDA when available and + falls back to CPU. + """ + device = self._resolve_device(device) + self.to(device) + print(f"Training on: {device}") + + os.makedirs(self.save_dir, exist_ok=True) + optimizer = torch.optim.Adam(self.halo_model.parameters(), lr=self._lr) + + checkpoint_path = os.path.join(self.save_dir, "halo_model") + if os.path.exists(checkpoint_path): + checkpoint = torch.load(checkpoint_path, map_location=self.device) + self.halo_model.load_state_dict(checkpoint["model"]) + optimizer.load_state_dict(checkpoint["optimizer"]) + + train_loader = get_dataloader( + train_dataset, batch_size=self._batch_size, shuffle=True + ) + + global_loss = 1e10 + for epoch in tqdm(range(self._epochs), desc="Epochs"): + self.halo_model.train() + batch_iter = tqdm(train_loader, desc=f"Epoch {epoch}", leave=False) + for batch in batch_iter: + visits = batch["visits"].to(self.device) + batch_ehr, batch_mask = self._encode_visits(visits) + + optimizer.zero_grad() + loss, _, _ = self.halo_model( + batch_ehr, + position_ids=None, + ehr_labels=batch_ehr, + ehr_masks=batch_mask, + pos_loss_weight=self.config.pos_loss_weight, + ) + loss.backward() + optimizer.step() + batch_iter.set_postfix(loss=f"{loss.item():.4f}") + + if val_dataset is not None: + self.halo_model.eval() + val_loader = get_dataloader( + val_dataset, batch_size=self._batch_size, shuffle=False + ) + val_losses = [] + with torch.no_grad(): + for val_batch in val_loader: + visits = val_batch["visits"].to(self.device) + batch_ehr, batch_mask = self._encode_visits(visits) + val_loss, _, _ = self.halo_model( + batch_ehr, + position_ids=None, + ehr_labels=batch_ehr, + ehr_masks=batch_mask, + pos_loss_weight=self.config.pos_loss_weight, + ) + val_losses.append(val_loss.item()) + + cur_val_loss = float(np.mean(val_losses)) + print(f"Epoch {epoch} Validation Loss: {cur_val_loss:.7f}") + if cur_val_loss < global_loss: + global_loss = cur_val_loss + state = { + "model": self.halo_model.state_dict(), + "optimizer": optimizer.state_dict(), + "epoch": epoch, + } + torch.save(state, checkpoint_path) + print("------------ Save best model ------------") + + # ------------------------------------------------------------------ + # Synthesis + # ------------------------------------------------------------------ + def generate( + self, num_samples: int, random_sampling: bool = True, device=None + ) -> List[Dict]: + """Generate synthetic patients using the trained HALO model. + + Autoregressive sampling: feed a start token and repeatedly call + ``halo_model.sample`` until an end token is produced or ``n_ctx`` steps + are reached, then decode code indices back to code strings. + + Args: + num_samples: Number of synthetic patients to generate. + random_sampling: If True, Bernoulli sampling (stochastic). If False, + rounding (deterministic). Default: True. + device: Device to generate on, e.g. ``"cuda"``, ``"cuda:1"``, or + ``"cpu"``. If ``None`` (default), uses CUDA when available and + falls back to CPU. + + Returns: + List of dicts, each ``{"patient_id": "synthetic_i", + "visits": [[code, ...], ...]}`` with decoded code strings. + """ + device = self._resolve_device(device) + self.to(device) + + cfg = self.config + index_to_code = {v: k for k, v in self.visits_processor.code_vocab.items()} + end_token_idx = cfg.code_vocab_size + cfg.label_vocab_size + 1 + start_token_idx = cfg.code_vocab_size + cfg.label_vocab_size + + self.halo_model.eval() + synthetic_dataset: List[Dict] = [] + sample_batch_size = min(num_samples, 256) + generated = 0 + pbar = tqdm(total=num_samples, desc="Generating patients") + + with torch.no_grad(): + while generated < num_samples: + bs = min(sample_batch_size, num_samples - generated) + stoken = torch.zeros( + cfg.total_vocab_size, device=self.device, dtype=torch.float32 + ) + stoken[start_token_idx] = 1 + prev = stoken.unsqueeze(0).unsqueeze(0).repeat(bs, 1, 1) + empty = torch.zeros( + bs, 1, cfg.total_vocab_size, + device=self.device, dtype=torch.float32, + ) + + for _ in range(cfg.n_ctx - 1): + prev = self.halo_model.sample( + torch.cat((prev, empty), dim=1), random_sampling + ) + has_end = prev[:, :, end_token_idx].sum(dim=1).bool() + if has_end.all(): + break + + batch_ehrs = prev.cpu().detach().numpy() + for i in range(bs): + ehr = batch_ehrs[i] # (seq_len, total_vocab_size) + visits_out: List[List[str]] = [] + # Position 0 is the start token; visits occupy positions 1+. + for j in range(1, len(ehr)): + indices = np.nonzero(ehr[j])[0] + visit_codes: List[str] = [] + hit_end = False + for idx in indices: + if idx < cfg.code_vocab_size: + code = index_to_code.get(int(idx)) + if code not in (None, "", ""): + visit_codes.append(code) + elif idx == end_token_idx: + hit_end = True + if visit_codes: + visits_out.append(visit_codes) + if hit_end: + break + + synthetic_dataset.append( + { + "patient_id": f"synthetic_{generated + i}", + "visits": visits_out, + } + ) + generated += bs + pbar.update(bs) + pbar.close() + + return synthetic_dataset diff --git a/pyhealth/models/generators/medgan.py b/pyhealth/models/generators/medgan.py new file mode 100644 index 000000000..3e0ee06c0 --- /dev/null +++ b/pyhealth/models/generators/medgan.py @@ -0,0 +1,517 @@ +"""MedGAN: Medical Generative Adversarial Network for synthetic EHR generation. + +This is a port of the reference implementation +(https://github.com/mp2893/medgan and the PyTorch reimplementation under +``reference/cor-gan/Generative/medGAN/MIMIC/pytorch/MLP/medGAN.py``) wrapped +as a PyHealth ``BaseModel`` so it consumes the standard +``dataset -> SampleDataset -> model`` pipeline. + +MedGAN treats each patient as a flat bag-of-codes (no visit structure), so it +expects an input feature named ``visits`` backed by a ``MultiHotProcessor``. +The training procedure has two phases (mirroring the reference): + +* a **linear autoencoder** is pre-trained with binary cross-entropy + reconstruction loss, and +* an **adversarial training** phase where the generator emits latent codes, + the autoencoder's decoder projects them back to a multi-hot patient vector, + and a discriminator with optional minibatch averaging tries to distinguish + real from synthetic. + +The ``MedGANAutoencoder``, ``MedGANGenerator`` and ``MedGANDiscriminator`` +modules below mirror the reference. The public ``MedGAN`` class follows the +same API style as :class:`pyhealth.models.generators.HALO` +(``train_model`` / ``generate`` / ``save_model`` / ``load_model``). +""" + +import os +from typing import Dict, List, Optional + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.utils.data import DataLoader, Dataset, RandomSampler +from tqdm import tqdm + +from pyhealth.models import BaseModel + + +# ---------------------------------------------------------------------------- +# Building blocks (ported from reference medgan.py / PyTorch reimplementation) +# ---------------------------------------------------------------------------- +class _MultiHotDataset(Dataset): + """Tiny ``torch.utils.data.Dataset`` over a multi-hot numpy matrix.""" + + def __init__(self, data: np.ndarray): + self.data = data.astype(np.float32) + + def __len__(self) -> int: + return len(self.data) + + def __getitem__(self, idx): + return torch.from_numpy(self.data[idx]) + + +class MedGANAutoencoder(nn.Module): + """Linear autoencoder for MedGAN pretraining. + + Mirrors the reference single-layer encoder/decoder + (``Linear -> Tanh`` and ``Linear -> Sigmoid``). + + Args: + input_dim: Vocabulary size (number of distinct codes). + embedding_dim: Latent dimensionality. Default: 128. + """ + + def __init__(self, input_dim: int, embedding_dim: int = 128): + super().__init__() + self.encoder = nn.Sequential( + nn.Linear(input_dim, embedding_dim), + nn.Tanh(), + ) + self.decoder = nn.Sequential( + nn.Linear(embedding_dim, input_dim), + nn.Sigmoid(), + ) + + def forward(self, x): + return self.decoder(self.encoder(x)) + + def encode(self, x): + return self.encoder(x) + + def decode(self, x): + return self.decoder(x) + + +class MedGANGenerator(nn.Module): + """Two-layer MLP generator with residual connections (per reference).""" + + def __init__(self, latent_dim: int = 128, hidden_dim: int = 128): + super().__init__() + self.linear1 = nn.Linear(latent_dim, hidden_dim) + self.bn1 = nn.BatchNorm1d(hidden_dim, eps=0.001, momentum=0.01) + self.act1 = nn.ReLU() + + self.linear2 = nn.Linear(hidden_dim, hidden_dim) + self.bn2 = nn.BatchNorm1d(hidden_dim, eps=0.001, momentum=0.01) + self.act2 = nn.Tanh() + + def forward(self, x): + residual = x + out = self.act1(self.bn1(self.linear1(x))) + residual + + residual = out + out = self.act2(self.bn2(self.linear2(out))) + residual + return out + + +class MedGANDiscriminator(nn.Module): + """MLP discriminator with optional minibatch averaging (per reference).""" + + def __init__( + self, + input_dim: int, + hidden_dim: int = 256, + minibatch_averaging: bool = True, + ): + super().__init__() + self.minibatch_averaging = minibatch_averaging + model_input_dim = input_dim * 2 if minibatch_averaging else input_dim + + self.model = nn.Sequential( + nn.Linear(model_input_dim, hidden_dim), + nn.ReLU(), + nn.Linear(hidden_dim, hidden_dim // 2), + nn.ReLU(), + nn.Linear(hidden_dim // 2, 1), + nn.Sigmoid(), + ) + + def forward(self, x): + if self.minibatch_averaging: + # Average over the batch and concatenate to each sample, exactly + # as in the reference (medGAN.py). + x_mean = torch.mean(x, dim=0).repeat(x.shape[0], 1) + x = torch.cat((x, x_mean), dim=1) + return self.model(x) + + +def _weights_init(m): + """Xavier-uniform for Linear, N(1, 0.02) gamma / 0 beta for BatchNorm.""" + if isinstance(m, nn.Linear): + nn.init.xavier_uniform_(m.weight) + if m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.BatchNorm1d): + nn.init.normal_(m.weight, mean=1.0, std=0.02) + nn.init.constant_(m.bias, 0) + + +def _autoencoder_loss(x_output, y_target): + """Sparse-friendly BCE: sum over features, mean over batch. + + Equivalent to ``BCELoss(reduction='sum') / batch_size`` and matches the + reference; ``BCELoss(reduction='mean')`` would also mean over features + which dilutes the signal for sparse code vectors. + """ + epsilon = 1e-12 + term = y_target * torch.log(x_output + epsilon) + ( + 1.0 - y_target + ) * torch.log(1.0 - x_output + epsilon) + return torch.mean(-torch.sum(term, dim=1), dim=0) + + +# ---------------------------------------------------------------------------- +# PyHealth BaseModel wrapper +# ---------------------------------------------------------------------------- +class MedGAN(BaseModel): + """MedGAN synthetic-EHR generator, wrapped as a PyHealth ``BaseModel``. + + Generates synthetic binary EHR records via the two-phase procedure from + Choi et al. (MLHC 2017): pretrain a linear autoencoder, then run BCE-GAN + adversarial training where the generator maps noise to the autoencoder's + latent space and the decoder projects back to a multi-hot patient vector. + + Generation is **unconditional**: each synthetic patient is a flat bag of + codes (no visit structure), matching the ``multi_hot`` input schema. + + Args: + dataset: A fitted ``SampleDataset`` whose ``input_schema`` contains + ``{"visits": "multi_hot"}`` and whose ``output_schema`` is empty. + latent_dim: Generator noise dimensionality. Default: 128. The + generator's residual connection requires ``latent_dim == + hidden_dim``; if they differ, ``latent_dim`` is silently aligned to + ``hidden_dim``. + hidden_dim: Generator hidden width (also the autoencoder embedding + dimension). Default: 128. + discriminator_hidden_dim: Discriminator hidden width. Default: 256. + minibatch_averaging: Concatenate per-batch mean to each discriminator + input. Default: True. + batch_size: Training batch size. Default: 512. + ae_epochs: Autoencoder pre-training epochs. Default: 100. + gan_epochs: Adversarial training epochs. Default: 200. + ae_lr: Autoencoder learning rate. Default: 1e-3. + gan_lr: GAN learning rate. Default: 1e-3. + save_dir: Checkpoint directory used by ``train_model``. + Default: ``"./save/medgan/"``. + + Examples: + >>> from pyhealth.datasets import create_sample_dataset + >>> samples = [ + ... {"patient_id": "p1", "visits": ["A", "B", "C"]}, + ... {"patient_id": "p2", "visits": ["A", "C", "D"]}, + ... ] + >>> dataset = create_sample_dataset( + ... samples=samples, + ... input_schema={"visits": "multi_hot"}, + ... output_schema={}, + ... ) + >>> model = MedGAN(dataset, latent_dim=16, hidden_dim=16, batch_size=2) + >>> isinstance(model, MedGAN) + True + """ + + def __init__( + self, + dataset, + latent_dim: int = 128, + hidden_dim: int = 128, + discriminator_hidden_dim: int = 256, + minibatch_averaging: bool = True, + batch_size: int = 512, + ae_epochs: int = 100, + gan_epochs: int = 200, + ae_lr: float = 1e-3, + gan_lr: float = 1e-3, + save_dir: str = "./save/medgan/", + ) -> None: + super().__init__(dataset) + + if "visits" not in dataset.input_processors: + raise ValueError( + "MedGAN expects an input feature named 'visits' backed by a " + "MultiHotProcessor." + ) + + # The generator's residual connection (``out + residual`` with + # ``residual`` being the noise input) requires latent_dim == hidden_dim. + # Align silently if the user mismatched, mirroring CorGAN. + if latent_dim != hidden_dim: + latent_dim = hidden_dim + self.latent_dim = latent_dim + self.hidden_dim = hidden_dim + self._batch_size = batch_size + self._ae_epochs = ae_epochs + self._gan_epochs = gan_epochs + self._ae_lr = ae_lr + self._gan_lr = gan_lr + self.save_dir = save_dir + + # Code vocab from the MultiHotProcessor's label_vocab. + self.visits_processor = dataset.input_processors["visits"] + self.input_dim = self.visits_processor.size() + self._idx_to_code: List[Optional[str]] = [None] * self.input_dim + for code, idx in self.visits_processor.label_vocab.items(): + self._idx_to_code[idx] = code + + self.autoencoder = MedGANAutoencoder( + input_dim=self.input_dim, + embedding_dim=hidden_dim, + ) + self.generator = MedGANGenerator( + latent_dim=latent_dim, + hidden_dim=hidden_dim, + ) + self.discriminator = MedGANDiscriminator( + input_dim=self.input_dim, + hidden_dim=discriminator_hidden_dim, + minibatch_averaging=minibatch_averaging, + ) + + self.autoencoder.apply(_weights_init) + self.generator.apply(_weights_init) + self.discriminator.apply(_weights_init) + + # ------------------------------------------------------------------ + # forward -- required by BaseModel + # ------------------------------------------------------------------ + def forward(self, **kwargs) -> Dict[str, torch.Tensor]: + """MedGAN does not have a single supervised forward pass. + + Use :meth:`train_model` for training and :meth:`generate` for + synthesis. ``forward`` is implemented only to satisfy the + ``BaseModel`` abstract contract. + """ + raise NotImplementedError( + "MedGAN is a GAN: use train_model() and generate() instead of " + "forward()." + ) + + # ------------------------------------------------------------------ + # Custom training loop + # ------------------------------------------------------------------ + @staticmethod + def _resolve_device(device=None) -> torch.device: + """Resolve a user-supplied device, defaulting to CUDA when available.""" + if device is None: + return torch.device("cuda" if torch.cuda.is_available() else "cpu") + return torch.device(device) + + def _build_dataloader(self, dataset) -> DataLoader: + """Stack the multi-hot tensors of ``dataset`` into a DataLoader. + + The fitted ``MultiHotProcessor`` has already converted each patient's + ``visits`` field into a ``(input_dim,)`` float32 tensor, so we can + simply stack and wrap. + """ + tensors = [dataset[i]["visits"] for i in range(len(dataset))] + matrix = torch.stack(tensors).numpy() + wrapped = _MultiHotDataset(matrix) + sampler = RandomSampler(wrapped, replacement=True) + return DataLoader( + wrapped, + batch_size=self._batch_size, + shuffle=False, + num_workers=0, + drop_last=True, + sampler=sampler, + ) + + def train_model(self, train_dataset, val_dataset=None, device=None) -> None: + """Train MedGAN with a custom two-phase loop. + + Named ``train_model`` (not ``train``) to avoid shadowing + ``nn.Module.train()``. Phase 1 pre-trains the autoencoder with + sparse-friendly BCE reconstruction loss; phase 2 runs standard + BCE-GAN adversarial training where the generator+decoder are + optimised against a binary discriminator. + + Args: + train_dataset: ``SampleDataset`` for training. + val_dataset: Unused; accepted for API symmetry with other PyHealth + trainers. + device: Device to train on (``"cuda"``, ``"cpu"``, etc.). If + ``None``, uses CUDA when available. + """ + device = self._resolve_device(device) + self.to(device) + print(f"Training MedGAN on: {device}") + + os.makedirs(self.save_dir, exist_ok=True) + dataloader = self._build_dataloader(train_dataset) + + # ---- Phase 1: Autoencoder pretraining ---- + optimizer_ae = torch.optim.Adam( + self.autoencoder.parameters(), lr=self._ae_lr + ) + for epoch in tqdm(range(self._ae_epochs), desc="AE pretrain"): + self.autoencoder.train() + total_loss, n_batches = 0.0, 0 + for batch in dataloader: + real = batch.to(self.device) + recon = self.autoencoder(real) + loss = _autoencoder_loss(recon, real) + + optimizer_ae.zero_grad() + loss.backward() + optimizer_ae.step() + + total_loss += loss.item() + n_batches += 1 + + # ---- Phase 2: Adversarial training ---- + # Generator + the autoencoder's decoder are trained jointly, matching + # the reference (the decoder is what makes synthetic samples valid). + optimizer_g = torch.optim.Adam( + list(self.generator.parameters()) + + list(self.autoencoder.decoder.parameters()), + lr=self._gan_lr, + ) + optimizer_d = torch.optim.Adam( + self.discriminator.parameters(), lr=self._gan_lr + ) + + best_d_loss = float("inf") + for epoch in tqdm(range(self._gan_epochs), desc="GAN train"): + self.generator.train() + self.discriminator.train() + self.autoencoder.eval() + self.autoencoder.decoder.train() + + epoch_d_loss, epoch_g_loss, n_batches = 0.0, 0.0, 0 + for batch in dataloader: + real = batch.to(self.device) + bs = real.size(0) + + # --- Train Discriminator --- + optimizer_d.zero_grad() + noise = torch.randn(bs, self.latent_dim, device=self.device) + fake = self.autoencoder.decode(self.generator(noise)) + + real_pred = self.discriminator(real) + fake_pred = self.discriminator(fake.detach()) + + d_loss = F.binary_cross_entropy( + real_pred, torch.ones_like(real_pred) + ) + F.binary_cross_entropy( + fake_pred, torch.zeros_like(fake_pred) + ) + d_loss.backward() + optimizer_d.step() + + # --- Train Generator (+ decoder) --- + optimizer_g.zero_grad() + fake_pred = self.discriminator(fake) + g_loss = F.binary_cross_entropy( + fake_pred, torch.ones_like(fake_pred) + ) + g_loss.backward() + optimizer_g.step() + + epoch_d_loss += d_loss.item() + epoch_g_loss += g_loss.item() + n_batches += 1 + + avg_d = epoch_d_loss / max(n_batches, 1) + if avg_d < best_d_loss: + best_d_loss = avg_d + self.save_model(os.path.join(self.save_dir, "best.pt")) + + self.save_model(os.path.join(self.save_dir, "final.pt")) + + # ------------------------------------------------------------------ + # Synthesis + # ------------------------------------------------------------------ + def generate( + self, + num_samples: int, + random_sampling: bool = False, + device=None, + ) -> List[Dict]: + """Generate synthetic patient records. + + Each synthetic patient is decoded from a generated multi-hot vector + by thresholding (or, optionally, Bernoulli sampling) at 0.5 and + mapping the indices back to code strings. + + Args: + num_samples: Number of synthetic patients to generate. + random_sampling: If True, Bernoulli-sample the decoder output; + otherwise threshold at 0.5 (the reference's behaviour). + Default: False. + device: Device to generate on. If ``None``, uses CUDA when + available. + + Returns: + List of dicts + ``{"patient_id": "synthetic_i", "visits": [[code, ...]]}``. + ``visits`` is a list containing a **single** visit (matching + HALO's nested-list output structure). MedGAN is a bag-of-codes + model -- following the reference ``process_mimic.py``, each + patient is represented by the union of codes across all of + their historical visits -- so the single inner list is that + aggregate bag. The inner list may be empty if the generator + produced an all-zero vector. + """ + device = self._resolve_device(device) + self.to(device) + + self.generator.eval() + self.autoencoder.eval() + + bs = min(self._batch_size, max(num_samples, 1)) + rows = np.zeros((num_samples, self.input_dim), dtype=np.float32) + pbar = tqdm(total=num_samples, desc="Generating patients") + with torch.no_grad(): + i = 0 + while i < num_samples: + cur = min(bs, num_samples - i) + z = torch.randn(cur, self.latent_dim, device=self.device) + probs = self.autoencoder.decode(self.generator(z)) + if random_sampling: + sample = torch.bernoulli(probs) + else: + sample = (probs >= 0.5).float() + rows[i : i + cur] = sample.cpu().numpy() + i += cur + pbar.update(cur) + pbar.close() + + results: List[Dict] = [] + for i in range(num_samples): + codes = [ + self._idx_to_code[idx] + for idx in np.nonzero(rows[i])[0] + if self._idx_to_code[idx] not in (None, "", "") + ] + # Wrap in a single-visit list to mirror HALO's nested output. + # MedGAN models the patient as one aggregate bag of codes. + results.append({"patient_id": f"synthetic_{i}", "visits": [codes]}) + return results + + # ------------------------------------------------------------------ + # Checkpoint I/O + # ------------------------------------------------------------------ + def save_model(self, path: str) -> None: + """Save weights and the code vocabulary needed for decoding.""" + torch.save( + { + "autoencoder": self.autoencoder.state_dict(), + "generator": self.generator.state_dict(), + "discriminator": self.discriminator.state_dict(), + "input_dim": self.input_dim, + "latent_dim": self.latent_dim, + "idx_to_code": self._idx_to_code, + }, + path, + ) + + def load_model(self, path: str) -> None: + """Load weights and the code vocabulary from a checkpoint.""" + ckpt = torch.load(path, map_location=self.device) + self.autoencoder.load_state_dict(ckpt["autoencoder"]) + self.generator.load_state_dict(ckpt["generator"]) + self.discriminator.load_state_dict(ckpt["discriminator"]) + if "idx_to_code" in ckpt: + self._idx_to_code = ckpt["idx_to_code"] diff --git a/pyhealth/models/generators/promptehr.py b/pyhealth/models/generators/promptehr.py new file mode 100644 index 000000000..4622e72c6 --- /dev/null +++ b/pyhealth/models/generators/promptehr.py @@ -0,0 +1,517 @@ +"""PromptEHR: prompt-learning BART for synthetic EHR generation. + +This is a PyHealth ``BaseModel`` port of PromptEHR (Wang & Sun, EMNLP'22, +https://github.com/RyanWangZf/PromptEHR), wrapped so it consumes the standard +``dataset -> set_task -> SampleDataset -> model`` pipeline and shares the same +:class:`~pyhealth.tasks.EHRGeneration` task as +:class:`~pyhealth.models.HALO` and :class:`~pyhealth.models.GPT2`. + +PromptEHR treats sequential EHRs as a *neural database* and learns to fill in +patient records with a conditional **BART** (sequence-to-sequence denoising +autoencoder) trained with **prompt learning**. The three ideas that define the +reference implementation are preserved here: + +* **BART seq2seq core.** Generation is encoder-decoder, not decoder-only. The + reference subclasses ``BartForEHRSimulation`` from ``BartPretrainedModel``; + this port wraps :class:`transformers.BartForConditionalGeneration`, mirroring + the way :class:`~pyhealth.models.GPT2` wraps ``GPT2LMHeadModel``. +* **Prompt learning.** The reference reparameterizes a learnable prompt from + patient baseline demographics and prepends it to the encoder/decoder + (``ConditionalPrompt``). PyHealth's :class:`~pyhealth.tasks.EHRGeneration` + task is *unconditional* (only ``visits``, no baseline features -- exactly like + HALO/GPT2), so the prompt reduces to a learnable continuous **soft prefix** + prepended to the encoder. This is the prompt-tuning core without the + demographic reparameterization. +* **Span-infilling objective.** The reference learns by masking spans of codes + and reconstructing them. Here the encoder sees a BART-style span-infilled + copy of the patient's code stream -- random non-overlapping spans with + lengths drawn from ``Poisson(mean_span_len)`` are each replaced by a single + ``[MASK]`` sentinel until roughly ``mask_prob`` of the stream is covered -- + and the decoder reconstructs the full stream. This matches the original BART + text-infilling objective used by PromptEHR rather than per-token masking. + +Each patient's visits are serialized into a single code stream:: + + [CODE_PROMPT] [VISIT_DELIM] ... [EOS] + +The reference handles several code types (diagnosis / procedure / drug / lab) +each with its own modality prompt token; the PyHealth ``EHRGeneration`` task +exposes a single ``visits`` modality, so a single ``[CODE_PROMPT]`` token marks +it. The code vocabulary is taken from the dataset's +``NestedSequenceProcessor`` (which already reserves index 0 for ```` and +index 1 for ````); five special tokens (BOS, EOS, VISIT_DELIM, MASK, +CODE_PROMPT) are appended, and ```` (index 0) is reused as the pad token. +""" + +import os +from typing import Dict, List, Optional + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from tqdm import tqdm +from transformers import BartConfig, BartForConditionalGeneration + +from pyhealth.datasets import get_dataloader +from pyhealth.models import BaseModel + + +class PromptEHR(BaseModel): + """PromptEHR synthetic-EHR generator, wrapped as a PyHealth ``BaseModel``. + + Trains a BART denoising autoencoder with a learnable soft prompt on patient + visit-code streams, then generates synthetic patients by prompt-conditioned + encoder-decoder sampling. Generation is **unconditional** (no demographic + conditioning), matching the :class:`~pyhealth.tasks.EHRGeneration` task. + + Args: + dataset: A fitted ``SampleDataset`` whose ``input_schema`` contains + ``{"visits": NestedSequenceProcessor}`` and whose ``output_schema`` + is empty. + embed_dim: BART model dimension (``d_model``). Must be divisible by + ``n_heads``. Default: 256. + n_heads: Number of attention heads (encoder and decoder). Default: 8. + n_layers: Number of encoder and decoder layers each. Default: 6. + ffn_dim: Feed-forward dimension. Default: 4 * ``embed_dim``. + prompt_length: Number of learnable soft-prompt positions prepended to + the encoder. Default: 8. + max_len: Maximum code-stream length (``max_position_embeddings``); + streams are truncated to this length. Default: 512. + mask_prob: Target fraction of the (non-sentinel) code stream covered + by masked spans in the encoder input. Default: 0.15. + mean_span_len: Mean of the Poisson distribution used to sample span + lengths for BART-style span infilling. Default: 3.0. + batch_size: Training batch size. Default: 16. + epochs: Number of training epochs. Default: 50. + lr: Learning rate for the Adam optimizer. Default: 1e-4. + save_dir: Directory for checkpoints written by ``train_model``. + Default: ``"./save/"``. + + Examples: + >>> from pyhealth.datasets import create_sample_dataset + >>> samples = [ + ... {"patient_id": "p1", "visits": [["A", "B"], ["C"]]}, + ... {"patient_id": "p2", "visits": [["A"], ["B", "C"]]}, + ... ] + >>> dataset = create_sample_dataset( + ... samples=samples, + ... input_schema={"visits": "nested_sequence"}, + ... output_schema={}, + ... ) + >>> model = PromptEHR( + ... dataset, embed_dim=16, n_heads=2, n_layers=2, max_len=64 + ... ) + >>> isinstance(model, PromptEHR) + True + """ + + def __init__( + self, + dataset, + embed_dim: int = 256, + n_heads: int = 8, + n_layers: int = 6, + ffn_dim: Optional[int] = None, + prompt_length: int = 8, + max_len: int = 512, + mask_prob: float = 0.15, + mean_span_len: float = 3.0, + batch_size: int = 16, + epochs: int = 50, + lr: float = 1e-4, + save_dir: str = "./save/", + ) -> None: + super(PromptEHR, self).__init__(dataset) + + if "visits" not in dataset.input_processors: + raise ValueError( + "PromptEHR expects an input feature named 'visits' backed by a " + "NestedSequenceProcessor." + ) + + self.save_dir = save_dir + self._batch_size = batch_size + self._epochs = epochs + self._lr = lr + self.max_len = max_len + self.mask_prob = mask_prob + self.mean_span_len = mean_span_len + self.prompt_length = prompt_length + + # Code vocab from the NestedSequenceProcessor (includes =0, =1). + self.visits_processor = dataset.input_processors["visits"] + self.code_vocab_size = self.visits_processor.vocab_size() + # Append five special tokens after the code vocab; reuse =0 as PAD. + self.bos_id = self.code_vocab_size + self.eos_id = self.code_vocab_size + 1 + self.delim_id = self.code_vocab_size + 2 # visit separator + self.mask_id = self.code_vocab_size + 3 # denoising mask token + self.code_prompt_id = self.code_vocab_size + 4 # modality prompt token + self.pad_id = 0 + total_vocab_size = self.code_vocab_size + 5 + + ffn_dim = ffn_dim if ffn_dim is not None else 4 * embed_dim + config = BartConfig( + vocab_size=total_vocab_size, + max_position_embeddings=max_len, + d_model=embed_dim, + encoder_layers=n_layers, + decoder_layers=n_layers, + encoder_attention_heads=n_heads, + decoder_attention_heads=n_heads, + encoder_ffn_dim=ffn_dim, + decoder_ffn_dim=ffn_dim, + pad_token_id=self.pad_id, + bos_token_id=self.bos_id, + eos_token_id=self.eos_id, + decoder_start_token_id=self.eos_id, # BART convention + forced_bos_token_id=None, + forced_eos_token_id=None, + ) + # Registered as sub-modules so .parameters()/.to() work. + self.bart = BartForConditionalGeneration(config) + # Learnable soft prompt (prompt learning), prepended to the encoder. + self.soft_prompt = nn.Parameter(torch.zeros(prompt_length, embed_dim)) + nn.init.normal_(self.soft_prompt, std=config.init_std) + + # ------------------------------------------------------------------ + @staticmethod + def _resolve_device(device=None) -> torch.device: + """Resolve a user-supplied device, defaulting to CUDA when available.""" + if device is None: + return torch.device("cuda" if torch.cuda.is_available() else "cpu") + return torch.device(device) + + # ------------------------------------------------------------------ + # Visit index tensor -> denoising seq2seq tensors + # ------------------------------------------------------------------ + def _serialize(self, visits: torch.Tensor) -> List[List[int]]: + """Serialize each patient's visits into a flat code stream. + + Layout (decoder target): ``[CODE_PROMPT] codes_v1 [VS] codes_v2 ... + [EOS]``. Index 0 (````) is skipped. + """ + streams: List[List[int]] = [] + for i in range(visits.shape[0]): + n_visits = int((visits[i].sum(dim=-1) > 0).sum().item()) + seq: List[int] = [self.code_prompt_id] + for j in range(n_visits): + codes = [int(c) for c in visits[i, j].tolist() if c > 0] + seq.extend(codes) + if j < n_visits - 1: + seq.append(self.delim_id) + seq.append(self.eos_id) + # Truncate but always keep the trailing EOS. + if len(seq) > self.max_len: + seq = seq[: self.max_len - 1] + [self.eos_id] + streams.append(seq) + return streams + + def _corrupt(self, stream: List[int]) -> List[int]: + """Build the encoder input: BOS + a BART span-infilled copy of the stream. + + Selects random non-overlapping spans inside the stream (excluding the + leading ``[CODE_PROMPT]`` modality marker and the trailing ``[EOS]``) + with lengths drawn from ``Poisson(mean_span_len)`` until roughly + ``mask_prob`` of the stream is covered, then replaces each span with a + single ``[MASK]`` sentinel. Visit separators and code tokens are both + eligible for masking, matching the original BART text-infilling + objective used by PromptEHR. + """ + bos = [self.bos_id] + n = len(stream) + # Inner range excludes the leading [CODE_PROMPT] and trailing [EOS]. + inner_lo, inner_hi = 1, n - 1 + inner_len = inner_hi - inner_lo + if inner_len <= 0: + return bos + list(stream) + + target_masked = int(round(self.mask_prob * inner_len)) + if target_masked <= 0: + return bos + list(stream) + + spans: List[tuple] = [] # (start, end), half-open, in stream coords + masked = 0 + # Cap attempts to avoid pathological loops on tiny / fully-packed streams. + for _ in range(4 * inner_len): + if masked >= target_masked: + break + span_len = max(1, int(np.random.poisson(self.mean_span_len))) + span_len = min(span_len, inner_len) + start = int(np.random.randint(inner_lo, inner_hi - span_len + 1)) + end = start + span_len + if any(start < e and end > s for (s, e) in spans): + continue + spans.append((start, end)) + masked += span_len + + if not spans: + return bos + list(stream) + + spans.sort() + out: List[int] = bos + list(stream[:inner_lo]) + cur = inner_lo + for (s, e) in spans: + out.extend(stream[cur:s]) + out.append(self.mask_id) + cur = e + out.extend(stream[cur:inner_hi]) + out.extend(stream[inner_hi:]) + return out + + def _encode_batch(self, visits: torch.Tensor): + """Convert padded visit indices to encoder inputs and decoder labels. + + Returns: + enc_input_ids: LongTensor ``(batch, L_enc)`` corrupted streams. + enc_attention_mask: LongTensor ``(batch, L_enc)``. + labels: LongTensor ``(batch, L_dec)`` full streams, padding -> -100. + """ + streams = self._serialize(visits) + enc_streams = [self._corrupt(s) for s in streams] + + enc_input_ids = self._pad_stack(enc_streams, self.pad_id) + enc_attention_mask = (enc_input_ids != self.pad_id).long() + # Position 0 is BOS, never masked out by the pad check; force it on. + enc_attention_mask[:, 0] = 1 + + labels = self._pad_stack(streams, self.pad_id) + labels[labels == self.pad_id] = -100 + return enc_input_ids, enc_attention_mask, labels + + def _pad_stack(self, seqs: List[List[int]], pad_value: int) -> torch.Tensor: + """Right-pad a list of int lists into a 2D LongTensor on ``self.device``.""" + length = max(len(s) for s in seqs) + out = torch.full( + (len(seqs), length), pad_value, dtype=torch.long, device=self.device + ) + for i, s in enumerate(seqs): + out[i, : len(s)] = torch.tensor(s, device=self.device) + return out + + def _encoder_inputs_embeds(self, input_ids: torch.Tensor, attention_mask: torch.Tensor): + """Prepend the learnable soft prompt to the encoder token embeddings. + + Returns the prompt-augmented ``inputs_embeds`` and the matching + attention mask (soft-prompt positions are always attended to). + """ + token_embeds = self.bart.get_input_embeddings()(input_ids) + bsz = input_ids.shape[0] + prompt = self.soft_prompt.unsqueeze(0).expand(bsz, -1, -1) + inputs_embeds = torch.cat([prompt, token_embeds], dim=1) + prompt_mask = torch.ones( + bsz, self.prompt_length, dtype=attention_mask.dtype, device=self.device + ) + attention_mask = torch.cat([prompt_mask, attention_mask], dim=1) + return inputs_embeds, attention_mask + + # ------------------------------------------------------------------ + # forward -- required by BaseModel + # ------------------------------------------------------------------ + def forward(self, visits: torch.Tensor, **kwargs) -> Dict[str, torch.Tensor]: + """Forward pass (denoising seq2seq reconstruction). + + Args: + visits: LongTensor ``(batch, max_visits, max_codes_per_visit)`` from + the ``NestedSequenceProcessor``. + **kwargs: Any other batch keys are ignored. + + Returns: + Dict with ``loss`` (scalar seq2seq cross-entropy) and ``y_prob`` + (decoder next-token probabilities, shape ``(batch, L_dec, vocab)``). + """ + visits = visits.to(self.device) + enc_input_ids, enc_attention_mask, labels = self._encode_batch(visits) + inputs_embeds, enc_attention_mask = self._encoder_inputs_embeds( + enc_input_ids, enc_attention_mask + ) + out = self.bart( + inputs_embeds=inputs_embeds, + attention_mask=enc_attention_mask, + labels=labels, + ) + return {"loss": out.loss, "y_prob": F.softmax(out.logits, dim=-1)} + + # ------------------------------------------------------------------ + # Custom training loop + # ------------------------------------------------------------------ + def train_model(self, train_dataset, val_dataset=None, device=None) -> None: + """Train PromptEHR with a custom loop. + + Named ``train_model`` (not ``train``) to avoid shadowing + ``nn.Module.train()``. Uses the standard ``get_dataloader``, an Adam + optimizer, and the BART denoising loss. When ``val_dataset`` is given, + validation loss is computed after each epoch and the best checkpoint is + saved to ``self.save_dir``. + + Args: + train_dataset: ``SampleDataset`` for training. + val_dataset: Optional ``SampleDataset`` for validation. + device: Device to train on, e.g. ``"cuda"``, ``"cuda:1"``, or + ``"cpu"``. If ``None`` (default), uses CUDA when available and + falls back to CPU. + """ + device = self._resolve_device(device) + self.to(device) + print(f"Training on: {device}") + + os.makedirs(self.save_dir, exist_ok=True) + optimizer = torch.optim.Adam(self.parameters(), lr=self._lr) + + checkpoint_path = os.path.join(self.save_dir, "promptehr_model") + if os.path.exists(checkpoint_path): + checkpoint = torch.load(checkpoint_path, map_location=self.device) + self.load_state_dict(checkpoint["model"]) + optimizer.load_state_dict(checkpoint["optimizer"]) + + train_loader = get_dataloader( + train_dataset, batch_size=self._batch_size, shuffle=True + ) + + global_loss = 1e10 + for epoch in tqdm(range(self._epochs), desc="Epochs"): + self.bart.train() + batch_iter = tqdm(train_loader, desc=f"Epoch {epoch}", leave=False) + for batch in batch_iter: + visits = batch["visits"].to(self.device) + + optimizer.zero_grad() + ret = self.forward(visits=visits) + loss = ret["loss"] + loss.backward() + optimizer.step() + batch_iter.set_postfix(loss=f"{loss.item():.4f}") + + if val_dataset is not None: + self.bart.eval() + val_loader = get_dataloader( + val_dataset, batch_size=self._batch_size, shuffle=False + ) + val_losses = [] + with torch.no_grad(): + for val_batch in val_loader: + visits = val_batch["visits"].to(self.device) + val_losses.append(self.forward(visits=visits)["loss"].item()) + + cur_val_loss = float(np.mean(val_losses)) + print(f"Epoch {epoch} Validation Loss: {cur_val_loss:.7f}") + if cur_val_loss < global_loss: + global_loss = cur_val_loss + state = { + "model": self.state_dict(), + "optimizer": optimizer.state_dict(), + "epoch": epoch, + } + torch.save(state, checkpoint_path) + print("------------ Save best model ------------") + + # ------------------------------------------------------------------ + # Synthesis + # ------------------------------------------------------------------ + def _decode_ids(self, ids: List[int], index_to_code: Dict[int, str]) -> List[List[str]]: + """Decode a generated decoder token stream into per-visit code lists.""" + visits_out: List[List[str]] = [] + current: List[str] = [] + for tid in ids: + if tid in (self.bos_id, self.pad_id, self.code_prompt_id, self.mask_id): + continue + if tid == self.eos_id: + break + if tid == self.delim_id: + if current: + visits_out.append(current) + current = [] + continue + if tid < self.code_vocab_size: + code = index_to_code.get(int(tid)) + if code not in (None, "", ""): + current.append(code) + if current: + visits_out.append(current) + return visits_out + + def generate( + self, + num_samples: int, + device=None, + top_k: int = 50, + top_p: float = 0.95, + ) -> List[Dict]: + """Generate synthetic patients with the trained PromptEHR model. + + Feeds the encoder a fully-masked seed stream (so generation is driven by + the learned soft prompt), precomputes the prompt-augmented encoder + states, and autoregressively samples a decoder stream with + ``top_k``/``top_p`` sampling, then decodes it into per-visit code lists. + + Args: + num_samples: Number of synthetic patients to generate. + device: Device to generate on, e.g. ``"cuda"``, ``"cuda:1"``, or + ``"cpu"``. If ``None`` (default), uses CUDA when available and + falls back to CPU. + top_k: Top-k sampling cutoff. Default: 50. + top_p: Nucleus (top-p) sampling cutoff. Default: 0.95. + + Returns: + List of dicts, each ``{"patient_id": "synthetic_i", + "visits": [[code, ...], ...]}`` with decoded code strings. + """ + device = self._resolve_device(device) + self.to(device) + + index_to_code = {v: k for k, v in self.visits_processor.code_vocab.items()} + + self.bart.eval() + synthetic_dataset: List[Dict] = [] + sample_batch_size = min(num_samples, 256) + generated = 0 + pbar = tqdm(total=num_samples, desc="Generating patients") + + # Fully-masked seed: [BOS] [CODE_PROMPT] [MASK] [EOS]. + seed = [self.bos_id, self.code_prompt_id, self.mask_id, self.eos_id] + + with torch.no_grad(): + while generated < num_samples: + bs = min(sample_batch_size, num_samples - generated) + enc_input_ids = torch.tensor( + [seed] * bs, dtype=torch.long, device=self.device + ) + enc_attention_mask = torch.ones_like(enc_input_ids) + inputs_embeds, enc_attention_mask = self._encoder_inputs_embeds( + enc_input_ids, enc_attention_mask + ) + encoder_outputs = self.bart.get_encoder()( + inputs_embeds=inputs_embeds, + attention_mask=enc_attention_mask, + return_dict=True, + ) + out_ids = self.bart.generate( + encoder_outputs=encoder_outputs, + attention_mask=enc_attention_mask, + max_length=self.max_len, + do_sample=True, + top_k=top_k, + top_p=top_p, + num_beams=1, + pad_token_id=self.pad_id, + eos_token_id=self.eos_id, + decoder_start_token_id=self.eos_id, + ) + for i in range(bs): + # BART's generate prepends decoder_start_token_id (= eos_id) + # at position 0; skip it so the real eos terminates decoding. + visits_out = self._decode_ids( + out_ids[i].tolist()[1:], index_to_code + ) + synthetic_dataset.append( + { + "patient_id": f"synthetic_{generated + i}", + "visits": visits_out, + } + ) + generated += bs + pbar.update(bs) + pbar.close() + + return synthetic_dataset diff --git a/pyhealth/tasks/__init__.py b/pyhealth/tasks/__init__.py index a32618f9c..2140f23ed 100644 --- a/pyhealth/tasks/__init__.py +++ b/pyhealth/tasks/__init__.py @@ -45,6 +45,13 @@ from .mortality_prediction_stagenet_mimic4 import ( MortalityPredictionStageNetMIMIC4, ) +from .generate_ehr import ( + EHRGeneration, + EHRGenerationMIMIC3, + EHRGenerationMIMIC4, + decode_dataset, + to_evaluation_dataframe, +) from .patient_linkage import patient_linkage_mimic3_fn from .readmission_prediction import ( ReadmissionPredictionEICU, diff --git a/pyhealth/tasks/generate_ehr.py b/pyhealth/tasks/generate_ehr.py new file mode 100644 index 000000000..6fb23da9d --- /dev/null +++ b/pyhealth/tasks/generate_ehr.py @@ -0,0 +1,253 @@ +"""EHR sequence-generation tasks for PyHealth generative models. + +This is the shared task for every generator in +:mod:`pyhealth.models.generators` (HALO, MedGAN, CorGAN, PromptEHR, ...). It +extracts, for each patient, the ordered list of visits where each visit is the +list of medical codes recorded in that admission. The single input feature +``visits`` is processed by :class:`~pyhealth.processors.NestedSequenceProcessor`; +there is no prediction label, so ``output_schema`` is empty. + +:class:`EHRGeneration` holds all the extraction logic; dataset-specific +subclasses only declare which event type and code attribute to read. + +Evaluating generated data +------------------------- +The privacy/utility metrics in :mod:`pyhealth.metrics.generative` (``utils.py``, +``privacy.py``, ``utility.py`` -- exposed through ``evaluate_synthetic_ehr``) +consume **long-form** dataframes: one row per ``(patient, visit, code)`` with +columns ``id`` / ``time`` / ``visit_codes`` / ``labels``. ``id`` is the patient +identifier, ``time`` the (integer) visit index, ``visit_codes`` a single code +string, and ``labels`` a patient-level binary label (reduced via ``max`` over +the patient's rows). + +Both the real task samples and a generator's ``generate()`` output use the same +``{"visits": [[code, ...], ...]}`` record shape, so +:func:`to_evaluation_dataframe` converts either into that long-form table. A +processed ``SampleDataset`` can be turned back into records with +:func:`decode_dataset`. Subjects are renumbered sequentially (0, 1, 2, ...) in +the ``id`` column -- synthetic patients do not correspond to real ones, so any +original ``patient_id`` is ignored. + +.. code-block:: python + + from pyhealth.tasks.generate_ehr import decode_dataset, to_evaluation_dataframe + from pyhealth.metrics.generative import evaluate_synthetic_ehr + + # Real train/test EHR come from the processed SampleDataset(s): + train_df = to_evaluation_dataframe(decode_dataset(train_dataset)) + test_df = to_evaluation_dataframe(decode_dataset(test_dataset)) + + # Synthetic EHR comes straight from the trained generator (HALO, GPT2, ...): + synthetic = model.generate(num_samples=len(train_dataset)) + syn_df = to_evaluation_dataframe(synthetic) + + # Privacy metrics need no labels: + results = evaluate_synthetic_ehr(train_df, test_df, syn_df, metrics="privacy") + +The **utility** metrics (machine-learning efficacy, next-visit prediction) +additionally require a meaningful binary ``labels`` column. Since this task is +unconditional (no labels), pass a ``label_fn`` to derive one per patient -- e.g. +``label_fn=lambda r: any("250" in c for v in r["visits"] for c in v)`` for a +diabetes flag -- and the same ``label_fn`` must be applied to the real and +synthetic frames. With no label available, restrict to ``metrics="privacy"``. + +Note: + The MLE component currently hard-codes the downstream task to + next-visit prediction, which is degenerate for bag-of-codes + generators (MedGAN, CorGAN) that emit a single aggregate visit per + patient. A future revision will let callers plug in static-label + tasks (e.g. mortality, readmission, "ever diagnosed with X") so MLE + is meaningful for both sequential (HALO, GPT2, PromptEHR) and + bag-of-codes generators. Until then, restrict bag-of-codes + evaluation to ``metrics="privacy"`` plus the prevalence metrics. +""" + +import logging +from typing import Callable, Dict, List, Optional, Type, Union + +from pyhealth.data.data import Patient +from pyhealth.processors import NestedSequenceProcessor + +from .base_task import BaseTask + +logger = logging.getLogger(__name__) + + +class EHRGeneration(BaseTask): + """Generic per-visit code-sequence task for unconditional EHR generators. + + Builds one sample per qualifying patient: the ordered list of visits, each + visit being the list of codes (read from ``code_attr`` on ``event_type`` + events) recorded in that admission. Patients with fewer than ``min_visits`` + qualifying visits are skipped. + + Subclass and override the class attributes for a specific dataset, or set + them on an instance. The defaults read MIMIC-III ICD-9 diagnosis codes. + + Args: + task_name: Name of the task. + input_schema: ``{"visits": NestedSequenceProcessor}``. + output_schema: empty (generative task, no labels). + event_type: Event type to pull per admission. Default + ``"diagnoses_icd"``. + code_attr: Event attribute holding the code string. Default + ``"icd9_code"``. + min_visits: Minimum qualifying visits to keep a patient. Default 2. + """ + + task_name: str = "ehr_generation" + input_schema: Dict[str, Union[str, Type]] = {"visits": NestedSequenceProcessor} + output_schema: Dict[str, Union[str, Type]] = {} + + event_type: str = "diagnoses_icd" + code_attr: str = "icd9_code" + min_visits: int = 2 + + def __call__(self, patient: Patient) -> List[Dict]: + """Extract the per-visit code sequence for a patient.""" + visits: List[List[str]] = [] + admissions = patient.get_events(event_type="admissions") + for admission in admissions: + events = patient.get_events( + event_type=self.event_type, + filters=[("hadm_id", "==", admission.hadm_id)], + ) + codes = [ + getattr(event, self.code_attr) + for event in events + if getattr(event, self.code_attr, None) + ] + if codes: + visits.append(codes) + + if len(visits) < self.min_visits: + return [] + + return [{"patient_id": patient.patient_id, "visits": visits}] + + +class EHRGenerationMIMIC3(EHRGeneration): + """EHR generation task for MIMIC-III (ICD-9 diagnosis codes). + + Examples: + >>> from pyhealth.datasets import MIMIC3Dataset + >>> from pyhealth.tasks import EHRGenerationMIMIC3 + >>> dataset = MIMIC3Dataset( + ... root="/path/to/mimic-iii/1.4", + ... tables=["diagnoses_icd"], + ... ) + >>> samples = dataset.set_task(EHRGenerationMIMIC3()) + """ + + task_name: str = "ehr_generation_mimic3" + event_type: str = "diagnoses_icd" + code_attr: str = "icd9_code" + + +class EHRGenerationMIMIC4(EHRGeneration): + """EHR generation task for MIMIC-IV (ICD diagnosis codes). + + Examples: + >>> from pyhealth.datasets import MIMIC4Dataset + >>> from pyhealth.tasks import EHRGenerationMIMIC4 + >>> dataset = MIMIC4Dataset( + ... ehr_root="/path/to/mimiciv/2.2/", + ... ehr_tables=["patients", "admissions", "diagnoses_icd"], + ... ) + >>> samples = dataset.set_task(EHRGenerationMIMIC4()) + """ + + task_name: str = "ehr_generation_mimic4" + event_type: str = "diagnoses_icd" + code_attr: str = "icd_code" + + +# ---------------------------------------------------------------------------- +# Conversion helpers for pyhealth.metrics.generative.evaluate_synthetic_ehr +# ---------------------------------------------------------------------------- +def to_evaluation_dataframe( + records, + label_fn: Optional[Callable[[Dict], int]] = None, + subject_col: str = "id", + visit_col: str = "time", + code_col: str = "visit_codes", + label_col: str = "labels", +): + """Flatten EHR-generation records into the long-form evaluation dataframe. + + Produces the one-row-per-``(patient, visit, code)`` table consumed by + :func:`pyhealth.metrics.generative.evaluate_synthetic_ehr` (and the + ``utils.py`` / ``privacy.py`` / ``utility.py`` functions beneath it). + + Subjects are numbered **sequentially** (0, 1, 2, ...) in ``subject_col``; + any ``"patient_id"`` on the records is ignored, since synthetic patients do + not correspond to real ones. + + Args: + records: Iterable of ``{"visits": [[code, ...], ...]}`` dicts. Both the + :class:`EHRGeneration` task output and a generator's ``generate()`` + output have this shape. + label_fn: Optional callable mapping a record to a binary patient label + (0/1) used by the utility metrics. Defaults to all-zeros. + subject_col: Output patient-id column. Default ``"id"``. + visit_col: Output visit-index column. Default ``"time"``. + code_col: Output single-code column. Default ``"visit_codes"``. + label_col: Output binary-label column. Default ``"labels"``. + + Returns: + ``pandas.DataFrame`` with columns + ``[subject_col, visit_col, code_col, label_col]``. + """ + import pandas as pd + + rows = [] + for subject_id, record in enumerate(records): + label = 0 if label_fn is None else int(label_fn(record)) + for visit_idx, visit in enumerate(record["visits"]): + for code in visit: + rows.append( + { + subject_col: subject_id, + visit_col: visit_idx, + code_col: code, + label_col: label, + } + ) + return pd.DataFrame( + rows, columns=[subject_col, visit_col, code_col, label_col] + ) + + +def decode_dataset(sample_dataset, feature_key: str = "visits") -> List[Dict]: + """Decode a processed EHRGeneration ``SampleDataset`` back into records. + + Inverts the :class:`~pyhealth.processors.NestedSequenceProcessor` encoding + using its vocabulary (skipping ````/````), yielding one + ``{"visits": [[code_str, ...], ...]}`` record per sample. Use this to build + the real train/test frames that ``evaluate_synthetic_ehr`` compares against. + + Args: + sample_dataset: A ``SampleDataset`` produced by :class:`EHRGeneration`. + feature_key: Input feature key holding the nested code sequence. + Default ``"visits"``. + + Returns: + List of ``{"visits": [[code_str, ...], ...]}`` records. + """ + processor = sample_dataset.input_processors[feature_key] + index_to_code = {idx: code for code, idx in processor.code_vocab.items()} + + records: List[Dict] = [] + for i in range(len(sample_dataset)): + sample = sample_dataset[i] + visits: List[List[str]] = [] + for row in sample[feature_key].tolist(): + codes = [ + index_to_code[int(idx)] + for idx in row + if index_to_code.get(int(idx)) not in (None, "", "") + ] + if codes: + visits.append(codes) + records.append({"visits": visits}) + return records diff --git a/tests/core/test_corgan.py b/tests/core/test_corgan.py new file mode 100644 index 000000000..7c5f54963 --- /dev/null +++ b/tests/core/test_corgan.py @@ -0,0 +1,228 @@ +import tempfile +import unittest + +import torch + +from pyhealth.datasets import create_sample_dataset +from pyhealth.models import CorGAN + + +class TestCorGAN(unittest.TestCase): + """Test cases for the CorGAN synthetic-EHR generator.""" + + def setUp(self): + """Bag-of-codes generative dataset (no labels) and a tiny model.""" + self.samples = [ + {"patient_id": "patient-0", "visits": ["A05B", "A05C", "A11D", "C129"]}, + {"patient_id": "patient-1", "visits": ["A05B", "A04A", "B035"]}, + {"patient_id": "patient-2", "visits": ["C129", "A11D", "A05C", "A04A"]}, + {"patient_id": "patient-3", "visits": ["B035", "A05B", "C129"]}, + ] + self.input_schema = {"visits": "multi_hot"} + self.output_schema = {} + + self.dataset = create_sample_dataset( + samples=self.samples, + input_schema=self.input_schema, + output_schema=self.output_schema, + dataset_name="test_corgan", + ) + + # Small vocab -> linear autoencoder is auto-selected. + self.model = CorGAN( + dataset=self.dataset, + latent_dim=8, + hidden_dim=8, + discriminator_hidden_dim=16, + batch_size=2, + ae_epochs=1, + gan_epochs=1, + n_iter_D=1, + ) + + def test_model_initialization(self): + """Vocab is derived from MultiHotProcessor; generation is unconditional.""" + self.assertIsInstance(self.model, CorGAN) + self.assertEqual(self.model.feature_keys, ["visits"]) + self.assertEqual(self.model.label_keys, []) + + proc_vocab = self.dataset.input_processors["visits"].size() + self.assertEqual(self.model.input_dim, proc_vocab) + + def test_small_vocab_falls_back_to_linear(self): + """For tiny vocabs the 6-layer CNN can't compress -- expect linear AE.""" + self.assertEqual(self.model.autoencoder_type, "linear") + + def test_components_present(self): + """Autoencoder, generator, and critic are all registered submodules.""" + self.assertTrue(hasattr(self.model, "autoencoder")) + self.assertTrue(hasattr(self.model, "generator")) + self.assertTrue(hasattr(self.model, "critic")) + + # Critic is a Wasserstein critic -> unbounded scalar (no sigmoid). + x = torch.zeros(2, self.model.input_dim) + with torch.no_grad(): + c_out = self.model.critic(x) + self.assertEqual(c_out.shape, (2, 1)) + + def test_forward_raises(self): + """CorGAN's BaseModel forward intentionally errors out.""" + with self.assertRaises(NotImplementedError): + self.model.forward() + + def test_train_model_runs(self): + """train_model completes a tiny two-phase loop on CPU and returns history.""" + with tempfile.TemporaryDirectory() as tmp: + model = CorGAN( + dataset=self.dataset, + latent_dim=8, + hidden_dim=8, + discriminator_hidden_dim=16, + batch_size=2, + ae_epochs=1, + gan_epochs=1, + n_iter_D=1, + save_dir=tmp, + ) + history = model.train_model(self.dataset, device="cpu") + + self.assertIn("autoencoder_loss", history) + self.assertIn("critic_loss", history) + self.assertIn("generator_loss", history) + self.assertEqual(len(history["autoencoder_loss"]), 1) + self.assertEqual(len(history["critic_loss"]), 1) + self.assertEqual(len(history["generator_loss"]), 1) + self.assertEqual(next(model.parameters()).device.type, "cpu") + + def test_generate(self): + """generate() returns the requested number of decoded synthetic patients. + + CorGAN is a bag-of-codes model, so each patient gets a single visit + containing the aggregate set of codes; the outer ``visits`` list + wraps that single visit to match HALO's nested format. + """ + synthetic = self.model.generate(num_samples=4, device="cpu") + self.assertEqual(len(synthetic), 4) + for i, patient in enumerate(synthetic): + self.assertEqual(patient["patient_id"], f"synthetic_{i}") + self.assertIsInstance(patient["visits"], list) + # Exactly one aggregate visit per patient. + self.assertEqual(len(patient["visits"]), 1) + visit = patient["visits"][0] + self.assertIsInstance(visit, list) + for code in visit: + self.assertIsInstance(code, str) + self.assertNotIn(code, ("", "")) + + def test_generate_random_sampling(self): + """random_sampling=True still produces well-formed patients.""" + synthetic = self.model.generate( + num_samples=3, random_sampling=True, device="cpu" + ) + self.assertEqual(len(synthetic), 3) + for patient in synthetic: + self.assertIn("patient_id", patient) + self.assertIn("visits", patient) + + def test_save_and_load_roundtrip(self): + """save_model + load_model preserves weights and vocabulary.""" + with tempfile.TemporaryDirectory() as tmp: + path = f"{tmp}/corgan.pt" + self.model.save_model(path) + + other = CorGAN( + dataset=self.dataset, + latent_dim=8, + hidden_dim=8, + discriminator_hidden_dim=16, + batch_size=2, + ae_epochs=1, + gan_epochs=1, + n_iter_D=1, + save_dir=tmp, + ) + other.load_model(path) + + for p1, p2 in zip( + self.model.generator.parameters(), + other.generator.parameters(), + ): + self.assertTrue(torch.allclose(p1, p2)) + self.assertEqual(other._idx_to_code, self.model._idx_to_code) + + def test_missing_visits_processor_raises(self): + """A dataset without a 'visits' feature should be rejected.""" + bad = create_sample_dataset( + samples=[ + {"patient_id": "p1", "codes": ["A", "B"]}, + {"patient_id": "p2", "codes": ["B"]}, + ], + input_schema={"codes": "multi_hot"}, + output_schema={}, + ) + with self.assertRaises(ValueError): + CorGAN(bad, latent_dim=8, hidden_dim=8) + + def test_unknown_autoencoder_type_raises(self): + """An unknown autoencoder_type is rejected.""" + with self.assertRaises(ValueError): + CorGAN(self.dataset, autoencoder_type="rnn") + + def test_cnn_path_with_large_vocab(self): + """A vocabulary big enough survives the 6-layer CNN chain end-to-end.""" + all_codes = [f"C{i:04d}" for i in range(1200)] + samples = [ + { + "patient_id": f"p{i}", + "visits": all_codes[i * 200 : (i + 1) * 200] + + all_codes[1100:1200], + } + for i in range(6) + ] + dataset = create_sample_dataset( + samples=samples, + input_schema={"visits": "multi_hot"}, + output_schema={}, + dataset_name="cnn_corgan", + ) + with tempfile.TemporaryDirectory() as tmp: + model = CorGAN( + dataset=dataset, + batch_size=2, + ae_epochs=1, + gan_epochs=1, + n_iter_D=1, + save_dir=tmp, + ) + self.assertEqual(model.autoencoder_type, "cnn") + # CNN bottleneck is fixed at 128 -- generator should track it. + self.assertEqual(model.latent_dim, 128) + self.assertEqual(model.hidden_dim, 128) + model.train_model(dataset, device="cpu") + out = model.generate(num_samples=2, device="cpu") + self.assertEqual(len(out), 2) + + @unittest.skipUnless(torch.cuda.is_available(), "CUDA not available") + def test_train_and_generate_on_cuda(self): + """When CUDA is available, the device arg moves training to GPU.""" + with tempfile.TemporaryDirectory() as tmp: + model = CorGAN( + dataset=self.dataset, + latent_dim=8, + hidden_dim=8, + discriminator_hidden_dim=16, + batch_size=2, + ae_epochs=1, + gan_epochs=1, + n_iter_D=1, + save_dir=tmp, + ) + model.train_model(self.dataset, device="cuda") + self.assertTrue(next(model.parameters()).is_cuda) + + synthetic = model.generate(num_samples=2, device="cuda") + self.assertEqual(len(synthetic), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/core/test_generative_metrics.py b/tests/core/test_generative_metrics.py new file mode 100644 index 000000000..27b8a9796 --- /dev/null +++ b/tests/core/test_generative_metrics.py @@ -0,0 +1,472 @@ +"""Unit tests for pyhealth.metrics.generative (synthetic-EHR metrics). + +Run with:: + + python -m unittest tests.core.test_generative_metrics -v +""" + +import unittest + +import numpy as np +import pandas as pd + +from pyhealth.metrics.generative import ( + calc_membership_inference, + calc_nnaar, + compute_discriminator_privacy, + compute_mle, + compute_prevalence_metrics, + evaluate_synthetic_ehr, +) +from pyhealth.metrics.generative.utils import ( + convert_cols_to_multihot, + train_lstm_model, + train_sklearn_model, +) + +SUBJECT_COL, VISIT_COL, CODE_COL, LABEL_COL = "id", "time", "visit_codes", "labels" + + +def _make_dataframes(): + """Builds small synthetic train/test/synthetic EHR dataframes.""" + train_ehr = pd.DataFrame( + { + "visit_codes": [0, 1, 3, 4, 1, 2, 0, 3, 2, 4, 1, 0, 2, 3, 4, + 1, 0, 2, 3, 4, 1, 0, 2, 3, 4], + "labels": [0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, + 1, 0, 0, 1, 0, 1, 0, 0, 1, 0], + "time": [0, 0, 1, 1, 0, 1, 2, 2, 3, 3, 1, 2, 3, 4, 4, + 0, 1, 2, 3, 4, 1, 2, 3, 4, 5], + "id": [0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, + 3, 3, 3, 3, 3, 4, 4, 4, 4, 4], + } + ).astype({"visit_codes": str, "labels": int, "time": int, "id": str}) + + test_ehr = pd.DataFrame( + { + "visit_codes": [1, 2, 0, 3, 4, 2, 1, 0, 3, 4, 1, 2, 3, 0, 4, + 2, 1, 3, 0, 4], + "labels": [0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, + 0, 0, 1, 0, 1], + "time": [0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 0, 1, 1, 2, 2, + 3, 3, 3, 4, 4], + "id": [0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2], + } + ).astype({"visit_codes": str, "labels": int, "time": int, "id": str}) + + syn_ehr = pd.DataFrame( + { + "visit_codes": [2, 3, 1, 4, 0, 2, 3, 1, 0, 4, 1, 2, 3, 4, 0, + 2, 1, 3, 4, 0, 2, 1, 3, 4, 0, 1, 2, 3, 4, 0], + "labels": [0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, + 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1], + "time": [0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, + 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5], + "id": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], + } + ).astype({"visit_codes": str, "labels": int, "time": int, "id": str}) + + return train_ehr, test_ehr, syn_ehr + + +def _generate_ehr( + n_patients, vocab, seed, id_offset=0, + n_visits_range=(2, 7), n_codes_range=(2, 6), +): + """Generates a random EHR dataframe with patients drawn from ``vocab``.""" + rng = np.random.default_rng(seed) + rows = [] + for i in range(n_patients): + pid = str(id_offset + i) + n_visits = int(rng.integers(*n_visits_range)) + label = int(rng.integers(0, 2)) + for t in range(n_visits): + n_codes = int(rng.integers(*n_codes_range)) + codes = rng.choice( + vocab, size=min(n_codes, len(vocab)), replace=False + ) + for code in codes: + rows.append( + {"id": pid, "time": t, + "visit_codes": str(code), "labels": label} + ) + return pd.DataFrame(rows).astype( + {"visit_codes": str, "labels": int, "time": int, "id": str} + ) + + +def _perturb_ehr(df, vocab, frac, seed): + """Returns a copy of ``df`` with a fraction of codes randomly replaced.""" + rng = np.random.default_rng(seed) + df = df.copy().reset_index(drop=True) + mask = rng.random(len(df)) < frac + new_codes = rng.choice(vocab, size=int(mask.sum())) + df.loc[mask, "visit_codes"] = [str(c) for c in new_codes] + return df + + +class GenerativeMetricsTestCase(unittest.TestCase): + """Shared fixtures and assertion helpers for the generative metrics.""" + + def setUp(self): + np.random.seed(0) + self.train_ehr, self.test_ehr, self.syn_ehr = _make_dataframes() + self.cols = dict( + subject_col=SUBJECT_COL, + visit_col=VISIT_COL, + code_col=CODE_COL, + label_col=LABEL_COL, + ) + + def assertSummary(self, summary, expected_keys): + """Asserts a metrics summary has the expected (mean, std) structure.""" + self.assertIsInstance(summary, dict) + for key in expected_keys: + self.assertIn(key, summary) + value = summary[key] + self.assertIsInstance(value, tuple) + self.assertEqual(len(value), 2) + mean, std = value + self.assertTrue(np.isfinite(mean), f"{key} mean not finite") + self.assertTrue(np.isfinite(std), f"{key} std not finite") + self.assertGreaterEqual(std, 0.0) + + +class TestNNAAR(GenerativeMetricsTestCase): + def test_calc_nnaar(self): + summary = calc_nnaar( + self.train_ehr, self.test_ehr, self.syn_ehr, + **self.cols, sample_size=10, n_runs=3, + ) + self.assertSummary(summary, ["nnaar", "aa_es", "aa_ts"]) + for key in ("aa_es", "aa_ts"): + self.assertGreaterEqual(summary[key][0], 0.0) + self.assertLessEqual(summary[key][0], 1.0) + self.assertGreaterEqual(summary["nnaar"][0], -1.0) + self.assertLessEqual(summary["nnaar"][0], 1.0) + + +class TestMembershipInference(GenerativeMetricsTestCase): + def test_calc_membership_inference(self): + summary = calc_membership_inference( + self.train_ehr, self.test_ehr, self.syn_ehr, + **self.cols, num_attack_samples=10, n_runs=3, + ) + keys = ["MIA_F1", "MIA_Precision", "MIA_Recall", "MIA_Accuracy"] + self.assertSummary(summary, keys) + for key in keys: + self.assertGreaterEqual(summary[key][0], 0.0) + self.assertLessEqual(summary[key][0], 1.0) + + +class TestDiscriminatorPrivacy(GenerativeMetricsTestCase): + def test_discriminator_privacy_lstm(self): + summary = compute_discriminator_privacy( + train_fn=train_lstm_model, + train_ehr=self.train_ehr, test_ehr=self.test_ehr, + syn_ehr=self.syn_ehr, **self.cols, n_bootstraps=3, + embed_dim=8, hidden_dim=8, batch_size=8, epochs=2, verbose=False, + ) + keys = ["Privacy_Discriminator_Accuracy", "Privacy_Score"] + self.assertSummary(summary, keys) + self.assertGreaterEqual(summary["Privacy_Score"][0], 0.0) + self.assertLessEqual(summary["Privacy_Score"][0], 1.0) + + def test_discriminator_privacy_rf(self): + summary = compute_discriminator_privacy( + train_fn=train_sklearn_model, + train_ehr=self.train_ehr, test_ehr=self.test_ehr, + syn_ehr=self.syn_ehr, **self.cols, n_bootstraps=3, model="rf", + ) + self.assertSummary( + summary, ["Privacy_Discriminator_Accuracy", "Privacy_Score"] + ) + + +class TestMLE(GenerativeMetricsTestCase): + def test_compute_mle_lstm(self): + summary = compute_mle( + train_fn=train_lstm_model, + train_ehr=self.train_ehr, test_ehr=self.test_ehr, + syn_ehr=self.syn_ehr, **self.cols, n_bootstraps=3, + embed_dim=8, hidden_dim=8, batch_size=8, epochs=2, verbose=False, + ) + keys = [ + "MLE_Real_Accuracy", "MLE_Synth_Accuracy", "MLE_Difference", + "MLE_Ratio", "MLE_Real_F1", "MLE_Synth_F1", + ] + self.assertSummary(summary, keys) + for key in ("MLE_Real_Accuracy", "MLE_Synth_Accuracy"): + self.assertGreaterEqual(summary[key][0], 0.0) + self.assertLessEqual(summary[key][0], 1.0) + + def test_compute_mle_rf(self): + summary = compute_mle( + train_fn=train_sklearn_model, + train_ehr=self.train_ehr, test_ehr=self.test_ehr, + syn_ehr=self.syn_ehr, **self.cols, n_bootstraps=3, model="rf", + ) + self.assertSummary(summary, ["MLE_Real_Accuracy", "MLE_Synth_Accuracy"]) + + +class TestPrevalenceMetrics(GenerativeMetricsTestCase): + def test_compute_prevalence_metrics(self): + summary = compute_prevalence_metrics( + self.train_ehr, self.syn_ehr, + subject_col=SUBJECT_COL, code_col=CODE_COL, n_bootstraps=3, + ) + keys = ["Prevalence_R2", "Prevalence_Pearson", "Prevalence_RMSE"] + self.assertSummary(summary, keys) + self.assertGreaterEqual(summary["Prevalence_Pearson"][0], -1.0) + self.assertLessEqual(summary["Prevalence_Pearson"][0], 1.0) + self.assertGreaterEqual(summary["Prevalence_RMSE"][0], 0.0) + + +class TestConvertColsToMultihot(GenerativeMetricsTestCase): + def test_convert_cols_to_multihot(self): + df = self.train_ehr.copy() + df["gender"] = ["M", "F"] * 12 + ["M"] + df["age"] = np.arange(len(df), dtype=float) + out = convert_cols_to_multihot( + df, code_col=CODE_COL, visit_col=VISIT_COL, + cat_cols=["gender"], num_cols=["age"], bins_per_num=2, + ) + self.assertIn("combined_codes", out.columns) + self.assertEqual(len(out), len(df)) + # Each combined code should fold in the code, the category and the bin. + first = out["combined_codes"].iloc[0] + self.assertIn("gender_", first) + self.assertIn("age_", first) + # The original dataframe must not be mutated. + self.assertNotIn("combined_codes", df.columns) + + +class TestEvaluateSyntheticEHR(GenerativeMetricsTestCase): + def test_evaluate_all_lstm(self): + out = evaluate_synthetic_ehr( + self.train_ehr, self.test_ehr, self.syn_ehr, **self.cols, + sample_size=10, mode="lstm", metrics="all", + lstm_params={"embed_dim": 8, "hidden_dim": 8, + "batch_size": 8, "epochs": 2}, + n_bootstraps=3, n_runs=3, + ) + for key in ("nnaar", "MIA_F1", "MLE_Real_Accuracy", + "Privacy_Score", "Prevalence_RMSE"): + self.assertIn(key, out) + + def test_evaluate_privacy_only_rf(self): + out = evaluate_synthetic_ehr( + self.train_ehr, self.test_ehr, self.syn_ehr, **self.cols, + sample_size=10, mode="rf", metrics="privacy", + n_bootstraps=3, n_runs=3, + ) + self.assertIn("nnaar", out) + self.assertNotIn("MLE_Real_Accuracy", out) + + def test_evaluate_utility_only_rf(self): + out = evaluate_synthetic_ehr( + self.train_ehr, self.test_ehr, self.syn_ehr, **self.cols, + mode="rf", metrics="utility", n_bootstraps=3, + ) + self.assertIn("MLE_Real_Accuracy", out) + self.assertNotIn("nnaar", out) + + def test_invalid_mode_raises(self): + with self.assertRaises(ValueError): + evaluate_synthetic_ehr( + self.train_ehr, self.test_ehr, self.syn_ehr, **self.cols, + mode="bad", + ) + + def test_invalid_metrics_raises(self): + with self.assertRaises(ValueError): + evaluate_synthetic_ehr( + self.train_ehr, self.test_ehr, self.syn_ehr, **self.cols, + metrics="bad", + ) + + +class TestMetricsBehavior(unittest.TestCase): + """Sanity checks: metrics should respond to how close synthetic data is. + + Three synthetic datasets are compared against the same real data: + + - ``exact``: an exact copy of the real training data, + - ``similar``: the training data with ~15% of codes randomly changed, + - ``different``: independent data over a disjoint code vocabulary. + + A well-behaved metric should rank these consistently (e.g. an exact copy + is the worst case for privacy and the best case for fidelity). + """ + + VOCAB_REAL = list(range(50)) + VOCAB_DIFF = list(range(100, 150)) + + @classmethod + def setUpClass(cls): + cls.train_ehr = _generate_ehr(60, cls.VOCAB_REAL, seed=1, id_offset=0) + cls.test_ehr = _generate_ehr( + 60, cls.VOCAB_REAL, seed=2, id_offset=10000 + ) + cls.syn_exact = cls.train_ehr.copy() + cls.syn_similar = _perturb_ehr( + cls.train_ehr, cls.VOCAB_REAL, frac=0.15, seed=3 + ) + cls.syn_different = _generate_ehr( + 60, cls.VOCAB_DIFF, seed=4, id_offset=20000 + ) + cls.cols = dict( + subject_col=SUBJECT_COL, + visit_col=VISIT_COL, + code_col=CODE_COL, + label_col=LABEL_COL, + ) + + def test_prevalence_orders_by_similarity(self): + # Prevalence similarity should degrade monotonically: exact > similar + # > different. + results = {} + for name, syn in [ + ("exact", self.syn_exact), + ("similar", self.syn_similar), + ("different", self.syn_different), + ]: + np.random.seed(0) + results[name] = compute_prevalence_metrics( + self.train_ehr, syn, + subject_col=SUBJECT_COL, code_col=CODE_COL, n_bootstraps=10, + ) + + rmse = {k: v["Prevalence_RMSE"][0] for k, v in results.items()} + r2 = {k: v["Prevalence_R2"][0] for k, v in results.items()} + pearson = {k: v["Prevalence_Pearson"][0] for k, v in results.items()} + + # An exact copy has identical code prevalence. + self.assertAlmostEqual(rmse["exact"], 0.0, places=9) + self.assertAlmostEqual(r2["exact"], 1.0, places=6) + self.assertAlmostEqual(pearson["exact"], 1.0, places=6) + + # Error grows / agreement shrinks as synthetic data drifts away. + self.assertLess(rmse["exact"], rmse["similar"]) + self.assertLess(rmse["similar"], rmse["different"]) + self.assertGreater(r2["exact"], r2["similar"]) + self.assertGreater(r2["similar"], r2["different"]) + self.assertGreaterEqual(pearson["exact"], pearson["similar"]) + self.assertGreater(pearson["similar"], pearson["different"]) + + def test_nnaar_flags_exact_copies(self): + # NNAAR should be high when synthetic data memorizes the training set + # and near zero otherwise. With proper self-exclusion in the + # within-set nearest-neighbor search, both exact copies and near-copies + # (15% perturbed) leak training membership -> high NNAAR, while + # independent synthetic data (disjoint vocabulary) does not -> ~0. + nnaar = {} + for name, syn in [ + ("exact", self.syn_exact), + ("similar", self.syn_similar), + ("different", self.syn_different), + ]: + np.random.seed(0) + nnaar[name] = calc_nnaar( + self.train_ehr, self.test_ehr, syn, + **self.cols, sample_size=1000, n_runs=3, + )["nnaar"][0] + + # Memorized / near-memorized synthetic data leaks -> high NNAAR. + self.assertGreater(nnaar["exact"], 0.3) + self.assertGreater(nnaar["similar"], 0.3) + # An exact copy is at least as leaky as a perturbed near-copy. + self.assertGreaterEqual(nnaar["exact"], nnaar["similar"]) + # Independent synthetic data does not leak -> NNAAR ~ 0. + self.assertLess(abs(nnaar["different"]), 0.15) + self.assertGreater(nnaar["exact"], nnaar["different"]) + self.assertGreater(nnaar["similar"], nnaar["different"]) + + def test_membership_inference_detects_training_data(self): + # The attack should succeed when synthetic data is derived from the + # training set and be near chance when it is unrelated. + acc = {} + for name, syn in [ + ("exact", self.syn_exact), + ("similar", self.syn_similar), + ("different", self.syn_different), + ]: + np.random.seed(0) + acc[name] = calc_membership_inference( + self.train_ehr, self.test_ehr, syn, + **self.cols, num_attack_samples=1000, n_runs=5, + )["MIA_Accuracy"][0] + + self.assertGreater(acc["exact"], 0.8) + self.assertGreater(acc["exact"], acc["different"]) + self.assertGreater(acc["similar"], acc["different"]) + self.assertLess(acc["different"], 0.7) + + def test_discriminator_privacy_orders_by_similarity(self): + # A discriminator easily separates a disjoint-vocabulary synthetic set + # (accuracy ~1, privacy score ~0) but not data derived from the real + # data (lower accuracy, higher privacy score). + score, acc = {}, {} + for name, syn in [ + ("exact", self.syn_exact), + ("similar", self.syn_similar), + ("different", self.syn_different), + ]: + np.random.seed(0) + result = compute_discriminator_privacy( + train_fn=train_sklearn_model, + train_ehr=self.train_ehr, test_ehr=self.test_ehr, + syn_ehr=syn, **self.cols, n_bootstraps=10, model="rf", + ) + score[name] = result["Privacy_Score"][0] + acc[name] = result["Privacy_Discriminator_Accuracy"][0] + + # The disjoint-vocabulary set is trivially detected. + self.assertGreater(acc["different"], 0.8) + self.assertLess(score["different"], 0.1) + # Data derived from the real data is harder to flag. + self.assertGreater(acc["different"], acc["exact"]) + self.assertGreater(acc["different"], acc["similar"]) + self.assertGreater(score["exact"], score["different"]) + self.assertGreater(score["similar"], score["different"]) + + def test_mle_orders_by_similarity(self): + # Utility should be highest for an exact copy and degrade as the + # synthetic data drifts away from the real data. + mle = {} + for name, syn in [ + ("exact", self.syn_exact), + ("similar", self.syn_similar), + ("different", self.syn_different), + ]: + np.random.seed(0) + mle[name] = compute_mle( + train_fn=train_sklearn_model, + train_ehr=self.train_ehr, test_ehr=self.test_ehr, + syn_ehr=syn, **self.cols, n_bootstraps=10, model="rf", + ) + + # An exact copy reproduces real utility exactly. + exact = mle["exact"] + self.assertAlmostEqual(exact["MLE_Difference"][0], 0.0, places=9) + self.assertAlmostEqual(exact["MLE_Difference"][1], 0.0, places=9) + self.assertAlmostEqual(exact["MLE_Ratio"][0], 1.0, places=9) + self.assertAlmostEqual( + exact["MLE_Synth_Accuracy"][0], exact["MLE_Real_Accuracy"][0], + places=9, + ) + + # Synthetic-trained accuracy degrades monotonically. + diff = {k: abs(v["MLE_Difference"][0]) for k, v in mle.items()} + ratio = {k: v["MLE_Ratio"][0] for k, v in mle.items()} + self.assertLessEqual(diff["exact"], diff["similar"]) + self.assertLess(diff["similar"], diff["different"]) + self.assertGreaterEqual(ratio["exact"], ratio["similar"]) + self.assertGreater(ratio["similar"], ratio["different"]) + self.assertLess(ratio["different"], 1.0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/core/test_gpt2.py b/tests/core/test_gpt2.py new file mode 100644 index 000000000..4e9519214 --- /dev/null +++ b/tests/core/test_gpt2.py @@ -0,0 +1,151 @@ +import tempfile +import unittest + +import torch + +from pyhealth.datasets import create_sample_dataset, get_dataloader +from pyhealth.models import GPT2 + + +class TestGPT2(unittest.TestCase): + """Test cases for the GPT-2 baseline synthetic-EHR generator.""" + + def setUp(self): + """Set up a synthetic generative dataset (no labels) and a tiny model.""" + self.samples = [ + {"patient_id": "patient-0", "visits": [["A05B", "A05C"], ["A11D"], ["C129"]]}, + {"patient_id": "patient-1", "visits": [["A05B"], ["A04A", "B035"]]}, + {"patient_id": "patient-2", "visits": [["C129", "A11D"], ["A05C"], ["A04A"]]}, + {"patient_id": "patient-3", "visits": [["B035"], ["A05B", "C129"]]}, + ] + + # Generative task: one nested-sequence input feature, no output labels. + self.input_schema = {"visits": "nested_sequence"} + self.output_schema = {} + + self.dataset = create_sample_dataset( + samples=self.samples, + input_schema=self.input_schema, + output_schema=self.output_schema, + dataset_name="test_gpt2", + ) + + # Small model; embed_dim must be divisible by n_heads. + self.model = GPT2( + dataset=self.dataset, + embed_dim=16, + n_heads=2, + n_layers=2, + max_len=64, + batch_size=2, + epochs=1, + ) + + def test_model_initialization(self): + """Vocab/special-token ids are derived from the processor.""" + self.assertIsInstance(self.model, GPT2) + self.assertEqual(self.model.feature_keys, ["visits"]) + self.assertEqual(self.model.label_keys, []) + + proc_vocab = self.dataset.input_processors["visits"].vocab_size() + self.assertEqual(self.model.code_vocab_size, proc_vocab) + self.assertEqual(self.model.bos_id, proc_vocab) + self.assertEqual(self.model.eos_id, proc_vocab + 1) + self.assertEqual(self.model.delim_id, proc_vocab + 2) + self.assertEqual(self.model.gpt2.config.vocab_size, proc_vocab + 3) + + def test_forward_input_format(self): + """The standard dataloader pads the visit dimension into a 3D tensor.""" + loader = get_dataloader(self.dataset, batch_size=2, shuffle=False) + batch = next(iter(loader)) + self.assertIsInstance(batch["visits"], torch.Tensor) + self.assertEqual(batch["visits"].dim(), 3) # (B, max_visits, max_codes) + + def test_model_forward(self): + """Forward returns a finite scalar loss and a probability tensor.""" + loader = get_dataloader(self.dataset, batch_size=2, shuffle=False) + batch = next(iter(loader)) + + with torch.no_grad(): + ret = self.model(**batch) + + self.assertIn("loss", ret) + self.assertIn("y_prob", ret) + self.assertEqual(ret["loss"].dim(), 0) + self.assertTrue(torch.isfinite(ret["loss"]).all()) + # y_prob: (B, L, vocab_size) + self.assertEqual(ret["y_prob"].shape[0], 2) + self.assertEqual(ret["y_prob"].shape[2], self.model.gpt2.config.vocab_size) + + def test_model_backward(self): + """Backward populates gradients on model parameters.""" + loader = get_dataloader(self.dataset, batch_size=2, shuffle=False) + batch = next(iter(loader)) + + ret = self.model(**batch) + ret["loss"].backward() + + has_gradient = any( + param.requires_grad and param.grad is not None + for param in self.model.parameters() + ) + self.assertTrue(has_gradient, "No parameters have gradients after backward") + + def test_generate(self): + """generate() returns the requested number of decoded synthetic patients.""" + synthetic = self.model.generate(num_samples=4) + + self.assertEqual(len(synthetic), 4) + for i, patient in enumerate(synthetic): + self.assertEqual(patient["patient_id"], f"synthetic_{i}") + self.assertIsInstance(patient["visits"], list) + for visit in patient["visits"]: + self.assertIsInstance(visit, list) + for code in visit: + self.assertIsInstance(code, str) + self.assertNotIn(code, ("", "")) + + def _build_model(self, save_dir): + return GPT2( + dataset=self.dataset, + embed_dim=16, + n_heads=2, + n_layers=2, + max_len=64, + batch_size=2, + epochs=1, + save_dir=save_dir, + ) + + def test_train_and_generate_accept_device_arg(self): + """train_model/generate accept an explicit device arg (CPU always works).""" + with tempfile.TemporaryDirectory() as tmp: + model = self._build_model(tmp) + model.train_model(self.dataset, device="cpu") + self.assertEqual(next(model.parameters()).device.type, "cpu") + + synthetic = model.generate(num_samples=2, device="cpu") + self.assertEqual(len(synthetic), 2) + + @unittest.skipUnless(torch.cuda.is_available(), "CUDA not available") + def test_train_and_generate_on_cuda(self): + """When CUDA is available, the device arg moves training/generation to GPU.""" + with tempfile.TemporaryDirectory() as tmp: + model = self._build_model(tmp) + model.train_model(self.dataset, device="cuda") + self.assertTrue(next(model.parameters()).is_cuda) + + # forward should now run on CUDA without an explicit move. + loader = get_dataloader(self.dataset, batch_size=2, shuffle=False) + batch = next(iter(loader)) + with torch.no_grad(): + ret = model(**batch) + self.assertTrue(ret["y_prob"].is_cuda) + self.assertTrue(torch.isfinite(ret["loss"]).all()) + + synthetic = model.generate(num_samples=2, device="cuda") + self.assertEqual(len(synthetic), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/core/test_halo.py b/tests/core/test_halo.py new file mode 100644 index 000000000..be9788d62 --- /dev/null +++ b/tests/core/test_halo.py @@ -0,0 +1,152 @@ +import tempfile +import unittest + +import torch + +from pyhealth.datasets import create_sample_dataset, get_dataloader +from pyhealth.models import HALO + + +class TestHALO(unittest.TestCase): + """Test cases for the HALO synthetic-EHR generator.""" + + def setUp(self): + """Set up a synthetic generative dataset (no labels) and a tiny model.""" + self.samples = [ + {"patient_id": "patient-0", "visits": [["A05B", "A05C"], ["A11D"], ["C129"]]}, + {"patient_id": "patient-1", "visits": [["A05B"], ["A04A", "B035"]]}, + {"patient_id": "patient-2", "visits": [["C129", "A11D"], ["A05C"], ["A04A"]]}, + {"patient_id": "patient-3", "visits": [["B035"], ["A05B", "C129"]]}, + ] + + # Generative task: one nested-sequence input feature, no output labels. + self.input_schema = {"visits": "nested_sequence"} + self.output_schema = {} + + self.dataset = create_sample_dataset( + samples=self.samples, + input_schema=self.input_schema, + output_schema=self.output_schema, + dataset_name="test_halo", + ) + + # Small model; embed_dim must be divisible by n_heads. + self.model = HALO( + dataset=self.dataset, + embed_dim=16, + n_heads=2, + n_layers=2, + n_ctx=8, + batch_size=2, + epochs=1, + ) + + def test_model_initialization(self): + """Vocab sizes are derived from the processor; generation is unconditional.""" + self.assertIsInstance(self.model, HALO) + self.assertEqual(self.model.feature_keys, ["visits"]) + self.assertEqual(self.model.label_keys, []) + + proc_vocab = self.dataset.input_processors["visits"].vocab_size() + self.assertEqual(self.model.config.code_vocab_size, proc_vocab) + self.assertEqual(self.model.config.label_vocab_size, 0) + self.assertEqual(self.model.config.total_vocab_size, proc_vocab + 3) + + def test_forward_input_format(self): + """The standard dataloader pads the visit dimension into a 3D tensor.""" + loader = get_dataloader(self.dataset, batch_size=2, shuffle=False) + batch = next(iter(loader)) + self.assertIsInstance(batch["visits"], torch.Tensor) + self.assertEqual(batch["visits"].dim(), 3) # (B, max_visits, max_codes) + + def test_model_forward(self): + """Forward returns a finite scalar loss and a probability tensor.""" + loader = get_dataloader(self.dataset, batch_size=2, shuffle=False) + batch = next(iter(loader)) + + with torch.no_grad(): + ret = self.model(**batch) + + self.assertIn("loss", ret) + self.assertIn("y_prob", ret) + self.assertEqual(ret["loss"].dim(), 0) + self.assertTrue(torch.isfinite(ret["loss"]).all()) + # y_prob: (B, n_ctx - 1, total_vocab_size) + self.assertEqual(ret["y_prob"].shape[0], 2) + self.assertEqual(ret["y_prob"].shape[1], self.model.config.n_ctx - 1) + self.assertEqual( + ret["y_prob"].shape[2], self.model.config.total_vocab_size + ) + + def test_model_backward(self): + """Backward populates gradients on model parameters.""" + loader = get_dataloader(self.dataset, batch_size=2, shuffle=False) + batch = next(iter(loader)) + + ret = self.model(**batch) + ret["loss"].backward() + + has_gradient = any( + param.requires_grad and param.grad is not None + for param in self.model.parameters() + ) + self.assertTrue(has_gradient, "No parameters have gradients after backward") + + def test_generate(self): + """generate() returns the requested number of decoded synthetic patients.""" + synthetic = self.model.generate(num_samples=4, random_sampling=True) + + self.assertEqual(len(synthetic), 4) + for i, patient in enumerate(synthetic): + self.assertEqual(patient["patient_id"], f"synthetic_{i}") + self.assertIsInstance(patient["visits"], list) + for visit in patient["visits"]: + self.assertIsInstance(visit, list) + for code in visit: + self.assertIsInstance(code, str) + self.assertNotIn(code, ("", "")) + + def _build_model(self, save_dir): + return HALO( + dataset=self.dataset, + embed_dim=16, + n_heads=2, + n_layers=2, + n_ctx=8, + batch_size=2, + epochs=1, + save_dir=save_dir, + ) + + def test_train_and_generate_accept_device_arg(self): + """train_model/generate accept an explicit device arg (CPU always works).""" + with tempfile.TemporaryDirectory() as tmp: + model = self._build_model(tmp) + model.train_model(self.dataset, device="cpu") + self.assertEqual(next(model.parameters()).device.type, "cpu") + + synthetic = model.generate(num_samples=2, device="cpu") + self.assertEqual(len(synthetic), 2) + + @unittest.skipUnless(torch.cuda.is_available(), "CUDA not available") + def test_train_and_generate_on_cuda(self): + """When CUDA is available, the device arg moves training/generation to GPU.""" + with tempfile.TemporaryDirectory() as tmp: + model = self._build_model(tmp) + model.train_model(self.dataset, device="cuda") + self.assertTrue(next(model.parameters()).is_cuda) + + # forward should now run on CUDA without an explicit move. + loader = get_dataloader(self.dataset, batch_size=2, shuffle=False) + batch = next(iter(loader)) + with torch.no_grad(): + ret = model(**batch) + self.assertTrue(ret["y_prob"].is_cuda) + self.assertTrue(torch.isfinite(ret["loss"]).all()) + + synthetic = model.generate(num_samples=2, device="cuda") + self.assertEqual(len(synthetic), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/core/test_medgan.py b/tests/core/test_medgan.py new file mode 100644 index 000000000..9ed01c5f1 --- /dev/null +++ b/tests/core/test_medgan.py @@ -0,0 +1,197 @@ +import tempfile +import unittest + +import torch + +from pyhealth.datasets import create_sample_dataset +from pyhealth.models import MedGAN + + +class TestMedGAN(unittest.TestCase): + """Test cases for the MedGAN synthetic-EHR generator.""" + + def setUp(self): + """Bag-of-codes generative dataset (no labels) and a tiny model.""" + self.samples = [ + {"patient_id": "patient-0", "visits": ["A05B", "A05C", "A11D", "C129"]}, + {"patient_id": "patient-1", "visits": ["A05B", "A04A", "B035"]}, + {"patient_id": "patient-2", "visits": ["C129", "A11D", "A05C", "A04A"]}, + {"patient_id": "patient-3", "visits": ["B035", "A05B", "C129"]}, + ] + self.input_schema = {"visits": "multi_hot"} + self.output_schema = {} + + self.dataset = create_sample_dataset( + samples=self.samples, + input_schema=self.input_schema, + output_schema=self.output_schema, + dataset_name="test_medgan", + ) + + self.model = MedGAN( + dataset=self.dataset, + latent_dim=8, + hidden_dim=8, + discriminator_hidden_dim=16, + batch_size=2, + ae_epochs=1, + gan_epochs=1, + ) + + def test_model_initialization(self): + """Vocab size is derived from MultiHotProcessor; generation is unconditional.""" + self.assertIsInstance(self.model, MedGAN) + self.assertEqual(self.model.feature_keys, ["visits"]) + self.assertEqual(self.model.label_keys, []) + + proc_vocab = self.dataset.input_processors["visits"].size() + self.assertEqual(self.model.input_dim, proc_vocab) + + def test_latent_dim_aligned_to_hidden_dim(self): + """The generator's residual requires latent_dim == hidden_dim; a + mismatched latent_dim is silently aligned instead of crashing.""" + with tempfile.TemporaryDirectory() as tmp: + model = MedGAN( + dataset=self.dataset, + latent_dim=4, + hidden_dim=8, + discriminator_hidden_dim=16, + batch_size=2, + ae_epochs=1, + gan_epochs=1, + save_dir=tmp, + ) + self.assertEqual(model.latent_dim, 8) + self.assertEqual(model.hidden_dim, 8) + # The aligned model trains and generates without a shape mismatch. + model.train_model(self.dataset, device="cpu") + out = model.generate(num_samples=2, device="cpu") + self.assertEqual(len(out), 2) + + def test_components_present(self): + """Autoencoder, generator, and discriminator are all registered submodules.""" + self.assertTrue(hasattr(self.model, "autoencoder")) + self.assertTrue(hasattr(self.model, "generator")) + self.assertTrue(hasattr(self.model, "discriminator")) + + # Discriminator output is a probability (sigmoid). + x = torch.zeros(2, self.model.input_dim) + with torch.no_grad(): + d_out = self.model.discriminator(x) + self.assertEqual(d_out.shape, (2, 1)) + self.assertTrue(((d_out >= 0) & (d_out <= 1)).all()) + + def test_forward_raises(self): + """MedGAN's BaseModel forward intentionally errors out.""" + with self.assertRaises(NotImplementedError): + self.model.forward() + + def test_train_model_runs(self): + """train_model completes a tiny two-phase loop on CPU.""" + with tempfile.TemporaryDirectory() as tmp: + model = MedGAN( + dataset=self.dataset, + latent_dim=8, + hidden_dim=8, + discriminator_hidden_dim=16, + batch_size=2, + ae_epochs=1, + gan_epochs=1, + save_dir=tmp, + ) + model.train_model(self.dataset, device="cpu") + self.assertEqual(next(model.parameters()).device.type, "cpu") + + def test_generate(self): + """generate() returns the requested number of decoded synthetic patients. + + MedGAN is a bag-of-codes model, so each patient gets a single visit + containing the aggregate set of codes; the outer ``visits`` list + wraps that single visit to match HALO's nested format. + """ + synthetic = self.model.generate(num_samples=4, device="cpu") + self.assertEqual(len(synthetic), 4) + for i, patient in enumerate(synthetic): + self.assertEqual(patient["patient_id"], f"synthetic_{i}") + self.assertIsInstance(patient["visits"], list) + # Exactly one aggregate visit per patient. + self.assertEqual(len(patient["visits"]), 1) + visit = patient["visits"][0] + self.assertIsInstance(visit, list) + for code in visit: + self.assertIsInstance(code, str) + self.assertNotIn(code, ("", "")) + + def test_generate_random_sampling(self): + """random_sampling=True still produces well-formed patients.""" + synthetic = self.model.generate( + num_samples=3, random_sampling=True, device="cpu" + ) + self.assertEqual(len(synthetic), 3) + for patient in synthetic: + self.assertIn("patient_id", patient) + self.assertIn("visits", patient) + + def test_save_and_load_roundtrip(self): + """save_model + load_model preserves weights and vocabulary.""" + with tempfile.TemporaryDirectory() as tmp: + path = f"{tmp}/medgan.pt" + self.model.save_model(path) + + # Build a fresh model and overwrite from disk; weights should match. + other = MedGAN( + dataset=self.dataset, + latent_dim=8, + hidden_dim=8, + discriminator_hidden_dim=16, + batch_size=2, + ae_epochs=1, + gan_epochs=1, + save_dir=tmp, + ) + other.load_model(path) + + for p1, p2 in zip( + self.model.generator.parameters(), + other.generator.parameters(), + ): + self.assertTrue(torch.allclose(p1, p2)) + self.assertEqual(other._idx_to_code, self.model._idx_to_code) + + def test_missing_visits_processor_raises(self): + """A dataset without a 'visits' feature should be rejected.""" + # Build a dataset with a different input feature name. + bad = create_sample_dataset( + samples=[ + {"patient_id": "p1", "codes": ["A", "B"]}, + {"patient_id": "p2", "codes": ["B"]}, + ], + input_schema={"codes": "multi_hot"}, + output_schema={}, + ) + with self.assertRaises(ValueError): + MedGAN(bad, latent_dim=8, hidden_dim=8) + + @unittest.skipUnless(torch.cuda.is_available(), "CUDA not available") + def test_train_and_generate_on_cuda(self): + """When CUDA is available, the device arg moves training to GPU.""" + with tempfile.TemporaryDirectory() as tmp: + model = MedGAN( + dataset=self.dataset, + latent_dim=8, + hidden_dim=8, + discriminator_hidden_dim=16, + batch_size=2, + ae_epochs=1, + gan_epochs=1, + save_dir=tmp, + ) + model.train_model(self.dataset, device="cuda") + self.assertTrue(next(model.parameters()).is_cuda) + + synthetic = model.generate(num_samples=2, device="cuda") + self.assertEqual(len(synthetic), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/core/test_promptehr.py b/tests/core/test_promptehr.py new file mode 100644 index 000000000..4323e3313 --- /dev/null +++ b/tests/core/test_promptehr.py @@ -0,0 +1,161 @@ +import tempfile +import unittest + +import torch + +from pyhealth.datasets import create_sample_dataset, get_dataloader +from pyhealth.models import PromptEHR + + +class TestPromptEHR(unittest.TestCase): + """Test cases for the PromptEHR synthetic-EHR generator.""" + + def setUp(self): + """Set up a synthetic generative dataset (no labels) and a tiny model.""" + self.samples = [ + {"patient_id": "patient-0", "visits": [["A05B", "A05C"], ["A11D"], ["C129"]]}, + {"patient_id": "patient-1", "visits": [["A05B"], ["A04A", "B035"]]}, + {"patient_id": "patient-2", "visits": [["C129", "A11D"], ["A05C"], ["A04A"]]}, + {"patient_id": "patient-3", "visits": [["B035"], ["A05B", "C129"]]}, + ] + + # Generative task: one nested-sequence input feature, no output labels. + self.input_schema = {"visits": "nested_sequence"} + self.output_schema = {} + + self.dataset = create_sample_dataset( + samples=self.samples, + input_schema=self.input_schema, + output_schema=self.output_schema, + dataset_name="test_promptehr", + ) + + # Small model; embed_dim must be divisible by n_heads. + self.model = PromptEHR( + dataset=self.dataset, + embed_dim=16, + n_heads=2, + n_layers=2, + prompt_length=4, + max_len=64, + batch_size=2, + epochs=1, + ) + + def test_model_initialization(self): + """Vocab/special-token ids are derived from the processor.""" + self.assertIsInstance(self.model, PromptEHR) + self.assertEqual(self.model.feature_keys, ["visits"]) + self.assertEqual(self.model.label_keys, []) + + proc_vocab = self.dataset.input_processors["visits"].vocab_size() + self.assertEqual(self.model.code_vocab_size, proc_vocab) + self.assertEqual(self.model.bos_id, proc_vocab) + self.assertEqual(self.model.eos_id, proc_vocab + 1) + self.assertEqual(self.model.delim_id, proc_vocab + 2) + self.assertEqual(self.model.mask_id, proc_vocab + 3) + self.assertEqual(self.model.code_prompt_id, proc_vocab + 4) + self.assertEqual(self.model.bart.config.vocab_size, proc_vocab + 5) + # The learnable soft prompt is part of the module parameters. + self.assertEqual( + tuple(self.model.soft_prompt.shape), (4, 16) + ) + + def test_forward_input_format(self): + """The standard dataloader pads the visit dimension into a 3D tensor.""" + loader = get_dataloader(self.dataset, batch_size=2, shuffle=False) + batch = next(iter(loader)) + self.assertIsInstance(batch["visits"], torch.Tensor) + self.assertEqual(batch["visits"].dim(), 3) # (B, max_visits, max_codes) + + def test_model_forward(self): + """Forward returns a finite scalar loss and a probability tensor.""" + loader = get_dataloader(self.dataset, batch_size=2, shuffle=False) + batch = next(iter(loader)) + + with torch.no_grad(): + ret = self.model(**batch) + + self.assertIn("loss", ret) + self.assertIn("y_prob", ret) + self.assertEqual(ret["loss"].dim(), 0) + self.assertTrue(torch.isfinite(ret["loss"]).all()) + # y_prob: (B, L_dec, vocab_size) + self.assertEqual(ret["y_prob"].shape[0], 2) + self.assertEqual(ret["y_prob"].shape[2], self.model.bart.config.vocab_size) + + def test_model_backward(self): + """Backward populates gradients on model parameters (incl. soft prompt).""" + loader = get_dataloader(self.dataset, batch_size=2, shuffle=False) + batch = next(iter(loader)) + + ret = self.model(**batch) + ret["loss"].backward() + + has_gradient = any( + param.requires_grad and param.grad is not None + for param in self.model.parameters() + ) + self.assertTrue(has_gradient, "No parameters have gradients after backward") + # The soft prompt specifically should receive a gradient. + self.assertIsNotNone(self.model.soft_prompt.grad) + + def test_generate(self): + """generate() returns the requested number of decoded synthetic patients.""" + synthetic = self.model.generate(num_samples=4) + + self.assertEqual(len(synthetic), 4) + for i, patient in enumerate(synthetic): + self.assertEqual(patient["patient_id"], f"synthetic_{i}") + self.assertIsInstance(patient["visits"], list) + for visit in patient["visits"]: + self.assertIsInstance(visit, list) + for code in visit: + self.assertIsInstance(code, str) + self.assertNotIn(code, ("", "")) + + def _build_model(self, save_dir): + return PromptEHR( + dataset=self.dataset, + embed_dim=16, + n_heads=2, + n_layers=2, + prompt_length=4, + max_len=64, + batch_size=2, + epochs=1, + save_dir=save_dir, + ) + + def test_train_and_generate_accept_device_arg(self): + """train_model/generate accept an explicit device arg (CPU always works).""" + with tempfile.TemporaryDirectory() as tmp: + model = self._build_model(tmp) + model.train_model(self.dataset, device="cpu") + self.assertEqual(next(model.parameters()).device.type, "cpu") + + synthetic = model.generate(num_samples=2, device="cpu") + self.assertEqual(len(synthetic), 2) + + @unittest.skipUnless(torch.cuda.is_available(), "CUDA not available") + def test_train_and_generate_on_cuda(self): + """When CUDA is available, the device arg moves training/generation to GPU.""" + with tempfile.TemporaryDirectory() as tmp: + model = self._build_model(tmp) + model.train_model(self.dataset, device="cuda") + self.assertTrue(next(model.parameters()).is_cuda) + + # forward should now run on CUDA without an explicit move. + loader = get_dataloader(self.dataset, batch_size=2, shuffle=False) + batch = next(iter(loader)) + with torch.no_grad(): + ret = model(**batch) + self.assertTrue(ret["y_prob"].is_cuda) + self.assertTrue(torch.isfinite(ret["loss"]).all()) + + synthetic = model.generate(num_samples=2, device="cuda") + self.assertEqual(len(synthetic), 2) + + +if __name__ == "__main__": + unittest.main() From fb481d9dd455c62a3f548eec5ad8b5d0df210627 Mon Sep 17 00:00:00 2001 From: John Wu <54558896+jhnwu3@users.noreply.github.com> Date: Sun, 7 Jun 2026 14:53:05 -0700 Subject: [PATCH 13/61] transfer FHIR pipeline to branch (#1155) * transfer FHIR pipeline to branch * fix * fix unit test using fast json readers * Replace editdistance with rapidfuzz for Python 3.13 compatibility editdistance 0.8.1 only ships cp311 wheels and has no Python 3.13 binary, causing CI installs to fail on Linux. rapidfuzz>=3.0.0 ships wheels for all major platforms including cp313 and provides an equivalent Levenshtein.distance() API. https://claude.ai/code/session_01L5qHpvAZQSgmZyc6tMTX6d * copilot fixes * revert ignore error change --------- Co-authored-by: Claude --- docs/api/datasets.rst | 2 + .../pyhealth.datasets.FHIRDataset.rst | 306 +++++++ .../datasets/pyhealth.datasets.MIMIC4FHIR.rst | 78 ++ docs/api/models.rst | 1 + .../models/pyhealth.models.EHRMambaCEHR.rst | 12 + docs/api/tasks.rst | 1 + ...pyhealth.tasks.mpf_clinical_prediction.rst | 12 + examples/mimic4fhir_mpf_ehrmamba.py | 61 ++ pyhealth/datasets/__init__.py | 1 + pyhealth/datasets/fhir/__init__.py | 16 + pyhealth/datasets/fhir/base.py | 415 +++++++++ .../datasets/fhir/configs/mimic4fhir.yaml | 210 +++++ pyhealth/datasets/fhir/mimic4.py | 42 + pyhealth/datasets/fhir/utils.py | 724 ++++++++++++++++ pyhealth/models/__init__.py | 1 + pyhealth/models/cehr_embeddings.py | 112 +++ pyhealth/models/ehrmamba_cehr.py | 117 +++ pyhealth/models/utils.py | 28 + pyhealth/nlp/metrics.py | 6 +- pyhealth/processors/__init__.py | 3 + pyhealth/processors/cehr_processor.py | 175 ++++ pyhealth/tasks/__init__.py | 8 + pyhealth/tasks/mpf_clinical_prediction.py | 315 +++++++ pyproject.toml | 3 +- tests/core/test_ehrmamba_cehr.py | 126 +++ tests/core/test_fhir_dataset.py | 817 ++++++++++++++++++ tests/core/test_fhir_ndjson_fixtures.py | 110 +++ tests/core/test_mpf_task.py | 99 +++ 28 files changed, 3797 insertions(+), 4 deletions(-) create mode 100644 docs/api/datasets/pyhealth.datasets.FHIRDataset.rst create mode 100644 docs/api/datasets/pyhealth.datasets.MIMIC4FHIR.rst create mode 100644 docs/api/models/pyhealth.models.EHRMambaCEHR.rst create mode 100644 docs/api/tasks/pyhealth.tasks.mpf_clinical_prediction.rst create mode 100644 examples/mimic4fhir_mpf_ehrmamba.py create mode 100644 pyhealth/datasets/fhir/__init__.py create mode 100644 pyhealth/datasets/fhir/base.py create mode 100644 pyhealth/datasets/fhir/configs/mimic4fhir.yaml create mode 100644 pyhealth/datasets/fhir/mimic4.py create mode 100644 pyhealth/datasets/fhir/utils.py create mode 100644 pyhealth/models/cehr_embeddings.py create mode 100644 pyhealth/models/ehrmamba_cehr.py create mode 100644 pyhealth/processors/cehr_processor.py create mode 100644 pyhealth/tasks/mpf_clinical_prediction.py create mode 100644 tests/core/test_ehrmamba_cehr.py create mode 100644 tests/core/test_fhir_dataset.py create mode 100644 tests/core/test_fhir_ndjson_fixtures.py create mode 100644 tests/core/test_mpf_task.py diff --git a/docs/api/datasets.rst b/docs/api/datasets.rst index 1875698ae..a23efb3d2 100644 --- a/docs/api/datasets.rst +++ b/docs/api/datasets.rst @@ -224,6 +224,8 @@ Available Datasets datasets/pyhealth.datasets.SampleDataset datasets/pyhealth.datasets.MIMIC3Dataset datasets/pyhealth.datasets.MIMIC4Dataset + datasets/pyhealth.datasets.FHIRDataset + datasets/pyhealth.datasets.MIMIC4FHIR datasets/pyhealth.datasets.MedicalTranscriptionsDataset datasets/pyhealth.datasets.CardiologyDataset datasets/pyhealth.datasets.eICUDataset diff --git a/docs/api/datasets/pyhealth.datasets.FHIRDataset.rst b/docs/api/datasets/pyhealth.datasets.FHIRDataset.rst new file mode 100644 index 000000000..16dbefc13 --- /dev/null +++ b/docs/api/datasets/pyhealth.datasets.FHIRDataset.rst @@ -0,0 +1,306 @@ +pyhealth.datasets.FHIRDataset +===================================== + +A generic, config-driven NDJSON ingest for `HL7 FHIR +`_ datasets. The whole pipeline is described by **a +single YAML config** with three top-level sections — what files to read, how to +turn each FHIR resource into a flat row, and how those rows appear as events +downstream. A custom FHIR ingest is "point at a YAML" — no Python required. + +The bundled :class:`~pyhealth.datasets.MIMIC4FHIR` subclass uses this engine +with the ``pyhealth/datasets/fhir/configs/mimic4fhir.yaml`` config tuned for +PhysioNet's MIMIC-IV on FHIR export. See the sub-page below for the quick-start. + +.. contents:: On this page + :local: + :depth: 1 + + +Quick start +----------- + +.. code-block:: python + + from pyhealth.datasets import MIMIC4FHIR, get_dataloader, split_by_patient + from pyhealth.tasks.mpf_clinical_prediction import MPFClinicalPredictionTask + from pyhealth.models import EHRMambaCEHR + from pyhealth.trainer import Trainer + + def main(): + ds = MIMIC4FHIR(root="/data/mimic-iv-fhir") + sample_ds = ds.set_task(MPFClinicalPredictionTask(), num_workers=1) + train, val, test = split_by_patient(sample_ds, [0.7, 0.1, 0.2]) + vocab_size = sample_ds.input_processors["concept_ids"].vocab.vocab_size + model = EHRMambaCEHR(dataset=sample_ds, vocab_size=vocab_size) + Trainer(model=model).train( + train_dataloader=get_dataloader(train, batch_size=8, shuffle=True), + val_dataloader=get_dataloader(val, batch_size=8), + epochs=2, + ) + + if __name__ == "__main__": + main() + +(``if __name__ == "__main__":`` matters — :meth:`~pyhealth.datasets.BaseDataset.set_task` +forks Dask workers; without the guard the workers re-import and re-spawn.) + + +Pipeline at a glance +-------------------- + +:: + + NDJSON shards on disk + | + | (Phase A) — stream line by line, route by resourceType, + | project via the YAML's resource_specs + v + flattened_tables/.parquet <- cache #1 + | + | (Phase B) — load_table, dd.concat, sort by patient_id (Dask) + v + global_event_df.parquet/part-*.parquet <- cache #2 + | + | (Phase C) — task_transform per-patient sample emit + v + task_df.ld/ <- cache #3a + | + | fit CehrProcessor vocab via SampleBuilder.fit(dataset) + | proc_transform per-sample tensorisation + v + samples_*.ld/ <- cache #3b ──> SampleDataset + +Each of the three cache tiers has its own existence check; re-running with +identical inputs skips every phase. Cache identity hashes the YAML byte digest, +glob patterns, ``max_patients``, and engine schema version — any meaningful +config change invalidates everything below it. See +:class:`~pyhealth.datasets.BaseDataset` for the Phase B/C internals that are +shared with all other PyHealth datasets. + + +The unified YAML config +----------------------- + +A FHIR ingest YAML has three top-level sections. The bundled +``mimic4fhir.yaml`` is the canonical worked example; what follows is the +section-by-section reference. + +Section 1: ``glob_patterns:`` (which files to read) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: yaml + + glob_patterns: + - "**/MimicPatient*.ndjson.gz" + - "**/MimicEncounter*.ndjson.gz" + # ... one pattern per resource-type shard family + +Defaults to ``["**/*.ndjson.gz"]`` when omitted. Only worth setting when your +export has a per-resource-type file-naming convention you want to exploit for +speed — PhysioNet MIMIC-IV FHIR ships shards as ``MimicPatient*.ndjson.gz``, +``MimicEncounter*.ndjson.gz``, etc., and filtering at the file level avoids +decompressing ~10% of the export that contains only unconfigured resource +types. For a generic export where everything is in ``bundles.ndjson.gz``, omit +this block and the streamer will filter by ``resourceType`` after parsing. + +Override at runtime via ``MIMIC4FHIR(glob_pattern=...)`` or +``MIMIC4FHIR(glob_patterns=[...])``. + +Section 2: ``resource_specs:`` (how to project JSON into rows) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Keys are FHIR ``resourceType`` strings. For each, declare a ``table`` name and +an ordered ``columns`` mapping: + +.. code-block:: yaml + + resource_specs: + + Patient: + table: patient + columns: + patient_id: { locate: ["id"], required: true } + birth_date: { locate: ["birthDate"] } + gender: { locate: ["gender"] } + deceased_boolean: { locate: ["deceasedBoolean"], transform: bool_norm } + + Observation: + table: observation + columns: + patient_id: { locate: ["subject.reference"], transform: ref_id, required: true } + resource_id: { locate: ["id"] } + encounter_id: { locate: ["encounter.reference"], transform: ref_id } + event_time: { locate: ["effectiveDateTime", "effectivePeriod.start", "issued"] } + concept_key: { locate: ["code"], transform: coding_key } + +Each column entry has three fields: + +``locate`` *(required, list of dotted paths)* + Ordered JSON paths into the resource; the first that resolves to a non-null + value wins. This is how FHIR choice-types (``onset[x]``, ``effective[x]``, + ``performed[x]``, …) are handled — list every variant explicitly. A single + string is accepted as shorthand for a one-element list. + +``transform`` *(optional, name of a built-in transform, default ``identity``)* + Maps the located leaf to a flat scalar string. See the registry below. + +``required`` *(optional, bool, default false)* + When ``true``, a resource whose ``locate`` cannot be resolved is **dropped** + (and logged) rather than emitted with a null. Use this on the patient + reference column so events without a discoverable patient never reach the + global event frame. + +Transform registry +^^^^^^^^^^^^^^^^^^ + +Available transforms (defined in +``pyhealth/datasets/fhir/utils.py`` ``TRANSFORMS`` dict): + +================== =========================================================== +``identity`` Pass the value through. Stringifies non-string scalars. +``ref_id`` Reference object or ``"Patient/p1"`` -> ``"p1"``. +``coding_key`` CodeableConcept -> ``"system|code"`` of its first coding. +``bool_norm`` JSON boolean / ``"true"``/``"false"`` -> ``"true"``/``"false"``/None. +``med_concept`` MedicationRequest medication[x] -> codeable-concept or + ``"MedicationRequest/reference|"`` fallback. +================== =========================================================== + +Adding a new transform is a one-liner: register a callable in ``TRANSFORMS`` +in ``utils.py`` and reference it by name from the YAML. + +Section 3: ``tables:`` (how rows are exposed as events) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Keys here must match the ``table:`` values from Section 2. Each entry tells +:meth:`~pyhealth.datasets.BaseDataset.load_table` how to read the flat parquet: + +.. code-block:: yaml + + tables: + patient: + file_path: "patient.parquet" + patient_id: "patient_id" + timestamp: "birth_date" + attributes: ["birth_date", "gender", "deceased_boolean"] + + observation: + file_path: "observation.parquet" + patient_id: "patient_id" + timestamp: "event_time" + attributes: ["resource_id", "encounter_id", "event_time", "concept_key"] + +``file_path`` is the parquet filename inside the cached +``flattened_tables/`` directory. ``patient_id`` and ``timestamp`` name the +columns to surface as the normalised ``patient_id`` and ``timestamp`` on each +event. ``attributes`` is the list of columns surfaced as event attributes — in +the global event frame they're renamed to ``{table}/{attr}`` and later show up +on ``patient.get_events(event_type=...).attr_name``. + +Cross-section validation +~~~~~~~~~~~~~~~~~~~~~~~~ + +At load time the dataset checks that every ``table:`` value declared in +Section 2 has a matching ``tables.`` block in Section 3. Typos surface +as a config error at startup, not silent empty parquets. + + +Customising for a non-MIMIC FHIR export +--------------------------------------- + +Step 1 — write your YAML. +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Copy ``pyhealth/datasets/fhir/configs/mimic4fhir.yaml`` and adapt the +``resource_specs:`` and ``tables:`` blocks for the resources you care about. +For an export that adds Immunizations: + +.. code-block:: yaml + + resource_specs: + Patient: + table: patient + columns: + patient_id: { locate: ["id"], required: true } + birth_date: { locate: ["birthDate"] } + Immunization: + table: immunization + columns: + patient_id: { locate: ["patient.reference"], transform: ref_id, required: true } + resource_id: { locate: ["id"] } + event_time: { locate: ["occurrenceDateTime", "recorded"] } + concept_key: { locate: ["vaccineCode"], transform: coding_key } + + tables: + patient: + file_path: "patient.parquet" + patient_id: "patient_id" + timestamp: "birth_date" + attributes: ["birth_date"] + immunization: + file_path: "immunization.parquet" + patient_id: "patient_id" + timestamp: "event_time" + attributes: ["resource_id", "event_time", "concept_key"] + +Step 2 — instantiate +~~~~~~~~~~~~~~~~~~~~ + +Either pass ``config_path=...`` directly: + +.. code-block:: python + + from pyhealth.datasets import FHIRDataset + + ds = FHIRDataset( + root="/data/my_fhir_export", + config_path="/path/to/my_export.yaml", + ) + +or write a 3-line subclass that bundles your config: + +.. code-block:: python + + from pyhealth.datasets import FHIRDataset + + class MyFHIR(FHIRDataset): + DEFAULT_CONFIG_PATH = "/path/to/my_export.yaml" + + ds = MyFHIR(root="/data/my_fhir_export") + +Step 3 — that's it. +~~~~~~~~~~~~~~~~~~~ + +Everything downstream — :meth:`~pyhealth.datasets.BaseDataset.set_task`, +:meth:`~pyhealth.datasets.BaseDataset.iter_patients`, +:meth:`~pyhealth.datasets.BaseDataset.get_patient` — works the same as for any +other PyHealth dataset. + + +Notes on resource use +--------------------- + +Streaming ingest avoids loading the whole NDJSON corpus into RAM, but downstream +steps still scale with cohort size. For a **smoke run** the bundled example +fixtures fit on any laptop. For a **laptop-scale real subset**, set +``max_patients=`` and/or narrow ``glob_patterns`` to keep cache and task passes +manageable; ≥16 GB system RAM is a comfort target for Polars + the trainer. +For the **full PhysioNet export**, prefer fast SSD, large disk, and plenty of +RAM — total work scales with the corpus size even if RAM ingest is bounded. + + +Bundled FHIR datasets +--------------------- + +.. toctree:: + :maxdepth: 1 + + pyhealth.datasets.MIMIC4FHIR + + +API reference +------------- + +.. autoclass:: pyhealth.datasets.FHIRDataset + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/datasets/pyhealth.datasets.MIMIC4FHIR.rst b/docs/api/datasets/pyhealth.datasets.MIMIC4FHIR.rst new file mode 100644 index 000000000..344f60cf7 --- /dev/null +++ b/docs/api/datasets/pyhealth.datasets.MIMIC4FHIR.rst @@ -0,0 +1,78 @@ +pyhealth.datasets.MIMIC4FHIR +============================ + +A pre-bundled :class:`~pyhealth.datasets.FHIRDataset` for the PhysioNet +`MIMIC-IV on FHIR `_ export +(R4, demo 2.1.0 and full release). All ingest logic — file globs, per-resource +projection, downstream event schema — is described by the bundled YAML at +``pyhealth/datasets/fhir/configs/mimic4fhir.yaml``; this class only points at +that path. + +For everything outside the MIMIC-specific defaults (transform registry, +``Col`` / ``ResourceSpec`` syntax, the three-tier cache story), see the parent +page: :doc:`pyhealth.datasets.FHIRDataset`. + +Quick start +----------- + +.. code-block:: python + + from pyhealth.datasets import MIMIC4FHIR + from pyhealth.tasks.mpf_clinical_prediction import MPFClinicalPredictionTask + + def main(): + ds = MIMIC4FHIR(root="/data/mimic-iv-fhir") + sample_ds = ds.set_task(MPFClinicalPredictionTask(), num_workers=1) + # ... split / dataloader / model / trainer ... + + if __name__ == "__main__": + main() + +For the full end-to-end demo (training EHR-Mamba on MPF samples) see +``examples/mimic4fhir_mpf_ehrmamba.py``. + +Resource coverage +----------------- + +The bundled config flattens six FHIR resource types out of the PhysioNet +export: + +========================== ============================ =============================== +FHIR resourceType Output table Key columns +========================== ============================ =============================== +``Patient`` ``patient.parquet`` ``patient_id``, ``birth_date``, ``gender``, ``deceased_*`` +``Encounter`` ``encounter.parquet`` ``patient_id``, ``encounter_id``, ``event_time``, ``encounter_class`` +``Condition`` ``condition.parquet`` ``patient_id``, ``encounter_id``, ``event_time``, ``concept_key`` +``Observation`` ``observation.parquet`` ``patient_id``, ``encounter_id``, ``event_time``, ``concept_key`` +``MedicationRequest`` ``medication_request.parquet`` ``patient_id``, ``encounter_id``, ``event_time``, ``concept_key`` +``Procedure`` ``procedure.parquet`` ``patient_id``, ``encounter_id``, ``event_time``, ``concept_key`` +========================== ============================ =============================== + +PhysioNet shards that contain only other resource types +(``MedicationAdministration``, ``Specimen``, ``Organization``, …) are skipped +at the file level by the bundled ``glob_patterns``. To include them, override +``glob_patterns=`` at the constructor and add a ``resource_specs:`` entry plus +matching ``tables:`` entry in a copy of the YAML. + +Customising +----------- + +The bundled config is the easiest starting point for authoring a similar ingest +for other FHIR exports. Copy +``pyhealth/datasets/fhir/configs/mimic4fhir.yaml``, edit the +``resource_specs:`` and ``tables:`` blocks for the resources you care about, +and either: + +* pass ``config_path=...`` directly to ``FHIRDataset(root=..., config_path=...)``, or +* subclass ``FHIRDataset`` and set ``DEFAULT_CONFIG_PATH`` on the subclass. + +See the "Customising for a non-MIMIC FHIR export" section of +:doc:`pyhealth.datasets.FHIRDataset` for the step-by-step. + +API reference +------------- + +.. autoclass:: pyhealth.datasets.MIMIC4FHIR + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/models.rst b/docs/api/models.rst index e98f74f5c..d695b20db 100644 --- a/docs/api/models.rst +++ b/docs/api/models.rst @@ -186,6 +186,7 @@ API Reference models/pyhealth.models.MoleRec models/pyhealth.models.Deepr models/pyhealth.models.EHRMamba + models/pyhealth.models.EHRMambaCEHR models/pyhealth.models.JambaEHR models/pyhealth.models.ContraWR models/pyhealth.models.SparcNet diff --git a/docs/api/models/pyhealth.models.EHRMambaCEHR.rst b/docs/api/models/pyhealth.models.EHRMambaCEHR.rst new file mode 100644 index 000000000..c15a09962 --- /dev/null +++ b/docs/api/models/pyhealth.models.EHRMambaCEHR.rst @@ -0,0 +1,12 @@ +pyhealth.models.EHRMambaCEHR +=================================== + +EHRMambaCEHR applies CEHR-style embeddings (:class:`~pyhealth.models.cehr_embeddings.MambaEmbeddingsForCEHR`) +and a stack of :class:`~pyhealth.models.MambaBlock` layers to a single FHIR token stream, for use with +:class:`~pyhealth.tasks.mpf_clinical_prediction.MPFClinicalPredictionTask` and +:class:`~pyhealth.datasets.fhir.FHIRDataset`. + +.. autoclass:: pyhealth.models.EHRMambaCEHR + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/tasks.rst b/docs/api/tasks.rst index 0ff286bfc..8724176a8 100644 --- a/docs/api/tasks.rst +++ b/docs/api/tasks.rst @@ -215,6 +215,7 @@ Available Tasks EHR Generation Length of Stay Prediction Medical Transcriptions Classification + MPF Clinical Prediction (FHIR) Mortality Prediction (Next Visit) Mortality Prediction (StageNet MIMIC-IV) Patient Linkage (MIMIC-III) diff --git a/docs/api/tasks/pyhealth.tasks.mpf_clinical_prediction.rst b/docs/api/tasks/pyhealth.tasks.mpf_clinical_prediction.rst new file mode 100644 index 000000000..27331905f --- /dev/null +++ b/docs/api/tasks/pyhealth.tasks.mpf_clinical_prediction.rst @@ -0,0 +1,12 @@ +pyhealth.tasks.mpf_clinical_prediction +====================================== + +Multitask Prompted Fine-tuning (MPF) style binary clinical prediction on FHIR +token timelines, paired with :class:`~pyhealth.datasets.FHIRDataset` and +:class:`~pyhealth.models.EHRMambaCEHR`. Based on CEHR / EHRMamba ideas +(EHRMamba, arXiv:2405.14567): https://arxiv.org/abs/2405.14567. + +.. autoclass:: pyhealth.tasks.MPFClinicalPredictionTask + :members: + :undoc-members: + :show-inheritance: diff --git a/examples/mimic4fhir_mpf_ehrmamba.py b/examples/mimic4fhir_mpf_ehrmamba.py new file mode 100644 index 000000000..33df598af --- /dev/null +++ b/examples/mimic4fhir_mpf_ehrmamba.py @@ -0,0 +1,61 @@ +"""EHRMambaCEHR on the local MIMIC-IV FHIR demo. + +Barebones path: Dataset -> task -> model -> trainer -> evaluate. + +Runs against the bundled demo at +``datasets/physionet.org/mimic-iv-fhir-demo/2.1.0/fhir`` and persists the +flattened-table cache under ``datasets/.cache/pyhealth/fhir-demo`` so a +second run hits the cache. + + PYTHONPATH=. python examples/mimic4fhir_mpf_ehrmamba.py +""" + +from __future__ import annotations + +from pathlib import Path + +from pyhealth.datasets import MIMIC4FHIR, get_dataloader, split_by_patient +from pyhealth.models import EHRMambaCEHR +from pyhealth.tasks.mpf_clinical_prediction import MPFClinicalPredictionTask +from pyhealth.trainer import Trainer + +# Absolute paths to the bundled PhysioNet MIMIC-IV-on-FHIR demo and its cache. +DEMO_ROOT = Path( + "/home/johnwu3/projects/PyHealth_Branch_Testing/datasets/" + "physionet.org/mimic-iv-fhir-demo/2.1.0/fhir" +) +CACHE_DIR = Path( + "/home/johnwu3/projects/PyHealth_Branch_Testing/datasets/.cache/pyhealth/fhir-demo" +) + + +def main() -> None: + dataset = MIMIC4FHIR(root=str(DEMO_ROOT), cache_dir=str(CACHE_DIR)) + sample_dataset = dataset.set_task(MPFClinicalPredictionTask(), num_workers=1) + + train_ds, val_ds, test_ds = split_by_patient(sample_dataset, [0.7, 0.1, 0.2]) + train_loader = get_dataloader(train_ds, batch_size=8, shuffle=True) + val_loader = get_dataloader(val_ds, batch_size=8, shuffle=False) + test_loader = get_dataloader(test_ds, batch_size=8, shuffle=False) + + vocab_size = sample_dataset.input_processors["concept_ids"].vocab.vocab_size + model = EHRMambaCEHR( + dataset=sample_dataset, + vocab_size=vocab_size, + embedding_dim=32, + num_layers=2, + dropout=0.1, + ) + + trainer = Trainer(model=model, metrics=["roc_auc", "pr_auc"]) + trainer.train( + train_dataloader=train_loader, + val_dataloader=val_loader, + epochs=2, + monitor="roc_auc", + ) + print(trainer.evaluate(test_loader)) + + +if __name__ == "__main__": + main() diff --git a/pyhealth/datasets/__init__.py b/pyhealth/datasets/__init__.py index 50b1b3887..c29955e7d 100644 --- a/pyhealth/datasets/__init__.py +++ b/pyhealth/datasets/__init__.py @@ -59,6 +59,7 @@ def __init__(self, *args, **kwargs): from .medical_transcriptions import MedicalTranscriptionsDataset from .mimic3 import MIMIC3Dataset from .mimic4 import MIMIC4CXRDataset, MIMIC4Dataset, MIMIC4EHRDataset, MIMIC4NoteDataset +from .fhir import FHIRDataset, MIMIC4FHIR from .mimicextract import MIMICExtractDataset from .omop import OMOPDataset from .physionet_deid import PhysioNetDeIDDataset diff --git a/pyhealth/datasets/fhir/__init__.py b/pyhealth/datasets/fhir/__init__.py new file mode 100644 index 000000000..adcbbea85 --- /dev/null +++ b/pyhealth/datasets/fhir/__init__.py @@ -0,0 +1,16 @@ +"""FHIR datasets: a generic engine + per-source subclasses. + +- :class:`~pyhealth.datasets.fhir.base.FHIRDataset` — generic, config-driven base. +- :class:`~pyhealth.datasets.fhir.mimic4.MIMIC4FHIR` — MIMIC-IV-on-FHIR (R4). +- :mod:`~pyhealth.datasets.fhir.utils` — the stateless flattening engine + (``Col``, ``ResourceSpec``, ``flatten_resource``, …). + +Authors: + John Wu and Evan Febrianto +""" + +from .base import FHIRDataset +from .mimic4 import MIMIC4FHIR +from .utils import Col, ResourceSpec + +__all__ = ["FHIRDataset", "MIMIC4FHIR", "Col", "ResourceSpec"] diff --git a/pyhealth/datasets/fhir/base.py b/pyhealth/datasets/fhir/base.py new file mode 100644 index 000000000..517d062bb --- /dev/null +++ b/pyhealth/datasets/fhir/base.py @@ -0,0 +1,415 @@ +"""Generic FHIR ingestion using flattened resource tables. + +Architecture +------------ +1. Stream NDJSON/NDJSON.GZ FHIR resources from disk. +2. Normalize each resource type into a 2D table via a declarative + :class:`~pyhealth.datasets.fhir.utils.ResourceSpec` registry + (``self.resource_specs``) — see :mod:`~pyhealth.datasets.fhir.utils`. +3. Feed those tables through the standard YAML-driven + :class:`~pyhealth.datasets.BaseDataset` pipeline so downstream task + processing operates on :class:`~pyhealth.data.Patient` and + ``global_event_df`` rows. + +``FHIRDataset`` is generic: it owns the streaming/cache/validation machinery but +no specific resource specs or config. Use it directly by passing +``resource_specs=`` + ``config_path=``, or subclass it for a concrete source +(e.g. :class:`~pyhealth.datasets.fhir.mimic4.MIMIC4FHIR`) that bakes those in as +class attributes. + +Authors: + John Wu and Evan Febrianto +""" + +from __future__ import annotations + +import functools +import hashlib +import logging +import operator +import shutil +import uuid +from pathlib import Path +from typing import Any, Dict, List, Mapping, Optional, Sequence + +import dask.dataframe as dd +import narwhals as nw +import orjson +import pandas as pd +import platformdirs +from yaml import safe_load + +from ..base_dataset import BaseDataset +from .utils import ( + FHIR_SCHEMA_VERSION, + SUPPORTED_OUTPUT_FORMATS, + ResourceSpec, + filter_flat_tables_by_patient_ids, + load_resource_specs_from_yaml, + sorted_patient_ids_from_flat_tables, + stream_fhir_ndjson_to_flat_tables, + table_file_name, + tables_from_specs, +) + +logger = logging.getLogger(__name__) + + +def read_fhir_settings_yaml(path: str) -> Dict[str, Any]: + with open(path, encoding="utf-8") as stream: + data = safe_load(stream) + return data if isinstance(data, dict) else {} + + +def _strip_tz_to_naive_ms(part: pd.Series) -> pd.Series: + if getattr(part.dtype, "tz", None) is not None: + part = part.dt.tz_localize(None) + return part.astype("datetime64[ms]") + + +class FHIRDataset(BaseDataset): + """FHIR resources flattened into per-type tables, then the standard pipeline. + + Streams raw FHIR NDJSON/NDJSON.GZ exports into flattened tables (one per + configured resource type) and pipelines them through + :class:`~pyhealth.datasets.BaseDataset` for downstream task processing + (global event dataframe, patient iteration, task sampling). + + The entire ingest is driven by a single YAML config with three top-level + sections — ``glob_patterns:`` (which NDJSON files to open), + ``resource_specs:`` (how to project each FHIR resource type into a flat + row), and ``tables:`` (how those rows are exposed as events downstream). + See ``pyhealth/datasets/fhir/configs/mimic4fhir.yaml`` for a complete + worked example and the FHIRDataset rst page for a section-by-section guide. + + Pass ``config_path=...`` directly, or subclass and set + ``DEFAULT_CONFIG_PATH`` to bundle a default (see + :class:`~pyhealth.datasets.fhir.mimic4.MIMIC4FHIR`). + + Args: + root: Path to the NDJSON/NDJSON.GZ export directory. + config_path: Path to the FHIR ingest YAML. Defaults to the class + attribute ``DEFAULT_CONFIG_PATH``. The YAML must contain a + ``resource_specs:`` block; any ``glob_patterns:`` and ``tables:`` + blocks are also read from here. + glob_pattern: Single glob for NDJSON files; overrides the YAML's + ``glob_patterns``. Mutually exclusive with *glob_patterns*. + glob_patterns: Multiple glob patterns; overrides the YAML's + ``glob_patterns``. Mutually exclusive with *glob_pattern*. + output_format: Flat-table format, one of ``parquet`` (default), + ``csv``, ``tsv``. Defaults to the class attribute + ``DEFAULT_OUTPUT_FORMAT``. + max_patients: Limit ingest to the first *N* unique patient IDs. + ingest_num_shards: Ignored; retained for API compatibility. + cache_dir: Cache directory root (UUID subdir appended per config). + num_workers: Worker processes for task sampling. + dev: Development mode; limits to 1000 patients if *max_patients* is + ``None``. + + Examples: + >>> # ad-hoc, no subclass + >>> ds = FHIRDataset( + ... root="/data/fhir", + ... config_path="my_fhir.yaml", + ... ) + >>> # or a preconfigured source subclass + >>> from pyhealth.datasets import MIMIC4FHIR + >>> ds = MIMIC4FHIR(root="/data/mimic-iv-fhir", max_patients=500) + """ + + #: Default ingest YAML path; set by source subclasses to bundle a config. + DEFAULT_CONFIG_PATH: Optional[str] = None + #: Default flat-table output format. + DEFAULT_OUTPUT_FORMAT: str = "parquet" + #: Dataset name used for cache identity / logging. + DATASET_NAME: str = "fhir" + + def __init__( + self, + root: str, + config_path: Optional[str] = None, + glob_pattern: Optional[str] = None, + glob_patterns: Optional[Sequence[str]] = None, + output_format: Optional[str] = None, + max_patients: Optional[int] = None, + ingest_num_shards: Optional[int] = None, + cache_dir: Optional[str | Path] = None, + num_workers: int = 1, + dev: bool = False, + ) -> None: + del ingest_num_shards + + resolved_config = config_path or type(self).DEFAULT_CONFIG_PATH + if resolved_config is None: + raise ValueError( + "FHIRDataset requires config_path: pass config_path=... or use a " + "subclass that defines DEFAULT_CONFIG_PATH." + ) + self._fhir_config_path = str(Path(resolved_config).resolve()) + self._fhir_settings = read_fhir_settings_yaml(self._fhir_config_path) + + # Section 2 of the YAML: how each FHIR resource type projects into a row. + self.resource_specs: Mapping[str, ResourceSpec] = ( + load_resource_specs_from_yaml(self._fhir_settings) + ) + + # Cross-validate: every table the specs declare must have a downstream + # `tables:` block (Section 3). Catches typos at startup. + spec_tables = set(tables_from_specs(self.resource_specs)) + declared_tables = set((self._fhir_settings.get("tables") or {}).keys()) + missing = spec_tables - declared_tables + if missing: + raise ValueError( + f"config {self._fhir_config_path}: resource_specs references " + f"table(s) {sorted(missing)} not declared in the 'tables:' " + f"block. Add a matching tables. entry (patient_id, " + f"timestamp, attributes) for each." + ) + + self.output_format = output_format or type(self).DEFAULT_OUTPUT_FORMAT + if self.output_format not in SUPPORTED_OUTPUT_FORMATS: + raise ValueError( + f"Unsupported output_format {self.output_format!r}; " + f"expected one of {SUPPORTED_OUTPUT_FORMATS}." + ) + + if glob_pattern is not None and glob_patterns is not None: + raise ValueError("Pass at most one of glob_pattern and glob_patterns.") + if glob_patterns is not None: + self.glob_patterns: List[str] = list(glob_patterns) + elif glob_pattern is not None: + self.glob_patterns = [glob_pattern] + else: + raw_list = self._fhir_settings.get("glob_patterns") + if raw_list: + if not isinstance(raw_list, list): + raise TypeError("config glob_patterns must be a list of strings.") + self.glob_patterns = [str(x) for x in raw_list] + elif self._fhir_settings.get("glob_pattern") is not None: + self.glob_patterns = [str(self._fhir_settings["glob_pattern"])] + else: + self.glob_patterns = ["**/*.ndjson.gz"] + + self.glob_pattern = ( + self.glob_patterns[0] + if len(self.glob_patterns) == 1 + else "; ".join(self.glob_patterns) + ) + self.max_patients = 1000 if dev and max_patients is None else max_patients + + self._fhir_tables = tables_from_specs(self.resource_specs) + + resolved_root = str(Path(root).expanduser().resolve()) + super().__init__( + root=resolved_root, + tables=list(self._fhir_tables), + dataset_name=type(self).DATASET_NAME, + config_path=self._fhir_config_path, + cache_dir=cache_dir, + num_workers=num_workers, + dev=dev, + ) + + # ------------------------------------------------------------------ + # Cache identity + # ------------------------------------------------------------------ + + def _init_cache_dir(self, cache_dir: str | Path | None) -> Path: + try: + yaml_digest = hashlib.sha256( + Path(self._fhir_config_path).read_bytes() + ).hexdigest()[:16] + except OSError: + yaml_digest = "missing" + identity = orjson.dumps( + { + "root": self.root, + "tables": sorted(self.tables), + "dataset_name": self.dataset_name, + "dev": self.dev, + "glob_patterns": self.glob_patterns, + "max_patients": self.max_patients, + "output_format": self.output_format, + "fhir_schema_version": FHIR_SCHEMA_VERSION, + "fhir_yaml_digest16": yaml_digest, + }, + option=orjson.OPT_SORT_KEYS, + ).decode("utf-8") + cache_id = str(uuid.uuid5(uuid.NAMESPACE_DNS, identity)) + out = ( + Path(platformdirs.user_cache_dir(appname="pyhealth")) / cache_id + if cache_dir is None + else Path(cache_dir) / cache_id + ) + out.mkdir(parents=True, exist_ok=True) + logger.info(f"Cache dir: {out}") + return out + + # ------------------------------------------------------------------ + # NDJSON -> flat tables ingest + # ------------------------------------------------------------------ + + @property + def prepared_tables_dir(self) -> Path: + return self.cache_dir / "flattened_tables" + + def _ensure_prepared_tables(self) -> None: + root = Path(self.root) + if not root.is_dir(): + raise FileNotFoundError(f"FHIR root not found: {root}") + + expected = [ + self.prepared_tables_dir / table_file_name(t, self.output_format) + for t in self._fhir_tables + ] + if all(p.is_file() for p in expected): + return + if self.prepared_tables_dir.exists(): + shutil.rmtree(self.prepared_tables_dir) + + try: + staging_root = self.create_tmpdir() + staging = staging_root / "flattened_fhir_tables" + staging.mkdir(parents=True, exist_ok=True) + stream_fhir_ndjson_to_flat_tables( + root, + self.glob_patterns, + staging, + self.resource_specs, + self.output_format, + ) + if self.max_patients is None: + shutil.move(str(staging), str(self.prepared_tables_dir)) + return + + filtered_root = self.create_tmpdir() + filtered = filtered_root / "filtered" + pids = sorted_patient_ids_from_flat_tables( + staging, self._fhir_tables, self.output_format + ) + filter_flat_tables_by_patient_ids( + staging, + filtered, + pids[: self.max_patients], + self._fhir_tables, + self.output_format, + ) + shutil.move(str(filtered), str(self.prepared_tables_dir)) + finally: + self.clean_tmpdir() + + def _event_transform(self, output_dir: Path) -> None: + self._ensure_prepared_tables() + super()._event_transform(output_dir) + + # ------------------------------------------------------------------ + # Table loading (flat tables instead of source CSVs) + # ------------------------------------------------------------------ + + def _read_flat_table(self, path: Path) -> dd.DataFrame: + if self.output_format == "parquet": + return dd.read_parquet( + str(path), split_row_groups=True, blocksize="64MB" + ).replace("", pd.NA) + sep = "\t" if self.output_format == "tsv" else "," + return dd.read_csv( + str(path), sep=sep, dtype=str, blocksize="64MB" + ).replace("", pd.NA) + + def load_table(self, table_name: str) -> dd.DataFrame: + """Load one flattened table into the standard event schema. + + Deviations from ``BaseDataset.load_table`` (CSV via ``_scan_csv_tsv_gz``): + + * Reads pre-built flat tables (parquet/csv/tsv) under + ``prepared_tables_dir``. + * Timestamp parsing uses ``errors="coerce"`` + ``utc=True`` (FHIR ISO + strings include timezone suffix or partial dates). + * Strips tz-aware timestamps to naive UTC for Dask compat. + * Drops rows with null ``patient_id`` before returning. + """ + assert self.config is not None + if table_name not in self.config.tables: + raise ValueError(f"Table {table_name} not found in config") + + table_cfg = self.config.tables[table_name] + path = self.prepared_tables_dir / table_file_name( + table_name, self.output_format + ) + if not path.exists(): + raise FileNotFoundError(f"Flattened table not found: {path}") + + logger.info(f"Scanning FHIR flattened table: {table_name} from {path}") + df: dd.DataFrame = self._read_flat_table(path) + df = df.rename(columns=str.lower) + + preprocess_func = getattr(self, f"preprocess_{table_name}", None) + if preprocess_func is not None: + logger.info( + f"Preprocessing FHIR table: {table_name} " + f"with {preprocess_func.__name__}" + ) + df = preprocess_func(nw.from_native(df)).to_native() # type: ignore[union-attr] + + for join_cfg in table_cfg.join: + join_path = self.prepared_tables_dir / Path(join_cfg.file_path).name + if not join_path.exists(): + raise FileNotFoundError(f"FHIR join table not found: {join_path}") + logger.info(f"Joining FHIR table {table_name} with {join_path}") + join_df: dd.DataFrame = self._read_flat_table(join_path) + join_df = join_df.rename(columns=str.lower) + join_key = join_cfg.on.lower() + cols = [c.lower() for c in join_cfg.columns] + df = df.merge(join_df[[join_key] + cols], on=join_key, how=join_cfg.how) + + ts_col = table_cfg.timestamp + if ts_col: + ts = ( + functools.reduce( + operator.add, + (df[c].astype("string") for c in ts_col), + ) + if isinstance(ts_col, list) + else df[ts_col].astype("string") + ) + ts = dd.to_datetime( + ts, format=table_cfg.timestamp_format, errors="coerce", utc=True + ) + df = df.assign(timestamp=ts.map_partitions(_strip_tz_to_naive_ms)) + else: + df = df.assign(timestamp=pd.NaT) + + if table_cfg.patient_id: + df = df.assign(patient_id=df[table_cfg.patient_id].astype("string")) + else: + df = df.reset_index(drop=True) + df = df.assign(patient_id=df.index.astype("string")) + + df = df.dropna(subset=["patient_id"]) + df = df.assign(event_type=table_name) + rename_attr = { + attr.lower(): f"{table_name}/{attr}" for attr in table_cfg.attributes + } + df = df.rename(columns=rename_attr) + return df[ + ["patient_id", "event_type", "timestamp"] + + [rename_attr[a.lower()] for a in table_cfg.attributes] + ] + + # ------------------------------------------------------------------ + # Patient IDs (deterministic sorted order) + # ------------------------------------------------------------------ + + @property + def unique_patient_ids(self) -> List[str]: + if self._unique_patient_ids is None: + self._unique_patient_ids = ( + self.global_event_df.select("patient_id") + .unique() + .sort("patient_id") + .collect(engine="streaming") + .to_series() + .to_list() + ) + logger.info(f"Found {len(self._unique_patient_ids)} unique patient IDs") + return self._unique_patient_ids diff --git a/pyhealth/datasets/fhir/configs/mimic4fhir.yaml b/pyhealth/datasets/fhir/configs/mimic4fhir.yaml new file mode 100644 index 000000000..1f0f1b697 --- /dev/null +++ b/pyhealth/datasets/fhir/configs/mimic4fhir.yaml @@ -0,0 +1,210 @@ +# MIMIC-IV-on-FHIR (R4) ingest config — single source of truth +# ============================================================ +# +# Authors: John Wu and Evan Febrianto +# +# This YAML drives the entire ingest pipeline for one FHIR export: +# +# 1. ``glob_patterns:`` which NDJSON files on disk to read +# 2. ``resource_specs:`` how to project each FHIR resource type into a row +# 3. ``tables:`` how those rows are exposed as events downstream +# +# Use this file as a complete worked example when authoring a YAML for any +# other FHIR export (BigQuery dumps, Synthea, etc.). To use it as-is, just +# instantiate :class:`~pyhealth.datasets.MIMIC4FHIR` with no extra arguments. +# +# To customise for a different export: +# * subclass FHIRDataset and point ``DEFAULT_CONFIG_PATH`` at your YAML, or +# * pass ``config_path=`` directly to ``FHIRDataset(...)``. + +version: "fhir_r4_flattened" + + +# --------------------------------------------------------------------------- +# Section 1: glob patterns — which NDJSON files to open +# --------------------------------------------------------------------------- +# +# Defaults to ``["**/*.ndjson.gz"]`` when omitted. Only useful when the export +# has per-resource-type file naming (PhysioNet MIMIC-IV FHIR ships shards as +# ``MimicPatient*.ndjson.gz``, ``MimicEncounter*.ndjson.gz``, etc.). Filtering +# at the file level avoids decompressing ~10% of the export that contains only +# unconfigured resource types (MedicationAdministration, Specimen, Organization). +# +# For a generic export where everything lives in ``bundles.ndjson.gz`` / +# ``**/*.ndjson.gz``, leave this commented out and the streamer will filter +# resources by ``resourceType`` after parsing — correct, just slower. +# +# Override at runtime via ``MIMIC4FHIR(glob_pattern=...)`` or +# ``MIMIC4FHIR(glob_patterns=[...])``. + +glob_patterns: + - "**/MimicPatient*.ndjson.gz" + - "**/MimicEncounter*.ndjson.gz" + - "**/MimicCondition*.ndjson.gz" + - "**/MimicObservation*.ndjson.gz" + - "**/MimicMedicationRequest*.ndjson.gz" + - "**/MimicProcedure*.ndjson.gz" + + +# --------------------------------------------------------------------------- +# Section 2: resource_specs — how to turn one FHIR JSON document into a row +# --------------------------------------------------------------------------- +# +# Keys are FHIR ``resourceType`` strings. For each resource type we declare: +# +# table: output table name (also the per-type Parquet filename stem). +# columns: ordered mapping of output column name -> Col spec. +# +# Each Col spec lists ordered JSON paths (``locate``); the first path that +# resolves to a non-null value wins (this is how FHIR choice-types like +# ``onset[x]`` and ``effective[x]`` are handled). ``transform`` names a function +# in the engine's TRANSFORMS registry that maps the located leaf to a flat +# string; ``required: true`` drops the resource if the leaf can't be resolved. +# +# Available transforms (defined in pyhealth/datasets/fhir/utils.py): +# +# identity Pass through (default; stringifies non-string scalars). +# ref_id "{ "reference": "Patient/p1" }" -> "p1". +# coding_key CodeableConcept -> "system|code" of its first coding. +# bool_norm JSON boolean / "true"/"false" string -> "true"/"false"/None. +# med_concept MedicationRequest.medication[x] -> codeable-concept or +# "MedicationRequest/reference|" fallback. +# +# Adding a new transform: register it in TRANSFORMS in utils.py; reference it +# by name here. + +resource_specs: + + Patient: + table: patient + columns: + patient_id: { locate: ["id"], required: true } + patient_fhir_id: { locate: ["id"] } + birth_date: { locate: ["birthDate"] } + gender: { locate: ["gender"] } + deceased_boolean: { locate: ["deceasedBoolean"], transform: bool_norm } + deceased_datetime: { locate: ["deceasedDateTime"] } + + Encounter: + table: encounter + columns: + patient_id: { locate: ["subject.reference"], transform: ref_id, required: true } + resource_id: { locate: ["id"] } + encounter_id: { locate: ["id"] } + event_time: { locate: ["period.start"] } + encounter_class: { locate: ["class.code"] } + encounter_end: { locate: ["period.end"] } + + Condition: + table: condition + columns: + patient_id: { locate: ["subject.reference"], transform: ref_id, required: true } + resource_id: { locate: ["id"] } + encounter_id: { locate: ["encounter.reference"], transform: ref_id } + event_time: { locate: ["onsetDateTime", "onsetPeriod.start", "recordedDate"] } + concept_key: { locate: ["code"], transform: coding_key } + + Observation: + table: observation + columns: + patient_id: { locate: ["subject.reference"], transform: ref_id, required: true } + resource_id: { locate: ["id"] } + encounter_id: { locate: ["encounter.reference"], transform: ref_id } + event_time: { locate: ["effectiveDateTime", "effectivePeriod.start", "issued"] } + concept_key: { locate: ["code"], transform: coding_key } + + MedicationRequest: + table: medication_request + columns: + patient_id: { locate: ["subject.reference"], transform: ref_id, required: true } + resource_id: { locate: ["id"] } + encounter_id: { locate: ["encounter.reference"], transform: ref_id } + event_time: { locate: ["authoredOn"] } + concept_key: { locate: ["medicationCodeableConcept", "medicationReference"], transform: med_concept } + + Procedure: + table: procedure + columns: + patient_id: { locate: ["subject.reference"], transform: ref_id, required: true } + resource_id: { locate: ["id"] } + encounter_id: { locate: ["encounter.reference"], transform: ref_id } + event_time: { locate: ["performedDateTime", "performedPeriod.start", "recordedDate"] } + concept_key: { locate: ["code"], transform: coding_key } + + +# --------------------------------------------------------------------------- +# Section 3: tables — how flat rows are exposed as events downstream +# --------------------------------------------------------------------------- +# +# Keys here must match the ``table:`` values in section 2. Each entry declares +# how BaseDataset.load_table reads the flat parquet: +# +# file_path: parquet filename (relative to the cached flattened_tables dir). +# patient_id: column name that holds the patient id. +# timestamp: column name to parse as the event timestamp. +# attributes: list of columns to surface as event attributes; they are +# renamed to ``{table}/{attr}`` in the global event frame and +# later show up on ``Patient.get_events(...).attr_name``. + +tables: + patient: + file_path: "patient.parquet" + patient_id: "patient_id" + timestamp: "birth_date" + attributes: + - "patient_fhir_id" + - "birth_date" + - "gender" + - "deceased_boolean" + - "deceased_datetime" + + encounter: + file_path: "encounter.parquet" + patient_id: "patient_id" + timestamp: "event_time" + attributes: + - "resource_id" + - "encounter_id" + - "event_time" + - "encounter_class" + - "encounter_end" + + condition: + file_path: "condition.parquet" + patient_id: "patient_id" + timestamp: "event_time" + attributes: + - "resource_id" + - "encounter_id" + - "event_time" + - "concept_key" + + observation: + file_path: "observation.parquet" + patient_id: "patient_id" + timestamp: "event_time" + attributes: + - "resource_id" + - "encounter_id" + - "event_time" + - "concept_key" + + medication_request: + file_path: "medication_request.parquet" + patient_id: "patient_id" + timestamp: "event_time" + attributes: + - "resource_id" + - "encounter_id" + - "event_time" + - "concept_key" + + procedure: + file_path: "procedure.parquet" + patient_id: "patient_id" + timestamp: "event_time" + attributes: + - "resource_id" + - "encounter_id" + - "event_time" + - "concept_key" diff --git a/pyhealth/datasets/fhir/mimic4.py b/pyhealth/datasets/fhir/mimic4.py new file mode 100644 index 000000000..5732a6776 --- /dev/null +++ b/pyhealth/datasets/fhir/mimic4.py @@ -0,0 +1,42 @@ +"""MIMIC-IV-on-FHIR (R4) dataset. + +A thin :class:`~pyhealth.datasets.fhir.base.FHIRDataset` wrapper that points at +the bundled YAML for the PhysioNet MIMIC-IV on FHIR export. The whole ingest +contract (resource projection + downstream table schema + glob hints) lives in +the YAML; this class only names its default path. + +Use this YAML as the worked example when authoring a config for a different +FHIR export — copy ``pyhealth/datasets/fhir/configs/mimic4fhir.yaml`` and +adapt the ``resource_specs:`` and ``tables:`` blocks. + +Authors: + John Wu and Evan Febrianto +""" + +from __future__ import annotations + +import os + +from .base import FHIRDataset + + +class MIMIC4FHIR(FHIRDataset): + """MIMIC-IV-on-FHIR (R4) dataset. + + Streams the PhysioNet MIMIC-IV on FHIR NDJSON.GZ export into flattened + Patient/Encounter/Condition/Observation/MedicationRequest/Procedure tables, + then runs the standard :class:`~pyhealth.datasets.BaseDataset` pipeline. + + The bundled config at ``pyhealth/datasets/fhir/configs/mimic4fhir.yaml`` + matches both the PhysioNet 2.1.0 demo and the full release. Override + ``config_path=`` to point at a customised copy. + + Examples: + >>> ds = MIMIC4FHIR(root="/data/mimic-iv-fhir", max_patients=500) + >>> sample_ds = ds.set_task(task, num_workers=4) + """ + + DEFAULT_CONFIG_PATH = os.path.join( + os.path.dirname(__file__), "configs", "mimic4fhir.yaml" + ) + DATASET_NAME = "mimic4fhir" diff --git a/pyhealth/datasets/fhir/utils.py b/pyhealth/datasets/fhir/utils.py new file mode 100644 index 000000000..40911cc43 --- /dev/null +++ b/pyhealth/datasets/fhir/utils.py @@ -0,0 +1,724 @@ +"""FHIR NDJSON parsing, generic flattening, and tabular table writing. + +This module is the **stateless engine** behind FHIR-to-tabular conversion. It +knows nothing about any specific FHIR source or resource type: the per-resource +projection is supplied as a declarative registry of :class:`ResourceSpec` objects +(see ``MIMIC4FHIR.RESOURCE_SPECS`` for an example) and applied generically by +:func:`flatten_resource`. + +Key public API +-------------- +flatten_resource(resource, specs) + Project one FHIR resource dict into ``(table_name, row_dict)`` using a spec + registry, or ``None`` if the resource is unconfigured / missing a required + field. + +stream_fhir_ndjson_to_flat_tables(root, glob_pattern, out_dir, specs, output_format) + Stream all matching NDJSON/NDJSON.GZ resources into per-type flat tables + (parquet/csv/tsv), validating + counting drops along the way. + +sorted_ndjson_files(root, glob_pattern) + List matching NDJSON files under root (deduplicated, sorted). + +filter_flat_tables_by_patient_ids(source_dir, out_dir, keep_ids, tables, output_format) + Subset existing flattened tables to a specific patient cohort. + +Authors: + John Wu and Evan Febrianto +""" + +from __future__ import annotations + +import gzip +import logging +from collections import Counter +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, Iterator, List, Mapping, Optional, Sequence, Tuple + +import orjson +import polars as pl +import pyarrow as pa +import pyarrow.csv as pa_csv +import pyarrow.parquet as pq + +logger = logging.getLogger(__name__) + +GlobPatternArg = str | Sequence[str] +"""Single glob string or sequence of strings for NDJSON file discovery.""" + +__all__ = [ + # Types + "GlobPatternArg", + "Col", + "ResourceSpec", + # Constants + "FHIR_SCHEMA_VERSION", + "SUPPORTED_OUTPUT_FORMATS", + "TRANSFORMS", + # Spec helpers + "tables_from_specs", + "columns_from_specs", + "table_file_name", + "load_resource_specs_from_yaml", + # Datetime helpers + "parse_dt", + "as_naive", + # FHIR iteration + "iter_ndjson_objects", + "iter_resources_from_ndjson_obj", + # Extraction + "flatten_resource", + # Pipeline + "sorted_ndjson_files", + "stream_fhir_ndjson_to_flat_tables", + "filter_flat_tables_by_patient_ids", + "sorted_patient_ids_from_flat_tables", +] + +# Bump when the flattening engine or its output layout changes; folded into the +# dataset cache identity so stale caches rebuild automatically. +FHIR_SCHEMA_VERSION = 4 + +SUPPORTED_OUTPUT_FORMATS = ("parquet", "csv", "tsv") + + +# --------------------------------------------------------------------------- +# Declarative extraction spec (the registry's value type) +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class Col: + """How to project one flat column out of a FHIR resource. + + Typically constructed indirectly by :meth:`ResourceSpec.from_dict` while + loading the dataset's YAML config; direct instantiation is supported for + programmatic use. + + Attributes: + locate: Ordered dotted paths into the resource; the first that resolves + to a non-null value wins. This is how FHIR choice-types (``onset[x]``, + ``effective[x]``, …) are handled — list every variant explicitly. + transform: Name of a value transform in :data:`TRANSFORMS` that converts + the located leaf into a flat scalar string. + required: When ``True``, a resource whose ``locate`` cannot be resolved is + dropped (and counted) rather than emitted with a null. + """ + + locate: Tuple[str, ...] + transform: str = "identity" + required: bool = False + + @classmethod + def from_dict(cls, data: Mapping[str, Any], *, ctx: str = "") -> "Col": + """Build a :class:`Col` from a YAML-style dict. + + Expected shape:: + + { locate: ["path.a", "path.b"], transform: "ref_id", required: false } + + ``transform`` defaults to ``"identity"`` and must name an entry in + :data:`TRANSFORMS`. ``required`` defaults to ``False``. A missing or + empty ``locate`` field raises ``ValueError``. + + Args: + data: Mapping containing ``locate`` (required) and the optional + ``transform`` / ``required`` keys. + ctx: Optional context string used in error messages + (e.g. ``"Patient.patient_id"``). + """ + if not isinstance(data, Mapping): + raise ValueError( + f"{ctx or 'Col'}: expected a mapping, got {type(data).__name__}." + ) + raw_locate = data.get("locate") + if not raw_locate: + raise ValueError( + f"{ctx or 'Col'}: missing required field 'locate'." + ) + if isinstance(raw_locate, str): + locate: Tuple[str, ...] = (raw_locate,) + else: + locate = tuple(str(p) for p in raw_locate) + if not locate: + raise ValueError( + f"{ctx or 'Col'}: 'locate' must list at least one path." + ) + transform = str(data.get("transform", "identity")) + if transform not in TRANSFORMS: + allowed = ", ".join(sorted(TRANSFORMS.keys())) + raise ValueError( + f"{ctx or 'Col'}: unknown transform {transform!r}. " + f"Allowed: {allowed}." + ) + required = bool(data.get("required", False)) + return cls(locate=locate, transform=transform, required=required) + + +@dataclass(frozen=True) +class ResourceSpec: + """How to project one FHIR resource type into a flat table. + + Typically constructed indirectly by :func:`load_resource_specs_from_yaml` + while loading the dataset's YAML config; direct instantiation is supported + for programmatic use. + + Attributes: + table: Output table name (also the per-type file stem). + columns: Mapping of output column name -> :class:`Col`. Insertion order + defines the table's column order. + """ + + table: str + columns: Mapping[str, Col] + + @classmethod + def from_dict( + cls, resource_type: str, data: Mapping[str, Any] + ) -> "ResourceSpec": + """Build a :class:`ResourceSpec` from a YAML-style dict. + + Expected shape:: + + { + table: "patient", + columns: { + patient_id: { locate: ["id"], required: true }, + birth_date: { locate: ["birthDate"] }, + ... + }, + } + + Args: + resource_type: FHIR resourceType string this spec describes + (e.g. ``"Patient"``). Used only for error messages. + data: Mapping containing ``table`` (required, str) and + ``columns`` (required, mapping of column name -> Col-shaped + mapping). + """ + if not isinstance(data, Mapping): + raise ValueError( + f"resource_specs.{resource_type}: expected a mapping, " + f"got {type(data).__name__}." + ) + table = data.get("table") + if not isinstance(table, str) or not table: + raise ValueError( + f"resource_specs.{resource_type}: missing required field " + f"'table' (string)." + ) + raw_columns = data.get("columns") + if not isinstance(raw_columns, Mapping) or not raw_columns: + raise ValueError( + f"resource_specs.{resource_type}: missing required field " + f"'columns' (non-empty mapping)." + ) + columns: Dict[str, Col] = {} + for col_name, col_data in raw_columns.items(): + columns[str(col_name)] = Col.from_dict( + col_data, + ctx=f"resource_specs.{resource_type}.columns.{col_name}", + ) + return cls(table=str(table), columns=columns) + + +def load_resource_specs_from_yaml( + raw: Mapping[str, Any], +) -> Dict[str, ResourceSpec]: + """Build the spec registry from a parsed YAML's ``resource_specs:`` block. + + Args: + raw: The full parsed YAML mapping (top-level dict). The + ``resource_specs`` key, if present, must be a mapping of FHIR + resourceType -> ResourceSpec-shaped dict. + + Returns: + Insertion-ordered mapping of resourceType to :class:`ResourceSpec`. + + Raises: + ValueError: If the ``resource_specs`` block is missing, empty, or + contains a malformed entry. + """ + block = raw.get("resource_specs") + if not isinstance(block, Mapping) or not block: + raise ValueError( + "config: missing or empty top-level 'resource_specs:' block. " + "Declare at least one FHIR resourceType -> spec mapping." + ) + specs: Dict[str, ResourceSpec] = {} + for resource_type, data in block.items(): + specs[str(resource_type)] = ResourceSpec.from_dict( + str(resource_type), data + ) + return specs + + +def tables_from_specs(specs: Mapping[str, ResourceSpec]) -> List[str]: + """Ordered, de-duplicated list of output table names declared by *specs*.""" + seen: Dict[str, None] = {} + for spec in specs.values(): + seen.setdefault(spec.table, None) + return list(seen.keys()) + + +def columns_from_specs(specs: Mapping[str, ResourceSpec]) -> Dict[str, List[str]]: + """Map each output table name to its ordered column names.""" + return {spec.table: list(spec.columns.keys()) for spec in specs.values()} + + +def table_file_name(table_name: str, output_format: str = "parquet") -> str: + """Filename for a flattened table given the output format.""" + ext = "parquet" if output_format == "parquet" else output_format + return f"{table_name}.{ext}" + + +# --------------------------------------------------------------------------- +# Datetime helpers (kept for external callers) +# --------------------------------------------------------------------------- + + +def parse_dt(s: Optional[str]) -> Optional[datetime]: + if not s: + return None + try: + dt = datetime.fromisoformat(s.replace("Z", "+00:00")) + except ValueError: + dt = None + if dt is None and len(s) >= 10: + try: + dt = datetime.strptime(s[:10], "%Y-%m-%d") + except ValueError: + return None + if dt is None: + return None + return dt.replace(tzinfo=None) if dt.tzinfo is not None else dt + + +def as_naive(dt: Optional[datetime]) -> Optional[datetime]: + if dt is None: + return None + return dt.replace(tzinfo=None) if dt.tzinfo is not None else dt + + +# --------------------------------------------------------------------------- +# FHIR JSON helpers +# --------------------------------------------------------------------------- + + +def _coding_key(coding: Dict[str, Any]) -> str: + return f"{coding.get('system') or 'unknown'}|{coding.get('code') or 'unknown'}" + + +def _first_coding(obj: Optional[Dict[str, Any]]) -> Optional[str]: + """CodeableConcept -> ``"system|code"`` for its first coding (or None).""" + if not isinstance(obj, dict): + return None + codings = obj.get("coding") or [] + if not codings and "concept" in obj: + codings = (obj.get("concept") or {}).get("coding") or [] + return _coding_key(codings[0]) if codings else None + + +def _ref_id(ref: Optional[Any]) -> Optional[str]: + """``{"reference": "Patient/p1"}`` or ``"Patient/p1"`` -> ``"p1"``.""" + if isinstance(ref, dict): + ref = ref.get("reference") + if not ref: + return None + return ref.rsplit("/", 1)[-1] if "/" in ref else ref + + +def _normalize_deceased_boolean_for_storage(value: Any) -> Optional[str]: + """Map Patient.deceasedBoolean to stored "true"/"false"/None. + + FHIR JSON uses real booleans; some exports use strings. Python's + bool("false") is True, so we must not coerce with bool(). + """ + if value is None: + return None + if value is True: + return "true" + if value is False: + return "false" + if isinstance(value, str): + key = value.strip().lower() + if key in ("true", "1", "yes", "y", "t"): + return "true" + if key in ("false", "0", "no", "n", "f", ""): + return "false" + return None + if isinstance(value, (int, float)) and not isinstance(value, bool): + if value == 0: + return "false" + if value == 1: + return "true" + return None + return None + + +def _medication_concept_key(value: Any) -> Optional[str]: + """MedicationRequest medication[x] -> a stable concept key. + + Accepts either a ``medicationCodeableConcept`` (-> ``"system|code"``) or a + ``medicationReference`` (-> ``"MedicationRequest/reference|"``). + """ + if not isinstance(value, dict): + return None + if "coding" in value or "concept" in value: + key = _first_coding(value) + if key: + return key + ref = value.get("reference") + if ref: + return f"MedicationRequest/reference|{_ref_id(ref) or ref}" + return None + + +def _identity(value: Any) -> Optional[str]: + if value is None or isinstance(value, str): + return value + return str(value) + + +# Transform registry: how a located leaf becomes a flat scalar string. +TRANSFORMS = { + "identity": _identity, + "ref_id": _ref_id, + "coding_key": _first_coding, + "bool_norm": _normalize_deceased_boolean_for_storage, + "med_concept": _medication_concept_key, +} + + +def _unwrap_resource_dict(raw: Any) -> Optional[Dict[str, Any]]: + if not isinstance(raw, dict): + return None + resource = raw.get("resource") if "resource" in raw else raw + return resource if isinstance(resource, dict) else None + + +def iter_resources_from_ndjson_obj(obj: Dict[str, Any]) -> Iterator[Dict[str, Any]]: + """Yield resource dicts from one parsed NDJSON object (Bundle or bare resource).""" + if "entry" in obj: + for entry in obj.get("entry") or []: + resource = entry.get("resource") + if isinstance(resource, dict): + yield resource + return + resource = _unwrap_resource_dict(obj) + if resource is not None: + yield resource + + +def iter_ndjson_objects(path: Path) -> Iterator[Dict[str, Any]]: + """Yield parsed JSON objects from a plain or gzip-compressed NDJSON file.""" + opener = ( + gzip.open(path, "rt", encoding="utf-8", errors="replace") + if path.suffix == ".gz" + else open(path, encoding="utf-8", errors="replace") + ) + with opener as stream: + for line in stream: + line = line.strip() + if not line: + continue + try: + parsed = orjson.loads(line) + except orjson.JSONDecodeError as e: + logger.warning("Skipping invalid JSON line in %s: %s", path, e) + continue + if isinstance(parsed, dict): + yield parsed + + +# --------------------------------------------------------------------------- +# Generic extraction engine +# --------------------------------------------------------------------------- + + +def _get_path(obj: Any, path: str) -> Any: + """Walk a dotted path (e.g. ``"encounter.reference"``) safely; None if absent.""" + cur = obj + for part in path.split("."): + if not isinstance(cur, dict): + return None + cur = cur.get(part) + return cur + + +def _first_located(resource: Dict[str, Any], paths: Tuple[str, ...]) -> Any: + """First non-null value among the ordered ``paths`` (choice-type resolution).""" + for path in paths: + value = _get_path(resource, path) + if value is not None: + return value + return None + + +def flatten_resource( + resource: Dict[str, Any], + specs: Mapping[str, ResourceSpec], +) -> Optional[Tuple[str, Dict[str, Optional[str]]]]: + """Project one FHIR resource into ``(table_name, row)`` via *specs*. + + Returns ``None`` if the resource type is not configured in *specs*, or if a + column marked ``required`` cannot be resolved (a dropped/corrupted resource). + """ + spec = specs.get(resource.get("resourceType")) + if spec is None: + return None + row: Dict[str, Optional[str]] = {} + for name, col in spec.columns.items(): + raw = _first_located(resource, col.locate) + if raw is None and col.required: + return None + row[name] = TRANSFORMS[col.transform](raw) + return spec.table, row + + +# --------------------------------------------------------------------------- +# Tabular writer (parquet / csv / tsv) +# --------------------------------------------------------------------------- + + +def _table_schema(columns: Sequence[str]) -> pa.Schema: + return pa.schema([(col, pa.string()) for col in columns]) + + +class _BufferedTableWriter: + """Buffered, streaming writer for one flat table in parquet/csv/tsv.""" + + def __init__( + self, + path: Path, + schema: pa.Schema, + output_format: str = "parquet", + batch_size: int = 50_000, + ) -> None: + self.path = path + self.schema = schema + self.output_format = output_format + self.batch_size = batch_size + self.rows: List[Dict[str, Any]] = [] + self._pq_writer: Optional[pq.ParquetWriter] = None + self._fh = None + self._csv_header_written = False + self._delimiter = "\t" if output_format == "tsv" else "," + self.path.parent.mkdir(parents=True, exist_ok=True) + + def add(self, row: Dict[str, Any]) -> None: + self.rows.append(row) + if len(self.rows) >= self.batch_size: + self.flush() + + def flush(self) -> None: + if not self.rows: + return + table = pa.Table.from_pylist(self.rows, schema=self.schema) + if self.output_format == "parquet": + if self._pq_writer is None: + self._pq_writer = pq.ParquetWriter(str(self.path), self.schema) + self._pq_writer.write_table(table) + else: + if self._fh is None: + self._fh = open(self.path, "wb") + pa_csv.write_csv( + table, + self._fh, + write_options=pa_csv.WriteOptions( + include_header=not self._csv_header_written, + delimiter=self._delimiter, + ), + ) + self._csv_header_written = True + self.rows.clear() + + def close(self) -> None: + self.flush() + if self.output_format == "parquet": + if self._pq_writer is None: + pq.write_table( + pa.Table.from_pylist([], schema=self.schema), str(self.path) + ) + else: + self._pq_writer.close() + return + if self._fh is None: + # Empty table: still write a header-only file for a stable schema. + self._fh = open(self.path, "wb") + pa_csv.write_csv( + pa.Table.from_pylist([], schema=self.schema), + self._fh, + write_options=pa_csv.WriteOptions( + include_header=True, delimiter=self._delimiter + ), + ) + self._fh.close() + + +# --------------------------------------------------------------------------- +# Pipeline +# --------------------------------------------------------------------------- + + +def sorted_ndjson_files(root: Path, glob_pattern: GlobPatternArg) -> List[Path]: + """Return sorted unique file paths under root matching glob pattern(s). + + Args: + root: Root directory to search under. + glob_pattern: Single glob string or sequence of glob strings. + + Returns: + Sorted list of matching files. Empty if no matches. + """ + patterns = [glob_pattern] if isinstance(glob_pattern, str) else list(glob_pattern) + files: set[Path] = set() + for pat in patterns: + files.update(p for p in root.glob(pat) if p.is_file()) + return sorted(files, key=lambda p: str(p)) + + +def stream_fhir_ndjson_to_flat_tables( + root: Path, + glob_pattern: GlobPatternArg, + out_dir: Path, + specs: Mapping[str, ResourceSpec], + output_format: str = "parquet", +) -> None: + """Stream NDJSON resources into normalized per-resource flat tables. + + Resources are validated as they stream: anything whose type is not in + *specs*, or which is missing a ``required`` field, is dropped and counted; a + summary is logged at the end so corruption is visible rather than silent. + + Args: + root: Root directory containing NDJSON/NDJSON.GZ files. + glob_pattern: Single glob string or sequence of glob strings. + out_dir: Output directory for per-resource-type tables. + specs: Registry mapping FHIR resourceType -> :class:`ResourceSpec`. + output_format: One of :data:`SUPPORTED_OUTPUT_FORMATS`. + """ + if output_format not in SUPPORTED_OUTPUT_FORMATS: + raise ValueError( + f"Unsupported output_format {output_format!r}; " + f"expected one of {SUPPORTED_OUTPUT_FORMATS}." + ) + out_dir.mkdir(parents=True, exist_ok=True) + tables = tables_from_specs(specs) + columns = columns_from_specs(specs) + writers = { + name: _BufferedTableWriter( + path=out_dir / table_file_name(name, output_format), + schema=_table_schema(columns[name]), + output_format=output_format, + ) + for name in tables + } + + ingested: Counter = Counter() + dropped: Counter = Counter() + skipped_unconfigured: Counter = Counter() + try: + for file_path in sorted_ndjson_files(root, glob_pattern): + for ndjson_obj in iter_ndjson_objects(file_path): + for resource in iter_resources_from_ndjson_obj(ndjson_obj): + resource_type = resource.get("resourceType") + result = flatten_resource(resource, specs) + if result is None: + if resource_type in specs: + dropped[resource_type] += 1 + else: + skipped_unconfigured[resource_type] += 1 + continue + table_name, row = result + writers[table_name].add(row) + ingested[table_name] += 1 + finally: + for writer in writers.values(): + writer.close() + + logger.info( + "FHIR flatten complete (%s): %s", + output_format, + {name: ingested[name] for name in tables}, + ) + for resource_type, count in dropped.items(): + logger.warning( + "FHIR flatten: dropped %d %s resource(s) missing a required field " + "(e.g. patient reference).", + count, + resource_type, + ) + if skipped_unconfigured: + total = sum(skipped_unconfigured.values()) + logger.info( + "FHIR flatten: skipped %d resource(s) of %d unconfigured type(s): %s", + total, + len(skipped_unconfigured), + dict(skipped_unconfigured), + ) + + +def _scan_flat_table(path: Path, output_format: str) -> pl.LazyFrame: + if output_format == "parquet": + return pl.scan_parquet(str(path)) + sep = "\t" if output_format == "tsv" else "," + # infer_schema_length=0 keeps every column as Utf8 (flat tables are all strings). + return pl.scan_csv(str(path), separator=sep, infer_schema_length=0) + + +def sorted_patient_ids_from_flat_tables( + table_dir: Path, + tables: Sequence[str], + output_format: str = "parquet", +) -> List[str]: + """Return sorted unique patient IDs from a directory of flattened tables.""" + patient_path = table_dir / table_file_name("patient", output_format) + if "patient" in tables and patient_path.exists(): + return ( + _scan_flat_table(patient_path, output_format) + .select("patient_id") + .unique() + .sort("patient_id") + .collect(engine="streaming")["patient_id"] + .to_list() + ) + frames = [ + _scan_flat_table( + table_dir / table_file_name(t, output_format), output_format + ).select("patient_id") + for t in tables + if t != "patient" + ] + return ( + pl.concat(frames) + .unique() + .sort("patient_id") + .collect(engine="streaming")["patient_id"] + .to_list() + ) + + +def filter_flat_tables_by_patient_ids( + source_dir: Path, + out_dir: Path, + keep_ids: Sequence[str], + tables: Sequence[str], + output_format: str = "parquet", +) -> None: + """Filter all flattened tables to only include rows for the given patient IDs.""" + out_dir.mkdir(parents=True, exist_ok=True) + keep_set = set(keep_ids) + for name in tables: + src = source_dir / table_file_name(name, output_format) + dst = out_dir / table_file_name(name, output_format) + lf = _scan_flat_table(src, output_format).filter( + pl.col("patient_id").is_in(keep_set) + ) + if output_format == "parquet": + lf.sink_parquet(str(dst)) + else: + sep = "\t" if output_format == "tsv" else "," + lf.sink_csv(str(dst), separator=sep) diff --git a/pyhealth/models/__init__.py b/pyhealth/models/__init__.py index 18500b9c0..a4fe5dc85 100644 --- a/pyhealth/models/__init__.py +++ b/pyhealth/models/__init__.py @@ -39,6 +39,7 @@ from .transformer import Transformer, TransformerLayer from .transformers_model import TransformersModel from .ehrmamba import EHRMamba, MambaBlock +from .ehrmamba_cehr import EHRMambaCEHR from .vae import VAE from .vision_embedding import VisionEmbeddingModel from .text_embedding import TextEmbedding diff --git a/pyhealth/models/cehr_embeddings.py b/pyhealth/models/cehr_embeddings.py new file mode 100644 index 000000000..7974a699e --- /dev/null +++ b/pyhealth/models/cehr_embeddings.py @@ -0,0 +1,112 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2024 Vector Institute / Odyssey authors +# +# Derived from Odyssey (https://github.com/VectorInstitute/odyssey): +# odyssey/models/embeddings.py — MambaEmbeddingsForCEHR, TimeEmbeddingLayer, VisitEmbedding +# Modifications: removed HuggingFace MambaConfig dependency; explicit constructor args. + +from __future__ import annotations + +from typing import Any, Optional + +import torch +from torch import nn + + +class TimeEmbeddingLayer(nn.Module): + """Embedding layer for time features (sinusoidal).""" + + def __init__(self, embedding_size: int, is_time_delta: bool = False): + super().__init__() + self.embedding_size = embedding_size + self.is_time_delta = is_time_delta + self.w = nn.Parameter(torch.empty(1, self.embedding_size)) + self.phi = nn.Parameter(torch.empty(1, self.embedding_size)) + nn.init.xavier_uniform_(self.w) + nn.init.xavier_uniform_(self.phi) + + def forward(self, time_stamps: torch.Tensor) -> torch.Tensor: + if self.is_time_delta: + time_stamps = torch.cat( + (time_stamps[:, 0:1] * 0, time_stamps[:, 1:] - time_stamps[:, :-1]), + dim=-1, + ) + time_stamps = time_stamps.float() + next_input = time_stamps.unsqueeze(-1) * self.w + self.phi + return torch.sin(next_input) + + +class VisitEmbedding(nn.Module): + """Embedding layer for visit segments.""" + + def __init__(self, visit_order_size: int, embedding_size: int): + super().__init__() + self.embedding = nn.Embedding(visit_order_size, embedding_size) + + def forward(self, visit_segments: torch.Tensor) -> torch.Tensor: + return self.embedding(visit_segments) + + +class MambaEmbeddingsForCEHR(nn.Module): + """CEHR-style combined embeddings for Mamba (concept + type + time + age + visit).""" + + def __init__( + self, + vocab_size: int, + hidden_size: int, + pad_token_id: int = 0, + type_vocab_size: int = 9, + max_num_visits: int = 512, + time_embeddings_size: int = 32, + visit_order_size: int = 3, + layer_norm_eps: float = 1e-12, + hidden_dropout_prob: float = 0.1, + ) -> None: + super().__init__() + self.hidden_size = hidden_size + self.pad_token_id = pad_token_id + self.type_vocab_size = type_vocab_size + self.max_num_visits = max_num_visits + self.word_embeddings = nn.Embedding( + vocab_size, hidden_size, padding_idx=pad_token_id + ) + self.token_type_embeddings = nn.Embedding(type_vocab_size, hidden_size) + self.visit_order_embeddings = nn.Embedding(max_num_visits, hidden_size) + self.time_embeddings = TimeEmbeddingLayer( + embedding_size=time_embeddings_size, is_time_delta=True + ) + self.age_embeddings = TimeEmbeddingLayer( + embedding_size=time_embeddings_size, is_time_delta=False + ) + self.visit_segment_embeddings = VisitEmbedding( + visit_order_size=visit_order_size, embedding_size=hidden_size + ) + self.scale_back_concat_layer = nn.Linear( + hidden_size + 2 * time_embeddings_size, hidden_size + ) + self.tanh = nn.Tanh() + self.LayerNorm = nn.LayerNorm(hidden_size, eps=layer_norm_eps) + self.dropout = nn.Dropout(hidden_dropout_prob) + + def forward( + self, + input_ids: torch.Tensor, + token_type_ids_batch: torch.Tensor, + time_stamps: torch.Tensor, + ages: torch.Tensor, + visit_orders: torch.Tensor, + visit_segments: torch.Tensor, + ) -> torch.Tensor: + inputs_embeds = self.word_embeddings(input_ids) + time_stamps_embeds = self.time_embeddings(time_stamps) + ages_embeds = self.age_embeddings(ages) + visit_segments_embeds = self.visit_segment_embeddings(visit_segments) + visit_order_embeds = self.visit_order_embeddings(visit_orders) + token_type_embeds = self.token_type_embeddings(token_type_ids_batch) + concat_in = torch.cat( + (inputs_embeds, time_stamps_embeds, ages_embeds), dim=-1 + ) + h = self.tanh(self.scale_back_concat_layer(concat_in)) + embeddings = h + token_type_embeds + visit_order_embeds + visit_segments_embeds + embeddings = self.dropout(embeddings) + return self.LayerNorm(embeddings) diff --git a/pyhealth/models/ehrmamba_cehr.py b/pyhealth/models/ehrmamba_cehr.py new file mode 100644 index 000000000..cd555629c --- /dev/null +++ b/pyhealth/models/ehrmamba_cehr.py @@ -0,0 +1,117 @@ +"""EHRMamba with CEHR-style embeddings for single-stream FHIR token sequences.""" + +from __future__ import annotations + +from typing import Any, Dict, Optional + +import torch +from torch import nn + +from pyhealth.datasets import SampleDataset + +from .base_model import BaseModel +from .cehr_embeddings import MambaEmbeddingsForCEHR +from .ehrmamba import MambaBlock +from .utils import get_rightmost_masked_timestep + + +class EHRMambaCEHR(BaseModel): + """Mamba backbone over CEHR embeddings (FHIR / MPF pipeline). + + Args: + dataset: Fitted :class:`~pyhealth.datasets.SampleDataset` with MPF task schema. + vocab_size: Concept embedding vocabulary size (typically ``task.vocab.vocab_size``). + embedding_dim: Hidden size (``hidden_size`` in CEHR embeddings). + num_layers: Number of :class:`~pyhealth.models.ehrmamba.MambaBlock` layers. + pad_token_id: Padding id for masking (default 0). + state_size: SSM state size per channel. + conv_kernel: Causal conv kernel in each block. + dropout: Dropout before classifier. + """ + + def __init__( + self, + dataset: SampleDataset, + vocab_size: int, + embedding_dim: int = 128, + num_layers: int = 2, + pad_token_id: int = 0, + state_size: int = 16, + conv_kernel: int = 4, + dropout: float = 0.1, + type_vocab_size: int = 16, + max_num_visits: int = 512, + time_embeddings_size: int = 32, + visit_segment_vocab: int = 3, + ): + super().__init__(dataset=dataset) + self.embedding_dim = embedding_dim + self.num_layers = num_layers + self.pad_token_id = pad_token_id + self.vocab_size = vocab_size + + assert len(self.label_keys) == 1, "EHRMambaCEHR supports single label key only" + self.label_key = self.label_keys[0] + self.mode = self.dataset.output_schema[self.label_key] + + self.embeddings = MambaEmbeddingsForCEHR( + vocab_size=vocab_size, + hidden_size=embedding_dim, + pad_token_id=pad_token_id, + type_vocab_size=type_vocab_size, + max_num_visits=max_num_visits, + time_embeddings_size=time_embeddings_size, + visit_order_size=visit_segment_vocab, + ) + self.blocks = nn.ModuleList( + [ + MambaBlock( + d_model=embedding_dim, + state_size=state_size, + conv_kernel=conv_kernel, + ) + for _ in range(num_layers) + ] + ) + self.dropout = nn.Dropout(dropout) + out_dim = self.get_output_size() + self.fc = nn.Linear(embedding_dim, out_dim) + self._forecasting_head: Optional[nn.Module] = None + + def forward_forecasting(self, **kwargs: Any) -> Optional[torch.Tensor]: + """Optional next-token / forecasting head (extension point; not implemented).""" + + return None + + def forward(self, **kwargs: Any) -> Dict[str, torch.Tensor]: + concept_ids = kwargs["concept_ids"].to(self.device).long() + token_type_ids = kwargs["token_type_ids"].to(self.device).long() + time_stamps = kwargs["time_stamps"].to(self.device).float() + ages = kwargs["ages"].to(self.device).float() + visit_orders = kwargs["visit_orders"].to(self.device).long() + visit_segments = kwargs["visit_segments"].to(self.device).long() + + x = self.embeddings( + input_ids=concept_ids, + token_type_ids_batch=token_type_ids, + time_stamps=time_stamps, + ages=ages, + visit_orders=visit_orders, + visit_segments=visit_segments, + ) + mask = concept_ids != self.pad_token_id + for blk in self.blocks: + x = blk(x) + pooled = get_rightmost_masked_timestep(x, mask) + logits = self.fc(self.dropout(pooled)) + y_true = kwargs[self.label_key].to(self.device).float() + if y_true.dim() == 1: + y_true = y_true.unsqueeze(-1) + loss = self.get_loss_function()(logits, y_true) + y_prob = self.prepare_y_prob(logits) + return { + "loss": loss, + "y_prob": y_prob, + "y_true": y_true, + "logit": logits, + } diff --git a/pyhealth/models/utils.py b/pyhealth/models/utils.py index 67edc010e..45cd6608d 100644 --- a/pyhealth/models/utils.py +++ b/pyhealth/models/utils.py @@ -44,3 +44,31 @@ def get_last_visit(hidden_states, mask): last_hidden_states = torch.gather(hidden_states, 1, last_visit) last_hidden_state = last_hidden_states[:, 0, :] return last_hidden_state + + +def get_rightmost_masked_timestep(hidden_states, mask): + """Gather hidden state at the last True position in ``mask`` per row. + + Unlike :func:`get_last_visit`, this does **not** assume valid tokens form a + contiguous prefix; it picks the maximum index where ``mask`` is True. + Use for MPF / CEHR layouts where padding can appear between boundary tokens. + + Args: + hidden_states: ``[batch, seq_len, hidden_size]``. + mask: ``[batch, seq_len]`` bool. + + Returns: + Tensor ``[batch, hidden_size]``. + """ + if mask is None: + return hidden_states[:, -1, :] + batch, seq_len, hidden = hidden_states.shape + device = hidden_states.device + idx = torch.arange(seq_len, device=device, dtype=torch.long).unsqueeze(0).expand( + batch, -1 + ) + idx_m = torch.where(mask, idx, torch.full_like(idx, -1)) + last_idx = idx_m.max(dim=1).values.clamp(min=0) + last_idx = last_idx.view(batch, 1, 1).expand(batch, 1, hidden) + gathered = torch.gather(hidden_states, 1, last_idx) + return gathered[:, 0, :] diff --git a/pyhealth/nlp/metrics.py b/pyhealth/nlp/metrics.py index 667a6665b..a61ec3a2d 100644 --- a/pyhealth/nlp/metrics.py +++ b/pyhealth/nlp/metrics.py @@ -353,13 +353,13 @@ class LevenshteinDistanceScoreMethod(ScoreMethod): """ @classmethod def _get_external_modules(cls: Type) -> Tuple[str, ...]: - return ('editdistance~=0.8.1',) + return ('rapidfuzz>=3.0.0',) def _score(self, meth: str, context: ScoreContext) -> Iterable[FloatScore]: - import editdistance + from rapidfuzz.distance import Levenshtein for s1, s2 in context.pairs: - val: int = editdistance.eval(s1, s2) + val: int = Levenshtein.distance(s1, s2) if self.normalize: text_len: int = max(len(s1), len(s2)) val = 1. - (val / text_len) diff --git a/pyhealth/processors/__init__.py b/pyhealth/processors/__init__.py index b48072270..4568a5ece 100644 --- a/pyhealth/processors/__init__.py +++ b/pyhealth/processors/__init__.py @@ -50,6 +50,7 @@ def get_processor(name: str): from .ignore_processor import IgnoreProcessor from .temporal_timeseries_processor import TemporalTimeseriesProcessor from .tuple_time_text_processor import TupleTimeTextProcessor +from .cehr_processor import CehrProcessor, ConceptVocab # Expose public API from .base_processor import ( @@ -79,4 +80,6 @@ def get_processor(name: str): "GraphProcessor", "AudioProcessor", "TupleTimeTextProcessor", + "CehrProcessor", + "ConceptVocab", ] diff --git a/pyhealth/processors/cehr_processor.py b/pyhealth/processors/cehr_processor.py new file mode 100644 index 000000000..9199f51d7 --- /dev/null +++ b/pyhealth/processors/cehr_processor.py @@ -0,0 +1,175 @@ +"""Concept vocabulary and CEHR feature processor for FHIR timelines. + +Public API +---------- +ConceptVocab + Token-to-dense-id mapping with PAD/UNK reserved at 0 and 1. JSON-serialisable. +ensure_special_tokens(vocab) + Add CEHR/MPF specials (````, ````, ````, ````) and + return their ids. +CehrProcessor + Standard :class:`~pyhealth.processors.FeatureProcessor` that maps a sample's + list of concept-key strings (already boundary-padded by the task) to a 1-D + ``torch.long`` tensor of token ids. Vocab growth happens inside the standard + ``SampleBuilder.fit(samples)`` loop -- no warm-up or freeze flag needed. + +The per-patient timeline-extraction helpers (`collect_cehr_timeline_events`, +`build_cehr_sequences`, `infer_mortality_label`, etc.) live with the task that +owns that logic: :mod:`pyhealth.tasks.mpf_clinical_prediction`. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, Iterable, List + +import orjson +import torch + +from . import register_processor +from .base_processor import FeatureProcessor + +DEFAULT_PAD = 0 +DEFAULT_UNK = 1 +PAD_TOKEN = "" +UNK_TOKEN = "" + +__all__ = [ + "DEFAULT_PAD", + "DEFAULT_UNK", + "PAD_TOKEN", + "UNK_TOKEN", + "ConceptVocab", + "ensure_special_tokens", + "CehrProcessor", +] + + +# --------------------------------------------------------------------------- +# Vocabulary +# --------------------------------------------------------------------------- + + +@dataclass +class ConceptVocab: + """Maps concept keys to dense ids with PAD/UNK reserved at 0 and 1.""" + + token_to_id: Dict[str, int] = field(default_factory=dict) + pad_id: int = DEFAULT_PAD + unk_id: int = DEFAULT_UNK + _next_id: int = 2 + + def __post_init__(self) -> None: + if not self.token_to_id: + self.token_to_id = {PAD_TOKEN: self.pad_id, UNK_TOKEN: self.unk_id} + self._next_id = 2 + + def add_token(self, key: str) -> int: + if key in self.token_to_id: + return self.token_to_id[key] + tid = self._next_id + self.token_to_id[key] = tid + self._next_id += 1 + return tid + + def __getitem__(self, key: str) -> int: + return self.token_to_id.get(key, self.unk_id) + + @property + def vocab_size(self) -> int: + return self._next_id + + def to_json(self) -> Dict[str, Any]: + return { + "token_to_id": self.token_to_id, + "next_id": self._next_id, + "pad_id": self.pad_id, + "unk_id": self.unk_id, + } + + @classmethod + def from_json(cls, data: Dict[str, Any]) -> "ConceptVocab": + pad_id = int(data.get("pad_id", DEFAULT_PAD)) + unk_id = int(data.get("unk_id", DEFAULT_UNK)) + vocab = cls(pad_id=pad_id, unk_id=unk_id) + loaded = dict(data.get("token_to_id") or {}) + if loaded: + vocab.token_to_id = loaded + vocab._next_id = int(data.get("next_id", max(loaded.values()) + 1)) + else: + vocab._next_id = int(data.get("next_id", 2)) + return vocab + + def save(self, path: str) -> None: + Path(path).parent.mkdir(parents=True, exist_ok=True) + Path(path).write_bytes(orjson.dumps(self.to_json(), option=orjson.OPT_SORT_KEYS)) + + @classmethod + def load(cls, path: str) -> "ConceptVocab": + return cls.from_json(orjson.loads(Path(path).read_bytes())) + + +def ensure_special_tokens(vocab: ConceptVocab) -> Dict[str, int]: + """Add EHRMamba/CEHR special tokens and return their ids.""" + return {name: vocab.add_token(name) for name in ("", "", "", "")} + + +# --------------------------------------------------------------------------- +# Processor +# --------------------------------------------------------------------------- + + +@register_processor("cehr") +class CehrProcessor(FeatureProcessor): + """Map a sample's list of concept-key strings to a 1-D LongTensor of ids. + + The task is expected to have already done all boundary-token insertion + (```` / ```` / ````) and left-padding with ````. This + processor's only state is a :class:`ConceptVocab`, grown during the + standard :meth:`~pyhealth.datasets.sample_dataset.SampleBuilder.fit` + pass over cached samples. + """ + + def __init__(self, max_len: int = 512) -> None: + self.vocab = ConceptVocab() + ensure_special_tokens(self.vocab) + self.max_len = max_len + + def fit(self, samples: Iterable[Dict[str, Any]], field: str) -> "CehrProcessor": + for sample in samples: + keys = sample.get(field) + if not keys: + continue + for key in keys: + if isinstance(key, str): + self.vocab.add_token(key) + return self + + def process(self, value: List[Any]) -> torch.Tensor: + ids = [ + self.vocab[k] if isinstance(k, str) else int(k) + for k in value + ] + return torch.tensor(ids, dtype=torch.long) + + def save(self, path: str) -> None: + self.vocab.save(path) + + def load(self, path: str) -> None: + self.vocab = ConceptVocab.load(path) + + def is_token(self) -> bool: + return True + + def schema(self) -> tuple[str, ...]: + return ("value",) + + def dim(self) -> tuple[int, ...]: + return (1,) + + def spatial(self) -> tuple[bool, ...]: + return (True,) + + def __repr__(self) -> str: + return f"CehrProcessor(max_len={self.max_len})" diff --git a/pyhealth/tasks/__init__.py b/pyhealth/tasks/__init__.py index 2140f23ed..cc95ef94e 100644 --- a/pyhealth/tasks/__init__.py +++ b/pyhealth/tasks/__init__.py @@ -74,3 +74,11 @@ VariantClassificationClinVar, ) from .patient_linkage_mimic3 import PatientLinkageMIMIC3Task + + +def __getattr__(name: str): + if name == "MPFClinicalPredictionTask": + from .mpf_clinical_prediction import MPFClinicalPredictionTask + + return MPFClinicalPredictionTask + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/pyhealth/tasks/mpf_clinical_prediction.py b/pyhealth/tasks/mpf_clinical_prediction.py new file mode 100644 index 000000000..f2a489035 --- /dev/null +++ b/pyhealth/tasks/mpf_clinical_prediction.py @@ -0,0 +1,315 @@ +"""Multitask Prompted Fine-tuning (MPF) clinical prediction on FHIR timelines. + +The task reads per-patient events via :meth:`pyhealth.data.Patient.get_events` +and :class:`~pyhealth.data.Event` attribute access (the standard PyHealth +idiom). It builds six aligned CEHR feature sequences, inserts MPF boundary +specials, and left-pads to ``max_len``. + +Concept-key → integer-id mapping happens later, inside the standard pipeline: +``SampleBuilder.fit`` walks the cached ``task_df.ld`` and fits a +:class:`~pyhealth.processors.CehrProcessor` on the ``concept_ids`` field; +that processor's vocab is then applied per sample by ``_proc_transform``. +The other five sequences are plain numeric lists handled by the standard +tensor processor. +""" + +from __future__ import annotations + +from datetime import datetime +from typing import Any, Dict, List, Optional, Tuple + +import torch + +import polars as pl + +from pyhealth.data import Event, Patient +from pyhealth.processors.cehr_processor import PAD_TOKEN + +from .base_task import BaseTask + +__all__ = [ + "EVENT_TYPE_TO_TOKEN_TYPE", + "MPFClinicalPredictionTask", + "collect_cehr_timeline_events", + "infer_mortality_label", +] + + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +EVENT_TYPE_TO_TOKEN_TYPE: Dict[str, int] = { + "encounter": 1, + "condition": 2, + "medication_request": 3, + "observation": 4, + "procedure": 5, +} + +_CLINICAL_EVENT_TYPES: Tuple[str, ...] = ( + "condition", + "observation", + "medication_request", + "procedure", +) + + +# --------------------------------------------------------------------------- +# Small pure helpers +# --------------------------------------------------------------------------- + + +def _deceased_boolean_column_means_dead(value: Any) -> bool: + """True only for an explicit ``"true"`` flag (not Python truthiness).""" + if value is None: + return False + return str(value).strip().lower() == "true" + + +def _encounter_concept_key(event: Any) -> str: + enc_class = getattr(event, "encounter_class", None) + if enc_class: + return f"encounter|{enc_class}" + return "encounter|unknown" + + +def _sequential_visit_idx_for_time( + event_time: Optional[datetime], + visit_encounters: List[Tuple[datetime, int]], +) -> int: + """Bucket an unlinked event into the nearest preceding encounter's index.""" + if not visit_encounters: + return 0 + if event_time is None: + return visit_encounters[-1][1] + chosen = visit_encounters[0][1] + for encounter_start, visit_idx in visit_encounters: + if encounter_start <= event_time: + chosen = visit_idx + else: + break + return chosen + + +def _birth_datetime_from_patient(patient: Patient) -> Optional[datetime]: + """Patient's birth date. + + The ``patient`` table's yaml entry declares ``timestamp: birth_date``, so + the Event's ``timestamp`` field is the birth date itself. + """ + events = patient.get_events(event_type="patient") + return events[0].timestamp if events else None + + +# --------------------------------------------------------------------------- +# Timeline extraction +# --------------------------------------------------------------------------- + + +def collect_cehr_timeline_events( + patient: Patient, +) -> List[Tuple[datetime, str, str, int]]: + """Collect ``(time, concept_key, event_type, visit_idx)`` tuples for one patient. + + Encounters define the visit boundaries. Clinical events that reference a + known encounter id are linked directly; events without a matching + encounter reference are bucketed into the chronologically nearest + preceding visit. + """ + # Only well-formed encounters (real id + non-null timestamp) define visit + # indices. We have to inspect the raw polars frame here: + # ``Event.__init__`` silently coerces ``timestamp=None`` to + # ``datetime.now()`` (data.py:43-45), so by the time we get back an Event + # we can no longer tell which encounters were timestamp-less. + encounters_df = patient.get_events(event_type="encounter", return_df=True) + valid_encounters = [ + Event.from_dict(row) + for row in encounters_df.filter( + pl.col("timestamp").is_not_null() + & pl.col("encounter/encounter_id").is_not_null() + ).iter_rows(named=True) + ] + + encounter_visit_idx: Dict[str, int] = {} + encounter_start_by_id: Dict[str, datetime] = {} + visit_encounters: List[Tuple[datetime, int]] = [] + for idx, enc in enumerate(valid_encounters): + enc_id = enc.encounter_id + encounter_visit_idx[enc_id] = idx + encounter_start_by_id[enc_id] = enc.timestamp + visit_encounters.append((enc.timestamp, idx)) + + events: List[Tuple[datetime, str, str, int]] = [] + unlinked: List[Tuple[Optional[datetime], str, str]] = [] + + for enc in valid_encounters: + events.append( + ( + enc.timestamp, + _encounter_concept_key(enc), + "encounter", + encounter_visit_idx[enc.encounter_id], + ) + ) + + for et in _CLINICAL_EVENT_TYPES: + for ev in patient.get_events(event_type=et): + concept_key = getattr(ev, "concept_key", None) or f"{et}|unknown" + enc_id = getattr(ev, "encounter_id", None) + # ``ev.timestamp`` is coerced (``None`` -> ``datetime.now()`` by + # ``Event.__init__``), so it can't reveal a missing time. The raw, + # null-aware signal is the ``event_time`` attribute (the yaml + # surfaces it for every clinical table); use it as the null sentinel + # and ``ev.timestamp`` for the already-parsed value. + t = ev.timestamp if getattr(ev, "event_time", None) is not None else None + if enc_id and enc_id in encounter_visit_idx: + if t is None: + t = encounter_start_by_id.get(enc_id) + if t is None: + continue + events.append((t, concept_key, et, encounter_visit_idx[enc_id])) + else: + unlinked.append((t, concept_key, et)) + + for t, concept_key, et in unlinked: + idx = _sequential_visit_idx_for_time(t, visit_encounters) + if t is None: + if not visit_encounters: + continue + # Use the start of the chosen visit; fall back to the latest encounter. + t = next( + (start for start, v_idx in visit_encounters if v_idx == idx), + visit_encounters[-1][0], + ) + events.append((t, concept_key, et, idx)) + + events.sort(key=lambda item: item[0]) + return events + + +# --------------------------------------------------------------------------- +# Label +# --------------------------------------------------------------------------- + + +def infer_mortality_label(patient: Patient) -> int: + """Heuristic binary mortality label from flattened patient rows.""" + for ev in patient.get_events(event_type="patient"): + if _deceased_boolean_column_means_dead(getattr(ev, "deceased_boolean", None)): + return 1 + if getattr(ev, "deceased_datetime", None): + return 1 + for ev in patient.get_events(event_type="condition"): + ck = (getattr(ev, "concept_key", None) or "").lower() + if any(token in ck for token in ("death", "deceased", "mortality")): + return 1 + return 0 + + +# --------------------------------------------------------------------------- +# Task +# --------------------------------------------------------------------------- + + +class MPFClinicalPredictionTask(BaseTask): + """Binary mortality prediction from FHIR CEHR sequences with optional MPF tokens. + + The task does timeline extraction and emits **raw** per-event lists, + including concept keys as strings. Tokenization is the + :class:`~pyhealth.processors.CehrProcessor`'s job, fit during the + standard ``SampleBuilder.fit(dataset)`` pass. + + Attributes: + max_len: Output sequence length (must be >= 2 for boundary tokens). + use_mpf: If True, prepend ```` to the sequence; else ````. + The closing ```` is always emitted. + """ + + task_name: str = "MPFClinicalPredictionFHIR" + output_schema: Dict[str, str] = {"label": "binary"} + + def __init__(self, max_len: int = 512, use_mpf: bool = True) -> None: + if max_len < 2: + raise ValueError("max_len must be >= 2 for MPF boundary tokens") + self.max_len = max_len + self.use_mpf = use_mpf + self.boundary_start = "" if use_mpf else "" + self.boundary_end = "" + self.input_schema: Dict[str, Any] = { + "concept_ids": ("cehr", {"max_len": max_len}), + "token_type_ids": ("tensor", {"dtype": torch.long}), + "time_stamps": ("tensor", {"dtype": torch.float32}), + "ages": ("tensor", {"dtype": torch.float32}), + "visit_orders": ("tensor", {"dtype": torch.long}), + "visit_segments": ("tensor", {"dtype": torch.long}), + } + + def __call__(self, patient: Patient) -> List[Dict[str, Any]]: + """Build one labeled sample dict per patient.""" + timeline = collect_cehr_timeline_events(patient) + birth = _birth_datetime_from_patient(patient) + + clinical_cap = self.max_len - 2 + tail = timeline[-clinical_cap:] if clinical_cap > 0 else [] + base_time = tail[0][0] if tail else None + + # Build the six aligned sequences in a single pass. + keys: List[str] = [self.boundary_start] + token_types: List[int] = [0] + time_stamps: List[float] = [0.0] + ages: List[float] = [0.0] + vis_o: List[int] = [0] + vis_s: List[int] = [0] + + for event_time, concept_key, event_type, visit_idx in tail: + time_delta = ( + float((event_time - base_time).total_seconds()) + if base_time is not None and event_time is not None + else 0.0 + ) + age_years = ( + (event_time - birth).days / 365.25 + if birth is not None and event_time is not None + else 0.0 + ) + keys.append(concept_key) + token_types.append(EVENT_TYPE_TO_TOKEN_TYPE.get(event_type, 0)) + time_stamps.append(time_delta) + ages.append(age_years) + vis_o.append(min(visit_idx, 511)) + vis_s.append(visit_idx % 2) + + keys.append(self.boundary_end) + token_types.append(0) + time_stamps.append(0.0) + ages.append(0.0) + vis_o.append(0) + vis_s.append(0) + + ml = self.max_len + keys = _left_pad(keys, ml, PAD_TOKEN) + token_types = _left_pad(token_types, ml, 0) + time_stamps = _left_pad(time_stamps, ml, 0.0) + ages = _left_pad(ages, ml, 0.0) + vis_o = _left_pad(vis_o, ml, 0) + vis_s = _left_pad(vis_s, ml, 0) + + return [ + { + "patient_id": patient.patient_id, + "concept_ids": keys, + "token_type_ids": token_types, + "time_stamps": time_stamps, + "ages": ages, + "visit_orders": vis_o, + "visit_segments": vis_s, + "label": infer_mortality_label(patient), + } + ] + + +def _left_pad(seq: List[Any], max_len: int, pad: Any) -> List[Any]: + if len(seq) >= max_len: + return seq[-max_len:] + return [pad] * (max_len - len(seq)) + seq diff --git a/pyproject.toml b/pyproject.toml index 98f88d47b..65e0e2757 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,7 @@ dependencies = [ "dask[complete]~=2025.11.0", "litdata~=0.2.59", "pyarrow~=22.0.0", + "orjson~=3.10", "narwhals~=2.13.0", "more-itertools~=10.8.0", "einops>=0.8.0", @@ -64,7 +65,7 @@ graph = [ "torch-geometric>=2.6.0", ] nlp = [ - "editdistance~=0.8.1", + "rapidfuzz>=3.0.0", "rouge_score~=0.1.2", "nltk~=3.9.1", ] diff --git a/tests/core/test_ehrmamba_cehr.py b/tests/core/test_ehrmamba_cehr.py new file mode 100644 index 000000000..81c995f28 --- /dev/null +++ b/tests/core/test_ehrmamba_cehr.py @@ -0,0 +1,126 @@ +import unittest + +import torch + +from pyhealth.datasets import create_sample_dataset, get_dataloader +from pyhealth.models import EHRMambaCEHR +from pyhealth.tasks.mpf_clinical_prediction import MPFClinicalPredictionTask + + +def _tiny_samples(seq: int = 16) -> tuple: + """Build hand-crafted samples in the new task's emitted format. + + ``concept_ids`` carries raw string tokens (``""`` / ``""`` / a + filler concept key); the ``CehrProcessor`` registered via the task's + ``input_schema`` does the string → integer-id mapping during + ``SampleBuilder.fit``. + """ + task = MPFClinicalPredictionTask(max_len=seq, use_mpf=True) + samples = [] + for lab in (0, 1): + samples.append( + { + "patient_id": f"p{lab}", + "visit_id": f"v{lab}", + "concept_ids": [""] + ["test|filler"] * (seq - 2) + [""], + "token_type_ids": [0] * seq, + "time_stamps": [0.0] * seq, + "ages": [50.0] * seq, + "visit_orders": [0] * seq, + "visit_segments": [0] * seq, + "label": lab, + } + ) + return samples, task + + +class TestEHRMambaCEHR(unittest.TestCase): + def test_readout_pools_rightmost_non_pad(self) -> None: + """MPF padding between tokens must not make pooling pick a pad position.""" + + from pyhealth.models.utils import ( + get_last_visit, + get_rightmost_masked_timestep, + ) + + h = torch.tensor([[[1.0, 0.0], [2.0, 0.0], [0.0, 0.0], [99.0, 0.0]]]) + m = torch.tensor([[True, True, False, True]]) + out = get_rightmost_masked_timestep(h, m) + self.assertTrue(torch.allclose(out[0], torch.tensor([99.0, 0.0]))) + wrong = get_last_visit(h, m) + self.assertFalse(torch.allclose(out[0], wrong[0])) + + def test_end_to_end_fhir_pipeline(self) -> None: + import tempfile + from pathlib import Path + + from pyhealth.datasets import MIMIC4FHIR, create_sample_dataset + from pyhealth.datasets import get_dataloader + + from tests.core.test_fhir_ndjson_fixtures import run_task, write_two_class_ndjson + + task = MPFClinicalPredictionTask(max_len=32, use_mpf=True) + with tempfile.TemporaryDirectory() as tmp: + write_two_class_ndjson(Path(tmp)) + ds = MIMIC4FHIR( + root=tmp, glob_pattern="*.ndjson", cache_dir=tmp + ) + samples = run_task(ds, task) + sample_ds = create_sample_dataset( + samples=samples, + input_schema=task.input_schema, + output_schema=task.output_schema, + dataset_name="fhir_test", + ) + vocab_size = sample_ds.input_processors["concept_ids"].vocab.vocab_size + model = EHRMambaCEHR( + dataset=sample_ds, + vocab_size=vocab_size, + embedding_dim=64, + num_layers=1, + ) + batch = next( + iter(get_dataloader(sample_ds, batch_size=2, shuffle=False)) + ) + out = model(**batch) + self.assertIn("loss", out) + out["loss"].backward() + + def test_forward_backward(self) -> None: + samples, task = _tiny_samples() + ds = create_sample_dataset( + samples=samples, + input_schema=task.input_schema, + output_schema=task.output_schema, + ) + vocab_size = ds.input_processors["concept_ids"].vocab.vocab_size + model = EHRMambaCEHR( + dataset=ds, + vocab_size=vocab_size, + embedding_dim=64, + num_layers=1, + state_size=8, + ) + batch = next(iter(get_dataloader(ds, batch_size=2, shuffle=False))) + out = model(**batch) + self.assertEqual(out["logit"].shape[0], 2) + out["loss"].backward() + + def test_eval_mode(self) -> None: + samples, task = _tiny_samples() + ds = create_sample_dataset( + samples=samples, + input_schema=task.input_schema, + output_schema=task.output_schema, + ) + vocab_size = ds.input_processors["concept_ids"].vocab.vocab_size + model = EHRMambaCEHR(dataset=ds, vocab_size=vocab_size, embedding_dim=32, num_layers=1) + model.eval() + with torch.no_grad(): + batch = next(iter(get_dataloader(ds, batch_size=2, shuffle=False))) + out = model(**batch) + self.assertIn("y_prob", out) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/core/test_fhir_dataset.py b/tests/core/test_fhir_dataset.py new file mode 100644 index 000000000..1f8557f9a --- /dev/null +++ b/tests/core/test_fhir_dataset.py @@ -0,0 +1,817 @@ +import gzip +import shutil +import tempfile +import unittest +from pathlib import Path +from typing import Dict, List, Tuple + +import orjson +import polars as pl + +from pyhealth.data import Patient +from pyhealth.datasets import MIMIC4FHIR +from pyhealth.datasets.fhir.utils import ( + flatten_resource, + load_resource_specs_from_yaml, +) +from pyhealth.processors.cehr_processor import ConceptVocab +from pyhealth.tasks.mpf_clinical_prediction import ( + MPFClinicalPredictionTask, + collect_cehr_timeline_events, + infer_mortality_label, +) + + +def _mimic4_specs(): + """Load the bundled MIMIC4 ResourceSpec registry from its YAML.""" + import yaml as _yaml + with open(MIMIC4FHIR.DEFAULT_CONFIG_PATH, encoding="utf-8") as _f: + return load_resource_specs_from_yaml(_yaml.safe_load(_f)) + + +_MIMIC4_SPECS = _mimic4_specs() + + +def _flatten_resource_to_table_row(resource): + """Flatten one resource via the MIMIC4 spec registry (test convenience).""" + return flatten_resource(resource, _MIMIC4_SPECS) + + +def _clinical_slice(sample: Dict[str, object]) -> Tuple[List[str], List[int], List[int]]: + """Drop ```` and the leading/trailing boundary tokens from a sample. + + Returns the per-event ``(concept_keys, visit_orders, visit_segments)`` + lists for the patient's clinical events only. + """ + keys = list(sample["concept_ids"]) # type: ignore[arg-type] + v_o = list(sample["visit_orders"]) # type: ignore[arg-type] + v_s = list(sample["visit_segments"]) # type: ignore[arg-type] + non_pad = [ + (k, o, s) for k, o, s in zip(keys, v_o, v_s) if k != "" + ] + # Strip leading boundary (/) and trailing . + middle = non_pad[1:-1] if len(non_pad) >= 2 else [] + return ( + [k for k, _, _ in middle], + [o for _, o, _ in middle], + [s for _, _, s in middle], + ) + +from tests.core.test_fhir_ndjson_fixtures import ( + ndjson_two_class_text, + write_one_patient_ndjson, + write_two_class_ndjson, +) + + +def _third_patient_loinc_resources() -> List[Dict[str, object]]: + return [ + { + "resourceType": "Patient", + "id": "p-synth-3", + "birthDate": "1960-01-01", + }, + { + "resourceType": "Encounter", + "id": "e3", + "subject": {"reference": "Patient/p-synth-3"}, + "period": {"start": "2020-08-01T10:00:00Z"}, + "class": {"code": "IMP"}, + }, + { + "resourceType": "Observation", + "id": "o3", + "subject": {"reference": "Patient/p-synth-3"}, + "encounter": {"reference": "Encounter/e3"}, + "effectiveDateTime": "2020-08-01T12:00:00Z", + "code": {"coding": [{"system": "http://loinc.org", "code": "999-9"}]}, + }, + ] + + +def write_two_class_plus_third_ndjson(directory: Path, *, name: str = "fixture.ndjson") -> Path: + lines = ndjson_two_class_text().strip().split("\n") + lines.extend(orjson.dumps(r).decode("utf-8") for r in _third_patient_loinc_resources()) + path = directory / name + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + return path + + +def _patient_from_rows(patient_id: str, rows: List[Dict[str, object]]) -> Patient: + """Build a Patient whose ``timestamp`` column is a real datetime, matching + the shape ``FHIRDataset.load_table`` produces in production. + + Production flat tables always carry a ``{event_type}/event_time`` column + (null when the source had no time), and ``timestamp`` is derived from it. + Mirror that here so ``Event`` attribute access sees ``event_time`` (the + null-aware signal the timeline uses): inject ``{event_type}/event_time`` = + the row's ``timestamp`` when not already set. + """ + rows = [dict(r) for r in rows] + for r in rows: + et = r.get("event_type") + if et and f"{et}/event_time" not in r: + r[f"{et}/event_time"] = r.get("timestamp") + df = pl.DataFrame(rows).with_columns( + pl.col("timestamp").str.to_datetime(strict=False) + ) + return Patient(patient_id=patient_id, data_source=df) + + +class TestDeceasedBooleanFlattening(unittest.TestCase): + def test_string_false_not_coerced_by_python_bool(self) -> None: + """Non-conformant ``\"false\"`` string must not become stored ``\"true\"``.""" + row = _flatten_resource_to_table_row( + { + "resourceType": "Patient", + "id": "p-str-false", + "deceasedBoolean": "false", + } + ) + self.assertIsNotNone(row) + _table, payload = row + self.assertEqual(payload.get("deceased_boolean"), "false") + + def test_string_true_parsed(self) -> None: + row = _flatten_resource_to_table_row( + { + "resourceType": "Patient", + "id": "p-str-true", + "deceasedBoolean": "true", + } + ) + self.assertIsNotNone(row) + self.assertEqual(row[1].get("deceased_boolean"), "true") + + def test_json_booleans_unchanged(self) -> None: + for raw, expected in ((True, "true"), (False, "false")): + with self.subTest(raw=raw): + row = _flatten_resource_to_table_row( + { + "resourceType": "Patient", + "id": "p-bool", + "deceasedBoolean": raw, + } + ) + self.assertIsNotNone(row) + self.assertEqual(row[1].get("deceased_boolean"), expected) + + def test_unknown_deceased_type_stored_as_none(self) -> None: + row = _flatten_resource_to_table_row( + { + "resourceType": "Patient", + "id": "p-garbage", + "deceasedBoolean": {"unexpected": "object"}, + } + ) + self.assertIsNotNone(row) + self.assertIsNone(row[1].get("deceased_boolean")) + + def test_infer_mortality_respects_string_false_row(self) -> None: + patient = _patient_from_rows( + "p1", + [ + { + "event_type": "patient", + "timestamp": "2020-01-01T00:00:00", + "patient/deceased_boolean": "false", + }, + ], + ) + self.assertEqual(infer_mortality_label(patient), 0) + + +class TestFHIRDataset(unittest.TestCase): + def test_concept_vocab_from_json_empty_token_to_id(self) -> None: + v = ConceptVocab.from_json({"token_to_id": {}}) + self.assertIn("", v.token_to_id) + self.assertIn("", v.token_to_id) + self.assertEqual(v._next_id, 2) + + def test_concept_vocab_from_json_empty_respects_next_id(self) -> None: + v = ConceptVocab.from_json({"token_to_id": {}, "next_id": 50}) + self.assertEqual(v._next_id, 50) + + def test_sorted_ndjson_files_accepts_sequence_and_dedupes(self) -> None: + from pyhealth.datasets.fhir.utils import sorted_ndjson_files + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "MimicPatient.ndjson.gz").write_text("x", encoding="utf-8") + (root / "MimicMedication.ndjson.gz").write_text("y", encoding="utf-8") + (root / "notes.txt").write_text("z", encoding="utf-8") + wide = sorted_ndjson_files(root, "**/*.ndjson.gz") + narrow = sorted_ndjson_files( + root, + ["MimicPatient*.ndjson.gz", "**/MimicPatient*.ndjson.gz"], + ) + self.assertEqual(len(wide), 2) + self.assertEqual(len(narrow), 1) + self.assertEqual(narrow[0].name, "MimicPatient.ndjson.gz") + + def test_dataset_accepts_glob_patterns_kwarg(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + write_one_patient_ndjson(Path(tmp)) + ds = MIMIC4FHIR( + root=tmp, glob_patterns=["*.ndjson"], cache_dir=tmp + ) + self.assertEqual(ds.glob_patterns, ["*.ndjson"]) + + def test_dataset_rejects_both_glob_kwargs(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + with self.assertRaises(ValueError): + MIMIC4FHIR( + root=tmp, + glob_pattern="*.ndjson", + glob_patterns=["*.ndjson"], + cache_dir=tmp, + ) + + def test_disk_ingest_gz_and_max_patients(self) -> None: + """gzip ingest path + ``max_patients`` cap, covered in one build. + + The heavier build/schema/set_task/pre_filter assertions now live in + ``TestFHIRSharedWorkflow`` (one shared build), so this is the only + ingest-variant build left in this class. + """ + with tempfile.TemporaryDirectory() as tmp: + gz_path = Path(tmp) / "fixture.ndjson.gz" + with gzip.open(gz_path, "wt", encoding="utf-8") as gz: + gz.write(ndjson_two_class_text()) + ds = MIMIC4FHIR(root=tmp, glob_pattern="*.ndjson.gz", max_patients=5) + self.assertEqual(len(ds.unique_patient_ids), 2) + + def test_encounter_reference_requires_exact_id(self) -> None: + patient = _patient_from_rows( + "p1", + [ + { + "patient_id": "p1", + "event_type": "patient", + "timestamp": None, + "patient/birth_date": "1950-01-01", + }, + { + "patient_id": "p1", + "event_type": "encounter", + "timestamp": "2020-06-01T10:00:00", + "encounter/encounter_id": "e1", + "encounter/encounter_class": "AMB", + }, + { + "patient_id": "p1", + "event_type": "encounter", + "timestamp": "2020-07-02T10:00:00", + "encounter/encounter_id": "e10", + "encounter/encounter_class": "IMP", + }, + { + "patient_id": "p1", + "event_type": "condition", + "timestamp": "2020-07-02T11:00:00", + "condition/encounter_id": "e10", + "condition/concept_key": "http://hl7.org/fhir/sid/icd-10-cm|I99", + }, + ], + ) + sample = MPFClinicalPredictionTask(max_len=64, use_mpf=True)(patient)[0] + self.assertEqual( + sample["concept_ids"].count("http://hl7.org/fhir/sid/icd-10-cm|I99"), + 1, + ) + + def test_unlinked_condition_emitted_once_with_two_encounters(self) -> None: + patient = _patient_from_rows( + "p1", + [ + { + "patient_id": "p1", + "event_type": "patient", + "timestamp": None, + "patient/birth_date": "1950-01-01", + }, + { + "patient_id": "p1", + "event_type": "encounter", + "timestamp": "2020-06-01T10:00:00", + "encounter/encounter_id": "ea", + "encounter/encounter_class": "AMB", + }, + { + "patient_id": "p1", + "event_type": "encounter", + "timestamp": "2020-07-01T10:00:00", + "encounter/encounter_id": "eb", + "encounter/encounter_class": "IMP", + }, + { + "patient_id": "p1", + "event_type": "condition", + "timestamp": "2020-06-15T12:00:00", + "condition/concept_key": "http://hl7.org/fhir/sid/icd-10-cm|Z00", + }, + ], + ) + sample = MPFClinicalPredictionTask(max_len=64, use_mpf=True)(patient)[0] + self.assertEqual( + sample["concept_ids"].count("http://hl7.org/fhir/sid/icd-10-cm|Z00"), + 1, + ) + + def test_max_len_two_keeps_only_boundary_tokens(self) -> None: + """``max_len=2`` leaves room for only the two boundary tokens; the + clinical timeline is truncated away. + """ + patient = _patient_from_rows( + "p1", + [ + { + "patient_id": "p1", + "event_type": "patient", + "timestamp": None, + "patient/birth_date": "1950-01-01", + }, + { + "patient_id": "p1", + "event_type": "encounter", + "timestamp": "2020-06-01T10:00:00", + "encounter/encounter_id": "e1", + "encounter/encounter_class": "AMB", + }, + { + "patient_id": "p1", + "event_type": "condition", + "timestamp": "2020-06-01T11:00:00", + "condition/encounter_id": "e1", + "condition/concept_key": "http://hl7.org/fhir/sid/icd-10-cm|I10", + }, + ], + ) + sample = MPFClinicalPredictionTask(max_len=2, use_mpf=True)(patient)[0] + self.assertEqual(sample["concept_ids"], ["", ""]) + self.assertEqual(sample["visit_segments"], [0, 0]) + + def test_visit_segments_alternate_by_visit_index(self) -> None: + patient = _patient_from_rows( + "p1", + [ + { + "patient_id": "p1", + "event_type": "patient", + "timestamp": None, + "patient/birth_date": "1950-01-01", + }, + { + "patient_id": "p1", + "event_type": "encounter", + "timestamp": "2020-06-01T10:00:00", + "encounter/encounter_id": "e0", + "encounter/encounter_class": "AMB", + }, + { + "patient_id": "p1", + "event_type": "condition", + "timestamp": "2020-06-01T11:00:00", + "condition/encounter_id": "e0", + "condition/concept_key": "http://hl7.org/fhir/sid/icd-10-cm|I10", + }, + { + "patient_id": "p1", + "event_type": "encounter", + "timestamp": "2020-07-01T10:00:00", + "encounter/encounter_id": "e1", + "encounter/encounter_class": "IMP", + }, + { + "patient_id": "p1", + "event_type": "condition", + "timestamp": "2020-07-01T11:00:00", + "condition/encounter_id": "e1", + "condition/concept_key": "http://hl7.org/fhir/sid/icd-10-cm|I20", + }, + ], + ) + sample = MPFClinicalPredictionTask(max_len=64, use_mpf=True)(patient)[0] + _, _, visit_segments = _clinical_slice(sample) + self.assertEqual(visit_segments, [0, 0, 1, 1]) + + def test_unlinked_visit_idx_matches_sequential_counter(self) -> None: + patient = _patient_from_rows( + "p1", + [ + { + "patient_id": "p1", + "event_type": "patient", + "timestamp": None, + "patient/birth_date": "1950-01-01", + }, + { + "patient_id": "p1", + "event_type": "encounter", + "timestamp": None, + "encounter/encounter_id": "e_bad", + "encounter/encounter_class": "AMB", + }, + { + "patient_id": "p1", + "event_type": "encounter", + "timestamp": "2020-03-01T10:00:00", + "encounter/encounter_id": "e_ok", + "encounter/encounter_class": "IMP", + }, + { + "patient_id": "p1", + "event_type": "condition", + "timestamp": "2020-03-05T11:00:00", + "condition/encounter_id": "e_ok", + "condition/concept_key": "http://hl7.org/fhir/sid/icd-10-cm|I10", + }, + { + "patient_id": "p1", + "event_type": "condition", + "timestamp": "2020-03-15T12:00:00", + "condition/concept_key": "http://hl7.org/fhir/sid/icd-10-cm|Z00", + }, + ], + ) + sample = MPFClinicalPredictionTask(max_len=64, use_mpf=True)(patient)[0] + keys = sample["concept_ids"] + i_link = keys.index("http://hl7.org/fhir/sid/icd-10-cm|I10") + i_free = keys.index("http://hl7.org/fhir/sid/icd-10-cm|Z00") + self.assertEqual(sample["visit_orders"][i_link], sample["visit_orders"][i_free]) + self.assertEqual(sample["visit_segments"][i_link], sample["visit_segments"][i_free]) + + def test_medication_request_uses_medication_codeable_concept(self) -> None: + patient = _patient_from_rows( + "p1", + [ + { + "patient_id": "p1", + "event_type": "patient", + "timestamp": None, + "patient/birth_date": "1950-01-01", + }, + { + "patient_id": "p1", + "event_type": "encounter", + "timestamp": "2020-06-01T10:00:00", + "encounter/encounter_id": "e1", + "encounter/encounter_class": "IMP", + }, + { + "patient_id": "p1", + "event_type": "medication_request", + "timestamp": "2020-06-01T11:00:00", + "medication_request/encounter_id": "e1", + "medication_request/concept_key": "http://www.nlm.nih.gov/research/umls/rxnorm|111", + }, + { + "patient_id": "p1", + "event_type": "medication_request", + "timestamp": "2020-06-01T12:00:00", + "medication_request/encounter_id": "e1", + "medication_request/concept_key": "http://www.nlm.nih.gov/research/umls/rxnorm|222", + }, + ], + ) + sample = MPFClinicalPredictionTask(max_len=64, use_mpf=True)(patient)[0] + keys = sample["concept_ids"] + ka = "http://www.nlm.nih.gov/research/umls/rxnorm|111" + kb = "http://www.nlm.nih.gov/research/umls/rxnorm|222" + self.assertEqual(keys.count(ka), 1) + self.assertEqual(keys.count(kb), 1) + + def test_medication_request_medication_reference_token(self) -> None: + patient = _patient_from_rows( + "p1", + [ + { + "patient_id": "p1", + "event_type": "patient", + "timestamp": None, + "patient/birth_date": "1950-01-01", + }, + { + "patient_id": "p1", + "event_type": "encounter", + "timestamp": "2020-06-01T10:00:00", + "encounter/encounter_id": "e1", + "encounter/encounter_class": "IMP", + }, + { + "patient_id": "p1", + "event_type": "medication_request", + "timestamp": "2020-06-01T11:00:00", + "medication_request/encounter_id": "e1", + "medication_request/concept_key": "MedicationRequest/reference|med-abc", + }, + ], + ) + sample = MPFClinicalPredictionTask(max_len=64, use_mpf=True)(patient)[0] + key = "MedicationRequest/reference|med-abc" + self.assertIn(key, sample["concept_ids"]) + self.assertEqual(sample["concept_ids"].count(key), 1) + + def test_collect_cehr_timeline_events_orders_by_timestamp(self) -> None: + patient = _patient_from_rows( + "p1", + [ + { + "patient_id": "p1", + "event_type": "patient", + "timestamp": None, + "patient/birth_date": "1950-01-01", + }, + { + "patient_id": "p1", + "event_type": "encounter", + "timestamp": "2020-06-01T10:00:00", + "encounter/encounter_id": "e1", + "encounter/encounter_class": "AMB", + }, + { + "patient_id": "p1", + "event_type": "condition", + "timestamp": "2020-06-01T11:00:00", + "condition/encounter_id": "e1", + "condition/concept_key": "a|1", + }, + { + "patient_id": "p1", + "event_type": "observation", + "timestamp": "2020-06-01T12:00:00", + "observation/encounter_id": "e1", + "observation/concept_key": "b|2", + }, + ], + ) + events = collect_cehr_timeline_events(patient) + self.assertEqual([event[1] for event in events], ["encounter|AMB", "a|1", "b|2"]) + + def test_timestampless_clinical_event_uses_encounter_start(self) -> None: + """A clinical event with no ``event_time`` that is linked to an encounter + must be placed at the encounter's start, not coerced to ~``now()``. + + ``Event.__init__`` coerces ``timestamp=None`` to ``datetime.now()``, so + the timeline relies on the raw ``event_time`` attribute as the null + sentinel; this locks that fallback. + """ + patient = _patient_from_rows( + "p1", + [ + { + "patient_id": "p1", + "event_type": "patient", + "timestamp": None, + "patient/birth_date": "1950-01-01", + }, + { + "patient_id": "p1", + "event_type": "encounter", + "timestamp": "2020-06-01T10:00:00", + "encounter/encounter_id": "e1", + "encounter/encounter_class": "IMP", + }, + { + "patient_id": "p1", + "event_type": "condition", + "timestamp": None, # timestamp-less clinical event + "condition/encounter_id": "e1", + "condition/concept_key": "http://hl7.org/fhir/sid/icd-10-cm|I10", + }, + ], + ) + events = collect_cehr_timeline_events(patient) + enc_time = next(t for t, _ck, et, _v in events if et == "encounter") + cond_time = next( + t + for t, ck, _et, _v in events + if ck == "http://hl7.org/fhir/sid/icd-10-cm|I10" + ) + self.assertEqual(cond_time, enc_time) + + def test_observation_effective_period_start_yields_event_time(self) -> None: + """Choice-type fix: an Observation carrying only ``effectivePeriod.start`` + (no ``effectiveDateTime``) must still resolve a non-null event_time. + The pre-refactor extractor silently dropped this variant. + """ + row = _flatten_resource_to_table_row( + { + "resourceType": "Observation", + "id": "o-period", + "subject": {"reference": "Patient/p"}, + "effectivePeriod": {"start": "2022-02-02T00:00:00Z"}, + "code": {"coding": [{"system": "http://loinc.org", "code": "1-1"}]}, + } + ) + self.assertIsNotNone(row) + self.assertEqual(row[1]["event_time"], "2022-02-02T00:00:00Z") + + def test_new_resource_type_via_registry_flows_through(self) -> None: + """A resource type absent from MIMIC4's specs flows end-to-end purely by + adding a YAML entry — no engine change. + + Also exercises the directly-usable generic ``FHIRDataset`` (whole + ingest contract authored in a single YAML, no subclass). + """ + from pyhealth.datasets import FHIRDataset + + resources = [ + {"resourceType": "Patient", "id": "imm-1", "birthDate": "1970-01-01"}, + { + "resourceType": "Immunization", + "id": "i1", + "patient": {"reference": "Patient/imm-1"}, + "occurrenceDateTime": "2021-01-01T00:00:00Z", + "vaccineCode": { + "coding": [{"system": "http://hl7.org/fhir/sid/cvx", "code": "208"}] + }, + }, + ] + config_yaml = ( + "version: test\n" + "resource_specs:\n" + " Patient:\n" + " table: patient\n" + " columns:\n" + " patient_id: { locate: [id], required: true }\n" + " birth_date: { locate: [birthDate] }\n" + " Immunization:\n" + " table: immunization\n" + " columns:\n" + " patient_id: { locate: [patient.reference], transform: ref_id, required: true }\n" + " resource_id: { locate: [id] }\n" + " encounter_id: { locate: [encounter.reference], transform: ref_id }\n" + " event_time: { locate: [occurrenceDateTime, recorded] }\n" + " concept_key: { locate: [vaccineCode], transform: coding_key }\n" + "tables:\n" + " patient:\n" + " file_path: patient.parquet\n" + " patient_id: patient_id\n" + " timestamp: birth_date\n" + " attributes: [birth_date]\n" + " immunization:\n" + " file_path: immunization.parquet\n" + " patient_id: patient_id\n" + " timestamp: event_time\n" + " attributes: [resource_id, encounter_id, event_time, concept_key]\n" + ) + with tempfile.TemporaryDirectory() as tmp: + tmpp = Path(tmp) + (tmpp / "fx.ndjson").write_text( + "\n".join(orjson.dumps(r).decode("utf-8") for r in resources) + "\n", + encoding="utf-8", + ) + cfg = tmpp / "immun.yaml" + cfg.write_text(config_yaml, encoding="utf-8") + ds = FHIRDataset( + root=str(tmpp), + config_path=str(cfg), + glob_pattern="*.ndjson", + cache_dir=str(tmpp), + ) + df = ds.global_event_df.collect(engine="streaming") + self.assertIn("immunization/concept_key", df.columns) + keys = ( + df.filter(pl.col("event_type") == "immunization")[ + "immunization/concept_key" + ] + .to_list() + ) + self.assertIn("http://hl7.org/fhir/sid/cvx|208", keys) + + def test_fhir_dataset_requires_specs(self) -> None: + """Bare ``FHIRDataset`` (no specs, no subclass) errors clearly.""" + from pyhealth.datasets import FHIRDataset + + with tempfile.TemporaryDirectory() as tmp: + with self.assertRaises(ValueError): + FHIRDataset(root=tmp, glob_pattern="*.ndjson", cache_dir=tmp) + + +class TestFHIRSharedWorkflow(unittest.TestCase): + """Build the dataset and run ``set_task`` ONCE, then assert over the shared + artifacts. Mirrors a realistic "ingest once, do many things" workflow and + keeps the suite fast: a single Dask build (plus one canonical ``set_task``) + shared by every assertion, instead of rebuilding per test. + """ + + @classmethod + def setUpClass(cls) -> None: + cls._tmp = tempfile.mkdtemp() + write_two_class_plus_third_ndjson(Path(cls._tmp)) + cls.ds = MIMIC4FHIR( + root=cls._tmp, glob_pattern="*.ndjson", cache_dir=cls._tmp, num_workers=1 + ) + # The single Dask build for the whole class. + cls.global_df = cls.ds.global_event_df.collect(engine="streaming") + # The canonical set_task (reuses the build above; no rebuild). + cls.sample_ds = cls.ds.set_task( + MPFClinicalPredictionTask(max_len=48, use_mpf=True), num_workers=1 + ) + cls.samples = sorted( + [cls.sample_ds[i] for i in range(len(cls.sample_ds))], + key=lambda s: s["patient_id"], + ) + + @classmethod + def tearDownClass(cls) -> None: + shutil.rmtree(cls._tmp, ignore_errors=True) + + def test_build_produces_expected_tables_and_schema(self) -> None: + """Flat parquet tables exist, and the global event frame has the + expected long-format + namespaced columns with a patient's events. + """ + prepared = self.ds.prepared_tables_dir + for name in ("patient", "encounter", "condition", "observation"): + self.assertTrue((prepared / f"{name}.parquet").is_file()) + for col in ( + "patient_id", + "timestamp", + "event_type", + "condition/concept_key", + "observation/concept_key", + "patient/deceased_boolean", + ): + self.assertIn(col, self.global_df.columns) + sub = self.global_df.filter(pl.col("patient_id") == "p-synth-1") + self.assertGreaterEqual(len(sub), 2) + + def test_set_task_builds_vocab(self) -> None: + vocab = self.sample_ds.input_processors["concept_ids"].vocab + self.assertGreater(vocab.vocab_size, 6) + + def test_set_task_produces_correct_samples(self) -> None: + self.assertEqual(len(self.samples), 3) + self.assertEqual( + {s["patient_id"] for s in self.samples}, + {"p-synth-1", "p-synth-2", "p-synth-3"}, + ) + for s in self.samples: + self.assertIn("concept_ids", s) + self.assertIn("label", s) + self.assertEqual({int(s["label"]) for s in self.samples}, {0, 1}) + + def test_cehr_sequence_shapes(self) -> None: + patient = self.ds.get_patient("p-synth-1") + sample = MPFClinicalPredictionTask(max_len=32, use_mpf=True)(patient)[0] + n = len(sample["concept_ids"]) + self.assertEqual(n, 32) + for key in ( + "token_type_ids", "time_stamps", "ages", "visit_orders", "visit_segments", + ): + self.assertEqual(len(sample[key]), n) + non_special = { + k for k in sample["concept_ids"] + if k not in ("", "", "", "") + } + self.assertGreater(len(non_special), 0) + + def test_mpf_pre_filter_single_patient_limits_effective_workers(self) -> None: + """Pre-filter yielding one patient caps effective_workers to 1 (the + formula is verified directly; a 1-patient ``set_task`` would raise on a + single label class). + """ + class OnePatientMPFTask(MPFClinicalPredictionTask): + def pre_filter(self, df: pl.LazyFrame) -> pl.LazyFrame: + return df.filter(pl.col("patient_id") == "p-synth-1") + + warmup_pids = ( + OnePatientMPFTask(max_len=48, use_mpf=True) + .pre_filter(self.ds.global_event_df) + .select("patient_id") + .unique() + .collect(engine="streaming") + .to_series() + .sort() + .to_list() + ) + self.assertEqual(warmup_pids, ["p-synth-1"]) + effective_workers = min(2, len(warmup_pids)) if warmup_pids else 1 + self.assertEqual(effective_workers, 1) + + def test_mpf_pre_filter_excludes_dropped_patients_from_vocab(self) -> None: + """A task ``pre_filter`` that drops a patient also drops their concept + keys from the fitted vocab. Reuses the shared build; a distinct + ``task_name`` keeps this run's sample cache separate from the canonical + ``set_task`` in setUpClass (identical params would otherwise collide on + the shared ``cache_dir``). + """ + class TwoPatientMPFTask(MPFClinicalPredictionTask): + task_name = "MPFClinicalPredictionFHIR_prefilter" + + def pre_filter(self, df: pl.LazyFrame) -> pl.LazyFrame: + return df.filter( + pl.col("patient_id").is_in(["p-synth-1", "p-synth-2"]) + ) + + sample_ds = self.ds.set_task( + TwoPatientMPFTask(max_len=48, use_mpf=True), num_workers=1 + ) + vocab = sample_ds.input_processors["concept_ids"].vocab + self.assertNotIn("http://loinc.org|999-9", vocab.token_to_id) + self.assertIn("http://loinc.org|789-0", vocab.token_to_id) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/core/test_fhir_ndjson_fixtures.py b/tests/core/test_fhir_ndjson_fixtures.py new file mode 100644 index 000000000..7311d4802 --- /dev/null +++ b/tests/core/test_fhir_ndjson_fixtures.py @@ -0,0 +1,110 @@ +"""NDJSON file bodies for :mod:`tests.core.test_fhir_dataset` (disk-only ingest).""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, List + +import orjson + + +# --------------------------------------------------------------------------- +# Synthetic in-memory FHIR resources +# --------------------------------------------------------------------------- + + +def _one_patient_resources() -> List[Dict[str, Any]]: + return [ + {"resourceType": "Patient", "id": "p-synth-1", "birthDate": "1950-01-01", "gender": "female"}, + { + "resourceType": "Encounter", + "id": "e1", + "subject": {"reference": "Patient/p-synth-1"}, + "period": {"start": "2020-06-01T10:00:00Z"}, + "class": {"code": "IMP"}, + }, + { + "resourceType": "Condition", + "id": "c1", + "subject": {"reference": "Patient/p-synth-1"}, + "encounter": {"reference": "Encounter/e1"}, + "code": {"coding": [{"system": "http://hl7.org/fhir/sid/icd-10-cm", "code": "I10"}]}, + "onsetDateTime": "2020-06-01T11:00:00Z", + }, + ] + + +def _two_patient_resources() -> List[Dict[str, Any]]: + return [ + *_one_patient_resources(), + {"resourceType": "Patient", "id": "p-synth-2", "birthDate": "1940-05-05", "deceasedBoolean": True}, + { + "resourceType": "Encounter", + "id": "e-dead", + "subject": {"reference": "Patient/p-synth-2"}, + "period": {"start": "2020-07-01T10:00:00Z"}, + "class": {"code": "IMP"}, + }, + { + "resourceType": "Observation", + "id": "o-dead", + "subject": {"reference": "Patient/p-synth-2"}, + "encounter": {"reference": "Encounter/e-dead"}, + "effectiveDateTime": "2020-07-01T12:00:00Z", + "code": {"coding": [{"system": "http://loinc.org", "code": "789-0"}]}, + }, + ] + + +# --------------------------------------------------------------------------- +# Text serialisers +# --------------------------------------------------------------------------- + + +def ndjson_one_patient_text() -> str: + return "\n".join(orjson.dumps(r).decode("utf-8") for r in _one_patient_resources()) + "\n" + + +def ndjson_two_class_text() -> str: + return "\n".join(orjson.dumps(r).decode("utf-8") for r in _two_patient_resources()) + "\n" + + +# --------------------------------------------------------------------------- +# Disk writers +# --------------------------------------------------------------------------- + + +def write_two_class_ndjson(directory: Path, *, name: str = "fixture.ndjson") -> Path: + path = directory / name + path.write_text(ndjson_two_class_text(), encoding="utf-8") + return path + + +def write_one_patient_ndjson(directory: Path, *, name: str = "fixture.ndjson") -> Path: + path = directory / name + path.write_text(ndjson_one_patient_text(), encoding="utf-8") + return path + + +# --------------------------------------------------------------------------- +# Shared test helper +# --------------------------------------------------------------------------- + + +def run_task(ds: Any, task: Any) -> List[Dict[str, Any]]: + """Run *task* over every patient in *ds* without the LitData caching pipeline. + + This helper mirrors the direct-iteration path that the old + ``FHIRDataset.gather_samples`` provided. It is intentionally kept + here (the shared fixture module) so all FHIR test files can import it + rather than each maintaining their own copy. + + Args: + ds: A :class:`~pyhealth.datasets.FHIRDataset` instance whose + ``global_event_df`` has already been built. + task: A :class:`~pyhealth.tasks.MPFClinicalPredictionTask` instance. + + Returns: + Flat list of sample dicts, one per patient. + """ + return [s for patient in ds.iter_patients() for s in task(patient)] diff --git a/tests/core/test_mpf_task.py b/tests/core/test_mpf_task.py new file mode 100644 index 000000000..a1b261719 --- /dev/null +++ b/tests/core/test_mpf_task.py @@ -0,0 +1,99 @@ +import shutil +import tempfile +import unittest +from pathlib import Path + +from pyhealth.datasets import MIMIC4FHIR +from pyhealth.processors.cehr_processor import PAD_TOKEN +from pyhealth.tasks.mpf_clinical_prediction import MPFClinicalPredictionTask + +from tests.core.test_fhir_ndjson_fixtures import ( + run_task, + write_two_class_ndjson, +) + + +class TestMPFClinicalPredictionTask(unittest.TestCase): + """Verifies the task emits boundary-marker strings at the expected + positions in its raw output. Vocab → integer-id mapping is the + ``CehrProcessor``'s job and is exercised separately via the standard + ``SampleBuilder.fit`` pipeline. + """ + + @classmethod + def setUpClass(cls) -> None: + cls._tmp = tempfile.mkdtemp() + write_two_class_ndjson(Path(cls._tmp)) + # One shared build for the whole class; tests reuse it via run_task + # (run_task just applies the task to cached patients — no rebuild). + cls.ds = MIMIC4FHIR( + root=cls._tmp, glob_pattern="*.ndjson", cache_dir=cls._tmp + ) + + @classmethod + def tearDownClass(cls) -> None: + shutil.rmtree(cls._tmp, ignore_errors=True) + + def test_max_len_validation(self) -> None: + with self.assertRaises(ValueError): + MPFClinicalPredictionTask(max_len=1, use_mpf=True) + + def test_mpf_sets_boundary_tokens(self) -> None: + task = MPFClinicalPredictionTask(max_len=32, use_mpf=True) + samples = run_task(self.ds, task) + self.assertGreater(len(samples), 0) + keys = samples[0]["concept_ids"] + first = next(i for i, x in enumerate(keys) if x != PAD_TOKEN) + last_nz = next( + i for i in range(len(keys) - 1, -1, -1) if keys[i] != PAD_TOKEN + ) + self.assertEqual(keys[first], "") + self.assertEqual(keys[last_nz], "") + self.assertEqual(keys[-1], "") + + def test_no_mpf_uses_cls_reg(self) -> None: + task = MPFClinicalPredictionTask(max_len=32, use_mpf=False) + samples = run_task(self.ds, task) + keys = samples[0]["concept_ids"] + first = next(i for i, x in enumerate(keys) if x != PAD_TOKEN) + last_nz = next( + i for i in range(len(keys) - 1, -1, -1) if keys[i] != PAD_TOKEN + ) + self.assertEqual(keys[first], "") + self.assertEqual(keys[last_nz], "") + self.assertEqual(keys[-1], "") + + def test_schema_keys(self) -> None: + task = MPFClinicalPredictionTask(max_len=16, use_mpf=True) + samples = run_task(self.ds, task) + for k in task.input_schema: + self.assertIn(k, samples[0]) + self.assertIn("label", samples[0]) + + def test_max_len_two_keeps_boundary_tokens(self) -> None: + """At ``max_len=2`` the sequence is exactly ``[, ]``.""" + + task = MPFClinicalPredictionTask(max_len=2, use_mpf=True) + samples = run_task(self.ds, task) + for s in samples: + keys = s["concept_ids"] + self.assertEqual(len(keys), 2) + self.assertEqual(keys[0], "") + self.assertEqual(keys[1], "") + + def test_fixed_length_alignment(self) -> None: + """All six per-event lists must be the same length (max_len).""" + + task = MPFClinicalPredictionTask(max_len=24, use_mpf=True) + samples = run_task(self.ds, task) + for s in samples: + self.assertEqual(len(s["concept_ids"]), 24) + self.assertEqual(len(s["token_type_ids"]), 24) + self.assertEqual(len(s["time_stamps"]), 24) + self.assertEqual(len(s["ages"]), 24) + self.assertEqual(len(s["visit_orders"]), 24) + self.assertEqual(len(s["visit_segments"]), 24) + + +if __name__ == "__main__": + unittest.main() From 17338b4fed44294e8efa1c5a179ab3fe22187c29 Mon Sep 17 00:00:00 2001 From: Sean Nian Date: Sat, 13 Jun 2026 21:01:22 -0700 Subject: [PATCH 14/61] [DL4H Sp26] Add MedFuse model for multi-modal EHR + chest X-ray fusion (#1003) * Add MedFuse multimodal model for EHR+CXR fusion * Address PR feedback: docstring, mask doc, test cleanup, paper citations --- docs/api/models.rst | 1 + docs/api/models/pyhealth.models.MedFuse.rst | 24 ++ examples/mimic4_mortality_medfuse.py | 385 +++++++++++++++++ pyhealth/models/__init__.py | 1 + pyhealth/models/medfuse.py | 456 ++++++++++++++++++++ tests/core/test_medfuse.py | 201 +++++++++ 6 files changed, 1068 insertions(+) create mode 100644 docs/api/models/pyhealth.models.MedFuse.rst create mode 100644 examples/mimic4_mortality_medfuse.py create mode 100644 pyhealth/models/medfuse.py create mode 100644 tests/core/test_medfuse.py diff --git a/docs/api/models.rst b/docs/api/models.rst index d695b20db..4187c123b 100644 --- a/docs/api/models.rst +++ b/docs/api/models.rst @@ -196,6 +196,7 @@ API Reference models/pyhealth.models.ConCare models/pyhealth.models.Agent models/pyhealth.models.GRASP + models/pyhealth.models.MedFuse models/pyhealth.models.MedLink models/pyhealth.models.TCN models/pyhealth.models.TFMTokenizer diff --git a/docs/api/models/pyhealth.models.MedFuse.rst b/docs/api/models/pyhealth.models.MedFuse.rst new file mode 100644 index 000000000..c39bcb374 --- /dev/null +++ b/docs/api/models/pyhealth.models.MedFuse.rst @@ -0,0 +1,24 @@ +pyhealth.models.MedFuse +======================= + +Overview +-------- + +MedFuse is an LSTM-based multi-modal fusion model for combining clinical +time-series (EHR) and chest X-ray (CXR) representations. + +Reference: Hayat et al., "MedFuse: Multi-modal fusion with clinical +time-series data and chest X-ray images." MLHC 2022. + +API Reference +------------- + +.. autoclass:: pyhealth.models.MedFuseLayer + :members: + :undoc-members: + :show-inheritance: + +.. autoclass:: pyhealth.models.MedFuse + :members: + :undoc-members: + :show-inheritance: diff --git a/examples/mimic4_mortality_medfuse.py b/examples/mimic4_mortality_medfuse.py new file mode 100644 index 000000000..265022c80 --- /dev/null +++ b/examples/mimic4_mortality_medfuse.py @@ -0,0 +1,385 @@ +"""MedFuse Ablation Study for In-Hospital Mortality Prediction. + +This script demonstrates MedFuse with varying hyperparameters on synthetic +mortality-like data to validate model behavior quickly. + +Paper: + Hayat et al. "MedFuse: Multi-modal fusion with clinical time-series data + and chest X-ray images." MLHC 2022. + +Ablations: + 1. EHR hidden dim: 64, 128, 256, 512 + 2. Fusion hidden dim: 128, 256, 512 + 3. Dropout: 0.0, 0.3, 0.5, 0.7 + 4. Learning rate: 1e-5, 1e-4, 1e-3 + 5. Modality: EHR-only vs EHR+CXR + +The script is intentionally lightweight by default: + - Synthetic data only + - Small dataset + - Few epochs +""" + +from __future__ import annotations + +import argparse +import math +import random +from typing import Dict, List, Tuple + +import numpy as np +import torch +from sklearn.metrics import average_precision_score, roc_auc_score + +from pyhealth.datasets import create_sample_dataset, get_dataloader +from pyhealth.models import MedFuse + + +def set_seed(seed: int) -> None: + """Sets all random seeds for reproducibility.""" + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + + +def make_synthetic_samples( + num_samples: int, + seq_len: int, + ehr_dim: int, + image_size: int, + seed: int, +) -> List[Dict[str, object]]: + """Builds synthetic samples with weakly learnable mortality signal.""" + generator = torch.Generator().manual_seed(seed) + samples: List[Dict[str, object]] = [] + + for index in range(num_samples): + ehr = torch.randn(seq_len, ehr_dim, generator=generator) + cxr = torch.randn(3, image_size, image_size, generator=generator) + + score = 0.7 * float(ehr[:, 0].mean()) + score += 0.3 * float(cxr.mean()) + score += float(torch.randn(1, generator=generator)) * 0.1 + label = 1 if score > 0 else 0 + + samples.append( + { + "patient_id": f"patient-{index}", + "visit_id": f"visit-{index}", + "ehr": ehr.tolist(), + "cxr": cxr.tolist(), + "label": label, + } + ) + + return samples + + +def build_datasets( + train_size: int, + val_size: int, + seq_len: int, + ehr_dim: int, + image_size: int, + seed: int, +): + """Creates synthetic train/validation datasets.""" + train_samples = make_synthetic_samples( + num_samples=train_size, + seq_len=seq_len, + ehr_dim=ehr_dim, + image_size=image_size, + seed=seed, + ) + val_samples = make_synthetic_samples( + num_samples=val_size, + seq_len=seq_len, + ehr_dim=ehr_dim, + image_size=image_size, + seed=seed + 1, + ) + + input_schema = {"ehr": "tensor", "cxr": "tensor"} + output_schema = {"label": "binary"} + + train_dataset = create_sample_dataset( + samples=train_samples, + input_schema=input_schema, + output_schema=output_schema, + dataset_name="medfuse_ablation_train", + ) + val_dataset = create_sample_dataset( + samples=val_samples, + input_schema=input_schema, + output_schema=output_schema, + dataset_name="medfuse_ablation_val", + ) + return train_dataset, val_dataset + + +def compute_binary_metrics( + y_true: List[float], + y_prob: List[float], +) -> Dict[str, float]: + """Computes AUROC and AUPRC with robust fallbacks.""" + metrics = { + "auroc": math.nan, + "auprc": math.nan, + } + + if len(set(y_true)) > 1: + metrics["auroc"] = float(roc_auc_score(y_true, y_prob)) + if any(y_true): + metrics["auprc"] = float(average_precision_score(y_true, y_prob)) + + return metrics + + +def train_and_evaluate( + train_dataset, + val_dataset, + use_cxr: bool, + ehr_hidden_dim: int, + fusion_hidden_dim: int, + dropout: float, + learning_rate: float, + epochs: int, + batch_size: int, + cxr_backbone: str, +) -> Dict[str, float]: + """Trains one MedFuse configuration and reports validation metrics.""" + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model = MedFuse( + dataset=train_dataset, + ehr_feature_key="ehr", + cxr_feature_key="cxr", + cxr_mask_key="cxr_mask", + ehr_hidden_dim=ehr_hidden_dim, + ehr_num_layers=2, + cxr_backbone=cxr_backbone, + cxr_pretrained=False, + fusion_hidden_dim=fusion_hidden_dim, + projection_dim=ehr_hidden_dim, + dropout=dropout, + ) + model.to(device) + + optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) + train_loader = get_dataloader(train_dataset, batch_size=batch_size, shuffle=True) + val_loader = get_dataloader(val_dataset, batch_size=batch_size, shuffle=False) + + for _ in range(epochs): + model.train() + for batch in train_loader: + optimizer.zero_grad() + model_inputs = { + "ehr": batch["ehr"].to(device), + "label": batch["label"].to(device), + } + if use_cxr: + model_inputs["cxr"] = batch["cxr"].to(device) + + outputs = model(**model_inputs) + outputs["loss"].backward() + optimizer.step() + + model.eval() + all_probs: List[float] = [] + all_true: List[float] = [] + + with torch.no_grad(): + for batch in val_loader: + model_inputs = {"ehr": batch["ehr"].to(device)} + if use_cxr: + model_inputs["cxr"] = batch["cxr"].to(device) + + outputs = model(**model_inputs) + all_probs.extend(outputs["y_prob"].view(-1).cpu().tolist()) + all_true.extend(batch["label"].view(-1).cpu().tolist()) + + return compute_binary_metrics(all_true, all_probs) + + +def best_config(rows: List[Tuple[str, Dict[str, float]]]) -> Tuple[str, float]: + """Returns the setting name and AUROC of the best configuration.""" + best_name = rows[0][0] + best_auroc = -1.0 + for name, metrics in rows: + auroc = metrics["auroc"] + if not math.isnan(auroc) and auroc > best_auroc: + best_auroc = auroc + best_name = name + return best_name, best_auroc + + +def format_metric(value: float) -> str: + """Formats metric values for table output.""" + if math.isnan(value): + return "nan" + return f"{value:.4f}" + + +def print_table(title: str, rows: List[Tuple[str, Dict[str, float]]]) -> None: + """Prints a compact ablation result table.""" + print(f"\n{title}") + print("| setting | AUROC | AUPRC |") + print("|---------|-------|-------|") + for setting, metrics in rows: + auroc = format_metric(metrics["auroc"]) + auprc = format_metric(metrics["auprc"]) + print(f"| {setting} | {auroc} | {auprc} |") + + +def run_ablation_study(args: argparse.Namespace) -> None: + """Runs all required MedFuse ablations.""" + set_seed(args.seed) + train_dataset, val_dataset = build_datasets( + train_size=args.train_size, + val_size=args.val_size, + seq_len=args.seq_len, + ehr_dim=args.ehr_dim, + image_size=args.image_size, + seed=args.seed, + ) + + base_config = { + "use_cxr": True, + "ehr_hidden_dim": 128, + "fusion_hidden_dim": 256, + "dropout": 0.3, + "learning_rate": 1e-4, + "epochs": args.epochs, + "batch_size": args.batch_size, + "cxr_backbone": args.cxr_backbone, + } + + hidden_results: List[Tuple[str, Dict[str, float]]] = [] + for hidden_dim in [64, 128, 256, 512]: + metrics = train_and_evaluate( + train_dataset=train_dataset, + val_dataset=val_dataset, + use_cxr=True, + ehr_hidden_dim=hidden_dim, + fusion_hidden_dim=base_config["fusion_hidden_dim"], + dropout=base_config["dropout"], + learning_rate=base_config["learning_rate"], + epochs=base_config["epochs"], + batch_size=base_config["batch_size"], + cxr_backbone=base_config["cxr_backbone"], + ) + hidden_results.append((f"ehr_hidden_dim={hidden_dim}", metrics)) + + fusion_results: List[Tuple[str, Dict[str, float]]] = [] + for fusion_dim in [128, 256, 512]: + metrics = train_and_evaluate( + train_dataset=train_dataset, + val_dataset=val_dataset, + use_cxr=True, + ehr_hidden_dim=base_config["ehr_hidden_dim"], + fusion_hidden_dim=fusion_dim, + dropout=base_config["dropout"], + learning_rate=base_config["learning_rate"], + epochs=base_config["epochs"], + batch_size=base_config["batch_size"], + cxr_backbone=base_config["cxr_backbone"], + ) + fusion_results.append((f"fusion_hidden_dim={fusion_dim}", metrics)) + + dropout_results: List[Tuple[str, Dict[str, float]]] = [] + for dropout_value in [0.0, 0.3, 0.5, 0.7]: + metrics = train_and_evaluate( + train_dataset=train_dataset, + val_dataset=val_dataset, + use_cxr=True, + ehr_hidden_dim=base_config["ehr_hidden_dim"], + fusion_hidden_dim=base_config["fusion_hidden_dim"], + dropout=dropout_value, + learning_rate=base_config["learning_rate"], + epochs=base_config["epochs"], + batch_size=base_config["batch_size"], + cxr_backbone=base_config["cxr_backbone"], + ) + dropout_results.append((f"dropout={dropout_value}", metrics)) + + lr_results: List[Tuple[str, Dict[str, float]]] = [] + for lr in [1e-5, 1e-4, 1e-3]: + metrics = train_and_evaluate( + train_dataset=train_dataset, + val_dataset=val_dataset, + use_cxr=True, + ehr_hidden_dim=base_config["ehr_hidden_dim"], + fusion_hidden_dim=base_config["fusion_hidden_dim"], + dropout=base_config["dropout"], + learning_rate=lr, + epochs=base_config["epochs"], + batch_size=base_config["batch_size"], + cxr_backbone=base_config["cxr_backbone"], + ) + lr_results.append((f"learning_rate={lr:.0e}", metrics)) + + modality_results: List[Tuple[str, Dict[str, float]]] = [] + for use_cxr in [False, True]: + metrics = train_and_evaluate( + train_dataset=train_dataset, + val_dataset=val_dataset, + use_cxr=use_cxr, + ehr_hidden_dim=base_config["ehr_hidden_dim"], + fusion_hidden_dim=base_config["fusion_hidden_dim"], + dropout=base_config["dropout"], + learning_rate=base_config["learning_rate"], + epochs=base_config["epochs"], + batch_size=base_config["batch_size"], + cxr_backbone=base_config["cxr_backbone"], + ) + name = "EHR+CXR" if use_cxr else "EHR-only" + modality_results.append((name, metrics)) + + all_ablations = [ + ("EHR Hidden Dim Ablation", hidden_results), + ("Fusion Hidden Dim Ablation", fusion_results), + ("Dropout Ablation", dropout_results), + ("Learning Rate Ablation", lr_results), + ("Modality Ablation", modality_results), + ] + + for title, rows in all_ablations: + print_table(title, rows) + + print("\nFindings (auto-generated from results above):") + for title, rows in all_ablations: + name, auroc = best_config(rows) + print(f"- {title}: best config = {name} (AUROC={format_metric(auroc)})") + + print( + "\nNote: These results are on small synthetic data with minimal training " + "(default 32 train samples, 1 epoch). They validate that the model is " + "sensitive to hyperparameter choices but should not be interpreted as " + "representative of real-world performance. For reference, Hayat et al. " + "(2022) report the following for in-hospital mortality on MIMIC-IV + " + "MIMIC-CXR:\n" + " - Table 2 (paired EHR+CXR test set): MedFuse (OPTIMAL) 0.865 AUROC " + "/ 0.594 AUPRC, vs. Unified (Hayat et al., 2021a) 0.835 / 0.495.\n" + " - Table 3 (partial test set): MedFuse (OPTIMAL) 0.874 AUROC / 0.567 " + "AUPRC." + ) + + +def parse_args() -> argparse.Namespace: + """Parses CLI arguments.""" + parser = argparse.ArgumentParser( + description="Run MedFuse synthetic mortality ablation study." + ) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--epochs", type=int, default=1) + parser.add_argument("--batch-size", type=int, default=8) + parser.add_argument("--train-size", type=int, default=32) + parser.add_argument("--val-size", type=int, default=16) + parser.add_argument("--seq-len", type=int, default=5) + parser.add_argument("--ehr-dim", type=int, default=10) + parser.add_argument("--image-size", type=int, default=32) + parser.add_argument("--cxr-backbone", type=str, default="resnet18") + return parser.parse_args() + + +if __name__ == "__main__": + run_ablation_study(parse_args()) \ No newline at end of file diff --git a/pyhealth/models/__init__.py b/pyhealth/models/__init__.py index a4fe5dc85..2f30ae673 100644 --- a/pyhealth/models/__init__.py +++ b/pyhealth/models/__init__.py @@ -16,6 +16,7 @@ from .graph_torchvision_model import Graph_TorchvisionModel from .graphcare import GraphCare from .grasp import GRASP, GRASPLayer +from .medfuse import MedFuse, MedFuseLayer from .medlink import MedLink from .micron import MICRON, MICRONLayer from .mlp import MLP diff --git a/pyhealth/models/medfuse.py b/pyhealth/models/medfuse.py new file mode 100644 index 000000000..8e9f8e112 --- /dev/null +++ b/pyhealth/models/medfuse.py @@ -0,0 +1,456 @@ +# Authors: Sean Nian, Tony Hong, Yaqi Qiao +# Description: MedFuse model implementation for PyHealth. + +from __future__ import annotations + +from typing import Dict, Optional, Tuple, cast + +import torch +import torch.nn as nn +import torch.nn.utils.rnn as rnn_utils + +from pyhealth.datasets import SampleDataset +from pyhealth.models.base_model import BaseModel + + +class MedFuseLayer(nn.Module): + """MedFuse fusion layer. + + Fuses EHR time-series and chest X-ray (CXR) representations using an + LSTM-based sequential fusion strategy that supports missing CXR modality. + Follows Paper §3.2, Eq. 4: the fusion LSTM consumes the token sequence + ``v_fusion = [v_ehr, v*_cxr]`` (length 1 when CXR is missing). The EHR + representation is the first input token, not an initial hidden state. + + Paper: + Hayat, N., Geras, K. J., & Shamout, F. E. (2022). + MedFuse: Multi-modal fusion with clinical time-series data and chest + X-ray images. MLHC 2022. + + Args: + ehr_input_dim: Dimension of EHR features at each timestep. + ehr_hidden_dim: Hidden dimension of the EHR LSTM encoder. + Default is 256. + ehr_num_layers: Number of stacked LSTM layers for EHR. + Default is 2. + cxr_backbone: ResNet variant for CXR encoder. + Default is ``"resnet34"``. + cxr_pretrained: Whether to use pretrained CXR encoder weights. + Default is True. + fusion_hidden_dim: Hidden dimension of fusion LSTM. + Default is 512. + projection_dim: Dimension to project CXR features to. + Must match ``ehr_hidden_dim``. + dropout: Dropout rate. + num_labels: Number of output labels. + """ + + SUPPORTED_CXR_BACKBONES = ("resnet18", "resnet34", "resnet50") + + def __init__( + self, + ehr_input_dim: int, + ehr_hidden_dim: int = 256, + ehr_num_layers: int = 2, + cxr_backbone: str = "resnet34", + cxr_pretrained: bool = True, + fusion_hidden_dim: int = 512, + projection_dim: int = 256, + dropout: float = 0.5, + num_labels: int = 1, + ) -> None: + super().__init__() + if projection_dim != ehr_hidden_dim: + raise ValueError( + "projection_dim must match ehr_hidden_dim for fusion sequencing." + ) + if cxr_backbone not in self.SUPPORTED_CXR_BACKBONES: + raise ValueError( + f"Unsupported cxr_backbone: {cxr_backbone}. " + f"Supported values: {self.SUPPORTED_CXR_BACKBONES}." + ) + + self.ehr_encoder = nn.LSTM( + input_size=ehr_input_dim, + hidden_size=ehr_hidden_dim, + num_layers=ehr_num_layers, + batch_first=True, + dropout=dropout if ehr_num_layers > 1 else 0.0, + ) + + self.cxr_encoder, cxr_feature_dim = self._build_cxr_encoder( + cxr_backbone=cxr_backbone, + cxr_pretrained=cxr_pretrained, + ) + self.projection = nn.Linear(cxr_feature_dim, projection_dim) + + self.fusion_lstm = nn.LSTM( + input_size=projection_dim, + hidden_size=fusion_hidden_dim, + num_layers=1, + batch_first=True, + ) + self.dropout_layer = nn.Dropout(dropout) + self.classifier = nn.Linear(fusion_hidden_dim, num_labels) + + def _build_cxr_encoder( + self, + cxr_backbone: str, + cxr_pretrained: bool, + ) -> Tuple[nn.Module, int]: + """Builds the CXR backbone and returns encoder + feature dimension.""" + try: + import torchvision.models as tv_models + except ImportError as exc: + raise ImportError( + "torchvision is required to use MedFuse CXR backbones." + ) from exc + + constructor = getattr(tv_models, cxr_backbone) + + if cxr_pretrained: + try: + weights = tv_models.get_model_weights(cxr_backbone).DEFAULT + backbone = constructor(weights=weights) + except Exception: + backbone = constructor(pretrained=True) + else: + try: + backbone = constructor(weights=None) + except TypeError: + backbone = constructor(pretrained=False) + + if not hasattr(backbone, "fc") or not isinstance(backbone.fc, nn.Linear): + raise ValueError( + f"Backbone {cxr_backbone} must expose a linear `fc` layer." + ) + + feature_dim = backbone.fc.in_features + backbone.fc = nn.Identity() + return backbone, feature_dim + + def _encode_cxr(self, cxr_input: torch.Tensor) -> torch.Tensor: + """Encodes CXR images and projects them to the fusion dimension.""" + if cxr_input.dim() != 4: + raise ValueError( + "cxr_input must have shape [batch, channels, height, width]." + ) + + if cxr_input.size(1) == 1: + cxr_input = cxr_input.repeat(1, 3, 1, 1) + elif cxr_input.size(1) != 3: + raise ValueError("cxr_input must have 1 or 3 channels.") + + cxr_features = self.cxr_encoder(cxr_input) + if cxr_features.dim() > 2: + cxr_features = torch.flatten(cxr_features, start_dim=1) + + projected = self.projection(cxr_features) + return projected + + def forward( + self, + ehr_input: torch.Tensor, + cxr_input: Optional[torch.Tensor] = None, + cxr_mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Runs MedFuse fusion. + + Args: + ehr_input: EHR tensor of shape ``[batch, seq_len, ehr_input_dim]``. + cxr_input: Optional CXR tensor of shape + ``[batch, channels, height, width]``. + cxr_mask: Optional tensor of shape ``[batch]`` indicating CXR + availability (1 = present, 0 = missing). + + Returns: + Logits tensor of shape ``[batch, num_labels]``. + """ + if ehr_input.dim() != 3: + raise ValueError( + "ehr_input must have shape [batch, seq_len, ehr_input_dim]." + ) + + _, (ehr_hidden, _) = self.ehr_encoder(ehr_input) + ehr_repr = self.dropout_layer(ehr_hidden[-1]) + + batch_size = ehr_repr.size(0) + sequence_lengths = torch.ones( + batch_size, + dtype=torch.long, + device=ehr_input.device, + ) + fusion_tokens = [ehr_repr.unsqueeze(1)] + + if cxr_input is not None: + if cxr_input.size(0) != batch_size: + raise ValueError( + "cxr_input batch size must match ehr_input batch size." + ) + + if cxr_mask is None: + cxr_present_mask = torch.ones( + batch_size, + dtype=torch.bool, + device=ehr_input.device, + ) + else: + cxr_present_mask = cxr_mask.to(ehr_input.device).view(-1).bool() + if cxr_present_mask.numel() != batch_size: + raise ValueError("cxr_mask must have shape [batch].") + + cxr_projected = torch.zeros( + batch_size, + self.projection.out_features, + dtype=ehr_repr.dtype, + device=ehr_input.device, + ) + + if cxr_present_mask.any(): + present_cxr = cxr_input[cxr_present_mask] + present_projected = self._encode_cxr(present_cxr) + cxr_projected[cxr_present_mask] = present_projected + sequence_lengths[cxr_present_mask] = 2 + + fusion_tokens.append(cxr_projected.unsqueeze(1)) + + fusion_sequence = torch.cat(fusion_tokens, dim=1) + packed_sequence = rnn_utils.pack_padded_sequence( + fusion_sequence, + lengths=sequence_lengths.cpu(), + batch_first=True, + enforce_sorted=False, + ) + _, (fusion_hidden, _) = self.fusion_lstm(packed_sequence) + fused_repr = self.dropout_layer(fusion_hidden[-1]) + logits = self.classifier(fused_repr) + return logits + + +class MedFuse(BaseModel): + """MedFuse model for multi-modal clinical prediction. + + This model fuses clinical time-series (EHR) and chest X-ray (CXR) + representations using an LSTM-based fusion module. It handles missing CXR + via variable sequence lengths, where each sample contributes either: + + - ``[v_ehr]`` (EHR only), or + - ``[v_ehr, v_cxr]`` (EHR + CXR). + + Paper: + Hayat, N., Geras, K. J., & Shamout, F. E. (2022). + MedFuse: Multi-modal fusion with clinical time-series data and chest + X-ray images. MLHC 2022. + + Args: + dataset: Sample dataset used for model configuration. + ehr_feature_key: Input key for EHR tensor. Default is ``"ehr"``. + cxr_feature_key: Input key for CXR tensor. Default is ``"cxr"``. + cxr_mask_key: Optional input key for per-sample CXR availability mask. + Default is ``"cxr_mask"``. To surface the mask through the + standard dataloader, include ``"cxr_mask": "tensor"`` in your + dataset's ``input_schema`` with per-sample availability values + (1 = CXR present, 0 = missing). + ehr_hidden_dim: Hidden dimension for EHR encoder. + ehr_num_layers: Number of EHR LSTM layers. + cxr_backbone: CXR ResNet backbone. Default is ``"resnet34"``. + cxr_pretrained: Whether to use pretrained CXR backbone. + fusion_hidden_dim: Hidden dimension for fusion LSTM. + projection_dim: CXR projection output dimension. + dropout: Dropout rate. + + Examples: + >>> from pyhealth.datasets import create_sample_dataset, get_dataloader + >>> from pyhealth.models import MedFuse + >>> samples = [ + ... { + ... "patient_id": "p0", + ... "visit_id": "v0", + ... "ehr": torch.randn(5, 10).tolist(), + ... "cxr": torch.randn(3, 32, 32).tolist(), + ... "label": 1, + ... }, + ... { + ... "patient_id": "p1", + ... "visit_id": "v1", + ... "ehr": torch.randn(5, 10).tolist(), + ... "cxr": torch.randn(3, 32, 32).tolist(), + ... "label": 0, + ... }, + ... ] + >>> dataset = create_sample_dataset( + ... samples=samples, + ... input_schema={"ehr": "tensor", "cxr": "tensor"}, + ... output_schema={"label": "binary"}, + ... ) + >>> loader = get_dataloader(dataset, batch_size=2) + >>> model = MedFuse(dataset=dataset, cxr_pretrained=False) + >>> batch = next(iter(loader)) + >>> output = model(**batch) + >>> output["y_prob"].shape + torch.Size([2, 1]) + """ + + def __init__( + self, + dataset: SampleDataset, + ehr_feature_key: str = "ehr", + cxr_feature_key: str = "cxr", + cxr_mask_key: Optional[str] = "cxr_mask", + ehr_hidden_dim: int = 256, + ehr_num_layers: int = 2, + cxr_backbone: str = "resnet34", + cxr_pretrained: bool = True, + fusion_hidden_dim: int = 512, + projection_dim: int = 256, + dropout: float = 0.5, + ) -> None: + super().__init__(dataset=dataset) + + if len(self.label_keys) != 1: + raise ValueError("MedFuse supports exactly one label key.") + + if ehr_feature_key not in self.feature_keys: + raise ValueError( + f"ehr_feature_key '{ehr_feature_key}' not found in dataset " + f"features: {self.feature_keys}." + ) + + self.ehr_feature_key = ehr_feature_key + self.cxr_feature_key = cxr_feature_key + self.cxr_mask_key = cxr_mask_key + self.label_key = self.label_keys[0] + self.mode = self._resolve_mode(self.dataset.output_schema[self.label_key]) + + self.has_cxr_feature = cxr_feature_key in self.feature_keys + self.ehr_input_dim = self._infer_ehr_input_dim(ehr_feature_key) + + output_size = self.get_output_size() + self.layer = MedFuseLayer( + ehr_input_dim=self.ehr_input_dim, + ehr_hidden_dim=ehr_hidden_dim, + ehr_num_layers=ehr_num_layers, + cxr_backbone=cxr_backbone, + cxr_pretrained=cxr_pretrained, + fusion_hidden_dim=fusion_hidden_dim, + projection_dim=projection_dim, + dropout=dropout, + num_labels=output_size, + ) + + def _extract_feature_value( + self, + feature_key: str, + feature: torch.Tensor | tuple[torch.Tensor, ...], + ) -> torch.Tensor: + """Extracts the semantic ``value`` tensor from a feature payload.""" + if isinstance(feature, torch.Tensor): + return feature + + if feature_key not in self.dataset.input_processors: + raise ValueError( + f"Feature '{feature_key}' is not defined in dataset processors." + ) + + schema = self.dataset.input_processors[feature_key].schema() + if "value" in schema: + value = feature[schema.index("value")] + if isinstance(value, torch.Tensor): + return value + + raise ValueError( + f"Feature '{feature_key}' must provide a tensor value in its schema." + ) + + def _infer_ehr_input_dim(self, ehr_feature_key: str) -> int: + """Infers EHR input feature dimension from processed dataset samples.""" + for sample in self.dataset: + if ehr_feature_key not in sample: + continue + feature = sample[ehr_feature_key] + if isinstance(feature, tuple): + value = self._extract_feature_value(ehr_feature_key, feature) + elif isinstance(feature, torch.Tensor): + value = feature + else: + value = torch.as_tensor(feature) + + if value.dim() >= 2: + return int(value.shape[-1]) + + raise ValueError( + "Unable to infer EHR input dimension. Ensure EHR feature tensors " + "have shape [seq_len, feature_dim]." + ) + + def forward( + self, + **kwargs: torch.Tensor | tuple[torch.Tensor, ...], + ) -> Dict[str, torch.Tensor]: + """Forward propagation. + + Args: + **kwargs: Model inputs. Expected keys include: + - ``ehr_feature_key`` (required): EHR tensor. + - ``cxr_feature_key`` (optional): CXR tensor. + - ``cxr_mask_key`` (optional): CXR availability mask. + - ``label_key`` (optional): Label tensor for loss computation. + + Returns: + A dictionary containing: + - ``logit``: Raw logits. + - ``y_prob``: Predicted probabilities. + - ``loss``: Loss tensor when labels are provided. + - ``y_true``: Ground-truth labels when provided. + """ + if self.ehr_feature_key not in kwargs: + raise ValueError( + f"Missing required EHR feature key: '{self.ehr_feature_key}'." + ) + + ehr_input = self._extract_feature_value( + self.ehr_feature_key, + cast(torch.Tensor | tuple[torch.Tensor, ...], kwargs[self.ehr_feature_key]), + ) + ehr_input = ehr_input.to(self.device).float() + + cxr_input: Optional[torch.Tensor] = None + if self.cxr_feature_key in kwargs: + cxr_input = self._extract_feature_value( + self.cxr_feature_key, + cast( + torch.Tensor | tuple[torch.Tensor, ...], + kwargs[self.cxr_feature_key], + ), + ) + cxr_input = cxr_input.to(self.device).float() + + cxr_mask: Optional[torch.Tensor] = None + if self.cxr_mask_key is not None and self.cxr_mask_key in kwargs: + raw_mask = kwargs[self.cxr_mask_key] + if isinstance(raw_mask, tuple): + raise ValueError("cxr_mask must be provided as a tensor.") + if isinstance(raw_mask, torch.Tensor): + cxr_mask = raw_mask.to(self.device) + else: + cxr_mask = torch.as_tensor(raw_mask, device=self.device) + + logits = self.layer( + ehr_input=ehr_input, + cxr_input=cxr_input, + cxr_mask=cxr_mask, + ) + y_prob = self.prepare_y_prob(logits) + + results: Dict[str, torch.Tensor] = { + "logit": logits, + "y_prob": y_prob, + } + + if self.label_key in kwargs: + y_true = cast(torch.Tensor, kwargs[self.label_key]).to(self.device) + loss = self.get_loss_function()(logits, y_true) + results["loss"] = loss + results["y_true"] = y_true + + return results diff --git a/tests/core/test_medfuse.py b/tests/core/test_medfuse.py new file mode 100644 index 000000000..514463909 --- /dev/null +++ b/tests/core/test_medfuse.py @@ -0,0 +1,201 @@ +import unittest + +import torch +import torch.nn as nn + +from pyhealth.datasets import create_sample_dataset, get_dataloader +from pyhealth.models import MedFuse, MedFuseLayer + +try: + import torchvision # noqa: F401 + + HAS_TORCHVISION = True +except ImportError: + HAS_TORCHVISION = False + + +@unittest.skipUnless(HAS_TORCHVISION, "torchvision is required for MedFuse tests") +class TestMedFuse(unittest.TestCase): + """Test cases for the MedFuse model.""" + + @classmethod + def setUpClass(cls) -> None: + cls.binary_dataset = cls._create_binary_dataset() + cls.multilabel_dataset = cls._create_multilabel_dataset() + + @staticmethod + def _create_binary_dataset(): + generator = torch.Generator().manual_seed(7) + samples = [] + for index in range(4): + samples.append( + { + "patient_id": f"patient-{index}", + "visit_id": f"visit-{index}", + "ehr": torch.randn(5, 10, generator=generator).tolist(), + "cxr": torch.randn(3, 32, 32, generator=generator).tolist(), + "label": index % 2, + } + ) + + return create_sample_dataset( + samples=samples, + input_schema={"ehr": "tensor", "cxr": "tensor"}, + output_schema={"label": "binary"}, + dataset_name="medfuse_binary_test", + ) + + @staticmethod + def _create_multilabel_dataset(): + generator = torch.Generator().manual_seed(17) + labels = [[0], [1], [0, 1], []] + samples = [] + for index, label in enumerate(labels): + samples.append( + { + "patient_id": f"patient-multi-{index}", + "visit_id": f"visit-multi-{index}", + "ehr": torch.randn(5, 10, generator=generator).tolist(), + "cxr": torch.randn(3, 32, 32, generator=generator).tolist(), + "label": label, + } + ) + + return create_sample_dataset( + samples=samples, + input_schema={"ehr": "tensor", "cxr": "tensor"}, + output_schema={"label": "multilabel"}, + dataset_name="medfuse_multilabel_test", + ) + + @staticmethod + def _build_model(dataset): + model = MedFuse( + dataset=dataset, + ehr_feature_key="ehr", + cxr_feature_key="cxr", + cxr_mask_key="cxr_mask", + ehr_hidden_dim=8, + ehr_num_layers=1, + cxr_backbone="resnet18", + cxr_pretrained=False, + fusion_hidden_dim=16, + projection_dim=8, + dropout=0.0, + ) + # Speed up unit tests: keep MedFuse API/logic but replace heavyweight + # image encoder compute with a tiny synthetic module. + projection_in_features = model.layer.projection.in_features + model.layer.cxr_encoder = nn.Sequential( + nn.Conv2d(3, 8, kernel_size=3, stride=1, padding=1), + nn.ReLU(), + nn.AdaptiveAvgPool2d((1, 1)), + nn.Flatten(start_dim=1), + nn.Linear(8, projection_in_features), + ) + return model + + @staticmethod + def _next_batch(dataset): + loader = get_dataloader(dataset, batch_size=2, shuffle=False) + return next(iter(loader)) + + def test_instantiation(self): + """Model can be created with valid args.""" + model = self._build_model(self.binary_dataset) + self.assertIsInstance(model, MedFuse) + self.assertIsInstance(model.layer, MedFuseLayer) + self.assertEqual(model.ehr_input_dim, 10) + + def test_forward_pass_both_modalities(self): + """Forward pass works with both EHR and CXR input.""" + model = self._build_model(self.binary_dataset) + batch = self._next_batch(self.binary_dataset) + + with torch.no_grad(): + output = model(**batch) + + self.assertIn("loss", output) + self.assertIn("y_prob", output) + self.assertIn("y_true", output) + self.assertIn("logit", output) + + def test_forward_pass_ehr_only(self): + """Forward pass works with EHR only (missing CXR).""" + model = self._build_model(self.binary_dataset) + batch = self._next_batch(self.binary_dataset) + + with torch.no_grad(): + output = model(ehr=batch["ehr"], label=batch["label"]) + + self.assertIn("loss", output) + self.assertIn("y_prob", output) + self.assertEqual(output["logit"].shape, (2, 1)) + + def test_output_shape(self): + """Output tensor has correct shape [batch_size, num_labels].""" + model = self._build_model(self.binary_dataset) + batch = self._next_batch(self.binary_dataset) + + with torch.no_grad(): + output = model(**batch) + + self.assertEqual(output["logit"].shape, (2, 1)) + self.assertEqual(output["y_prob"].shape, (2, 1)) + + def test_gradient_computation(self): + """Gradients flow through the entire network.""" + model = self._build_model(self.binary_dataset) + batch = self._next_batch(self.binary_dataset) + + output = model(**batch) + loss = output["loss"] + loss.backward() + + for name, parameter in model.named_parameters(): + # _dummy_param is a 0-element BaseModel helper, skip grad check + if name == "_dummy_param": + continue + if parameter.requires_grad: + self.assertIsNotNone(parameter.grad) + + def test_missing_modality_robustness(self): + """Model produces valid output for mixed modality availability.""" + model = self._build_model(self.binary_dataset) + batch = self._next_batch(self.binary_dataset) + cxr_mask = torch.tensor([1, 0], dtype=torch.long) + + with torch.no_grad(): + output = model( + ehr=batch["ehr"], + cxr=batch["cxr"], + cxr_mask=cxr_mask, + label=batch["label"], + ) + + self.assertEqual(output["logit"].shape, (2, 1)) + self.assertFalse(torch.isnan(output["y_prob"]).any().item()) + + def test_binary_vs_multilabel_mode(self): + """Model works in both binary and multilabel modes.""" + binary_model = self._build_model(self.binary_dataset) + binary_batch = self._next_batch(self.binary_dataset) + + with torch.no_grad(): + binary_output = binary_model(**binary_batch) + + self.assertEqual(binary_output["logit"].shape[1], 1) + + multilabel_model = self._build_model(self.multilabel_dataset) + multilabel_batch = self._next_batch(self.multilabel_dataset) + + with torch.no_grad(): + multilabel_output = multilabel_model(**multilabel_batch) + + self.assertEqual(multilabel_output["logit"].shape[1], 2) + self.assertTrue(torch.all(multilabel_output["y_prob"] >= 0).item()) + self.assertTrue(torch.all(multilabel_output["y_prob"] <= 1).item()) + + +if __name__ == "__main__": + unittest.main() From 820e5eef39e2893cf71ad27a0d6c63df7c5e40f0 Mon Sep 17 00:00:00 2001 From: Felipe Bonchristiano <47675223+fbonc@users.noreply.github.com> Date: Sun, 28 Jun 2026 15:15:25 -0300 Subject: [PATCH 15/61] Add attention rollout interpretability method (Abnar & Zuidema 2020) (#1158) * Attention Rollout skeleton * attention_rollout.py done * tests/core/test_attention_rollout.py done * attention rollout integrated into example scripts * attention rollout docs * attention rollout docs/interpret/pyhealth.interpret.methods.attention_rollout.rst added * Stip trailing whitespace and rename example keys to rollout * attention rollout: doc and style changes * attention rollout: module docstring header --- docs/api/interpret.rst | 2 + ...th.interpret.methods.attention_rollout.rst | 84 +++++ docs/why_pyhealth.rst | 2 +- .../dka_stageattn_mimic4_interpret.py | 1 + .../dka_transformer_mimic4_interpret.py | 1 + .../los_stageattn_mimic4_interpret.py | 1 + .../los_transformer_mimic4_interpret.py | 1 + .../mp_stageattn_mimic4_interpret.py | 1 + .../mp_transformer_mimic4_interpret.py | 1 + pyhealth/interpret/methods/__init__.py | 2 + .../interpret/methods/attention_rollout.py | 285 ++++++++++++++++ tests/core/test_attention_rollout.py | 311 ++++++++++++++++++ 12 files changed, 691 insertions(+), 1 deletion(-) create mode 100644 docs/api/interpret/pyhealth.interpret.methods.attention_rollout.rst create mode 100644 pyhealth/interpret/methods/attention_rollout.py create mode 100644 tests/core/test_attention_rollout.py diff --git a/docs/api/interpret.rst b/docs/api/interpret.rst index 747f05b56..d121d6ddb 100644 --- a/docs/api/interpret.rst +++ b/docs/api/interpret.rst @@ -57,6 +57,7 @@ New to interpretability in PyHealth? Check out these complete examples: - Train a ViT model on COVID-19 chest X-ray classification - Use CheferRelevance for gradient-weighted attention attribution - Visualize which image patches contribute to predictions + **LIME Example:** - ``examples/lime_stagenet_mimic4.py`` - Demonstrates LIME (Local Interpretable Model-agnostic Explanations) for StageNet mortality prediction. Shows how to: @@ -78,6 +79,7 @@ Attribution Methods interpret/pyhealth.interpret.methods.gim interpret/pyhealth.interpret.methods.basic_gradient interpret/pyhealth.interpret.methods.chefer + interpret/pyhealth.interpret.methods.attention_rollout interpret/pyhealth.interpret.methods.deeplift interpret/pyhealth.interpret.methods.integrated_gradients interpret/pyhealth.interpret.methods.shap diff --git a/docs/api/interpret/pyhealth.interpret.methods.attention_rollout.rst b/docs/api/interpret/pyhealth.interpret.methods.attention_rollout.rst new file mode 100644 index 000000000..35875f3d5 --- /dev/null +++ b/docs/api/interpret/pyhealth.interpret.methods.attention_rollout.rst @@ -0,0 +1,84 @@ +pyhealth.interpret.methods.attention_rollout +============================================= + +Overview +-------- + +Attention Rollout provides token-level relevance scores for Transformer models +in PyHealth. It quantifies how attention propagates information across layers by +composing the per-layer attention matrices (with a residual-connection +correction), yielding a single importance score per input token (e.g. diagnosis +codes, procedure codes, medications) for a given patient sample. + +Unlike :class:`~pyhealth.interpret.methods.CheferRelevance`, which is +gradient-weighted and **class-specific**, attention rollout is **forward-pass +only**, **gradient-free**, and **class-agnostic**: it explains how information +flows through the attention mechanism independent of any target class. It serves +as the standard baseline that gradient-based attention methods are compared +against, and complements Chefer rather than replacing it. + +This method is particularly useful for: + +- **Clinical decision support**: Understanding which medical codes drove a particular prediction +- **Model debugging**: Identifying whether the model attends to clinically meaningful features +- **Feature importance**: Ranking tokens by how much attention flows to them +- **Trust and transparency**: Providing interpretable, class-agnostic explanations for model predictions + +The implementation follows the paper by Abnar & Zuidema (2020): "Quantifying +Attention Flow in Transformers" (https://arxiv.org/abs/2005.00928). + +Key Features +------------ + +- **Multi-modal support**: Works with multiple feature types (conditions, procedures, drugs, labs, etc.) +- **Gradient-free**: Computed from a single forward pass; no backward pass is used in the attribution math +- **Class-agnostic**: Independent of the predicted/target class (``target_class_idx`` is accepted but ignored) +- **Layer-wise composition**: Composes per-layer attention as ``rollout = Â_L @ ... @ Â_1`` with the residual correction ``Â = 0.5 * (A + I)`` +- **Distribution over tokens**: Because each ``Â`` is row-stochastic, so is their product; per-token relevance sums to 1 (before the input-shape expansion) +- **Model-agnostic by duck-typing**: Works with any model exposing the attention-readout methods ``set_attention_hooks``, ``get_attention_layers`` and ``get_relevance_tensor`` (currently :class:`~pyhealth.models.Transformer` and :class:`~pyhealth.models.StageAttentionNet`), not just one named model + +Usage Notes +----------- + +1. **Batch size**: For interpretability, use ``batch_size=1`` to get per-sample explanations. +2. **Do not wrap in** ``torch.no_grad()``: Although rollout is gradient-free in its math, the shared attention-readout plumbing registers a gradient hook on the attention tensors during the forward pass, so calling ``attribute(**batch)`` inside ``torch.no_grad()`` raises a ``RuntimeError``. Call it under the default (grad-enabled) context; no backward pass is performed. +3. **Model compatibility**: Works with any model that exposes ``set_attention_hooks``, ``get_attention_layers`` and ``get_relevance_tensor`` — not restricted to the Transformer. Incompatible models raise ``TypeError`` at construction. +4. **Class specification**: ``target_class_idx`` is accepted for API compatibility but ignored, since rollout is class-agnostic. + +Quick Start +----------- + +.. code-block:: python + + from pyhealth.models import Transformer + from pyhealth.interpret.methods import AttentionRollout + from pyhealth.datasets import get_dataloader + + # Assume you have a trained transformer model and dataset + model = Transformer(dataset=sample_dataset, ...) + # ... train the model ... + + # Create interpretability object + rollout = AttentionRollout(model) + + # Get a test sample (batch_size=1) + test_loader = get_dataloader(test_dataset, batch_size=1, shuffle=False) + batch = next(iter(test_loader)) + + # Compute attributions (target_class_idx is accepted but ignored) + scores = rollout.attribute(**batch) + + # Analyze results + for feature_key, attribution in scores.items(): + print(f"{feature_key}: {attribution.shape}") + top_tokens = attribution[0].topk(5).indices + print(f" Top 5 most relevant tokens: {top_tokens}") + +API Reference +------------- + +.. autoclass:: pyhealth.interpret.methods.AttentionRollout + :members: + :undoc-members: + :show-inheritance: + :member-order: bysource diff --git a/docs/why_pyhealth.rst b/docs/why_pyhealth.rst index 4ecfe031d..41635e6ab 100644 --- a/docs/why_pyhealth.rst +++ b/docs/why_pyhealth.rst @@ -175,7 +175,7 @@ Go beyond standard metrics with comprehensive model assessment: - Gradient-based: Integrated Gradients, DeepLift, Saliency Maps, GIM - Perturbation-based: LIME, SHAP (with healthcare-optimized implementations) -- Attention-based: Chefer relevance propagation for transformers +- Attention-based: Chefer relevance propagation and attention rollout for transformers - Visualization tools for clinical decision support **Uncertainty quantification:** diff --git a/examples/interpretability/dka_stageattn_mimic4_interpret.py b/examples/interpretability/dka_stageattn_mimic4_interpret.py index 3b405bd52..3001a8d46 100644 --- a/examples/interpretability/dka_stageattn_mimic4_interpret.py +++ b/examples/interpretability/dka_stageattn_mimic4_interpret.py @@ -135,6 +135,7 @@ def count_labels(ds): "deeplift": DeepLift(model, use_embeddings=True), "gim": GIM(model), "chefer": CheferRelevance(model), + "rollout": AttentionRollout(model), "shap": ShapExplainer(model, use_embeddings=True), "lime": LimeExplainer(model, use_embeddings=True, n_samples=200), } diff --git a/examples/interpretability/dka_transformer_mimic4_interpret.py b/examples/interpretability/dka_transformer_mimic4_interpret.py index d2617d652..1a7f87799 100644 --- a/examples/interpretability/dka_transformer_mimic4_interpret.py +++ b/examples/interpretability/dka_transformer_mimic4_interpret.py @@ -135,6 +135,7 @@ def count_labels(ds): "deeplift": DeepLift(model, use_embeddings=True), "gim": GIM(model), "chefer": CheferRelevance(model), + "rollout": AttentionRollout(model), "shap": ShapExplainer(model, use_embeddings=True), "lime": LimeExplainer(model, use_embeddings=True, n_samples=200), } diff --git a/examples/interpretability/los_stageattn_mimic4_interpret.py b/examples/interpretability/los_stageattn_mimic4_interpret.py index 51d253f25..35a7c95ec 100644 --- a/examples/interpretability/los_stageattn_mimic4_interpret.py +++ b/examples/interpretability/los_stageattn_mimic4_interpret.py @@ -121,6 +121,7 @@ def main(): "deeplift": DeepLift(model, use_embeddings=True), "gim": GIM(model), "chefer": CheferRelevance(model), + "rollout": AttentionRollout(model), "shap": ShapExplainer(model, use_embeddings=True), "lime": LimeExplainer(model, use_embeddings=True, n_samples=200), } diff --git a/examples/interpretability/los_transformer_mimic4_interpret.py b/examples/interpretability/los_transformer_mimic4_interpret.py index ccb06c707..cb5911cd2 100644 --- a/examples/interpretability/los_transformer_mimic4_interpret.py +++ b/examples/interpretability/los_transformer_mimic4_interpret.py @@ -121,6 +121,7 @@ def main(): "deeplift": DeepLift(model, use_embeddings=True), "gim": GIM(model), "chefer": CheferRelevance(model), + "rollout": AttentionRollout(model), "shap": ShapExplainer(model, use_embeddings=True), "lime": LimeExplainer(model, use_embeddings=True, n_samples=200), } diff --git a/examples/interpretability/mp_stageattn_mimic4_interpret.py b/examples/interpretability/mp_stageattn_mimic4_interpret.py index e42b9aca6..e7eaa8541 100644 --- a/examples/interpretability/mp_stageattn_mimic4_interpret.py +++ b/examples/interpretability/mp_stageattn_mimic4_interpret.py @@ -121,6 +121,7 @@ def main(): "deeplift": DeepLift(model, use_embeddings=True), "gim": GIM(model), "chefer": CheferRelevance(model), + "rollout": AttentionRollout(model), "shap": ShapExplainer(model, use_embeddings=True), "lime": LimeExplainer(model, use_embeddings=True, n_samples=200), } diff --git a/examples/interpretability/mp_transformer_mimic4_interpret.py b/examples/interpretability/mp_transformer_mimic4_interpret.py index dcdb55215..e3ffb4f85 100644 --- a/examples/interpretability/mp_transformer_mimic4_interpret.py +++ b/examples/interpretability/mp_transformer_mimic4_interpret.py @@ -121,6 +121,7 @@ def main(): "deeplift": DeepLift(model, use_embeddings=True), "gim": GIM(model), "chefer": CheferRelevance(model), + "rollout": AttentionRollout(model), "shap": ShapExplainer(model, use_embeddings=True), "lime": LimeExplainer(model, use_embeddings=True, n_samples=200), } diff --git a/pyhealth/interpret/methods/__init__.py b/pyhealth/interpret/methods/__init__.py index 6c92cb6e4..3f012f1b0 100644 --- a/pyhealth/interpret/methods/__init__.py +++ b/pyhealth/interpret/methods/__init__.py @@ -1,6 +1,7 @@ from pyhealth.interpret.methods.base_interpreter import BaseInterpreter from pyhealth.interpret.methods.baseline import RandomBaseline from pyhealth.interpret.methods.chefer import CheferRelevance +from pyhealth.interpret.methods.attention_rollout import AttentionRollout from pyhealth.interpret.methods.basic_gradient import BasicGradientSaliencyMaps from pyhealth.interpret.methods.deeplift import DeepLift from pyhealth.interpret.methods.gim import GIM @@ -15,6 +16,7 @@ __all__ = [ "BaseInterpreter", "CheferRelevance", + "AttentionRollout", "DeepLift", "GIM", "IntegratedGradientGIM", diff --git a/pyhealth/interpret/methods/attention_rollout.py b/pyhealth/interpret/methods/attention_rollout.py new file mode 100644 index 000000000..fc4e709f5 --- /dev/null +++ b/pyhealth/interpret/methods/attention_rollout.py @@ -0,0 +1,285 @@ +"""Attention rollout for transformer interpretability. + +This module implements the canonical attention rollout method, a +forward-pass-only, gradient-free, class-agnostic attention-flow baseline. +It complements the gradient-weighted, class-specific +:class:`~pyhealth.interpret.methods.CheferRelevance`. + +Paper: + Abnar, Samira, and Willem Zuidema. + "Quantifying Attention Flow in Transformers." + Proceedings of the 58th Annual Meeting of the Association for + Computational Linguistics (ACL), 2020. + https://arxiv.org/abs/2005.00928 +""" + +from typing import Dict, Optional + +import torch + +from pyhealth.models.base_model import BaseModel +from .base_interpreter import BaseInterpreter + + +class AttentionRollout(BaseInterpreter): + """Attention rollout for transformer interpretability. + + Implements the canonical attention rollout method of Abnar & Zuidema, + "Quantifying Attention Flow in Transformers" (2020), + https://arxiv.org/abs/2005.00928. + + Unlike :class:`~pyhealth.interpret.methods.CheferRelevance`, which is + gradient-weighted and class-specific, rollout is **forward-pass only**, + **gradient-free**, and **class-agnostic**: it quantifies how attention + propagates information across layers, independent of any target class. + It serves as the standard baseline that gradient-based attention methods + are compared against. + + .. note:: + "Gradient-free" refers to the attribution **math**: no backward pass + is run and no gradients enter the rollout computation. It does **not** + mean the call is safe inside ``torch.no_grad()``. The shared + attention-readout plumbing registers a gradient hook on the attention + tensors during the forward pass, so running ``attribute(**batch)`` + under ``torch.no_grad()`` raises a ``RuntimeError``. Call it under the + default (grad-enabled) context. + + This interpreter works with any model that exposes the attention-readout + methods ``set_attention_hooks``, ``get_attention_layers``, and + ``get_relevance_tensor`` (currently :class:`~pyhealth.models.Transformer` + and :class:`~pyhealth.models.StageAttentionNet`). Compatibility is checked + by duck-typing in ``__init__`` rather than by requiring a named interface, + since these methods are general attention readout and not specific to any + one method. + + The algorithm, per feature key: + + 1. Enable attention hooks via ``model.set_attention_hooks(True)`` and run a + single forward pass (no backward pass). + 2. Retrieve per-layer attention maps via ``model.get_attention_layers()``, + discarding the gradient element of each ``(attn_map, attn_grad)`` pair. + 3. Fuse heads (mean) to get one ``[batch, seq, seq]`` matrix per layer. + 4. Account for residual connections: ``A_hat = 0.5 * (A + I)``. + 5. Compose layers by matrix product: ``rollout = A_hat_L @ ... @ A_hat_1``. + 6. Reduce to per-token scores via ``model.get_relevance_tensor()``, then + expand to raw input value shapes. + + Because each ``A_hat`` is row-stochastic, so is their product; the + per-token relevance therefore forms a distribution over tokens (sums to 1 + before the input-shape expansion). + + Args: + model (BaseModel): A trained PyHealth model exposing the attention- + readout methods listed above. + head_fusion (str): How to combine attention heads into a single matrix + per layer. Currently only ``"mean"`` is supported (the canonical + choice from the paper). Defaults to ``"mean"``. + + Example: + >>> from pyhealth.datasets import create_sample_dataset, get_dataloader + >>> from pyhealth.models import Transformer + >>> from pyhealth.interpret.methods import AttentionRollout + >>> + >>> samples = [ + ... { + ... "patient_id": "p0", + ... "visit_id": "v0", + ... "conditions": ["A05B", "A05C", "A06A"], + ... "procedures": ["P01", "P02"], + ... "label": 1, + ... }, + ... { + ... "patient_id": "p0", + ... "visit_id": "v1", + ... "conditions": ["A05B"], + ... "procedures": ["P01"], + ... "label": 0, + ... }, + ... ] + >>> dataset = create_sample_dataset( + ... samples=samples, + ... input_schema={"conditions": "sequence", "procedures": "sequence"}, + ... output_schema={"label": "binary"}, + ... dataset_name="ehr_example", + ... ) + >>> model = Transformer(dataset=dataset) + >>> # ... train the model ... + >>> + >>> interpreter = AttentionRollout(model) + >>> batch = next(iter(get_dataloader(dataset, batch_size=2))) + >>> + >>> attributions = interpreter.attribute(**batch) + >>> # Returns dict: {"conditions": tensor, "procedures": tensor} + >>> print(attributions["conditions"].shape) # [batch, num_tokens] + >>> + >>> # target_class_idx is accepted but ignored (rollout is class-agnostic) + >>> same = interpreter.attribute(target_class_idx=1, **batch) + """ + + def __init__(self, model: BaseModel, head_fusion: str = "mean"): + if head_fusion != "mean": + raise ValueError( + f"Unsupported head_fusion='{head_fusion}'. " + "Currently supported values: mean." + ) + + required_methods = [ + "set_attention_hooks", + "get_attention_layers", + "get_relevance_tensor", + ] + missing_methods = [m for m in required_methods if not hasattr(model, m)] + + if missing_methods: + raise TypeError( + "AttentionRollout requires a model that exposes the attention " + "interpretability methods: " + f"{', '.join(required_methods)}. " + f"Missing: {', '.join(missing_methods)}." + ) + + super().__init__(model) + self.head_fusion = head_fusion + + + def attribute( + self, + target_class_idx: Optional[int] = None, + **data, + ) -> Dict[str, torch.Tensor]: + """Compute class-agnostic attention rollout attributions. + + Args: + target_class_idx: Accepted for API compatibility with class-specific + interpreters. Attention rollout is class-agnostic, so this argument + is ignored. + **data: Batch input passed directly to the model. + + Returns: + Dict[str, torch.Tensor]: A dict keyed by the model's feature keys. + Each value holds the rollout relevance for that feature — the + CLS-token row of the composed attention-rollout matrix, reduced + to one score per token by ``model.get_relevance_tensor()`` and + then expanded to the raw input value shape by + ``_map_to_input_shapes``. For flat sequence features this is + ``[batch, num_tokens]``; for nested sequences the per-visit + score is replicated across the codes within each visit. + Scores are non-negative and, before the input-shape expansion, + sum to 1 across tokens (a consequence of composing + row-stochastic matrices). + + Note: + Do not call this method inside a ``torch.no_grad()`` context. Even + though rollout uses no gradients, enabling attention hooks registers + a gradient hook during the forward pass, which requires grad-enabled + tensors and otherwise raises a ``RuntimeError``. + """ + + self.model.set_attention_hooks(True) + try: + self.model(**data) + finally: + self.model.set_attention_hooks(False) + + attention_layers = self.model.get_attention_layers() + R = {} + + for feature_key, layers in attention_layers.items(): + rollout = None + + for attn_map, _ in layers: + if attn_map is None: + raise RuntimeError( + "AttentionRollout expected attention maps to be captured " + f"for feature '{feature_key}', but found None." + ) + + attn = self._fuse_heads(attn_map) + attn = self._add_residual(attn) + + if rollout is None: + batch_size, seq_len, _ = attn.shape + rollout = torch.eye( + seq_len, + device=attn.device, + dtype=attn.dtype, + ) + rollout = rollout.unsqueeze(0).expand( + batch_size, + seq_len, + seq_len, + ) + + rollout = torch.bmm(attn, rollout) + + if rollout is None: + raise RuntimeError( + "AttentionRollout expected at least one attention layer " + f"for feature '{feature_key}', but found none." + ) + + R[feature_key] = rollout + + attributions = self.model.get_relevance_tensor(R, **data) + return self._map_to_input_shapes(attributions, data) + + def _fuse_heads(self, attn_map: torch.Tensor) -> torch.Tensor: + """Fuse attention heads from [batch, heads, seq, seq] to [batch, seq, seq].""" + if (self.head_fusion == "mean"): + return attn_map.mean(dim=1) + + def _map_to_input_shapes( + self, + attributions: Dict[str, torch.Tensor], + data: dict, + ) -> Dict[str, torch.Tensor]: + """Expand attributions to match raw input value shapes. + + For nested sequences the attention operates on a pooled + (visit-level) sequence, but downstream consumers (e.g. ablation + metrics) expect attributions to match the raw input value shape. + Per-visit relevance scores are replicated across all codes + within each visit. + + Args: + attributions: Per-feature attribution tensors returned by + ``model.get_relevance_tensor()``. + data: Original ``**data`` kwargs from the dataloader batch. + + Returns: + Attributions expanded to raw input value shapes where needed. + """ + result: Dict[str, torch.Tensor] = {} + for key, attr in attributions.items(): + feature = data.get(key) + if feature is not None: + if isinstance(feature, torch.Tensor): + val = feature + else: + schema = self.model.dataset.input_processors[key].schema() + val = ( + feature[schema.index("value")] + if "value" in schema + else None + ) + if val is not None and val.dim() > attr.dim(): + for _ in range(val.dim() - attr.dim()): + attr = attr.unsqueeze(-1) + attr = attr.expand_as(val) + result[key] = attr + return result + + @staticmethod + def _add_residual(attn: torch.Tensor) -> torch.Tensor: + """ + Add canonical rollout residual connection: 0.5 * (A + I). + 0.5 * (A + I) stays row-stochastic only because A is (soft-max ouput). + """ + + batch, seq_len, _ = attn.shape + identity = torch.eye( + seq_len, + device=attn.device, + dtype=attn.dtype, + ).unsqueeze(0) + return 0.5 * (attn + identity) diff --git a/tests/core/test_attention_rollout.py b/tests/core/test_attention_rollout.py new file mode 100644 index 000000000..45c745163 --- /dev/null +++ b/tests/core/test_attention_rollout.py @@ -0,0 +1,311 @@ +# Author: Felipe Amaral Bonchristiano +# NetID: felipea5 +# Description: Unit tests for the AttentionRollout interpretability method +# (Abnar & Zuidema, 2020, https://arxiv.org/abs/2005.00928). + +import unittest + +import torch +import torch.nn as nn + +from pyhealth.datasets import create_sample_dataset, get_dataloader +from pyhealth.interpret.methods import AttentionRollout +from pyhealth.models import Transformer + + +def _make_dataset(samples, input_schema): + """Build a tiny sample dataset.""" + return create_sample_dataset( + samples=samples, + input_schema=input_schema, + output_schema={"label": "binary"}, + dataset_name="rollout_test", + ) + + +class TestAttentionRollout(unittest.TestCase): + """Tests for :class:`AttentionRollout`.""" + + def setUp(self): + torch.manual_seed(42) + + self.samples = [ + { + "patient_id": "p0", + "visit_id": "v0", + "conditions": ["A05B", "A05C", "A06A"], + "procedures": ["P01", "P02"], + "label": 1, + }, + { + "patient_id": "p1", + "visit_id": "v0", + "conditions": ["A05B"], + "procedures": ["P01"], + "label": 0, + }, + ] + self.input_schema = { + "conditions": "sequence", + "procedures": "sequence", + } + self.dataset = _make_dataset(self.samples, self.input_schema) + self.model = Transformer( + dataset=self.dataset, + embedding_dim=8, + heads=2, + num_layers=2, + ) + self.interpreter = AttentionRollout(self.model) + self.loader = get_dataloader(self.dataset, batch_size=2, shuffle=False) + self.batch = next(iter(self.loader)) + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _capture_relevance(self, model): + """Wrap ``get_relevance_tensor`` to capture pre/post-expansion tensors. + + Returns ``(captured_R, captured_pre)`` dicts that are populated as a + side effect of the next ``attribute`` call: + + * ``captured_R`` — the composed rollout matrices ``[batch, seq, seq]``. + * ``captured_pre`` — the per-token relevance *before* the + input-shape expansion done by ``_map_to_input_shapes``. + """ + original = model.get_relevance_tensor + captured_R = {} + captured_pre = {} + + def spy(R, **data): + for key, value in R.items(): + captured_R[key] = value.detach().clone() + out = original(R, **data) + for key, value in out.items(): + captured_pre[key] = value.detach().clone() + return out + + model.get_relevance_tensor = spy + return captured_R, captured_pre + + # ------------------------------------------------------------------ + # Tests + # ------------------------------------------------------------------ + + def test_returns_dict_keyed_by_feature_keys(self): + """attribute() returns a dict keyed by exactly the model feature keys.""" + + attributions = self.interpreter.attribute(**self.batch) + + self.assertIsInstance(attributions, dict) + self.assertEqual( + set(attributions.keys()), + set(self.model.feature_keys), + ) + + def test_output_shape_matches_input_seq_length(self): + """Each attribution matches its input feature's shape (seq length).""" + + attributions = self.interpreter.attribute(**self.batch) + + for key in self.model.feature_keys: + self.assertIsInstance(attributions[key], torch.Tensor) + # Flat sequence inputs are [batch, seq_len] - the attribution + # must line up token-for-token with the raw input. + self.assertEqual(attributions[key].shape, self.batch[key].shape) + self.assertEqual(attributions[key].shape[0], 2) # batch size + + def test_multi_feature_key_model(self): + """A model with several feature streams yields one entry per stream.""" + + attributions = self.interpreter.attribute(**self.batch) + + self.assertIn("conditions", attributions) + self.assertIn("procedures", attributions) + self.assertEqual(len(attributions), 2) + + def test_row_stochastic_invariant(self): + """Pre-expansion relevance is a distribution over tokens (sums to 1).""" + + _, captured_pre = self._capture_relevance(self.model) + self.interpreter.attribute(**self.batch) + + self.assertTrue(captured_pre) # something was captured + for key, relevance in captured_pre.items(): + token_sums = relevance.sum(dim=-1) + self.assertTrue( + torch.allclose(token_sums, torch.ones_like(token_sums), atol=1e-5), + msg=f"feature '{key}' relevance does not sum to 1: {token_sums}", + ) + # Rollout produces non-negative relevance. + self.assertTrue(torch.all(relevance >= 0)) + + def test_rollout_matrices_are_row_stochastic(self): + """Every composed rollout matrix has rows summing to 1.""" + + captured_R, _ = self._capture_relevance(self.model) + self.interpreter.attribute(**self.batch) + + for key, rollout in captured_R.items(): + row_sums = rollout.sum(dim=-1) + self.assertTrue( + torch.allclose(row_sums, torch.ones_like(row_sums), atol=1e-5), + msg=f"feature '{key}' rollout not row-stochastic: {row_sums}", + ) + + def test_identity_attention_gives_identity_rollout(self): + """If attention is the identity at every layer, rollout is identity.""" + + original_layers = self.model.get_attention_layers + + def identity_layers(): + # Reuse real shapes captured during the forward pass, but + # overwrite each attention map with an identity per head. + real = original_layers() + patched = {} + for key, layers in real.items(): + new_layers = [] + for attn_map, grad in layers: + batch, heads, seq, _ = attn_map.shape + eye = ( + torch.eye(seq, dtype=attn_map.dtype, device=attn_map.device) + .reshape(1, 1, seq, seq) + .expand(batch, heads, seq, seq) + .contiguous() + ) + new_layers.append((eye, grad)) + patched[key] = new_layers + return patched + + self.model.get_attention_layers = identity_layers + captured_R, _ = self._capture_relevance(self.model) + self.interpreter.attribute(**self.batch) + + self.assertTrue(captured_R) + for key, rollout in captured_R.items(): + batch, seq, _ = rollout.shape + expected = ( + torch.eye(seq, dtype=rollout.dtype) + .unsqueeze(0) + .expand(batch, seq, seq) + ) + self.assertTrue( + torch.allclose(rollout, expected, atol=1e-6), + msg=f"feature '{key}' rollout is not identity", + ) + + def test_single_layer(self): + """num_layers=1 still produces valid, row-stochastic attributions.""" + + torch.manual_seed(42) + model = Transformer( + dataset=self.dataset, embedding_dim=8, heads=2, num_layers=1 + ) + interpreter = AttentionRollout(model) + _, captured_pre = self._capture_relevance(model) + + attributions = interpreter.attribute(**self.batch) + + self.assertEqual(set(attributions.keys()), set(model.feature_keys)) + for relevance in captured_pre.values(): + token_sums = relevance.sum(dim=-1) + self.assertTrue( + torch.allclose(token_sums, torch.ones_like(token_sums), atol=1e-5) + ) + + def test_single_head(self): + """heads=1 (no head fusion needed) produces valid attributions.""" + + torch.manual_seed(42) + model = Transformer( + dataset=self.dataset, embedding_dim=8, heads=1, num_layers=2 + ) + interpreter = AttentionRollout(model) + _, captured_pre = self._capture_relevance(model) + + attributions = interpreter.attribute(**self.batch) + + self.assertEqual(set(attributions.keys()), set(model.feature_keys)) + for relevance in captured_pre.values(): + token_sums = relevance.sum(dim=-1) + self.assertTrue( + torch.allclose(token_sums, torch.ones_like(token_sums), atol=1e-5) + ) + + def test_masked_padded_sequence(self): + """Padded batches (uneven sequence lengths) stay row-stochastic.""" + # Sanity check that padding actually occurred. + self.assertEqual(self.batch["conditions"].shape[1], 3) + + _, captured_pre = self._capture_relevance(self.model) + self.interpreter.attribute(**self.batch) + + for key, relevance in captured_pre.items(): + token_sums = relevance.sum(dim=-1) + self.assertTrue( + torch.allclose(token_sums, torch.ones_like(token_sums), atol=1e-5), + msg=f"padded feature '{key}' relevance does not sum to 1", + ) + + def test_target_class_idx_is_a_noop(self): + """Rollout is class-agnostic: target_class_idx must not change output.""" + + baseline = self.interpreter.attribute(**self.batch) + with_target = self.interpreter.attribute(target_class_idx=1, **self.batch) + + for key in baseline: + self.assertTrue( + torch.allclose(baseline[key], with_target[key], atol=1e-6), + msg=f"target_class_idx changed attributions for '{key}'", + ) + + def test_incompatible_model_raises_type_error(self): + """Model lacking the attention-readout methods raises TypeError.""" + + class PlainModel(nn.Module): + def forward(self, **data): + return {"logit": torch.zeros(1, 1)} + + with self.assertRaises(TypeError): + AttentionRollout(PlainModel()) + + def test_unsupported_head_fusion_raises_value_error(self): + """An unsupported head_fusion value raises ValueError (not AttributeError).""" + with self.assertRaises(ValueError): + AttentionRollout(self.model, head_fusion="max") + + def test_model_is_in_eval_mode(self): + """Constructing the interpreter puts the model in eval mode, disabling dropout + and making attributions deterministic for a given input. + """ + + self.assertFalse(self.model.training) + + def test_attribute_is_deterministic(self): + """Repeated calls on the same batch produce identical attributions.""" + + first = self.interpreter.attribute(**self.batch) + second = self.interpreter.attribute(**self.batch) + + for key in first: + self.assertTrue( + torch.allclose(first[key], second[key], atol=1e-6), + msg=f"attributions for '{key}' are not deterministic", + ) + + def test_callable_interface_matches_attribute(self): + """Calling the interpreter directly is equivalent to attribute().""" + + via_attribute = self.interpreter.attribute(**self.batch) + via_call = self.interpreter(**self.batch) + + self.assertEqual(set(via_attribute.keys()), set(via_call.keys())) + for key in via_attribute: + self.assertTrue( + torch.allclose(via_attribute[key], via_call[key], atol=1e-6) + ) + + +if __name__ == "__main__": + unittest.main() From c6a63ee8a804f00c3495a5dce751816ca497c526 Mon Sep 17 00:00:00 2001 From: John Wu <54558896+jhnwu3@users.noreply.github.com> Date: Thu, 9 Jul 2026 13:34:38 +0900 Subject: [PATCH 16/61] Add CI gate enforcing PR contribution rules for pyhealth/ changes (#1176) Any PR touching pyhealth/**/*.py must also update docs/ and examples/, keep added/modified lines free of ruff lint violations, and give new or modified top-level public classes/functions a '>>>' docstring example. Co-authored-by: Claude --- .github/workflows/pr_contribution_rules.yml | 33 +++++ CONTRIBUTING.md | 18 +++ pyproject.toml | 10 ++ tools/check_pr_rules.py | 155 ++++++++++++++++++++ 4 files changed, 216 insertions(+) create mode 100644 .github/workflows/pr_contribution_rules.yml create mode 100644 tools/check_pr_rules.py diff --git a/.github/workflows/pr_contribution_rules.yml b/.github/workflows/pr_contribution_rules.yml new file mode 100644 index 000000000..63540da5a --- /dev/null +++ b/.github/workflows/pr_contribution_rules.yml @@ -0,0 +1,33 @@ +name: PR Contribution Rules + +on: + pull_request: + types: [opened, synchronize, reopened] + +jobs: + contribution-rules: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Fetch PR base and head commits + run: | + git fetch origin ${{ github.event.pull_request.base.sha }} --depth=1 + git fetch origin ${{ github.event.pull_request.head.sha }} --depth=1 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Install ruff + run: pip install 'ruff~=0.15' + + - name: Check PR contribution rules + run: | + python tools/check_pr_rules.py \ + --base ${{ github.event.pull_request.base.sha }} \ + --head ${{ github.event.pull_request.head.sha }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7aef2b3ae..5be4ccc82 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -57,6 +57,24 @@ is set to 88 characters. We follow the [Google style](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings) for docstrings. +## PR Contribution Rules (enforced in CI) + +Any pull request that modifies a file under `pyhealth/` must also: + +- Update at least one file under `docs/` and one file under `examples/`. +- Keep newly added/modified lines free of [ruff](https://docs.astral.sh/ruff/) + lint violations (`ruff check`, 88-char line length). Pre-existing lint + issues elsewhere in a touched file are not blocked. +- Give every new or modified top-level public class/function a `>>>` usage + example in its docstring, so it renders as example code in the API docs. + +These rules are checked by `.github/workflows/pr_contribution_rules.yml`, +which runs `tools/check_pr_rules.py`. You can run the same check locally: + +```bash +python tools/check_pr_rules.py --base --head HEAD +``` + ## Community We welcome you to join our community diff --git a/pyproject.toml b/pyproject.toml index 65e0e2757..b4626e649 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,6 +69,9 @@ nlp = [ "rouge_score~=0.1.2", "nltk~=3.9.1", ] +lint = [ + "ruff~=0.15", +] [project.urls] Homepage = "https://github.com/sunlabuiuc/PyHealth" @@ -82,6 +85,13 @@ requires = ["hatchling"] build-backend = "hatchling.build" +### Ruff +# +[tool.ruff] +line-length = 88 +target-version = "py313" + + ### Hatchling # [tool.hatch.build.targets.wheel] diff --git a/tools/check_pr_rules.py b/tools/check_pr_rules.py new file mode 100644 index 000000000..1bfbcbd38 --- /dev/null +++ b/tools/check_pr_rules.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +""" +CI gate enforcing PyHealth's PR contribution rules. + +Rules enforced whenever a PR touches pyhealth/**/*.py: + + 1. Docs/examples: the PR must also modify at least one file under + docs/** and one file under examples/**. + 2. Lint: lines added or modified in touched pyhealth/**/*.py files must + be free of ruff violations. Pre-existing violations elsewhere in a + touched file are not flagged. + 3. Docstring examples: new or modified top-level public classes/ + functions in pyhealth/**/*.py must include a '>>>' usage example in + their docstring. + +Usage: + python tools/check_pr_rules.py --base --head +""" +import argparse +import ast +import json +import subprocess +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def sh(*args): + return subprocess.run( + args, cwd=REPO_ROOT, capture_output=True, text=True, check=True + ).stdout + + +def changed_files(base, head): + out = sh("git", "diff", "--name-only", "--diff-filter=ACMR", f"{base}..{head}") + return [line.strip() for line in out.splitlines() if line.strip()] + + +def added_lines(base, head, path): + """Line numbers in `path` at `head` that were added or modified vs `base`.""" + out = sh("git", "diff", "--unified=0", f"{base}..{head}", "--", path) + lines = set() + for line in out.splitlines(): + if not line.startswith("@@"): + continue + plus = line.split("+")[1].split(" ")[0] + if "," in plus: + start, count = (int(x) for x in plus.split(",")) + else: + start, count = int(plus), 1 + lines |= set(range(start, start + count)) + return lines + + +def check_docs_examples(files): + if not any(f.startswith("pyhealth/") and f.endswith(".py") for f in files): + return [] + problems = [] + if not any(f.startswith("docs/") for f in files): + problems.append( + "PR modifies pyhealth/ source files but no file under docs/ " + "was updated." + ) + if not any(f.startswith("examples/") for f in files): + problems.append( + "PR modifies pyhealth/ source files but no file under " + "examples/ was updated." + ) + return problems + + +def check_lint(files, base, head): + py_files = [ + f + for f in files + if f.startswith("pyhealth/") and f.endswith(".py") and (REPO_ROOT / f).exists() + ] + if not py_files: + return [] + result = subprocess.run( + ["ruff", "check", "--output-format=json", *py_files], + cwd=REPO_ROOT, + capture_output=True, + text=True, + ) + if not result.stdout.strip(): + return [] + problems = [] + for v in json.loads(result.stdout): + path = Path(v["filename"]).resolve().relative_to(REPO_ROOT).as_posix() + line = v["location"]["row"] + if line in added_lines(base, head, path): + problems.append(f"{path}:{line}: {v['code']} {v['message']}") + return problems + + +def check_docstring_examples(files, base, head): + problems = [] + for path in files: + if not (path.startswith("pyhealth/") and path.endswith(".py")): + continue + full = REPO_ROOT / path + if not full.exists(): + continue + added = added_lines(base, head, path) + if not added: + continue + try: + tree = ast.parse(full.read_text()) + except SyntaxError: + continue + for node in ast.iter_child_nodes(tree): + if not isinstance(node, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)): + continue + if node.name.startswith("_"): + continue + span = set(range(node.lineno, node.end_lineno + 1)) + if not span & added: + continue + doc = ast.get_docstring(node) + if not doc or ">>>" not in doc: + kind = "class" if isinstance(node, ast.ClassDef) else "function" + problems.append( + f"{path}:{node.lineno}: public {kind} '{node.name}' is " + "new/modified but its docstring has no '>>>' usage example." + ) + return problems + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base", required=True, help="base commit SHA") + parser.add_argument("--head", required=True, help="head commit SHA") + args = parser.parse_args() + + files = changed_files(args.base, args.head) + problems = ( + check_docs_examples(files) + + check_lint(files, args.base, args.head) + + check_docstring_examples(files, args.base, args.head) + ) + + if problems: + print("PR contribution rules failed:\n") + for p in problems: + print(f" - {p}") + print(f"\n{len(problems)} issue(s) found. See CONTRIBUTING.md for details.") + sys.exit(1) + + print("All PR contribution rules passed.") + + +if __name__ == "__main__": + main() From 546c1ad2510838ed8117082b1d949b31f0030ece Mon Sep 17 00:00:00 2001 From: vihaan101 <247244351+vihaan101@users.noreply.github.com> Date: Tue, 14 Jul 2026 22:35:39 +0530 Subject: [PATCH 17/61] Add EEGBCI dataset and tasks (#1177) * feat: add EEGBCI helper functions * feat: add EEGBCI dataset * feat: add EEGBCI tasks * test: add opt-in EEGBCI real-data smoke test * docs: add EEGBCI pattern discovery example * docs: add EEGBCI API docs * chore: record EEGBCI verification * docs: refine EEGBCI moment report design * Add EEGBCI moment report constants * Add EEGBCI rest baseline helpers * Add EEGBCI state scoring helpers * Add EEGBCI task state quality helpers * Add EEGBCI moment row annotation * Add EEGBCI representative windows * Render EEGBCI moment summary * Wire EEGBCI moment report main flow * Document EEGBCI moment report outputs * Fix EEGBCI moment report review findings * Polish EEGBCI report artifact * Exclude EEG pattern discovery notes * fix: address EEGBCI review feedback * fix: address EEGBCI review feedback * fix: satisfy PR contribution rules --- docs/api/datasets.rst | 1 + .../pyhealth.datasets.EEGBCIDataset.rst | 7 + docs/api/tasks.rst | 1 + docs/api/tasks/pyhealth.tasks.eegbci.rst | 7 + examples/eeg/eegbci/README.md | 73 + .../eeg/eegbci/eegbci_pattern_discovery.py | 716 ++++++++ pyhealth/datasets/__init__.py | 1 + pyhealth/datasets/configs/eegbci.yaml | 13 + pyhealth/datasets/eegbci.py | 249 +++ pyhealth/tasks/__init__.py | 4 + pyhealth/tasks/eegbci.py | 586 ++++++ tests/core/test_eegbci.py | 1572 +++++++++++++++++ 12 files changed, 3230 insertions(+) create mode 100644 docs/api/datasets/pyhealth.datasets.EEGBCIDataset.rst create mode 100644 docs/api/tasks/pyhealth.tasks.eegbci.rst create mode 100644 examples/eeg/eegbci/README.md create mode 100644 examples/eeg/eegbci/eegbci_pattern_discovery.py create mode 100644 pyhealth/datasets/configs/eegbci.yaml create mode 100644 pyhealth/datasets/eegbci.py create mode 100644 pyhealth/tasks/eegbci.py create mode 100644 tests/core/test_eegbci.py diff --git a/docs/api/datasets.rst b/docs/api/datasets.rst index a23efb3d2..592aed487 100644 --- a/docs/api/datasets.rst +++ b/docs/api/datasets.rst @@ -241,6 +241,7 @@ Available Datasets datasets/pyhealth.datasets.COVID19CXRDataset datasets/pyhealth.datasets.ChestXray14Dataset datasets/pyhealth.datasets.PhysioNetDeIDDataset + datasets/pyhealth.datasets.EEGBCIDataset datasets/pyhealth.datasets.TUABDataset datasets/pyhealth.datasets.TUEVDataset datasets/pyhealth.datasets.ClinVarDataset diff --git a/docs/api/datasets/pyhealth.datasets.EEGBCIDataset.rst b/docs/api/datasets/pyhealth.datasets.EEGBCIDataset.rst new file mode 100644 index 000000000..8f4d427e9 --- /dev/null +++ b/docs/api/datasets/pyhealth.datasets.EEGBCIDataset.rst @@ -0,0 +1,7 @@ +pyhealth.datasets.EEGBCIDataset +================================ + +.. autoclass:: pyhealth.datasets.EEGBCIDataset + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/tasks.rst b/docs/api/tasks.rst index 8724176a8..c7910e626 100644 --- a/docs/api/tasks.rst +++ b/docs/api/tasks.rst @@ -223,6 +223,7 @@ Available Tasks Sleep Staging Sleep Staging (SleepEDF) Temple University EEG Tasks + EEGBCI Tasks Sleep Staging v2 Benchmark EHRShot ChestX-ray14 Binary Classification diff --git a/docs/api/tasks/pyhealth.tasks.eegbci.rst b/docs/api/tasks/pyhealth.tasks.eegbci.rst new file mode 100644 index 000000000..b2682057f --- /dev/null +++ b/docs/api/tasks/pyhealth.tasks.eegbci.rst @@ -0,0 +1,7 @@ +pyhealth.tasks.eegbci +===================== + +.. automodule:: pyhealth.tasks.eegbci + :members: + :undoc-members: + :show-inheritance: diff --git a/examples/eeg/eegbci/README.md b/examples/eeg/eegbci/README.md new file mode 100644 index 000000000..268f2bfb6 --- /dev/null +++ b/examples/eeg/eegbci/README.md @@ -0,0 +1,73 @@ +# EEGBCI Pattern Discovery + +This example uses `EEGBCIDataset` and `EEGBCIPatternDiscovery` to create +2-second EEGBCI windows with task labels, Welch bandpower features, and cautious +frequency-profile interpretations. + +The interpretations are exploratory signal metadata. They are not clinical +diagnoses and do not prove a subject's cognition. + +Run a tiny real-data example: + +```bash +python examples/eeg/eegbci/eegbci_pattern_discovery.py \ + --subjects 1 \ + --runs 3 \ + --max-windows 20 \ + --download +``` + +Outputs are written to `outputs/eegbci_pattern_discovery/` by default: + +- `eegbci_pattern_windows.csv` +- `eegbci_pattern_summary.md` + +The CSV has one row per emitted 2-second window. Key columns include subject/run +metadata, `event_code`, decoded `task_label`, PyHealth task-class identifier +(`eegbci_label` / `label`), PyHealth model-local label (`model_label`), +absolute window timing, band powers, relative band powers, `dominant_band`, +frequency ratios, and `interpretation`. + +The moment-report columns add analysis-grade fields: + +- `analysis_version` +- `state_hypothesis`, `state_confidence`, and `evidence_score` +- `evidence_summary` +- `rest_reference_scope` and rest-normalized relative band deltas +- `task_state_relation`, `task_state_rationale`, and `task_state_confidence` +- `is_low_confidence`, `is_possible_artifact`, and `is_mixed_or_ambiguous` + +The `interpretation` column is report-level text derived from these moment-report +fields. Legacy task-level fields such as `brain_state_hypothesis`, `confidence`, +and `quality_flags` are intentionally not written to the CSV. + +The Markdown report summarizes state counts, task-label/state agreement, +rest-normalized bandpower deltas, confidence and quality flags, representative +windows, limitations, and next checks. These labels are signal-pattern +summaries from short EEG windows, not clinical findings or evidence of a +subject's cognition. + +## Data source and citation + +The example uses PhysioNet's [EEG Motor Movement/Imagery Dataset +(eegmmidb), version 1.0.0](https://physionet.org/content/eegmmidb/1.0.0/). +The dataset files are distributed under the +[Open Data Commons Attribution License v1.0](https://opendatacommons.org/licenses/by/1-0/). + +Please cite: + +- Schalk, G. (2009). *EEG Motor Movement/Imagery Dataset* (version 1.0.0). + PhysioNet. https://doi.org/10.13026/C28G6P +- Schalk, G., McFarland, D. J., Hinterberger, T., Birbaumer, N., & Wolpaw, + J. R. (2004). BCI2000: A General-Purpose Brain-Computer Interface (BCI) + System. *IEEE Transactions on Biomedical Engineering, 51*(6), 1034-1043. +- Goldberger, A. L., Amaral, L. A. N., Glass, L., Hausdorff, J. M., Ivanov, + P. C., Mark, R. G., Mietus, J. E., Moody, G. B., Peng, C.-K., & Stanley, + H. E. (2000). PhysioBank, PhysioToolkit, and PhysioNet: Components of a new + research resource for complex physiologic signals. *Circulation, 101*(23), + e215-e220. + +`--root` points to the local EEGBCI data directory. With `--download`, MNE +downloads any missing EDF files under that root. PyHealth task caches are stored +under the configured PyHealth cache directory and are keyed by the requested +subject/run selection. diff --git a/examples/eeg/eegbci/eegbci_pattern_discovery.py b/examples/eeg/eegbci/eegbci_pattern_discovery.py new file mode 100644 index 000000000..447d02509 --- /dev/null +++ b/examples/eeg/eegbci/eegbci_pattern_discovery.py @@ -0,0 +1,716 @@ +from __future__ import annotations + +import argparse +import math +import sys +from collections import Counter +from pathlib import Path + +import pandas as pd + +REPO_ROOT = Path(__file__).resolve().parents[3] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from pyhealth.datasets import EEGBCIDataset +from pyhealth.tasks import EEGBCIPatternDiscovery + + +ANALYSIS_VERSION = "eegbci_pattern_moment_report_v1" +REPORT_BANDS = ("delta", "theta", "alpha", "beta", "gamma") +STATE_CONFIDENCE_RANK = {"low": 0, "medium": 1, "high": 2} + + +def scalar_value(value): + if hasattr(value, "item"): + return value.item() + return value + + +def parse_int_list(value: str) -> list[int]: + items: list[int] = [] + for raw_part in value.split(","): + part = raw_part.strip() + if not part: + raise ValueError("Empty value in integer list") + if "-" in part: + start_text, end_text = part.split("-", 1) + start = int(start_text.strip()) + end = int(end_text.strip()) + if start > end: + raise ValueError("Range start must be <= range end") + items.extend(range(start, end + 1)) + else: + items.append(int(part)) + return items + + +def sample_to_row(sample: dict) -> dict: + bandpower = sample["bandpower"] + model_label = scalar_value(sample["label"]) + eegbci_label = scalar_value(sample.get("eegbci_label", model_label)) + return { + "patient_id": sample["patient_id"], + "record_id": sample["record_id"], + "subject_id": sample["subject_id"], + "run": sample["run"], + "run_type": sample["run_type"], + "trial_id": sample["trial_id"], + "event_code": sample["event_code"], + "task_label": sample["task_label"], + "label_family": sample["label_family"], + "label": eegbci_label, + "eegbci_label": eegbci_label, + "model_label": model_label, + "start_time": sample["start_time"], + "end_time": sample["end_time"], + "dominant_band": bandpower["dominant_band"], + "alpha_beta_ratio": bandpower["alpha_beta_ratio"], + "theta_beta_ratio": bandpower["theta_beta_ratio"], + **{key: value for key, value in bandpower.items() if key.endswith("_power")}, + **{key: value for key, value in bandpower.items() if key.endswith("_relative")}, + } + + +def _mean_band_values(rows: list[dict]) -> dict: + means = {} + for band in REPORT_BANDS: + key = f"{band}_relative" + values = [float(row[key]) for row in rows if row.get(key) not in ("", None)] + if values: + means[key] = sum(values) / len(values) + return means + + +def build_rest_baselines(rows: list[dict]) -> dict: + rest_rows = [row for row in rows if row.get("task_label") == "rest"] + same_subject_run = {} + same_subject_all_runs = {} + + subject_run_keys = sorted({(row["subject_id"], row["run"]) for row in rest_rows}) + for key in subject_run_keys: + subject_id, run = key + grouped = [ + row + for row in rest_rows + if row["subject_id"] == subject_id and row["run"] == run + ] + same_subject_run[key] = _mean_band_values(grouped) + + subject_keys = sorted({row["subject_id"] for row in rest_rows}) + for subject_id in subject_keys: + grouped = [row for row in rest_rows if row["subject_id"] == subject_id] + same_subject_all_runs[subject_id] = _mean_band_values(grouped) + + return { + "same_subject_run": same_subject_run, + "same_subject_all_runs": same_subject_all_runs, + "global_rest": _mean_band_values(rest_rows) if rest_rows else None, + } + + +def _baseline_for_row(row: dict, baselines: dict) -> tuple[str, dict | None]: + subject_run_key = (row["subject_id"], row["run"]) + if subject_run_key in baselines["same_subject_run"]: + return "same_subject_run", baselines["same_subject_run"][subject_run_key] + if row["subject_id"] in baselines["same_subject_all_runs"]: + return "same_subject_all_runs", baselines["same_subject_all_runs"][row["subject_id"]] + if baselines["global_rest"]: + return "global_rest", baselines["global_rest"] + return "unavailable", None + + +def _clip01(value: float) -> float: + return max(0.0, min(1.0, value)) + + +def derive_state_hypothesis(row: dict) -> dict: + delta_bands = { + band: row.get(f"rest_{band}_relative_delta") for band in REPORT_BANDS + } + try: + normalized_deltas = { + band: float(value) + for band, value in delta_bands.items() + if value not in ("", None) + } + except (TypeError, ValueError): + normalized_deltas = {} + has_rest_deltas = len(normalized_deltas) == len(REPORT_BANDS) and all( + math.isfinite(value) for value in normalized_deltas.values() + ) + + if has_rest_deltas: + evidence_values = normalized_deltas + scores = { + "idle_alpha_profile": max(evidence_values["alpha"], 0.0) + + max(-evidence_values["beta"], 0.0), + "sensorimotor_engagement_profile": max( + -evidence_values["alpha"], 0.0 + ) + + max(evidence_values["beta"], 0.0) + + max(evidence_values["gamma"], 0.0), + "slow_wave_dominant_pattern": max(evidence_values["delta"], 0.0) + + max(evidence_values["theta"], 0.0), + "possible_artifact_profile": max( + evidence_values["gamma"] - 0.03, 0.0 + ) + * 2.0, + } + evidence_basis = "rest_normalized_delta" + else: + evidence_values = { + band: float(row.get(f"{band}_relative", 0.0) or 0.0) + for band in REPORT_BANDS + } + alpha_beta = float(row.get("alpha_beta_ratio", 0.0) or 0.0) + theta_beta = float(row.get("theta_beta_ratio", 0.0) or 0.0) + scores = { + "idle_alpha_profile": _clip01( + (evidence_values["alpha"] - 0.25) + + min(alpha_beta / 8.0, 0.40) + ), + "sensorimotor_engagement_profile": _clip01( + (evidence_values["beta"] - 0.20) + + max(evidence_values["gamma"] - 0.12, 0.0) + + max(0.0, 1.5 - alpha_beta) / 6.0 + ), + "slow_wave_dominant_pattern": _clip01( + (evidence_values["delta"] + evidence_values["theta"]) + - 0.45 + + min(theta_beta / 8.0, 0.20) + ), + "possible_artifact_profile": _clip01( + (evidence_values["gamma"] - 0.22) * 2.0 + + max(evidence_values["delta"] - 0.50, 0.0) + ), + } + evidence_basis = "absolute_band_profile" + + ordered = sorted(scores.items(), key=lambda item: item[1], reverse=True) + winner, winning_score = ordered[0] + margin = winning_score - ordered[1][1] + + if has_rest_deltas: + if winning_score < 0.02 or margin < 0.01: + state = "mixed_ambiguous_profile" + confidence = "low" + elif winning_score >= 0.15 and margin >= 0.08: + state = winner + confidence = "high" + elif winning_score >= 0.06 and margin >= 0.025: + state = winner + confidence = "medium" + else: + state = winner + confidence = "low" + evidence_score = round(min(winning_score / 0.20, 1.0), 3) + else: + if winning_score < 0.20 or margin < 0.08: + state = "mixed_ambiguous_profile" + evidence_score = round(max(winning_score, 0.10), 3) + confidence = "low" + else: + state = winner + evidence_score = round(winning_score, 3) + if winning_score >= 0.65 and margin >= 0.20: + confidence = "high" + elif winning_score >= 0.35 and margin >= 0.12: + confidence = "medium" + else: + confidence = "low" + + return { + "state_hypothesis": state, + "state_confidence": confidence, + "evidence_score": evidence_score, + "evidence_summary": ( + f"basis={evidence_basis}; delta={evidence_values['delta']:.3f}; " + f"theta={evidence_values['theta']:.3f}; " + f"alpha={evidence_values['alpha']:.3f}; " + f"beta={evidence_values['beta']:.3f}; " + f"gamma={evidence_values['gamma']:.3f}; margin={margin:.3f}" + ), + } + + +def derive_task_state_relation(row: dict) -> dict: + label_family = row.get("label_family", "") + task_label = row.get("task_label", "") + state = row.get("state_hypothesis", "") + + if state == "possible_artifact_profile": + relation = "not_applicable" + confidence = "medium" + rationale = ( + "Artifact-like frequency evidence is flagged for inspection instead of " + "task-label comparison." + ) + elif state == "mixed_ambiguous_profile": + relation = "ambiguous" + confidence = "low" + rationale = ( + "No frequency-profile state won clearly enough to compare strongly with " + "the task label." + ) + elif task_label == "rest" and state == "idle_alpha_profile": + relation = "supports_label" + confidence = "medium" + rationale = "The idle-like alpha profile is consistent with a rest-labeled EEGBCI window." + elif label_family == "motor_execution" and state == "sensorimotor_engagement_profile": + relation = "supports_label" + confidence = "medium" + rationale = ( + "The motor-engaged frequency profile is consistent with an " + "execution-labeled window." + ) + elif label_family == "motor_imagery" and state == "sensorimotor_engagement_profile": + relation = "adds_detail" + confidence = "medium" + rationale = ( + "The motor-engaged frequency profile adds signal detail to an " + "imagery-labeled window." + ) + elif label_family in {"motor_execution", "motor_imagery"} and state == "idle_alpha_profile": + relation = "disagrees" + confidence = "medium" + rationale = "The idle-like alpha profile does not align with a motor-labeled EEGBCI window." + elif state == "slow_wave_dominant_pattern": + relation = "adds_detail" + confidence = "low" + rationale = "The slow-wave dominant pattern adds frequency detail but is not a direct task match." + else: + relation = "ambiguous" + confidence = "low" + rationale = ( + "The task label and frequency-profile state do not have a stronger " + "deterministic mapping." + ) + + return { + "task_state_relation": relation, + "task_state_rationale": rationale, + "task_state_confidence": confidence, + } + + +def derive_quality_columns(row: dict) -> dict: + flags = str(row.get("quality_flags", "")) + state = row.get("state_hypothesis", "") + confidence = row.get("state_confidence", row.get("confidence", "")) + return { + "is_low_confidence": confidence == "low", + "is_possible_artifact": state == "possible_artifact_profile" + or "artifact" in flags + or "high_gamma" in flags, + "is_mixed_or_ambiguous": state == "mixed_ambiguous_profile" + or "ambiguous" in flags, + } + + +def derive_moment_interpretation(row: dict) -> str: + state = row.get("state_hypothesis", "missing") + confidence = row.get("state_confidence", "missing") + evidence = row.get("evidence_score", "") + relation = row.get("task_state_relation", "missing") + task = row.get("task_label", "missing") + dominant = row.get("dominant_band", "missing") + scope = row.get("rest_reference_scope", "missing") + return ( + f"The segment is consistent with `{state}` based on a `{dominant}`-dominant " + f"frequency profile ({confidence} confidence, evidence {evidence}). " + f"The task label is `{task}`, the task/state relation is `{relation}`, " + f"and the rest reference is `{scope}`." + ) + + +BASE_OUTPUT_COLUMNS = ( + "patient_id", + "record_id", + "subject_id", + "run", + "run_type", + "trial_id", + "event_code", + "task_label", + "label_family", + "label", + "eegbci_label", + "model_label", + "start_time", + "end_time", + "dominant_band", + "alpha_beta_ratio", + "theta_beta_ratio", + "interpretation", + "delta_power", + "theta_power", + "alpha_power", + "beta_power", + "gamma_power", + "delta_relative", + "theta_relative", + "alpha_relative", + "beta_relative", + "gamma_relative", +) + +MOMENT_REPORT_COLUMNS = ( + "analysis_version", + "state_hypothesis", + "state_confidence", + "evidence_score", + "evidence_summary", + "rest_reference_scope", + "rest_delta_relative_delta", + "rest_theta_relative_delta", + "rest_alpha_relative_delta", + "rest_beta_relative_delta", + "rest_gamma_relative_delta", + "task_state_relation", + "task_state_rationale", + "task_state_confidence", + "is_low_confidence", + "is_possible_artifact", + "is_mixed_or_ambiguous", +) + +OUTPUT_COLUMNS = BASE_OUTPUT_COLUMNS + MOMENT_REPORT_COLUMNS + + +def annotate_moment_rows(rows: list[dict], baselines: dict) -> list[dict]: + annotated = [] + for row in rows: + next_row = dict(row) + scope, baseline = _baseline_for_row(next_row, baselines) + next_row["analysis_version"] = ANALYSIS_VERSION + next_row["rest_reference_scope"] = scope + + for band in REPORT_BANDS: + source_key = f"{band}_relative" + delta_key = f"rest_{band}_relative_delta" + if baseline and source_key in baseline and next_row.get(source_key) not in ("", None): + next_row[delta_key] = round( + float(next_row[source_key]) - float(baseline[source_key]), 6 + ) + else: + next_row[delta_key] = "" + + next_row.update(derive_state_hypothesis(next_row)) + next_row.update(derive_task_state_relation(next_row)) + next_row["interpretation"] = derive_moment_interpretation(next_row) + next_row.update(derive_quality_columns(next_row)) + annotated.append(next_row) + return annotated + + +def _stable_row_key(row: dict) -> tuple: + return ( + row.get("subject_id", 0), + row.get("run", 0), + float(row.get("start_time", 0.0) or 0.0), + ) + + +def _strongest_row(rows: list[dict]) -> dict | None: + if not rows: + return None + return sorted( + rows, + key=lambda row: ( + -float(row.get("evidence_score", 0.0) or 0.0), + -STATE_CONFIDENCE_RANK.get(row.get("state_confidence", "low"), 0), + *_stable_row_key(row), + ), + )[0] + + +def select_representative_windows(rows: list[dict]) -> dict: + definitions = { + "strongest_idle_like": "idle_alpha_profile", + "strongest_motor_engaged": "sensorimotor_engagement_profile", + "strongest_slow_wave": "slow_wave_dominant_pattern", + "strongest_artifact_like": "possible_artifact_profile", + } + cards = {} + absent = [] + + for card_name, state in definitions.items(): + candidate = _strongest_row( + [row for row in rows if row.get("state_hypothesis") == state] + ) + if candidate is None: + absent.append(card_name) + else: + cards[card_name] = candidate + + ambiguous = [ + row for row in rows if row.get("state_hypothesis") == "mixed_ambiguous_profile" + ] + if ambiguous: + cards["most_ambiguous"] = sorted( + ambiguous, + key=lambda row: ( + float(row.get("evidence_score", 0.0) or 0.0), + -STATE_CONFIDENCE_RANK.get(row.get("state_confidence", "low"), 0), + *_stable_row_key(row), + ), + )[0] + else: + absent.append("most_ambiguous") + + disagreement = _strongest_row( + [row for row in rows if row.get("task_state_relation") == "disagrees"] + ) + if disagreement is None: + absent.append("strongest_task_state_disagreement") + else: + cards["strongest_task_state_disagreement"] = disagreement + + return {"cards": cards, "absent": absent} + + +def _format_count_lines(counter: Counter) -> list[str]: + if not counter: + return ["- None"] + return [f"- {label}: {count}" for label, count in counter.most_common()] + + +def _format_card(row: dict) -> list[str]: + bands = ", ".join( + f"{band}={float(row.get(f'{band}_relative', 0.0) or 0.0):.3f}" + for band in REPORT_BANDS + ) + deltas = ", ".join( + f"{band}={row.get(f'rest_{band}_relative_delta', '')}" + for band in REPORT_BANDS + ) + return [ + f"- Subject {row.get('subject_id')} run {row.get('run')} trial {row.get('trial_id')}", + f" - Task: {row.get('task_label')} from {row.get('start_time')}s to {row.get('end_time')}s", + ( + f" - State: {row.get('state_hypothesis')} " + f"({row.get('state_confidence')}, evidence {row.get('evidence_score')})" + ), + f" - Dominant band: {row.get('dominant_band')}; relative bands: {bands}", + f" - Rest deltas: {deltas}; scope: {row.get('rest_reference_scope')}", + ( + f" - Task relation: {row.get('task_state_relation')} " + f"({row.get('task_state_confidence')})" + ), + ( + f" - Flags: low_confidence={row.get('is_low_confidence')}, " + f"possible_artifact={row.get('is_possible_artifact')}, " + f"mixed_or_ambiguous={row.get('is_mixed_or_ambiguous')}" + ), + f" - Rationale: {row.get('task_state_rationale')}", + ] + + +def render_summary(rows: list[dict], config: dict) -> str: + state_counts = Counter(row.get("state_hypothesis", "missing") for row in rows) + task_counts = Counter(row.get("task_label", "missing") for row in rows) + confidence_counts = Counter(row.get("state_confidence", "missing") for row in rows) + relation_counts = Counter(row.get("task_state_relation", "missing") for row in rows) + unavailable_rest = sum( + row.get("rest_reference_scope") == "unavailable" for row in rows + ) + low_confidence = sum(bool(row.get("is_low_confidence")) for row in rows) + artifacts = sum(bool(row.get("is_possible_artifact")) for row in rows) + ambiguous = sum(bool(row.get("is_mixed_or_ambiguous")) for row in rows) + representatives = select_representative_windows(rows) + + executive = [] + if not rows: + executive.append("No windows were produced for the requested configuration.") + else: + top_state, top_state_count = state_counts.most_common(1)[0] + executive.append( + f"Processed {len(rows)} windows. Most common state: `{top_state}` " + f"({top_state_count}/{len(rows)})." + ) + if low_confidence == len(rows): + executive.append("Every window is low confidence.") + if len(state_counts) == 1: + executive.append( + "Every window maps to the same state; broaden coverage or review thresholds." + ) + if unavailable_rest == len(rows): + executive.append("No rest baseline was available for the emitted rows.") + if config.get("output_was_capped"): + executive.append("Output was capped by `--max-windows`.") + + lines = [ + "# EEGBCI Pattern Discovery Moment Report", + "", + f"Analysis version: `{ANALYSIS_VERSION}`", + "", + "## Executive Result", + "", + *[f"- {item}" for item in executive], + "", + "## Run Configuration", + "", + f"- Subjects: {config.get('subjects')}", + f"- Runs: {config.get('runs')}", + f"- Max windows: {config.get('max_windows')}", + f"- Baseline source rows: {config.get('baseline_row_count')}", + "", + "## Window Coverage", + "", + f"- Output windows: {len(rows)}", + f"- Task labels: {dict(task_counts)}", + "", + "## Moment-State Summary", + "", + *_format_count_lines(state_counts), + "", + "## Task Label x State Matrix", + "", + ] + + matrix = Counter( + (row.get("task_label", "missing"), row.get("state_hypothesis", "missing")) + for row in rows + ) + if matrix: + for (task_label, state), count in sorted(matrix.items()): + lines.append(f"- {task_label} x {state}: {count}") + else: + lines.append("- None") + + lines.extend( + [ + "", + "## Rest-Normalized Bandpower Summary", + "", + f"- Rows with unavailable rest baseline: {unavailable_rest}", + ] + ) + for band in REPORT_BANDS: + key = f"rest_{band}_relative_delta" + values = [float(row.get(key)) for row in rows if row.get(key) not in ("", None)] + if values: + lines.append(f"- {band}: mean delta {sum(values) / len(values):.3f}") + else: + lines.append(f"- {band}: unavailable") + + lines.extend( + [ + "", + "## Confidence and Quality Audit", + "", + f"- State confidence: {dict(confidence_counts)}", + f"- Task-state relations: {dict(relation_counts)}", + f"- Low-confidence rows: {low_confidence}", + f"- Possible artifact rows: {artifacts}", + f"- Mixed or ambiguous rows: {ambiguous}", + "", + "## Representative Windows", + "", + ] + ) + if representatives["cards"]: + for card_name, row in representatives["cards"].items(): + lines.append(f"### {card_name.replace('_', ' ').title()}") + lines.extend(_format_card(row)) + lines.append("") + else: + lines.append("- None") + if representatives["absent"]: + lines.append( + f"- Absent representative classes: {', '.join(representatives['absent'])}" + ) + + lines.extend( + [ + "", + "## Limitations", + "", + ( + "- These labels are signal-pattern summaries from short EEG windows. " + "They are not clinical findings and should not be read as evidence " + "of a subject's cognition." + ), + ] + ) + if unavailable_rest: + lines.append("- No rest baseline was available for at least one emitted row.") + if config.get("output_was_capped"): + lines.append( + "- The output was capped, so the artifact may not represent all requested windows." + ) + + lines.extend( + [ + "", + "## Next Checks", + "", + "- Run with broader subjects/runs to verify that state diversity improves.", + "- Inspect possible artifact rows before drawing conclusions from state counts.", + "- Compare rest-normalized deltas against the raw relative band shares.", + ] + ) + return "\n".join(lines).rstrip() + "\n" + + +def write_summary(rows: list[dict], path: Path, config: dict) -> None: + path.write_text(render_summary(rows, config), encoding="utf-8") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--root", default="~/.cache/pyhealth/eegbci") + parser.add_argument("--subjects", default="1,2,3") + parser.add_argument("--runs", default="3-14") + parser.add_argument("--output-dir", default="outputs/eegbci_pattern_discovery") + parser.add_argument("--max-windows", type=int, default=None) + parser.add_argument("--download", action="store_true") + args = parser.parse_args() + + output_dir = Path(args.output_dir).expanduser() + output_dir.mkdir(parents=True, exist_ok=True) + + requested_subjects = parse_int_list(args.subjects) + requested_runs = parse_int_list(args.runs) + dataset = EEGBCIDataset( + root=str(Path(args.root).expanduser()), + subjects=requested_subjects, + runs=requested_runs, + download=args.download, + ) + sample_dataset = dataset.set_task(EEGBCIPatternDiscovery(compute_stft=False)) + + all_rows = [sample_to_row(sample) for sample in sample_dataset] + baseline_row_count = sum(row.get("task_label") == "rest" for row in all_rows) + baselines = build_rest_baselines(all_rows) + annotated_rows = annotate_moment_rows(all_rows, baselines) + output_rows = ( + annotated_rows[: args.max_windows] + if args.max_windows is not None + else annotated_rows + ) + output_was_capped = ( + args.max_windows is not None and len(annotated_rows) > len(output_rows) + ) + + csv_path = output_dir / "eegbci_pattern_windows.csv" + summary_path = output_dir / "eegbci_pattern_summary.md" + pd.DataFrame(output_rows, columns=OUTPUT_COLUMNS).to_csv(csv_path, index=False) + write_summary( + output_rows, + summary_path, + { + "subjects": getattr(dataset, "subjects", requested_subjects), + "runs": getattr(dataset, "runs", requested_runs), + "max_windows": args.max_windows, + "baseline_row_count": baseline_row_count, + "output_was_capped": output_was_capped, + }, + ) + print(f"Wrote {csv_path}") + print(f"Wrote {summary_path}") + + +if __name__ == "__main__": + main() diff --git a/pyhealth/datasets/__init__.py b/pyhealth/datasets/__init__.py index c29955e7d..99d90aa62 100644 --- a/pyhealth/datasets/__init__.py +++ b/pyhealth/datasets/__init__.py @@ -82,6 +82,7 @@ def __init__(self, *args, **kwargs): split_by_visit, split_by_visit_conformal, ) +from .eegbci import EEGBCIDataset as EEGBCIDataset # noqa: E402 from .tuab import TUABDataset from .tuev import TUEVDataset from .utils import ( diff --git a/pyhealth/datasets/configs/eegbci.yaml b/pyhealth/datasets/configs/eegbci.yaml new file mode 100644 index 000000000..edb0e5b7a --- /dev/null +++ b/pyhealth/datasets/configs/eegbci.yaml @@ -0,0 +1,13 @@ +version: "1.0.0" +tables: + records: + file_path: "eegbci-pyhealth.csv" + patient_id: "patient_id" + timestamp: null + attributes: + - "record_id" + - "subject_id" + - "run" + - "run_type" + - "signal_file" + - "source" diff --git a/pyhealth/datasets/eegbci.py b/pyhealth/datasets/eegbci.py new file mode 100644 index 000000000..60cbd2ef5 --- /dev/null +++ b/pyhealth/datasets/eegbci.py @@ -0,0 +1,249 @@ +from __future__ import annotations + +import hashlib +import json +import logging +from pathlib import Path +from typing import Optional + +import mne +import pandas as pd + +from .base_dataset import BaseDataset +from pyhealth.tasks.eegbci import EEGMotorImageryEEGBCI, run_type_for_run + +logger = logging.getLogger(__name__) + +EEGBCI_METADATA_COLUMNS = { + "patient_id", + "record_id", + "subject_id", + "run", + "run_type", + "signal_file", + "source", +} + + +class EEGBCIDataset(BaseDataset): + """PhysioNet EEG Motor Movement/Imagery metadata dataset. + + The source dataset is PhysioNet's EEG Motor Movement/Imagery Dataset + (``eegmmidb``), version 1.0.0, licensed under the + Open Data Commons Attribution License v1.0. Cite Schalk (2009), + https://doi.org/10.13026/C28G6P. + + Args: + root: Directory containing or receiving EEGBCI EDF files and metadata. + dataset_name: Optional dataset name prefix. Defaults to ``"eegbci"``. + config_path: Optional dataset configuration path. + subjects: Subject identifiers to include. Defaults to ``[1, 2, 3]``. + runs: Run identifiers to include. Defaults to runs 3 through 14. + download: Whether MNE may download missing EDF files. + **kwargs: Additional arguments forwarded to :class:`BaseDataset`. + + Raises: + FileNotFoundError: If a requested EDF is unavailable and downloading is + disabled. + + Examples: + >>> dataset = EEGBCIDataset( + ... root="/path/to/eegbci", subjects=[1], runs=[3], download=True + ... ) + >>> dataset.stats() + """ + + def __init__( + self, + root: str, + dataset_name: Optional[str] = None, + config_path: Optional[str] = None, + subjects: Optional[list[int]] = None, + runs: Optional[list[int]] = None, + download: bool = False, + **kwargs, + ) -> None: + if config_path is None: + config_path = Path(__file__).parent / "configs" / "eegbci.yaml" + self.root = root + self.subjects = self._normalize_selection( + list(subjects) if subjects is not None else [1, 2, 3] + ) + self.runs = self._normalize_selection( + list(runs) if runs is not None else list(range(3, 15)) + ) + self.download = download + self.selection_key = self._build_selection_key() + self.metadata_file_name = self._metadata_file_name() + self.prepare_metadata() + metadata_key = self._metadata_cache_key() + dataset_name = dataset_name or "eegbci" + super().__init__( + root=root, + tables=["records"], + dataset_name=f"{dataset_name}_{self.selection_key}_{metadata_key}", + config_path=config_path, + **kwargs, + ) + if self.config is not None: + self.config.tables["records"].file_path = self.metadata_file_name + + @staticmethod + def _normalize_selection(values: list[int]) -> list[int]: + """Normalize identifiers to sorted, unique integers. + + Args: + values: Subject or run identifiers. + + Returns: + The normalized identifiers. + """ + return sorted({int(value) for value in values}) + + def _build_selection_key(self) -> str: + """Build a stable cache identity for the subject/run selection. + + Returns: + The selection key. + """ + payload = { + "subjects": [int(subject) for subject in self.subjects], + "runs": [int(run) for run in self.runs], + } + digest = hashlib.sha1( + json.dumps(payload, sort_keys=True).encode("utf-8") + ).hexdigest()[:10] + subject_part = "-".join(f"{int(subject):03d}" for subject in self.subjects) + run_part = "-".join(f"{int(run):02d}" for run in self.runs) + return f"s{subject_part}_r{run_part}_{digest}" + + def _metadata_file_name(self) -> str: + """Return the selection-specific metadata filename. + + Returns: + The CSV filename. + """ + return f"eegbci-pyhealth-{self.selection_key}.csv" + + def _metadata_cache_key(self) -> str: + """Return a content fingerprint for derived dataset caches. + + Returns: + A stable fingerprint of the selection-specific metadata CSV. + """ + csv_path = Path(self.root) / self.metadata_file_name + return hashlib.sha1(csv_path.read_bytes()).hexdigest()[:10] + + def _find_local_edf(self, subject: int, run: int) -> Path | None: + """Find the canonical EDF path before a recursive fallback search. + + Args: + subject: PhysioNet subject identifier. + run: EEGBCI run identifier. + + Returns: + The local EDF path, or ``None`` when no matching file exists. + """ + root = Path(self.root) + filename = f"S{subject:03d}R{run:02d}.edf" + canonical_path = ( + root / "files" / "eegmmidb" / "1.0.0" / f"S{subject:03d}" / filename + ) + if canonical_path.exists(): + return canonical_path + matches = sorted(root.rglob(filename)) + return matches[0] if matches else None + + def _requested_pairs(self) -> list[tuple[int, int]]: + """Return requested subject/run pairs in stable order. + + Returns: + The sorted subject/run pairs. + """ + return sorted( + (int(subject), int(run)) + for subject in self.subjects + for run in self.runs + ) + + def _metadata_matches_request(self, csv_path: Path) -> bool: + """Check whether cached metadata can be safely reused. + + Valid metadata has the required columns and requested subject/run pairs, + and every referenced EDF path still exists. + + Args: + csv_path: Metadata CSV path. + + Returns: + Whether the cached metadata is reusable. + """ + try: + df = pd.read_csv(csv_path) + except Exception: + return False + if not EEGBCI_METADATA_COLUMNS.issubset(df.columns): + return False + pairs = sorted((int(row.subject_id), int(row.run)) for row in df.itertuples()) + if pairs != self._requested_pairs(): + return False + return all(Path(str(row.signal_file)).is_file() for row in df.itertuples()) + + def prepare_metadata(self) -> None: + """Reuse valid metadata or write rows for every requested EDF. + + Raises: + FileNotFoundError: If a requested EDF is unavailable and downloading + is disabled. + """ + root = Path(self.root) + csv_path = root / self.metadata_file_name + if csv_path.exists() and self._metadata_matches_request(csv_path): + return + + rows: list[dict] = [] + for subject in self.subjects: + paths_by_run: dict[int, Path] = {} + if self.download: + downloaded = mne.datasets.eegbci.load_data( + subject, self.runs, path=str(root), update_path=False + ) + for path in downloaded: + p = Path(path) + for run in self.runs: + if p.name == f"S{subject:03d}R{run:02d}.edf": + paths_by_run[run] = p + for run in self.runs: + signal_file = paths_by_run.get(run) or self._find_local_edf(subject, run) + if signal_file is None: + raise FileNotFoundError( + f"Missing EEGBCI EDF for subject {subject}, run {run}. " + "Pass download=True to fetch it with MNE." + ) + rows.append( + { + "patient_id": f"S{subject:03d}", + "record_id": f"R{run:02d}", + "subject_id": int(subject), + "run": int(run), + "run_type": run_type_for_run(run), + "signal_file": str(signal_file), + "source": "physionet_eegbci", + } + ) + + df = pd.DataFrame(rows) + df.sort_values(["subject_id", "run"], inplace=True) + df.reset_index(drop=True, inplace=True) + csv_path.parent.mkdir(parents=True, exist_ok=True) + df.to_csv(csv_path, index=False) + logger.info("Wrote EEGBCI metadata to %s", csv_path) + + @property + def default_task(self) -> EEGMotorImageryEEGBCI: + """Return the canonical supervised EEGBCI task. + + Returns: + An :class:`EEGMotorImageryEEGBCI` task. + """ + return EEGMotorImageryEEGBCI() diff --git a/pyhealth/tasks/__init__.py b/pyhealth/tasks/__init__.py index cc95ef94e..406b457f2 100644 --- a/pyhealth/tasks/__init__.py +++ b/pyhealth/tasks/__init__.py @@ -69,6 +69,10 @@ EEGEventsTUEV, EEGAbnormalTUAB ) +from .eegbci import ( + EEGBCIPatternDiscovery as EEGBCIPatternDiscovery, + EEGMotorImageryEEGBCI as EEGMotorImageryEEGBCI, +) from .variant_classification import ( MutationPathogenicityPrediction, VariantClassificationClinVar, diff --git a/pyhealth/tasks/eegbci.py b/pyhealth/tasks/eegbci.py new file mode 100644 index 000000000..b000bcda3 --- /dev/null +++ b/pyhealth/tasks/eegbci.py @@ -0,0 +1,586 @@ +"""Tasks and signal helpers for PhysioNet EEGBCI recordings.""" + +from __future__ import annotations + +from typing import Any, Dict, List, Tuple + +import mne +import numpy as np +import torch + +from pyhealth.tasks import BaseTask + +EEGBCI_RUN_TYPES = { + 3: "motor_execution_left_right", + 4: "motor_imagery_left_right", + 5: "motor_execution_fists_feet", + 6: "motor_imagery_fists_feet", + 7: "motor_execution_left_right", + 8: "motor_imagery_left_right", + 9: "motor_execution_fists_feet", + 10: "motor_imagery_fists_feet", + 11: "motor_execution_left_right", + 12: "motor_imagery_left_right", + 13: "motor_execution_fists_feet", + 14: "motor_imagery_fists_feet", +} + +EEGBCI_LABELS = { + "rest": 0, + "execute_left_fist": 1, + "execute_right_fist": 2, + "imagine_left_fist": 3, + "imagine_right_fist": 4, + "execute_both_fists": 5, + "execute_both_feet": 6, + "imagine_both_fists": 7, + "imagine_both_feet": 8, +} + + +def run_type_for_run(run: int) -> str: + """Return the experimental condition for an EEGBCI run. + + Args: + run: EEGBCI run identifier. + + Returns: + The experimental condition name. + + Raises: + ValueError: If the run is not supported. + + Examples: + >>> run_type_for_run(3) + 'motor_execution_left_right' + """ + try: + return EEGBCI_RUN_TYPES[int(run)] + except KeyError as exc: + raise ValueError(f"Unsupported EEGBCI run: {run}") from exc + + +def label_family_for_run(run: int) -> str: + """Return the execution, imagery, or baseline family for a run. + + Args: + run: EEGBCI run identifier. + + Returns: + The label family. + + Raises: + ValueError: If the run is not supported. + + Examples: + >>> label_family_for_run(4) + 'motor_imagery' + """ + run_type = run_type_for_run(run) + if "execution" in run_type: + return "motor_execution" + if "imagery" in run_type: + return "motor_imagery" + return "baseline" + + +def task_label_for_event(run: int, event_code: str) -> str: + """Decode a T0/T1/T2 annotation using its run context. + + Args: + run: EEGBCI run identifier. + event_code: Annotation code such as ``"T0"``, ``"T1"``, or ``"T2"``. + + Returns: + The semantic motor-task label. + + Raises: + ValueError: If the run or event code is not supported. + + Examples: + >>> task_label_for_event(3, "T1") + 'execute_left_fist' + """ + code = str(event_code).strip() + if code == "T0": + return "rest" + run_type = run_type_for_run(run) + mapping = { + "motor_execution_left_right": { + "T1": "execute_left_fist", + "T2": "execute_right_fist", + }, + "motor_imagery_left_right": { + "T1": "imagine_left_fist", + "T2": "imagine_right_fist", + }, + "motor_execution_fists_feet": { + "T1": "execute_both_fists", + "T2": "execute_both_feet", + }, + "motor_imagery_fists_feet": { + "T1": "imagine_both_fists", + "T2": "imagine_both_feet", + }, + } + try: + return mapping[run_type][code] + except KeyError as exc: + raise ValueError(f"Unsupported EEGBCI event {event_code!r} for run {run}") from exc + + +def numeric_label_for_task(task_label: str) -> int: + """Return the stable PyHealth 0-8 task-class identifier. + + Args: + task_label: Semantic EEGBCI task label. + + Returns: + The PyHealth task-class identifier. + + Raises: + ValueError: If the task label is not supported. + + Examples: + >>> numeric_label_for_task("imagine_both_feet") + 8 + """ + try: + return EEGBCI_LABELS[task_label] + except KeyError as exc: + raise ValueError(f"Unsupported EEGBCI task label: {task_label}") from exc + + +EEGBCI_COMPAT_CHANNELS = ( + "FC5", + "FC3", + "FC1", + "FC2", + "FC4", + "FC6", + "C5", + "C3", + "C1", + "C2", + "C4", + "C6", + "CP5", + "CP3", + "CP4", + "CP6", +) + + +def normalize_eegbci_channel_name(name: str) -> str: + """Normalize an EDF channel name and known aliases. + + Args: + name: Source channel name. + + Returns: + The normalized channel name. + + Examples: + >>> normalize_eegbci_channel_name("EEG C3-REF") + 'C3' + """ + clean = name.upper().replace(".", "").replace("EEG ", "").replace("-REF", "") + aliases = { + "T9": "FT9", + "T10": "FT10", + } + return aliases.get(clean, clean) + + +def select_eegbci_channels( + data: np.ndarray, + ch_names: List[str], + channel_mode: str = "compat16", +) -> Tuple[np.ndarray, List[str]]: + """Select all EEG channels or the compatibility montage. + + Args: + data: EEG data with shape ``(channels, time)``. + ch_names: Channel names matching the first data dimension. + channel_mode: ``"compat16"`` or ``"all"``. + + Returns: + The selected data and corresponding channel names. + + Raises: + ValueError: If the mode is invalid or required channels are missing. + + Examples: + >>> data = np.zeros((len(EEGBCI_COMPAT_CHANNELS), 400)) + >>> selected, _ = select_eegbci_channels( + ... data, list(EEGBCI_COMPAT_CHANNELS) + ... ) + >>> selected.shape + (16, 400) + """ + if channel_mode == "all": + return data, list(ch_names) + if channel_mode != "compat16": + raise ValueError("channel_mode must be one of {'compat16', 'all'}") + + normalized_to_index = { + normalize_eegbci_channel_name(name): idx for idx, name in enumerate(ch_names) + } + missing = [ch for ch in EEGBCI_COMPAT_CHANNELS if ch not in normalized_to_index] + if missing: + raise ValueError(f"Missing EEGBCI channels for compat16 mode: {missing}") + indices = [normalized_to_index[ch] for ch in EEGBCI_COMPAT_CHANNELS] + return data[indices], list(EEGBCI_COMPAT_CHANNELS) + + +def normalize_signal(signal: np.ndarray, mode: str | None) -> np.ndarray: + """Apply the configured per-channel signal normalization. + + Args: + signal: EEG signal with time on the final dimension. + mode: ``"95th_percentile"``, ``"div_by_100"``, or ``None``. + + Returns: + The normalized signal. + + Raises: + ValueError: If the normalization mode is unsupported. + + Examples: + >>> normalize_signal(np.array([[0.0, 100.0]]), "div_by_100").tolist() + [[0.0, 1.0]] + """ + if mode is None: + return signal + if mode == "95th_percentile": + scale = np.quantile( + np.abs(signal), q=0.95, axis=-1, method="linear", keepdims=True + ) + return signal / (scale + 1e-8) + if mode == "div_by_100": + return signal / 100.0 + raise ValueError("normalization must be one of {None, '95th_percentile', 'div_by_100'}") + + +BANDS = { + "delta": (0.5, 4.0), + "theta": (4.0, 8.0), + "alpha": (8.0, 13.0), + "beta": (13.0, 30.0), + "gamma": (30.0, 45.0), +} + + +def compute_band_powers(data: np.ndarray, sfreq: float) -> Dict[str, float | str]: + """Compute absolute and relative Welch band powers and ratios. + + Args: + data: EEG data with shape ``(channels, time)``. + sfreq: Sampling rate in hertz. + + Returns: + Band powers, relative powers, ratios, and the dominant band. + + Raises: + ValueError: If data is not two-dimensional. + + Examples: + >>> time = np.arange(400) / 200 + >>> signal = np.sin(2 * np.pi * 10 * time) + >>> compute_band_powers(signal[None, :], 200)["dominant_band"] + 'alpha' + """ + from scipy.signal import welch + + if data.ndim != 2: + raise ValueError("data must have shape (channels, time)") + nperseg = min(data.shape[-1], int(sfreq * 2)) + freqs, psd = welch(data, fs=sfreq, nperseg=nperseg, axis=-1) + mean_psd = psd.mean(axis=0) + + features: Dict[str, float | str] = {} + total_power = 0.0 + band_values: Dict[str, float] = {} + for band, (low, high) in BANDS.items(): + mask = (freqs >= low) & (freqs < high) + value = float(np.trapezoid(mean_psd[mask], freqs[mask])) if np.any(mask) else 0.0 + features[f"{band}_power"] = value + band_values[band] = value + total_power += value + + denom = total_power + 1e-12 + for band, value in band_values.items(): + features[f"{band}_relative"] = float(value / denom) + + features["dominant_band"] = max(band_values, key=band_values.get) + features["alpha_beta_ratio"] = float( + band_values["alpha"] / (band_values["beta"] + 1e-12) + ) + features["theta_beta_ratio"] = float( + band_values["theta"] / (band_values["beta"] + 1e-12) + ) + return features + + +def interpret_band_profile(features: Dict[str, float | str]) -> Dict[str, str]: + """Produce cautious exploratory interpretation metadata. + + Args: + features: Band-power features from :func:`compute_band_powers`. + + Returns: + A signal-pattern hypothesis, confidence, quality flags, and summary. + + Examples: + >>> features = { + ... "dominant_band": "alpha", + ... "alpha_relative": 0.6, + ... "alpha_beta_ratio": 3.0, + ... } + >>> interpret_band_profile(features)["brain_state_hypothesis"] + 'relaxed_or_idle' + """ + dominant = str(features["dominant_band"]) + alpha_rel = float(features.get("alpha_relative", 0.0)) + beta_rel = float(features.get("beta_relative", 0.0)) + theta_rel = float(features.get("theta_relative", 0.0)) + gamma_rel = float(features.get("gamma_relative", 0.0)) + alpha_beta = float(features.get("alpha_beta_ratio", 0.0)) + theta_beta = float(features.get("theta_beta_ratio", 0.0)) + + quality_flags: List[str] = [] + hypothesis = "mixed_frequency_profile" + confidence = "low" + + if dominant == "alpha" and alpha_rel >= 0.45 and alpha_beta >= 2.0: + hypothesis = "relaxed_or_idle" + confidence = "medium" + elif dominant == "beta" and beta_rel >= 0.35: + hypothesis = "active_sensorimotor_processing" + confidence = "medium" + elif dominant == "theta" and theta_rel >= 0.35 and theta_beta >= 1.5: + hypothesis = "slow_wave_or_drowsy_pattern" + confidence = "medium" + elif dominant == "gamma" and gamma_rel >= 0.30: + hypothesis = "high_frequency_or_artifact_pattern" + confidence = "low" + quality_flags.append("possible_muscle_artifact") + + if confidence == "low": + quality_flags.append("low_confidence") + + return { + "brain_state_hypothesis": hypothesis, + "confidence": confidence, + "quality_flags": ";".join(quality_flags) if quality_flags else "none", + "interpretation": ( + f"The segment is consistent with {hypothesis} based on a " + f"{dominant}-dominant frequency profile." + ), + } + + +def iter_annotation_windows( + raw: mne.io.BaseRaw, + run: int, + window_size: float = 2.0, +) -> List[Dict[str, Any]]: + """Convert T0/T1/T2 annotations into complete fixed-duration windows. + + Args: + raw: Loaded MNE recording with annotations. + run: EEGBCI run identifier used to decode task labels. + window_size: Window duration in seconds. + + Returns: + Window metadata dictionaries for complete annotation windows. + + Raises: + ValueError: If a supported annotation cannot be decoded for the run. + + Examples: + >>> info = mne.create_info(["C3"], 200, "eeg") + >>> raw = mne.io.RawArray(np.zeros((1, 400)), info, verbose="error") + >>> _ = raw.set_annotations(mne.Annotations([0.0], [2.0], ["T0"])) + >>> len(iter_annotation_windows(raw, run=3)) + 1 + """ + sfreq = float(raw.info["sfreq"]) + window_samples = int(round(window_size * sfreq)) + windows: List[Dict[str, Any]] = [] + for idx, annotation in enumerate(raw.annotations): + event_code = str(annotation["description"]) + if event_code not in {"T0", "T1", "T2"}: + continue + start_sample = int( + raw.time_as_index([float(annotation["onset"])], use_rounding=True)[0] + ) + duration_samples = int(round(float(annotation["duration"]) * sfreq)) + n_full_windows = duration_samples // window_samples + for window_idx in range(n_full_windows): + s0 = start_sample + window_idx * window_samples + s1 = s0 + window_samples + task_label = task_label_for_event(run, event_code) + windows.append( + { + "trial_id": f"ann{idx:04d}_win{window_idx:03d}", + "event_code": event_code, + "task_label": task_label, + "label_family": label_family_for_run(run), + "label": numeric_label_for_task(task_label), + "start_time": s0 / sfreq, + "end_time": s1 / sfreq, + "start_sample": s0, + "end_sample": s1, + } + ) + return windows + + +class EEGMotorImageryEEGBCI(BaseTask): + """Build fixed-duration EEGBCI motor-task samples. + + Args: + window_size: Window duration in seconds. + resample_rate: Target sampling rate, or ``None`` to retain the source rate. + bandpass_filter: Low and high cutoff frequencies, or ``None`` to disable + filtering. + channel_mode: ``"compat16"`` for the shared 16-channel montage or ``"all"`` + for all EEG channels. + normalization: ``"95th_percentile"``, ``"div_by_100"``, or ``None``. + compute_stft: Whether to include an STFT tensor. + + Each emitted sample includes patient/run/trial metadata, ``signal``, semantic + ``task_label`` and processor ``label`` strings, integer ``eegbci_label`` as a + PyHealth task-class identifier, channel names, sample rate, and window timing. + When enabled, ``stft`` is also included. + + Examples: + >>> task = EEGMotorImageryEEGBCI(compute_stft=False) + >>> task.task_name + 'EEGBCI_motor_imagery' + """ + + task_name: str = "EEGBCI_motor_imagery" + input_schema: Dict[str, str] = {"signal": "tensor", "stft": "tensor"} + output_schema: Dict[str, str] = {"label": "multiclass"} + + def __init__( + self, + window_size: float = 2.0, + resample_rate: float | None = 200, + bandpass_filter: Tuple[float, float] | None = (0.5, 45.0), + channel_mode: str = "compat16", + normalization: str | None = "95th_percentile", + compute_stft: bool = True, + ) -> None: + super().__init__() + self.cache_version = "semantic_labels_v1" + self.window_size = window_size + self.resample_rate = resample_rate + self.bandpass_filter = bandpass_filter + self.channel_mode = channel_mode + self.normalization = normalization + self.compute_stft = compute_stft + if not compute_stft: + self.input_schema = {"signal": "tensor"} + + def __call__(self, patient: Any) -> List[Dict[str, Any]]: + return self._base_samples_from_patient(patient) + + def read_raw(self, signal_file: str) -> mne.io.BaseRaw: + """Load an EDF and apply configured filtering and resampling. + + Args: + signal_file: EDF file path. + + Returns: + The preprocessed MNE recording. + """ + raw = mne.io.read_raw_edf(signal_file, preload=True, verbose="error") + raw.pick_types(eeg=True, stim=False, exclude=[]) + if self.bandpass_filter is not None: + raw.filter( + l_freq=self.bandpass_filter[0], + h_freq=self.bandpass_filter[1], + verbose="error", + ) + if self.resample_rate is not None: + raw.resample(self.resample_rate, n_jobs=1, verbose="error") + return raw + + def _base_samples_from_patient(self, patient: Any) -> List[Dict[str, Any]]: + samples: List[Dict[str, Any]] = [] + for event in patient.get_events("records"): + raw = self.read_raw(event.signal_file) + data = raw.get_data(units="uV") + selected, selected_names = select_eegbci_channels( + data, raw.ch_names, self.channel_mode + ) + selected = normalize_signal(selected, self.normalization) + sfreq = float(raw.info["sfreq"]) + for idx, window in enumerate( + iter_annotation_windows(raw, int(event.run), self.window_size) + ): + signal_np = selected[:, window["start_sample"] : window["end_sample"]] + if signal_np.shape[-1] != int(round(self.window_size * sfreq)): + continue + signal = torch.FloatTensor(signal_np) + sample = { + "patient_id": patient.patient_id, + "record_id": event.record_id, + "subject_id": int(event.subject_id), + "run": int(event.run), + "run_type": event.run_type, + "signal_file": event.signal_file, + "trial_id": f"{patient.patient_id}_{event.record_id}_{idx:04d}", + "event_code": window["event_code"], + "task_label": window["task_label"], + "label_family": window["label_family"], + "label": str(window["task_label"]), + "eegbci_label": int(window["label"]), + "signal": signal, + "channel_names": selected_names, + "start_time": window["start_time"], + "end_time": window["end_time"], + "sample_rate": sfreq, + } + if self.compute_stft: + from pyhealth.models.tfm_tokenizer import get_stft_torch + + sample["stft"] = get_stft_torch( + signal.unsqueeze(0), resampling_rate=int(round(sfreq)) + ).squeeze(0) + samples.append(sample) + raw.close() + return samples + + +class EEGBCIPatternDiscovery(EEGMotorImageryEEGBCI): + """Extend EEGBCI motor-task samples with exploratory band metadata. + + Each emitted sample contains the supervised-task fields from + :class:`EEGMotorImageryEEGBCI` plus ``bandpower``, + ``brain_state_hypothesis``, ``confidence``, ``quality_flags``, and + ``interpretation``. These fields describe signal patterns and are not clinical + diagnoses. + + Examples: + >>> task = EEGBCIPatternDiscovery(compute_stft=False) + >>> task.task_name + 'EEGBCI_pattern_discovery' + """ + + task_name: str = "EEGBCI_pattern_discovery" + + def __call__(self, patient: Any) -> List[Dict[str, Any]]: + samples = self._base_samples_from_patient(patient) + for sample in samples: + features = compute_band_powers( + sample["signal"].detach().cpu().numpy(), + float(sample["sample_rate"]), + ) + interpretation = interpret_band_profile(features) + sample["bandpower"] = features + sample.update(interpretation) + return samples diff --git a/tests/core/test_eegbci.py b/tests/core/test_eegbci.py new file mode 100644 index 000000000..e775510df --- /dev/null +++ b/tests/core/test_eegbci.py @@ -0,0 +1,1572 @@ +import os +import sys +import unittest +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import List +from unittest.mock import patch + +import numpy as np +import pandas as pd +import torch + +from pyhealth.tasks.eegbci import ( + EEGBCI_LABELS, + label_family_for_run, + numeric_label_for_task, + run_type_for_run, + task_label_for_event, +) + + +class TestEEGBCIHelpers(unittest.TestCase): + def test_run_type_for_run(self): + self.assertEqual(run_type_for_run(3), "motor_execution_left_right") + self.assertEqual(run_type_for_run(4), "motor_imagery_left_right") + self.assertEqual(run_type_for_run(5), "motor_execution_fists_feet") + self.assertEqual(run_type_for_run(6), "motor_imagery_fists_feet") + self.assertEqual(run_type_for_run(14), "motor_imagery_fists_feet") + + def test_task_label_for_event_is_run_aware(self): + self.assertEqual(task_label_for_event(3, "T0"), "rest") + self.assertEqual(task_label_for_event(3, "T1"), "execute_left_fist") + self.assertEqual(task_label_for_event(3, "T2"), "execute_right_fist") + self.assertEqual(task_label_for_event(4, "T1"), "imagine_left_fist") + self.assertEqual(task_label_for_event(4, "T2"), "imagine_right_fist") + self.assertEqual(task_label_for_event(5, "T1"), "execute_both_fists") + self.assertEqual(task_label_for_event(5, "T2"), "execute_both_feet") + self.assertEqual(task_label_for_event(6, "T1"), "imagine_both_fists") + self.assertEqual(task_label_for_event(6, "T2"), "imagine_both_feet") + + def test_label_family_and_numeric_labels(self): + self.assertEqual(label_family_for_run(3), "motor_execution") + self.assertEqual(label_family_for_run(4), "motor_imagery") + self.assertEqual(numeric_label_for_task("rest"), 0) + self.assertEqual(numeric_label_for_task("execute_left_fist"), 1) + self.assertEqual(numeric_label_for_task("imagine_both_feet"), 8) + + def test_invalid_run_and_event_raise_clear_errors(self): + with self.assertRaisesRegex(ValueError, "Unsupported EEGBCI run"): + run_type_for_run(2) + with self.assertRaisesRegex(ValueError, "Unsupported EEGBCI event"): + task_label_for_event(3, "BAD") + + def test_select_eegbci_channels_compat16(self): + from pyhealth.tasks.eegbci import EEGBCI_COMPAT_CHANNELS, select_eegbci_channels + + ch_names = list(EEGBCI_COMPAT_CHANNELS) + ["EXTRA"] + data = np.arange(len(ch_names) * 100, dtype=float).reshape(len(ch_names), 100) + selected, selected_names = select_eegbci_channels(data, ch_names, "compat16") + self.assertEqual(selected.shape, (16, 100)) + self.assertEqual(selected_names, list(EEGBCI_COMPAT_CHANNELS)) + np.testing.assert_allclose(selected[0], data[0]) + + def test_select_eegbci_channels_all(self): + from pyhealth.tasks.eegbci import select_eegbci_channels + + data = np.ones((64, 50)) + ch_names = [f"CH{i}" for i in range(64)] + selected, selected_names = select_eegbci_channels(data, ch_names, "all") + self.assertEqual(selected.shape, (64, 50)) + self.assertEqual(selected_names, ch_names) + + def test_select_eegbci_channels_missing_channel_raises(self): + from pyhealth.tasks.eegbci import select_eegbci_channels + + with self.assertRaisesRegex(ValueError, "Missing EEGBCI channels"): + select_eegbci_channels(np.ones((2, 20)), ["C3", "C4"], "compat16") + + def test_normalize_signal_95th_percentile(self): + from pyhealth.tasks.eegbci import normalize_signal + + signal = np.array([[0.0, 1.0, 2.0, 100.0], [0.0, -2.0, 2.0, 4.0]]) + normalized = normalize_signal(signal, "95th_percentile") + self.assertEqual(normalized.shape, signal.shape) + self.assertLess(np.max(np.abs(normalized[0])), 2.0) + + def test_compute_band_powers_detects_alpha_sinusoid(self): + from pyhealth.tasks.eegbci import compute_band_powers + + sfreq = 200.0 + times = np.arange(0, 2, 1 / sfreq) + alpha = np.sin(2 * np.pi * 10 * times) + data = np.stack([alpha, alpha]) + features = compute_band_powers(data, sfreq) + self.assertEqual(features["dominant_band"], "alpha") + self.assertGreater(features["alpha_relative"], 0.5) + self.assertGreater(features["alpha_beta_ratio"], 1.0) + + def test_compute_band_powers_detects_beta_sinusoid(self): + from pyhealth.tasks.eegbci import compute_band_powers + + sfreq = 200.0 + times = np.arange(0, 2, 1 / sfreq) + beta = np.sin(2 * np.pi * 20 * times) + data = np.stack([beta, beta]) + features = compute_band_powers(data, sfreq) + self.assertEqual(features["dominant_band"], "beta") + self.assertGreater(features["beta_relative"], 0.5) + + def test_interpret_band_profile_returns_cautious_metadata(self): + from pyhealth.tasks.eegbci import interpret_band_profile + + interpretation = interpret_band_profile( + { + "dominant_band": "alpha", + "alpha_relative": 0.65, + "beta_relative": 0.10, + "theta_relative": 0.10, + "gamma_relative": 0.05, + "alpha_beta_ratio": 6.5, + "theta_beta_ratio": 1.0, + } + ) + self.assertEqual(interpretation["brain_state_hypothesis"], "relaxed_or_idle") + self.assertIn(interpretation["confidence"], {"low", "medium", "high"}) + self.assertIn("consistent with", interpretation["interpretation"]) + self.assertNotIn( + "This is exploratory signal metadata", interpretation["interpretation"] + ) + self.assertNotIn("clinical diagnosis", interpretation["interpretation"]) + + +from pyhealth.datasets.eegbci import EEGBCIDataset + + +class TestEEGBCIDataset(unittest.TestCase): + def _set_metadata_identity(self, ds): + ds.selection_key = ds._build_selection_key() + ds.metadata_file_name = ds._metadata_file_name() + + def test_prepare_metadata_with_existing_files(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + edf = root / "files" / "eegmmidb" / "1.0.0" / "S001" / "S001R03.edf" + edf.parent.mkdir(parents=True) + edf.write_bytes(b"") + + ds = EEGBCIDataset.__new__(EEGBCIDataset) + ds.root = str(root) + ds.subjects = [1] + ds.runs = [3] + ds.download = False + self._set_metadata_identity(ds) + ds.prepare_metadata() + + csv_path = root / ds.metadata_file_name + self.assertTrue(csv_path.exists()) + df = pd.read_csv(csv_path) + self.assertEqual(len(df), 1) + self.assertEqual(df.loc[0, "patient_id"], "S001") + self.assertEqual(df.loc[0, "record_id"], "R03") + self.assertEqual(df.loc[0, "subject_id"], 1) + self.assertEqual(df.loc[0, "run"], 3) + self.assertEqual(df.loc[0, "run_type"], "motor_execution_left_right") + self.assertEqual(df.loc[0, "source"], "physionet_eegbci") + + edf.unlink() + self.assertFalse(ds._metadata_matches_request(csv_path)) + + def test_metadata_content_changes_cache_identity(self): + with tempfile.TemporaryDirectory() as tmp: + ds = EEGBCIDataset.__new__(EEGBCIDataset) + ds.root = tmp + ds.metadata_file_name = "metadata.csv" + csv_path = Path(tmp) / ds.metadata_file_name + + csv_path.write_text("signal_file\n/old/path.edf\n", encoding="utf-8") + first_key = ds._metadata_cache_key() + csv_path.write_text("signal_file\n/new/path.edf\n", encoding="utf-8") + + self.assertNotEqual(first_key, ds._metadata_cache_key()) + + def test_selection_inputs_are_normalized_for_stable_identity(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for subject, run in [(1, 3), (1, 4), (2, 3), (2, 4)]: + edf = ( + root + / "files" + / "eegmmidb" + / "1.0.0" + / f"S{subject:03d}" + / f"S{subject:03d}R{run:02d}.edf" + ) + edf.parent.mkdir(parents=True, exist_ok=True) + edf.write_bytes(b"") + + first = EEGBCIDataset( + root=str(root), + subjects=[2, 1, 1], + runs=[4, 3, 4], + download=False, + ) + second = EEGBCIDataset( + root=str(root), + subjects=[1, 2], + runs=[3, 4], + download=False, + ) + + self.assertEqual(first.subjects, [1, 2]) + self.assertEqual(first.runs, [3, 4]) + self.assertEqual(first.selection_key, second.selection_key) + self.assertEqual(first.dataset_name, second.dataset_name) + + def test_prepare_metadata_uses_selection_specific_files(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + first = root / "files" / "eegmmidb" / "1.0.0" / "S001" / "S001R03.edf" + second = root / "files" / "eegmmidb" / "1.0.0" / "S002" / "S002R04.edf" + first.parent.mkdir(parents=True) + second.parent.mkdir(parents=True) + first.write_bytes(b"") + second.write_bytes(b"") + + ds_first = EEGBCIDataset( + root=str(root), subjects=[1], runs=[3], download=False + ) + ds_second = EEGBCIDataset( + root=str(root), subjects=[2], runs=[4], download=False + ) + + first_csv = root / ds_first.metadata_file_name + second_csv = root / ds_second.metadata_file_name + self.assertNotEqual(first_csv, second_csv) + self.assertTrue(first_csv.exists()) + self.assertTrue(second_csv.exists()) + + first_df = pd.read_csv(first_csv) + second_df = pd.read_csv(second_csv) + self.assertEqual(first_df.loc[0, "subject_id"], 1) + self.assertEqual(first_df.loc[0, "run"], 3) + self.assertEqual(second_df.loc[0, "subject_id"], 2) + self.assertEqual(second_df.loc[0, "run"], 4) + + def test_find_local_edf_checks_canonical_mne_path_first(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + canonical = root / "files" / "eegmmidb" / "1.0.0" / "S001" / "S001R03.edf" + fallback = root / "other" / "S001R03.edf" + canonical.parent.mkdir(parents=True) + fallback.parent.mkdir(parents=True) + canonical.write_bytes(b"") + fallback.write_bytes(b"") + + ds = EEGBCIDataset.__new__(EEGBCIDataset) + ds.root = str(root) + + self.assertEqual(ds._find_local_edf(1, 3), canonical) + + def test_prepare_metadata_download_uses_mne_loader(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + fake_path = root / "S001R04.edf" + fake_path.write_bytes(b"") + ds = EEGBCIDataset.__new__(EEGBCIDataset) + ds.root = str(root) + ds.subjects = [1] + ds.runs = [4] + ds.download = True + self._set_metadata_identity(ds) + + with patch( + "pyhealth.datasets.eegbci.mne.datasets.eegbci.load_data", + return_value=[str(fake_path)], + ) as load_data: + ds.prepare_metadata() + + load_data.assert_called_once_with(1, [4], path=str(root), update_path=False) + df = pd.read_csv(root / ds.metadata_file_name) + self.assertEqual(df.loc[0, "record_id"], "R04") + self.assertEqual(df.loc[0, "run_type"], "motor_imagery_left_right") + + def test_prepare_metadata_missing_local_file_raises(self): + with tempfile.TemporaryDirectory() as tmp: + ds = EEGBCIDataset.__new__(EEGBCIDataset) + ds.root = tmp + ds.subjects = [1] + ds.runs = [3] + ds.download = False + self._set_metadata_identity(ds) + with self.assertRaisesRegex(FileNotFoundError, "download=True"): + ds.prepare_metadata() + + def test_default_task_returns_motor_imagery(self): + from pyhealth.tasks.eegbci import EEGMotorImageryEEGBCI + + ds = EEGBCIDataset.__new__(EEGBCIDataset) + self.assertIs(type(ds.default_task), EEGMotorImageryEEGBCI) + + def test_dataset_set_task_offline_integration(self): + import mne + from pyhealth.tasks.eegbci import EEGBCI_COMPAT_CHANNELS + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + edf = root / "files" / "eegmmidb" / "1.0.0" / "S001" / "S001R03.edf" + edf.parent.mkdir(parents=True) + edf.write_bytes(b"") + sfreq = 160.0 + times = np.arange(0, 2, 1 / sfreq) + signal = np.sin(2 * np.pi * 10 * times) + raw = mne.io.RawArray( + np.tile(signal, (16, 1)), + mne.create_info( + list(EEGBCI_COMPAT_CHANNELS), sfreq=sfreq, ch_types=["eeg"] * 16 + ), + verbose="error", + ) + raw.set_annotations( + mne.Annotations(onset=[0.0], duration=[2.0], description=["T1"]) + ) + dataset = EEGBCIDataset( + root=str(root), + subjects=[1], + runs=[3], + download=False, + cache_dir=root / "cache", + ) + + with patch("pyhealth.tasks.eegbci.mne.io.read_raw_edf", return_value=raw): + sample_dataset = dataset.set_task(num_workers=1) + + self.assertEqual(len(sample_dataset), 1) + self.assertEqual(sample_dataset.task_name, "EEGBCI_motor_imagery") + sample = sample_dataset[0] + self.assertEqual(sample["task_label"], "execute_left_fist") + self.assertEqual(sample["eegbci_label"], 1) + self.assertEqual(tuple(sample["signal"].shape), (16, 400)) + self.assertIn("stft", sample) + + +from pyhealth.tasks.eegbci import EEGBCIPatternDiscovery, EEGMotorImageryEEGBCI + + +@dataclass +class _EEGBCIEvent: + signal_file: str + record_id: str = "R03" + subject_id: int = 1 + run: int = 3 + run_type: str = "motor_execution_left_right" + source: str = "physionet_eegbci" + + +class _EEGBCIPatient: + def __init__(self, patient_id: str, events: List[_EEGBCIEvent]): + self.patient_id = patient_id + self._events = events + + def get_events(self, event_type=None) -> List[_EEGBCIEvent]: + if event_type not in (None, "records"): + return [] + return self._events + + +class TestEEGBCITasks(unittest.TestCase): + def test_task_schema_attributes(self): + task = EEGMotorImageryEEGBCI() + self.assertEqual(task.task_name, "EEGBCI_motor_imagery") + self.assertEqual(task.input_schema, {"signal": "tensor", "stft": "tensor"}) + self.assertEqual(task.output_schema, {"label": "multiclass"}) + self.assertEqual(task.cache_version, "semantic_labels_v1") + + def test_task_schema_without_stft(self): + task = EEGMotorImageryEEGBCI(compute_stft=False) + self.assertEqual(task.input_schema, {"signal": "tensor"}) + + def test_pattern_discovery_schema_attributes(self): + task = EEGBCIPatternDiscovery(compute_stft=False) + self.assertEqual(task.task_name, "EEGBCI_pattern_discovery") + self.assertEqual(task.input_schema, {"signal": "tensor"}) + + def test_iter_annotation_windows_uses_full_2s_windows(self): + import mne + from pyhealth.tasks.eegbci import iter_annotation_windows + + sfreq = 200.0 + raw = mne.io.RawArray( + np.zeros((2, int(sfreq * 6))), + mne.create_info(["C3", "C4"], sfreq=sfreq, ch_types=["eeg", "eeg"]), + verbose="error", + ) + raw.set_annotations( + mne.Annotations(onset=[0.5, 2.0], duration=[1.0, 3.0], description=["T0", "T1"]) + ) + windows = iter_annotation_windows(raw, run=3, window_size=2.0) + self.assertEqual(len(windows), 1) + self.assertEqual(windows[0]["event_code"], "T1") + self.assertEqual(windows[0]["task_label"], "execute_left_fist") + self.assertEqual(windows[0]["start_sample"], 400) + self.assertEqual(windows[0]["end_sample"], 800) + + def test_motor_imagery_task_returns_samples_from_raw(self): + import mne + + sfreq = 200.0 + raw = mne.io.RawArray( + np.ones((16, int(sfreq * 5))), + mne.create_info( + list( + __import__( + "pyhealth.tasks.eegbci", fromlist=["EEGBCI_COMPAT_CHANNELS"] + ).EEGBCI_COMPAT_CHANNELS + ), + sfreq=sfreq, + ch_types=["eeg"] * 16, + ), + verbose="error", + ) + raw.set_annotations(mne.Annotations(onset=[0.0], duration=[2.0], description=["T1"])) + patient = _EEGBCIPatient("S001", [_EEGBCIEvent(signal_file="dummy.edf")]) + task = EEGMotorImageryEEGBCI(compute_stft=False, resample_rate=None, bandpass_filter=None) + + with patch("pyhealth.tasks.eegbci.mne.io.read_raw_edf", return_value=raw): + samples = task(patient) + + self.assertEqual(len(samples), 1) + sample = samples[0] + self.assertEqual(sample["patient_id"], "S001") + self.assertEqual(sample["record_id"], "R03") + self.assertEqual(sample["event_code"], "T1") + self.assertEqual(sample["task_label"], "execute_left_fist") + self.assertEqual(sample["label"], "execute_left_fist") + self.assertEqual(sample["eegbci_label"], 1) + self.assertEqual(tuple(sample["signal"].shape), (16, 400)) + + def test_sparse_task_labels_remain_distinct_for_multiclass_processing(self): + from pyhealth.processors import MultiClassLabelProcessor + + labels = ["rest", "execute_both_fists", "execute_both_feet"] + processor = MultiClassLabelProcessor() + processor.fit([{"label": label} for label in labels], "label") + + self.assertEqual( + len({processor.process(label).item() for label in labels}), len(labels) + ) + + def test_stft_uses_current_sample_rate(self): + import mne + from pyhealth.tasks.eegbci import EEGBCI_COMPAT_CHANNELS + + sfreq = 100.0 + raw = mne.io.RawArray( + np.ones((16, int(sfreq * 2))), + mne.create_info( + list(EEGBCI_COMPAT_CHANNELS), sfreq=sfreq, ch_types=["eeg"] * 16 + ), + verbose="error", + ) + raw.set_annotations( + mne.Annotations(onset=[0.0], duration=[2.0], description=["T1"]) + ) + patient = _EEGBCIPatient("S001", [_EEGBCIEvent(signal_file="dummy.edf")]) + task = EEGMotorImageryEEGBCI(resample_rate=None, bandpass_filter=None) + + with patch("pyhealth.tasks.eegbci.mne.io.read_raw_edf", return_value=raw): + samples = task(patient) + + self.assertEqual(len(samples), 1) + self.assertEqual(tuple(samples[0]["stft"].shape), (16, 50, 3)) + + def test_pattern_discovery_adds_bandpower_metadata(self): + import mne + from pyhealth.tasks.eegbci import EEGBCI_COMPAT_CHANNELS + + sfreq = 200.0 + times = np.arange(0, 2, 1 / sfreq) + alpha = np.sin(2 * np.pi * 10 * times) + raw = mne.io.RawArray( + np.tile(alpha, (16, 1)), + mne.create_info(list(EEGBCI_COMPAT_CHANNELS), sfreq=sfreq, ch_types=["eeg"] * 16), + verbose="error", + ) + raw.set_annotations(mne.Annotations(onset=[0.0], duration=[2.0], description=["T0"])) + patient = _EEGBCIPatient("S001", [_EEGBCIEvent(signal_file="dummy.edf")]) + task = EEGBCIPatternDiscovery(compute_stft=False, resample_rate=None, bandpass_filter=None) + + with patch("pyhealth.tasks.eegbci.mne.io.read_raw_edf", return_value=raw): + samples = task(patient) + + self.assertEqual(len(samples), 1) + sample = samples[0] + self.assertEqual(sample["bandpower"]["dominant_band"], "alpha") + self.assertEqual(sample["brain_state_hypothesis"], "relaxed_or_idle") + self.assertIn("interpretation", sample) + + +class TestEEGBCIMomentReportHelpers(unittest.TestCase): + def _moment_row(self, **overrides): + row = { + "patient_id": "S001", + "record_id": "R03", + "subject_id": 1, + "run": 3, + "run_type": "motor_execution_left_right", + "trial_id": "S001_R03_T0_0", + "event_code": "T0", + "task_label": "rest", + "label_family": "rest", + "label": 0, + "eegbci_label": 0, + "model_label": 0, + "start_time": 0.0, + "end_time": 2.0, + "dominant_band": "alpha", + "delta_relative": 0.05, + "theta_relative": 0.10, + "alpha_relative": 0.55, + "beta_relative": 0.20, + "gamma_relative": 0.10, + "alpha_beta_ratio": 2.75, + "theta_beta_ratio": 0.50, + } + row.update(overrides) + return row + + def _sample(self, **overrides): + sample = { + "patient_id": "S001", + "record_id": "R03", + "subject_id": 1, + "run": 3, + "run_type": "motor_execution_left_right", + "trial_id": "S001_R03_T0_0", + "event_code": "T0", + "task_label": "rest", + "label_family": "rest", + "label": 0, + "eegbci_label": 0, + "start_time": 0.0, + "end_time": 2.0, + "brain_state_hypothesis": "relaxed_or_idle", + "confidence": "medium", + "quality_flags": "", + "interpretation": "Alpha-dominant profile.", + "bandpower": { + "dominant_band": "alpha", + "alpha_beta_ratio": 2.75, + "theta_beta_ratio": 0.50, + "delta_power": 0.05, + "theta_power": 0.10, + "alpha_power": 0.55, + "beta_power": 0.20, + "gamma_power": 0.10, + "delta_relative": 0.05, + "theta_relative": 0.10, + "alpha_relative": 0.55, + "beta_relative": 0.20, + "gamma_relative": 0.10, + }, + } + sample.update(overrides) + return sample + + def test_analysis_version_constant(self): + from examples.eeg.eegbci.eegbci_pattern_discovery import ANALYSIS_VERSION + + self.assertEqual(ANALYSIS_VERSION, "eegbci_pattern_moment_report_v1") + + def test_parse_int_list_strips_whitespace(self): + from examples.eeg.eegbci.eegbci_pattern_discovery import parse_int_list + + self.assertEqual(parse_int_list("1, 2, 4-6"), [1, 2, 4, 5, 6]) + + def test_parse_int_list_rejects_descending_ranges(self): + from examples.eeg.eegbci.eegbci_pattern_discovery import parse_int_list + + with self.assertRaisesRegex(ValueError, "Range start must be <= range end"): + parse_int_list("5-3") + + def test_build_rest_baselines_uses_rest_rows_only(self): + from examples.eeg.eegbci.eegbci_pattern_discovery import build_rest_baselines + + rows = [ + self._moment_row(task_label="rest", subject_id=1, run=3, alpha_relative=0.50), + self._moment_row( + task_label="execute_left_fist", subject_id=1, run=3, alpha_relative=0.90 + ), + self._moment_row(task_label="rest", subject_id=1, run=4, alpha_relative=0.70), + ] + + baselines = build_rest_baselines(rows) + + self.assertAlmostEqual( + baselines["same_subject_run"][(1, 3)]["alpha_relative"], 0.50 + ) + self.assertAlmostEqual( + baselines["same_subject_all_runs"][1]["alpha_relative"], 0.60 + ) + self.assertAlmostEqual(baselines["global_rest"]["alpha_relative"], 0.60) + + def test_build_rest_baselines_handles_no_rest_rows(self): + from examples.eeg.eegbci.eegbci_pattern_discovery import build_rest_baselines + + rows = [ + self._moment_row( + task_label="execute_left_fist", label_family="motor_execution" + ) + ] + + baselines = build_rest_baselines(rows) + + self.assertEqual(baselines["same_subject_run"], {}) + self.assertEqual(baselines["same_subject_all_runs"], {}) + self.assertIsNone(baselines["global_rest"]) + + def test_render_summary_reports_rest_baseline_source_rows(self): + from examples.eeg.eegbci.eegbci_pattern_discovery import render_summary + + rows = [ + self._moment_row(task_label="rest"), + self._moment_row( + task_label="execute_left_fist", label_family="motor_execution" + ), + self._moment_row(task_label="rest", run=4), + ] + + summary = render_summary( + rows, + { + "subjects": [1], + "runs": [3, 4], + "max_windows": None, + "baseline_row_count": 2, + "output_was_capped": False, + }, + ) + + self.assertIn("- Baseline source rows: 2", summary) + + def test_annotate_rest_fallback_scopes(self): + from examples.eeg.eegbci.eegbci_pattern_discovery import ( + annotate_moment_rows, + build_rest_baselines, + ) + + rows = [ + self._moment_row(task_label="rest", subject_id=1, run=3, alpha_relative=0.50), + self._moment_row(task_label="rest", subject_id=1, run=4, alpha_relative=0.70), + self._moment_row( + task_label="execute_left_fist", + label_family="motor_execution", + subject_id=1, + run=3, + alpha_relative=0.80, + ), + self._moment_row( + task_label="execute_left_fist", + label_family="motor_execution", + subject_id=1, + run=5, + alpha_relative=0.80, + ), + self._moment_row( + task_label="execute_left_fist", + label_family="motor_execution", + subject_id=2, + run=8, + alpha_relative=0.80, + ), + ] + + annotated = annotate_moment_rows(rows, build_rest_baselines(rows)) + + self.assertEqual(annotated[2]["rest_reference_scope"], "same_subject_run") + self.assertEqual(annotated[3]["rest_reference_scope"], "same_subject_all_runs") + self.assertEqual(annotated[4]["rest_reference_scope"], "global_rest") + + def test_derive_state_hypothesis_detects_profiles(self): + from examples.eeg.eegbci.eegbci_pattern_discovery import derive_state_hypothesis + + cases = [ + ( + self._moment_row( + alpha_relative=0.60, + beta_relative=0.12, + gamma_relative=0.05, + alpha_beta_ratio=5.0, + ), + "idle_alpha_profile", + ), + ( + self._moment_row( + alpha_relative=0.12, + beta_relative=0.48, + gamma_relative=0.16, + alpha_beta_ratio=0.25, + ), + "sensorimotor_engagement_profile", + ), + ( + self._moment_row( + delta_relative=0.42, + theta_relative=0.36, + alpha_relative=0.08, + beta_relative=0.08, + ), + "slow_wave_dominant_pattern", + ), + ( + self._moment_row( + gamma_relative=0.48, alpha_relative=0.10, beta_relative=0.12 + ), + "possible_artifact_profile", + ), + ( + self._moment_row( + delta_relative=0.18, + theta_relative=0.20, + alpha_relative=0.22, + beta_relative=0.21, + gamma_relative=0.19, + alpha_beta_ratio=1.05, + ), + "mixed_ambiguous_profile", + ), + ] + + for row, expected in cases: + with self.subTest(expected=expected): + result = derive_state_hypothesis(row) + self.assertEqual(result["state_hypothesis"], expected) + self.assertIn(result["state_confidence"], {"low", "medium", "high"}) + self.assertGreaterEqual(result["evidence_score"], 0.0) + self.assertLessEqual(result["evidence_score"], 1.0) + self.assertIn("alpha=", result["evidence_summary"]) + + def test_state_hypothesis_uses_only_finite_rest_deltas(self): + from examples.eeg.eegbci.eegbci_pattern_discovery import derive_state_hypothesis + + delta_profiles = ( + ({"alpha": 0.10, "beta": -0.05}, "idle_alpha_profile"), + ( + {"alpha": -0.10, "beta": 0.08, "gamma": 0.02}, + "sensorimotor_engagement_profile", + ), + ({"delta": 0.10, "theta": 0.08}, "slow_wave_dominant_pattern"), + ) + for deltas, expected in delta_profiles: + row = self._moment_row() + for band in ("delta", "theta", "alpha", "beta", "gamma"): + row[f"rest_{band}_relative_delta"] = deltas.get(band, 0.0) + result = derive_state_hypothesis(row) + self.assertEqual(result["state_hypothesis"], expected) + self.assertIn("basis=rest_normalized_delta", result["evidence_summary"]) + + for invalid in (float("nan"), float("inf")): + row = self._moment_row(alpha_relative=0.65, alpha_beta_ratio=6.5) + for band in ("delta", "theta", "alpha", "beta", "gamma"): + row[f"rest_{band}_relative_delta"] = invalid + result = derive_state_hypothesis(row) + self.assertTrue(np.isfinite(result["evidence_score"])) + self.assertIn("basis=absolute_band_profile", result["evidence_summary"]) + + def test_state_confidence_requires_margin(self): + from examples.eeg.eegbci.eegbci_pattern_discovery import ( + STATE_CONFIDENCE_RANK, + derive_state_hypothesis, + ) + + clear = derive_state_hypothesis( + self._moment_row( + alpha_relative=0.70, + beta_relative=0.10, + gamma_relative=0.04, + alpha_beta_ratio=6.0, + ) + ) + weaker = derive_state_hypothesis( + self._moment_row( + alpha_relative=0.40, + beta_relative=0.22, + gamma_relative=0.10, + alpha_beta_ratio=2.0, + ) + ) + + self.assertEqual(clear["state_hypothesis"], weaker["state_hypothesis"]) + self.assertGreater( + STATE_CONFIDENCE_RANK[clear["state_confidence"]], + STATE_CONFIDENCE_RANK[weaker["state_confidence"]], + ) + + def test_task_state_relation_table_is_deterministic(self): + from examples.eeg.eegbci.eegbci_pattern_discovery import ( + derive_task_state_relation, + ) + + cases = [ + ("rest", "rest", "idle_alpha_profile", "supports_label"), + ("rest", "rest", "mixed_ambiguous_profile", "ambiguous"), + ("rest", "rest", "possible_artifact_profile", "not_applicable"), + ( + "execute_left_fist", + "motor_execution", + "sensorimotor_engagement_profile", + "supports_label", + ), + ( + "imagine_left_fist", + "motor_imagery", + "sensorimotor_engagement_profile", + "adds_detail", + ), + ("execute_left_fist", "motor_execution", "idle_alpha_profile", "disagrees"), + ( + "imagine_left_fist", + "motor_imagery", + "slow_wave_dominant_pattern", + "adds_detail", + ), + ] + + for task_label, label_family, state, expected in cases: + with self.subTest(state=state, label_family=label_family): + result = derive_task_state_relation( + self._moment_row( + task_label=task_label, + label_family=label_family, + state_hypothesis=state, + ) + ) + self.assertEqual(result["task_state_relation"], expected) + self.assertIn(result["task_state_confidence"], {"low", "medium", "high"}) + self.assertGreater(len(result["task_state_rationale"]), 20) + + def test_quality_booleans_are_parseable(self): + from examples.eeg.eegbci.eegbci_pattern_discovery import derive_quality_columns + + flags = derive_quality_columns( + self._moment_row( + state_hypothesis="possible_artifact_profile", + state_confidence="low", + quality_flags="low_confidence; high_gamma", + ) + ) + + self.assertTrue(flags["is_low_confidence"]) + self.assertTrue(flags["is_possible_artifact"]) + self.assertFalse(flags["is_mixed_or_ambiguous"]) + + def test_quality_booleans_do_not_depend_on_string_parsing_only(self): + from examples.eeg.eegbci.eegbci_pattern_discovery import derive_quality_columns + + flags = derive_quality_columns( + self._moment_row( + state_hypothesis="mixed_ambiguous_profile", + state_confidence="medium", + quality_flags="", + ) + ) + + self.assertTrue(flags["is_mixed_or_ambiguous"]) + + def test_quality_booleans_do_not_conflate_legacy_low_confidence(self): + from examples.eeg.eegbci.eegbci_pattern_discovery import derive_quality_columns + + flags = derive_quality_columns( + self._moment_row( + state_hypothesis="idle_alpha_profile", + state_confidence="medium", + quality_flags="low_confidence", + ) + ) + + self.assertFalse(flags["is_low_confidence"]) + + def test_annotate_moment_rows_adds_required_fields(self): + from examples.eeg.eegbci.eegbci_pattern_discovery import ( + ANALYSIS_VERSION, + MOMENT_REPORT_COLUMNS, + annotate_moment_rows, + build_rest_baselines, + ) + + rows = [ + self._moment_row(task_label="rest", alpha_relative=0.50, beta_relative=0.20), + self._moment_row( + task_label="execute_left_fist", + label_family="motor_execution", + alpha_relative=0.20, + beta_relative=0.45, + ), + ] + + annotated = annotate_moment_rows(rows, build_rest_baselines(rows)) + + for annotated_row in annotated: + for column in MOMENT_REPORT_COLUMNS: + self.assertIn(column, annotated_row) + row = annotated[1] + self.assertEqual(row["analysis_version"], ANALYSIS_VERSION) + self.assertIn( + row["state_hypothesis"], + { + "idle_alpha_profile", + "sensorimotor_engagement_profile", + "slow_wave_dominant_pattern", + "possible_artifact_profile", + "mixed_ambiguous_profile", + }, + ) + self.assertIn("rest_alpha_relative_delta", row) + self.assertAlmostEqual(row["rest_alpha_relative_delta"], -0.30) + self.assertIn("task_state_relation", row) + self.assertIn("task_state_rationale", row) + self.assertIn("is_low_confidence", row) + + def test_annotate_moment_rows_marks_unavailable_rest(self): + from examples.eeg.eegbci.eegbci_pattern_discovery import ( + annotate_moment_rows, + build_rest_baselines, + ) + + rows = [ + self._moment_row( + task_label="execute_left_fist", label_family="motor_execution" + ) + ] + + annotated = annotate_moment_rows(rows, build_rest_baselines(rows)) + + self.assertEqual(annotated[0]["rest_reference_scope"], "unavailable") + for band in ("delta", "theta", "alpha", "beta", "gamma"): + self.assertEqual(annotated[0][f"rest_{band}_relative_delta"], "") + + def test_rest_delta_values_are_band_specific(self): + from examples.eeg.eegbci.eegbci_pattern_discovery import ( + annotate_moment_rows, + build_rest_baselines, + ) + + rows = [ + self._moment_row( + task_label="rest", + delta_relative=0.10, + theta_relative=0.20, + alpha_relative=0.30, + beta_relative=0.25, + gamma_relative=0.15, + ), + self._moment_row( + task_label="execute_left_fist", + label_family="motor_execution", + delta_relative=0.15, + theta_relative=0.18, + alpha_relative=0.25, + beta_relative=0.35, + gamma_relative=0.07, + ), + ] + + annotated = annotate_moment_rows(rows, build_rest_baselines(rows)) + + self.assertAlmostEqual(annotated[1]["rest_delta_relative_delta"], 0.05) + self.assertAlmostEqual(annotated[1]["rest_theta_relative_delta"], -0.02) + self.assertAlmostEqual(annotated[1]["rest_alpha_relative_delta"], -0.05) + self.assertAlmostEqual(annotated[1]["rest_beta_relative_delta"], 0.10) + self.assertAlmostEqual(annotated[1]["rest_gamma_relative_delta"], -0.08) + + def test_annotate_moment_rows_adds_report_interpretation(self): + from examples.eeg.eegbci.eegbci_pattern_discovery import ( + annotate_moment_rows, + build_rest_baselines, + ) + + row = self._moment_row() + + annotated = annotate_moment_rows([row], build_rest_baselines([row])) + + self.assertIn("consistent with", annotated[0]["interpretation"]) + self.assertIn("task label", annotated[0]["interpretation"]) + self.assertIn(annotated[0]["state_hypothesis"], annotated[0]["interpretation"]) + + def test_annotate_moment_rows_does_not_mutate_input_rows(self): + from examples.eeg.eegbci.eegbci_pattern_discovery import ( + annotate_moment_rows, + build_rest_baselines, + ) + + rows = [self._moment_row()] + original = [dict(row) for row in rows] + + annotate_moment_rows(rows, build_rest_baselines(rows)) + + self.assertEqual(rows, original) + + def test_select_representative_windows_is_deterministic(self): + from examples.eeg.eegbci.eegbci_pattern_discovery import ( + select_representative_windows, + ) + + rows = [ + self._moment_row( + subject_id=2, + run=4, + start_time=6.0, + state_hypothesis="idle_alpha_profile", + state_confidence="medium", + evidence_score=0.80, + ), + self._moment_row( + subject_id=1, + run=3, + start_time=4.0, + state_hypothesis="idle_alpha_profile", + state_confidence="medium", + evidence_score=0.80, + ), + self._moment_row( + subject_id=1, + run=3, + start_time=8.0, + state_hypothesis="sensorimotor_engagement_profile", + state_confidence="high", + evidence_score=0.90, + ), + self._moment_row( + subject_id=1, + run=3, + start_time=10.0, + state_hypothesis="mixed_ambiguous_profile", + state_confidence="low", + evidence_score=0.12, + ), + self._moment_row( + subject_id=1, + run=3, + start_time=12.0, + state_hypothesis="idle_alpha_profile", + task_state_relation="disagrees", + state_confidence="medium", + evidence_score=0.70, + ), + ] + + selected = select_representative_windows(rows) + + self.assertEqual(selected["cards"]["strongest_idle_like"]["subject_id"], 1) + self.assertEqual( + selected["cards"]["strongest_motor_engaged"]["state_hypothesis"], + "sensorimotor_engagement_profile", + ) + self.assertEqual(selected["cards"]["most_ambiguous"]["start_time"], 10.0) + self.assertEqual( + selected["cards"]["strongest_task_state_disagreement"][ + "task_state_relation" + ], + "disagrees", + ) + self.assertIn("strongest_artifact_like", selected["absent"]) + + def test_select_representative_windows_picks_lowest_evidence_ambiguous(self): + from examples.eeg.eegbci.eegbci_pattern_discovery import ( + select_representative_windows, + ) + + rows = [ + self._moment_row( + subject_id=2, + run=3, + start_time=4.0, + state_hypothesis="mixed_ambiguous_profile", + state_confidence="low", + evidence_score=0.20, + ), + self._moment_row( + subject_id=1, + run=3, + start_time=8.0, + state_hypothesis="mixed_ambiguous_profile", + state_confidence="low", + evidence_score=0.10, + ), + ] + + selected = select_representative_windows(rows) + + self.assertEqual(selected["cards"]["most_ambiguous"]["subject_id"], 1) + + def test_select_representative_windows_picks_strongest_disagreement(self): + from examples.eeg.eegbci.eegbci_pattern_discovery import ( + select_representative_windows, + ) + + rows = [ + self._moment_row( + subject_id=1, + run=3, + start_time=4.0, + state_hypothesis="idle_alpha_profile", + task_state_relation="disagrees", + state_confidence="medium", + evidence_score=0.50, + ), + self._moment_row( + subject_id=2, + run=3, + start_time=6.0, + state_hypothesis="idle_alpha_profile", + task_state_relation="disagrees", + state_confidence="medium", + evidence_score=0.80, + ), + ] + + selected = select_representative_windows(rows) + + self.assertEqual( + selected["cards"]["strongest_task_state_disagreement"]["subject_id"], 2 + ) + + def test_render_summary_contains_required_sections_and_limitations(self): + from examples.eeg.eegbci.eegbci_pattern_discovery import ( + ANALYSIS_VERSION, + annotate_moment_rows, + build_rest_baselines, + render_summary, + ) + + rows = [ + self._moment_row( + task_label="execute_left_fist", label_family="motor_execution" + ) + ] + annotated = annotate_moment_rows(rows, build_rest_baselines(rows)) + summary = render_summary( + annotated, + { + "subjects": [1], + "runs": [3], + "max_windows": 1, + "baseline_row_count": 1, + "output_was_capped": True, + }, + ) + + self.assertIn(ANALYSIS_VERSION, summary.splitlines()[2]) + for heading in [ + "## Executive Result", + "## Run Configuration", + "## Window Coverage", + "## Moment-State Summary", + "## Task Label x State Matrix", + "## Rest-Normalized Bandpower Summary", + "## Confidence and Quality Audit", + "## Representative Windows", + "## Limitations", + "## Next Checks", + ]: + self.assertIn(heading, summary) + self.assertIn("No rest baseline was available", summary) + self.assertIn("Output was capped by `--max-windows`", summary) + self.assertNotIn( + "Brain-state hypotheses are exploratory signal metadata", + summary.splitlines()[2], + ) + + def test_render_summary_handles_empty_rows(self): + from examples.eeg.eegbci.eegbci_pattern_discovery import render_summary + + summary = render_summary( + [], + { + "subjects": [1], + "runs": [3], + "max_windows": 0, + "baseline_row_count": 0, + "output_was_capped": True, + }, + ) + + self.assertIn("No windows were produced", summary) + self.assertIn("## Limitations", summary) + + def test_render_summary_reports_all_low_confidence_and_same_state(self): + from examples.eeg.eegbci.eegbci_pattern_discovery import render_summary + + rows = [ + self._moment_row( + state_hypothesis="mixed_ambiguous_profile", + state_confidence="low", + evidence_score=0.10, + task_state_relation="ambiguous", + task_state_confidence="low", + rest_reference_scope="unavailable", + is_low_confidence=True, + is_possible_artifact=False, + is_mixed_or_ambiguous=True, + ), + self._moment_row( + start_time=2.0, + state_hypothesis="mixed_ambiguous_profile", + state_confidence="low", + evidence_score=0.12, + task_state_relation="ambiguous", + task_state_confidence="low", + rest_reference_scope="unavailable", + is_low_confidence=True, + is_possible_artifact=False, + is_mixed_or_ambiguous=True, + ), + ] + + summary = render_summary( + rows, + { + "subjects": [1], + "runs": [3], + "max_windows": None, + "baseline_row_count": 2, + "output_was_capped": False, + }, + ) + + self.assertIn("Every window is low confidence", summary) + self.assertIn("Every window maps to the same state", summary) + + def test_render_summary_reports_task_state_matrix(self): + from examples.eeg.eegbci.eegbci_pattern_discovery import render_summary + + rows = [ + self._moment_row( + task_label="rest", + state_hypothesis="idle_alpha_profile", + state_confidence="medium", + evidence_score=0.60, + task_state_relation="supports_label", + task_state_confidence="medium", + rest_reference_scope="same_subject_run", + ), + self._moment_row( + task_label="execute_left_fist", + label_family="motor_execution", + state_hypothesis="sensorimotor_engagement_profile", + state_confidence="medium", + evidence_score=0.70, + task_state_relation="supports_label", + task_state_confidence="medium", + rest_reference_scope="same_subject_run", + ), + ] + + summary = render_summary( + rows, + { + "subjects": [1], + "runs": [3], + "max_windows": None, + "baseline_row_count": 2, + "output_was_capped": False, + }, + ) + + self.assertIn("rest x idle_alpha_profile: 1", summary) + self.assertIn( + "execute_left_fist x sensorimotor_engagement_profile: 1", summary + ) + + def test_render_summary_includes_representative_window_details(self): + from examples.eeg.eegbci.eegbci_pattern_discovery import render_summary + + row = self._moment_row( + state_hypothesis="idle_alpha_profile", + state_confidence="medium", + evidence_score=0.75, + task_state_relation="supports_label", + task_state_confidence="medium", + task_state_rationale="The idle-like alpha profile is consistent with rest.", + rest_reference_scope="same_subject_run", + rest_delta_relative_delta=0.01, + rest_theta_relative_delta=0.02, + rest_alpha_relative_delta=0.03, + rest_beta_relative_delta=-0.02, + rest_gamma_relative_delta=-0.01, + is_low_confidence=False, + is_possible_artifact=False, + is_mixed_or_ambiguous=False, + ) + + summary = render_summary( + [row], + { + "subjects": [1], + "runs": [3], + "max_windows": None, + "baseline_row_count": 1, + "output_was_capped": False, + }, + ) + + for text in [ + "Subject 1 run 3 trial S001_R03_T0_0", + "Task: rest from 0.0s to 2.0s", + "State: idle_alpha_profile", + "Dominant band: alpha", + "Rest deltas:", + "Task relation: supports_label", + "low_confidence=False", + "Rationale: The idle-like alpha profile", + ]: + self.assertIn(text, summary) + + def test_render_summary_moves_nonclinical_warning_to_limitations(self): + from examples.eeg.eegbci.eegbci_pattern_discovery import render_summary + + summary = render_summary( + [self._moment_row(state_hypothesis="idle_alpha_profile")], + { + "subjects": [1], + "runs": [3], + "max_windows": None, + "baseline_row_count": 1, + "output_was_capped": False, + }, + ) + + opening = "\n".join(summary.splitlines()[:6]) + limitations = summary.split("## Limitations", 1)[1] + self.assertNotIn("clinical findings", opening) + self.assertIn("clinical findings", limitations) + + def test_summary_text_does_not_repeat_old_row_level_caveat(self): + from examples.eeg.eegbci.eegbci_pattern_discovery import ( + annotate_moment_rows, + build_rest_baselines, + render_summary, + ) + + rows = [ + self._moment_row( + interpretation="This is exploratory signal metadata, not a diagnosis." + ) + ] + annotated = annotate_moment_rows(rows, build_rest_baselines(rows)) + + summary = render_summary( + annotated, + { + "subjects": [1], + "runs": [3], + "max_windows": None, + "baseline_row_count": 1, + "output_was_capped": False, + }, + ) + + self.assertNotIn("This is exploratory signal metadata", summary) + + def test_moment_report_columns_are_declared(self): + from examples.eeg.eegbci.eegbci_pattern_discovery import ( + MOMENT_REPORT_COLUMNS, + OUTPUT_COLUMNS, + ) + + for column in [ + "patient_id", + "task_label", + "alpha_relative", + "analysis_version", + "state_hypothesis", + "state_confidence", + "evidence_score", + "evidence_summary", + "rest_reference_scope", + "rest_alpha_relative_delta", + "task_state_relation", + "task_state_rationale", + "task_state_confidence", + "interpretation", + "is_low_confidence", + "is_possible_artifact", + "is_mixed_or_ambiguous", + ]: + self.assertIn(column, OUTPUT_COLUMNS) + self.assertIn("analysis_version", MOMENT_REPORT_COLUMNS) + + def test_output_columns_remove_legacy_task_fields(self): + from examples.eeg.eegbci.eegbci_pattern_discovery import OUTPUT_COLUMNS + + for legacy_column in [ + "brain_state_hypothesis", + "confidence", + "quality_flags", + "legacy_brain_state_hypothesis", + "legacy_confidence", + "legacy_quality_flags", + "legacy_interpretation", + ]: + self.assertNotIn(legacy_column, OUTPUT_COLUMNS) + + def test_empty_dataframe_uses_output_columns(self): + from examples.eeg.eegbci.eegbci_pattern_discovery import OUTPUT_COLUMNS + + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "empty.csv" + pd.DataFrame([], columns=OUTPUT_COLUMNS).to_csv(path, index=False) + + df = pd.read_csv(path) + + self.assertEqual(len(df), 0) + self.assertEqual(list(df.columns), list(OUTPUT_COLUMNS)) + + def test_main_max_windows_zero_writes_empty_artifacts(self): + from examples.eeg.eegbci import eegbci_pattern_discovery as example + + class FakeDataset: + def __init__(self, *args, **kwargs): + pass + + def set_task(self, task): + return [self_sample] + + self_sample = self._sample() + with tempfile.TemporaryDirectory() as tmp: + argv = [ + "eegbci_pattern_discovery.py", + "--subjects", + "1", + "--runs", + "3", + "--max-windows", + "0", + "--output-dir", + tmp, + ] + with patch.object(sys, "argv", argv), patch.object( + example, "EEGBCIDataset", FakeDataset + ): + example.main() + + csv_path = Path(tmp) / "eegbci_pattern_windows.csv" + summary_path = Path(tmp) / "eegbci_pattern_summary.md" + df = pd.read_csv(csv_path) + summary = summary_path.read_text(encoding="utf-8") + + self.assertEqual(len(df), 0) + self.assertEqual(list(df.columns), list(example.OUTPUT_COLUMNS)) + self.assertIn("No windows were produced", summary) + self.assertIn("Output was capped by `--max-windows`", summary) + + def test_main_baseline_uses_uncapped_rows(self): + from examples.eeg.eegbci import eegbci_pattern_discovery as example + + first = self._sample( + task_label="execute_left_fist", + label_family="motor_execution", + alpha_beta_ratio=0.5, + bandpower={ + **self._sample()["bandpower"], + "dominant_band": "beta", + "alpha_relative": 0.20, + "beta_relative": 0.45, + "alpha_beta_ratio": 0.5, + }, + ) + rest = self._sample( + task_label="rest", + start_time=2.0, + bandpower={ + **self._sample()["bandpower"], + "alpha_relative": 0.50, + "beta_relative": 0.20, + }, + ) + + class FakeDataset: + def __init__(self, *args, **kwargs): + pass + + def set_task(self, task): + return [first, rest] + + with tempfile.TemporaryDirectory() as tmp: + argv = [ + "eegbci_pattern_discovery.py", + "--subjects", + "1", + "--runs", + "3", + "--max-windows", + "1", + "--output-dir", + tmp, + ] + with patch.object(sys, "argv", argv), patch.object( + example, "EEGBCIDataset", FakeDataset + ): + example.main() + + df = pd.read_csv(Path(tmp) / "eegbci_pattern_windows.csv") + + self.assertEqual(len(df), 1) + self.assertEqual(df.loc[0, "rest_reference_scope"], "same_subject_run") + self.assertAlmostEqual(df.loc[0, "rest_alpha_relative_delta"], -0.30) + + def test_main_writes_analysis_version_to_every_csv_row(self): + from examples.eeg.eegbci import eegbci_pattern_discovery as example + + samples = [self._sample(), self._sample(start_time=2.0, trial_id="second")] + + class FakeDataset: + def __init__(self, *args, **kwargs): + pass + + def set_task(self, task): + return samples + + with tempfile.TemporaryDirectory() as tmp: + argv = [ + "eegbci_pattern_discovery.py", + "--subjects", + "1", + "--runs", + "3", + "--output-dir", + tmp, + ] + with patch.object(sys, "argv", argv), patch.object( + example, "EEGBCIDataset", FakeDataset + ): + example.main() + + df = pd.read_csv(Path(tmp) / "eegbci_pattern_windows.csv") + + self.assertEqual(len(df), 2) + self.assertTrue((df["analysis_version"] == example.ANALYSIS_VERSION).all()) + + def test_parse_int_list_rejects_invalid_input_loudly(self): + from examples.eeg.eegbci.eegbci_pattern_discovery import parse_int_list + + with self.assertRaises(ValueError): + parse_int_list("a") + with self.assertRaises(ValueError): + parse_int_list("3-a") + + def test_parse_int_list_accepts_ranges_and_singletons(self): + from examples.eeg.eegbci.eegbci_pattern_discovery import parse_int_list + + self.assertEqual(parse_int_list("1,3-5"), [1, 3, 4, 5]) + + +@unittest.skipUnless( + os.environ.get("PYHEALTH_RUN_REAL_EEGBCI") == "1", + "Set PYHEALTH_RUN_REAL_EEGBCI=1 to download and test real EEGBCI data.", +) +class TestEEGBCIRealDataSmoke(unittest.TestCase): + def test_real_eegbci_subject_1_run_3_pattern_discovery(self): + with tempfile.TemporaryDirectory() as tmp: + dataset = EEGBCIDataset(root=tmp, subjects=[1], runs=[3], download=True) + sample_dataset = dataset.set_task( + EEGBCIPatternDiscovery(compute_stft=False, window_size=2.0) + ) + self.assertGreater(len(sample_dataset), 0) + sample = sample_dataset[0] + self.assertIn("signal", sample) + self.assertEqual(sample["signal"].shape[0], 16) + self.assertIn(sample["task_label"], set(EEGBCI_LABELS)) + self.assertIn("bandpower", sample) + self.assertIn("brain_state_hypothesis", sample) From 132c9fbcd760fd823722ec766abe55382e7f5994 Mon Sep 17 00:00:00 2001 From: Arjun Chatterjee Date: Tue, 21 Jul 2026 11:51:23 -0700 Subject: [PATCH 18/61] Covariate Shift Conformal Prediction Fixes (#1180) * Covariate CP fixes * small edits to pass checks --- docs/api/calib.rst | 2 +- examples/cxr/covid19cxr_conformal.py | 3 +- .../predictionset/base_conformal/__init__.py | 19 ++++-- .../covariate/covariate_label.py | 60 +++++++++++++------ tests/core/test_covariate_label.py | 47 +++++++++++++++ 5 files changed, 105 insertions(+), 26 deletions(-) diff --git a/docs/api/calib.rst b/docs/api/calib.rst index 7599b2a10..7814b4719 100644 --- a/docs/api/calib.rst +++ b/docs/api/calib.rst @@ -21,7 +21,7 @@ confidence levels: - :class:`~pyhealth.calib.predictionset.LABEL`: Conformal prediction with bounded error - :class:`~pyhealth.calib.predictionset.SCRIB`: Class-specific risk control - :class:`~pyhealth.calib.predictionset.FavMac`: Value-maximizing sets with cost control -- :class:`~pyhealth.calib.predictionset.CovariateLabel`: Covariate shift adaptive conformal +- :class:`~pyhealth.calib.predictionset.CovariateLabel`: Covariate shift adaptive conformal prediction with a finite-sample correction for the calibration/test weighting - :class:`~pyhealth.calib.predictionset.ClusterLabel`: K-means cluster-based conformal prediction - :class:`~pyhealth.calib.predictionset.NeighborhoodLabel`: Neighborhood Conformal Prediction (NCP) diff --git a/examples/cxr/covid19cxr_conformal.py b/examples/cxr/covid19cxr_conformal.py index 1d1bf8508..47e2eebbc 100644 --- a/examples/cxr/covid19cxr_conformal.py +++ b/examples/cxr/covid19cxr_conformal.py @@ -171,7 +171,8 @@ print("\nCreating CovariateLabel predictor...") covariate_predictor = CovariateLabel(model=resnet, alpha=alpha) -# Calibrate with embeddings (KDEs will be fitted automatically) +# Calibrate with embeddings (KDEs will be fitted automatically). The +# calibration weights include a finite-sample correction for the test point. print("Calibrating CovariateLabel predictor...") print(" - Fitting KDEs for covariate shift correction...") covariate_predictor.calibrate( diff --git a/pyhealth/calib/predictionset/base_conformal/__init__.py b/pyhealth/calib/predictionset/base_conformal/__init__.py index 3451f9062..54a47ea55 100644 --- a/pyhealth/calib/predictionset/base_conformal/__init__.py +++ b/pyhealth/calib/predictionset/base_conformal/__init__.py @@ -109,9 +109,17 @@ class BaseConformal(SetPredictor): alpha: Target miscoverage rate(s). Can be: - float: marginal coverage P(Y not in C(X)) <= alpha - array: class-conditional P(Y not in C(X) | Y=k) <= alpha[k] - score_type: Type of conformity score to use. Options: - - "aps": Adaptive Prediction Sets (default, uses probability scores) - - "threshold": Simple threshold on probabilities + score_type: Type of conformity score to use. Currently only one score + is implemented: + - "threshold" (default): NC score = 1 - p(true class), the score + from Sadinle, Lei, and Wasserman (2019) ("LABEL"). + - "aps": accepted as a backward-compatible alias for + "threshold". Despite the name, this does **not** implement + Adaptive Prediction Sets (Romano, Sesia, and Candes 2020) -- + that method uses a different score (cumulative sorted class + probabilities) which is not implemented here. If you need + genuine APS, do not rely on this option; it is kept only so + existing calls with ``score_type="aps"`` keep working. debug: Whether to use debug mode (processes fewer samples) Examples: @@ -158,7 +166,7 @@ def __init__( self, model: BaseModel, alpha: Union[float, np.ndarray], - score_type: str = "aps", + score_type: str = "threshold", debug: bool = False, **kwargs, ) -> None: @@ -201,8 +209,7 @@ def _compute_nc_scores( Non-conformity scores of shape (N,) — higher means less conforming. """ N = len(y_true) - if self.score_type == "aps" or self.score_type == "threshold": - # NC score = 1 - p(true class); higher = less conforming + if self.score_type == "threshold" or self.score_type == "aps": scores = 1.0 - y_prob[np.arange(N), y_true] else: raise ValueError(f"Unknown score_type: {self.score_type}") diff --git a/pyhealth/calib/predictionset/covariate/covariate_label.py b/pyhealth/calib/predictionset/covariate/covariate_label.py index f90430be8..e0d66ee90 100644 --- a/pyhealth/calib/predictionset/covariate/covariate_label.py +++ b/pyhealth/calib/predictionset/covariate/covariate_label.py @@ -170,25 +170,49 @@ def _compute_likelihood_ratio( def _query_weighted_quantile( - scores: np.ndarray, alpha: float, weights: np.ndarray + scores: np.ndarray, + alpha: float, + weights: np.ndarray, + test_weight: float = 0.0, ) -> float: - """Compute weighted quantile of scores. + """Compute the weighted conformal quantile of scores. + + Implements the finite-sample correction for weighted conformal prediction + under covariate shift (Tibshirani et al. 2019). Args: - scores: Array of conformity scores - alpha: Quantile level (between 0 and 1) - weights: Weights for each score + scores: Array of conformity scores (higher = more conforming). + alpha: Quantile level (between 0 and 1). + weights: Un-normalized weights (likelihood ratios) for each score. + test_weight: Un-normalized weight representing the test point. + Reserves ``test_weight / (sum(weights) + test_weight)`` of + probability mass at conformity ``-inf``. Default 0.0 recovers the + old, uncorrected behavior (kept for backward compatibility with + direct callers of this helper). Returns: - The weighted alpha-quantile of scores + The weighted alpha-quantile of scores. Returns ``-inf`` if the + reserved test-point mass alone already meets or exceeds ``alpha``, + since there isn't enough calibration mass to justify a stricter, + finite threshold without risking under-coverage. """ - # Sort scores and corresponding weights sorted_indices = np.argsort(scores) sorted_scores = scores[sorted_indices] sorted_weights = weights[sorted_indices] - # Compute cumulative weights - cum_weights = np.cumsum(sorted_weights) / np.sum(sorted_weights) + total_weight = np.sum(sorted_weights) + test_weight + if total_weight <= 0: + return -np.inf + + p_test = test_weight / total_weight + if p_test >= alpha: + # Not enough calibration mass to reach the target coverage without + # dipping into the mass reserved for the test point itself: fall + # back to the maximally permissive (safe) threshold. + return -np.inf + + # Compute cumulative weights over the reserved-mass-inclusive total. + cum_weights = np.cumsum(sorted_weights) / total_weight # Find the index where cumulative weight exceeds alpha idx = np.searchsorted(cum_weights, alpha, side="left") @@ -197,7 +221,7 @@ def _query_weighted_quantile( if idx >= len(sorted_scores): idx = len(sorted_scores) - 1 - return sorted_scores[idx] + return float(sorted_scores[idx]) class CovariateLabel(SetPredictor): @@ -458,8 +482,7 @@ def calibrate( self.kde_test, self.kde_cal, X ) - # Normalize weights - weights = likelihood_ratios / np.sum(likelihood_ratios) + # Keep weights un-normalized here self._sum_cal_weights = np.sum(likelihood_ratios) # Extract conformity scores (probabilities of true class) @@ -467,8 +490,10 @@ def calibrate( # Compute weighted quantile thresholds if isinstance(self.alpha, float): - # Marginal coverage: single threshold - t = _query_weighted_quantile(conformity_scores, self.alpha, weights) + test_weight = float(np.mean(likelihood_ratios)) + t = _query_weighted_quantile( + conformity_scores, self.alpha, likelihood_ratios, test_weight + ) else: # Class-conditional coverage: one threshold per class t = [] @@ -476,11 +501,10 @@ def calibrate( mask = y_true == k if np.sum(mask) > 0: class_scores = conformity_scores[mask] - class_weights = weights[mask] - # Renormalize class weights - class_weights = class_weights / np.sum(class_weights) + class_weights = likelihood_ratios[mask] + class_test_weight = float(np.mean(class_weights)) t_k = _query_weighted_quantile( - class_scores, self.alpha[k], class_weights + class_scores, self.alpha[k], class_weights, class_test_weight ) else: # If no calibration examples, use -inf (include all) diff --git a/tests/core/test_covariate_label.py b/tests/core/test_covariate_label.py index b5aa0aaf6..14c38cd46 100644 --- a/tests/core/test_covariate_label.py +++ b/tests/core/test_covariate_label.py @@ -317,12 +317,59 @@ def test_weighted_quantile_function(self): weights = np.array([0.1, 0.2, 0.3, 0.2, 0.2]) alpha = 0.5 + # Default test_weight=0.0 preserves the old (uncorrected) behavior + # for any direct caller that doesn't opt into the finite-sample + # correction. quantile = _query_weighted_quantile(scores, alpha, weights) self.assertIsInstance(quantile, (float, np.floating)) self.assertGreaterEqual(quantile, scores.min()) self.assertLessEqual(quantile, scores.max()) + def test_weighted_quantile_reserves_test_point_mass(self): + """The finite-sample correction should recover the standard (N+1) + reserved-mass fraction in the no-shift limit (uniform weights, + test_weight = mean of calibration weights).""" + from pyhealth.calib.predictionset.covariate.covariate_label import ( + _query_weighted_quantile, + ) + + N = 6 + weights = np.ones(N) + test_weight = float(np.mean(weights)) + p_test = test_weight / (np.sum(weights) + test_weight) + + self.assertAlmostEqual(p_test, 1.0 / (N + 1), places=10) + + def test_weighted_quantile_small_calibration_set_is_conservative(self): + """With very few calibration examples relative to the requested + alpha, the corrected quantile should fall back to -inf (maximally + permissive / safe) rather than returning an overconfident finite + threshold, since there isn't enough calibration mass to support the + target coverage without dipping into the reserved test-point mass.""" + from pyhealth.calib.predictionset.covariate.covariate_label import ( + _query_weighted_quantile, + ) + + scores = np.array([0.4, 0.6]) # N=2 + weights = np.array([1.1, 1.1]) + test_weight = float(np.mean(weights)) + + # 1/(N+1) = 1/3 ~= 0.333 > alpha=0.3, so there isn't enough + # calibration mass to safely support this target. + result = _query_weighted_quantile(scores, 0.3, weights, test_weight) + self.assertEqual(result, -np.inf) + + # A larger, adequately-sized calibration set at the same alpha + # should NOT hit the same fallback. + scores_large = np.linspace(0.0, 1.0, 50) + weights_large = np.ones(50) + test_weight_large = float(np.mean(weights_large)) + result_large = _query_weighted_quantile( + scores_large, 0.3, weights_large, test_weight_large + ) + self.assertTrue(np.isfinite(result_large)) + def test_likelihood_ratio_function(self): """Test the likelihood ratio computation.""" from pyhealth.calib.predictionset.covariate.covariate_label import ( From ef489180ddd5fa40c10f7bff530aac9fa46e9cc6 Mon Sep 17 00:00:00 2001 From: Arjun Chatterjee Date: Wed, 22 Jul 2026 14:25:25 -0700 Subject: [PATCH 19/61] Add missing citations in PyHealth code (#1181) * Add missing citations for TFMTokenizer, EHRMambaCEHR, CEHR embeddings, comprehensiveness/sufficiency metrics, MLE, and NNAAR * to pass the pr checks --- .../metrics/pyhealth.metrics.generative.rst | 3 ++- .../conformal_eeg/test_tfm_tuev_inference.py | 2 ++ pyhealth/metrics/generative/privacy.py | 6 +++++ pyhealth/metrics/generative/utility.py | 7 +++++ .../interpretability/comprehensiveness.py | 5 ++++ .../metrics/interpretability/sufficiency.py | 5 ++++ pyhealth/models/cehr_embeddings.py | 27 ++++++++++++++++++- pyhealth/models/ehrmamba_cehr.py | 25 +++++++++++++++++ pyhealth/models/tfm_tokenizer.py | 10 +++++-- 9 files changed, 86 insertions(+), 4 deletions(-) diff --git a/docs/api/metrics/pyhealth.metrics.generative.rst b/docs/api/metrics/pyhealth.metrics.generative.rst index 85e448a52..2116903d2 100644 --- a/docs/api/metrics/pyhealth.metrics.generative.rst +++ b/docs/api/metrics/pyhealth.metrics.generative.rst @@ -2,7 +2,8 @@ pyhealth.metrics.generative =================================== Evaluation metrics for synthetic (generative) EHR data, covering privacy, -utility, and statistical fidelity. +utility, and statistical fidelity. See each function's docstring for the +paper it implements. .. currentmodule:: pyhealth.metrics.generative diff --git a/examples/conformal_eeg/test_tfm_tuev_inference.py b/examples/conformal_eeg/test_tfm_tuev_inference.py index fede48d4d..d25eff7ac 100644 --- a/examples/conformal_eeg/test_tfm_tuev_inference.py +++ b/examples/conformal_eeg/test_tfm_tuev_inference.py @@ -5,6 +5,8 @@ present in this repo. No training — pure inference to verify weights and normalization are correct. +Model: TFMTokenizer, see pyhealth.models.tfm_tokenizer for the paper citation. + Usage: python examples/conformal_eeg/test_tfm_tuev_inference.py python examples/conformal_eeg/test_tfm_tuev_inference.py --gpu_id 1 diff --git a/pyhealth/metrics/generative/privacy.py b/pyhealth/metrics/generative/privacy.py index 70edc4957..9f9696caa 100644 --- a/pyhealth/metrics/generative/privacy.py +++ b/pyhealth/metrics/generative/privacy.py @@ -55,6 +55,12 @@ def calc_nnaar( ) -> Dict[str, Tuple[float, float]]: """Computes the Nearest Neighbor Adversarial Accuracy Risk (NNAAR). + Paper: + Yale, Andrew, Saloni Dash, Ritik Dutta, Isabelle Guyon, Adrien Pavao, + and Kristin P. Bennett. "Generation and Evaluation of Privacy + Preserving Synthetic Health Data." Neurocomputing 416 (2020): + 244-255. https://doi.org/10.1016/j.neucom.2019.12.136 + NNAAR measures whether the synthetic data sits closer to the real training data than to held-out test data, which would indicate memorization:: diff --git a/pyhealth/metrics/generative/utility.py b/pyhealth/metrics/generative/utility.py index d36ffcacd..3d10778eb 100644 --- a/pyhealth/metrics/generative/utility.py +++ b/pyhealth/metrics/generative/utility.py @@ -53,6 +53,13 @@ def compute_mle( (Train-Synthetic-Test-Real, TSTR). Both are evaluated on the same real test set. Synthetic accuracy/F1 close to real accuracy/F1 indicates high utility. + Paper: + Esteban, Cristobal, Stephanie L. Hyland, and Gunnar Ratsch. + "Real-valued (Medical) Time Series Generation with Recurrent + Conditional GANs." arXiv:1706.02633 (2017). Introduces and names + the "Train on Synthetic, Test on Real" (TSTR) evaluation protocol + this function implements. + Note: The current implementation hard-codes the downstream task to next-visit prediction (built via diff --git a/pyhealth/metrics/interpretability/comprehensiveness.py b/pyhealth/metrics/interpretability/comprehensiveness.py index 755daf47d..5374c622b 100644 --- a/pyhealth/metrics/interpretability/comprehensiveness.py +++ b/pyhealth/metrics/interpretability/comprehensiveness.py @@ -18,6 +18,11 @@ class ComprehensivenessMetric(RemovalBasedMetric): are REMOVED (ablated). Higher scores indicate more faithful interpretations. + Paper: + DeYoung, Jay, Sarthak Jain, Nazneen Fatema Rajani, Eric Lehman, + Caiming Xiong, Richard Socher, and Byron C. Wallace. + "ERASER: A Benchmark to Evaluate Rationalized NLP Models." ACL 2020. + The metric is computed as: COMP = (1/|B|) × Σ[p_c(x)(x) - p_c(x)(x \\ x:q%)] q∈B diff --git a/pyhealth/metrics/interpretability/sufficiency.py b/pyhealth/metrics/interpretability/sufficiency.py index c6c6a8fa5..cb6ab0efc 100644 --- a/pyhealth/metrics/interpretability/sufficiency.py +++ b/pyhealth/metrics/interpretability/sufficiency.py @@ -18,6 +18,11 @@ class SufficiencyMetric(RemovalBasedMetric): features are KEPT (all others removed). Lower scores indicate more faithful interpretations. + Paper: + DeYoung, Jay, Sarthak Jain, Nazneen Fatema Rajani, Eric Lehman, + Caiming Xiong, Richard Socher, and Byron C. Wallace. + "ERASER: A Benchmark to Evaluate Rationalized NLP Models." ACL 2020. + The metric is computed as: SUFF = (1/|B|) × Σ[p_c(x)(x) - p_c(x)(x:q%)] q∈B diff --git a/pyhealth/models/cehr_embeddings.py b/pyhealth/models/cehr_embeddings.py index 7974a699e..c8ed4e88f 100644 --- a/pyhealth/models/cehr_embeddings.py +++ b/pyhealth/models/cehr_embeddings.py @@ -48,7 +48,32 @@ def forward(self, visit_segments: torch.Tensor) -> torch.Tensor: class MambaEmbeddingsForCEHR(nn.Module): - """CEHR-style combined embeddings for Mamba (concept + type + time + age + visit).""" + """CEHR-style combined embeddings for Mamba (concept + type + time + age + visit). + + Paper: Same paper as :class:`~pyhealth.models.ehrmamba.EHRMamba` -- + EHRMAMBA: Towards Generalizable and Scalable Foundation Models for + Electronic Health Records (arxiv 2405.14567). This embedding scheme is + part of that paper's own Odyssey toolkit (see module header for the + code source). + + Examples: + >>> import torch + >>> from pyhealth.models.cehr_embeddings import MambaEmbeddingsForCEHR + >>> embeddings = MambaEmbeddingsForCEHR(vocab_size=100, hidden_size=32) + >>> batch_size, seq_len = 2, 5 + >>> input_ids = torch.randint(0, 100, (batch_size, seq_len)) + >>> token_type_ids = torch.zeros(batch_size, seq_len, dtype=torch.long) + >>> time_stamps = torch.zeros(batch_size, seq_len) + >>> ages = torch.zeros(batch_size, seq_len) + >>> visit_orders = torch.zeros(batch_size, seq_len, dtype=torch.long) + >>> visit_segments = torch.zeros(batch_size, seq_len, dtype=torch.long) + >>> out = embeddings( + ... input_ids, token_type_ids, time_stamps, ages, + ... visit_orders, visit_segments, + ... ) + >>> out.shape + torch.Size([2, 5, 32]) + """ def __init__( self, diff --git a/pyhealth/models/ehrmamba_cehr.py b/pyhealth/models/ehrmamba_cehr.py index cd555629c..711b032c6 100644 --- a/pyhealth/models/ehrmamba_cehr.py +++ b/pyhealth/models/ehrmamba_cehr.py @@ -18,6 +18,13 @@ class EHRMambaCEHR(BaseModel): """Mamba backbone over CEHR embeddings (FHIR / MPF pipeline). + Paper: Same paper as :class:`~pyhealth.models.ehrmamba.EHRMamba` -- + EHRMAMBA: Towards Generalizable and Scalable Foundation Models for + Electronic Health Records (arxiv 2405.14567). This class combines that + paper's Mamba backbone (:class:`~pyhealth.models.ehrmamba.MambaBlock`) + with CEHR-style embeddings (see + :class:`~pyhealth.models.cehr_embeddings.MambaEmbeddingsForCEHR`). + Args: dataset: Fitted :class:`~pyhealth.datasets.SampleDataset` with MPF task schema. vocab_size: Concept embedding vocabulary size (typically ``task.vocab.vocab_size``). @@ -27,6 +34,24 @@ class EHRMambaCEHR(BaseModel): state_size: SSM state size per channel. conv_kernel: Causal conv kernel in each block. dropout: Dropout before classifier. + + Examples: + >>> from pyhealth.datasets import MIMIC4FHIR, split_by_patient + >>> from pyhealth.tasks.mpf_clinical_prediction import ( + ... MPFClinicalPredictionTask, + ... ) + >>> from pyhealth.models import EHRMambaCEHR + >>> dataset = MIMIC4FHIR(root="/path/to/mimic-iv-fhir-demo") + >>> sample_dataset = dataset.set_task(MPFClinicalPredictionTask()) + >>> train_ds, val_ds, test_ds = split_by_patient( + ... sample_dataset, [0.7, 0.1, 0.2] + ... ) + >>> vocab_size = ( + ... sample_dataset.input_processors["concept_ids"].vocab.vocab_size + ... ) + >>> model = EHRMambaCEHR( + ... dataset=sample_dataset, vocab_size=vocab_size, embedding_dim=32 + ... ) """ def __init__( diff --git a/pyhealth/models/tfm_tokenizer.py b/pyhealth/models/tfm_tokenizer.py index 38bd6de04..04f83cc57 100644 --- a/pyhealth/models/tfm_tokenizer.py +++ b/pyhealth/models/tfm_tokenizer.py @@ -714,10 +714,16 @@ def load_embedding_weights(source_model, target_model): class TFMTokenizer(BaseModel): """TFM-Tokenizer model. - + This model uses VQ-VAE with transformers to tokenize EEG signals. It can extract discrete tokens and continuous embeddings for downstream tasks. - + + Paper: + Pradeepkumar, Jathurshan, Xihao Piao, Zheng Chen, and Jimeng Sun. + "Tokenizing Single-Channel EEG with Time-Frequency Motif Learning." + ICLR 2026. https://arxiv.org/abs/2502.16060 + Code: https://github.com/Jathurshan0330/TFM-Tokenizer + The model expects two inputs: - STFT spectrogram: shape (batch, n_freq, n_time) - Raw temporal signal: shape (batch, n_samples) From 841be53a4d872dba190d0950fbd8cc986fc9598e Mon Sep 17 00:00:00 2001 From: "Axel.Cffrd.Dnty" <150222552+AxelNoun@users.noreply.github.com> Date: Sun, 9 Aug 2026 05:14:21 +0200 Subject: [PATCH 20/61] Add MEDS dataset support (MEDSDataset + typed Parquet scan path) (#1179) * feat(datasets): add Parquet scan path to BaseDataset Route .parquet/.pq files, globs, and directories through a typed _scan_parquet scanner; keep CSV/TSV(.gz) on the existing path. Add a datetime fast-path in load_table that skips the string round-trip and casts to datetime64[ms], preserving NaT for static events. * feat(datasets): add MEDSDataset for the Medical Event Data Standard Declarative YAML wrapper over the shared Parquet scan path, with split_source subset selection (metadata or directory layout), distinct processing caches per subset, and a construction-time Parquet footer schema guard that rejects missing, non-timestamp, or timezone-aware time columns. * test(datasets): add MEDS synthetic and demo smoke tests Deterministic sharded Parquet fixtures cover nested splits, subset filtering, cache isolation, set_task smoke, and construction-time schema-guard TypeErrors. Demo smoke stays skip-gated behind MEDS_DEMO_ROOT / test-resources/meds_demo (gitignored). * docs(examples): add MEDS example and API docs Document MEDSDataset in the API reference and add an end-to-end examples/meds_demo.py against the public PhysioNet MIMIC-IV MEDS demo. * feat(tasks): add InHospitalMortalityMEDS MEDS-native in-hospital mortality task: one sample per completed stay, reconstructed by joining HOSPITAL_ADMISSION/HOSPITAL_DISCHARGE events on hadm_id. Half-open [admit, prediction_time) observation window (full_stay default; first_hours early-warning variant), label from the HOSPITAL_DISCHARGE//DIED discharge code. Discharge and MEDS_DEATH events are excluded from features to prevent label leakage. hadm_id is dataset-specific (not part of the core MEDS schema), so it is exposed via a bundled configs/meds_with_hadm.yaml rather than the default config. Verified on the public MIMIC-IV demo in MEDS: 12 positive / 238 stays (rate 0.0504); set_task sample count 238. - pyhealth/tasks/in_hospital_mortality_meds.py (+ __init__ export) - pyhealth/datasets/configs/meds_with_hadm.yaml - tests/core/test_in_hospital_mortality_meds.py - examples/verify_meds_mortality.py - docs/api/tasks/pyhealth.tasks.InHospitalMortalityMEDS.rst (+ tasks.rst toctree) Co-authored-by: Cursor * fix(tasks): re-export InHospitalMortalityMEDS with explicit alias Satisfies ruff F401 on the newly added __init__ line under tools/check_pr_rules scoped lint (same pattern as eegbci). Co-authored-by: Cursor * docs: link MEDS schema docs for subject_splits mapping Co-authored-by: Cursor * refactor: rename _reconstruct_stays to _group_stays and clarify summarize docstring Co-authored-by: Cursor * docs: add end-to-end RNN training to MEDS demo Co-authored-by: Cursor * docs: link subject_splits in MEDSDataset API page; qualify demo metrics output Co-authored-by: Cursor * refactor: drop summarize helper from InHospitalMortalityMEDS Co-authored-by: Cursor * style: modernize typing annotations, drop unused noqa (UP006/UP035/UP045/RUF100) Co-authored-by: Cursor --------- Co-authored-by: Cursor --- .gitignore | 16 +- docs/api/datasets.rst | 1 + .../pyhealth.datasets.MEDSDataset.rst | 9 + docs/api/tasks.rst | 1 + ...pyhealth.tasks.InHospitalMortalityMEDS.rst | 7 + examples/meds_demo.py | 138 ++++++++ examples/verify_meds_mortality.py | 62 ++++ pyhealth/datasets/__init__.py | 1 + pyhealth/datasets/base_dataset.py | 99 +++++- pyhealth/datasets/configs/meds.yaml | 34 ++ pyhealth/datasets/configs/meds_with_hadm.yaml | 22 ++ pyhealth/datasets/meds.py | 305 +++++++++++++++++ pyhealth/tasks/__init__.py | 3 + pyhealth/tasks/in_hospital_mortality_meds.py | 252 ++++++++++++++ tests/core/test_in_hospital_mortality_meds.py | 278 ++++++++++++++++ tests/core/test_meds.py | 309 ++++++++++++++++++ 16 files changed, 1526 insertions(+), 11 deletions(-) create mode 100644 docs/api/datasets/pyhealth.datasets.MEDSDataset.rst create mode 100644 docs/api/tasks/pyhealth.tasks.InHospitalMortalityMEDS.rst create mode 100644 examples/meds_demo.py create mode 100644 examples/verify_meds_mortality.py create mode 100644 pyhealth/datasets/configs/meds.yaml create mode 100644 pyhealth/datasets/configs/meds_with_hadm.yaml create mode 100644 pyhealth/datasets/meds.py create mode 100644 pyhealth/tasks/in_hospital_mortality_meds.py create mode 100644 tests/core/test_in_hospital_mortality_meds.py create mode 100644 tests/core/test_meds.py diff --git a/.gitignore b/.gitignore index 086c8da3f..f0c1ff431 100644 --- a/.gitignore +++ b/.gitignore @@ -140,4 +140,18 @@ data/physionet.org/ .codex # Model weight files (large binaries, distributed separately) -weightfiles/ \ No newline at end of file +weightfiles/ + +# Local / personal (never commit) +CLAUDE.local.md +.claude/settings.local.json + +# Local test fixtures (download separately; not part of the repo) +test-resources/core/chestxray14/ +test-resources/meds_demo/ + +# Local Python environments (not .venv — that pattern is already ignored) +.venv312/ + +# Tool caches +.ruff_cache/ \ No newline at end of file diff --git a/docs/api/datasets.rst b/docs/api/datasets.rst index 592aed487..c9a88b7ff 100644 --- a/docs/api/datasets.rst +++ b/docs/api/datasets.rst @@ -225,6 +225,7 @@ Available Datasets datasets/pyhealth.datasets.MIMIC3Dataset datasets/pyhealth.datasets.MIMIC4Dataset datasets/pyhealth.datasets.FHIRDataset + datasets/pyhealth.datasets.MEDSDataset datasets/pyhealth.datasets.MIMIC4FHIR datasets/pyhealth.datasets.MedicalTranscriptionsDataset datasets/pyhealth.datasets.CardiologyDataset diff --git a/docs/api/datasets/pyhealth.datasets.MEDSDataset.rst b/docs/api/datasets/pyhealth.datasets.MEDSDataset.rst new file mode 100644 index 000000000..92bc93e0d --- /dev/null +++ b/docs/api/datasets/pyhealth.datasets.MEDSDataset.rst @@ -0,0 +1,9 @@ +pyhealth.datasets.MEDSDataset +=================================== + +Dataset class for data in the `Medical Event Data Standard (MEDS) `_, a minimal event-based schema for machine learning over EHR data (MEDS Working Group / Arnrich et al., ICLR 2024 Workshop on Learning from Time Series For Health; `openreview:IsHy2ebjIG `_). Sharded Parquet event files are read with their native types, and standard MEDS splits (train / tuning / held_out) can be selected directly via the ``subset`` argument. The canonical subject-to-split mapping is defined in ``metadata/subject_splits.parquet``; see the `MEDS schema documentation `_. + +.. autoclass:: pyhealth.datasets.MEDSDataset + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/tasks.rst b/docs/api/tasks.rst index c7910e626..bdaa9599a 100644 --- a/docs/api/tasks.rst +++ b/docs/api/tasks.rst @@ -207,6 +207,7 @@ Available Tasks Base Task In-Hospital Mortality (MIMIC-IV) + In-Hospital Mortality (MEDS) MIMIC-III ICD-9 Coding Cardiology Detection COVID-19 CXR Classification diff --git a/docs/api/tasks/pyhealth.tasks.InHospitalMortalityMEDS.rst b/docs/api/tasks/pyhealth.tasks.InHospitalMortalityMEDS.rst new file mode 100644 index 000000000..9ca8c55d2 --- /dev/null +++ b/docs/api/tasks/pyhealth.tasks.InHospitalMortalityMEDS.rst @@ -0,0 +1,7 @@ +pyhealth.tasks.InHospitalMortalityMEDS +====================================== + +.. autoclass:: pyhealth.tasks.in_hospital_mortality_meds.InHospitalMortalityMEDS + :members: + :undoc-members: + :show-inheritance: diff --git a/examples/meds_demo.py b/examples/meds_demo.py new file mode 100644 index 000000000..faebaf78d --- /dev/null +++ b/examples/meds_demo.py @@ -0,0 +1,138 @@ +"""End-to-end example: loading a MEDS dataset with PyHealth. + +This example uses the public *MIMIC-IV demo data in the Medical Event Data +Standard (MEDS)* (PhysioNet, v0.0.1, ODbL v1.0, ~100 subjects): +https://doi.org/10.13026/t2y8-ea41 + +Download it once (open access, ~a few MB): + + wget -r -N -c -np https://physionet.org/files/mimic-iv-demo-meds/0.0.1/ + +Then run: + + python examples/meds_demo.py \\ + --root physionet.org/files/mimic-iv-demo-meds/0.0.1 + +Any dataset following the MEDS layout (``data/**.parquet`` + +``metadata/subject_splits.parquet``) works the same way. See the MEDS +specification: https://github.com/Medical-Event-Data-Standard/meds +""" + +import argparse +import tempfile +from pathlib import Path +from typing import Any, List + +import polars as pl +import pyhealth.datasets.configs as meds_configs + +from pyhealth.datasets import MEDSDataset, get_dataloader, split_by_patient +from pyhealth.models import RNN +from pyhealth.tasks import InHospitalMortalityMEDS +from pyhealth.trainer import Trainer + +# Full-stay MEDS code sequences are often thousands of events long. Keeping +# every code makes a vanilla RNN prohibitively slow on CPU, so the training +# block below keeps only the *most recent* codes per stay. This is a demo +# ergonomics choice, not a benchmark configuration: production runs should +# use the unmodified task (``InHospitalMortalityMEDS()``) and an appropriate +# model/window for the sequence lengths involved. +_DEMO_MAX_SEQ_LEN = 256 + + +class _DemoMortalityTask(InHospitalMortalityMEDS): + """Demo-only wrapper that tail-truncates ``codes`` before ``set_task``.""" + + def __init__(self, max_seq_len: int = _DEMO_MAX_SEQ_LEN, **kwargs) -> None: + super().__init__(**kwargs) + self.max_seq_len = max_seq_len + + def __call__(self, patient: Any) -> List[dict]: + samples = super().__call__(patient) + for sample in samples: + codes = sample["codes"] + if len(codes) > self.max_seq_len: + sample["codes"] = codes[-self.max_seq_len :] + return samples + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--root", + required=True, + help="Root of the MEDS dataset (directory containing data/ and metadata/)", + ) + parser.add_argument( + "--subset", + default="train", + help="Split to load as a subset (default: train)", + ) + args = parser.parse_args() + + # 1) Load the full dataset: every Parquet shard under data/ is read, + # including nested split directories (data//.parquet). + dataset = MEDSDataset(root=args.root) + dataset.stats() + + # 2) Peek at the canonical event frame (typed straight from Parquet: + # string patient ids, datetime64[ms] timestamps, float values). + events = dataset.global_event_df + print(events.head(5).collect()) + + # 3) Load a split-restricted subset. Subjects are selected through the + # metadata/subject_splits.parquet assignment; each subset uses its + # own processing cache. + subset = MEDSDataset(root=args.root, subset=args.subset) + n_subset = len(subset.unique_patient_ids) + n_total = len(dataset.unique_patient_ids) + print(f"Subjects in subset '{args.subset}': {n_subset} / {n_total}") + + # 4) Static (null-time) MEDS events, e.g. demographics, are preserved. + n_static = ( + events.filter(pl.col("timestamp").is_null()).select(pl.len()).collect().item() + ) + print(f"Static (null-time) events: {n_static}") + + # 5) In-hospital mortality task + minimal RNN training loop (1 epoch). + # Sequences are tail-truncated via _DemoMortalityTask (see module note). + cfg = Path(meds_configs.__file__).parent / "meds_with_hadm.yaml" + task_cache = tempfile.mkdtemp(prefix="meds_demo_task_") + cohort = MEDSDataset( + root=args.root, + config_path=str(cfg), + subset=args.subset, + cache_dir=task_cache, + ) + samples = cohort.set_task(_DemoMortalityTask()) + print( + f"Mortality task samples ({args.subset}, codes tail-truncated to " + f"{_DEMO_MAX_SEQ_LEN}): {len(samples)}" + ) + + train_dataset, val_dataset, test_dataset = split_by_patient( + samples, [0.8, 0.1, 0.1] + ) + train_dataloader = get_dataloader(train_dataset, batch_size=32, shuffle=True) + val_dataloader = get_dataloader(val_dataset, batch_size=32, shuffle=False) + test_dataloader = get_dataloader(test_dataset, batch_size=32, shuffle=False) + + model = RNN(dataset=samples, embedding_dim=64, hidden_dim=64) + trainer = Trainer(model=model) + trainer.train( + train_dataloader=train_dataloader, + val_dataloader=val_dataloader, + epochs=1, + monitor="roc_auc", + ) + metrics = trainer.evaluate(test_dataloader) + print( + "Test metrics (smoke test only — 1 epoch, tail-truncated sequences; " + "not interpretable as model quality):" + ) + print(metrics) + + +if __name__ == "__main__": + # BaseDataset spawns Dask worker processes; keep the main-module guard. + main() diff --git a/examples/verify_meds_mortality.py b/examples/verify_meds_mortality.py new file mode 100644 index 000000000..8c199db84 --- /dev/null +++ b/examples/verify_meds_mortality.py @@ -0,0 +1,62 @@ +"""Standalone verification of the MEDS in-hospital mortality cohort. + +Applies ``InHospitalMortalityMEDS`` via ``set_task`` on a MEDS dataset and +prints basic cohort counts (expected ~12/238 positives on the public +MIMIC-IV demo). Prefer this path over a per-patient ``task(get_patient)`` +loop: ``set_task`` uses the library's lazy loading and parallel processing. + +Usage: + # Download the public demo once (open access, ODbL v1.0): + # wget -r -N -c -np https://physionet.org/files/mimic-iv-demo-meds/0.0.1/ + python examples/verify_meds_mortality.py \\ + --root physionet.org/files/mimic-iv-demo-meds/0.0.1 + +The task needs hadm_id; this script uses the bundled +``configs/meds_with_hadm.yaml`` automatically. +""" + +import argparse +from pathlib import Path + +import pyhealth.datasets.configs as meds_configs +from pyhealth.datasets import MEDSDataset +from pyhealth.tasks import InHospitalMortalityMEDS + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--root", + required=True, + help="Root of the MEDS dataset (contains data/ and metadata/).", + ) + parser.add_argument( + "--observation-window", + default="full_stay", + choices=["full_stay", "first_hours"], + ) + parser.add_argument("--window-hours", type=float, default=48.0) + args = parser.parse_args() + + cfg = Path(meds_configs.__file__).parent / "meds_with_hadm.yaml" + dataset = MEDSDataset(root=args.root, config_path=str(cfg)) + task = InHospitalMortalityMEDS( + observation_window=args.observation_window, + window_hours=args.window_hours, + ) + + samples = dataset.set_task(task) + n = len(samples) + n_positive = sum(int(samples[i]["mortality"]) for i in range(n)) + n_patients = len({samples[i]["patient_id"] for i in range(n)}) + + print(f"root : {args.root}") + print(f"observation_window : {args.observation_window}") + print(f"n_samples (stays) : {n}") + print(f"n_patients : {n_patients}") + print(f"n_positive (died) : {n_positive}") + print(f"positive_rate : {((n_positive / n) if n else 0.0):.4f}") + + +if __name__ == "__main__": + main() diff --git a/pyhealth/datasets/__init__.py b/pyhealth/datasets/__init__.py index 99d90aa62..57a9956c2 100644 --- a/pyhealth/datasets/__init__.py +++ b/pyhealth/datasets/__init__.py @@ -57,6 +57,7 @@ def __init__(self, *args, **kwargs): from .eicu import eICUDataset from .isruc import ISRUCDataset from .medical_transcriptions import MedicalTranscriptionsDataset +from .meds import MEDSDataset as MEDSDataset from .mimic3 import MIMIC3Dataset from .mimic4 import MIMIC4CXRDataset, MIMIC4Dataset, MIMIC4EHRDataset, MIMIC4NoteDataset from .fhir import FHIRDataset, MIMIC4FHIR diff --git a/pyhealth/datasets/base_dataset.py b/pyhealth/datasets/base_dataset.py index 0e4280aab..3d449d579 100644 --- a/pyhealth/datasets/base_dataset.py +++ b/pyhealth/datasets/base_dataset.py @@ -315,6 +315,15 @@ class BaseDataset(ABC): config (dict): Configuration loaded from a YAML file. global_event_df (pl.LazyFrame): The global event data frame. dev (bool): Whether to enable dev mode (limit to 1000 patients). + + Examples: + >>> from pyhealth.datasets import BaseDataset + >>> dataset = BaseDataset( + ... root="/path/to/source", + ... tables=["patients", "diagnoses"], + ... config_path="/path/to/config.yaml", + ... ) + >>> dataset.stats() """ def __init__( @@ -425,6 +434,68 @@ def clean_tmpdir(self) -> None: if tmp_dir.exists(): shutil.rmtree(tmp_dir) + def _scan_table(self, source_path: str) -> dd.DataFrame: + """Routes a table source to the appropriate scanner based on its format. + + Parquet sources (``.parquet``/``.pq`` files, glob patterns targeting + such files, or directories of Parquet shards) are handled by + :meth:`_scan_parquet`. Any other source falls back to the existing + CSV/TSV(.gz) scanner, preserving prior behavior for all datasets. + + Args: + source_path (str): Path to the table source. + + Returns: + dd.DataFrame: The Dask DataFrame for the table source. + """ + stripped = source_path.rstrip("/") + if stripped.endswith((".parquet", ".pq")) or ( + not is_url(source_path) and Path(source_path).is_dir() + ): + return self._scan_parquet(source_path) + return self._scan_csv_tsv_gz(source_path) + + def _scan_parquet(self, source_path: str) -> dd.DataFrame: + """Scans a Parquet source and returns a Dask DataFrame. + + The source may be a single ``.parquet``/``.pq`` file, a glob pattern, + or a directory that is scanned recursively — which supports sharded + datasets such as MEDS, laid out as ``data//.parquet``. + + Unlike :meth:`_scan_csv_tsv_gz`, no all-string schema coercion is + applied: Parquet files embed their schema, so source dtypes (native + timestamps, numeric columns, nullable strings) are preserved and + handled downstream by :meth:`load_table`. + + Args: + source_path (str): Path to a Parquet file, directory, or glob. + + Returns: + dd.DataFrame: The Dask DataFrame backed by the Parquet source. + + Raises: + FileNotFoundError: If the source path does not exist, or if a + directory source contains no Parquet files. + """ + path = Path(source_path) + is_glob = any(ch in source_path for ch in "*?[") + if not is_glob: + if not path.exists(): + raise FileNotFoundError( + f"Parquet source does not exist: {source_path}" + ) + if path.is_dir() and not any( + itertools.chain(path.rglob("*.parquet"), path.rglob("*.pq")) + ): + raise FileNotFoundError( + f"Directory contains no Parquet files: {source_path}" + ) + return dd.read_parquet( + source_path, + split_row_groups=True, # type: ignore + blocksize="64MB", + ) + def _scan_csv_tsv_gz(self, source_path: str) -> dd.DataFrame: """Scans a CSV/TSV file (possibly gzipped) and returns a Dask DataFrame. @@ -596,7 +667,8 @@ def load_table(self, table_name: str) -> dd.DataFrame: Raises: ValueError: If the table is not found in the config. - FileNotFoundError: If the CSV file for the table or join is not found. + FileNotFoundError: If the source file (CSV/TSV or Parquet) for the + table or join is not found. """ assert self.config is not None, "Config must be provided to load tables" @@ -608,7 +680,7 @@ def load_table(self, table_name: str) -> dd.DataFrame: csv_path = clean_path(csv_path) logger.info(f"Scanning table: {table_name} from {csv_path}") - df = self._scan_csv_tsv_gz(csv_path) + df = self._scan_table(csv_path) # Convert column names to lowercase before calling preprocess_func df = df.rename(columns=str.lower) @@ -627,7 +699,7 @@ def load_table(self, table_name: str) -> dd.DataFrame: other_csv_path = f"{self.root}/{join_cfg.file_path}" other_csv_path = clean_path(other_csv_path) logger.info(f"Joining with table: {other_csv_path}") - join_df = self._scan_csv_tsv_gz(other_csv_path) + join_df = self._scan_table(other_csv_path) join_df = join_df.rename(columns=str.lower) join_key = join_cfg.on columns = join_cfg.columns @@ -651,14 +723,21 @@ def load_table(self, table_name: str) -> dd.DataFrame: timestamp_series: dd.Series = functools.reduce( operator.add, (df[col].astype("string") for col in timestamp_col) ) + timestamp_series = dd.to_datetime( + timestamp_series, + format=timestamp_format, + errors="raise", + ) + elif pd.api.types.is_datetime64_any_dtype(df[timestamp_col].dtype): + # Typed sources (e.g. Parquet) already carry native timestamps: + # skip the string round-trip and only normalize the unit below. + timestamp_series: dd.Series = df[timestamp_col] else: - timestamp_series: dd.Series = df[timestamp_col].astype("string") - - timestamp_series: dd.Series = dd.to_datetime( - timestamp_series, - format=timestamp_format, - errors="raise", - ) + timestamp_series = dd.to_datetime( + df[timestamp_col].astype("string"), + format=timestamp_format, + errors="raise", + ) df: dd.DataFrame = df.assign( timestamp=timestamp_series.astype("datetime64[ms]") ) diff --git a/pyhealth/datasets/configs/meds.yaml b/pyhealth/datasets/configs/meds.yaml new file mode 100644 index 000000000..a251854ee --- /dev/null +++ b/pyhealth/datasets/configs/meds.yaml @@ -0,0 +1,34 @@ +# MEDS (Medical Event Data Standard) tables. +# +# MEDS is already flat (one row per measurement), so a single event table +# covers the data shards. The canonical subject-to-split mapping is exposed +# as a second, ordinary event table -- the same pattern as the `splits` +# table in ehrshot.yaml. TableConfig has no format field: Parquet reading is +# handled by BaseDataset._scan_table / _scan_parquet, so no config-schema +# change is needed for the 21 existing datasets. +# +# Paths match mimic-iv-demo-meds: data//*.parquet + +# metadata/subject_splits.parquet. +version: "1.0" +tables: + meds: + # Directory of shards; dd.read_parquet reads the nested tree whole. + file_path: "data" + patient_id: "subject_id" + # Native datetime64[us] in MEDS Parquet, narrowed to [ms] by the + # BaseDataset typed-timestamp fast-path. No timestamp_format: that + # would imply text parsing. MEDSDataset rejects non-timestamp / + # tz-aware columns at construction (footer schema guard). + timestamp: "time" + attributes: + - "code" + - "numeric_value" + + # Canonical split map as events (attribute `subject_splits/split`). + # Opt-in: tables=["meds", "subject_splits"]. + subject_splits: + file_path: "metadata/subject_splits.parquet" + patient_id: "subject_id" + timestamp: null + attributes: + - "split" diff --git a/pyhealth/datasets/configs/meds_with_hadm.yaml b/pyhealth/datasets/configs/meds_with_hadm.yaml new file mode 100644 index 000000000..508e901bc --- /dev/null +++ b/pyhealth/datasets/configs/meds_with_hadm.yaml @@ -0,0 +1,22 @@ +version: "1.0" +# Stay-aware MEDS config: identical to the default configs/meds.yaml but +# additionally exposes `hadm_id`, which stay-based tasks (e.g. +# InHospitalMortalityMEDS) need to reconstruct admissions/discharges. +# The default config keeps `attributes` minimal; opt into this one when a +# task consumes hadm_id. +tables: + meds: + file_path: "data" + patient_id: "subject_id" + timestamp: "time" + attributes: + - "code" + - "numeric_value" + - "hadm_id" + + subject_splits: + file_path: "metadata/subject_splits.parquet" + patient_id: "subject_id" + timestamp: null + attributes: + - "split" diff --git a/pyhealth/datasets/meds.py b/pyhealth/datasets/meds.py new file mode 100644 index 000000000..e6d5047ee --- /dev/null +++ b/pyhealth/datasets/meds.py @@ -0,0 +1,305 @@ +"""MEDS (Medical Event Data Standard) dataset for PyHealth. + +MEDS distributes event data as *typed*, sharded Parquet, already flattened to +one row per measurement -- ``(subject_id, time, code, numeric_value, ...)`` -- +plus a canonical subject-to-split mapping at +``metadata/subject_splits.parquet``. See the MEDS schema documentation for +the canonical subject-to-split mapping: +https://medical-event-data-standard.github.io/ +This maps almost one-to-one onto +PyHealth's canonical event schema +(``patient_id | event_type | timestamp |
/``). + +Parquet scanning and the typed-timestamp fast-path live in +:class:`BaseDataset`. ``MEDSDataset`` adds three MEDS-specific pieces: + +1. **Schema contract at construction.** :meth:`_validate_event_schema` reads + Parquet footers only and raises ``TypeError`` when the configured + timestamp column is missing, not a timestamp type, or timezone-aware + (MEDS reference ``DataSchema`` is ``timestamp[us]``, tz-naive). +2. **Split-aware loading.** ``subset=`` keeps only the patients of one + canonical split, via ``split_source`` (``"metadata"`` or ``"directory"``). +3. **Cache disambiguation.** Subset instances nest a dedicated cache + directory so different splits never share a processing cache. + +MEDS spec: https://github.com/Medical-Event-Data-Standard/meds +""" + +import logging +from pathlib import Path +from typing import Literal + +import dask.dataframe as dd +import pandas as pd +import pyarrow as pa +import pyarrow.dataset as pa_ds + +from .base_dataset import BaseDataset, clean_path + +logger = logging.getLogger(__name__) + +#: Canonical MEDS split names, in canonical order (MEDS spec). +MEDS_SPLITS: tuple[str, ...] = ("train", "tuning", "held_out") + +#: MEDS-normative locations, relative to the dataset root. +DATA_RELPATH = "data" +SUBJECT_SPLITS_RELPATH = "metadata/subject_splits.parquet" + +SplitSource = Literal["metadata", "directory"] + + +class MEDSDataset(BaseDataset): + """Dataset for MEDS (Medical Event Data Standard) sources. + + MEDS data is distributed as sharded, typed Parquet under per-split + directories (``data/train/*.parquet``, ``data/tuning/*.parquet``, + ``data/held_out/*.parquet``) plus a canonical subject-to-split map at + ``metadata/subject_splits.parquet``. See the MEDS schema documentation: + https://medical-event-data-standard.github.io/ + + ``time`` must be a timezone-naive timestamp (MEDS reference schema); + violations raise ``TypeError`` at construction. + + Split handling: + The canonical split is available two ways, both optional: + + * as **events**: load the ``subject_splits`` table + (``tables=["meds", "subject_splits"]``) and each subject carries one + ``subject_splits`` event with attribute ``subject_splits/split`` -- + the exact pattern of EHRShot's ``splits`` table, usable from + ``Task.pre_filter`` or per-patient logic; + * as a **loader filter**: ``subset="train"`` (or ``"tuning"`` / + ``"held_out"``) keeps only that split's patients in every loaded + table, via the same patient-``isin`` mechanic as dev mode. + + ``split_source`` controls where ``subset`` gets its patient list: + ``"metadata"`` (default) reads the canonical mapping file -- + authoritative per the MEDS spec and independent of directory layout; + ``"directory"`` derives it from which ``data//`` directory + subjects appear in -- useful when an export omits the metadata file. + The two sources *should* agree; whether PyHealth must verify that + equivalence is an open question for the upstream maintainer, so + this class does not silently pick one when they could diverge: it + uses exactly the source you asked for, and caches them separately. + + Note: + ``event_type`` is the table name (``"meds"``) for every row; the + clinically meaningful event kind lives in the ``meds/code`` + attribute. This mirrors EHRShot, whose single ``ehrshot`` table also + carries an event vocabulary in a ``code`` attribute. Whether upstream + prefers mapping MEDS ``code`` onto ``event_type`` instead is a design + question for the maintainer. + + Args: + root: Root directory of the MEDS dataset (the directory that + contains ``data/`` and ``metadata/``). + tables: Tables to load, as named in ``configs/meds.yaml``. Defaults + to ``["meds"]``; add ``"subject_splits"`` to expose the canonical + split as events. + subset: ``"train"``, ``"tuning"``, ``"held_out"``, or ``"all"`` + (default). Anything but ``"all"`` filters every loaded table to + that split's patients. + split_source: Where ``subset`` gets its patient list from; see + above. Ignored when ``subset="all"``. + dataset_name: Dataset name. Defaults to ``"meds"``. + config_path: Path to the YAML config. Defaults to + ``configs/meds.yaml``. + **kwargs: Forwarded to :class:`BaseDataset` (``cache_dir``, + ``num_workers``, ``dev``). Note dev mode's 1000-patient cap is + applied downstream of ``load_table`` (in + ``BaseDataset._event_transform``), so it composes with + ``subset`` with no extra handling here. + + Examples: + >>> from pyhealth.datasets import MEDSDataset + >>> dataset = MEDSDataset( + ... root="/path/to/mimic-iv-demo-meds/0.0.1", + ... ) # doctest: +SKIP + >>> dataset.stats() # doctest: +SKIP + >>> # Canonical training split only, split map exposed as events: + >>> train = MEDSDataset( + ... root="/path/to/mimic-iv-demo-meds/0.0.1", + ... tables=["meds", "subject_splits"], + ... subset="train", + ... ) # doctest: +SKIP + """ + + def __init__( + self, + root: str, + tables: list[str] | None = None, + subset: str = "all", + split_source: SplitSource = "metadata", + dataset_name: str | None = None, + config_path: str | None = None, + **kwargs, + ) -> None: + if subset not in (*MEDS_SPLITS, "all"): + raise ValueError( + f"subset must be one of {(*MEDS_SPLITS, 'all')}, got {subset!r}" + ) + if split_source not in ("metadata", "directory"): + raise ValueError( + f"split_source must be 'metadata' or 'directory', got {split_source!r}" + ) + + # Set before super().__init__: _init_cache_dir (called by the base + # constructor) reads them. + self.subset = subset + self.split_source = split_source + self._subset_patient_ids_cache: list[str] | None = None + + if config_path is None: + logger.info("No config path provided, using default MEDS config") + config_path = Path(__file__).parent / "configs" / "meds.yaml" + + if tables is None: + tables = ["meds"] + + super().__init__( + root=root, + tables=tables, + dataset_name=dataset_name or "meds", + config_path=config_path, + **kwargs, + ) + + # Fail fast on schema-contract violations (footer read only). + self._validate_event_schema() + + # ------------------------------------------------------------------ + # Cache keying + # ------------------------------------------------------------------ + + def _init_cache_dir(self, cache_dir) -> Path: + """Nest a subset-specific directory under the standard cache key. + + The base cache key hashes only ``{root, tables, dataset_name, dev}`` + (``BaseDataset._init_cache_dir``); ``subset`` changes the *content* + of the cached ``global_event_df`` because rows are filtered in + ``load_data``, so instances with different subsets (or different + split sources) must not share a cache. ``subset="all"`` (default) + keeps the exact upstream cache layout. + """ + base = super()._init_cache_dir(cache_dir) + if self.subset == "all": + return base + sub = base / f"subset-{self.split_source}-{self.subset}" + sub.mkdir(parents=True, exist_ok=True) + return sub + + # ------------------------------------------------------------------ + # Split handling (canonical split as events + optional subset filter) + # ------------------------------------------------------------------ + + def _subset_patient_ids(self) -> list[str] | None: + """Patient IDs belonging to ``self.subset``; ``None`` for ``"all"``. + + * ``split_source="metadata"``: read the canonical mapping file. One + row per subject, so plain pandas is enough. Column names + ``subject_id`` / ``split``. + * ``split_source="directory"``: subjects found under + ``data//``. Column projection keeps the read cheap. + + Computed once per instance and reused across tables, so multi-table + loads pay the read a single time. + """ + if self.subset == "all": + return None + if self._subset_patient_ids_cache is None: + if self.split_source == "metadata": + path = Path(clean_path(f"{self.root}/{SUBJECT_SPLITS_RELPATH}")) + if not path.exists(): + raise FileNotFoundError( + f"subset={self.subset!r} with split_source='metadata' " + f"requires {SUBJECT_SPLITS_RELPATH} under " + f"{self.root!r}. Pass split_source='directory' to " + "derive the split from the data// layout, or " + "use subset='all'." + ) + splits = pd.read_parquet(path).rename(columns=str.lower) + ids = splits.loc[splits["split"] == self.subset, "subject_id"] + else: # "directory" + split_dir = Path( + clean_path(f"{self.root}/{DATA_RELPATH}/{self.subset}") + ) + if not split_dir.is_dir(): + raise FileNotFoundError( + f"subset={self.subset!r} with split_source=" + f"'directory' requires the directory " + f"{DATA_RELPATH}/{self.subset} under {self.root!r}." + ) + ids = ( + self._scan_parquet(str(split_dir))["subject_id"].unique().compute() + ) + self._subset_patient_ids_cache = ids.astype("string").dropna().tolist() + logger.info( + f"MEDS subset={self.subset!r} via split_source=" + f"{self.split_source!r}: " + f"{len(self._subset_patient_ids_cache)} patients" + ) + return self._subset_patient_ids_cache + + def load_data(self) -> dd.DataFrame: + """Load all configured tables, restricted to the subset if any. + + Returns: + dd.DataFrame: The concatenated event frame, filtered to the + subjects of ``self.subset`` when a split was requested. + """ + df = super().load_data() + subset_ids = self._subset_patient_ids() + if subset_ids is not None: + df = df[df["patient_id"].isin(subset_ids)] + return df + + def _validate_event_schema(self) -> None: + """Fails fast when a Parquet event table violates the MEDS contract. + + Only Parquet footers are read (no data, no Dask). For every selected + table whose source is Parquet and whose timestamp is a single column, + that column must exist and be a timezone-naive timestamp type: the + MEDS reference ``DataSchema`` defines ``time`` as ``timestamp[us]`` + without a timezone (verified against the ``meds`` 0.4.1 package). + + This closes, at construction time, the silent-parse hazard of + date-like integers: an ``int64`` column holding ``20240101`` is + rejected here by dtype instead of being parsed as a date deep + inside the Dask graph. + + Raises: + TypeError: If the timestamp column is missing from the Parquet + schema, is not a timestamp type, or is timezone-aware. + """ + for name in self.tables: + table_cfg = self.config.tables.get(name.lower()) + if table_cfg is None: + continue # unknown table: load_table raises the proper error + ts_col = table_cfg.timestamp + if not ts_col or isinstance(ts_col, list): + continue + source = Path(clean_path(f"{self.root}/{table_cfg.file_path}")) + if source.suffix not in (".parquet", ".pq") and not source.is_dir(): + continue # non-Parquet source: string-parse contract applies + schema = pa_ds.dataset(str(source), format="parquet").schema + fields = {field.name.lower(): field for field in schema} + field = fields.get(ts_col.lower()) + if field is None: + raise TypeError( + f"MEDS table '{name}': timestamp column '{ts_col}' is " + f"missing from the Parquet schema {schema.names}." + ) + if not pa.types.is_timestamp(field.type): + raise TypeError( + f"MEDS table '{name}': column '{ts_col}' must be a " + f"timestamp in the Parquet schema, got '{field.type}'. " + "Date-like integers or strings parse unreliably; convert " + "the column upstream (e.g. with MEDS-Transform)." + ) + if field.type.tz is not None: + raise TypeError( + f"MEDS table '{name}': column '{ts_col}' is timezone-" + f"aware ('{field.type}'), but the MEDS reference schema " + "is timezone-naive (timestamp[us]). Normalize upstream, " + "e.g. tz_convert('UTC').tz_localize(None)." + ) diff --git a/pyhealth/tasks/__init__.py b/pyhealth/tasks/__init__.py index 406b457f2..df8411db0 100644 --- a/pyhealth/tasks/__init__.py +++ b/pyhealth/tasks/__init__.py @@ -22,6 +22,9 @@ drug_recommendation_mimic4_fn, drug_recommendation_omop_fn, ) +from .in_hospital_mortality_meds import ( + InHospitalMortalityMEDS as InHospitalMortalityMEDS, +) from .in_hospital_mortality_mimic4 import InHospitalMortalityMIMIC4 from .length_of_stay_prediction import ( LengthOfStayPredictioneICU, diff --git a/pyhealth/tasks/in_hospital_mortality_meds.py b/pyhealth/tasks/in_hospital_mortality_meds.py new file mode 100644 index 000000000..fd356abb0 --- /dev/null +++ b/pyhealth/tasks/in_hospital_mortality_meds.py @@ -0,0 +1,252 @@ +"""In-hospital mortality prediction for datasets in the Medical Event Data +Standard (MEDS). + +This module provides :class:`InHospitalMortalityMEDS`, the MEDS-native +counterpart of +:class:`~pyhealth.tasks.in_hospital_mortality_mimic4.InHospitalMortalityMIMIC4`. +The MIMIC-IV task anchors on a visit object and reads +``admission.hospital_expire_flag``; MEDS represents a hospitalization as two +separate events (``HOSPITAL_ADMISSION//*`` and ``HOSPITAL_DISCHARGE//*``) +that share a ``hadm_id``, so a stay is derived by grouping those events +on that identifier and the label is derived from the discharge code. + +Task definition +--------------- +Let a *stay* be the set of events sharing one ``hadm_id`` for a subject, +with admission time ``t_a`` (earliest admission event) and discharge time +``t_d`` (latest discharge event). For each completed stay +(``t_d > t_a``) the task produces one sample: + +* **Prediction time** ``t_p``. + ``observation_window="full_stay"`` (default) sets ``t_p = t_d``. + ``observation_window="first_hours"`` sets ``t_p = t_a + window_hours`` and + keeps only stays with length of stay strictly greater than + ``window_hours``, so the window is fully observed and the outcome is + strictly future. +* **Features.** The ordered sequence of MEDS ``code`` values in the + half-open interval ``[t_a, t_p)``, excluding every ``HOSPITAL_DISCHARGE//*`` + event and every ``MEDS_DEATH`` event. Both exclusions matter: the half-open + bound already removes the discharge event when ``t_p = t_d``, and dropping + ``MEDS_DEATH`` removes the canonical death sentinel (which the demo places a + few hours after discharge) so the outcome can never leak into the input. +* **Label.** ``mortality = 1`` iff the stay's discharge code is + ``HOSPITAL_DISCHARGE//DIED``. This is the in-hospital, same-stay + definition, consistent with ``hospital_expire_flag`` upstream. On the + public MIMIC-IV demo in MEDS it is a strict superset of ``MEDS_DEATH`` + occurring within a stay: ``MEDS_DEATH`` carries a null ``hadm_id`` there, + so it cannot be attached to a stay and is deliberately not used as the + label. Deaths outside the index stay are a subject-level problem and are + out of scope for this task. + +Configuration +------------- +The task reads ``hadm_id``, which is **not** part of the core MEDS schema +(``subject_id``/``time``/``code``/``numeric_value``/``text_value``) but is +present in MIMIC-derived MEDS datasets. It is therefore kept out of the +default ``configs/meds.yaml`` (selecting an absent attribute would raise for +generic MEDS data). A bundled ``configs/meds_with_hadm.yaml`` exposes it; +pass that config (or your own that lists ``hadm_id``) when using this task. + +Scope note +---------- +The MIMIC-IV task additionally drops pediatric admissions via +``anchor_age``. A MEDS-native age filter is derivable from ``MEDS_BIRTH`` but +is intentionally omitted here: its on-disk representation is not fixed across +MEDS datasets, and silently assuming one would be unsound. Age restriction is +therefore left to a preprocessing step or a future, explicitly parameterized +extension. + +References: + MEDS Working Group. Medical Event Data Standard (MEDS): Facilitating + Machine Learning for Health. ICLR 2024 Workshop on Learning from Time + Series For Health. https://openreview.net/forum?id=IsHy2ebjIG +""" + +from typing import Any, ClassVar + +import polars as pl + +from .base_task import BaseTask + +ADMISSION_PREFIX = "HOSPITAL_ADMISSION" +DISCHARGE_PREFIX = "HOSPITAL_DISCHARGE" +DIED_CODE = "HOSPITAL_DISCHARGE//DIED" +DEATH_CODE = "MEDS_DEATH" + +_FULL_STAY = "full_stay" +_FIRST_HOURS = "first_hours" +_VALID_WINDOWS = (_FULL_STAY, _FIRST_HOURS) + + +class InHospitalMortalityMEDS(BaseTask): + """In-hospital mortality prediction for MEDS datasets. + + One sample per completed hospital stay. The observation window is the + half-open interval ``[admission, prediction_time)`` and the binary label + is whether the stay ended in death (discharge code + ``HOSPITAL_DISCHARGE//DIED``). MEDS codes observed during the window, + excluding the terminating discharge event and any ``MEDS_DEATH``, form + the input sequence. See the module docstring for the full definition. + + Args: + observation_window (str): ``"full_stay"`` (default) observes the + entire stay, i.e. ``[admission, discharge)``. ``"first_hours"`` + observes only ``[admission, admission + window_hours)`` and keeps + stays whose length exceeds ``window_hours`` (an early-warning + setup with a strictly future outcome). + window_hours (float): Observation length used when + ``observation_window="first_hours"``. Ignored for ``"full_stay"``. + Defaults to ``48.0``, matching ``InHospitalMortalityMIMIC4``. + code_mapping (Optional[Dict[str, Tuple[str, str]]]): Optional vocab + mapping forwarded to :class:`BaseTask` (e.g. + ``{"codes": ("ICD10CM", "CCSCM")}``). + + Attributes: + task_name (str): The name of the task. + input_schema (Dict[str, str]): ``codes`` — the sequence of MEDS + codes observed during the window. + output_schema (Dict[str, str]): ``mortality`` — binary in-hospital + mortality. + + Raises: + ValueError: If ``observation_window`` is not one of + ``"full_stay"``/``"first_hours"``, or if ``window_hours`` is not + positive. + + Examples: + >>> from pathlib import Path + >>> import pyhealth.datasets.configs as meds_configs + >>> from pyhealth.datasets import MEDSDataset + >>> from pyhealth.tasks import InHospitalMortalityMEDS + >>> # A bundled stay-aware config exposes hadm_id (not a core MEDS + >>> # field, so it is kept out of the default configs/meds.yaml): + >>> cfg = Path(meds_configs.__file__).parent / "meds_with_hadm.yaml" + >>> dataset = MEDSDataset( + ... root="/path/to/mimic-iv-demo-meds/0.0.1", + ... config_path=str(cfg), + ... ) + >>> samples = dataset.set_task(InHospitalMortalityMEDS()) + >>> # Early-warning variant: first 48h, stays longer than 48h only + >>> early = InHospitalMortalityMEDS(observation_window="first_hours") + """ + + task_name: str = "InHospitalMortalityMEDS" + input_schema: ClassVar[dict[str, str]] = {"codes": "sequence"} + output_schema: ClassVar[dict[str, str]] = {"mortality": "binary"} + + def __init__( + self, + observation_window: str = _FULL_STAY, + window_hours: float = 48.0, + code_mapping: dict[str, tuple[str, str]] | None = None, + ) -> None: + if observation_window not in _VALID_WINDOWS: + raise ValueError( + f"observation_window must be one of {_VALID_WINDOWS}, " + f"got {observation_window!r}." + ) + if window_hours <= 0: + raise ValueError(f"window_hours must be positive, got {window_hours}.") + super().__init__(code_mapping=code_mapping) + self.observation_window = observation_window + self.window_hours = float(window_hours) + + def pre_filter(self, df: pl.LazyFrame) -> pl.LazyFrame: + """Restricts the global scan to MEDS events before per-patient calls. + + All MEDS data lives in a single ``meds`` event type, so this narrows + the frame once rather than per patient. + """ + return df.filter(pl.col("event_type") == "meds") + + def _group_stays(self, events: pl.DataFrame) -> pl.DataFrame: + """Builds one row per stay from admission/discharge events. + + Args: + events (pl.DataFrame): This patient's MEDS events, with an + integer ``_hadm`` column already attached. + + Returns: + pl.DataFrame: Columns ``_hadm``, ``admit``, ``discharge``, + ``discharge_code``, one row per ``hadm_id`` that has both an + admission and a discharge. Malformed duplicates collapse via + earliest-admission / latest-discharge aggregation. + """ + code = pl.col("meds/code") + admissions = ( + events.filter(code.str.starts_with(ADMISSION_PREFIX)) + .filter(pl.col("_hadm").is_not_null()) + .group_by("_hadm") + .agg(pl.col("timestamp").min().alias("admit")) + ) + discharges = ( + events.filter(code.str.starts_with(DISCHARGE_PREFIX)) + .filter(pl.col("_hadm").is_not_null()) + .group_by("_hadm") + .agg( + pl.col("timestamp").max().alias("discharge"), + code.sort_by("timestamp").last().alias("discharge_code"), + ) + ) + return admissions.join(discharges, on="_hadm", how="inner") + + def __call__(self, patient: Any) -> list[dict[str, Any]]: + events = patient.get_events(event_type="meds", return_df=True) + if events.height == 0: + return [] + + # A nullable integer id is promoted to float through the Dask/pandas + # pipeline whenever the column carries nulls (e.g. lab events and the + # MEDS_DEATH sentinel). Cast back to a nullable integer so stays join + # cleanly and emitted ids stay integral rather than "555.0". + events = events.with_columns( + pl.col("meds/hadm_id").cast(pl.Int64, strict=False).alias("_hadm") + ) + code = pl.col("meds/code") + + stays = self._group_stays(events) + if stays.height == 0: + return [] + + samples: list[dict[str, Any]] = [] + for stay in stays.sort("admit").iter_rows(named=True): + admit, discharge = stay["admit"], stay["discharge"] + if discharge <= admit: + continue # degenerate/zero-length stay + + if self.observation_window == _FIRST_HOURS: + duration_hours = (discharge - admit).total_seconds() / 3600.0 + if duration_hours <= self.window_hours: + continue # window not fully observed within this stay + predict_time = admit + _timedelta_hours(self.window_hours) + else: + predict_time = discharge + + window = events.filter( + (pl.col("timestamp") >= admit) + & (pl.col("timestamp") < predict_time) # half-open: excludes t_p + & (~code.str.starts_with(DISCHARGE_PREFIX)) + & (code != DEATH_CODE) + ).sort("timestamp") + + codes = window["meds/code"].to_list() + if not codes: + continue # no observable signal before the prediction time + + samples.append( + { + "patient_id": patient.patient_id, + "hadm_id": stay["_hadm"], + "codes": codes, + "mortality": int(stay["discharge_code"] == DIED_CODE), + } + ) + + return samples + + +def _timedelta_hours(hours: float): + """Returns a ``datetime.timedelta`` of ``hours`` (kept import-local).""" + from datetime import timedelta + + return timedelta(hours=hours) diff --git a/tests/core/test_in_hospital_mortality_meds.py b/tests/core/test_in_hospital_mortality_meds.py new file mode 100644 index 000000000..6ae62aae7 --- /dev/null +++ b/tests/core/test_in_hospital_mortality_meds.py @@ -0,0 +1,278 @@ +"""Tests for InHospitalMortalityMEDS. + +The synthetic tests build a small MEDS-shaped dataset (typed Parquet shards +with a ``hadm_id`` column, plus a stay-aware config) with no real data and no +downloads. Feature-content assertions apply the task directly to real +``Patient`` objects (``dataset.get_patient(...)``), because that yields the +task's raw ``List[Dict]`` output; going through ``set_task`` instead would +tokenize the ``codes`` sequence into integer indices (the ``sequence`` +processor), which is the correct user path but hides the string codes the +leakage tests must inspect. A separate test exercises ``set_task`` end to end +to confirm the intended path produces the expected number of samples. + +The central property under test is the absence of label leakage: for a stay +that ends in death, neither the discharge event nor the ``MEDS_DEATH`` +sentinel may appear in the emitted feature sequence. +``TestInHospitalMortalityMEDSDemoSmoke`` optionally exercises the public +MIMIC-IV demo in MEDS when it is available locally, and is skipped otherwise. +""" + +import os +import shutil +import tempfile +import unittest +from datetime import datetime, timedelta +from pathlib import Path + +import polars as pl + +from pyhealth.datasets import MEDSDataset +from pyhealth.tasks import InHospitalMortalityMEDS +from pyhealth.tasks.in_hospital_mortality_meds import ( + DEATH_CODE, + DISCHARGE_PREFIX, +) + +T0 = datetime(2024, 1, 1, 8, 0, 0) + +# Stay-aware config: exposes hadm_id, which the task requires. +_CONFIG = """version: "1.0" +tables: + meds: + file_path: "data" + patient_id: "subject_id" + timestamp: "time" + attributes: + - "code" + - "numeric_value" + - "hadm_id" +""" + + +def _events_to_frame(rows): + """Rows are (subject_id, offset_hours, code, hadm_id_or_None).""" + return pl.DataFrame( + { + "subject_id": pl.Series([r[0] for r in rows], dtype=pl.Int64), + "time": pl.Series( + [T0 + timedelta(hours=r[1]) for r in rows], dtype=pl.Datetime("us") + ), + "code": pl.Series([r[2] for r in rows], dtype=pl.String), + "numeric_value": pl.Series([None] * len(rows), dtype=pl.Float32), + "hadm_id": pl.Series( + [r[3] for r in rows], + dtype=pl.Int64, # nullable + ), + } + ) + + +class TestInHospitalMortalityMEDS(unittest.TestCase): + """Task behavior on a synthetic MEDS dataset via set_task.""" + + def setUp(self): + self.temp_dir = Path(tempfile.mkdtemp()) + self.root = self.temp_dir / "meds" + (self.root / "data").mkdir(parents=True) + self.cache_root = self.temp_dir / "cache" + self.config_path = self.temp_dir / "meds_hadm.yaml" + self.config_path.write_text(_CONFIG) + self._write_default_cohort() + + def tearDown(self): + if self.temp_dir.exists(): + # ignore_errors: litdata may keep chunk handles open on Windows. + shutil.rmtree(self.temp_dir, ignore_errors=True) + + def _write_default_cohort(self): + # Subject 1: a stay ending in death (hadm 555), a MEDS_DEATH a few + # hours later (null hadm), a post-death stray event, then a second + # stay ending at home (hadm 777). Subject 2: a survived stay whose + # subject later dies out of hospital. + rows = [ + (1, 0, "HOSPITAL_ADMISSION//EW", 555), + (1, 2, "LAB//50912", 555), + (1, 6, "MED//aspirin", 555), + (1, 10, "HOSPITAL_DISCHARGE//DIED", 555), + (1, 14, DEATH_CODE, None), + (1, 16, "LAB//stray", None), + (1, 120, "HOSPITAL_ADMISSION//OBS", 777), + (1, 121, "LAB//x", 777), + (1, 144, "HOSPITAL_DISCHARGE//HOME", 777), + (2, 0, "HOSPITAL_ADMISSION//EW", 999), + (2, 5, "HOSPITAL_DISCHARGE//HOME", 999), + (2, 200, DEATH_CODE, None), + ] + _events_to_frame(rows).write_parquet(self.root / "data" / "0.parquet") + + def _dataset(self): + return MEDSDataset( + root=str(self.root), + config_path=str(self.config_path), + cache_dir=self.cache_root, + ) + + def _apply(self, task=None): + """Applies the task to every patient, returning raw sample dicts. + + This mirrors what ``set_task`` does per patient but keeps the task's + untokenized output so feature sequences remain inspectable. + """ + task = task or InHospitalMortalityMEDS() + dataset = self._dataset() + samples = [] + for pid in dataset.unique_patient_ids: + samples.extend(task(dataset.get_patient(pid))) + return samples + + def _by_hadm(self, samples): + return {s["hadm_id"]: s for s in samples} + + def test_one_sample_per_completed_stay(self): + samples = self._apply() + # Three completed stays across the two subjects. + self.assertEqual(len(samples), 3) + self.assertEqual(sorted(s["hadm_id"] for s in samples), [555, 777, 999]) + # ids are integral, not promoted floats + self.assertTrue(all(isinstance(s["hadm_id"], int) for s in samples)) + + def test_labels_from_discharge_code(self): + by = self._by_hadm(self._apply()) + self.assertEqual(by[555]["mortality"], 1) # DIED + self.assertEqual(by[777]["mortality"], 0) # HOME + self.assertEqual(by[999]["mortality"], 0) # HOME (subject dies later) + + def test_no_label_leakage_in_positive_stay(self): + """The defining safety property: outcome never enters the features.""" + died = self._by_hadm(self._apply())[555] + for code in died["codes"]: + self.assertFalse(code.startswith(DISCHARGE_PREFIX)) + self.assertNotEqual(code, DEATH_CODE) + # Exactly the pre-discharge, non-death events, in order. + self.assertEqual( + died["codes"], + ["HOSPITAL_ADMISSION//EW", "LAB//50912", "MED//aspirin"], + ) + # The stray post-death event is excluded too. + self.assertNotIn("LAB//stray", died["codes"]) + + def test_meds_death_without_hadm_never_labels_a_stay(self): + # Subject 2 dies out of hospital; the in-hospital stay stays negative. + self.assertEqual(self._by_hadm(self._apply())[999]["mortality"], 0) + + def test_first_hours_requires_sufficient_length_of_stay(self): + # Default cohort: no stay exceeds 48h, so the early-warning variant + # yields nothing. + task = InHospitalMortalityMEDS(observation_window="first_hours") + self.assertEqual(self._apply(task), []) + + def test_first_hours_observes_only_the_window(self): + # A single long stay (LOS 100h) observed for its first 48h. + (self.root / "data" / "0.parquet").unlink() + rows = [ + (7, 0, "HOSPITAL_ADMISSION//EW", 900), + (7, 12, "LAB//a", 900), + (7, 47, "LAB//b", 900), # inside 48h + (7, 60, "LAB//c", 900), # outside 48h + (7, 100, "HOSPITAL_DISCHARGE//DIED", 900), + ] + _events_to_frame(rows).write_parquet(self.root / "data" / "0.parquet") + task = InHospitalMortalityMEDS( + observation_window="first_hours", window_hours=48.0 + ) + samples = self._apply(task) + self.assertEqual(len(samples), 1) + self.assertEqual(samples[0]["mortality"], 1) # eventual outcome + self.assertEqual( + samples[0]["codes"], + ["HOSPITAL_ADMISSION//EW", "LAB//a", "LAB//b"], + ) + self.assertNotIn("LAB//c", samples[0]["codes"]) + + def test_discharge_boundary_is_half_open(self): + (self.root / "data" / "0.parquet").unlink() + rows = [ + (8, 0, "HOSPITAL_ADMISSION//EW", 111), + (8, 4, "LAB//inside", 111), + (8, 5, "LAB//at_discharge", 111), # exactly at t_discharge + (8, 5, "HOSPITAL_DISCHARGE//HOME", 111), + ] + _events_to_frame(rows).write_parquet(self.root / "data" / "0.parquet") + samples = self._apply() + self.assertEqual(len(samples), 1) + self.assertEqual(samples[0]["codes"], ["HOSPITAL_ADMISSION//EW", "LAB//inside"]) + + def test_invalid_parameters_raise(self): + with self.assertRaises(ValueError): + InHospitalMortalityMEDS(observation_window="bogus") + with self.assertRaises(ValueError): + InHospitalMortalityMEDS(window_hours=0) + with self.assertRaises(ValueError): + InHospitalMortalityMEDS(window_hours=-3) + + def test_set_task_integration_yields_expected_count(self): + """The intended user path runs and produces one sample per stay. + + Codes are tokenized by the sequence processor here, so only the + sample count (structure) is asserted; content/leakage is covered by + the get_patient-based tests above. + """ + dataset = self._dataset() + sample_dataset = dataset.set_task(InHospitalMortalityMEDS()) + self.assertEqual(len(sample_dataset), 3) + + +def _demo_root() -> str: + env = os.environ.get("MEDS_DEMO_ROOT") + if env: + return env + test_dir = Path(__file__).parent.parent.parent + return str(test_dir / "test-resources" / "meds_demo") + + +@unittest.skipUnless( + Path(_demo_root()).is_dir(), + "MIMIC-IV demo in MEDS format not available locally " + "(set MEDS_DEMO_ROOT or place it under test-resources/meds_demo)", +) +class TestInHospitalMortalityMEDSDemoSmoke(unittest.TestCase): + """Smoke test on the public MIMIC-IV demo in MEDS format. + + Requires a config exposing hadm_id; this test writes one next to a + temporary cache. The demo (PhysioNet, https://doi.org/10.13026/t2y8-ea41, + ODbL v1.0) is never downloaded here. + """ + + def setUp(self): + self.temp_dir = Path(tempfile.mkdtemp()) + self.config_path = self.temp_dir / "meds_hadm.yaml" + self.config_path.write_text(_CONFIG) + + def tearDown(self): + if self.temp_dir.exists(): + shutil.rmtree(self.temp_dir, ignore_errors=True) + + def test_produces_stays_without_leakage(self): + dataset = MEDSDataset( + root=_demo_root(), + config_path=str(self.config_path), + cache_dir=self.temp_dir, + ) + task = InHospitalMortalityMEDS() + # Apply per patient to inspect raw (untokenized) code sequences. + samples = [] + for pid in dataset.unique_patient_ids: + samples.extend(task(dataset.get_patient(pid))) + self.assertGreater(len(samples), 0) + # No sample may contain a discharge or death code (leakage guard). + for sample in samples: + for code in sample["codes"]: + self.assertFalse(code.startswith(DISCHARGE_PREFIX)) + self.assertNotEqual(code, DEATH_CODE) + n_positive = sum(int(s["mortality"]) for s in samples) + self.assertGreater(n_positive, 0) + self.assertLess(n_positive, len(samples)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/core/test_meds.py b/tests/core/test_meds.py new file mode 100644 index 000000000..4ad39658c --- /dev/null +++ b/tests/core/test_meds.py @@ -0,0 +1,309 @@ +"""Tests for MEDSDataset (synthetic Parquet fixtures, fixed seeds). + +Optional smoke on a real export: set ``MEDS_DEMO_ROOT`` to the dataset version +directory (the folder containing ``data/`` and ``metadata/``). +""" + +from __future__ import annotations + +import os +import shutil +import tempfile +import unittest +from datetime import datetime +from pathlib import Path +from typing import Dict, List +from unittest.mock import patch + +import numpy as np +import pandas as pd +import polars as pl +import pyarrow as pa +import pyarrow.parquet as pq + +from pyhealth.datasets import MEDSDataset, MIMIC3Dataset +from pyhealth.tasks.base_task import BaseTask + +T1 = datetime(2024, 1, 1, 8, 0, 0) +T2 = datetime(2024, 1, 2, 9, 30, 0) + +MEDS_DEMO_ROOT = os.environ.get("MEDS_DEMO_ROOT") +_DEFAULT_DEMO = ( + Path(__file__).parent.parent.parent / "test-resources" / "meds_demo" +) +MEDS_DEMO_PATH = ( + Path(MEDS_DEMO_ROOT).expanduser() + if MEDS_DEMO_ROOT + else _DEFAULT_DEMO +) + +# Fixed split assignment for deterministic assertions. +_SYNTHETIC_SPLITS: Dict[str, List[int]] = { + "train": [1001, 1002, 1003, 1004], + "tuning": [1005, 1006], + "held_out": [1007, 1008], +} + + +def write_synthetic_meds(root: Path, *, seed: int = 42, rows_per_shard: int = 25) -> None: + """Write a MEDS-shaped tree under ``root`` (``data//*.parquet`` + metadata).""" + rng = np.random.default_rng(seed) + data_root = root / "data" + for split, subjects in _SYNTHETIC_SPLITS.items(): + split_dir = data_root / split + split_dir.mkdir(parents=True, exist_ok=True) + for shard in range(2): + n = rows_per_shard + pq.write_table( + pa.table( + { + "subject_id": pa.array(rng.choice(subjects, n), type=pa.int64()), + "time": pa.array( + pd.date_range("2020-01-01", periods=n, freq="h"), + type=pa.timestamp("us"), + ), + "code": pa.array( + [f"LAB//{i % 5}" for i in range(n)], type=pa.string() + ), + "numeric_value": pa.array( + rng.normal(size=n).astype(np.float32), type=pa.float32() + ), + } + ), + split_dir / f"{shard}.parquet", + ) + + meta = root / "metadata" + meta.mkdir(exist_ok=True) + all_subjects = [sid for ids in _SYNTHETIC_SPLITS.values() for sid in ids] + all_splits = [ + split for split, ids in _SYNTHETIC_SPLITS.items() for _ in ids + ] + pq.write_table( + pa.table( + { + "subject_id": pa.array(all_subjects, type=pa.int64()), + "split": pa.array(all_splits, type=pa.string()), + } + ), + meta / "subject_splits.parquet", + ) + + +class _MedsSmokeTask(BaseTask): + """Minimal task: one sample per patient with at least one meds event.""" + + task_name: str = "MedsSmokeTask" + input_schema: Dict[str, str] = {"codes": "sequence"} + output_schema: Dict[str, str] = {"has_events": "binary"} + + def __call__(self, patient): + meds = patient.get_events(event_type="meds") + if not meds: + return [] + codes = [event.code for event in meds if event.code] + if not codes: + return [] + return [ + { + "patient_id": patient.patient_id, + "codes": codes, + "has_events": int(patient.patient_id) % 2, + } + ] + + +class TestMEDSDatasetSynthetic(unittest.TestCase): + """MEDSDataset against a local synthetic MEDS export.""" + + @classmethod + def setUpClass(cls) -> None: + cls._tmp = tempfile.mkdtemp(prefix="meds_synthetic_") + cls.root = Path(cls._tmp) + write_synthetic_meds(cls.root) + + @classmethod + def tearDownClass(cls) -> None: + shutil.rmtree(cls._tmp, ignore_errors=True) + + def _dataset(self, **kwargs) -> MEDSDataset: + return MEDSDataset( + root=str(self.root), + cache_dir=self._tmp, + num_workers=1, + **kwargs, + ) + + def test_load_table_schema_and_dtypes(self) -> None: + ds = self._dataset() + df = ds.load_table("meds").compute() + self.assertIn("patient_id", df.columns) + self.assertIn("timestamp", df.columns) + self.assertIn("event_type", df.columns) + self.assertIn("meds/code", df.columns) + self.assertIn("meds/numeric_value", df.columns) + self.assertEqual(str(df["patient_id"].dtype), "string") + self.assertEqual(str(df["timestamp"].dtype), "datetime64[ms]") + self.assertTrue((df["event_type"] == "meds").all()) + + def test_loads_all_patients(self) -> None: + ds = self._dataset() + expected = {str(sid) for sid in _SYNTHETIC_SPLITS["train"]} + expected |= {str(sid) for sid in _SYNTHETIC_SPLITS["tuning"]} + expected |= {str(sid) for sid in _SYNTHETIC_SPLITS["held_out"]} + self.assertEqual(set(ds.unique_patient_ids), expected) + + def test_subset_train_via_metadata(self) -> None: + ds = self._dataset(subset="train", split_source="metadata") + expected = {str(sid) for sid in _SYNTHETIC_SPLITS["train"]} + self.assertEqual(set(ds.unique_patient_ids), expected) + + def test_subset_tuning_via_directory(self) -> None: + ds = self._dataset(subset="tuning", split_source="directory") + expected = {str(sid) for sid in _SYNTHETIC_SPLITS["tuning"]} + self.assertEqual(set(ds.unique_patient_ids), expected) + + def test_subject_splits_exposed_as_events(self) -> None: + ds = self._dataset(tables=["meds", "subject_splits"]) + patient_id = str(_SYNTHETIC_SPLITS["train"][0]) + patient = ds.get_patient(patient_id) + split_events = patient.get_events(event_type="subject_splits") + self.assertEqual(len(split_events), 1) + self.assertEqual(split_events[0].split, "train") + + def test_patient_meds_event_attributes(self) -> None: + ds = self._dataset(subset="train") + patient = ds.get_patient(str(_SYNTHETIC_SPLITS["train"][0])) + meds = patient.get_events(event_type="meds") + self.assertGreater(len(meds), 0) + self.assertTrue(str(meds[0].code).startswith("LAB//")) + + def test_invalid_subset_raises(self) -> None: + with self.assertRaises(ValueError): + self._dataset(subset="validation") + + def test_schema_violations_raise_type_error_at_construction(self): + """The footer guard rejects non-conforming `time` before any Dask. + + Covers the ADR 002 T5 hazard (date-like ints such as 20240101 would + otherwise parse silently) plus strings, timezone-aware timestamps, + and a missing column. + """ + cases = { + "int64": ( + "time", + pl.Series([20240101, 20240102], dtype=pl.Int64), + ), + "string": ( + "time", + pl.Series(["2024-01-01", "2024-01-02"], dtype=pl.String), + ), + "tz_aware": ( + "time", + pl.Series([T1, T2], dtype=pl.Datetime("us", "UTC")), + ), + "missing": ("ts", pl.Series([T1, T2], dtype=pl.Datetime("us"))), + } + for label, (col_name, series) in cases.items(): + with self.subTest(time=label): + bad_root = Path(self._tmp) / f"meds_bad_{label}" + (bad_root / "data").mkdir(parents=True) + pl.DataFrame( + { + "subject_id": pl.Series([1, 2], dtype=pl.Int64), + col_name: series, + "code": pl.Series(["A", "B"], dtype=pl.String), + "numeric_value": pl.Series( + [None, None], dtype=pl.Float32 + ), + } + ).write_parquet(bad_root / "data" / "0.parquet") + with self.assertRaises(TypeError): + MEDSDataset(root=str(bad_root), cache_dir=self._tmp) + + def test_cache_dir_varies_with_subset(self) -> None: + with patch( + "pyhealth.datasets.base_dataset.platformdirs.user_cache_dir", + return_value=self._tmp, + ): + all_ds = MEDSDataset( + root=str(self.root), + cache_dir=self._tmp, + num_workers=1, + ) + train_ds = MEDSDataset( + root=str(self.root), + cache_dir=self._tmp, + subset="train", + split_source="metadata", + num_workers=1, + ) + self.assertNotEqual(all_ds.cache_dir, train_ds.cache_dir) + + def test_set_task_smoke(self) -> None: + ds = self._dataset(subset="train") + sample_ds = ds.set_task(_MedsSmokeTask(), num_workers=1) + self.assertGreater(len(sample_ds), 0) + sample = sample_ds[0] + self.assertIn("codes", sample) + self.assertEqual(sample["has_events"], int(sample["patient_id"]) % 2) + + def test_mimic3_csv_path_unchanged(self) -> None: + """Non-regression: CSV-backed datasets still load after MEDSDataset addition.""" + demo = ( + Path(__file__).parent.parent.parent + / "test-resources" + / "core" + / "mimic3demo" + ) + ds = MIMIC3Dataset( + root=str(demo), + tables=["diagnoses_icd"], + cache_dir=self._tmp, + num_workers=1, + ) + self.assertGreater(len(ds.unique_patient_ids), 0) + + +@unittest.skipUnless( + MEDS_DEMO_PATH.is_dir() + and (MEDS_DEMO_PATH / "data").is_dir() + and (MEDS_DEMO_PATH / "metadata" / "subject_splits.parquet").is_file(), + "Download mimic-iv-demo-meds into test-resources/meds_demo or set MEDS_DEMO_ROOT", +) +class TestMEDSDatasetDemoSmoke(unittest.TestCase): + """Smoke on mimic-iv-demo-meds (partial export is enough for dtype checks).""" + + @classmethod + def setUpClass(cls) -> None: + cls.root = MEDS_DEMO_PATH.resolve() + cls.cache = tempfile.mkdtemp(prefix="meds_demo_") + + @classmethod + def tearDownClass(cls) -> None: + shutil.rmtree(cls.cache, ignore_errors=True) + + def test_demo_load_table_dtypes(self) -> None: + ds = MEDSDataset( + root=str(self.root), + cache_dir=self.cache, + num_workers=1, + ) + df = ds.load_table("meds").compute() + self.assertEqual(str(df["patient_id"].dtype), "string") + self.assertEqual(str(df["timestamp"].dtype), "datetime64[ms]") + self.assertGreater(len(df), 0) + + def test_demo_stats_and_subset(self) -> None: + ds = MEDSDataset( + root=str(self.root), + cache_dir=self.cache, + subset="train", + num_workers=1, + ) + ds.stats() + self.assertGreater(len(ds.unique_patient_ids), 0) + + +if __name__ == "__main__": + unittest.main() From a92c25a556df94de363196c843a45fad64c08c53 Mon Sep 17 00:00:00 2001 From: William Pang Date: Mon, 10 Aug 2026 20:27:57 -0700 Subject: [PATCH 21/61] Initial Push --- .../unified_embedding_e2e_mimic4.py | 500 ++++++++ pyhealth/models/bottleneck_transformer.py | 520 ++++++++ pyhealth/models/ehrmamba.py | 101 +- pyhealth/models/embedding/base.py | 52 + pyhealth/models/embedding/unified.py | 595 +++++++++ pyhealth/models/embedding/vanilla.py | 366 ++++++ pyhealth/models/embedding/vision.py | 384 ++++++ pyhealth/models/jamba_ehr.py | 93 +- pyhealth/models/rnn.py | 88 +- pyhealth/models/transformer.py | 140 ++- pyhealth/processors/time_image_processor.py | 105 +- .../processors/tuple_time_text_processor.py | 56 +- .../will/condor/labs_only/labs_only_rnn.sub | 45 + .../condor/labs_only/run_labs_only_rnn.sh | 168 +++ pyhealth/tasks/multimodal_mimic4.py | 1064 +++++++++++++++++ pyhealth/trainer.py | 157 ++- 16 files changed, 4292 insertions(+), 142 deletions(-) create mode 100644 examples/mortality_prediction/unified_embedding_e2e_mimic4.py create mode 100644 pyhealth/models/bottleneck_transformer.py create mode 100644 pyhealth/models/embedding/base.py create mode 100644 pyhealth/models/embedding/unified.py create mode 100644 pyhealth/models/embedding/vanilla.py create mode 100644 pyhealth/models/embedding/vision.py create mode 100644 pyhealth/scripts_delete_me/will/condor/labs_only/labs_only_rnn.sub create mode 100644 pyhealth/scripts_delete_me/will/condor/labs_only/run_labs_only_rnn.sh create mode 100644 pyhealth/tasks/multimodal_mimic4.py diff --git a/examples/mortality_prediction/unified_embedding_e2e_mimic4.py b/examples/mortality_prediction/unified_embedding_e2e_mimic4.py new file mode 100644 index 000000000..d7182955a --- /dev/null +++ b/examples/mortality_prediction/unified_embedding_e2e_mimic4.py @@ -0,0 +1,500 @@ +"""End-to-end protocol runner for Unified Embedding on MIMIC-IV. + +Trains and evaluates a unified-embedding model (RNN / Transformer / +BottleneckTransformer / EHRMamba / JambaEHR) on a MIMIC-IV mortality task, +then writes per-sample predictions to CSV. + +Tasks +----- +--task labs (default) + LabsMIMIC4: 10-dim lab vectors only. + +--task notes_labs (recommended for multimodal) + NotesLabsMIMIC4: notes + 10-dim lab vectors. + +--task notes_labs_cxr + NotesLabsCXRMIMIC4: notes + labs + chest-xray. + +Example +------- + python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \\ + --ehr-root /path/to/mimiciv/2.2 \\ + --task labs \\ + --model transformer \\ + --heads 4 --num-layers 2 \\ + --dev --device cpu \\ + --epochs 10 --batch-size 32 --lr 1e-3 \\ + --output-dir ./output/unified_e2e + + # EHRMamba on full dataset (no --dev): + python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \\ + --ehr-root /data/mimic-iv/2.2 --note-root /data/mimic-iv/note \\ + --task notes_labs --model ehrmamba \\ + --embedding-dim 128 --num-layers 2 --seed 42 + + # JambaEHR: + python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \\ + --ehr-root /data/mimic-iv/2.2 --note-root /data/mimic-iv/note \\ + --task notes_labs --model jambaehr \\ + --embedding-dim 128 --jamba-transformer-layers 2 --jamba-mamba-layers 6 +""" + +from __future__ import annotations + +import argparse +import csv +from pathlib import Path +from typing import Any, Dict, Optional, Tuple + +import numpy as np + +from pyhealth.datasets import ( + MIMIC4Dataset, + get_dataloader, + split_by_patient, + split_by_sample, +) +from pyhealth.models import RNN, Transformer, UnifiedMultimodalEmbeddingModel +from pyhealth.models.bottleneck_transformer import BottleneckTransformer +from pyhealth.models.ehrmamba import EHRMamba +from pyhealth.models.jamba_ehr import JambaEHR +from pyhealth.tasks.multimodal_mimic4 import ( + LabsMIMIC4, + NotesLabsCXRMIMIC4, + NotesLabsMIMIC4, +) +from pyhealth.trainer import Trainer +from pyhealth.utils import set_seed + + +class WandbLogger: + + def __init__( + self, + enabled: bool, + project: str, + entity: Optional[str], + run_name: str, + tags: list[str], + config: Dict[str, Any], + ) -> None: + self.enabled = enabled + self._run = None + if self.enabled: + import wandb + + self._run = wandb.init( + project=project, + entity=entity, + name=run_name, + tags=tags, + config=config, + ) + + def log(self, data: Dict[str, Any], step: Optional[int] = None) -> None: + if self.enabled: + self._run.log(data, step=step) + + def finish(self) -> None: + if self.enabled: + self._run.finish() + + +def _build_base_dataset(args: argparse.Namespace) -> MIMIC4Dataset: + ehr_tables = ["labevents"] + note_tables = None + cxr_kwargs = {} + + if args.task == "notes_labs": + if not args.note_root: + raise ValueError("--task notes_labs requires --note-root.") + note_tables = ["discharge", "radiology"] + + if args.task == "notes_labs_cxr": + if not args.note_root: + raise ValueError("--task notes_labs_cxr requires --note-root.") + if not args.cxr_root: + raise ValueError("--task notes_labs_cxr requires --cxr-root.") + note_tables = ["discharge", "radiology"] + cxr_kwargs = dict( + cxr_root=args.cxr_root, + cxr_variant=args.cxr_variant, + cxr_tables=["metadata", "negbio", "chexpert", "split"], + ) + + return MIMIC4Dataset( + ehr_root=args.ehr_root, + ehr_tables=ehr_tables, + note_root=args.note_root if note_tables else None, + note_tables=note_tables, + cache_dir=args.cache_dir, + dev=args.dev if args.dev else False, + num_workers=args.num_workers, + **cxr_kwargs, + ) + + +def _build_task(args: argparse.Namespace): + if args.task == "notes_labs": + return NotesLabsMIMIC4( + window_hours=args.observation_window_hours, + ) + if args.task == "notes_labs_cxr": + return NotesLabsCXRMIMIC4( + window_hours=args.observation_window_hours, + ) + if args.task == "labs": + return LabsMIMIC4(window_hours=args.observation_window_hours) + raise ValueError(f"Unknown task: {args.task}") + + +def _split_dataset(dataset: Any, seed: int) -> Tuple[Any, Any, Any]: + train_ds, val_ds, test_ds = split_by_patient(dataset, [0.8, 0.1, 0.1], seed=seed) + if len(train_ds) == 0 or len(test_ds) == 0: + train_ds, val_ds, test_ds = split_by_sample(dataset, [0.8, 0.1, 0.1], seed=seed) + return train_ds, val_ds, test_ds + + +def _build_model(args: argparse.Namespace, sample_dataset: Any): + unified = UnifiedMultimodalEmbeddingModel( + processors=sample_dataset.input_processors, + embedding_dim=args.embedding_dim, + freeze_text_encoder=args.freeze_encoder, + ) + + if args.model == "rnn": + return RNN( + dataset=sample_dataset, + embedding_dim=args.embedding_dim, + hidden_dim=args.hidden_dim, + unified_embedding=unified, + rnn_type=args.rnn_type, + num_layers=args.rnn_layers, + dropout=args.dropout, + bidirectional=args.bidirectional, + ) + if args.model == "transformer": + return Transformer( + dataset=sample_dataset, + embedding_dim=args.embedding_dim, + heads=args.heads, + num_layers=args.num_layers, + dropout=args.dropout, + unified_embedding=unified, + ) + if args.model == "bottleneck_transformer": + return BottleneckTransformer( + dataset=sample_dataset, + embedding_dim=args.embedding_dim, + bottlenecks_n=args.bottlenecks_n, + fusion_startidx=args.fusion_startidx, + num_layers=args.num_layers, + heads=args.heads, + dropout=args.dropout, + unified_embedding=unified, + ) + if args.model == "ehrmamba": + return EHRMamba( + dataset=sample_dataset, + embedding_dim=args.embedding_dim, + num_layers=args.num_layers, + state_size=args.mamba_state_size, + conv_kernel=args.mamba_conv_kernel, + dropout=args.dropout, + unified_embedding=unified, + ) + if args.model == "jambaehr": + return JambaEHR( + dataset=sample_dataset, + embedding_dim=args.embedding_dim, + num_transformer_layers=args.jamba_transformer_layers, + num_mamba_layers=args.jamba_mamba_layers, + heads=args.heads, + dropout=args.dropout, + state_size=args.mamba_state_size, + conv_kernel=args.mamba_conv_kernel, + unified_embedding=unified, + ) + raise ValueError(f"Unknown model: {args.model}") + + +def _write_predictions( + output_csv: Path, + patient_ids: list[str], + y_true: np.ndarray, + y_prob: np.ndarray, +) -> None: + output_csv.parent.mkdir(parents=True, exist_ok=True) + + y_true_flat = y_true.reshape(-1).tolist() + y_prob_flat = y_prob.reshape(-1).tolist() + + with output_csv.open("w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter( + f, + fieldnames=["patient_id", "y_true", "y_prob", "y_pred_threshold_0_5"], + ) + writer.writeheader() + for idx, prob in enumerate(y_prob_flat): + writer.writerow( + { + "patient_id": patient_ids[idx], + "y_true": int(y_true_flat[idx]), + "y_prob": float(prob), + "y_pred_threshold_0_5": int(float(prob) >= 0.5), + } + ) + + +def run(args: argparse.Namespace) -> Path: + set_seed(args.seed) + + base_dataset = _build_base_dataset(args) + task = _build_task(args) + sample_dataset = base_dataset.set_task(task, num_workers=args.num_workers) + + if len(sample_dataset) == 0: + raise RuntimeError( + "Task produced zero samples. Check roots/tables or adjust settings." + ) + + train_ds, val_ds, test_ds = _split_dataset(sample_dataset, seed=args.seed) + + model = _build_model(args, sample_dataset) + + train_loader = get_dataloader(train_ds, batch_size=args.batch_size, shuffle=True) + val_loader = ( + get_dataloader(val_ds, batch_size=args.batch_size, shuffle=False) + if len(val_ds) > 0 + else None + ) + test_loader = ( + get_dataloader(test_ds, batch_size=args.batch_size, shuffle=False) + if len(test_ds) > 0 + else None + ) + + exp_name = f"{args.model}_seed{args.seed}" + output_dir = Path(args.output_dir) + + wandb_logger = WandbLogger( + enabled=args.wandb, + project=args.wandb_project, + entity=args.wandb_entity, + run_name=args.wandb_run_name or exp_name, + tags=args.wandb_tags.split(",") if args.wandb_tags else [args.task, args.model], + config=vars(args), + ) + + trainer = Trainer( + model=model, + metrics=["pr_auc", "roc_auc", "f1", "accuracy"], + device=args.device, + enable_logging=True, + output_path=str(output_dir), + exp_name=exp_name, + ) + + # BottleneckTransformer is more fragile on full MIMIC-IV with no warmup. + # Use safer defaults unless explicitly overridden from CLI. + effective_lr = args.lr + effective_max_grad_norm = args.max_grad_norm + optimizer_params = {} + + if args.model == "bottleneck_transformer": + if effective_lr is None: + effective_lr = 1e-4 + if effective_max_grad_norm is None: + effective_max_grad_norm = 0.5 + optimizer_params["eps"] = args.adam_eps if args.adam_eps is not None else 1e-6 + else: + if effective_lr is None: + effective_lr = 1e-4 + if effective_max_grad_norm is None: + effective_max_grad_norm = 1.0 + if args.adam_eps is not None: + optimizer_params["eps"] = args.adam_eps + + optimizer_params["lr"] = effective_lr + + if args.epochs > 0 and len(train_ds) > 0: + metrics_history = trainer.train( + train_dataloader=train_loader, + val_dataloader=val_loader, + epochs=args.epochs, + optimizer_params=optimizer_params, + weight_decay=args.weight_decay, + max_grad_norm=effective_max_grad_norm, + monitor="pr_auc", + load_best_model_at_last=True, + patience=args.patience, + use_amp=args.use_amp, + amp_dtype=args.amp_dtype, + ) + for epoch_record in metrics_history: + wandb_logger.log(epoch_record, step=epoch_record["epoch"]) + + if wandb_logger.enabled and test_loader is not None: + test_scores = trainer.evaluate(test_loader) + wandb_logger.log({f"test_{k}": v for k, v in test_scores.items()}) + + inference_loader = test_loader or val_loader or train_loader + y_true, y_prob, _, patient_ids = trainer.inference( + inference_loader, return_patient_ids=True + ) + + output_csv = output_dir / exp_name / f"predictions_{args.model}.csv" + _write_predictions(output_csv, patient_ids, y_true, y_prob) + + wandb_logger.finish() + + return output_csv + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run E2E unified embedding on MIMIC-IV with any of six sequence heads." + ) + parser.add_argument("--ehr-root", type=str, required=True) + parser.add_argument("--note-root", type=str, default=None) + parser.add_argument("--cxr-root", type=str, default=None) + parser.add_argument("--cxr-variant", type=str, default="sunlab", choices=["default", "sunlab"]) + parser.add_argument("--cache-dir", type=str, default=None) + parser.add_argument("--output-dir", type=str, default="./output/unified_e2e") + + parser.add_argument( + "--task", + type=str, + choices=["labs", "notes_labs", "notes_labs_cxr"], + default="labs", + help=( + "notes_labs: admission-context text (CC/HPI/PMH/MedsOnAdm) + labs. " + "Recommended for multimodal. " + "notes_labs_cxr: notes_labs plus in-window chest X-rays; requires " + "--note-root and --cxr-root." + ), + ) + parser.add_argument( + "--model", + type=str, + choices=["rnn", "transformer", "bottleneck_transformer", + "ehrmamba", "jambaehr"], + default="rnn", + ) + + parser.add_argument("--embedding-dim", type=int, default=64) + parser.add_argument("--hidden-dim", type=int, default=64) + parser.add_argument("--dropout", type=float, default=0.1) + parser.add_argument("--epochs", type=int, default=1) + parser.add_argument("--batch-size", type=int, default=32) + parser.add_argument( + "--lr", + type=float, + default=None, + help="Learning rate. Default is 1e-4 for all models.", + ) + parser.add_argument( + "--adam-eps", + type=float, + default=None, + help=( + "Adam epsilon. Default is model-specific: 1e-8 for non-BT models, " + "1e-6 for bottleneck_transformer." + ), + ) + parser.add_argument("--weight-decay", type=float, default=0.0) + parser.add_argument("--device", type=str, default=None) + parser.add_argument( + "--use-amp", + action="store_true", + help="Enable automatic mixed precision training to reduce GPU memory usage.", + ) + parser.add_argument( + "--amp-dtype", + type=str, + default="bf16", + choices=["bf16", "fp16"], + help="AMP dtype when --use-amp is set. bf16 is more stable (default).", + ) + parser.add_argument("--num-workers", type=int, default=1) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--patience", type=int, default=None) + parser.add_argument( + "--dev", + nargs="?", + type=int, + const=1000, + default=0, + help=( + "Dev mode: limit dataset to N patients for fast iteration. " + "--dev (no value) defaults to 1000 patients. " + "--dev 5000 limits to 5000. Omit for full dataset." + ), + ) + parser.add_argument("--observation-window-hours", type=int, default=24) + parser.add_argument( + "--freeze-encoder", + action="store_true", + default=False, + help=( + "Freeze pretrained BERT text encoder weights and train only the " + "downstream backbone (RNN/Transformer head + projection layer). " + ), + ) + parser.add_argument("--rnn-type", type=str, default="GRU") + parser.add_argument("--rnn-layers", type=int, default=1) + parser.add_argument("--bidirectional", action="store_true") + + parser.add_argument("--heads", type=int, default=4) + parser.add_argument("--num-layers", type=int, default=2) + + parser.add_argument("--bottlenecks-n", type=int, default=4) + parser.add_argument("--fusion-startidx", type=int, default=1) + + parser.add_argument( + "--max-grad-norm", + type=float, + default=None, + help=( + "Gradient clipping max norm. Default is model-specific: None for " + "non-BT models, 0.5 for bottleneck_transformer." + ), + ) + + parser.add_argument( + "--wandb", + action="store_true", + default=False, + help="Log training/eval metrics to Weights & Biases.", + ) + parser.add_argument("--wandb-project", type=str, default="pyhealth-mortality") + parser.add_argument("--wandb-entity", type=str, default=None) + parser.add_argument( + "--wandb-run-name", + type=str, + default=None, + help="Defaults to '{model}_seed{seed}' if unset.", + ) + parser.add_argument( + "--wandb-tags", + type=str, + default=None, + help="Comma-separated wandb tags, e.g. 'labs,rnn'. Defaults to '{task},{model}' if unset.", + ) + + parser.add_argument("--mamba-state-size", type=int, default=16, + help="SSM state size for EHRMamba and JambaEHR blocks.") + parser.add_argument("--mamba-conv-kernel", type=int, default=4, + help="Causal conv kernel size for EHRMamba and JambaEHR blocks.") + parser.add_argument("--jamba-transformer-layers", type=int, default=2, + help="Number of Transformer (attention) layers in JambaEHR.") + parser.add_argument("--jamba-mamba-layers", type=int, default=6, + help="Number of Mamba (SSM) layers in JambaEHR.") + + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + output_csv_path = run(args) + print(f"Saved predictions to: {output_csv_path}") \ No newline at end of file diff --git a/pyhealth/models/bottleneck_transformer.py b/pyhealth/models/bottleneck_transformer.py new file mode 100644 index 000000000..9d20f2549 --- /dev/null +++ b/pyhealth/models/bottleneck_transformer.py @@ -0,0 +1,520 @@ +from typing import Any, Dict, List, Optional, Tuple, Union, cast + +import torch +import torch.nn as nn + +from pyhealth.datasets import SampleDataset +from pyhealth.models import BaseModel +from pyhealth.models.embedding import EmbeddingModel +from pyhealth.models.embedding.unified import UnifiedMultimodalEmbeddingModel + + +class MultimodalBottleneckTransformerEncoder(nn.Module): + """ + Generalized Bottleneck Transformer Encoder for N modalities. + Based on "Attention Bottlenecks for Multimodal Fusion" (Nagrani et al., NeurIPS 2021). + """ + + def __init__( + self, + n_modality: int, + bottlenecks_n: int, + fusion_startidx: int, + n_layers: int, + n_head: int, + d_model: int, + d_ff: int, + dropout: float = 0.1, + ): + super(MultimodalBottleneckTransformerEncoder, self).__init__() + + self.n_modality = n_modality + self.fusion_startidx = fusion_startidx + self.n_layers = n_layers + self.n_fusion_layers = n_layers - fusion_startidx + self.n_prefusion = fusion_startidx + self.d_model = d_model + self.n_bottlenecks = bottlenecks_n + + # Shared Bottleneck Tokens — small init to avoid early gradient explosion + self.bottlenecks = nn.Parameter(torch.randn(1, bottlenecks_n, d_model) * 0.02) + + # Prefusion Stacks: independent layers per modality + self.prefusion_stacks = nn.ModuleList([ + nn.ModuleList([ + nn.TransformerEncoderLayer( + d_model=d_model, + nhead=n_head, + dim_feedforward=d_ff, + dropout=dropout, + batch_first=True + ) for _ in range(n_modality) + ]) for _ in range(self.n_prefusion) + ]) + + # Fusion Stacks: processes [bottleneck_tokens || modality_tokens] + self.fusion_stacks = nn.ModuleList([ + nn.ModuleList([ + nn.TransformerEncoderLayer( + d_model=d_model, + nhead=n_head, + dim_feedforward=d_ff, + dropout=dropout, + batch_first=True + ) for _ in range(n_modality) + ]) for _ in range(self.n_fusion_layers) + ]) + + def forward_prefusion(self, enc_inputs: List[torch.Tensor], masks: List[torch.Tensor]) -> List[torch.Tensor]: + for enc_layers in self.prefusion_stacks: + enc_outputs = [] + for modal_idx, enc_layer in enumerate(enc_layers): + # Apply mask to padding tokens (src_key_padding_mask requires True for ignoring) + # True in mask = invalid/padding + enc_out = enc_layer(enc_inputs[modal_idx], src_key_padding_mask=~masks[modal_idx] if masks[modal_idx] is not None else None) + enc_outputs.append(enc_out) + enc_inputs = enc_outputs + return enc_inputs + + def forward_fusion(self, enc_inputs: List[torch.Tensor], masks: List[torch.Tensor], bottleneck_tokens: torch.Tensor, valid_modalities: List[torch.Tensor]) -> List[torch.Tensor]: + # valid_modalities: [B] list of boolean/float tensors indicating if modality is present + batch_size = enc_inputs[0].size(0) + + for modality_encoders in self.fusion_stacks: + enc_outputs = [] + bottleneck_tokens_modality_sum = torch.zeros_like(bottleneck_tokens) + sum_of_modalities = torch.zeros(batch_size, 1, 1, device=bottleneck_tokens.device) + + for idx, enc_layer in enumerate(modality_encoders): + # Concatenate bottleneck tokens with modality tokens + # bottleneck_tokens: [B, num_bottlenecks, d_model] + # enc_inputs[idx]: [B, seq_len, d_model] + fused_input = torch.cat([bottleneck_tokens, enc_inputs[idx]], dim=1) + + # Padding mask for bottleneck tokens is always False (i.e. valid) + # [B, num_bottlenecks] of False + b_mask = torch.zeros(batch_size, self.n_bottlenecks, dtype=torch.bool, device=fused_input.device) + + # Modality padding mask + m_mask = ~masks[idx] if masks[idx] is not None else torch.zeros(batch_size, enc_inputs[idx].size(1), dtype=torch.bool, device=fused_input.device) + + combined_mask = torch.cat([b_mask, m_mask], dim=1) + + # Pass through the layer + enc_out = enc_layer(fused_input, src_key_padding_mask=combined_mask) + + # The output consists of processed bottleneck tokens and modality tokens + # [B, num_bottlenecks, d_model] and [B, seq_len, d_model] + bottleneck_hidden_tokens = enc_out[:, :self.n_bottlenecks, :] + modality_hidden_tokens = enc_out[:, self.n_bottlenecks:, :] + enc_outputs.append(modality_hidden_tokens) + + # Average updated bottlenecks from valid modalities + modality_is_valid = valid_modalities[idx].view(batch_size, 1, 1) + bottleneck_tokens_modality_sum += bottleneck_hidden_tokens * modality_is_valid + sum_of_modalities += modality_is_valid + + # Prevent division by zero if all modalities are missing + # If sum_of_modalities is 0, just pass zeros (or keep previous bottleneck_tokens) + # sum_of_modalities = torch.clamp(sum_of_modalities, min=1.0) + avg_divisor = sum_of_modalities.clone() + avg_divisor[avg_divisor == 0] = 1.0 + + bottleneck_tokens = bottleneck_tokens_modality_sum / avg_divisor + enc_inputs = enc_outputs + + return enc_inputs + + def forward(self, enc_inputs: List[torch.Tensor], masks: List[torch.Tensor]) -> List[torch.Tensor]: + batch_size = enc_inputs[0].size(0) + + # Determine if a modality is valid for each instance in the batch + # A modality is valid if it has at least one True in its mask + valid_modalities = [] + for mask, inp in zip(masks, enc_inputs): + if mask is not None: + # [B] - True if there's any valid token (1/True) + valid = mask.any(dim=1).float() + else: + valid = torch.ones(batch_size, device=inp.device) + valid_modalities.append(valid) + + bottleneck_tokens = self.bottlenecks.expand(batch_size, -1, -1) + + enc_inputs = self.forward_prefusion(enc_inputs, masks) + enc_inputs = self.forward_fusion(enc_inputs, masks, bottleneck_tokens, valid_modalities) + + return enc_inputs + + +class BottleneckTransformer(BaseModel): + """Bottleneck Transformer model for PyHealth datasets. + + Per-field mode: each feature stream is embedded with :class:`EmbeddingModel`, + prefixed with a learnable per-modality ``[CLS]`` token, processed by + independent prefusion layers, then fused via shared bottleneck tokens. + The per-modality ``[CLS]`` embeddings are averaged and fed to the + classification head. + + Unified mode (``unified_embedding`` supplied): all temporal fields are + jointly embedded and time-sorted by + :class:`~pyhealth.models.embedding.unified.UnifiedMultimodalEmbeddingModel` + into a single interleaved sequence. A single ``[CLS]`` token is prepended + and the encoder runs with ``n_modality=1``, so the bottleneck tokens attend + over the full cross-modal timeline. + + Args: + dataset (SampleDataset): dataset providing processed inputs. + embedding_dim (int): shared embedding dimension. + bottlenecks_n (int): number of shared bottleneck tokens. + fusion_startidx (int): layer index at which bottleneck fusion starts. + Must satisfy ``0 <= fusion_startidx <= num_layers``. + num_layers (int): total transformer layers (prefusion + fusion). + heads (int): number of attention heads per transformer block. + dropout (float): dropout rate inside transformer blocks. + unified_embedding (UnifiedMultimodalEmbeddingModel, optional): when + provided, switches to unified mode. + + Examples: + >>> from pyhealth.datasets import create_sample_dataset, get_dataloader + >>> samples = [ + ... { + ... "patient_id": "patient-0", + ... "visit_id": "visit-0", + ... "conditions": ["A", "B", "C"], + ... "procedures": ["X", "Y"], + ... "label": 1, + ... }, + ... { + ... "patient_id": "patient-1", + ... "visit_id": "visit-0", + ... "conditions": ["D"], + ... "procedures": ["Z", "Y"], + ... "label": 0, + ... }, + ... ] + >>> input_schema = {"conditions": "sequence", "procedures": "sequence"} + >>> output_schema = {"label": "binary"} + >>> dataset = create_sample_dataset( + ... samples, + ... input_schema, + ... output_schema, + ... dataset_name="demo", + ... ) + >>> model = BottleneckTransformer(dataset=dataset, num_layers=3, fusion_startidx=1, bottlenecks_n=4) + >>> loader = get_dataloader(dataset, batch_size=2, shuffle=True) + >>> batch = next(iter(loader)) + >>> output = model(**batch) + >>> sorted(output.keys()) + ['logit', 'loss', 'y_prob', 'y_true'] + """ + + def __init__( + self, + dataset: SampleDataset, + embedding_dim: int = 128, + bottlenecks_n: int = 4, + fusion_startidx: int = 1, + num_layers: int = 3, + heads: int = 4, + dropout: float = 0.5, + unified_embedding: Optional[UnifiedMultimodalEmbeddingModel] = None, + ): + super().__init__(dataset=dataset) + self.embedding_dim = embedding_dim + self.bottlenecks_n = bottlenecks_n + self.fusion_startidx = fusion_startidx + self.num_layers = num_layers + self.heads = heads + self.dropout = dropout + self._use_unified = unified_embedding is not None + + assert ( + len(self.label_keys) == 1 + ), "Only one label key is supported if BottleneckTransformer is initialized" + self.label_key = self.label_keys[0] + self.mode = self.dataset.output_schema[self.label_key] + + assert 0 <= fusion_startidx <= num_layers, ( + f"fusion_startidx must be in [0, num_layers], got {fusion_startidx}" + ) + + output_size = self.get_output_size() + + if self._use_unified: + self.embedding_model = unified_embedding + # Single CLS token for the unified interleaved sequence + self.cls_token = nn.Parameter(torch.randn(1, 1, embedding_dim) * 0.02) + self.encoder = MultimodalBottleneckTransformerEncoder( + n_modality=1, + bottlenecks_n=bottlenecks_n, + fusion_startidx=fusion_startidx, + n_layers=num_layers, + n_head=heads, + d_model=embedding_dim, + d_ff=embedding_dim * 4, + dropout=dropout, + ) + else: + self.embedding_model = EmbeddingModel(dataset, embedding_dim) + self.n_modality = len(self.feature_keys) + # Per-modality CLS tokens + self.cls_token_per_modality = nn.ParameterList([ + nn.Parameter(torch.randn(1, 1, embedding_dim) * 0.02) + for _ in range(self.n_modality) + ]) + self.encoder = MultimodalBottleneckTransformerEncoder( + n_modality=self.n_modality, + bottlenecks_n=bottlenecks_n, + fusion_startidx=fusion_startidx, + n_layers=num_layers, + n_head=heads, + d_model=embedding_dim, + d_ff=embedding_dim * 4, + dropout=dropout, + ) + + # fc input is embedding_dim in both modes (CLS token, not concat) + self.fc = nn.Linear(embedding_dim, output_size) + + def _build_unified_inputs( + self, kwargs: Dict[str, Any] + ) -> Dict[str, Dict[str, torch.Tensor]]: + """Extract value/time/mask tensors for UnifiedMultimodalEmbeddingModel.""" + inputs: Dict[str, Dict[str, torch.Tensor]] = {} + for field_name in self.feature_keys: + feature = kwargs[field_name] + if isinstance(feature, torch.Tensor): + feature = (feature,) + schema = self.dataset.input_processors[field_name].schema() + field_dict: Dict[str, torch.Tensor] = {} + if "value" in schema: + field_dict["value"] = feature[schema.index("value")].to(self.device) + if "time" in schema: + field_dict["time"] = feature[schema.index("time")].to(self.device) + if "mask" in schema: + field_dict["mask"] = feature[schema.index("mask")].to(self.device) + inputs[field_name] = field_dict + return inputs + + def _forward_unified( + self, + **kwargs: Union[torch.Tensor, Tuple[torch.Tensor, ...]], + ) -> Dict[str, torch.Tensor]: + """Forward pass in unified-embedding mode. + + Calls UnifiedMultimodalEmbeddingModel to produce a single time-sorted + sequence, prepends a CLS token, encodes with the bottleneck encoder + (n_modality=1), and classifies from the CLS output. + """ + inputs = self._build_unified_inputs(kwargs) + out = self.embedding_model(inputs) + sequence = out["sequence"] # (B, S, E) + event_mask = out["mask"].bool() # (B, S) + + # Prepend CLS token + batch_size = sequence.size(0) + cls = self.cls_token.expand(batch_size, -1, -1) + sequence = torch.cat([cls, sequence], dim=1) + cls_mask = torch.ones(batch_size, 1, dtype=torch.bool, device=sequence.device) + event_mask = torch.cat([cls_mask, event_mask], dim=1) + + enc_outputs = self.encoder([sequence], [event_mask]) + patient_emb = enc_outputs[0][:, 0, :] # CLS token output + + logits = self.fc(patient_emb) + y_prob = self.prepare_y_prob(logits) + results: Dict[str, torch.Tensor] = {"logit": logits, "y_prob": y_prob} + if self.label_key in kwargs: + y_true = cast(torch.Tensor, kwargs[self.label_key]).to(self.device) + results["loss"] = self.get_loss_function()(logits, y_true) + results["y_true"] = y_true + return results + + @staticmethod + def _pool_embedding(x: torch.Tensor) -> torch.Tensor: + if x.dim() == 4: + x = x.sum(dim=2) + if x.dim() == 2: + x = x.unsqueeze(1) + return x + + @staticmethod + def _mask_from_embeddings(x: torch.Tensor) -> torch.Tensor: + mask = torch.any(torch.abs(x) > 0, dim=-1) + if mask.dim() == 1: + mask = mask.unsqueeze(1) + invalid_rows = ~mask.any(dim=1) + if invalid_rows.any(): + mask[invalid_rows, 0] = True + return mask.bool() + + def forward( + self, + **kwargs: Union[torch.Tensor, Tuple[torch.Tensor, ...]], + ) -> Dict[str, torch.Tensor]: + """Forward propagation. + + In unified mode dispatches to :meth:`_forward_unified`. Otherwise runs + per-field embedding + bottleneck fusion. + + Args: + **kwargs: keyword arguments for the model. + + Returns: + A dictionary with the following keys: + loss: a scalar tensor representing the final loss. + y_prob: a tensor of predicted probabilities. + y_true: a tensor representing the true labels. + logit: the raw logits before activation. + """ + if self._use_unified: + return self._forward_unified(**kwargs) + + enc_inputs = [] + masks = [] + + for idx, feature_key in enumerate(self.feature_keys): + feature = kwargs[feature_key] + + if isinstance(feature, torch.Tensor): + feature = (feature,) + + schema = self.dataset.input_processors[feature_key].schema() + + value = feature[schema.index("value")] if "value" in schema else None + mask = feature[schema.index("mask")] if "mask" in schema else None + + if len(feature) == len(schema) + 1 and mask is None: + mask = feature[-1] + + if value is None: + raise ValueError( + f"Feature '{feature_key}' must contain 'value' " + f"in the schema." + ) + else: + value = value.to(self.device) + + if mask is not None: + mask = mask.to(self.device) + value = self.embedding_model({feature_key: value}, masks={feature_key: mask})[feature_key] + else: + value = self.embedding_model({feature_key: value})[feature_key] + + value = self._pool_embedding(value) + + if mask is not None: + mask = mask.bool() + if mask.dim() == value.dim(): + mask = mask.any(dim=-1) + else: + mask = self._mask_from_embeddings(value) + + # Prepend Modality CLS token + batch_size = value.size(0) + cls_token = self.cls_token_per_modality[idx].expand(batch_size, -1, -1) + value = torch.cat([cls_token, value], dim=1) + + # Update mask for CLS token (always valid) + cls_mask = torch.ones(batch_size, 1, dtype=torch.bool, device=value.device) + mask = torch.cat([cls_mask, mask], dim=1) + + enc_inputs.append(value) + masks.append(mask) + + # Pass through Bottleneck Transformer Encoder + enc_outputs = self.encoder(enc_inputs, masks) + + # Extract CLS tokens + cls_tokens = [out[:, 0, :].unsqueeze(1) for out in enc_outputs] + cls_tokens = torch.cat(cls_tokens, dim=1) # [B, n_modality, embedding_dim] + + # Average CLS tokens across valid modalities + b_size = cls_tokens.size(0) + valid_modalities = [] + for mask in masks: + # We check if there's any valid token aside from the CLS token (index 0) + if mask.size(1) > 1: + valid = mask[:, 1:].any(dim=1).float() + else: + valid = mask[:, 0].float() # fallback + valid_modalities.append(valid.view(b_size, 1, 1)) + + valid_modality_tensor = torch.cat(valid_modalities, dim=1) # [B, n_modality, 1] + + # Apply valid mask + masked_cls = cls_tokens * valid_modality_tensor + sum_valid = valid_modality_tensor.sum(dim=1) # [B, 1] + + # Avoid division by zero + sum_valid[sum_valid == 0] = 1.0 + patient_emb = masked_cls.sum(dim=1) / sum_valid # [B, embedding_dim] + + logits = self.fc(patient_emb) + y_prob = self.prepare_y_prob(logits) + + results = { + "logit": logits, + "y_prob": y_prob, + } + + if self.label_key in kwargs: + y_true = cast(torch.Tensor, kwargs[self.label_key]).to(self.device) + loss = self.get_loss_function()(logits, y_true) + results["loss"] = loss + results["y_true"] = y_true + + return results + +if __name__ == "__main__": + from pyhealth.datasets import create_sample_dataset, get_dataloader + + samples = [ + { + "patient_id": "patient-0", + "visit_id": "visit-0", + "conditions": ["A", "B", "C"], + "procedures": ["X", "Y"], + "label": 1, + }, + { + "patient_id": "patient-1", + "visit_id": "visit-0", + "conditions": ["D"], + "procedures": ["Z", "Y"], + "label": 0, + }, + ] + + input_schema = { + "conditions": "sequence", + "procedures": "sequence", + } + output_schema = {"label": "binary"} + + dataset = create_sample_dataset( + samples=samples, + input_schema=input_schema, + output_schema=output_schema, + dataset_name="test", + ) + + train_loader = get_dataloader(dataset, batch_size=2, shuffle=True) + + model = BottleneckTransformer( + dataset=dataset, + embedding_dim=64, + bottlenecks_n=2, + fusion_startidx=1, + num_layers=3, + heads=2 + ) + + data_batch = next(iter(train_loader)) + + result = model(**data_batch) + print(result) + + result["loss"].backward() + print("Test completed successfully.") \ No newline at end of file diff --git a/pyhealth/models/ehrmamba.py b/pyhealth/models/ehrmamba.py index e24c5595f..afdbc8f35 100644 --- a/pyhealth/models/ehrmamba.py +++ b/pyhealth/models/ehrmamba.py @@ -6,6 +6,7 @@ from pyhealth.datasets import SampleDataset from pyhealth.models import BaseModel from pyhealth.models.embedding import EmbeddingModel +from pyhealth.models.embedding.unified import UnifiedMultimodalEmbeddingModel from pyhealth.models.utils import get_last_visit from pyhealth.processors import ( MultiHotProcessor, @@ -111,6 +112,11 @@ class EHRMamba(BaseModel): Electronic Health Records (arxiv 2405.14567). Uses Mamba (SSM) for linear complexity in sequence length; supports long EHR sequences. + When ``unified_embedding`` is supplied the model switches to **unified + mode**: all temporal fields are jointly embedded and time-sorted by + :class:`UnifiedMultimodalEmbeddingModel`, then processed by a *single* + stack of :class:`MambaBlock` layers rather than one stack per field. + Args: dataset: SampleDataset for token/embedding setup. embedding_dim: Embedding and hidden dimension. Default 128. @@ -118,6 +124,8 @@ class EHRMamba(BaseModel): state_size: SSM state size per channel. Default 16. conv_kernel: Causal conv kernel size in block. Default 4. dropout: Dropout before classification head. Default 0.1. + unified_embedding: Optional pre-built UnifiedMultimodalEmbeddingModel. + When provided, enables unified multi-modal mode. """ def __init__( @@ -128,6 +136,7 @@ def __init__( state_size: int = 16, conv_kernel: int = 4, dropout: float = 0.1, + unified_embedding: Optional[UnifiedMultimodalEmbeddingModel] = None, ): super().__init__(dataset=dataset) self.embedding_dim = embedding_dim @@ -135,19 +144,18 @@ def __init__( self.state_size = state_size self.conv_kernel = conv_kernel self.dropout_rate = dropout + self._use_unified = unified_embedding is not None assert len(self.label_keys) == 1, "EHRMamba supports single label key only" self.label_key = self.label_keys[0] self.mode = self.dataset.output_schema[self.label_key] - self.embedding_model = EmbeddingModel(dataset, embedding_dim) - self.feature_processors = { - k: self.dataset.input_processors[k] for k in self.feature_keys - } + output_size = self.get_output_size() + self.dropout = nn.Dropout(dropout) - self.blocks = nn.ModuleDict() - for feature_key in self.feature_keys: - self.blocks[feature_key] = nn.ModuleList( + if self._use_unified: + self.embedding_model = unified_embedding + self._unified_blocks = nn.ModuleList( [ MambaBlock( d_model=embedding_dim, @@ -157,10 +165,76 @@ def __init__( for _ in range(num_layers) ] ) - - output_size = self.get_output_size() - self.dropout = nn.Dropout(dropout) - self.fc = nn.Linear(len(self.feature_keys) * embedding_dim, output_size) + self.fc = nn.Linear(embedding_dim, output_size) + else: + self.embedding_model = EmbeddingModel(dataset, embedding_dim) + self.feature_processors = { + k: self.dataset.input_processors[k] for k in self.feature_keys + } + self.blocks = nn.ModuleDict() + for feature_key in self.feature_keys: + self.blocks[feature_key] = nn.ModuleList( + [ + MambaBlock( + d_model=embedding_dim, + state_size=state_size, + conv_kernel=conv_kernel, + ) + for _ in range(num_layers) + ] + ) + self.fc = nn.Linear(len(self.feature_keys) * embedding_dim, output_size) + + def _build_unified_inputs( + self, kwargs: Dict[str, Any] + ) -> Dict[str, Dict[str, torch.Tensor]]: + """Build the inputs dict required by UnifiedMultimodalEmbeddingModel.""" + inputs: Dict[str, Dict[str, torch.Tensor]] = {} + for field_name in self.feature_keys: + feature = kwargs[field_name] + if isinstance(feature, torch.Tensor): + feature = (feature,) + schema = self.dataset.input_processors[field_name].schema() + field_dict: Dict[str, torch.Tensor] = {} + if "value" in schema: + field_dict["value"] = feature[schema.index("value")].to(self.device) + if "time" in schema: + field_dict["time"] = feature[schema.index("time")].to(self.device) + if "mask" in schema: + field_dict["mask"] = feature[schema.index("mask")].to(self.device) + inputs[field_name] = field_dict + return inputs + + def _forward_unified(self, **kwargs) -> Dict[str, torch.Tensor]: + """Forward pass in unified-embedding mode. + + Calls UnifiedMultimodalEmbeddingModel to produce a single + temporally-sorted event sequence, then encodes it with one shared + MambaBlock stack and pools to the last valid event. + """ + inputs = self._build_unified_inputs(kwargs) + out = self.embedding_model(inputs) + x = out["sequence"] # (B, S_total, E) + mask = out["mask"].bool() # (B, S_total) + + for blk in self._unified_blocks: + x = blk(x) + + last_h = get_last_visit(x, mask) + logits = self.fc(self.dropout(last_h)) + y_prob = self.prepare_y_prob(logits) + results: Dict[str, torch.Tensor] = { + "loss": torch.tensor(0.0), # placeholder, overwritten below + "y_prob": y_prob, + "logit": logits, + } + if self.label_key in kwargs: + y_true = kwargs[self.label_key].to(self.device) + results["loss"] = self.get_loss_function()(logits, y_true) + results["y_true"] = y_true + if kwargs.get("embed", False): + results["embed"] = last_h + return results @staticmethod def _split_temporal(feature: Any) -> Tuple[Optional[torch.Tensor], Any]: @@ -211,6 +285,9 @@ def _pool_embedding(x: torch.Tensor) -> torch.Tensor: return x def forward(self, **kwargs) -> Dict[str, torch.Tensor]: + if self._use_unified: + return self._forward_unified(**kwargs) + patient_emb = [] embedding_inputs: Dict[str, torch.Tensor] = {} masks: Dict[str, torch.Tensor] = {} @@ -261,4 +338,4 @@ def forward(self, **kwargs) -> Dict[str, torch.Tensor]: batch = next(iter(loader)) out = model(**batch) print("keys:", sorted(out.keys())) - out["loss"].backward() + out["loss"].backward() \ No newline at end of file diff --git a/pyhealth/models/embedding/base.py b/pyhealth/models/embedding/base.py new file mode 100644 index 000000000..5ff1ab2a6 --- /dev/null +++ b/pyhealth/models/embedding/base.py @@ -0,0 +1,52 @@ +from abc import ABC, abstractmethod + + +class BaseEmbeddingModel(ABC): + """Abstract base class for all embedding models in PyHealth. + + All embedding models share a common contract: + + - They expose an ``embedding_dim`` property indicating the output vector dimension. + - Their ``forward`` method accepts processor output tensors and returns + vector embeddings. + + Concrete subclasses: + + - :class:`EmbeddingModel` – generic encoder for codes, sequences, timeseries + - :class:`VisionEmbeddingModel` – patch-based encoder for medical images (Josh) + - :class:`TextEmbeddingModel` – BERT-based encoder for clinical text (Rian) + - :class:`UnifiedMultimodalEmbeddingModel` – temporally-aligned multi-modal encoder + """ + + @property + @abstractmethod + def embedding_dim(self) -> int: + """Output embedding dimension shared across all modalities.""" + ... + + @abstractmethod + def forward(self, *args, **kwargs): + """Transform processor outputs into embeddings. + + Subclass return types + --------------------- + EmbeddingModel + ``Dict[str, Tensor]`` mapping each field name to its embedded + tensor. When ``output_mask=True`` is passed, returns a + ``(Dict[str, Tensor], Dict[str, Tensor])`` tuple of + (embeddings, masks). + + VisionEmbeddingModel + ``Tensor`` of shape ``[batch, embedding_dim]``. + + TextEmbeddingModel + ``(Tensor, BoolTensor)`` of shapes ``([B, T, E], [B, T])`` + when ``return_mask=True`` (default), or a plain + ``Tensor [B, T, E]`` when ``return_mask=False``. + + UnifiedMultimodalEmbeddingModel + ``Dict[str, Tensor]`` with keys ``"sequence"`` ``[B, S, E]``, + ``"mask"`` ``[B, S]``, ``"time"`` ``[B, S]``, and + ``"type_ids"`` ``[B, S]``. + """ + ... diff --git a/pyhealth/models/embedding/unified.py b/pyhealth/models/embedding/unified.py new file mode 100644 index 000000000..30f83a76d --- /dev/null +++ b/pyhealth/models/embedding/unified.py @@ -0,0 +1,595 @@ +"""UnifiedMultimodalEmbeddingModel, temporally aligned multimodal embedding. + +Takes K temporal features ( dict outputs from ``TemporalFeatureProcessor`` +subclasses ), embeds each event with a modality-specific encoder, then +interleaves all events on a shared timeline by sorting on timestamp and adding +sinusoidal time embeddings + learned modality-type embeddings. + +Output shape: ``(B, S_total, E')``, a single sequence of events usable by +any downstream sequence model (Transformer, Mamba, RNN, …). + +IMAGE encoding delegates to :class:`PatchEmbedding` from +:mod:`pyhealth.models.embedding.vision` (Josh's model), pooling patch tokens +to a single per-image vector via global mean pooling. + +TEXT encoding uses a pretrained BERT tokenizer model directly, extracting the +[CLS] token per note, the same BERT-based approach as +:class:`TextEmbeddingModel` (Rian's model). + +Unimodal model reuse via ``field_embeddings``:: + + vision_model = VisionEmbeddingModel(dataset, embedding_dim=128) + text_model = TextEmbeddingModel(embedding_dim=128) + unified = UnifiedMultimodalEmbeddingModel( + processors=dataset.input_processors, + embedding_dim=128, + field_embeddings={ + "chest_xray": vision_model, # reuses trained backbone + "notes": text_model, # reuses BERT + projection + }, + ) + +Quickstart:: + + from pyhealth.models.embedding import UnifiedMultimodalEmbeddingModel + from pyhealth.datasets.collate import collate_temporal + model = UnifiedMultimodalEmbeddingModel(dataset, embedding_dim=128) + # inside forward: + # inputs = {field: {"value": Tensor, "time": Tensor, ...}, ...} + out = model(inputs) + # out["sequence"]: (B, S_total, 128) + # out["mask"]: (B, S_total) , 1 = real event, 0 = padding + # out["time"]: (B, S_total) , hours from first event +""" + +from __future__ import annotations + +import math +import warnings +from typing import Any, Optional + +import torch +import torch.nn as nn + +from ...processors.base_processor import ModalityType, TemporalFeatureProcessor +from .base import BaseEmbeddingModel +from .vision import PatchEmbedding + + +# ── Helpers ─────────────────────────────────────────────────────────────────── + + +class SinusoidalTimeEmbedding(nn.Module): + """Continuous sinusoidal embedding for scalar time values (in hours). + + Identical in spirit to the positional encoding in "Attention is All You + Need" but operating on real-valued timestamps rather than integer positions. + + Args: + dim: Output embedding dimension (must be even). + max_hours: Maximum expected time value in hours. Values are normalised + to ``[0, 2π]`` before the sin/cos projection. Default 720 (30 days). + + Shape: + Input: ``(*, )`` float tensor of times in hours + Output: ``(*, dim)`` + """ + + def __init__(self, dim: int, max_hours: float = 720.0): + super().__init__() + assert dim % 2 == 0, f"dim must be even, got {dim}" + self.dim = dim + self.max_hours = max_hours + half = dim // 2 + freqs = torch.exp( + -math.log(10000.0) * torch.arange(half, dtype=torch.float32) / (half - 1) + ) + self.register_buffer("freqs", freqs) # (dim//2,) + + def forward(self, t: torch.Tensor) -> torch.Tensor: + """:param t: ``(...,)`` float, times in hours.""" + t_norm = t / self.max_hours * 2 * math.pi # (...,) + args = t_norm.unsqueeze(-1) * self.freqs # (..., dim//2) + return torch.cat([args.sin(), args.cos()], dim=-1) # (..., dim) + + +class _MeanPool(nn.Module): + """Pool a sequence of patch embeddings to a single vector via global mean.""" + + def forward(self, x: torch.Tensor) -> torch.Tensor: + # (B, num_patches, E) -> (B, E) + return x.mean(dim=1) + + +# ── Main model ─────────────────────────────────────────────────────────────── + + +class UnifiedMultimodalEmbeddingModel(nn.Module, BaseEmbeddingModel): + """Embed heterogeneous temporal features into a single aligned sequence. + + **All** input processors must be ``TemporalFeatureProcessor`` subclasses. + Non-temporal processors (e.g. ``SequenceProcessor``, ``MultiHotProcessor``) + are rejected with a clear error, use :class:`EmbeddingModel` for those fields. + + Modality routing: + + - **CODE**: ``nn.Embedding`` lookup. + - **TEXT**: Pretrained BERT (same approach as :class:`TextEmbeddingModel`), + CLS token extracted per note. + - **IMAGE**: :class:`PatchEmbedding` (from :class:`VisionEmbeddingModel`) + followed by global mean pooling to produce one vector per image event. + - **NUMERIC / SIGNAL**: ``nn.Linear`` projection. + + Unimodal model reuse: + + Pass pre-built :class:`EmbeddingModel`, :class:`VisionEmbeddingModel`, or + :class:`TextEmbeddingModel` instances via ``field_embeddings`` to reuse + their trained encoder weights instead of building new ones from scratch. + The core encoder module is extracted from each pre-built model: + + - ``EmbeddingModel`` → ``embedding_layers[field_name]`` (``nn.Embedding`` / + ``nn.Linear``) + - ``VisionEmbeddingModel`` → ``embedding_layers[field_name]`` backbone + + global mean pooling + - ``TextEmbeddingModel`` → ``transformer`` (BERT) + ``fc`` (projection) + + Algorithm + --------- + For each temporal field: + + 1. Route ``inputs[field]["value"]`` through a modality-specific encoder → + ``(B, N_i, E')`` per-event embeddings. + 2. Retrieve ``inputs[field]["time"]`` → ``(B, N_i)`` timestamps (hours). + 3. (Optional) Retrieve ``inputs[field]["mask"]`` → ``(B, N_i, L)`` or + ``(B, N_i)`` attention mask; reduced to event-level ``(B, N_i)`` if + token-level. + + Then: + + 4. Concatenate across all fields → ``(B, S_total, E')``. + 5. Sort events along dim=1 by timestamp (ascending). + 6. Add ``SinusoidalTimeEmbedding(time)`` + ``type_embedding(modality_idx)``. + 7. Return ``{"sequence", "time", "mask", "type_ids"}``. + + Args: + processors: ``dict[field_name, TemporalFeatureProcessor]``, the + processors for each temporal field in the dataset. Pass + ``dataset.input_processors`` directly. + embedding_dim: Shared embedding dimension ``E'``. + time_embedding: ``"sinusoidal"`` (default) or ``"learned"``. + max_time_hours: Normalisation constant for the time embedding. + Defaults to 720 h (30 days). + image_size: Image size (H=W) assumed for IMAGE fields when using + PatchEmbedding. Defaults to 224. + image_channels: Number of input channels for IMAGE fields. Defaults to 3. + patch_size: Patch size for IMAGE PatchEmbedding encoder. Defaults to 16. + image_pool: Pooling strategy applied to IMAGE patch tokens to produce + one vector per image event. Only ``"mean"`` (global mean pooling) + is currently implemented. Defaults to ``"mean"``. + field_embeddings: Optional mapping of field names to pre-built unimodal + embedding models. Supported types: + + - :class:`EmbeddingModel` (codes / numeric) — extracts + ``embedding_layers[field_name]``. + - :class:`VisionEmbeddingModel` — extracts the backbone layer and + wraps it with global mean pooling. + - :class:`TextEmbeddingModel` — reuses ``transformer`` and ``fc`` + for BERT-based CLS extraction. + + Fields not present in this dict fall back to the default + internally-built encoders. + + Example:: + + model = UnifiedMultimodalEmbeddingModel( + processors=dataset.input_processors, + embedding_dim=128, + ) + # inputs: {field: {"value": Tensor, "time": Tensor, "mask": Tensor}} + out = model(inputs) + seq = out["sequence"] # (B, S_total, 128) + mask = out["mask"] # (B, S_total) float, 1=valid 0=pad + + # With pre-built unimodal models: + vision = VisionEmbeddingModel(dataset, embedding_dim=128) + model = UnifiedMultimodalEmbeddingModel( + processors=dataset.input_processors, + embedding_dim=128, + field_embeddings={"chest_xray": vision}, + ) + """ + + def __init__( + self, + processors: dict[str, Any], + embedding_dim: int = 128, + time_embedding: str = "sinusoidal", + max_time_hours: float = 720.0, + image_size: int = 224, + image_channels: int = 3, + patch_size: int = 16, + image_pool: str = "mean", + field_embeddings: Optional[dict[str, Any]] = None, + freeze_text_encoder: bool = False, + ): + super().__init__() + if image_pool != "mean": + raise NotImplementedError( + f"Only image_pool='mean' is implemented, got {image_pool!r}." + ) + self._embedding_dim = embedding_dim + self._freeze_text_encoder = freeze_text_encoder + self.image_pool = image_pool + _field_embeddings = field_embeddings or {} + + self.encoders: nn.ModuleDict = nn.ModuleDict() + self.projections: nn.ModuleDict = nn.ModuleDict() + self.modality_types: dict[str, ModalityType] = {} + self._shared_text_field_by_model: dict[str, str] = {} + self._text_canonical: dict[str, str] = {} # field → first field sharing the same tokenizer + + for field_name, processor in processors.items(): + if not isinstance(processor, TemporalFeatureProcessor): + raise TypeError( + f"UnifiedMultimodalEmbeddingModel requires every input processor " + f"to be a TemporalFeatureProcessor subclass, but '{field_name}' " + f"uses {type(processor).__name__}. For non-temporal fields use " + f"EmbeddingModel." + ) + + m = processor.modality() + self.modality_types[field_name] = m + pre_built = _field_embeddings.get(field_name) + + if m == ModalityType.CODE: + self.encoders[field_name] = self._build_code_encoder( + field_name, processor, pre_built, embedding_dim + ) + + elif m == ModalityType.TEXT: + self._build_text_encoder( + field_name, processor, pre_built, embedding_dim, + freeze=freeze_text_encoder, + ) + + elif m == ModalityType.IMAGE: + self.encoders[field_name] = self._build_image_encoder( + field_name, + processor, + pre_built, + embedding_dim, + image_size, + image_channels, + patch_size, + image_pool, + ) + + elif m in (ModalityType.NUMERIC, ModalityType.SIGNAL): + self.encoders[field_name] = self._build_numeric_encoder( + field_name, processor, pre_built, embedding_dim + ) + + else: + raise NotImplementedError( + f"No encoder implemented for modality {m!r} (field '{field_name}')." + ) + + # Shared type embedding, one vector per unique modality in this dataset + unique_modalities = sorted(set(self.modality_types.values())) + self._modality_to_idx: dict[ModalityType, int] = { + mod: i for i, mod in enumerate(unique_modalities) + } + self.type_embedding = nn.Embedding(len(unique_modalities), embedding_dim) + self._warned_nested_code_flatten = False + + # Time embedding + if time_embedding == "sinusoidal": + self.time_embed = SinusoidalTimeEmbedding(embedding_dim, max_time_hours) + else: + raise NotImplementedError( + "Only 'sinusoidal' time embedding is implemented." + ) + + # ── Encoder builders ────────────────────────────────────────────────────── + + def _build_code_encoder( + self, + field_name: str, + processor: TemporalFeatureProcessor, + pre_built: Any, + embedding_dim: int, + ) -> nn.Module: + """Build CODE encoder: nn.Embedding, optionally from a pre-built EmbeddingModel.""" + if ( + pre_built is not None + and hasattr(pre_built, "embedding_layers") + and field_name in pre_built.embedding_layers + ): + layer = pre_built.embedding_layers[field_name] + pre_dim = getattr(pre_built, "embedding_dim", embedding_dim) + if pre_dim != embedding_dim: + return nn.Sequential(layer, nn.Linear(pre_dim, embedding_dim)) + return layer + + vocab_size = processor.value_dim() + return nn.Embedding(vocab_size, embedding_dim, padding_idx=0) + + def _build_text_encoder( + self, + field_name: str, + processor: TemporalFeatureProcessor, + pre_built: Any, + embedding_dim: int, + freeze: bool = False, + ) -> None: + """Build TEXT encoder: BERT + projection, optionally from TextEmbeddingModel.""" + + def _set_projection( + pre_dim: int, proj_source: Optional[nn.Module] = None + ) -> None: + if pre_dim != embedding_dim: + if proj_source is not None: + self.projections[field_name] = nn.Sequential( + proj_source, + nn.Linear(pre_dim, embedding_dim), + ) + else: + self.projections[field_name] = nn.Linear(pre_dim, embedding_dim) + elif proj_source is not None: + self.projections[field_name] = proj_source + + if ( + pre_built is not None + and hasattr(pre_built, "transformer") + and hasattr(pre_built, "fc") + ): + self.encoders[field_name] = pre_built.transformer + if freeze: + for p in pre_built.transformer.parameters(): + p.requires_grad = False + pre_dim = getattr(pre_built, "embedding_dim", embedding_dim) + _set_projection(pre_dim, pre_built.fc) + return + + if processor.is_token(): + from transformers import AutoModel + + bert = AutoModel.from_pretrained(processor.tokenizer_model) + if freeze: + for p in bert.parameters(): + p.requires_grad = False + self.encoders[field_name] = bert + hidden = bert.config.hidden_size + if hidden != embedding_dim: + self.projections[field_name] = nn.Linear(hidden, embedding_dim) + else: + raise ValueError( + f"TEXT processor '{field_name}' must either supply a pre-built " + f"TextEmbeddingModel via field_embeddings or use a tokenizer " + f"(set tokenizer_model=...) to be used with " + f"UnifiedMultimodalEmbeddingModel." + ) + + def _build_image_encoder( + self, + field_name: str, + processor: TemporalFeatureProcessor, + pre_built: Any, + embedding_dim: int, + image_size: int, + image_channels: int, + patch_size: int, + image_pool: str, + ) -> nn.Module: + """Build IMAGE encoder: backbone + pool, optionally from VisionEmbeddingModel.""" + pool_layers: dict[str, nn.Module] = {"mean": _MeanPool()} + pool_layer = pool_layers[image_pool] + + if ( + pre_built is not None + and hasattr(pre_built, "embedding_layers") + and field_name in pre_built.embedding_layers + ): + backbone = pre_built.embedding_layers[field_name] + pre_dim = getattr(pre_built, "embedding_dim", embedding_dim) + if pre_dim != embedding_dim: + return nn.Sequential( + backbone, pool_layer, nn.Linear(pre_dim, embedding_dim) + ) + return nn.Sequential(backbone, pool_layer) + + _image_size = getattr(processor, "image_size", image_size) + _in_channels = getattr(processor, "in_channels", image_channels) + return nn.Sequential( + PatchEmbedding(_image_size, patch_size, _in_channels, embedding_dim), + pool_layer, + ) + + def _build_numeric_encoder( + self, + field_name: str, + processor: TemporalFeatureProcessor, + pre_built: Any, + embedding_dim: int, + ) -> nn.Module: + """Build NUMERIC/SIGNAL encoder: nn.Linear, optionally from EmbeddingModel.""" + if ( + pre_built is not None + and hasattr(pre_built, "embedding_layers") + and field_name in pre_built.embedding_layers + ): + layer = pre_built.embedding_layers[field_name] + pre_dim = getattr(pre_built, "embedding_dim", embedding_dim) + if pre_dim != embedding_dim: + return nn.Sequential(layer, nn.Linear(pre_dim, embedding_dim)) + return layer + + in_features = processor.value_dim() + return nn.Linear(in_features, embedding_dim) + + @property + def embedding_dim(self) -> int: + return self._embedding_dim + + # ── Forward ─────────────────────────────────────────────────────────────── + + def forward( + self, + inputs: dict[str, dict[str, torch.Tensor]], + ) -> dict[str, torch.Tensor]: + """Encode and temporally align all temporal features. + + Args: + inputs: ``{field_name: {"value": Tensor, "time": Tensor, + "mask": Tensor (optional)}}`` + , one dict per temporal feature, exactly as produced by + ``collate_temporal``. + + Returns: + A dict with keys: + + * ``"sequence"``, ``(B, S_total, E')`` temporally-sorted events + (content + time + type embeddings) + * ``"time"`` , ``(B, S_total)`` timestamps (hours) + * ``"mask"`` , ``(B, S_total)`` 1=real event, 0=padding + * ``"type_ids"``, ``(B, S_total)`` modality index per event + * ``"token_emb"``, ``(B, S_total, E')`` content-only event embedding + (before time/type are added); the target for masked modeling. + """ + all_embeddings: list[torch.Tensor] = [] + all_times: list[torch.Tensor] = [] + all_masks: list[torch.Tensor] = [] + all_types: list[torch.Tensor] = [] + + for field_name, feat_dict in inputs.items(): + value = feat_dict["value"] # (B, N_i, ...) or (B, S, F) + time = feat_dict["time"] # (B, N_i) + mask = feat_dict.get("mask") + + if time is None: + # Fallback: treat every event as occurring at t=0 + time = torch.zeros(value.shape[:2], device=value.device) + + modality = self.modality_types[field_name] + encoder_key = self._text_canonical.get(field_name, field_name) + encoder = self.encoders[encoder_key] + + # ── Encode ──────────────────────────────────────────────────── + if modality == ModalityType.CODE: + # CODE values may be either: + # - flat indices: (B, S) + # - nested indices: (B, S, C) where C is codes-per-event + # For nested indices, flatten to (B, S*C, E') so code-level + # detail is preserved, and expand time/mask to match. + if value.dim() == 2: + emb = encoder(value) # (B, S, E') + elif value.dim() == 3: + bsz, seq_len, per_event_codes = value.shape + token_emb = encoder(value.long()) # (B, S, C, E') + emb = token_emb.reshape(bsz, seq_len * per_event_codes, -1) + + if not self._warned_nested_code_flatten: + warnings.warn( + ( + "UnifiedMultimodalEmbeddingModel detected " + f"nested CODE input for '{field_name}' with " + f"shape={tuple(value.shape)}. Flattening to " + f"(B, S*C, E) and repeating time along C." + ), + stacklevel=2, + ) + self._warned_nested_code_flatten = True + + if time is not None: + time = ( + time.unsqueeze(-1) + .expand(-1, -1, per_event_codes) + .reshape(bsz, seq_len * per_event_codes) + ) + + if mask is not None: + if mask.dim() == 2: + mask = ( + mask.unsqueeze(-1) + .expand(-1, -1, per_event_codes) + .reshape(bsz, seq_len * per_event_codes) + ) + elif mask.dim() == 3: + mask = mask.reshape(bsz, seq_len * per_event_codes) + else: + raise ValueError( + f"Unsupported CODE value rank for '{field_name}': " + f"shape={tuple(value.shape)}" + ) + + elif modality == ModalityType.TEXT: + b, n, l = value.shape + flat_ids = value.view(b * n, l) + flat_mask = mask.view(b * n, l) if mask is not None else None + out = encoder(input_ids=flat_ids, attention_mask=flat_mask) + cls_emb = out.last_hidden_state[:, 0, :] # (B*N, H) + if field_name in self.projections: + cls_emb = self.projections[field_name](cls_emb) + emb = cls_emb.view(b, n, -1) # (B, N, E') + + elif modality == ModalityType.IMAGE: + # encoder = Sequential(PatchEmbedding, _MeanPool) → (B*N, E') + b, n, c, h, w = value.shape + flat_imgs = value.view(b * n, c, h, w) + img_emb = encoder(flat_imgs) # (B*N, E') + emb = img_emb.view(b, n, -1) # (B, N, E') + + else: # NUMERIC / SIGNAL + emb = encoder(value) # (B, T, E') + + # ── Build event-level validity mask ─────────────────────────── + if mask is None: + event_mask = torch.ones(emb.shape[:2], device=emb.device) + else: + if mask.dim() > time.dim(): + # token-level (B, N, L) → event-level (B, N) + event_mask = (mask.sum(dim=-1) > 0).float() + else: + event_mask = mask.float() + + # ── Modality type indices ───────────────────────────────────── + type_idx = self._modality_to_idx[modality] + type_ids = torch.full( + emb.shape[:2], type_idx, dtype=torch.long, device=emb.device + ) + + all_embeddings.append(emb) + all_times.append(time) + all_masks.append(event_mask) + all_types.append(type_ids) + + # ── Concatenate across all fields ───────────────────────────────── + cat_emb = torch.cat(all_embeddings, dim=1) # (B, S_total, E') + cat_time = torch.cat(all_times, dim=1) # (B, S_total) + cat_mask = torch.cat(all_masks, dim=1) # (B, S_total) + cat_types = torch.cat(all_types, dim=1) # (B, S_total) + + # ── Sort by time ────────────────────────────────────────────────── + sort_idx = cat_time.argsort(dim=1) + cat_emb = cat_emb.gather(1, sort_idx.unsqueeze(-1).expand_as(cat_emb)) + cat_time = cat_time.gather(1, sort_idx) + cat_mask = cat_mask.gather(1, sort_idx) + cat_types = cat_types.gather(1, sort_idx) + + # ── Add time + type embeddings ──────────────────────────────────── + time_emb = self.time_embed(cat_time) # (B, S_total, E') + type_emb = self.type_embedding(cat_types) # (B, S_total, E') + final = cat_emb + time_emb + type_emb # (B, S_total, E') + + return { + "sequence": final, # (B, S_total, E') + "time": cat_time, # (B, S_total) + "mask": cat_mask, # (B, S_total) + "type_ids": cat_types, # (B, S_total) + # Per-event content embedding BEFORE time/type are added (same sort + # order as ``sequence``). Masked-modeling pretrainers should + # reconstruct THIS rather than ``sequence``: the time/type + # components are largely recoverable from event position, so + # including them in the target dilutes the content signal. + "token_emb": cat_emb, # (B, S_total, E') + } \ No newline at end of file diff --git a/pyhealth/models/embedding/vanilla.py b/pyhealth/models/embedding/vanilla.py new file mode 100644 index 000000000..9b684d0c0 --- /dev/null +++ b/pyhealth/models/embedding/vanilla.py @@ -0,0 +1,366 @@ +from __future__ import annotations + +from typing import Dict, Any, Optional, Union +import os + +import torch +import torch.nn as nn + +from ...datasets import SampleDataset +from ...processors import ( + MultiHotProcessor, + NestedFloatsProcessor, + NestedSequenceProcessor, + SequenceProcessor, + StageNetProcessor, + StageNetTensorProcessor, + TensorProcessor, + TimeseriesProcessor, + DeepNestedSequenceProcessor, + DeepNestedFloatsProcessor, +) +from ..base_model import BaseModel +from .base import BaseEmbeddingModel + + +def _iter_text_vectors( + path: str, + embedding_dim: int, + wanted_tokens: set[str], + encoding: str = "utf-8", +) -> Dict[str, torch.Tensor]: + """Loads word vectors from a text file (e.g., GloVe) for a subset of tokens. + + Expected format: one token per line followed by embedding_dim floats. + + This function reads the file line-by-line and only retains vectors for + tokens present in `wanted_tokens`. + """ + + if not os.path.exists(path): + raise FileNotFoundError(f"pretrained embedding file not found: {path}") + + vectors: Dict[str, torch.Tensor] = {} + with open(path, "r", encoding=encoding) as f: + for line in f: + line = line.strip() + if not line: + continue + parts = line.split() + # token + embedding_dim values + if len(parts) < embedding_dim + 1: + continue + token = parts[0] + if token not in wanted_tokens: + continue + try: + vec = torch.tensor( + [float(x) for x in parts[1 : embedding_dim + 1]], + dtype=torch.float, + ) + except ValueError: + continue + vectors[token] = vec + return vectors + + +def init_embedding_with_pretrained( + embedding: nn.Embedding, + code_vocab: Dict[Any, int], + pretrained_path: str, + embedding_dim: int, + pad_token: str = "", + unk_token: str = "", + normalize: bool = False, + freeze: bool = False, +) -> int: + """Initializes an nn.Embedding from a pretrained text-vector file. + + Tokens not found in the pretrained file are left as the module's existing + random initialization. + + Returns: + int: number of tokens successfully initialized from the file. + """ + + # Build wanted token set (stringified) + vocab_tokens = {str(t) for t in code_vocab.keys()} + vectors = _iter_text_vectors(pretrained_path, embedding_dim, vocab_tokens) + + loaded = 0 + with torch.no_grad(): + for tok, idx in code_vocab.items(): + tok_s = str(tok) + if tok_s in vectors: + vec = vectors[tok_s] + if normalize: + vec = vec / (vec.norm(p=2) + 1e-12) + embedding.weight[idx].copy_(vec) + loaded += 1 + + # Ensure pad row is zero + if pad_token in code_vocab: + embedding.weight[code_vocab[pad_token]].zero_() + # If embedding has a padding_idx, keep it consistent + if embedding.padding_idx is not None: + embedding.weight[embedding.padding_idx].zero_() + + if freeze: + embedding.weight.requires_grad_(False) + + return loaded + + +class EmbeddingModel(BaseModel): + """ + EmbeddingModel is responsible for creating embedding layers for different types of input data. + + This model automatically creates appropriate embedding transformations based on the processor type: + + - SequenceProcessor: nn.Embedding + Input: (batch, seq_len) + Output: (batch, seq_len, embedding_dim) + + - NestedSequenceProcessor: nn.Embedding + Input: (batch, num_visits, max_codes_per_visit) + Output: (batch, num_visits, max_codes_per_visit, embedding_dim) + + - DeepNestedSequenceProcessor: nn.Embedding + Input: (batch, num_groups, num_visits, max_codes_per_visit) + Output: (batch, num_groups, num_visits, max_codes_per_visit, embedding_dim) + + - TimeseriesProcessor / NestedFloatsProcessor / DeepNestedFloatsProcessor / StageNetTensorProcessor: + nn.Linear over the last dimension + Input: (..., size) + Output: (..., embedding_dim) + + - TensorProcessor: nn.Linear (size inferred from first sample) + + - MultiHotProcessor: nn.Linear over multi-hot vector + """ + + def __init__( + self, + dataset: SampleDataset, + embedding_dim: int = 128, + pretrained_emb_path: Optional[Union[str, Dict[str, str]]] = None, + freeze_pretrained: bool = False, + normalize_pretrained: bool = False, + ): + super().__init__(dataset) + # BaseEmbeddingModel declares `embedding_dim` as an abstract property, + # so we can't set self.embedding_dim directly (no setter). Use a + # private backing attribute and expose it through the property below. + self._embedding_dim = embedding_dim + self.embedding_layers = nn.ModuleDict() + + for field_name, processor in self.dataset.input_processors.items(): + # Deep categorical: use special module that collapses last dim to embedding_dim + + # Regular categorical sequences -> nn.Embedding (adds embedding dim) + if isinstance( + processor, + ( + SequenceProcessor, + StageNetProcessor, + NestedSequenceProcessor, + DeepNestedSequenceProcessor, + ), + ): + vocab_size = len(processor.code_vocab) + + # For NestedSequenceProcessor and DeepNestedSequenceProcessor, don't use padding_idx + # because empty visits/groups need non-zero embeddings. + if isinstance( + processor, (NestedSequenceProcessor, DeepNestedSequenceProcessor) + ): + self.embedding_layers[field_name] = nn.Embedding( + num_embeddings=vocab_size, + embedding_dim=embedding_dim, + padding_idx=None, + ) + else: + self.embedding_layers[field_name] = nn.Embedding( + num_embeddings=vocab_size, + embedding_dim=embedding_dim, + padding_idx=0, + ) + + # Optional pretrained initialization (e.g., GloVe). + if pretrained_emb_path is not None: + if isinstance(pretrained_emb_path, str): + path = pretrained_emb_path + else: + path = pretrained_emb_path.get(field_name) + if path: + init_embedding_with_pretrained( + self.embedding_layers[field_name], + processor.code_vocab, + path, + embedding_dim=embedding_dim, + normalize=normalize_pretrained, + freeze=freeze_pretrained, + ) + + # Numeric features (including deep nested floats) -> nn.Linear over last dim + elif isinstance( + processor, + ( + TimeseriesProcessor, + StageNetTensorProcessor, + NestedFloatsProcessor, + DeepNestedFloatsProcessor, + ), + ): + # Assuming processor.size() returns the last-dim size + in_features = processor.size() + self.embedding_layers[field_name] = nn.Linear( + in_features=in_features, out_features=embedding_dim + ) + + elif isinstance(processor, TensorProcessor): + # Infer size from first sample + sample_tensor = None + for sample in dataset: + if field_name in sample: + sample_tensor = processor.process(sample[field_name]) + break + if sample_tensor is not None: + input_size = ( + sample_tensor.shape[-1] if sample_tensor.dim() > 0 else 1 + ) + self.embedding_layers[field_name] = nn.Linear( + in_features=input_size, out_features=embedding_dim + ) + + elif isinstance(processor, MultiHotProcessor): + num_categories = processor.size() + self.embedding_layers[field_name] = nn.Linear( + in_features=num_categories, out_features=embedding_dim + ) + + # Smart Processor (Token-based) -> Transformers + elif hasattr(processor, "is_token") and processor.is_token(): + try: + from transformers import AutoModel + except ImportError: + raise ImportError( + "Please install `transformers` to use token-based processors." + ) + + # Load the model + self.embedding_layers[field_name] = AutoModel.from_pretrained( + processor.tokenizer_model + ) + + # Check if we need projection + if ( + self.embedding_layers[field_name].config.hidden_size + != self.embedding_dim + ): + self.embedding_layers[f"{field_name}_proj"] = nn.Linear( + self.embedding_layers[field_name].config.hidden_size, + self.embedding_dim, + ) + + else: + print( + "Warning: No embedding created for field due to lack of compatible processor:", + field_name, + ) + + def forward( + self, + inputs: Dict[str, torch.Tensor], + masks: Dict[str, torch.Tensor] = None, + output_mask: bool = False, + ) -> ( + Dict[str, torch.Tensor] + | tuple[Dict[str, torch.Tensor], Dict[str, torch.Tensor]] + ): + + embedded: Dict[str, torch.Tensor] = {} + out_masks: Dict[str, torch.Tensor] = {} if output_mask else None + + for field_name, tensor in inputs.items(): + processor = self.dataset.input_processors.get(field_name, None) + + if field_name not in self.embedding_layers: + # No embedding layer -> passthrough + embedded[field_name] = tensor + continue + + # Check if it's a transformer model + layer = self.embedding_layers[field_name] + + # Check for transformers.PreTrainedModel (but without importing if possible, use class name check) + # or check if it has 'config' attribute + if hasattr(layer, "config") and hasattr(layer, "forward"): + # It's likely a transformer + tensor = tensor.to(self.device).long() # Ensure LongTensor for IDs + + mask = None + if masks is not None and field_name in masks: + mask = masks[field_name].to(self.device) + + # Handle 3D input (Batch, Num_Notes, Seq_Len) + is_3d = inputs[field_name].dim() == 3 + + if is_3d: + b, n, l = inputs[field_name].shape + tensor = tensor.view(b * n, l) + if mask is not None: + mask = mask.view(b * n, l) + + # Forward pass through transformer + output = layer(input_ids=tensor, attention_mask=mask) + x = output.last_hidden_state # (Batch, Seq, Hidden) + + if is_3d: + # If we had 3D input, we MUST pool the sequence dim (L) to get one vector per note + # Resulting shape: (B, N, H) + + # Pool L dim -> (B*N, H) using CLS token (index 0) + x = x[:, 0, :] + + # Check projections + if f"{field_name}_proj" in self.embedding_layers: + x = self.embedding_layers[f"{field_name}_proj"](x) + + x = x.view(b, n, -1) + + else: + # 2D input (Batch, Seq) -> (Batch, Seq, Hidden) + # No pooling, treating as sequence of tokens (word embeddings) + if f"{field_name}_proj" in self.embedding_layers: + x = self.embedding_layers[f"{field_name}_proj"](x) + + embedded[field_name] = x + + else: + # Standard layers + tensor = tensor.to(self.device) + embedded[field_name] = layer(tensor) + + if output_mask: + # Generate a mask for this field + if masks is not None and field_name in masks: + out_masks[field_name] = masks[field_name].to(self.device) + elif hasattr(processor, "code_vocab"): + pad_idx = processor.code_vocab.get("", 0) + out_masks[field_name] = tensor != pad_idx + else: + # Default mask generation (e.g. for simple linear layers where 0 might be padding?) + # Be careful changing this behavior. + # Previous code: + # masks[field_name] = (tensor != pad_idx) -> where pad_idx was 0 default + pad_idx = 0 + out_masks[field_name] = tensor != pad_idx + + if output_mask: + return embedded, out_masks + else: + return embedded + + def __repr__(self) -> str: + return f"EmbeddingModel(embedding_layers={self.embedding_layers})" \ No newline at end of file diff --git a/pyhealth/models/embedding/vision.py b/pyhealth/models/embedding/vision.py new file mode 100644 index 000000000..57eedda22 --- /dev/null +++ b/pyhealth/models/embedding/vision.py @@ -0,0 +1,384 @@ +# Author: Josh Steier +# Description: Vision embedding model for medical imaging + +from typing import Any, Dict, Literal, Optional, Tuple, Union + +import torch +import torch.nn as nn +import shutil +from ...datasets import SampleDataset +from ..base_model import BaseModel +from ...processors import ImageProcessor +from .base import BaseEmbeddingModel + + +class Permute(nn.Module): + """Utility module to permute tensor dimensions in nn.Sequential. + + Args: + dims: Variable number of integers specifying the desired ordering + of dimensions. + + Example: + >>> permute = Permute(0, 2, 1) + >>> x = torch.randn(32, 256, 49) # (B, E, spatial) + >>> out = permute(x) # (B, spatial, E) + """ + + def __init__(self, *dims: int) -> None: + super().__init__() + self.dims = dims + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return x.permute(*self.dims) + + +class PatchEmbedding(nn.Module): + """Convert images to patch embeddings using ViT-style projection. + + Splits an image into non-overlapping patches and projects each patch + to an embedding vector using a convolutional layer. + + Args: + image_size: Input image size (assumes square images). + patch_size: Size of each square patch. + in_channels: Number of input channels. + embedding_dim: Output embedding dimension for each patch. + + Example: + >>> patch_embed = PatchEmbedding(224, 16, 3, 256) + >>> x = torch.randn(4, 3, 224, 224) + >>> patches = patch_embed(x) # (4, 196, 256) + """ + + def __init__( + self, + image_size: int = 224, + patch_size: int = 16, + in_channels: int = 3, + embedding_dim: int = 128, + ) -> None: + super().__init__() + if image_size % patch_size != 0: + raise ValueError( + f"image_size ({image_size}) must be divisible by " + f"patch_size ({patch_size})" + ) + self.patch_size = patch_size + self.num_patches = (image_size // patch_size) ** 2 + self.proj = nn.Conv2d( + in_channels, embedding_dim, kernel_size=patch_size, stride=patch_size + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + # (B, C, H, W) -> (B, E, H/P, W/P) -> (B, num_patches, E) + x = self.proj(x) + x = x.flatten(2).transpose(1, 2) + return x + + +class VisionEmbeddingModel(BaseModel, BaseEmbeddingModel): + """Vision embedding model for medical image inputs. + + Converts medical images to sequences of patch embeddings suitable for + attention-based fusion with other modalities (EHR, text). + + Supports multiple backbone types: + - "patch": ViT-style patch projection (lightweight) + - "cnn": Small CNN encoder (good inductive bias) + - "resnet18"/"resnet50": Pretrained backbones + + Output shape: (batch, num_patches, embedding_dim) + + Args: + dataset: SampleDataset with ImageProcessor fields. + embedding_dim: Output embedding dimension. Default 128. + patch_size: Patch size for "patch" backbone. Default 16. + backbone: One of "patch", "cnn", "resnet18", "resnet50". + pretrained: Use ImageNet weights for ResNet. Default True. + freeze_backbone: Freeze pretrained weights. Default False. + dropout: Dropout rate. Default 0.0. + use_cls_token: Prepend learnable [CLS] token. Default False. + + Example: + >>> from pyhealth.datasets import create_sample_dataset + >>> model = VisionEmbeddingModel(dataset, embedding_dim=256) + >>> embeddings = model({"chest_xray": images}) + """ + + def __init__( + self, + dataset: SampleDataset, + embedding_dim: int = 128, + patch_size: int = 16, + backbone: Literal["patch", "cnn", "resnet18", "resnet50"] = "patch", + pretrained: bool = True, + freeze_backbone: bool = False, + dropout: float = 0.0, + use_cls_token: bool = False, + pool: Optional[Literal["mean"]] = None, + ) -> None: + super().__init__(dataset) + + self._embedding_dim = embedding_dim + self.patch_size = patch_size + self.pool = pool + self.backbone_type = backbone + self.use_cls_token = use_cls_token + + self.embedding_layers = nn.ModuleDict() + self.pos_embeddings = nn.ParameterDict() + self.cls_tokens = nn.ParameterDict() if use_cls_token else None + self.dropout = nn.Dropout(dropout) if dropout > 0 else nn.Identity() + + self._field_info: Dict[str, Dict[str, Any]] = {} + + for field_name, processor in self.dataset.input_processors.items(): + if not isinstance(processor, ImageProcessor): + continue + + image_size = processor.image_size + in_channels = self._infer_channels(processor) + + num_patches = self._build_embedding_layer( + field_name, image_size, in_channels, backbone, pretrained, freeze_backbone + ) + + num_positions = num_patches + 1 if use_cls_token else num_patches + self.pos_embeddings[field_name] = nn.Parameter( + torch.randn(1, num_positions, embedding_dim) * 0.02 + ) + + if use_cls_token: + self.cls_tokens[field_name] = nn.Parameter( + torch.randn(1, 1, embedding_dim) * 0.02 + ) + + self._field_info[field_name] = { + "num_patches": num_patches, + "image_size": image_size, + "in_channels": in_channels, + } + + @property + def embedding_dim(self) -> int: + return self._embedding_dim + + def _infer_channels(self, processor: ImageProcessor) -> int: + """Infer number of input channels from processor mode.""" + mode = getattr(processor, "mode", None) + if mode == "L": + return 1 + elif mode == "RGBA": + return 4 + return 3 + + def _build_embedding_layer( + self, + field_name: str, + image_size: int, + in_channels: int, + backbone: str, + pretrained: bool, + freeze_backbone: bool, + ) -> int: + """Build embedding layer and return number of output patches.""" + if backbone == "patch": + num_patches = (image_size // self.patch_size) ** 2 + self.embedding_layers[field_name] = PatchEmbedding( + image_size, self.patch_size, in_channels, self._embedding_dim + ) + + elif backbone == "cnn": + num_patches = 7 * 7 + self.embedding_layers[field_name] = nn.Sequential( + nn.Conv2d(in_channels, 64, 7, stride=2, padding=3), + nn.BatchNorm2d(64), + nn.ReLU(inplace=True), + nn.Conv2d(64, 128, 3, stride=2, padding=1), + nn.BatchNorm2d(128), + nn.ReLU(inplace=True), + nn.Conv2d(128, self._embedding_dim, 3, stride=2, padding=1), + nn.BatchNorm2d(self._embedding_dim), + nn.ReLU(inplace=True), + nn.AdaptiveAvgPool2d((7, 7)), + nn.Flatten(2), + Permute(0, 2, 1), + ) + + elif backbone in ("resnet18", "resnet50"): + num_patches = 7 * 7 + self.embedding_layers[field_name] = self._build_resnet_backbone( + backbone, in_channels, pretrained, freeze_backbone + ) + + else: + raise ValueError(f"Unknown backbone: {backbone}") + + return num_patches + + def _build_resnet_backbone( + self, backbone: str, in_channels: int, pretrained: bool, freeze: bool + ) -> nn.Module: + """Build pretrained ResNet backbone with spatial output.""" + try: + import torchvision.models as models + except ImportError as e: + raise ImportError("torchvision required for ResNet backbones") from e + + if backbone == "resnet18": + weights = models.ResNet18_Weights.DEFAULT if pretrained else None + resnet = models.resnet18(weights=weights) + feature_dim = 512 + else: + weights = models.ResNet50_Weights.DEFAULT if pretrained else None + resnet = models.resnet50(weights=weights) + feature_dim = 2048 + + if in_channels != 3: + resnet.conv1 = nn.Conv2d( + in_channels, 64, kernel_size=7, stride=2, padding=3, bias=False + ) + + layers = list(resnet.children())[:-2] + backbone_net = nn.Sequential(*layers) + + if freeze: + for param in backbone_net.parameters(): + param.requires_grad = False + + return nn.Sequential( + backbone_net, + nn.Conv2d(feature_dim, self._embedding_dim, kernel_size=1), + nn.Flatten(2), + Permute(0, 2, 1), + ) + + def forward( + self, + inputs: Dict[str, torch.Tensor], + output_mask: bool = False, + ) -> Union[Dict[str, torch.Tensor], Tuple[Dict[str, torch.Tensor], Dict[str, torch.Tensor]]]: + """Forward pass. + + Args: + inputs: Dict mapping field names to image tensors (B, C, H, W). + output_mask: If True, also return attention masks. + + Returns: + Dict of embeddings (B, num_patches, E), optionally with masks. + """ + embedded: Dict[str, torch.Tensor] = {} + masks: Dict[str, torch.Tensor] = {} if output_mask else None + + for field_name, tensor in inputs.items(): + if field_name not in self.embedding_layers: + embedded[field_name] = tensor + continue + + tensor = tensor.to(self.device) + batch_size = tensor.size(0) + + x = self.embedding_layers[field_name](tensor) + + if self.use_cls_token: + cls = self.cls_tokens[field_name].expand(batch_size, -1, -1) + x = torch.cat([cls, x], dim=1) + + x = x + self.pos_embeddings[field_name] + x = self.dropout(x) + + if self.pool == "mean": + x = x.mean(dim=1, keepdim=True) + + embedded[field_name] = x + + if output_mask: + masks[field_name] = torch.ones( + batch_size, x.size(1), dtype=torch.bool, device=x.device + ) + + return (embedded, masks) if output_mask else embedded + + def get_output_info(self, field_name: str) -> Dict[str, Any]: + """Get metadata about embedding output for a field.""" + if field_name not in self._field_info: + raise KeyError(f"Field '{field_name}' not found") + + info = self._field_info[field_name].copy() + info["embedding_dim"] = self._embedding_dim + info["has_cls_token"] = self.use_cls_token + if self.pool == "mean": + info["num_tokens"] = 1 + else: + info["num_tokens"] = info["num_patches"] + (1 if self.use_cls_token else 0) + return info + + def __repr__(self) -> str: + fields = list(self.embedding_layers.keys()) + return ( + f"VisionEmbeddingModel(backbone={self.backbone_type!r}, " + f"embedding_dim={self._embedding_dim}, fields={fields})" + ) + + +if __name__ == "__main__": + from pyhealth.datasets import create_sample_dataset + from pyhealth.datasets.utils import get_dataloader + import tempfile + import os + from PIL import Image + import numpy as np + + # Create synthetic images + temp_dir = tempfile.mkdtemp() + samples = [] + for i in range(10): + img_path = os.path.join(temp_dir, f"img_{i}.png") + img = Image.fromarray(np.random.randint(0, 255, (224, 224), dtype=np.uint8), mode="L") + img.save(img_path) + samples.append({ + "patient_id": f"p{i}", + "visit_id": f"v{i}", + "chest_xray": img_path, + "label": i % 2, + }) + + dataset = create_sample_dataset( + samples=samples, + input_schema={"chest_xray": ("image", {"image_size": 224, "mode": "L"})}, + output_schema={"label": "binary"}, + dataset_name="test_vision", + ) + + model = VisionEmbeddingModel( + dataset=dataset, + embedding_dim=128, + backbone="cnn", + use_cls_token=True, + ) + + model_pooled = VisionEmbeddingModel( + dataset=dataset, + embedding_dim=128, + backbone="cnn", + pool="mean", + ) + + + + loader = get_dataloader(dataset, batch_size=4, shuffle=False) + batch = next(iter(loader)) + + embeddings_pooled = model_pooled({"chest_xray": batch["chest_xray"]}) + print(f"Pooled output shape: {embeddings_pooled['chest_xray'].shape}") # expect (4, 1, 128) + print(f"Pooled output info: {model_pooled.get_output_info('chest_xray')}") # expect num_tokens=1 + + + embeddings = model({"chest_xray": batch["chest_xray"]}) + print(f"Input shape: {batch['chest_xray'].shape}") + print(f"Output shape: {embeddings['chest_xray'].shape}") + print(f"Output info: {model.get_output_info('chest_xray')}") + + # Cleanup + shutil.rmtree(temp_dir) diff --git a/pyhealth/models/jamba_ehr.py b/pyhealth/models/jamba_ehr.py index fea902bd1..37d738f3f 100644 --- a/pyhealth/models/jamba_ehr.py +++ b/pyhealth/models/jamba_ehr.py @@ -15,6 +15,7 @@ from pyhealth.datasets import SampleDataset from pyhealth.models import BaseModel from pyhealth.models.embedding import EmbeddingModel +from pyhealth.models.embedding.unified import UnifiedMultimodalEmbeddingModel from pyhealth.models.transformer import TransformerBlock from pyhealth.models.ehrmamba import MambaBlock from pyhealth.models.utils import get_last_visit @@ -177,6 +178,11 @@ class JambaEHR(BaseModel): by an independent :class:`JambaLayer`. The resulting patient embeddings are concatenated and projected through a classification head. + When ``unified_embedding`` is supplied the model switches to **unified + mode**: all temporal fields are jointly embedded and time-sorted by + :class:`UnifiedMultimodalEmbeddingModel`, then processed by a *single* + :class:`JambaLayer` rather than one layer per field. + Args: dataset (SampleDataset): Dataset providing processed inputs. embedding_dim (int): Embedding and hidden dimension. Default 128. @@ -186,6 +192,8 @@ class JambaEHR(BaseModel): dropout (float): Dropout rate. Default 0.3. state_size (int): SSM state size in Mamba blocks. Default 16. conv_kernel (int): Causal conv kernel in Mamba blocks. Default 4. + unified_embedding (UnifiedMultimodalEmbeddingModel, optional): when + provided, enables unified multi-modal mode with a single JambaLayer. Examples: >>> from pyhealth.datasets import create_sample_dataset, get_dataloader @@ -234,6 +242,7 @@ def __init__( dropout: float = 0.3, state_size: int = 16, conv_kernel: int = 4, + unified_embedding: Optional[UnifiedMultimodalEmbeddingModel] = None, ): super(JambaEHR, self).__init__(dataset=dataset) self.embedding_dim = embedding_dim @@ -243,6 +252,7 @@ def __init__( self.dropout_rate = dropout self.state_size = state_size self.conv_kernel = conv_kernel + self._use_unified = unified_embedding is not None assert ( len(self.label_keys) == 1 @@ -250,11 +260,12 @@ def __init__( self.label_key = self.label_keys[0] self.mode = self.dataset.output_schema[self.label_key] - self.embedding_model = EmbeddingModel(dataset, embedding_dim) + output_size = self.get_output_size() + self.dropout = nn.Dropout(dropout) - self.jamba: nn.ModuleDict = nn.ModuleDict() - for feature_key in self.feature_keys: - self.jamba[feature_key] = JambaLayer( + if self._use_unified: + self.embedding_model = unified_embedding + self._unified_jamba = JambaLayer( feature_size=embedding_dim, num_transformer_layers=num_transformer_layers, num_mamba_layers=num_mamba_layers, @@ -263,12 +274,66 @@ def __init__( state_size=state_size, conv_kernel=conv_kernel, ) + self.fc = nn.Linear(embedding_dim, output_size) + else: + self.embedding_model = EmbeddingModel(dataset, embedding_dim) + self.jamba: nn.ModuleDict = nn.ModuleDict() + for feature_key in self.feature_keys: + self.jamba[feature_key] = JambaLayer( + feature_size=embedding_dim, + num_transformer_layers=num_transformer_layers, + num_mamba_layers=num_mamba_layers, + heads=heads, + dropout=dropout, + state_size=state_size, + conv_kernel=conv_kernel, + ) + self.fc = nn.Linear(len(self.feature_keys) * embedding_dim, output_size) + + def _build_unified_inputs( + self, kwargs: Dict[str, Any] + ) -> Dict[str, Dict[str, torch.Tensor]]: + """Build the inputs dict required by UnifiedMultimodalEmbeddingModel.""" + inputs: Dict[str, Dict[str, torch.Tensor]] = {} + for field_name in self.feature_keys: + feature = kwargs[field_name] + if isinstance(feature, torch.Tensor): + feature = (feature,) + schema = self.dataset.input_processors[field_name].schema() + field_dict: Dict[str, torch.Tensor] = {} + if "value" in schema: + field_dict["value"] = feature[schema.index("value")].to(self.device) + if "time" in schema: + field_dict["time"] = feature[schema.index("time")].to(self.device) + if "mask" in schema: + field_dict["mask"] = feature[schema.index("mask")].to(self.device) + inputs[field_name] = field_dict + return inputs + + def _forward_unified(self, **kwargs: Any) -> Dict[str, torch.Tensor]: + """Forward pass in unified-embedding mode. + + Calls UnifiedMultimodalEmbeddingModel to produce a single + temporally-sorted event sequence, then encodes it with one shared + JambaLayer and pools to the last valid event. + """ + inputs = self._build_unified_inputs(kwargs) + out = self.embedding_model(inputs) + sequence = out["sequence"] # (B, S_total, E) + mask = out["mask"] # (B, S_total) float, 1=valid 0=pad - output_size = self.get_output_size() - self.dropout = nn.Dropout(dropout) - self.fc = nn.Linear( - len(self.feature_keys) * embedding_dim, output_size - ) + _, cls_emb = self._unified_jamba(sequence, mask) + logits = self.fc(self.dropout(cls_emb)) + y_prob = self.prepare_y_prob(logits) + + results: Dict[str, torch.Tensor] = {"logit": logits, "y_prob": y_prob} + if self.label_key in kwargs: + y_true = cast(torch.Tensor, kwargs[self.label_key]).to(self.device) + results["loss"] = self.get_loss_function()(logits, y_true) + results["y_true"] = y_true + if kwargs.get("embed", False): + results["embed"] = cls_emb + return results @staticmethod def _pool_embedding(x: torch.Tensor) -> torch.Tensor: @@ -317,9 +382,10 @@ def forward( ) -> Dict[str, torch.Tensor]: """Forward propagation. - Embeds each feature stream, encodes through the hybrid - Transformer-Mamba stack, concatenates per-stream patient - representations, and projects to label space. + In **unified mode** (when ``unified_embedding`` was supplied at init) + the model jointly embeds all temporal fields and processes them with a + single JambaLayer backbone. Otherwise each field is embedded and + encoded independently. Args: **kwargs: Must include all feature keys (tensors or tuples @@ -330,6 +396,9 @@ def forward( ``y_prob``, ``y_true``, ``logit``, and optionally ``embed`` if ``kwargs["embed"] is True``. """ + if self._use_unified: + return self._forward_unified(**kwargs) + patient_emb = [] for feature_key in self.feature_keys: diff --git a/pyhealth/models/rnn.py b/pyhealth/models/rnn.py index 3393d7287..94fcea0ad 100644 --- a/pyhealth/models/rnn.py +++ b/pyhealth/models/rnn.py @@ -1,4 +1,4 @@ -from typing import Dict, Optional, Tuple +from typing import Any, Dict, Optional, Tuple import torch import torch.nn as nn @@ -20,6 +20,7 @@ ) from .embedding import EmbeddingModel +from .embedding.unified import UnifiedMultimodalEmbeddingModel class RNNLayer(nn.Module): @@ -92,9 +93,7 @@ def forward( Args: x: a tensor of shape [batch size, sequence len, input size]. mask: an optional tensor of shape [batch size, sequence len], where - 1 indicates valid and 0 indicates invalid. Samples with all-zero - masks are clamped to length 1 to prevent pack_padded_sequence - from receiving zero-length sequences. + 1 indicates valid and 0 indicates invalid. Returns: outputs: a tensor of shape [batch size, sequence len, hidden size], @@ -111,13 +110,10 @@ def forward( ) else: lengths = torch.sum(mask.int(), dim=-1).cpu() - # Clamp lengths to at least 1 to handle empty sequences, - # matching TCNLayer (tcn.py:186). - lengths = torch.clamp(lengths, min=1) # Ensure tensor is contiguous for cuDNN compatibility x = x.contiguous() x = rnn_utils.pack_padded_sequence( - x, lengths, batch_first=True, enforce_sorted=False + x.contiguous(), lengths, batch_first=True, enforce_sorted=False ) outputs, _ = self.rnn(x) outputs, _ = rnn_utils.pad_packed_sequence(outputs, batch_first=True) @@ -211,6 +207,7 @@ def __init__( dataset: SampleDataset, embedding_dim: int = 128, hidden_dim: int = 128, + unified_embedding: Optional[UnifiedMultimodalEmbeddingModel] = None, **kwargs ): super(RNN, self).__init__( @@ -218,6 +215,7 @@ def __init__( ) self.embedding_dim = embedding_dim self.hidden_dim = hidden_dim + self._use_unified = unified_embedding is not None # validate kwargs for RNN layer if "input_size" in kwargs: raise ValueError("input_size is determined by embedding_dim") @@ -227,20 +225,71 @@ def __init__( self.label_key = self.label_keys[0] self.mode = self.dataset.output_schema[self.label_key] - self.embedding_model = EmbeddingModel(dataset, embedding_dim) - - self.rnn = nn.ModuleDict() - for feature_key in self.dataset.input_processors.keys(): - self.rnn[feature_key] = RNNLayer( - input_size=embedding_dim, hidden_size=hidden_dim, **kwargs - ) output_size = self.get_output_size() - self.fc = nn.Linear(len(self.feature_keys) * self.hidden_dim, output_size) + + if self._use_unified: + self.embedding_model = unified_embedding + self.rnn = nn.ModuleDict({ + "unified": RNNLayer(input_size=embedding_dim, hidden_size=hidden_dim, **kwargs) + }) + self.fc = nn.Linear(hidden_dim, output_size) + else: + self.embedding_model = EmbeddingModel(dataset, embedding_dim) + self.rnn = nn.ModuleDict() + for feature_key in self.dataset.input_processors.keys(): + self.rnn[feature_key] = RNNLayer( + input_size=embedding_dim, hidden_size=hidden_dim, **kwargs + ) + self.fc = nn.Linear(len(self.feature_keys) * self.hidden_dim, output_size) + + def _build_unified_inputs( + self, kwargs: Dict[str, Any] + ) -> Dict[str, Dict[str, torch.Tensor]]: + """Build the inputs dict required by UnifiedMultimodalEmbeddingModel.""" + inputs: Dict[str, Dict[str, torch.Tensor]] = {} + for field_name in self.feature_keys: + feature = kwargs[field_name] + if isinstance(feature, torch.Tensor): + feature = (feature,) + schema = self.dataset.input_processors[field_name].schema() + field_dict: Dict[str, torch.Tensor] = {} + if "value" in schema: + field_dict["value"] = feature[schema.index("value")].to(self.device) + if "time" in schema: + field_dict["time"] = feature[schema.index("time")].to(self.device) + if "mask" in schema: + field_dict["mask"] = feature[schema.index("mask")].to(self.device) + inputs[field_name] = field_dict + return inputs + + def _forward_unified(self, **kwargs: Any) -> Dict[str, torch.Tensor]: + """Forward pass in unified-embedding mode. + + Embeds all temporal fields jointly as a single time-sorted sequence + and processes it with one RNN backbone. + """ + inputs = self._build_unified_inputs(kwargs) + out = self.embedding_model(inputs) + sequence = out["sequence"] # (B, S, E) + mask = out["mask"].int() # (B, S) + + _, last_hidden = self.rnn["unified"](sequence, mask) # (B, hidden_dim) + logits = self.fc(last_hidden) + y_true = kwargs[self.label_key].to(self.device) + loss = self.get_loss_function()(logits, y_true) + y_prob = self.prepare_y_prob(logits) + results = {"loss": loss, "y_prob": y_prob, "y_true": y_true, "logit": logits} + if kwargs.get("embed", False): + results["embed"] = last_hidden + return results def forward(self, **kwargs) -> Dict[str, torch.Tensor]: """Forward propagation. - The label `kwargs[self.label_key]` is a list of labels for each patient. + In **unified mode** (when ``unified_embedding`` was supplied at init) + the model jointly embeds all temporal fields as a single time-sorted + sequence and processes it with one RNN. Otherwise each field is + embedded and encoded independently. Args: **kwargs: keyword arguments for the model. The keys must contain @@ -254,6 +303,9 @@ def forward(self, **kwargs) -> Dict[str, torch.Tensor]: - logit: a tensor representing the logits. - embed (optional): a tensor representing the patient embeddings if requested. """ + if self._use_unified: + return self._forward_unified(**kwargs) + patient_emb = [] # We need to preprocess kwargs to extract values and masks for EmbeddingModel @@ -569,4 +621,4 @@ def forward(self, **kwargs) -> Dict[str, torch.Tensor]: } if kwargs.get("embed", False): results["embed"] = patient_emb - return results + return results \ No newline at end of file diff --git a/pyhealth/models/transformer.py b/pyhealth/models/transformer.py index cc0dfc5ca..5bb2bdfe3 100644 --- a/pyhealth/models/transformer.py +++ b/pyhealth/models/transformer.py @@ -12,6 +12,7 @@ from pyhealth.datasets import SampleDataset from pyhealth.models import BaseModel from pyhealth.models.embedding import EmbeddingModel +from pyhealth.models.embedding.unified import UnifiedMultimodalEmbeddingModel from pyhealth.interpret.api import CheferInterpretable # VALID_OPERATION_LEVEL = ["visit", "event"] @@ -54,7 +55,7 @@ def forward( # Use -inf so softmax produces exact zeros on padded positions, # avoiding a second masked_fill after softmax (saves one full # [B, H, S, S] boolean allocation and an extra copy). - pad_mask = (mask == 0) + pad_mask = mask == 0 scores = scores.masked_fill(pad_mask, -1e9) p_attn = self.softmax(scores) if dropout is not None: @@ -164,7 +165,7 @@ def forward( self.attn_map = None # 3) "Concat" using a view and apply a final linear. x = x.transpose(1, 2).contiguous().view(batch_size, -1, self.h * self.d_k) - + return self.output_linear(x) @@ -246,7 +247,7 @@ def set_activation_hooks(self, hooks) -> None: """Deprecated compatibility stub; no-op.""" return None - def forward(self, x, mask=None, register_hook = False): + def forward(self, x, mask=None, register_hook=False): """Forward propagation. Args: @@ -256,7 +257,12 @@ def forward(self, x, mask=None, register_hook = False): Returns: A tensor of shape [batch_size, seq_len, hidden] """ - x = self.input_sublayer(x, lambda _x: self.attention(_x, _x, _x, mask=mask, register_hook=register_hook)) + x = self.input_sublayer( + x, + lambda _x: self.attention( + _x, _x, _x, mask=mask, register_hook=register_hook + ), + ) x = self.output_sublayer(x, lambda _x: self.feed_forward(_x, mask=mask)) return self.dropout(x) @@ -297,7 +303,10 @@ def set_activation_hooks(self, hooks) -> None: return None def forward( - self, x: torch.Tensor, mask: Optional[torch.Tensor] = None, register_hook: bool = False + self, + x: torch.Tensor, + mask: Optional[torch.Tensor] = None, + register_hook: bool = False, ) -> Tuple[torch.Tensor, torch.Tensor]: """Forward propagation. @@ -328,12 +337,21 @@ class Transformer(BaseModel, CheferInterpretable): an independent :class:`TransformerLayer`. The resulting [CLS]-style embeddings are concatenated and passed to a classification head. + When ``unified_embedding`` is supplied the model switches to **unified + mode**: all temporal fields are jointly embedded and time-sorted by + :class:`UnifiedMultimodalEmbeddingModel`, then processed by a *single* + :class:`TransformerLayer` rather than one layer per field. This allows + full cross-modal attention over the interleaved event sequence. + Args: dataset (SampleDataset): dataset providing processed inputs. embedding_dim (int): shared embedding dimension. heads (int): number of attention heads per transformer block. dropout (float): dropout rate applied inside transformer blocks. num_layers (int): number of transformer blocks per feature stream. + unified_embedding (UnifiedMultimodalEmbeddingModel, optional): when + provided, the model uses a single backbone over the unified + multi-modal sequence instead of per-field transformers. Examples: >>> from pyhealth.datasets import create_sample_dataset, get_dataloader @@ -377,6 +395,7 @@ def __init__( dropout: float = 0.5, num_layers: int = 1, max_seq_len: int = 1024, + unified_embedding: Optional[UnifiedMultimodalEmbeddingModel] = None, ): super().__init__(dataset=dataset) self.embedding_dim = embedding_dim @@ -385,6 +404,7 @@ def __init__( self.num_layers = num_layers self.max_seq_len = max_seq_len self._attention_hooks_enabled = False + self._use_unified = unified_embedding is not None assert ( len(self.label_keys) == 1 @@ -392,19 +412,28 @@ def __init__( self.label_key = self.label_keys[0] self.mode = self.dataset.output_schema[self.label_key] - self.embedding_model = EmbeddingModel(dataset, embedding_dim) + output_size = self.get_output_size() - self.transformer: nn.ModuleDict = nn.ModuleDict() - for feature_key in self.feature_keys: - self.transformer[feature_key] = TransformerLayer( + if self._use_unified: + self.embedding_model = unified_embedding + self._unified_backbone = TransformerLayer( feature_size=embedding_dim, heads=heads, dropout=dropout, num_layers=num_layers, ) - - output_size = self.get_output_size() - self.fc = nn.Linear(len(self.feature_keys) * embedding_dim, output_size) + self.fc = nn.Linear(embedding_dim, output_size) + else: + self.embedding_model = EmbeddingModel(dataset, embedding_dim) + self.transformer: nn.ModuleDict = nn.ModuleDict() + for feature_key in self.feature_keys: + self.transformer[feature_key] = TransformerLayer( + feature_size=embedding_dim, + heads=heads, + dropout=dropout, + num_layers=num_layers, + ) + self.fc = nn.Linear(len(self.feature_keys) * embedding_dim, output_size) def _pool_embedding(self, x: torch.Tensor) -> torch.Tensor: """Pool nested embeddings to ``[batch, seq_len, hidden]`` format. @@ -443,6 +472,61 @@ def _mask_from_embeddings(x: torch.Tensor) -> torch.Tensor: mask[invalid_rows, 0] = True return mask.bool() + def _build_unified_inputs( + self, kwargs: Dict[str, Any] + ) -> Dict[str, Dict[str, torch.Tensor]]: + """Build inputs expected by UnifiedMultimodalEmbeddingModel.""" + + inputs: Dict[str, Dict[str, torch.Tensor]] = {} + for field_name in self.feature_keys: + feature = kwargs[field_name] + if isinstance(feature, torch.Tensor): + feature = (feature,) + + schema = self.dataset.input_processors[field_name].schema() + field_dict: Dict[str, torch.Tensor] = {} + if "value" in schema: + field_dict["value"] = feature[schema.index("value")].to(self.device) + if "time" in schema: + field_dict["time"] = feature[schema.index("time")].to(self.device) + if "mask" in schema: + field_dict["mask"] = feature[schema.index("mask")].to(self.device) + inputs[field_name] = field_dict + + return inputs + + def _forward_unified( + self, + **kwargs: torch.Tensor | tuple[torch.Tensor, ...], + ) -> Dict[str, torch.Tensor]: + """Forward pass in unified-embedding mode.""" + + register_hook = self._attention_hooks_enabled + inputs = self._build_unified_inputs(cast(Dict[str, Any], kwargs)) + out = self.embedding_model(inputs) + sequence = cast(torch.Tensor, out["sequence"]) + event_mask = cast(torch.Tensor, out["mask"]).bool() + + _, patient_emb = self._unified_backbone(sequence, event_mask, register_hook) + + logits = self.fc(patient_emb) + y_prob = self.prepare_y_prob(logits) + + results: Dict[str, torch.Tensor] = { + "logit": logits, + "y_prob": y_prob, + } + + if self.label_key in kwargs: + y_true = cast(torch.Tensor, kwargs[self.label_key]).to(self.device) + loss = self.get_loss_function()(logits, y_true) + results["loss"] = loss + results["y_true"] = y_true + + if kwargs.get("embed", False): + results["embed"] = patient_emb + return results + def forward_from_embedding( self, **kwargs: torch.Tensor | tuple[torch.Tensor, ...], @@ -500,8 +584,7 @@ def forward_from_embedding( if value is None: raise ValueError( - f"Feature '{feature_key}' must contain 'value' " - f"in the schema." + f"Feature '{feature_key}' must contain 'value' " f"in the schema." ) else: value = value.to(self.device) @@ -515,9 +598,7 @@ def forward_from_embedding( else: mask = self._mask_from_embeddings(value).to(self.device) - _, cls_emb = self.transformer[feature_key]( - value, mask, register_hook - ) + _, cls_emb = self.transformer[feature_key](value, mask, register_hook) patient_emb.append(cls_emb) patient_emb = torch.cat(patient_emb, dim=1) @@ -545,6 +626,11 @@ def forward( ) -> Dict[str, torch.Tensor]: """Forward propagation. + In **unified mode** (when ``unified_embedding`` was supplied at init) + the model jointly embeds all temporal fields and processes them with a + single transformer backbone. Otherwise each field is embedded and + encoded independently. + Args: **kwargs: keyword arguments for the model. @@ -562,6 +648,9 @@ def forward( logit: the raw logits before activation. embed: (if embed=True in kwargs) the patient embedding. """ + if self._use_unified: + return self._forward_unified(**kwargs) + for feature_key in self.feature_keys: feature = kwargs[feature_key] @@ -575,15 +664,16 @@ def forward( if value is None: raise ValueError( - f"Feature '{feature_key}' must contain 'value' " - f"in the schema." + f"Feature '{feature_key}' must contain 'value' " f"in the schema." ) else: value = value.to(self.device) if mask is not None: mask = mask.to(self.device) - value = self.embedding_model({feature_key: value}, masks={feature_key: mask})[feature_key] + value = self.embedding_model( + {feature_key: value}, masks={feature_key: mask} + )[feature_key] else: value = self.embedding_model({feature_key: value})[feature_key] @@ -591,9 +681,9 @@ def forward( # Reconstruct tuple with embedded value # Note: we need to handle list/tuple conversion carefully # feature is a tuple. - + # Simple slice reconstruction - kwargs[feature_key] = feature[:i] + (value,) + feature[i + 1:] + kwargs[feature_key] = feature[:i] + (value,) + feature[i + 1 :] return self.forward_from_embedding(**kwargs) @@ -621,9 +711,7 @@ def get_attention_layers( cast(TransformerBlock, blk).attention.get_attn_map(), cast(TransformerBlock, blk).attention.get_attn_grad(), ) - for blk in cast( - TransformerLayer, self.transformer[key] - ).transformer + for blk in cast(TransformerLayer, self.transformer[key]).transformer ] for key in self.feature_keys } @@ -683,4 +771,4 @@ def get_relevance_tensor( result = model(**data_batch) print(result) - result["loss"].backward() + result["loss"].backward() \ No newline at end of file diff --git a/pyhealth/processors/time_image_processor.py b/pyhealth/processors/time_image_processor.py index 9d313e6bc..205782c7f 100644 --- a/pyhealth/processors/time_image_processor.py +++ b/pyhealth/processors/time_image_processor.py @@ -75,6 +75,11 @@ class TimeImageProcessor(TemporalFeatureProcessor): patient has more images, the most recent (by timestamp) are kept. If None, all images are kept. Defaults to None. + padding: Sentinel string that marks a missing image. When + a path equals this value, a zero tensor of shape + (C, H, W) is returned instead of loading from disk. + If None, all paths are treated as real file paths. + Defaults to None. Raises: ValueError: If normalize is True but mean or std is missing. @@ -107,6 +112,7 @@ def __init__( std: Optional[List[float]] = None, mode: Optional[str] = None, max_images: Optional[int] = None, + padding: Optional[str] = None, ) -> None: self.image_size = image_size self.to_tensor = to_tensor @@ -115,18 +121,14 @@ def __init__( self.std = std self.mode = mode self.max_images = max_images + self.padding = padding self.n_channels = None - if self.normalize and ( - self.mean is None or self.std is None - ): + if self.normalize and (self.mean is None or self.std is None): raise ValueError( - "Normalization requires both mean and std to be " - "provided." + "Normalization requires both mean and std to be " "provided." ) - if not self.normalize and ( - self.mean is not None or self.std is not None - ): + if not self.normalize and (self.mean is not None or self.std is not None): raise ValueError( "Mean and std are provided but normalize is set " "to False. Either provide normalize=True, or " @@ -146,36 +148,49 @@ def _build_transform(self) -> transforms.Compose: transform_list = [] if self.mode is not None: transform_list.append( - transforms.Lambda( - partial(_convert_mode, mode=self.mode) - ) + transforms.Lambda(partial(_convert_mode, mode=self.mode)) ) if self.image_size is not None: - transform_list.append( - transforms.Resize( - (self.image_size, self.image_size) - ) - ) + transform_list.append(transforms.Resize((self.image_size, self.image_size))) if self.to_tensor: transform_list.append(transforms.ToTensor()) if self.normalize: - transform_list.append( - transforms.Normalize( - mean=self.mean, std=self.std - ) - ) + transform_list.append(transforms.Normalize(mean=self.mean, std=self.std)) return transforms.Compose(transform_list) - def _load_single_image( - self, path: Union[str, Path] - ) -> torch.Tensor: + def _zero_image_tensor(self) -> torch.Tensor: + """Return a zero tensor matching the expected image shape (C, H, W). + + Used as a placeholder when an image path is an empty string. + Channel count is inferred from self.n_channels if available, + otherwise derived from self.mode ("L"→1, "RGBA"→4, else 3). + + Returns: + Zero tensor of shape (C, image_size, image_size). + """ + if self.n_channels is not None: + c = self.n_channels + elif self.mode == "L": + c = 1 + elif self.mode == "RGBA": + c = 4 + else: + c = 3 + return torch.zeros(c, self.image_size, self.image_size) + + def _load_single_image(self, path: Union[str, Path]) -> torch.Tensor: """Load and transform a single image from disk. + If path equals missing_path_token, returns a zero tensor of + the same shape as a normal image (C, H, W) via _zero_image_tensor. + Called internally by process() for each image path in the input list. Args: - path: Path to the image file. + path: Path to the image file. If this equals + missing_path_token, a zero-filled placeholder tensor + is returned instead. Returns: Transformed image tensor of shape (C, H, W). @@ -183,18 +198,16 @@ def _load_single_image( Raises: FileNotFoundError: If the image file does not exist. """ + if self.padding is not None and str(path) == self.padding: + return self._zero_image_tensor() image_path = Path(path) if not image_path.exists(): - raise FileNotFoundError( - f"Image file not found: {image_path}" - ) + raise FileNotFoundError(f"Image file not found: {image_path}") with Image.open(image_path) as img: img.load() return self.transform(img) - def fit( - self, samples: Iterable[Dict[str, Any]], field: str - ) -> None: + def fit(self, samples: Iterable[Dict[str, Any]], field: str) -> None: """Fit the processor by inferring n_channels from data. Scans samples to find the first valid entry for the given @@ -214,8 +227,10 @@ def fit( for sample in samples: if field in sample and sample[field] is not None: image_paths, _ = sample[field] - if len(image_paths) > 0: - path = Path(image_paths[0]) + for raw_path in image_paths: + if self.padding is not None and str(raw_path) == self.padding: + continue + path = Path(raw_path) if path.exists(): with Image.open(path) as img: if img.mode == "L": @@ -225,14 +240,14 @@ def fit( else: self.n_channels = 3 break + if self.n_channels is not None: + break if self.n_channels is None: self.n_channels = 3 def process( self, - value: Tuple[ - List[Union[str, Path]], List[float] - ], + value: Tuple[List[Union[str, Path]], List[float]], ) -> Tuple[torch.Tensor, torch.Tensor, str]: """Process paired image paths and timestamps. @@ -278,15 +293,10 @@ def process( if len(image_paths) == 0: raise ValueError("image_paths must be non-empty.") - paired = sorted( - zip(time_diffs, image_paths), key=lambda x: x[0] - ) + paired = sorted(zip(time_diffs, image_paths), key=lambda x: x[0]) - if ( - self.max_images is not None - and len(paired) > self.max_images - ): - paired = paired[-self.max_images:] + if self.max_images is not None and len(paired) > self.max_images: + paired = paired[-self.max_images :] timestamps = [] image_tensors = [] @@ -295,9 +305,7 @@ def process( timestamps.append(t) images = torch.stack(image_tensors, dim=0) - timestamps = torch.tensor( - timestamps, dtype=torch.float32 - ) + timestamps = torch.tensor(timestamps, dtype=torch.float32) if self.n_channels is None: self.n_channels = images.shape[1] @@ -344,5 +352,6 @@ def __repr__(self) -> str: f"mean={self.mean}, " f"std={self.std}, " f"mode={self.mode}, " - f"max_images={self.max_images})" + f"max_images={self.max_images}, " + f"padding={self.padding!r})" ) \ No newline at end of file diff --git a/pyhealth/processors/tuple_time_text_processor.py b/pyhealth/processors/tuple_time_text_processor.py index bbe74c4e6..cb1fe99a0 100644 --- a/pyhealth/processors/tuple_time_text_processor.py +++ b/pyhealth/processors/tuple_time_text_processor.py @@ -5,6 +5,7 @@ from . import register_processor logger = logging.getLogger(__name__) +_MISSING_TEXT_TOKEN = "[MISSING_TEXT]" @register_processor("tuple_time_text") class TupleTimeTextProcessor(TemporalFeatureProcessor): @@ -81,6 +82,41 @@ def process(self, value: Tuple[List[str], List[float]]) -> Union[Tuple[List[str] - str: Type tag """ texts, time_diffs = value + texts = list(texts or []) + time_diffs = list(time_diffs or []) + + # Keep text/time aligned and filter malformed text entries. + pair_count = min(len(texts), len(time_diffs)) + cleaned_texts: List[str] = [] + cleaned_times: List[float] = [] + for i in range(pair_count): + raw_text = texts[i] + raw_time = time_diffs[i] + + # Normalize text; skip null/whitespace-only entries. + if raw_text is None: + continue + text = str(raw_text).strip() + if text == "": + continue + + # Best-effort float normalization; skip unparseable timestamps. + try: + t = float(raw_time) + except (TypeError, ValueError): + continue + + cleaned_texts.append(text) + cleaned_times.append(t) + + # Fast tokenizer path crashes on empty batches; force a single + # missingness token when all notes are empty/malformed. + if len(cleaned_texts) == 0: + cleaned_texts = [_MISSING_TEXT_TOKEN] + cleaned_times = [0.0] + + texts = cleaned_texts + time_diffs = cleaned_times time_tensor = torch.tensor(time_diffs, dtype=torch.float32) if self.tokenizer is not None: @@ -93,16 +129,18 @@ def process(self, value: Tuple[List[str], List[float]]) -> Union[Tuple[List[str] return_tensors="pt" ) - input_ids = encoded["input_ids"] - attention_mask = encoded["attention_mask"] + input_ids = encoded.get("input_ids") + if input_ids is None: + raise ValueError("Tokenizer output is missing required `input_ids`.") + + attention_mask = encoded.get("attention_mask") + if attention_mask is None: + attention_mask = torch.ones_like(input_ids) # Not all tokenizers return token_type_ids (e.g. RoBERTa might not, BERT does) - if "token_type_ids" in encoded: - token_type_ids = encoded["token_type_ids"] - else: - # meaningful text usually 0, padding 0? BERT uses 0 for sent A. - # If not provided, we can just use zeros or omit. - # For consistency with schema, let's provide zeros if expected. + token_type_ids = encoded.get("token_type_ids") + if token_type_ids is None: + # Some tokenizers do not return token_type_ids. token_type_ids = torch.zeros_like(input_ids) return input_ids, attention_mask, token_type_ids, time_tensor, self.type_tag @@ -172,4 +210,4 @@ def process_temporal(self, value) -> dict: def __repr__(self): if self.tokenizer_model: return f"TupleTimeTextProcessor(type_tag='{self.type_tag}', tokenizer='{self.tokenizer_model}')" - return f"TupleTimeTextProcessor(type_tag='{self.type_tag}')" + return f"TupleTimeTextProcessor(type_tag='{self.type_tag}')" \ No newline at end of file diff --git a/pyhealth/scripts_delete_me/will/condor/labs_only/labs_only_rnn.sub b/pyhealth/scripts_delete_me/will/condor/labs_only/labs_only_rnn.sub new file mode 100644 index 000000000..b0768f201 --- /dev/null +++ b/pyhealth/scripts_delete_me/will/condor/labs_only/labs_only_rnn.sub @@ -0,0 +1,45 @@ +# HTCondor submission — labs-only RNN mortality run +# +# Condor equivalent of scripts/will/sunlab/tmux_run_labs_only_rnn_variant.py: +# same task (labs), same model (rnn), same hyperparameters. Runs unattended +# instead of in a tmux session; Condor assigns the GPU (no manual +# nvidia-smi / CUDA_VISIBLE_DEVICES step needed). +# +# To submit (from the project root): +# mkdir -p /home/wp14/logs/condor +# condor_submit scripts/will/condor/labs_only_rnn.sub +# +# Monitor: +# condor_q +# tail -f /home/wp14/logs/condor/labs_only_rnn__0.out + +initialdir = /home/wp14/PyHealth +executable = /home/wp14/PyHealth/scripts/will/condor/run_labs_only_rnn.sh +transfer_executable = False +arguments = $(seed) +getenv = True + +environment = "EHR_ROOT=/shared/rsaas/physionet.org/files/mimiciv/2.2 \ + CACHE_DIR=/shared/eng/wp14/pyhealth_cache_labs \ + OUTPUT_DIR=/home/wp14/output \ + USE_WANDB=1 \ + WANDB_PROJECT=pyhealth-multimodal-labs-only \ + WANDB_RUN_NAME=labs_rnn_seed$(seed)" + +output = /home/wp14/logs/condor/labs_only_rnn_$(ClusterId)_$(Process).out +error = /home/wp14/logs/condor/labs_only_rnn_$(ClusterId)_$(Process).err +log = /home/wp14/logs/condor/labs_only_rnn_$(ClusterId)_$(Process).log + +stream_output = False +stream_error = False + +request_gpus = 1 +request_cpus = 4 +request_memory = 163840MB +request_disk = 20GB + +Rank = TARGET.CUDAGlobalMemoryMb + +queue seed from ( + 12 +) \ No newline at end of file diff --git a/pyhealth/scripts_delete_me/will/condor/labs_only/run_labs_only_rnn.sh b/pyhealth/scripts_delete_me/will/condor/labs_only/run_labs_only_rnn.sh new file mode 100644 index 000000000..649eaccb1 --- /dev/null +++ b/pyhealth/scripts_delete_me/will/condor/labs_only/run_labs_only_rnn.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +# HTCondor executable — labs-only RNN mortality run. +# +# Condor equivalent of scripts/will/sunlab/tmux_run_labs_only_rnn_variant.py: +# same task (labs), same model (rnn), same hyperparameter defaults. GPU +# selection (nvidia-smi / CUDA_VISIBLE_DEVICES) is dropped since Condor +# assigns the GPU via request_gpus / cgroups. +# +# usage: run_labs_only_rnn.sh +# to remove logs: rm -rf logs/condor/* +# to remove cache: rm -rf /shared/eng/wp14/pyhealth_cache_labs/* +set -euo pipefail + +SEED="${1:?usage: run_labs_only_rnn.sh }" + +CONDA_ENV="${CONDA_ENV:-pyhealth2}" +PROJECT_DIR="${PROJECT_DIR:-/home/wp14/PyHealth}" +EHR_ROOT="${EHR_ROOT:-/shared/rsaas/physionet.org/files/mimiciv/2.2}" +CACHE_DIR="${CACHE_DIR:-/shared/eng/wp14/pyhealth_cache_labs}" +OUTPUT_DIR="${OUTPUT_DIR:-/home/wp14/output}" +CONDA_SH="${CONDA_SH:-}" + +DEV_MODE="${DEV_MODE:-0}" +EMBEDDING_DIM="${EMBEDDING_DIM:-64}" +HIDDEN_DIM="${HIDDEN_DIM:-64}" +RNN_TYPE="${RNN_TYPE:-GRU}" +RNN_LAYERS="${RNN_LAYERS:-1}" +DROPOUT="${DROPOUT:-0.1}" +EPOCHS="${EPOCHS:-15}" +BATCH_SIZE="${BATCH_SIZE:-32}" +LR="${LR:-1e-3}" +WEIGHT_DECAY="${WEIGHT_DECAY:-1e-5}" +PATIENCE="${PATIENCE:-5}" +NUM_WORKERS="${NUM_WORKERS:-4}" + +# Condor GPU cgroups can expose a truncated CUDA_VISIBLE_DEVICES UUID that +# distributed's NVML diagnostics can't resolve, crashing LocalCluster startup. +# Skip the distributed Dask cluster (falls back to the plain local scheduler) +# to avoid touching NVML during event-dataframe preprocessing. +export PYHEALTH_DISABLE_DASK_DISTRIBUTED="${PYHEALTH_DISABLE_DASK_DISTRIBUTED:-1}" + +USE_WANDB="${USE_WANDB:-0}" +WANDB_PROJECT="${WANDB_PROJECT:-pyhealth-multimodal-labs-only}" +WANDB_RUN_NAME="${WANDB_RUN_NAME:-}" + +resolve_conda_sh() { + if [[ -n "${CONDA_SH}" && -f "${CONDA_SH}" ]]; then + echo "${CONDA_SH}" + return 0 + fi + if command -v conda >/dev/null 2>&1; then + local base + base="$(conda info --base 2>/dev/null || true)" + if [[ -n "${base}" && -f "${base}/etc/profile.d/conda.sh" ]]; then + echo "${base}/etc/profile.d/conda.sh" + return 0 + fi + fi + if [ -f /etc/profile.d/modules.sh ]; then + source /etc/profile.d/modules.sh >/dev/null 2>&1 || true + if command -v module >/dev/null 2>&1; then + module load miniconda3 >/dev/null 2>&1 || true + module load anaconda3 >/dev/null 2>&1 || true + if command -v conda >/dev/null 2>&1; then + local mod_base + mod_base="$(conda info --base 2>/dev/null || true)" + if [[ -n "${mod_base}" && -f "${mod_base}/etc/profile.d/conda.sh" ]]; then + echo "${mod_base}/etc/profile.d/conda.sh" + return 0 + fi + fi + fi + fi + local user_name home_dir + user_name="${USER:-$(id -un 2>/dev/null || true)}" + home_dir="${HOME:-/home/${user_name}}" + local candidates=( + "${home_dir}/miniconda3/etc/profile.d/conda.sh" + "/home/${user_name}/miniconda3/etc/profile.d/conda.sh" + "${home_dir}/anaconda3/etc/profile.d/conda.sh" + "/home/${user_name}/anaconda3/etc/profile.d/conda.sh" + "/opt/miniconda3/etc/profile.d/conda.sh" + "/opt/anaconda3/etc/profile.d/conda.sh" + "/opt/conda/etc/profile.d/conda.sh" + ) + local c + for c in "${candidates[@]}"; do + if [[ -f "${c}" ]]; then + echo "${c}" + return 0 + fi + done + local found="" + found="$(find "${home_dir}" /opt /usr/local /shared -maxdepth 6 -type f -path '*/etc/profile.d/conda.sh' 2>/dev/null | head -n 1 || true)" + if [[ -n "${found}" && -f "${found}" ]]; then + echo "${found}" + return 0 + fi + return 1 +} + +CONDA_SH="$(resolve_conda_sh || true)" +if [[ -z "${CONDA_SH}" || ! -f "${CONDA_SH}" ]]; then + echo "ERROR: conda.sh not found. Set CONDA_SH explicitly." >&2 + exit 1 +fi +source "${CONDA_SH}" +eval "$(conda shell.bash hook)" +conda activate "${CONDA_ENV}" + +cd "${PROJECT_DIR}" +export PYTHONPATH="${PROJECT_DIR}:${PYTHONPATH:-}" + +JOB_TAG="rnn_labs_s${SEED}_c${_CONDOR_CLUSTER_ID:-local}_p${_CONDOR_PROCNO:-0}" + +echo "========================================================" +echo " Labs-only RNN run | ${JOB_TAG}" +echo " $(date -u '+%Y-%m-%d %H:%M:%S UTC')" +echo " Conda env : ${CONDA_ENV}" +echo " EHR root : ${EHR_ROOT}" +echo " Cache dir : ${CACHE_DIR}" +echo " Output dir: ${OUTPUT_DIR}" +echo " Seed : ${SEED}" +echo " Dev mode : ${DEV_MODE}" +echo "========================================================" + +if ! python -c "import pyhealth" >/dev/null 2>&1; then + echo "ERROR: pyhealth is not importable. Run: bash condor_setup.sh" >&2 + exit 1 +fi + +COMMON=( + --ehr-root "${EHR_ROOT}" + --cache-dir "${CACHE_DIR}" + --task labs + --model rnn + --embedding-dim "${EMBEDDING_DIM}" + --hidden-dim "${HIDDEN_DIM}" + --rnn-type "${RNN_TYPE}" + --rnn-layers "${RNN_LAYERS}" + --dropout "${DROPOUT}" + --epochs "${EPOCHS}" + --batch-size "${BATCH_SIZE}" + --lr "${LR}" + --weight-decay "${WEIGHT_DECAY}" + --patience "${PATIENCE}" + --num-workers "${NUM_WORKERS}" + --seed "${SEED}" + --output-dir "${OUTPUT_DIR}" +) + +if [[ "${DEV_MODE}" == "1" ]]; then + COMMON+=(--dev) +fi + +if [[ "${USE_WANDB}" == "1" ]]; then + COMMON+=(--wandb --wandb-project "${WANDB_PROJECT}") + if [[ -n "${WANDB_RUN_NAME}" ]]; then + COMMON+=(--wandb-run-name "${WANDB_RUN_NAME}") + fi +fi + +python examples/mortality_prediction/unified_embedding_e2e_mimic4.py "${COMMON[@]}" + +echo "========================================================" +echo " Completed ${JOB_TAG}" +echo " $(date -u '+%Y-%m-%d %H:%M:%S UTC')" +echo "========================================================" \ No newline at end of file diff --git a/pyhealth/tasks/multimodal_mimic4.py b/pyhealth/tasks/multimodal_mimic4.py new file mode 100644 index 000000000..c3b1308e9 --- /dev/null +++ b/pyhealth/tasks/multimodal_mimic4.py @@ -0,0 +1,1064 @@ +import logging +import re +from datetime import datetime, timedelta +from typing import Any, Dict, List, Optional, Union, Tuple, ClassVar + +from pyhealth.tasks.base_task import BaseTask + +logger = logging.getLogger(__name__) + + +class BaseMultimodalMIMIC4Task(BaseTask): + """Base class for multimodal MIMIC-IV tasks. + + Provides shared constants and utility methods used across all multimodal + task variants (notes, ICD codes, lab values). + """ + + MISSING_TEXT_TOKEN: ClassVar[str] = "" + MISSING_CODE_TOKEN: ClassVar[str] = "" + MISSING_FLOAT_TOKEN: ClassVar[float] = 0.0 + + LAB_CATEGORIES: ClassVar[Dict[str, List[str]]] = { + "Sodium": ["50824", "52455", "50983", "52623"], + "Potassium": ["50822", "52452", "50971", "52610"], + "Chloride": ["50806", "52434", "50902", "52535"], + "Bicarbonate": ["50803", "50804"], + "Glucose": ["50809", "52027", "50931", "52569"], + "Calcium": ["50808", "51624"], + "Magnesium": ["50960"], + "Anion Gap": ["50868", "52500"], + "Osmolality": ["52031", "50964", "51701"], + "Phosphate": ["50970"], + } + + LAB_CATEGORY_NAMES: ClassVar[List[str]] = [ + "Sodium", + "Potassium", + "Chloride", + "Bicarbonate", + "Glucose", + "Calcium", + "Magnesium", + "Anion Gap", + "Osmolality", + "Phosphate", + ] + + LABITEMS: ClassVar[List[str]] = [ + item for itemids in LAB_CATEGORIES.values() for item in itemids + ] + + RADIOLOGY_CLINICAL_HEADERS: ClassVar[List[str]] = [ + "indication", + "impression", + # "findings", + # "clinical history", + # "history", + # "comparison", + # "technique", + # "conclusion", + # "summary" + ] + + DISCHARGE_CLINICAL_HEADERS: ClassVar[List[str]] = [ + "chief complaint", + # "history of present illness", + # "hpi", + # "past medical history", + # "past medical and surgical history", + # "past medical/surgical history", + # "past surgical history", + # "medications on admission", + # "admission medications", + # "home medications", + # "social history", + # "family history", + # "allergies", + # "review of systems", + ] + + def __init__( + self, + window_hours: Optional[float] = None, + ): + self.window_hours = window_hours + + @staticmethod + def _clean_text(text: Optional[str]) -> Optional[str]: + """Return text if non-empty, otherwise None.""" + return text if text else None + + @staticmethod + def _parse_note_sections(text: str, note_type: str) -> Dict[str, str]: + """Split a note into {lowercased_header: content_text} pairs.""" + ext_text = text + '\n\n' + if note_type == "radiology": + section_re = re.compile(r'([a-zA-Z ]+):[ \t\n]+(.+?)\n{2,}', re.DOTALL) + elif note_type == "discharge": + section_re = re.compile(r'([a-zA-Z ]+):\n+(.+?)\n{2,}', re.DOTALL) + else: + raise ValueError(f"Note Type '{note_type}' not supported.") + return { + m.group(1).strip().lower(): m.group(2).strip() + for m in section_re.finditer(ext_text) + if m.end() - m.start() > 0 + } + + @staticmethod + def _parse_datetime(value: Any) -> Optional[datetime]: + if isinstance(value, datetime): + return value + if isinstance(value, str): + for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d"): + try: + return datetime.strptime(value, fmt) + except ValueError: + continue + return None + + @staticmethod + def _to_hours(delta_seconds: float) -> float: + return delta_seconds / 3600.0 + + def _compute_effective_window( + self, + admissions_to_process: List[Any], + ) -> Tuple[datetime, Optional[datetime]]: + """Compute effective start/end from the global span of processed admissions. + + Returns: + Tuple of (effective_start, effective_end). + """ + global_start = admissions_to_process[0].timestamp + global_end: Optional[datetime] = None + + for a in admissions_to_process: + dt = self._parse_datetime(getattr(a, "dischtime", None)) + if dt is not None and (global_end is None or dt > global_end): + global_end = dt + + if self.window_hours is not None: + effective_start = global_start + effective_end = effective_start + timedelta(hours=self.window_hours) + return effective_start, effective_end + + effective_start = global_start + effective_end = global_end + + return effective_start, effective_end + + def _build_admissions_to_process(self, patient: Any) -> Tuple[List[Any], int]: + """Build admissions to process and derive mortality label. + + Includes all admissions up to and including the first death admission. + Patients who die in their first (and only) admission are included as + positives — previously they were dropped, which silently discarded most + ICU mortality positives and collapsed positive rate from ~10% to ~2.7%. + This now matches stagenet's semantics: use all available admission data + and label as positive if any admission has hospital_expire_flag=1. + """ + admissions = patient.get_events(event_type="admissions") + if len(admissions) == 0: + return [], 0 + + admissions_to_process: List[Any] = [] + mortality_label = 0 + + for admission in admissions: + admissions_to_process.append(admission) + if admission.hospital_expire_flag in [1, "1"]: + mortality_label = 1 + break + + return admissions_to_process, mortality_label + + def _collect_icd_codes(self, patient: Any, hadm_id: Any) -> List[str]: + """Collect ICD diagnosis and procedure codes for one admission. + + Returns: + List of ICD code strings, or an empty list if none found. + """ + diagnoses_icd = patient.get_events( + event_type="diagnoses_icd", filters=[("hadm_id", "==", hadm_id)] + ) + procedures_icd = patient.get_events( + event_type="procedures_icd", filters=[("hadm_id", "==", hadm_id)] + ) + return [ + e.icd_code for e in diagnoses_icd if hasattr(e, "icd_code") and e.icd_code + ] + [ + e.icd_code for e in procedures_icd if hasattr(e, "icd_code") and e.icd_code + ] + + def _collect_labs( + self, + patient: Any, + admission_time: datetime, + end_time: datetime, + ) -> Tuple[List[float], List[List[float]], List[List[bool]]]: + """Collect lab values and observation masks for one admission. + + Args: + patient: Patient object. + admission_time: Start of the window; times are relative to this. + end_time: End of the window (inclusive). + + Returns: + Tuple of (lab_times, lab_values, lab_masks). ``lab_masks`` is a + parallel boolean tensor where ``True`` means observed and ``False`` + means imputed with 0.0. Falls back to a single missing placeholder + row when no valid lab events are found. + """ + try: + import polars as pl + except ImportError as exc: + raise ImportError("Polars is required for lab collection.") from exc + + labevents_df = patient.get_events( + event_type="labevents", + start=admission_time, + end=end_time, + return_df=True, + ) + + lab_times: List[float] = [] + lab_values: List[List[float]] = [] + lab_masks: List[List[bool]] = [] + + labevents_df = labevents_df.filter( + pl.col("labevents/itemid").is_in(self.LABITEMS) + ) + if labevents_df.height > 0: + labevents_df = labevents_df.with_columns( + pl.col("labevents/storetime").str.strptime( + pl.Datetime, "%Y-%m-%d %H:%M:%S" + ) + ) + labevents_df = labevents_df.filter( + pl.col("labevents/storetime") <= end_time + ) + if labevents_df.height > 0: + labevents_df = labevents_df.select( + pl.col("timestamp"), + pl.col("labevents/itemid"), + pl.col("labevents/valuenum").cast(pl.Float64), + ) + for lab_ts in sorted(labevents_df["timestamp"].unique().to_list()): + ts_labs = labevents_df.filter(pl.col("timestamp") == lab_ts) + lab_vector: List[float] = [] + lab_mask: List[bool] = [] + for category_name in self.LAB_CATEGORY_NAMES: + category_value = self.MISSING_FLOAT_TOKEN + observed = False + for itemid in self.LAB_CATEGORIES[category_name]: + matching = ts_labs.filter( + pl.col("labevents/itemid") == itemid + ) + if matching.height > 0: + category_value = matching["labevents/valuenum"][0] + observed = True + break + lab_vector.append(category_value) + lab_mask.append(observed) + lab_times.append( + self._to_hours((lab_ts - admission_time).total_seconds()) + ) + lab_values.append(lab_vector) + lab_masks.append(lab_mask) + else: # If missing lab for a given admission + lab_values.append( + [self.MISSING_FLOAT_TOKEN] * len(self.LAB_CATEGORY_NAMES) + ) + lab_masks.append([False] * len(self.LAB_CATEGORY_NAMES)) + lab_times.append(self.MISSING_FLOAT_TOKEN) + + if len(lab_values) == 0: # If missing lab for ALL admissions + lab_values.append([self.MISSING_FLOAT_TOKEN] * len(self.LAB_CATEGORY_NAMES)) + lab_masks.append([False] * len(self.LAB_CATEGORY_NAMES)) + lab_times.append(self.MISSING_FLOAT_TOKEN) + return lab_times, lab_values, lab_masks + + def _collect_notes( + self, + patient: Any, + note_event_type: str, + hadm_id: Any, + admission_time: datetime, + start_time: Optional[datetime] = None, + end_time: Optional[datetime] = None, + section_headers: Optional[List[str]] = None, + fallback_to_full_note: bool = False, + ) -> Tuple[List[str], List[float]]: + """Collect notes of a given type for one admission. + + Args: + patient: Patient object. + note_event_type: Event type string (e.g. "discharge", "radiology"). + hadm_id: Admission ID to filter by. + admission_time: Admission start time; used to compute time offsets. + start_time: Optional start of the time window. + end_time: Optional end of the time window. + section_headers: When provided, extract only these named sections + from each note (lowercased match against parsed headers). + fallback_to_full_note: When True (default), falls back to the full + note text if no matching sections are found. When False, notes + with no matching sections are dropped entirely. + + Returns: + Tuple of (texts, hours_from_admission). Falls back to + ``([MISSING_TEXT_TOKEN], [MISSING_FLOAT_TOKEN])`` when the events + list is empty. + """ + notes = patient.get_events( + event_type=note_event_type, + start=start_time, + end=end_time, + filters=[("hadm_id", "==", hadm_id)], + ) + + texts: List[str] = [] + note_times: List[float] = [] + for note in notes: + try: + note_text = self._clean_text(note.text) + if note_text: + if section_headers is not None: + parsed = self._parse_note_sections(note_text, note_type=note_event_type) + extracted = [f"{k}: {v}" for k, v in parsed.items() if k in section_headers and v] + if extracted: + note_text = " [SEP] ".join(extracted) + elif not fallback_to_full_note: + continue + + time_from_admission = self._to_hours( + (note.timestamp - admission_time).total_seconds() + ) + texts.append(note_text) + note_times.append(time_from_admission) + except ( + AttributeError + ): # note object is missing .text or .timestamp attribute (e.g. malformed note) + pass + + return texts, note_times + + +class ICDLabsMIMIC4(BaseMultimodalMIMIC4Task): + """Task for ICD codes + lab values mortality prediction using MIMIC-IV. + + A notes-free structured-EHR task that uses only: + + - **ICD codes**: diagnosis and procedure codes per admission, processed by + ``StageNetProcessor`` with inter-admission time offsets. + - **Lab values**: 10-dimensional lab vectors (one per lab category) at each + measurement timestamp, processed by ``StageNetTensorProcessor``. + + Examples: + >>> from pyhealth.datasets import MIMIC4Dataset + >>> from pyhealth.tasks.multimodal_mimic4 import ICDLabsMIMIC4 + >>> dataset = MIMIC4Dataset( + ... ehr_root="/path/to/mimic-iv/2.2", + ... ehr_tables=["diagnoses_icd", "procedures_icd", "labevents"], + ... ) + >>> task = ICDLabsMIMIC4() + >>> samples = dataset.set_task(task) + """ + + PADDING: int = 0 + + task_name: str = "ICDLabsMIMIC4" + input_schema: Dict[str, Union[str, Tuple[str, Dict]]] = { + "icd_codes": ("stagenet", {"padding": PADDING}), + "labs": ("stagenet_tensor", {}), + "labs_mask": ("stagenet_tensor", {}), + } + output_schema: Dict[str, str] = {"mortality": "binary"} + + def __call__(self, patient: Any) -> List[Dict[str, Any]]: + demographics = patient.get_events(event_type="patients") + if not demographics: + return [] + + admissions_to_process, mortality_label = self._build_admissions_to_process( + patient + ) + + if len(admissions_to_process) == 0: + return [] + + effective_start, effective_end = self._compute_effective_window( + admissions_to_process + ) + + all_icd_codes: List[List[str]] = [] + all_icd_times: List[float] = [] + all_lab_values: List[List[float]] = [] + all_lab_masks: List[List[bool]] = [] + all_lab_times: List[float] = [] + previous_admission_time = None + + for admission in admissions_to_process: + admission_time = admission.timestamp + + try: + admission_dischtime = datetime.strptime( + admission.dischtime, "%Y-%m-%d %H:%M:%S" + ) + except (ValueError, AttributeError): + admission_dischtime = admission_time + if admission_dischtime < admission_time: + admission_dischtime = admission_time + + visit_icd_codes = self._collect_icd_codes(patient, admission.hadm_id) + if visit_icd_codes: + if previous_admission_time is None: + time_from_previous = 0.0 + else: + time_from_previous = self._to_hours( + (admission_time - previous_admission_time).total_seconds() + ) + all_icd_codes.append(visit_icd_codes) + all_icd_times.append(time_from_previous) + else: + all_icd_codes.append([self.MISSING_TEXT_TOKEN]) + all_icd_times.append(self.MISSING_FLOAT_TOKEN) + + previous_admission_time = admission_time + + lab_times, lab_values, lab_masks = self._collect_labs( + patient=patient, + admission_time=admission_time, + end_time=admission_dischtime, + ) + all_lab_times.extend(lab_times) + all_lab_values.extend(lab_values) + all_lab_masks.extend(lab_masks) + + if len(all_lab_values) == 0: + all_lab_values.append( + [self.MISSING_FLOAT_TOKEN] * len(self.LAB_CATEGORY_NAMES) + ) + all_lab_masks.append([False] * len(self.LAB_CATEGORY_NAMES)) + all_lab_times.append(self.MISSING_FLOAT_TOKEN) + + if len(all_icd_codes) == 0: + all_icd_codes.append([self.MISSING_TEXT_TOKEN]) + all_icd_times.append(self.MISSING_FLOAT_TOKEN) + + single_patient_longitudinal_record = { + "patient_id": patient.patient_id, + "icd_codes": (all_icd_times, all_icd_codes), + "labs": (all_lab_times, all_lab_values), + "labs_mask": (all_lab_times, all_lab_masks), + "mortality": mortality_label, + "window_start": effective_start, + "window_end": effective_end, + } + + return [single_patient_longitudinal_record] + + +class NotesLabsMIMIC4(BaseMultimodalMIMIC4Task): + """Mortality prediction from admission-context notes and lab values. + + Follows the approach of Lee et al. (2023): use text that is clinically + available *at admission* rather than discharge notes. ICD codes are + excluded by default but can be re-enabled for ablation experiments. + + Text is extracted from the MIMIC-IV discharge note by parsing the Chief + Complaint, History of Present Illness, Past Medical History, and Medications + on Admission sections — all of which describe the patient's state at the + start of the stay. The extracted text is assigned timestamp 0.0. + + Radiology reports are also included, parsed for their Indication and + Impression sections and bounded to the same observation window as labs + (rather than timestamp 0.0), since — unlike the discharge summary — they + are written at exam time and describe findings from later in the stay. + + Fields: + admission_note_times: Admission-context discharge-note text at time + 0.0, plus in-window radiology note text at its exam-relative + timestamp. + labs: 10-dim lab vectors at each measurement timestamp. + labs_mask: Boolean observation mask parallel to ``labs``. + icd_codes: (only when ``include_icd=True``) Diagnosis + procedure codes + per admission with inter-admission time offsets. + + Args: + window_hours: Hours from admission for lab collection. ``None`` + collects for the full admission span. Default: 24. + include_icd: When ``True``, collect discharge-coded ICD codes and add + ``icd_codes`` to the sample dict / input schema. Default: ``False``. + """ + + PADDING: int = 0 + + task_name: str = "NotesLabsMIMIC4" + + _BASE_INPUT_SCHEMA: ClassVar[Dict] = { + "admission_note_times": ( + "tuple_time_text", + { + "tokenizer_model": "emilyalsentzer/Bio_ClinicalBERT", + "type_tag": "note", + }, + ), + "labs": ("stagenet_tensor", {}), + "labs_mask": ("stagenet_tensor", {}), + } + + input_schema: Dict[str, Union[str, Tuple[str, Dict]]] = _BASE_INPUT_SCHEMA + output_schema: Dict[str, str] = {"mortality": "binary"} + + def __init__( + self, + window_hours: Optional[float] = None, + include_icd: bool = False, + ) -> None: + super().__init__(window_hours=window_hours) + self.include_icd = include_icd + schema = dict(self._BASE_INPUT_SCHEMA) + if include_icd: + schema["icd_codes"] = ("stagenet", {"padding": self.PADDING}) + self.input_schema = schema + logger.info( + "NotesLabsMIMIC4: filtering discharge notes to sections: %s; " + "radiology notes to sections: %s", + self.DISCHARGE_CLINICAL_HEADERS, + self.RADIOLOGY_CLINICAL_HEADERS, + ) + + def __call__(self, patient: Any) -> List[Dict[str, Any]]: + if not patient.get_events(event_type="patients"): + return [] + + admissions_to_process, mortality_label = self._build_admissions_to_process( + patient + ) + if not admissions_to_process: + return [] + + effective_start, effective_end = self._compute_effective_window( + admissions_to_process + ) + + all_note_texts: List[str] = [] + all_note_times: List[float] = [] + all_lab_values: List[List[float]] = [] + all_lab_masks: List[List[bool]] = [] + all_lab_times: List[float] = [] + all_icd_codes: List[List[str]] = [] + all_icd_times: List[float] = [] + previous_admission_time = None + + for admission in admissions_to_process: + admission_time = admission.timestamp + + try: + admission_dischtime = datetime.strptime( + admission.dischtime, "%Y-%m-%d %H:%M:%S" + ) + except (ValueError, AttributeError): + admission_dischtime = admission_time + if admission_dischtime < admission_time: + admission_dischtime = admission_time + + note_texts, note_times = self._collect_notes( + patient, + "discharge", + admission.hadm_id, + admission_time, + section_headers=self.DISCHARGE_CLINICAL_HEADERS, + ) + all_note_texts.extend(note_texts) + all_note_times.extend(note_times) + + # Labs within the observation window + lab_end = ( + effective_end + if self.window_hours is not None + else admission_dischtime + ) + lab_times, lab_values, lab_masks = self._collect_labs( + patient=patient, + admission_time=admission_time, + end_time=lab_end, + ) + all_lab_times.extend(lab_times) + all_lab_values.extend(lab_values) + all_lab_masks.extend(lab_masks) + + # Radiology notes within the observation window. Unlike the + # discharge note (a retrospective summary parsed for its + # admission-context sections), radiology reports are written at + # exam time, so they're bounded to the same window as labs + # to avoid pulling in findings from later in the stay. + radiology_texts, radiology_times = self._collect_notes( + patient, + "radiology", + admission.hadm_id, + admission_time, + start_time=admission_time, + end_time=lab_end, + section_headers=self.RADIOLOGY_CLINICAL_HEADERS, + ) + all_note_texts.extend(radiology_texts) + all_note_times.extend(radiology_times) + + if self.include_icd: + visit_icd_codes = self._collect_icd_codes(patient, admission.hadm_id) + time_from_previous = ( + 0.0 + if previous_admission_time is None + else self._to_hours( + (admission_time - previous_admission_time).total_seconds() + ) + ) + if visit_icd_codes: + all_icd_codes.append(visit_icd_codes) + all_icd_times.append(time_from_previous) + else: + all_icd_codes.append([self.MISSING_TEXT_TOKEN]) + all_icd_times.append(self.MISSING_FLOAT_TOKEN) + previous_admission_time = admission_time + + if not all_lab_values: + all_lab_values.append( + [self.MISSING_FLOAT_TOKEN] * len(self.LAB_CATEGORY_NAMES) + ) + all_lab_masks.append([False] * len(self.LAB_CATEGORY_NAMES)) + all_lab_times.append(self.MISSING_FLOAT_TOKEN) + + if not all_note_texts: + all_note_texts = [self.MISSING_TEXT_TOKEN] + all_note_times = [self.MISSING_FLOAT_TOKEN] + + record: Dict[str, Any] = { + "patient_id": patient.patient_id, + "admission_note_times": (all_note_texts, all_note_times), + "labs": (all_lab_times, all_lab_values), + "labs_mask": (all_lab_times, all_lab_masks), + "mortality": mortality_label, + "window_start": effective_start, + "window_end": effective_end, + } + + if self.include_icd: + if not all_icd_codes: + all_icd_codes.append([self.MISSING_TEXT_TOKEN]) + all_icd_times.append(self.MISSING_FLOAT_TOKEN) + record["icd_codes"] = (all_icd_times, all_icd_codes) + + return [record] + + +class NotesLabsCXRMIMIC4(BaseMultimodalMIMIC4Task): + """Mortality prediction from admission-context notes, labs, and CXR. + + Extends ``NotesLabsMIMIC4`` with chest X-ray images: the same + admission-context discharge-note sections, in-window radiology reports, + and labs, plus CXR studies (StudyDate+StudyTime from the ``metadata`` + event table) bounded to the same observation window as labs/radiology. + ICD codes are excluded by default but can be re-enabled for ablation + experiments, same as ``NotesLabsMIMIC4``. + + Fields: + admission_note_times: Admission-context discharge-note text at time + 0.0, plus in-window radiology note text at its exam-relative + timestamp. + labs: 10-dim lab vectors at each measurement timestamp. + labs_mask: Boolean observation mask parallel to ``labs``. + cxr_image_times: In-window CXR image paths at their exam-relative + timestamp. + icd_codes: (only when ``include_icd=True``) Diagnosis + procedure codes + per admission with inter-admission time offsets. + + Args: + window_hours: Hours from admission for lab/CXR collection. + ``None`` (default) collects for the full admission span. + include_icd: When ``True``, collect discharge-coded ICD codes and add + ``icd_codes`` to the sample dict / input schema. Default: ``False``. + """ + + PADDING: int = 0 + + task_name: str = "NotesLabsCXRMIMIC4" + + _BASE_INPUT_SCHEMA: ClassVar[Dict] = { + "admission_note_times": ( + "tuple_time_text", + { + "tokenizer_model": "emilyalsentzer/Bio_ClinicalBERT", + "type_tag": "note", + }, + ), + "labs": ("stagenet_tensor", {}), + "labs_mask": ("stagenet_tensor", {}), + "cxr_image_times": ( + "time_image", + { + "image_size": 224, + "mode": "RGB", + "padding": "", + }, + ), + } + + input_schema: Dict[str, Union[str, Tuple[str, Dict]]] = _BASE_INPUT_SCHEMA + output_schema: Dict[str, str] = {"mortality": "binary"} + + def __init__( + self, + window_hours: Optional[float] = None, + include_icd: bool = False, + ) -> None: + super().__init__(window_hours=window_hours) + self.include_icd = include_icd + schema = dict(self._BASE_INPUT_SCHEMA) + if include_icd: + schema["icd_codes"] = ("stagenet", {"padding": self.PADDING}) + self.input_schema = schema + logger.info( + "NotesLabsCXRMIMIC4: filtering discharge notes to sections: %s; " + "radiology notes to sections: %s", + self.DISCHARGE_CLINICAL_HEADERS, + self.RADIOLOGY_CLINICAL_HEADERS, + ) + + def __call__(self, patient: Any) -> List[Dict[str, Any]]: + if not patient.get_events(event_type="patients"): + return [] + + admissions_to_process, mortality_label = self._build_admissions_to_process( + patient + ) + if not admissions_to_process: + return [] + + effective_start, effective_end = self._compute_effective_window( + admissions_to_process + ) + + all_note_texts: List[str] = [] + all_note_times: List[float] = [] + all_lab_values: List[List[float]] = [] + all_lab_masks: List[List[bool]] = [] + all_lab_times: List[float] = [] + all_icd_codes: List[List[str]] = [] + all_icd_times: List[float] = [] + all_cxr_paths: List[str] = [] + all_cxr_times: List[float] = [] + previous_admission_time = None + + for admission in admissions_to_process: + admission_time = admission.timestamp + + # Skip admissions that start at or after the observation window + # closes, prevents Polars searchsorted OverflowError in CXR lookup. + if effective_end is not None and admission_time >= effective_end: + continue + + try: + admission_dischtime = datetime.strptime( + admission.dischtime, "%Y-%m-%d %H:%M:%S" + ) + except (ValueError, AttributeError): + admission_dischtime = admission_time + if admission_dischtime < admission_time: + admission_dischtime = admission_time + + note_texts, note_times = self._collect_notes( + patient, + "discharge", + admission.hadm_id, + admission_time, + section_headers=self.DISCHARGE_CLINICAL_HEADERS, + ) + all_note_texts.extend(note_texts) + all_note_times.extend(note_times) + + # Labs within the observation window + lab_end = ( + effective_end + if self.window_hours is not None + else admission_dischtime + ) + lab_times, lab_values, lab_masks = self._collect_labs( + patient=patient, + admission_time=admission_time, + end_time=lab_end, + ) + all_lab_times.extend(lab_times) + all_lab_values.extend(lab_values) + all_lab_masks.extend(lab_masks) + + # Radiology notes within the observation window. Unlike the + # discharge note (a retrospective summary parsed for its + # admission-context sections), radiology reports are written at + # exam time, so they're bounded to the same window as labs/CXR + # to avoid pulling in findings from later in the stay. + radiology_texts, radiology_times = self._collect_notes( + patient, + "radiology", + admission.hadm_id, + admission_time, + start_time=admission_time, + end_time=lab_end, + section_headers=self.RADIOLOGY_CLINICAL_HEADERS, + ) + all_note_texts.extend(radiology_texts) + all_note_times.extend(radiology_times) + + # CXR studies within the same observation window as labs/radiology. + # CXR metadata is filtered by timestamp; this includes StudyTime. + metadata_events = patient.get_events( + event_type="metadata", + start=admission_time, + end=lab_end, + ) + for event in metadata_events: + try: + if event.image_path: + all_cxr_paths.append(event.image_path) + all_cxr_times.append( + self._to_hours( + (event.timestamp - admission_time).total_seconds() + ) + ) + except AttributeError: + continue + + if self.include_icd: + visit_icd_codes = self._collect_icd_codes(patient, admission.hadm_id) + time_from_previous = ( + 0.0 + if previous_admission_time is None + else self._to_hours( + (admission_time - previous_admission_time).total_seconds() + ) + ) + if visit_icd_codes: + all_icd_codes.append(visit_icd_codes) + all_icd_times.append(time_from_previous) + else: + all_icd_codes.append([self.MISSING_TEXT_TOKEN]) + all_icd_times.append(self.MISSING_FLOAT_TOKEN) + previous_admission_time = admission_time + + if not all_lab_values: + all_lab_values.append( + [self.MISSING_FLOAT_TOKEN] * len(self.LAB_CATEGORY_NAMES) + ) + all_lab_masks.append([False] * len(self.LAB_CATEGORY_NAMES)) + all_lab_times.append(self.MISSING_FLOAT_TOKEN) + + if not all_note_texts: + all_note_texts = [self.MISSING_TEXT_TOKEN] + all_note_times = [self.MISSING_FLOAT_TOKEN] + + # time_image processor expects at least one path/time pair. + if len(all_cxr_paths) == 0: + all_cxr_paths = [self.MISSING_TEXT_TOKEN] + all_cxr_times = [self.MISSING_FLOAT_TOKEN] + + record: Dict[str, Any] = { + "patient_id": patient.patient_id, + "admission_note_times": (all_note_texts, all_note_times), + "labs": (all_lab_times, all_lab_values), + "labs_mask": (all_lab_times, all_lab_masks), + "cxr_image_times": (all_cxr_paths, all_cxr_times), + "mortality": mortality_label, + "window_start": effective_start, + "window_end": effective_end, + } + + if self.include_icd: + if not all_icd_codes: + all_icd_codes.append([self.MISSING_TEXT_TOKEN]) + all_icd_times.append(self.MISSING_FLOAT_TOKEN) + record["icd_codes"] = (all_icd_times, all_icd_codes) + + return [record] + + +class LabsMIMIC4(BaseMultimodalMIMIC4Task): + """EHR-only mortality prediction using lab values — no notes, no ICD codes. + + Serves as the structured-EHR reference baseline for multimodal ablations. + Collecting only ``labevents`` keeps the dataset loader fast and avoids any + leakage from discharge-coded ICD tables. + + Schema mirrors the ``labs`` / ``labs_mask`` fields from ``NotesLabsMIMIC4`` + so the same backbone models (MLP, RNN, Transformer, etc.) work unchanged. + + Args: + window_hours: Hours from admission to collect lab measurements. + ``None`` collects for the full admission span. Default: 24. + """ + + PADDING: int = 0 + + task_name: str = "LabsMIMIC4" + + input_schema: ClassVar[Dict] = { + "labs": ("stagenet_tensor", {}), + "labs_mask": ("stagenet_tensor", {}), + } + output_schema: ClassVar[Dict] = {"mortality": "binary"} + + def __init__(self, window_hours: Optional[float] = 24) -> None: + super().__init__() + self.window_hours = window_hours + + def __call__(self, patient: Any) -> List[Dict[str, Any]]: # type: ignore[override] + admissions_to_process, mortality_label = self._build_admissions_to_process( + patient + ) + if not admissions_to_process: + return [] + + effective_start, effective_end = self._compute_effective_window( + admissions_to_process + ) + + all_lab_times: List[float] = [] + all_lab_values: List[List[float]] = [] + all_lab_masks: List[List[bool]] = [] + + for admission in admissions_to_process: + admission_time = admission.timestamp + + try: + admission_dischtime = datetime.strptime( + admission.dischtime, "%Y-%m-%d %H:%M:%S" + ) + except (ValueError, AttributeError): + admission_dischtime = admission_time + if admission_dischtime < admission_time: + admission_dischtime = admission_time + + lab_times, lab_values, lab_masks = self._collect_labs( + patient=patient, + admission_time=admission_time, + end_time=admission_dischtime, + ) + all_lab_times.extend(lab_times) + all_lab_values.extend(lab_values) + all_lab_masks.extend(lab_masks) + + if len(all_lab_values) == 0: + all_lab_values.append( + [self.MISSING_FLOAT_TOKEN] * len(self.LAB_CATEGORY_NAMES) + ) + all_lab_masks.append([False] * len(self.LAB_CATEGORY_NAMES)) + all_lab_times.append(self.MISSING_FLOAT_TOKEN) + + single_patient_longitudinal_record = { + "patient_id": patient.patient_id, + "labs": (all_lab_times, all_lab_values), + "labs_mask": (all_lab_times, all_lab_masks), + "mortality": mortality_label, + "window_start": effective_start, + "window_end": effective_end, + } + + return [single_patient_longitudinal_record] + + +class CXRMIMIC4(BaseMultimodalMIMIC4Task): + """CXR-only mortality prediction using chest X-ray images. + + Serves as the imaging-only reference baseline for multimodal ablations — + no notes, no ICD codes, no labs — isolating the chest X-ray + modality the same way ``LabsMIMIC4`` isolates labs. + + CXR studies are filtered by timestamp (StudyDate+StudyTime, from the + ``metadata`` event table) within each admission's observation window. + + Args: + window_hours: Hours from admission to collect CXR studies. ``None`` + collects for the full admission span. Default: None. + """ + + task_name: str = "CXRMIMIC4" + + input_schema: ClassVar[Dict] = { + "cxr_image_times": ( + "time_image", + { + "image_size": 224, + "mode": "RGB", + "padding": "", + }, + ), + } + output_schema: ClassVar[Dict] = {"mortality": "binary"} + + def __call__(self, patient: Any) -> List[Dict[str, Any]]: # type: ignore[override] + admissions_to_process, mortality_label = self._build_admissions_to_process( + patient + ) + if not admissions_to_process: + return [] + + effective_start, effective_end = self._compute_effective_window( + admissions_to_process + ) + + all_cxr_paths: List[str] = [] + all_cxr_times: List[float] = [] + + for admission in admissions_to_process: + admission_time = admission.timestamp + + # Skip admissions that start at or after the observation window + # closes, prevents Polars searchsorted OverflowError. + if effective_end is not None and admission_time >= effective_end: + continue + + try: + admission_dischtime = datetime.strptime( + admission.dischtime, "%Y-%m-%d %H:%M:%S" + ) + except (ValueError, AttributeError): + admission_dischtime = admission_time + if admission_dischtime < admission_time: + admission_dischtime = admission_time + + admission_end = admission_dischtime + if effective_end is not None and effective_end < admission_end: + admission_end = effective_end + + # CXR metadata is filtered by timestamp; this includes StudyTime. + metadata_events = patient.get_events( + event_type="metadata", + start=admission_time, + end=admission_end, + ) + for event in metadata_events: + try: + if event.image_path: + all_cxr_paths.append(event.image_path) + all_cxr_times.append( + self._to_hours( + (event.timestamp - admission_time).total_seconds() + ) + ) + except AttributeError: + continue + + # time_image processor expects at least one path/time pair. + if len(all_cxr_paths) == 0: + all_cxr_paths = [self.MISSING_TEXT_TOKEN] + all_cxr_times = [self.MISSING_FLOAT_TOKEN] + + single_patient_longitudinal_record = { + "patient_id": patient.patient_id, + "cxr_image_times": (all_cxr_paths, all_cxr_times), + "mortality": mortality_label, + "window_start": effective_start, + "window_end": effective_end, + } + + return [single_patient_longitudinal_record] \ No newline at end of file diff --git a/pyhealth/trainer.py b/pyhealth/trainer.py index bc6a28677..8a86cc709 100644 --- a/pyhealth/trainer.py +++ b/pyhealth/trainer.py @@ -1,5 +1,7 @@ +import json import logging import os +import time from datetime import datetime from typing import Callable, Dict, List, Optional, Type @@ -37,6 +39,15 @@ def set_logger(log_path: str) -> None: return +def _vram_stats(device: str) -> Dict[str, float]: + """Returns current and peak VRAM usage in MB for a CUDA device.""" + if not torch.cuda.is_available() or not str(device).startswith("cuda"): + return {} + allocated = torch.cuda.memory_allocated(device) / 1024**2 + peak = torch.cuda.max_memory_allocated(device) / 1024**2 + return {"vram_allocated_mb": allocated, "vram_peak_mb": peak} + + def get_metrics_fn(mode: str) -> Callable: if mode == "binary": return binary_metrics_fn @@ -126,6 +137,9 @@ def train( monitor_criterion: str = "max", load_best_model_at_last: bool = True, patience=None, + accumulation_steps: int = 1, + use_amp: bool = False, + amp_dtype: str = "bf16", ): """Trains the model. @@ -145,10 +159,25 @@ def train( Default is True. patience: Number of epochs to wait for improvement before early stopping. Default is None, which means no early stopping. + accumulation_steps: Gradient accumulation steps to simulate a larger + effective batch size. Default is 1 (no accumulation). + use_amp: Whether to use automatic mixed precision. Default is False. + amp_dtype: AMP dtype — "bf16" (stable, recommended) or "fp16". + Default is "bf16". """ if optimizer_params is None: optimizer_params = {"lr": 1e-3} + _amp_dtype = ( + torch.bfloat16 if amp_dtype == "bf16" else torch.float16 + ) + # GradScaler only needed for fp16; bf16 has fp32 dynamic range + scaler = ( + torch.cuda.amp.GradScaler() + if (use_amp and _amp_dtype == torch.float16) + else None + ) + # logging logger.info("Training:") logger.info(f"Batch size: {train_dataloader.batch_size}") @@ -161,6 +190,8 @@ def train( logger.info(f"Monitor criterion: {monitor_criterion}") logger.info(f"Epochs: {epochs}") logger.info(f"Patience: {patience}") + logger.info(f"Accumulation steps: {accumulation_steps}") + logger.info(f"AMP: {use_amp} (dtype={amp_dtype})") # set optimizer param = list(self.model.named_parameters()) @@ -184,50 +215,129 @@ def train( steps_per_epoch = len(train_dataloader) global_step = 0 patience_counter = 0 + metrics_history: List[Dict] = [] + train_start = time.perf_counter() + total_skipped_steps = 0 # epoch training loop - for epoch in range(epochs): + epoch_iterator = tqdm(range(epochs), desc="Epochs", unit="epoch") + for epoch in epoch_iterator: + epoch_iterator.set_postfix_str(f"{epoch + 1}/{epochs}", refresh=False) training_loss = [] + epoch_skipped_steps = 0 self.model.zero_grad() self.model.train() + if torch.cuda.is_available() and str(self.device).startswith("cuda"): + torch.cuda.reset_peak_memory_stats(self.device) + epoch_start = time.perf_counter() # batch training loop logger.info("") - for _ in trange( + for step_idx in trange( steps_per_epoch, - desc=f"Epoch {epoch} / {epochs}", + desc=f"Epoch {epoch + 1}/{epochs}", smoothing=0.05, + leave=False, ): try: data = next(data_iterator) except StopIteration: data_iterator = iter(train_dataloader) data = next(data_iterator) - # forward - output = self.model(**data) - loss = output["loss"] + # forward (with optional AMP) + if use_amp: + with torch.autocast(device_type="cuda", dtype=_amp_dtype): + output = self.model(**data) + loss = output["loss"] / accumulation_steps + else: + output = self.model(**data) + loss = output["loss"] / accumulation_steps # backward - loss.backward() - if max_grad_norm is not None: - torch.nn.utils.clip_grad_norm_( - self.model.parameters(), max_grad_norm + if scaler is not None: + scaler.scale(loss).backward() + else: + loss.backward() + training_loss.append(loss.item() * accumulation_steps) + # optimizer step every accumulation_steps batches or epoch end + is_update_step = ( + (step_idx + 1) % accumulation_steps == 0 + or (step_idx + 1) == steps_per_epoch + ) + if is_update_step: + if scaler is not None: + scaler.unscale_(optimizer) + # Always compute the grad norm (even with no clipping + # configured) so non-finite gradients can be detected and + # skipped before they permanently poison the model with + # NaN weights. + grad_norm = torch.nn.utils.clip_grad_norm_( + self.model.parameters(), + max_grad_norm if max_grad_norm is not None else float("inf"), ) - # update - optimizer.step() - optimizer.zero_grad() - training_loss.append(loss.item()) - global_step += 1 + step_ok = bool(torch.isfinite(grad_norm)) + if not step_ok: + epoch_skipped_steps += 1 + total_skipped_steps += 1 + logger.warning( + f"epoch-{epoch} step-{global_step}: non-finite " + f"gradient norm ({grad_norm}); skipping optimizer " + f"step." + ) + if scaler is not None: + if step_ok: + scaler.step(optimizer) + scaler.update() + elif step_ok: + optimizer.step() + optimizer.zero_grad() + global_step += 1 + + epoch_time = time.perf_counter() - epoch_start + vram = _vram_stats(self.device) + + epochs_done = epoch + 1 + epochs_left = epochs - epochs_done + elapsed_total = time.perf_counter() - train_start + avg_epoch_time = elapsed_total / epochs_done + eta_s = avg_epoch_time * epochs_left + eta_h, eta_rem = divmod(int(eta_s), 3600) + eta_m = eta_rem // 60 + eta_str = f"{eta_h}h{eta_m:02d}m" + # log and save logger.info(f"--- Train epoch-{epoch}, step-{global_step} ---") logger.info(f"loss: {sum(training_loss) / len(training_loss):.4f}") + logger.info(f"epoch_time: {epoch_time:.2f}s elapsed: {elapsed_total:.0f}s ETA: {eta_str} ({epochs_done}/{epochs} epochs)") + print(f"[ETA] epoch {epochs_done}/{epochs} done in {epoch_time:.0f}s — ETA to finish: {eta_str}", flush=True) + if vram: + logger.info( + f"vram_peak: {vram['vram_peak_mb']:.1f} MB " + f"vram_current: {vram['vram_allocated_mb']:.1f} MB" + ) + if epoch_skipped_steps > 0: + logger.warning( + f"Skipped {epoch_skipped_steps} optimizer step(s) this " + f"epoch due to non-finite gradients (total so far: " + f"{total_skipped_steps})." + ) if self.exp_path is not None: self.save_ckpt(os.path.join(self.exp_path, "last.ckpt")) + epoch_record: Dict = { + "epoch": epoch, + "global_step": global_step, + "train_loss": sum(training_loss) / len(training_loss), + "epoch_time_s": round(epoch_time, 3), + "skipped_steps": epoch_skipped_steps, + **{f"train_{k}": v for k, v in vram.items()}, + } + # validation if val_dataloader is not None: scores = self.evaluate(val_dataloader) logger.info(f"--- Eval epoch-{epoch}, step-{global_step} ---") for key in scores.keys(): logger.info("{}: {:.4f}".format(key, scores[key])) + epoch_record.update({f"val_{k}": v for k, v in scores.items()}) # save best model if monitor is not None: score = scores[monitor] @@ -247,8 +357,21 @@ def train( logger.info( f"Early stopping at epoch-{epoch}, step-{global_step}" ) + metrics_history.append(epoch_record) break + metrics_history.append(epoch_record) + + total_time = time.perf_counter() - train_start + logger.info(f"--- Training complete: {total_time:.2f}s total ---") + + # persist metrics history + if self.exp_path is not None: + history_path = os.path.join(self.exp_path, "metrics_history.json") + with open(history_path, "w") as f: + json.dump(metrics_history, f, indent=2) + logger.info(f"Metrics history saved to {history_path}") + # load best model if load_best_model_at_last and self.exp_path is not None and os.path.isfile( os.path.join(self.exp_path, "best.ckpt")): @@ -262,7 +385,7 @@ def train( for key in scores.keys(): logger.info("{}: {:.4f}".format(key, scores[key])) - return + return metrics_history def inference(self, dataloader, additional_outputs=None, return_patient_ids=False) -> Dict[str, float]: @@ -430,4 +553,4 @@ def forward(self, x, y, **kwargs): monitor="accuracy", epochs=5, test_dataloader=val_dataloader, - ) + ) \ No newline at end of file From bc053040ec91e125c6a2a9782d5cd4384157d884 Mon Sep 17 00:00:00 2001 From: William Pang Date: Tue, 11 Aug 2026 01:45:04 -0700 Subject: [PATCH 22/61] Updates --- .../will/condor/labs_only/labs_only_rnn.sub | 0 .../will/condor/labs_only/run_labs_only_rnn.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {pyhealth/scripts_delete_me => scripts}/will/condor/labs_only/labs_only_rnn.sub (100%) rename {pyhealth/scripts_delete_me => scripts}/will/condor/labs_only/run_labs_only_rnn.sh (100%) diff --git a/pyhealth/scripts_delete_me/will/condor/labs_only/labs_only_rnn.sub b/scripts/will/condor/labs_only/labs_only_rnn.sub similarity index 100% rename from pyhealth/scripts_delete_me/will/condor/labs_only/labs_only_rnn.sub rename to scripts/will/condor/labs_only/labs_only_rnn.sub diff --git a/pyhealth/scripts_delete_me/will/condor/labs_only/run_labs_only_rnn.sh b/scripts/will/condor/labs_only/run_labs_only_rnn.sh similarity index 100% rename from pyhealth/scripts_delete_me/will/condor/labs_only/run_labs_only_rnn.sh rename to scripts/will/condor/labs_only/run_labs_only_rnn.sh From 5483dee99dd04302fd31ef5996cb18862538b38b Mon Sep 17 00:00:00 2001 From: William Pang Date: Tue, 11 Aug 2026 03:59:00 -0500 Subject: [PATCH 23/61] New Changes --- scripts/will/condor/labs_only/labs_only_rnn.sub | 4 ++-- scripts/will/condor/labs_only/run_labs_only_rnn.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) mode change 100644 => 100755 scripts/will/condor/labs_only/run_labs_only_rnn.sh diff --git a/scripts/will/condor/labs_only/labs_only_rnn.sub b/scripts/will/condor/labs_only/labs_only_rnn.sub index b0768f201..e7d66a1b1 100644 --- a/scripts/will/condor/labs_only/labs_only_rnn.sub +++ b/scripts/will/condor/labs_only/labs_only_rnn.sub @@ -7,14 +7,14 @@ # # To submit (from the project root): # mkdir -p /home/wp14/logs/condor -# condor_submit scripts/will/condor/labs_only_rnn.sub +# condor_submit scripts/will/condor/labs_only/labs_only_rnn.sub # # Monitor: # condor_q # tail -f /home/wp14/logs/condor/labs_only_rnn__0.out initialdir = /home/wp14/PyHealth -executable = /home/wp14/PyHealth/scripts/will/condor/run_labs_only_rnn.sh +executable = /home/wp14/PyHealth/scripts/will/condor/labs_only/run_labs_only_rnn.sh transfer_executable = False arguments = $(seed) getenv = True diff --git a/scripts/will/condor/labs_only/run_labs_only_rnn.sh b/scripts/will/condor/labs_only/run_labs_only_rnn.sh old mode 100644 new mode 100755 index 649eaccb1..e05a8895f --- a/scripts/will/condor/labs_only/run_labs_only_rnn.sh +++ b/scripts/will/condor/labs_only/run_labs_only_rnn.sh @@ -20,7 +20,7 @@ CACHE_DIR="${CACHE_DIR:-/shared/eng/wp14/pyhealth_cache_labs}" OUTPUT_DIR="${OUTPUT_DIR:-/home/wp14/output}" CONDA_SH="${CONDA_SH:-}" -DEV_MODE="${DEV_MODE:-0}" +DEV_MODE="${DEV_MODE:-1}" EMBEDDING_DIM="${EMBEDDING_DIM:-64}" HIDDEN_DIM="${HIDDEN_DIM:-64}" RNN_TYPE="${RNN_TYPE:-GRU}" From a0f1422e4c885254ac2f9301c68f5689ac501b8e Mon Sep 17 00:00:00 2001 From: William Pang Date: Tue, 11 Aug 2026 04:21:41 -0500 Subject: [PATCH 24/61] New updates --- pyhealth/datasets/base_dataset.py | 187 ++++++++++---------------- pyhealth/models/embedding/__init__.py | 33 +++++ 2 files changed, 101 insertions(+), 119 deletions(-) create mode 100644 pyhealth/models/embedding/__init__.py diff --git a/pyhealth/datasets/base_dataset.py b/pyhealth/datasets/base_dataset.py index 3d449d579..5618f4e9c 100644 --- a/pyhealth/datasets/base_dataset.py +++ b/pyhealth/datasets/base_dataset.py @@ -3,7 +3,7 @@ import pickle from abc import ABC from pathlib import Path -from typing import Dict, Iterator, Iterable, List, Optional, Any, Callable +from typing import Dict, Iterator, Iterable, List, Optional, Any, Callable, Union import functools import operator from urllib.parse import urlparse, urlunparse @@ -84,7 +84,13 @@ def path_exists(path: str) -> bool: except requests.RequestException: return False else: - return Path(path).exists() + try: + return Path(path).exists() + except OSError: + # Treat unreadable paths (e.g. stale/corrupted filesystem + # entries that raise I/O errors on stat) as non-existent so + # callers can fall back to an alternate extension. + return False def _csv_tsv_gz_path(path: str) -> str: @@ -314,16 +320,7 @@ class BaseDataset(ABC): dataset_name (str): Name of the dataset. config (dict): Configuration loaded from a YAML file. global_event_df (pl.LazyFrame): The global event data frame. - dev (bool): Whether to enable dev mode (limit to 1000 patients). - - Examples: - >>> from pyhealth.datasets import BaseDataset - >>> dataset = BaseDataset( - ... root="/path/to/source", - ... tables=["patients", "diagnoses"], - ... config_path="/path/to/config.yaml", - ... ) - >>> dataset.stats() + dev (Union[bool, int]): Whether to enable dev mode. If True, limit to 1000 patients. If an int, limit to that many patients. """ def __init__( @@ -334,7 +331,7 @@ def __init__( config_path: Optional[str] = None, cache_dir: str | Path | None = None, num_workers: int = 1, - dev: bool = False, + dev: Union[bool, int] = False, ): """Initializes the BaseDataset. @@ -351,7 +348,7 @@ def __init__( - **str** or **Path**: Used as the root cache directory path. A UUID is appended to the provided path to capture dataset configuration. num_workers (int): Number of worker processes for parallel operations. - dev (bool): Whether to run in dev mode (limits to 1000 patients). + dev (Union[bool, int]): Whether to run in dev mode. If True, limits to 1000 patients. If an int, limits to that many patients. """ if len(set(tables)) != len(tables): logger.warning("Duplicate table names in tables list. Removing duplicates.") @@ -434,68 +431,6 @@ def clean_tmpdir(self) -> None: if tmp_dir.exists(): shutil.rmtree(tmp_dir) - def _scan_table(self, source_path: str) -> dd.DataFrame: - """Routes a table source to the appropriate scanner based on its format. - - Parquet sources (``.parquet``/``.pq`` files, glob patterns targeting - such files, or directories of Parquet shards) are handled by - :meth:`_scan_parquet`. Any other source falls back to the existing - CSV/TSV(.gz) scanner, preserving prior behavior for all datasets. - - Args: - source_path (str): Path to the table source. - - Returns: - dd.DataFrame: The Dask DataFrame for the table source. - """ - stripped = source_path.rstrip("/") - if stripped.endswith((".parquet", ".pq")) or ( - not is_url(source_path) and Path(source_path).is_dir() - ): - return self._scan_parquet(source_path) - return self._scan_csv_tsv_gz(source_path) - - def _scan_parquet(self, source_path: str) -> dd.DataFrame: - """Scans a Parquet source and returns a Dask DataFrame. - - The source may be a single ``.parquet``/``.pq`` file, a glob pattern, - or a directory that is scanned recursively — which supports sharded - datasets such as MEDS, laid out as ``data//.parquet``. - - Unlike :meth:`_scan_csv_tsv_gz`, no all-string schema coercion is - applied: Parquet files embed their schema, so source dtypes (native - timestamps, numeric columns, nullable strings) are preserved and - handled downstream by :meth:`load_table`. - - Args: - source_path (str): Path to a Parquet file, directory, or glob. - - Returns: - dd.DataFrame: The Dask DataFrame backed by the Parquet source. - - Raises: - FileNotFoundError: If the source path does not exist, or if a - directory source contains no Parquet files. - """ - path = Path(source_path) - is_glob = any(ch in source_path for ch in "*?[") - if not is_glob: - if not path.exists(): - raise FileNotFoundError( - f"Parquet source does not exist: {source_path}" - ) - if path.is_dir() and not any( - itertools.chain(path.rglob("*.parquet"), path.rglob("*.pq")) - ): - raise FileNotFoundError( - f"Directory contains no Parquet files: {source_path}" - ) - return dd.read_parquet( - source_path, - split_row_groups=True, # type: ignore - blocksize="64MB", - ) - def _scan_csv_tsv_gz(self, source_path: str) -> dd.DataFrame: """Scans a CSV/TSV file (possibly gzipped) and returns a Dask DataFrame. @@ -571,30 +506,52 @@ def _event_transform(self, output_dir: Path) -> None: compute_ok = False try: df = self.load_data() - with DaskCluster( - n_workers=self.num_workers, - threads_per_worker=1, - processes=not in_notebook(), - # Use cache_dir for Dask's scratch space to avoid filling up /tmp or home directory - local_directory=str(self.create_tmpdir()), - ) as cluster: - with DaskClient(cluster) as client: - if self.dev: - logger.info("Dev mode enabled: limiting to 1000 patients") - patients = df["patient_id"].unique().head(1000).tolist() - filter = df["patient_id"].isin(patients) - df = df[filter] - - logger.info(f"Caching event dataframe to {output_dir}...") - collection = df.sort_values("patient_id").to_parquet( - output_dir, - write_index=False, - compute=False, - ) - handle = client.compute(collection) - dask_progress(handle) - handle.result() # type: ignore - compute_ok = True # Data is fully written to disk + disable_distributed = os.environ.get( + "PYHEALTH_DISABLE_DASK_DISTRIBUTED", "0" + ) == "1" + + if disable_distributed: + logger.info( + "PYHEALTH_DISABLE_DASK_DISTRIBUTED=1 detected; using local dask scheduler." + ) + if self.dev: + n = 1000 if self.dev is True else int(self.dev) + logger.info(f"Dev mode enabled: limiting to {n} patients") + patients = df["patient_id"].unique().head(n, compute=True).tolist() + patient_filter = df["patient_id"].isin(patients) + df = df[patient_filter] + + logger.info(f"Caching event dataframe to {output_dir}...") + df.sort_values("patient_id").to_parquet( + output_dir, + write_index=False, + compute=True, + ) + else: + with DaskCluster( + n_workers=self.num_workers, + threads_per_worker=1, + processes=not in_notebook(), + # Use cache_dir for Dask's scratch space to avoid filling up /tmp or home directory + local_directory=str(self.create_tmpdir()), + ) as cluster: + with DaskClient(cluster) as client: + if self.dev: + logger.info(f"Dev mode enabled: limiting to {1000 if self.dev is True else int(self.dev)} patients") + patients = df["patient_id"].unique().head(1000 if self.dev is True else int(self.dev)).tolist() + filter = df["patient_id"].isin(patients) + df = df[filter] + + logger.info(f"Caching event dataframe to {output_dir}...") + collection = df.sort_values("patient_id").to_parquet( + output_dir, + write_index=False, + compute=False, + ) + handle = client.compute(collection) + dask_progress(handle) + handle.result() # type: ignore + compute_ok = True # Data is fully written to disk except TimeoutError: if compute_ok: # Cluster shutdown timed out after successful compute — data is intact @@ -667,8 +624,7 @@ def load_table(self, table_name: str) -> dd.DataFrame: Raises: ValueError: If the table is not found in the config. - FileNotFoundError: If the source file (CSV/TSV or Parquet) for the - table or join is not found. + FileNotFoundError: If the CSV file for the table or join is not found. """ assert self.config is not None, "Config must be provided to load tables" @@ -680,7 +636,7 @@ def load_table(self, table_name: str) -> dd.DataFrame: csv_path = clean_path(csv_path) logger.info(f"Scanning table: {table_name} from {csv_path}") - df = self._scan_table(csv_path) + df = self._scan_csv_tsv_gz(csv_path) # Convert column names to lowercase before calling preprocess_func df = df.rename(columns=str.lower) @@ -699,7 +655,7 @@ def load_table(self, table_name: str) -> dd.DataFrame: other_csv_path = f"{self.root}/{join_cfg.file_path}" other_csv_path = clean_path(other_csv_path) logger.info(f"Joining with table: {other_csv_path}") - join_df = self._scan_table(other_csv_path) + join_df = self._scan_csv_tsv_gz(other_csv_path) join_df = join_df.rename(columns=str.lower) join_key = join_cfg.on columns = join_cfg.columns @@ -723,21 +679,14 @@ def load_table(self, table_name: str) -> dd.DataFrame: timestamp_series: dd.Series = functools.reduce( operator.add, (df[col].astype("string") for col in timestamp_col) ) - timestamp_series = dd.to_datetime( - timestamp_series, - format=timestamp_format, - errors="raise", - ) - elif pd.api.types.is_datetime64_any_dtype(df[timestamp_col].dtype): - # Typed sources (e.g. Parquet) already carry native timestamps: - # skip the string round-trip and only normalize the unit below. - timestamp_series: dd.Series = df[timestamp_col] else: - timestamp_series = dd.to_datetime( - df[timestamp_col].astype("string"), - format=timestamp_format, - errors="raise", - ) + timestamp_series: dd.Series = df[timestamp_col].astype("string") + + timestamp_series: dd.Series = dd.to_datetime( + timestamp_series, + format=timestamp_format, + errors="raise", + ) df: dd.DataFrame = df.assign( timestamp=timestamp_series.astype("datetime64[ms]") ) @@ -1165,4 +1114,4 @@ def _main_guard(self, func_name: str): f"{func_name} method accessed from a non-main process. This may lead to unexpected behavior.\n" + "Consider use __name__ == '__main__' guard when using multiprocessing." ) - exit(1) + exit(1) \ No newline at end of file diff --git a/pyhealth/models/embedding/__init__.py b/pyhealth/models/embedding/__init__.py new file mode 100644 index 000000000..2bc66d73b --- /dev/null +++ b/pyhealth/models/embedding/__init__.py @@ -0,0 +1,33 @@ +"""Embedding models for PyHealth multimodal pipelines. + +All embedding models share the :class:`BaseEmbeddingModel` interface: +they expose an ``embedding_dim`` property and a ``forward`` method that +transforms processor output tensors into dense vector embeddings. + +Available models: + +- :class:`EmbeddingModel` — generic encoder for codes, sequences, timeseries +- :class:`VisionEmbeddingModel` — ViT-style patch encoder for medical images (Josh) +- :class:`UnifiedMultimodalEmbeddingModel` — temporally-aligned multi-modal encoder + +Helper utilities: + +- :class:`SinusoidalTimeEmbedding` — continuous time positional encoding +- :func:`init_embedding_with_pretrained` — load GloVe-style pretrained vectors +""" + +from .base import BaseEmbeddingModel +from .vanilla import EmbeddingModel, init_embedding_with_pretrained +from .vision import VisionEmbeddingModel, PatchEmbedding, Permute +from .unified import UnifiedMultimodalEmbeddingModel, SinusoidalTimeEmbedding + +__all__ = [ + "BaseEmbeddingModel", + "EmbeddingModel", + "VisionEmbeddingModel", + "PatchEmbedding", + "Permute", + "UnifiedMultimodalEmbeddingModel", + "SinusoidalTimeEmbedding", + "init_embedding_with_pretrained", +] \ No newline at end of file From b80f7efa29e1208715e6cba317927b359a2ceee6 Mon Sep 17 00:00:00 2001 From: William Pang Date: Tue, 11 Aug 2026 04:42:10 -0500 Subject: [PATCH 25/61] New Updates --- pyhealth/models/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyhealth/models/__init__.py b/pyhealth/models/__init__.py index 2f30ae673..7464f7a90 100644 --- a/pyhealth/models/__init__.py +++ b/pyhealth/models/__init__.py @@ -46,7 +46,7 @@ from .text_embedding import TextEmbedding from .sdoh import SdohClassifier from .medlink import MedLink -from .unified_embedding import UnifiedMultimodalEmbeddingModel, SinusoidalTimeEmbedding +from .embedding import UnifiedMultimodalEmbeddingModel, SinusoidalTimeEmbedding from .califorest import CaliForest from .generators.halo import HALO from .generators.gpt2 import GPT2 From f0187f929c7c5cce5b5861eb6d3a68aec9523abe Mon Sep 17 00:00:00 2001 From: William Pang Date: Tue, 11 Aug 2026 05:26:38 -0500 Subject: [PATCH 26/61] New Updates --- .../datasets/configs/mimic4_cxr_sunlab.yaml | 105 ++++++++++ pyhealth/datasets/mimic4.py | 138 ++++++++++-- .../labs_notes_cxr/labs_notes_cxr_rnn.sub | 61 ++++++ .../labs_notes_cxr/run_labs_notes_cxr_rnn.sh | 197 ++++++++++++++++++ 4 files changed, 489 insertions(+), 12 deletions(-) create mode 100644 pyhealth/datasets/configs/mimic4_cxr_sunlab.yaml create mode 100644 scripts/will/condor/labs_notes_cxr/labs_notes_cxr_rnn.sub create mode 100755 scripts/will/condor/labs_notes_cxr/run_labs_notes_cxr_rnn.sh diff --git a/pyhealth/datasets/configs/mimic4_cxr_sunlab.yaml b/pyhealth/datasets/configs/mimic4_cxr_sunlab.yaml new file mode 100644 index 000000000..e631de4aa --- /dev/null +++ b/pyhealth/datasets/configs/mimic4_cxr_sunlab.yaml @@ -0,0 +1,105 @@ +version: "2.1.0" +tables: + metadata: + file_path: "mimic-cxr-2.0.0-metadata-pyhealth-sunlab.csv" + patient_id: "subject_id" + timestamp: + - "studydate" + - "studytime" + timestamp_format: "%Y%m%d%H%M%S" + attributes: + - "image_path" + - "dicom_id" + - "study_id" + - "performedprocedurestepdescription" + - "viewposition" + - "rows" + - "columns" + - "procedurecodesequence_codemeaning" + - "viewcodesequence_codemeaning" + - "patientorientationcodesequence_codemeaning" + + chexpert: + file_path: "mimic-cxr-2.0.0-chexpert.csv" + patient_id: "subject_id" + join: + - file_path: "mimic-cxr-2.0.0-metadata-pyhealth-sunlab.csv" + "on": "study_id" + how: "inner" + columns: + - "studydate" + - "studytime" + - "dicom_id" + timestamp: + - "studydate" + - "studytime" + timestamp_format: "%Y%m%d%H%M%S" + attributes: + - "dicom_id" + - "study_id" + - "atelectasis" + - "cardiomegaly" + - "consolidation" + - "edema" + - "enlarged cardiomediastinum" + - "fracture" + - "lung lesion" + - "lung opacity" + - "no finding" + - "pleural effusion" + - "pleural other" + - "pneumonia" + - "pneumothorax" + - "support devices" + + negbio: + file_path: "mimic-cxr-2.0.0-negbio.csv" + patient_id: "subject_id" + join: + - file_path: "mimic-cxr-2.0.0-metadata-pyhealth-sunlab.csv" + "on": "study_id" + how: "inner" + columns: + - "studydate" + - "studytime" + - "dicom_id" + timestamp: + - "studydate" + - "studytime" + timestamp_format: "%Y%m%d%H%M%S" + attributes: + - "dicom_id" + - "study_id" + - "atelectasis" + - "cardiomegaly" + - "consolidation" + - "edema" + - "enlarged cardiomediastinum" + - "fracture" + - "lung lesion" + - "lung opacity" + - "no finding" + - "pleural effusion" + - "pleural other" + - "pneumonia" + - "pneumothorax" + - "support devices" + + split: + file_path: "mimic-cxr-2.0.0-split.csv" + patient_id: "subject_id" + join: + - file_path: "mimic-cxr-2.0.0-metadata-pyhealth-sunlab.csv" + "on": "dicom_id" + how: "inner" + columns: + - "studydate" + - "studytime" + timestamp: + - "studydate" + - "studytime" + timestamp_format: "%Y%m%d%H%M%S" + attributes: + - "dicom_id" + - "study_id" + - "split" \ No newline at end of file diff --git a/pyhealth/datasets/mimic4.py b/pyhealth/datasets/mimic4.py index 9d1aa55d8..f9a7da38c 100644 --- a/pyhealth/datasets/mimic4.py +++ b/pyhealth/datasets/mimic4.py @@ -1,7 +1,7 @@ import logging import os import warnings -from typing import List, Optional +from typing import List, Optional, Union import pandas as pd import dask.dataframe as dd @@ -223,6 +223,102 @@ def process_image_path(x): return +class MIMIC4CXRSunlabDataset(BaseDataset): + """ + Sunlab variant of the MIMIC-CXR Chest X-ray dataset. + + This variant uses the existing metadata CSV and derives flattened image + paths at ``images/{dicom_id}.jpg``. + """ + + def __init__( + self, + root: str, + tables: List[str], + dataset_name: str = "mimic4_cxr_sunlab", + config_path: Optional[str] = None, + cache_dir: Optional[str] = None, + **kwargs, + ): + if config_path is None: + config_path = os.path.join( + os.path.dirname(__file__), "configs", "mimic4_cxr_sunlab.yaml" + ) + logger.info(f"Using default Sunlab CXR config: {config_path}") + self.prepare_metadata(root) + log_memory_usage(f"Before initializing {dataset_name}") + super().__init__( + root=root, + tables=tables, + dataset_name=dataset_name, + config_path=config_path, + cache_dir=cache_dir, + **kwargs, + ) + log_memory_usage(f"After initializing {dataset_name}") + + @staticmethod + def _resolve_column_name(columns: List[str], target: str) -> str: + lower_to_original = {col.lower(): col for col in columns} + resolved = lower_to_original.get(target.lower()) + if resolved is None: + raise ValueError( + f"Expected column '{target}' in metadata, available columns: {columns}" + ) + return resolved + + def prepare_metadata(self, root: str) -> None: + metadata_path = os.path.join(root, "mimic-cxr-2.0.0-metadata.csv") + if not os.path.exists(metadata_path): + raise FileNotFoundError( + f"Sunlab metadata file not found: {metadata_path}. " + "Expected existing metadata linked by dicom_id/subject_id/study_id." + ) + + images_dir = os.path.join(root, "images") + if not os.path.isdir(images_dir): + raise FileNotFoundError( + f"Sunlab images directory not found: {images_dir}. " + "Expected flattened image files at images/{dicom_id}.jpg." + ) + + metadata = pd.read_csv(metadata_path, dtype=str) + + dicom_col = self._resolve_column_name(metadata.columns.tolist(), "dicom_id") + study_time_col = self._resolve_column_name( + metadata.columns.tolist(), "studytime" + ) + + # Normalize StudyTime so timestamps parse with %Y%m%d%H%M%S in config. + def normalize_studytime(value: Optional[str]) -> str: + if value is None: + return "000000" + value_str = str(value).strip() + if value_str == "" or value_str.lower() == "nan": + return "000000" + try: + return f"{int(float(value_str)):06d}" + except Exception: + digits = "".join(ch for ch in value_str if ch.isdigit()) + if digits == "": + return "000000" + return digits[:6].zfill(6) + + metadata[study_time_col] = metadata[study_time_col].apply(normalize_studytime) + + metadata["image_path"] = metadata[dicom_col].apply( + lambda dicom_id: os.path.join(root, "images", f"{dicom_id}.jpg") + ) + + # Align with existing config conventions by using lowercase headers. + metadata.columns = [col.lower() for col in metadata.columns] + + metadata.to_csv( + os.path.join(root, "mimic-cxr-2.0.0-metadata-pyhealth-sunlab.csv"), + index=False, + ) + + class MIMIC4Dataset(BaseDataset): """ Unified MIMIC-IV dataset with support for EHR, clinical notes, and X-rays. @@ -242,6 +338,7 @@ class MIMIC4Dataset(BaseDataset): ehr_config_path: Path to the EHR config file note_config_path: Path to the note config file cxr_config_path: Path to the CXR config file + cxr_variant: Which CXR variant to load ("default" or "sunlab") dataset_name: Name of the dataset dev: Whether to enable dev mode (limit to 1000 patients) @@ -279,8 +376,9 @@ def __init__( ehr_config_path: Optional[str] = None, note_config_path: Optional[str] = None, cxr_config_path: Optional[str] = None, + cxr_variant: str = "default", dataset_name: str = "mimic4", - dev: bool = False, + dev: Union[bool, int] = False, cache_dir: Optional[str] = None, num_workers: int = 1, ): @@ -340,17 +438,33 @@ def __init__( # Initialize CXR dataset if root is provided if cxr_root is not None: + if cxr_variant not in {"default", "sunlab"}: + raise ValueError( + f"Unknown cxr_variant '{cxr_variant}'. " + "Expected one of {'default', 'sunlab'}." + ) + logger.info( - f"Initializing MIMIC4CXRDataset with tables: {cxr_tables} (dev mode: {dev})" - ) - self.sub_datasets["cxr"] = MIMIC4CXRDataset( - root=cxr_root, - tables=cxr_tables, - config_path=cxr_config_path, - cache_dir=str(self.cache_dir), - dev=dev, - num_workers=num_workers, + f"Initializing MIMIC4 CXR variant '{cxr_variant}' with tables: {cxr_tables} (dev mode: {dev})" ) + if cxr_variant == "sunlab": + self.sub_datasets["cxr"] = MIMIC4CXRSunlabDataset( + root=cxr_root, + tables=cxr_tables, + config_path=cxr_config_path, + cache_dir=str(self.cache_dir), + dev=dev, + num_workers=num_workers, + ) + else: + self.sub_datasets["cxr"] = MIMIC4CXRDataset( + root=cxr_root, + tables=cxr_tables, + config_path=cxr_config_path, + cache_dir=str(self.cache_dir), + dev=dev, + num_workers=num_workers, + ) log_memory_usage("After CXR dataset initialization") log_memory_usage("Completed MIMIC4Dataset init") @@ -374,4 +488,4 @@ def load_data(self) -> dd.DataFrame: if len(frames) == 1: return frames[0] else: - return dd.concat(frames, axis=0, join="outer") + return dd.concat(frames, axis=0, join="outer") \ No newline at end of file diff --git a/scripts/will/condor/labs_notes_cxr/labs_notes_cxr_rnn.sub b/scripts/will/condor/labs_notes_cxr/labs_notes_cxr_rnn.sub new file mode 100644 index 000000000..118fa8149 --- /dev/null +++ b/scripts/will/condor/labs_notes_cxr/labs_notes_cxr_rnn.sub @@ -0,0 +1,61 @@ +# HTCondor submission — labs+notes+CXR RNN mortality run +# +# Condor equivalent of +# scripts/will/lambda-labs/labs_notes_cxr/tmux_run_labs_notes_cxr_rnn_variant.py: +# same task (notes_labs_cxr), same model (rnn), same hyperparameters. Runs +# unattended instead of in a tmux session; Condor assigns the GPU (no manual +# nvidia-smi / CUDA_VISIBLE_DEVICES step needed). +# +# To submit (from the project root): +# mkdir -p /home/wp14/logs/condor +# condor_submit scripts/will/condor/labs_notes_cxr/labs_notes_cxr_rnn.sub +# +# Monitor: +# condor_q +# tail -f /home/wp14/logs/condor/labs_notes_cxr_rnn__0.out + +initialdir = /home/wp14/PyHealth +executable = /home/wp14/PyHealth/scripts/will/condor/labs_notes_cxr/run_labs_notes_cxr_rnn.sh +transfer_executable = False +arguments = $(seed) +getenv = True + +environment = "EHR_ROOT=/shared/rsaas/physionet.org/files/mimiciv/2.2 \ + NOTE_ROOT=/shared/rsaas/physionet.org/files/mimic-note \ + CXR_ROOT=/shared/rsaas/physionet.org/files/MIMIC-CXR \ + CXR_VARIANT=sunlab \ + CACHE_DIR=/shared/rsaas/wp14/pyhealth_cache_labs_notes_cxr \ + OUTPUT_DIR=/home/wp14/output \ + USE_WANDB=1 \ + WANDB_PROJECT=pyhealth-multimodal-labs-notes-cxr \ + WANDB_RUN_NAME=labs_notes_cxr_rnn_seed$(seed) \ + FREEZE_ENCODER=1" + +output = /home/wp14/logs/condor/labs_notes_cxr_rnn_$(ClusterId)_$(Process).out +error = /home/wp14/logs/condor/labs_notes_cxr_rnn_$(ClusterId)_$(Process).err +log = /home/wp14/logs/condor/labs_notes_cxr_rnn_$(ClusterId)_$(Process).log + +stream_output = False +stream_error = False + +request_gpus = 1 +request_cpus = 8 +# Starting point copied from labs_notes_rnn.sub (400000MB was sized for the +# full-scale patient_id sort/shuffle there). Untested for this variant: the +# added CXR branch means more dataloader workers decoding JPEGs plus an +# image-metadata join during caching, so this may need to be raised further +# if the job gets cgroup-killed. +request_memory = 400000MB +request_disk = 20GB + +# Same reasoning as labs_notes_rnn.sub: FREEZE_ENCODER=1 (frozen +# Bio_ClinicalBERT text encoder) keeps VRAM down for the text branch, but the +# CXR image encoder is unfrozen here and adds its own footprint on top, so +# the GPU memory floor is a conservative starting guess pending a real OOM +# data point on this variant. +Requirements = (TARGET.GPUs_GlobalMemoryMb >= 40000) +Rank = TARGET.GPUs_GlobalMemoryMb + +queue seed from ( + 12 +) diff --git a/scripts/will/condor/labs_notes_cxr/run_labs_notes_cxr_rnn.sh b/scripts/will/condor/labs_notes_cxr/run_labs_notes_cxr_rnn.sh new file mode 100755 index 000000000..4f40d8801 --- /dev/null +++ b/scripts/will/condor/labs_notes_cxr/run_labs_notes_cxr_rnn.sh @@ -0,0 +1,197 @@ +#!/usr/bin/env bash +# HTCondor executable — labs+notes+CXR RNN mortality run. +# +# Condor equivalent of +# scripts/will/lambda-labs/labs_notes_cxr/tmux_run_labs_notes_cxr_rnn_variant.py: +# same task (notes_labs_cxr), same model (rnn), same hyperparameter defaults. GPU +# selection (nvidia-smi / CUDA_VISIBLE_DEVICES) is dropped since Condor +# assigns the GPU via request_gpus / cgroups. +# +# usage: run_labs_notes_cxr_rnn.sh +# to remove logs: rm -rf logs/condor/* +# to remove cache: rm -rf /shared/rsaas/wp14/pyhealth_cache_labs_notes_cxr/* +set -euo pipefail + +SEED="${1:?usage: run_labs_notes_cxr_rnn.sh }" + +CONDA_ENV="${CONDA_ENV:-pyhealth2}" +PROJECT_DIR="${PROJECT_DIR:-/home/wp14/PyHealth}" +EHR_ROOT="${EHR_ROOT:-/shared/rsaas/physionet.org/files/mimiciv/2.2}" +NOTE_ROOT="${NOTE_ROOT:-/shared/rsaas/physionet.org/files/mimic-note}" +CXR_ROOT="${CXR_ROOT:-/shared/rsaas/physionet.org/files/MIMIC-CXR}" +CXR_VARIANT="${CXR_VARIANT:-default}" +CACHE_DIR="${CACHE_DIR:-/shared/rsaas/wp14/pyhealth_cache_labs_notes_cxr}" +OUTPUT_DIR="${OUTPUT_DIR:-/home/wp14/output}" +CONDA_SH="${CONDA_SH:-}" + +DEV_MODE="${DEV_MODE:-1}" +EMBEDDING_DIM="${EMBEDDING_DIM:-128}" +HIDDEN_DIM="${HIDDEN_DIM:-128}" +RNN_TYPE="${RNN_TYPE:-GRU}" +RNN_LAYERS="${RNN_LAYERS:-2}" +DROPOUT="${DROPOUT:-0.1}" +EPOCHS="${EPOCHS:-15}" +BATCH_SIZE="${BATCH_SIZE:-32}" +LR="${LR:-1e-3}" +WEIGHT_DECAY="${WEIGHT_DECAY:-1e-5}" +PATIENCE="${PATIENCE:-5}" +NUM_WORKERS="${NUM_WORKERS:-4}" +FREEZE_ENCODER="${FREEZE_ENCODER:-0}" +INCLUDE_VITALS="${INCLUDE_VITALS:-0}" +USE_AMP="${USE_AMP:-0}" +AMP_DTYPE="${AMP_DTYPE:-bf16}" + +# Condor GPU cgroups can expose a truncated CUDA_VISIBLE_DEVICES UUID that +# distributed's NVML diagnostics can't resolve, crashing LocalCluster startup. +# Disabling the whole distributed cluster (PYHEALTH_DISABLE_DASK_DISTRIBUTED=1) +# also throws away its disk-spilling/memory limits, which OOM'd the full-scale +# patient_id sort during event-dataframe caching (job 10821, 160GB cgroup +# limit hit). Scope the fix to just the NVML probe instead, keeping the real +# distributed cluster (with spilling) for the sort. +export DASK_DISTRIBUTED__DIAGNOSTICS__NVML="${DASK_DISTRIBUTED__DIAGNOSTICS__NVML:-0}" + +USE_WANDB="${USE_WANDB:-0}" +WANDB_PROJECT="${WANDB_PROJECT:-pyhealth-multimodal-labs-notes-cxr}" +WANDB_RUN_NAME="${WANDB_RUN_NAME:-}" + +resolve_conda_sh() { + if [[ -n "${CONDA_SH}" && -f "${CONDA_SH}" ]]; then + echo "${CONDA_SH}" + return 0 + fi + if command -v conda >/dev/null 2>&1; then + local base + base="$(conda info --base 2>/dev/null || true)" + if [[ -n "${base}" && -f "${base}/etc/profile.d/conda.sh" ]]; then + echo "${base}/etc/profile.d/conda.sh" + return 0 + fi + fi + if [ -f /etc/profile.d/modules.sh ]; then + source /etc/profile.d/modules.sh >/dev/null 2>&1 || true + if command -v module >/dev/null 2>&1; then + module load miniconda3 >/dev/null 2>&1 || true + module load anaconda3 >/dev/null 2>&1 || true + if command -v conda >/dev/null 2>&1; then + local mod_base + mod_base="$(conda info --base 2>/dev/null || true)" + if [[ -n "${mod_base}" && -f "${mod_base}/etc/profile.d/conda.sh" ]]; then + echo "${mod_base}/etc/profile.d/conda.sh" + return 0 + fi + fi + fi + fi + local user_name home_dir + user_name="${USER:-$(id -un 2>/dev/null || true)}" + home_dir="${HOME:-/home/${user_name}}" + local candidates=( + "${home_dir}/miniconda3/etc/profile.d/conda.sh" + "/home/${user_name}/miniconda3/etc/profile.d/conda.sh" + "${home_dir}/anaconda3/etc/profile.d/conda.sh" + "/home/${user_name}/anaconda3/etc/profile.d/conda.sh" + "/opt/miniconda3/etc/profile.d/conda.sh" + "/opt/anaconda3/etc/profile.d/conda.sh" + "/opt/conda/etc/profile.d/conda.sh" + ) + local c + for c in "${candidates[@]}"; do + if [[ -f "${c}" ]]; then + echo "${c}" + return 0 + fi + done + local found="" + found="$(find "${home_dir}" /opt /usr/local /shared -maxdepth 6 -type f -path '*/etc/profile.d/conda.sh' 2>/dev/null | head -n 1 || true)" + if [[ -n "${found}" && -f "${found}" ]]; then + echo "${found}" + return 0 + fi + return 1 +} + +CONDA_SH="$(resolve_conda_sh || true)" +if [[ -z "${CONDA_SH}" || ! -f "${CONDA_SH}" ]]; then + echo "ERROR: conda.sh not found. Set CONDA_SH explicitly." >&2 + exit 1 +fi +source "${CONDA_SH}" +eval "$(conda shell.bash hook)" +conda activate "${CONDA_ENV}" + +cd "${PROJECT_DIR}" +export PYTHONPATH="${PROJECT_DIR}:${PYTHONPATH:-}" + +JOB_TAG="rnn_labs_notes_cxr_s${SEED}_c${_CONDOR_CLUSTER_ID:-local}_p${_CONDOR_PROCNO:-0}" + +echo "========================================================" +echo " Labs+notes+CXR RNN run | ${JOB_TAG}" +echo " $(date -u '+%Y-%m-%d %H:%M:%S UTC')" +echo " Conda env : ${CONDA_ENV}" +echo " EHR root : ${EHR_ROOT}" +echo " Note root : ${NOTE_ROOT}" +echo " CXR root : ${CXR_ROOT} (variant=${CXR_VARIANT})" +echo " Cache dir : ${CACHE_DIR}" +echo " Output dir : ${OUTPUT_DIR}" +echo " Seed : ${SEED}" +echo " Dev mode : ${DEV_MODE}" +echo " Use AMP : ${USE_AMP} (dtype=${AMP_DTYPE})" +echo "========================================================" + +if ! python -c "import pyhealth" >/dev/null 2>&1; then + echo "ERROR: pyhealth is not importable. Run: bash condor_setup.sh" >&2 + exit 1 +fi + +COMMON=( + --ehr-root "${EHR_ROOT}" + --note-root "${NOTE_ROOT}" + --cxr-root "${CXR_ROOT}" + --cxr-variant "${CXR_VARIANT}" + --cache-dir "${CACHE_DIR}" + --task notes_labs_cxr + --model rnn + --embedding-dim "${EMBEDDING_DIM}" + --hidden-dim "${HIDDEN_DIM}" + --rnn-type "${RNN_TYPE}" + --rnn-layers "${RNN_LAYERS}" + --dropout "${DROPOUT}" + --epochs "${EPOCHS}" + --batch-size "${BATCH_SIZE}" + --lr "${LR}" + --weight-decay "${WEIGHT_DECAY}" + --patience "${PATIENCE}" + --num-workers "${NUM_WORKERS}" + --seed "${SEED}" + --output-dir "${OUTPUT_DIR}" +) + +if [[ "${DEV_MODE}" == "1" ]]; then + COMMON+=(--dev) +fi + +if [[ "${FREEZE_ENCODER}" == "1" ]]; then + COMMON+=(--freeze-encoder) +fi + +if [[ "${INCLUDE_VITALS}" == "1" ]]; then + COMMON+=(--include-vitals) +fi + +if [[ "${USE_AMP}" == "1" ]]; then + COMMON+=(--use-amp --amp-dtype "${AMP_DTYPE}") +fi + +if [[ "${USE_WANDB}" == "1" ]]; then + COMMON+=(--wandb --wandb-project "${WANDB_PROJECT}") + if [[ -n "${WANDB_RUN_NAME}" ]]; then + COMMON+=(--wandb-run-name "${WANDB_RUN_NAME}") + fi +fi + +python examples/mortality_prediction/unified_embedding_e2e_mimic4.py "${COMMON[@]}" + +echo "========================================================" +echo " Completed ${JOB_TAG}" +echo " $(date -u '+%Y-%m-%d %H:%M:%S UTC')" +echo "========================================================" From b91ec9eaa59d9325f58b3f832c013b8d7343fb70 Mon Sep 17 00:00:00 2001 From: William Pang Date: Tue, 11 Aug 2026 05:46:58 -0500 Subject: [PATCH 27/61] New Updates --- .../will/condor/labs_notes/lab_notes_rnn.sub | 58 ++++++ .../condor/labs_notes/run_labs_notes_rnn.sh | 191 ++++++++++++++++++ 2 files changed, 249 insertions(+) create mode 100644 scripts/will/condor/labs_notes/lab_notes_rnn.sub create mode 100755 scripts/will/condor/labs_notes/run_labs_notes_rnn.sh diff --git a/scripts/will/condor/labs_notes/lab_notes_rnn.sub b/scripts/will/condor/labs_notes/lab_notes_rnn.sub new file mode 100644 index 000000000..a5ede861e --- /dev/null +++ b/scripts/will/condor/labs_notes/lab_notes_rnn.sub @@ -0,0 +1,58 @@ +# HTCondor submission — labs+notes RNN mortality run +# +# Condor equivalent of scripts/will/sunlab/labs_notes/tmux_run_labs_notes_rnn_variant.py: +# same task (notes_labs), same model (rnn), same hyperparameters. Runs unattended +# instead of in a tmux session; Condor assigns the GPU (no manual +# nvidia-smi / CUDA_VISIBLE_DEVICES step needed). +# +# To submit (from the project root): +# mkdir -p /home/wp14/logs/condor +# condor_submit scripts/will/condor/labs_notes/labs_notes_rnn.sub +# +# Monitor: +# condor_q +# tail -f /home/wp14/logs/condor/labs_notes_rnn__0.out + +initialdir = /home/wp14/PyHealth +executable = /home/wp14/PyHealth/scripts/will/condor/labs_notes/run_labs_notes_rnn.sh +transfer_executable = False +arguments = $(seed) +getenv = True + +environment = "EHR_ROOT=/shared/rsaas/physionet.org/files/mimiciv/2.2 \ + NOTE_ROOT=/shared/rsaas/physionet.org/files/mimic-note \ + CACHE_DIR=/shared/rsaas/wp14/pyhealth_cache_labs_notes \ + OUTPUT_DIR=/home/wp14/output \ + USE_WANDB=1 \ + WANDB_PROJECT=pyhealth-multimodal-labs-notes \ + WANDB_RUN_NAME=labs_notes_rnn_seed$(seed)" + +output = /home/wp14/logs/condor/labs_notes_rnn_$(ClusterId)_$(Process).out +error = /home/wp14/logs/condor/labs_notes_rnn_$(ClusterId)_$(Process).err +log = /home/wp14/logs/condor/labs_notes_rnn_$(ClusterId)_$(Process).log + +stream_output = False +stream_error = False + +request_gpus = 1 +request_cpus = 4 +# Was 163840MB (160GB) — cgroup-killed job 10821 at 162747MB during the +# full-scale patient_id sort/shuffle (see run_labs_notes_rnn.sh comment). +# Bumped for headroom now that the distributed cluster (with disk-spilling) +# is back in play; c02 has ~1TB total and is otherwise idle. +request_memory = 400000MB +request_disk = 20GB + +# Previously hardcoded to sunlab-c01 (A100 80GB) because the previous run +# OOM'd on a 47GB card with ~46.8GB resident. sunlab-c01's condor_startd is +# currently down (master alive, STARTD_StartTime=0), so it never matches — +# the only machine in the pool is sunlab-c02 (8x RTX 6000 Ada, 48509MB each). +# Match any GPU with enough headroom and prefer the biggest; FREEZE_ENCODER=1 +# below (frozen Bio_ClinicalBERT text encoder, ~50% less VRAM for the text +# branch) is what actually keeps this under 48GB instead of the hostname pin. +Requirements = (TARGET.GPUs_GlobalMemoryMb >= 40000) +Rank = TARGET.GPUs_GlobalMemoryMb + +queue seed from ( + 12 +) \ No newline at end of file diff --git a/scripts/will/condor/labs_notes/run_labs_notes_rnn.sh b/scripts/will/condor/labs_notes/run_labs_notes_rnn.sh new file mode 100755 index 000000000..0e26d3fc1 --- /dev/null +++ b/scripts/will/condor/labs_notes/run_labs_notes_rnn.sh @@ -0,0 +1,191 @@ +#!/usr/bin/env bash +# HTCondor executable — labs+notes RNN mortality run. +# +# Condor equivalent of scripts/will/sunlab/labs_notes/tmux_run_labs_notes_rnn_variant.py: +# same task (notes_labs), same model (rnn), same hyperparameter defaults. GPU +# selection (nvidia-smi / CUDA_VISIBLE_DEVICES) is dropped since Condor +# assigns the GPU via request_gpus / cgroups. +# +# usage: run_labs_notes_rnn.sh +# to remove logs: rm -rf logs/condor/* +# to remove cache: rm -rf /shared/rsaas/wp14/pyhealth_cache_labs_notes/* +set -euo pipefail + +SEED="${1:?usage: run_labs_notes_rnn.sh }" + +CONDA_ENV="${CONDA_ENV:-pyhealth2}" +PROJECT_DIR="${PROJECT_DIR:-/home/wp14/PyHealth}" +EHR_ROOT="${EHR_ROOT:-/shared/rsaas/physionet.org/files/mimiciv/2.2}" +NOTE_ROOT="${NOTE_ROOT:-/shared/rsaas/physionet.org/files/mimic-note}" +CACHE_DIR="${CACHE_DIR:-/shared/eng/wp14/pyhealth_cache_labs_notes}" +OUTPUT_DIR="${OUTPUT_DIR:-/home/wp14/output}" +CONDA_SH="${CONDA_SH:-}" + +DEV_MODE="${DEV_MODE:-1}" +EMBEDDING_DIM="${EMBEDDING_DIM:-128}" +HIDDEN_DIM="${HIDDEN_DIM:-128}" +RNN_TYPE="${RNN_TYPE:-GRU}" +RNN_LAYERS="${RNN_LAYERS:-2}" +DROPOUT="${DROPOUT:-0.1}" +EPOCHS="${EPOCHS:-15}" +BATCH_SIZE="${BATCH_SIZE:-32}" +LR="${LR:-1e-3}" +WEIGHT_DECAY="${WEIGHT_DECAY:-1e-5}" +PATIENCE="${PATIENCE:-5}" +NUM_WORKERS="${NUM_WORKERS:-4}" +FREEZE_ENCODER="${FREEZE_ENCODER:-0}" +INCLUDE_VITALS="${INCLUDE_VITALS:-0}" +USE_AMP="${USE_AMP:-0}" +AMP_DTYPE="${AMP_DTYPE:-bf16}" + +# Condor GPU cgroups can expose a truncated CUDA_VISIBLE_DEVICES UUID that +# distributed's NVML diagnostics can't resolve, crashing LocalCluster startup. +# Disabling the whole distributed cluster (PYHEALTH_DISABLE_DASK_DISTRIBUTED=1) +# also throws away its disk-spilling/memory limits, which OOM'd the full-scale +# patient_id sort during event-dataframe caching (job 10821, 160GB cgroup +# limit hit). Scope the fix to just the NVML probe instead, keeping the real +# distributed cluster (with spilling) for the sort. +export DASK_DISTRIBUTED__DIAGNOSTICS__NVML="${DASK_DISTRIBUTED__DIAGNOSTICS__NVML:-0}" + +USE_WANDB="${USE_WANDB:-0}" +WANDB_PROJECT="${WANDB_PROJECT:-pyhealth-multimodal-labs-notes}" +WANDB_RUN_NAME="${WANDB_RUN_NAME:-}" + +resolve_conda_sh() { + if [[ -n "${CONDA_SH}" && -f "${CONDA_SH}" ]]; then + echo "${CONDA_SH}" + return 0 + fi + if command -v conda >/dev/null 2>&1; then + local base + base="$(conda info --base 2>/dev/null || true)" + if [[ -n "${base}" && -f "${base}/etc/profile.d/conda.sh" ]]; then + echo "${base}/etc/profile.d/conda.sh" + return 0 + fi + fi + if [ -f /etc/profile.d/modules.sh ]; then + source /etc/profile.d/modules.sh >/dev/null 2>&1 || true + if command -v module >/dev/null 2>&1; then + module load miniconda3 >/dev/null 2>&1 || true + module load anaconda3 >/dev/null 2>&1 || true + if command -v conda >/dev/null 2>&1; then + local mod_base + mod_base="$(conda info --base 2>/dev/null || true)" + if [[ -n "${mod_base}" && -f "${mod_base}/etc/profile.d/conda.sh" ]]; then + echo "${mod_base}/etc/profile.d/conda.sh" + return 0 + fi + fi + fi + fi + local user_name home_dir + user_name="${USER:-$(id -un 2>/dev/null || true)}" + home_dir="${HOME:-/home/${user_name}}" + local candidates=( + "${home_dir}/miniconda3/etc/profile.d/conda.sh" + "/home/${user_name}/miniconda3/etc/profile.d/conda.sh" + "${home_dir}/anaconda3/etc/profile.d/conda.sh" + "/home/${user_name}/anaconda3/etc/profile.d/conda.sh" + "/opt/miniconda3/etc/profile.d/conda.sh" + "/opt/anaconda3/etc/profile.d/conda.sh" + "/opt/conda/etc/profile.d/conda.sh" + ) + local c + for c in "${candidates[@]}"; do + if [[ -f "${c}" ]]; then + echo "${c}" + return 0 + fi + done + local found="" + found="$(find "${home_dir}" /opt /usr/local /shared -maxdepth 6 -type f -path '*/etc/profile.d/conda.sh' 2>/dev/null | head -n 1 || true)" + if [[ -n "${found}" && -f "${found}" ]]; then + echo "${found}" + return 0 + fi + return 1 +} + +CONDA_SH="$(resolve_conda_sh || true)" +if [[ -z "${CONDA_SH}" || ! -f "${CONDA_SH}" ]]; then + echo "ERROR: conda.sh not found. Set CONDA_SH explicitly." >&2 + exit 1 +fi +source "${CONDA_SH}" +eval "$(conda shell.bash hook)" +conda activate "${CONDA_ENV}" + +cd "${PROJECT_DIR}" +export PYTHONPATH="${PROJECT_DIR}:${PYTHONPATH:-}" + +JOB_TAG="rnn_labs_notes_s${SEED}_c${_CONDOR_CLUSTER_ID:-local}_p${_CONDOR_PROCNO:-0}" + +echo "========================================================" +echo " Labs+notes RNN run | ${JOB_TAG}" +echo " $(date -u '+%Y-%m-%d %H:%M:%S UTC')" +echo " Conda env : ${CONDA_ENV}" +echo " EHR root : ${EHR_ROOT}" +echo " Note root : ${NOTE_ROOT}" +echo " Cache dir : ${CACHE_DIR}" +echo " Output dir: ${OUTPUT_DIR}" +echo " Seed : ${SEED}" +echo " Dev mode : ${DEV_MODE}" +echo " Use AMP : ${USE_AMP} (dtype=${AMP_DTYPE})" +echo "========================================================" + +if ! python -c "import pyhealth" >/dev/null 2>&1; then + echo "ERROR: pyhealth is not importable. Run: bash condor_setup.sh" >&2 + exit 1 +fi + +COMMON=( + --ehr-root "${EHR_ROOT}" + --note-root "${NOTE_ROOT}" + --cache-dir "${CACHE_DIR}" + --task notes_labs + --model rnn + --embedding-dim "${EMBEDDING_DIM}" + --hidden-dim "${HIDDEN_DIM}" + --rnn-type "${RNN_TYPE}" + --rnn-layers "${RNN_LAYERS}" + --dropout "${DROPOUT}" + --epochs "${EPOCHS}" + --batch-size "${BATCH_SIZE}" + --lr "${LR}" + --weight-decay "${WEIGHT_DECAY}" + --patience "${PATIENCE}" + --num-workers "${NUM_WORKERS}" + --seed "${SEED}" + --output-dir "${OUTPUT_DIR}" +) + +if [[ "${DEV_MODE}" == "1" ]]; then + COMMON+=(--dev) +fi + +if [[ "${FREEZE_ENCODER}" == "1" ]]; then + COMMON+=(--freeze-encoder) +fi + +if [[ "${INCLUDE_VITALS}" == "1" ]]; then + COMMON+=(--include-vitals) +fi + +if [[ "${USE_AMP}" == "1" ]]; then + COMMON+=(--use-amp --amp-dtype "${AMP_DTYPE}") +fi + +if [[ "${USE_WANDB}" == "1" ]]; then + COMMON+=(--wandb --wandb-project "${WANDB_PROJECT}") + if [[ -n "${WANDB_RUN_NAME}" ]]; then + COMMON+=(--wandb-run-name "${WANDB_RUN_NAME}") + fi +fi + +python examples/mortality_prediction/unified_embedding_e2e_mimic4.py "${COMMON[@]}" + +echo "========================================================" +echo " Completed ${JOB_TAG}" +echo " $(date -u '+%Y-%m-%d %H:%M:%S UTC')" +echo "========================================================" From 2b7b2c5d4fc0ef0d7fe19032aee0a8cc99a9a495 Mon Sep 17 00:00:00 2001 From: William Pang Date: Tue, 11 Aug 2026 16:24:07 -0500 Subject: [PATCH 28/61] New Updates --- pyhealth/data/data.py | 2 +- pyhealth/datasets/utils.py | 66 +++++++++---------- .../condor/labs_notes/run_labs_notes_rnn.sh | 2 +- .../labs_notes_cxr/run_labs_notes_cxr_rnn.sh | 2 +- 4 files changed, 35 insertions(+), 37 deletions(-) diff --git a/pyhealth/data/data.py b/pyhealth/data/data.py index 14b1b526c..39dd1206f 100644 --- a/pyhealth/data/data.py +++ b/pyhealth/data/data.py @@ -155,7 +155,7 @@ def _filter_by_time_range_fast(self, df: pl.DataFrame, start: Optional[datetime] start_idx = np.searchsorted(ts_col, np.datetime64(start, "ms"), side="left") if end is not None: end_idx = np.searchsorted(ts_col, np.datetime64(end, "ms"), side="right") - return df.slice(start_idx, end_idx - start_idx) + return df.slice(start_idx, max(0, end_idx - start_idx)) def _filter_by_event_type_regular(self, df: pl.DataFrame, event_type: Optional[str]) -> pl.DataFrame: """Regular filtering by event type. Time complexity: O(n).""" diff --git a/pyhealth/datasets/utils.py b/pyhealth/datasets/utils.py index 24c87a1d5..0125ab4f9 100644 --- a/pyhealth/datasets/utils.py +++ b/pyhealth/datasets/utils.py @@ -15,9 +15,10 @@ MODULE_CACHE_PATH = os.path.join(BASE_CACHE_PATH, "datasets") create_directory(MODULE_CACHE_PATH) -#PyG import for graph-based models +# PyG import for graph-based models try: from torch_geometric.data import Data as PyGData, Batch as PyGBatch + HAS_PYG = True except ImportError: HAS_PYG = False @@ -267,40 +268,37 @@ def collate_fn_dict_with_padding(batch: List[dict]) -> dict: for key in keys: values = [sample[key] for sample in batch] - # Check if this is a temporal feature tuple (time, values) - if isinstance(values[0], tuple) and len(values[0]) == 2: - # Handle (time, values) tuples from processors - time_tensors = [v[0] for v in values] - value_tensors = [v[1] for v in values] - - # Collate values - if value_tensors[0].dim() == 0: - # Scalars - collated_values = torch.stack(value_tensors) - elif all(v.shape == value_tensors[0].shape for v in value_tensors): - # All same shape - collated_values = torch.stack(value_tensors) - else: - # Variable shapes, use pad_sequence - collated_values = pad_sequence( - value_tensors, batch_first=True, padding_value=0 - ) - - # Collate times (if present) - collated_times = None - # Check if ALL samples have time (not just some) - if all(t is not None for t in time_tensors): - time_tensors_all = [t for t in time_tensors if t is not None] - if all(t.shape == time_tensors_all[0].shape for t in time_tensors_all): - collated_times = torch.stack(time_tensors_all) + if isinstance(values[0], tuple): + # Generic tuple collation for processor outputs, e.g. + # - (time, value) from StageNet processors + # - (value, mask, token_type_ids, time, type_tag) + # from TupleTimeTextProcessor with tokenizer. + transposed = list(zip(*values)) + collated_elems = [] + + for elem_vals in transposed: + first = elem_vals[0] + + if first is None and all(v is None for v in elem_vals): + collated_elems.append(None) + elif isinstance(first, torch.Tensor): + tensor_vals = list(elem_vals) + if all(v.shape == tensor_vals[0].shape for v in tensor_vals): + collated_elems.append(torch.stack(tensor_vals)) + else: + collated_elems.append( + pad_sequence( + tensor_vals, + batch_first=True, + padding_value=0, + ) + ) else: - collated_times = pad_sequence( - time_tensors_all, batch_first=True, padding_value=0 - ) + collated_elems.append(list(elem_vals)) + + collated[key] = tuple(collated_elems) - # Return as tuple (time, values) - collated[key] = (collated_times, collated_values) - # PyG Data objects (graph processor output) + # PyG Data objects (graph processor output) elif HAS_PYG and isinstance(values[0], PyGData): collated[key] = PyGBatch.from_data_list(values) @@ -453,4 +451,4 @@ def load_processors(processor_dir: str) -> Tuple[Dict, Dict]: print(list_nested_levels([[1, [2], [[3]]]])) print(is_homo_list([1, 2, 3])) print(is_homo_list([1, 2, [3]])) - print(is_homo_list([1, 2.0])) + print(is_homo_list([1, 2.0])) \ No newline at end of file diff --git a/scripts/will/condor/labs_notes/run_labs_notes_rnn.sh b/scripts/will/condor/labs_notes/run_labs_notes_rnn.sh index 0e26d3fc1..33661ff11 100755 --- a/scripts/will/condor/labs_notes/run_labs_notes_rnn.sh +++ b/scripts/will/condor/labs_notes/run_labs_notes_rnn.sh @@ -33,7 +33,7 @@ LR="${LR:-1e-3}" WEIGHT_DECAY="${WEIGHT_DECAY:-1e-5}" PATIENCE="${PATIENCE:-5}" NUM_WORKERS="${NUM_WORKERS:-4}" -FREEZE_ENCODER="${FREEZE_ENCODER:-0}" +FREEZE_ENCODER="${FREEZE_ENCODER:-1}" INCLUDE_VITALS="${INCLUDE_VITALS:-0}" USE_AMP="${USE_AMP:-0}" AMP_DTYPE="${AMP_DTYPE:-bf16}" diff --git a/scripts/will/condor/labs_notes_cxr/run_labs_notes_cxr_rnn.sh b/scripts/will/condor/labs_notes_cxr/run_labs_notes_cxr_rnn.sh index 4f40d8801..aa9037ae7 100755 --- a/scripts/will/condor/labs_notes_cxr/run_labs_notes_cxr_rnn.sh +++ b/scripts/will/condor/labs_notes_cxr/run_labs_notes_cxr_rnn.sh @@ -36,7 +36,7 @@ LR="${LR:-1e-3}" WEIGHT_DECAY="${WEIGHT_DECAY:-1e-5}" PATIENCE="${PATIENCE:-5}" NUM_WORKERS="${NUM_WORKERS:-4}" -FREEZE_ENCODER="${FREEZE_ENCODER:-0}" +FREEZE_ENCODER="${FREEZE_ENCODER:-1}" INCLUDE_VITALS="${INCLUDE_VITALS:-0}" USE_AMP="${USE_AMP:-0}" AMP_DTYPE="${AMP_DTYPE:-bf16}" From 0a75f99dba1fb96d6d9876790edecf252fd0d2f4 Mon Sep 17 00:00:00 2001 From: Arjun Chatterjee Date: Mon, 17 Aug 2026 17:17:09 -0700 Subject: [PATCH 29/61] Implements real APS & dynamic scoring for Conformal Prediction (#1189) * Implement real APS and add dynamic score_type to conformal methods Adds pyhealth/calib/predictionset/scores.py, a shared score module implementing genuine Adaptive Prediction Sets (Romano, Sesia, and Candes 2020): nonconformity score = cumulative sum of predicted probabilities for classes ranked above the target, plus a randomized U*p(target) term (U ~ Uniform(0,1), one draw per example, shared across all candidate classes). Previously, BaseConformal's score_type="aps" was silently just an alias for "threshold" and did not implement APS at all. Threads a new score_type parameter ("threshold" [default, unchanged behavior] or "aps") through BaseConformal, LABEL, ClusterLabel, CovariateLabel, and NeighborhoodLabel, each with an optional random_state for reproducible APS randomization. SCRIB and FavMac are intentionally excluded since their calibration isn't a score-then- quantile pattern. Verified via numpy-only synthetic tests: both score types hit ~90% empirical coverage at alpha=0.1, for marginal and class-conditional coverage, in both nonconformity and conformity sign conventions. * Add scores.py doctests, aps usage examples, tests, and docs Adds >>> usage examples to the 4 public functions in pyhealth/calib/predictionset/scores.py (verified against real computed output). Adds a score_type="aps" usage example to the docstrings of BaseConformal, LABEL, ClusterLabel, CovariateLabel, and NeighborhoodLabel. Adds tests/core/test_scores.py covering both score types: threshold backward-compatibility, the APS formula's hand-computable non-randomized case, monotonicity, reproducibility under a seeded RNG, nonconformity/ conformity complementarity, and empirical marginal coverage at the target alpha. Extends test_cluster_label.py, test_covariate_label.py, and test_neighborhood_label.py with score_type="aps" end-to-end cases. Documents the score_type argument and adds the previously-missing BaseConformal entry to docs/api/calib/pyhealth.calib.predictionset.rst. * Fix ruff lint violations flagged by CI (UP/RUF rules) CI's ruff install (pip install 'ruff~=0.15') resolves to the latest 0.x release under PEP 440 compatible-release semantics, which enabled more pyupgrade/ruff-specific default rules than the older cached ruff used for local verification. Fixes all 13 flagged violations: Optional[X]/Union[X, Y] -> X | Y, typing.Dict -> dict (including the now-modernized pre-existing forward() return annotations this forced), an unused unpacked variable, and an unsorted __all__. Verified by reproducing the CI's exact environment: a clean venv with `pip install 'ruff~=0.15'` (which also resolves to 0.16.3), confirming `tools/check_pr_rules.py` now passes. --- .../calib/pyhealth.calib.predictionset.rst | 34 ++- .../conformal_eeg/test_tfm_tuev_inference.py | 2 +- .../predictionset/base_conformal/__init__.py | 73 ++++-- .../predictionset/cluster/cluster_label.py | 45 +++- .../cluster/neighborhood_label.py | 39 +++- .../covariate/covariate_label.py | 43 +++- pyhealth/calib/predictionset/label.py | 65 ++++-- pyhealth/calib/predictionset/scores.py | 221 ++++++++++++++++++ tests/core/test_cluster_label.py | 34 +++ tests/core/test_covariate_label.py | 33 +++ tests/core/test_neighborhood_label.py | 22 ++ tests/core/test_scores.py | 163 +++++++++++++ 12 files changed, 725 insertions(+), 49 deletions(-) create mode 100644 pyhealth/calib/predictionset/scores.py create mode 100644 tests/core/test_scores.py diff --git a/docs/api/calib/pyhealth.calib.predictionset.rst b/docs/api/calib/pyhealth.calib.predictionset.rst index fe445ea1b..740b1c87c 100644 --- a/docs/api/calib/pyhealth.calib.predictionset.rst +++ b/docs/api/calib/pyhealth.calib.predictionset.rst @@ -1,10 +1,20 @@ pyhealth.calib.predictionset =================================== -Prediction set constructors that provide set-valued predictions with statistical -coverage guarantees. These methods are based on conformal prediction and related +Prediction set constructors that provide set-valued predictions with statistical +coverage guarantees. These methods are based on conformal prediction and related techniques for uncertainty quantification. +``BaseConformal``, ``LABEL``, ``ClusterLabel``, ``CovariateLabel``, and +``NeighborhoodLabel`` all accept a ``score_type`` argument selecting the +nonconformity/conformity score used for calibration and set construction: +either ``"threshold"`` (the default, unchanged from prior releases) or +``"aps"`` (Adaptive Prediction Sets, Romano, Sesia, and Candes 2020), which +adapts the prediction set size to the model's per-input confidence. See +:mod:`pyhealth.calib.predictionset.scores` for the exact score formulas. +``SCRIB`` and ``FavMac`` are not included since their calibration +procedures aren't a score-then-quantile pattern. + Available Methods ----------------- @@ -12,6 +22,7 @@ Available Methods :toctree: _autosummary :nosignatures: + pyhealth.calib.predictionset.BaseConformal pyhealth.calib.predictionset.LABEL pyhealth.calib.predictionset.SCRIB pyhealth.calib.predictionset.FavMac @@ -19,6 +30,14 @@ Available Methods pyhealth.calib.predictionset.ClusterLabel pyhealth.calib.predictionset.NeighborhoodLabel +BaseConformal (Standard Split Conformal Prediction) +---------------------------------------------------- + +.. autoclass:: pyhealth.calib.predictionset.BaseConformal + :members: + :undoc-members: + :show-inheritance: + LABEL (Least Ambiguous Set-valued Classifier) ---------------------------------------------- @@ -71,3 +90,14 @@ Helper Functions ---------------- .. autofunction:: pyhealth.calib.predictionset.covariate.fit_kde + +Score Functions +--------------- + +Shared, pluggable nonconformity/conformity score implementations backing +the ``score_type`` argument described above. + +.. autofunction:: pyhealth.calib.predictionset.scores.all_class_nc_scores +.. autofunction:: pyhealth.calib.predictionset.scores.all_class_conformity_scores +.. autofunction:: pyhealth.calib.predictionset.scores.true_class_nc_scores +.. autofunction:: pyhealth.calib.predictionset.scores.true_class_conformity_scores diff --git a/examples/conformal_eeg/test_tfm_tuev_inference.py b/examples/conformal_eeg/test_tfm_tuev_inference.py index d25eff7ac..d8350d46c 100644 --- a/examples/conformal_eeg/test_tfm_tuev_inference.py +++ b/examples/conformal_eeg/test_tfm_tuev_inference.py @@ -27,7 +27,7 @@ TUEV_ROOT = "/srv/local/data/TUH/tuh_eeg_events/v2.0.0/edf/" REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -TOKENIZER_WEIGHTS = os.path.join(REPO_ROOT, "weightfiles", "tfm_tokenizer_last.pth") +TOKENIZER_WEIGHTS = os.path.join(REPO_ROOT, "weightfiles", "tfm_tokenizer_last.pth") CLASSIFIER_WEIGHTS_DIR = os.path.join( REPO_ROOT, "weightfiles", "TFM_Tokenizer_multiple_finetuned_on_TUEV" ) diff --git a/pyhealth/calib/predictionset/base_conformal/__init__.py b/pyhealth/calib/predictionset/base_conformal/__init__.py index 54a47ea55..9dde35db5 100644 --- a/pyhealth/calib/predictionset/base_conformal/__init__.py +++ b/pyhealth/calib/predictionset/base_conformal/__init__.py @@ -10,18 +10,30 @@ Paper: Vovk, Vladimir, Alexander Gammerman, and Glenn Shafer. "Algorithmic learning in a random world." Springer, 2005. - + Papadopoulos, Harris, Kostas Proedrou, Volodya Vovk, and Alex Gammerman. "Inductive confidence machines for regression." ECML 2002. + + Sadinle, Mauricio, Jing Lei, and Larry Wasserman. "Least ambiguous + set-valued classifiers with bounded error levels." Journal of the + American Statistical Association (2019). [score_type="threshold"] + + Romano, Yaniv, Matteo Sesia, and Emmanuel Candes. "Classification with + valid and adaptive coverage." NeurIPS 2020. [score_type="aps"] """ -from typing import Dict, Union +from typing import Union import numpy as np import torch from torch.utils.data import IterableDataset from pyhealth.calib.base_classes import SetPredictor +from pyhealth.calib.predictionset.scores import ( + SUPPORTED_SCORE_TYPES, + all_class_nc_scores, + true_class_nc_scores, +) from pyhealth.calib.utils import prepare_numpy_dataset from pyhealth.models import BaseModel @@ -109,17 +121,20 @@ class BaseConformal(SetPredictor): alpha: Target miscoverage rate(s). Can be: - float: marginal coverage P(Y not in C(X)) <= alpha - array: class-conditional P(Y not in C(X) | Y=k) <= alpha[k] - score_type: Type of conformity score to use. Currently only one score - is implemented: + score_type: Type of nonconformity score to use: - "threshold" (default): NC score = 1 - p(true class), the score from Sadinle, Lei, and Wasserman (2019) ("LABEL"). - - "aps": accepted as a backward-compatible alias for - "threshold". Despite the name, this does **not** implement - Adaptive Prediction Sets (Romano, Sesia, and Candes 2020) -- - that method uses a different score (cumulative sorted class - probabilities) which is not implemented here. If you need - genuine APS, do not rely on this option; it is kept only so - existing calls with ``score_type="aps"`` keep working. + - "aps": Adaptive Prediction Sets (Romano, Sesia, and Candes + 2020). NC score for class k is the cumulative sum of predicted + probabilities for classes ranked above k, plus a randomized + U * p(k) term (U ~ Uniform(0,1), one draw per example, shared + across all candidate classes for that example). Unlike + "threshold", this adapts the prediction set size to how + peaked or flat the model's predicted distribution is for each + individual input. See :mod:`pyhealth.calib.predictionset.scores` + for the exact formula. + random_state: Optional int seed for the RNG used by score_type="aps" + (the U ~ Uniform(0,1) draws). Ignored for score_type="threshold". debug: Whether to use debug mode (processes fewer samples) Examples: @@ -160,6 +175,12 @@ class BaseConformal(SetPredictor): >>> conformal_model_cc = BaseConformal( ... model, alpha=[0.1, 0.15, 0.1, 0.1, 0.1]) >>> conformal_model_cc.calibrate(cal_dataset=val_data) + >>> + >>> # Use APS instead of the default threshold score (adapts set + >>> # size to how confident the model is on each individual input) + >>> conformal_model_aps = BaseConformal( + ... model, alpha=0.1, score_type="aps", random_state=0) + >>> conformal_model_aps.calibrate(cal_dataset=val_data) """ def __init__( @@ -167,6 +188,7 @@ def __init__( model: BaseModel, alpha: Union[float, np.ndarray], score_type: str = "threshold", + random_state: int | None = None, debug: bool = False, **kwargs, ) -> None: @@ -176,6 +198,11 @@ def __init__( raise NotImplementedError( "BaseConformal only supports multiclass classification" ) + if score_type not in SUPPORTED_SCORE_TYPES: + raise ValueError( + f"Unknown score_type: {score_type!r}. Supported: " + f"{SUPPORTED_SCORE_TYPES}." + ) self.mode = self.model.mode @@ -187,6 +214,7 @@ def __init__( self.device = model.device self.debug = debug self.score_type = score_type + self.rng = np.random.default_rng(random_state) # Store alpha if not isinstance(alpha, float): @@ -208,13 +236,9 @@ def _compute_nc_scores( Returns: Non-conformity scores of shape (N,) — higher means less conforming. """ - N = len(y_true) - if self.score_type == "threshold" or self.score_type == "aps": - scores = 1.0 - y_prob[np.arange(N), y_true] - else: - raise ValueError(f"Unknown score_type: {self.score_type}") - - return scores + return true_class_nc_scores( + y_prob, y_true, score_type=self.score_type, rng=self.rng + ) def calibrate(self, cal_dataset: IterableDataset): """Calibrate the thresholds for prediction set construction. @@ -268,7 +292,7 @@ def calibrate(self, cal_dataset: IterableDataset): if self.debug: print(f"Calibrated thresholds: {self.t}") - def forward(self, **kwargs) -> Dict[str, torch.Tensor]: + def forward(self, **kwargs) -> dict[str, torch.Tensor]: """Forward propagation with prediction set construction. Returns: @@ -284,8 +308,15 @@ def forward(self, **kwargs) -> Dict[str, torch.Tensor]: pred = self.model(**kwargs) - # Include class y if its NC score (1 - p(y)) <= NC threshold self.t - pred["y_predset"] = (1.0 - pred["y_prob"]) <= self.t + y_prob = pred["y_prob"].detach().cpu().numpy() + nc_scores = all_class_nc_scores( + y_prob, score_type=self.score_type, rng=self.rng + ) + nc_scores = torch.as_tensor( + nc_scores, device=pred["y_prob"].device, dtype=pred["y_prob"].dtype + ) + # Include class y if its NC score <= NC threshold self.t + pred["y_predset"] = nc_scores <= self.t return pred diff --git a/pyhealth/calib/predictionset/cluster/cluster_label.py b/pyhealth/calib/predictionset/cluster/cluster_label.py index f56a325fa..0c719973c 100644 --- a/pyhealth/calib/predictionset/cluster/cluster_label.py +++ b/pyhealth/calib/predictionset/cluster/cluster_label.py @@ -19,6 +19,11 @@ from pyhealth.calib.base_classes import SetPredictor from pyhealth.calib.predictionset.base_conformal import _query_quantile +from pyhealth.calib.predictionset.scores import ( + SUPPORTED_SCORE_TYPES, + all_class_nc_scores, + true_class_nc_scores, +) from pyhealth.calib.utils import extract_embeddings, prepare_numpy_dataset from pyhealth.models import BaseModel @@ -44,7 +49,13 @@ class ClusterLabel(SetPredictor): - float: marginal coverage P(Y not in C(X)) <= alpha - array: class-conditional P(Y not in C(X) | Y=k) <= alpha[k] n_clusters: Number of K-means clusters. Default is 5. - random_state: Random seed for K-means clustering. Default is 42. + random_state: Random seed for K-means clustering, and (if + score_type="aps") for the score's U ~ Uniform(0,1) draws. + Default is 42. + score_type: Nonconformity score to use: "threshold" (default, + NC score = 1 - p(true class), Sadinle, Lei, and Wasserman 2019) + or "aps" (Adaptive Prediction Sets, Romano, Sesia, and Candes + 2020). See :mod:`pyhealth.calib.predictionset.scores`. debug: Whether to use debug mode (processes fewer samples for faster iteration) @@ -89,6 +100,15 @@ class ClusterLabel(SetPredictor): ... y_true, y_prob, metrics=["accuracy", "miscoverage_ps"], ... y_predset=extra["y_predset"] ... ) + >>> + >>> # Use APS instead of the default threshold score + >>> cluster_predictor_aps = ClusterLabel( + ... model=model, alpha=0.1, n_clusters=5, score_type="aps") + >>> cluster_predictor_aps.calibrate( + ... cal_dataset=cal_ds, + ... train_embeddings=train_embeddings, + ... cal_embeddings=cal_embeddings, + ... ) """ def __init__( @@ -97,6 +117,7 @@ def __init__( alpha: Union[float, np.ndarray], n_clusters: int = 5, random_state: int = 42, + score_type: str = "threshold", debug: bool = False, **kwargs, ) -> None: @@ -106,6 +127,11 @@ def __init__( raise NotImplementedError( "ClusterLabel only supports multiclass classification" ) + if score_type not in SUPPORTED_SCORE_TYPES: + raise ValueError( + f"Unknown score_type: {score_type!r}. Supported: " + f"{SUPPORTED_SCORE_TYPES}." + ) self.mode = self.model.mode @@ -116,6 +142,7 @@ def __init__( self.device = model.device self.debug = debug + self.score_type = score_type # Store alpha if not isinstance(alpha, float): @@ -129,6 +156,7 @@ def __init__( ) self.n_clusters = n_clusters self.random_state = random_state + self.rng = np.random.default_rng(random_state) # Will be set during calibration self.kmeans_model = None @@ -215,7 +243,9 @@ def calibrate( print(f"Cluster assignments: {np.bincount(cal_cluster_labels)}") # Compute non-conformity scores (higher = less conforming) - conformity_scores = 1.0 - y_prob[np.arange(N), y_true] + conformity_scores = true_class_nc_scores( + y_prob, y_true, score_type=self.score_type, rng=self.rng + ) # Compute cluster-specific thresholds self.cluster_thresholds = {} @@ -313,8 +343,15 @@ def forward(self, **kwargs) -> Dict[str, torch.Tensor]: ) cluster_thresholds = cluster_thresholds.view(view_shape) - # Include class y if its NC score (1 - p(y)) <= NC threshold - pred["y_predset"] = (1.0 - pred["y_prob"]) <= cluster_thresholds + # Include class y if its NC score <= NC threshold + y_prob_np = pred["y_prob"].detach().cpu().numpy() + nc_scores = all_class_nc_scores( + y_prob_np, score_type=self.score_type, rng=self.rng + ) + nc_scores = torch.as_tensor( + nc_scores, device=pred["y_prob"].device, dtype=pred["y_prob"].dtype + ) + pred["y_predset"] = nc_scores <= cluster_thresholds pred.pop("embed", None) # do not expose internal embedding to caller return pred diff --git a/pyhealth/calib/predictionset/cluster/neighborhood_label.py b/pyhealth/calib/predictionset/cluster/neighborhood_label.py index 2d9f2dc6d..36fdbaa0a 100644 --- a/pyhealth/calib/predictionset/cluster/neighborhood_label.py +++ b/pyhealth/calib/predictionset/cluster/neighborhood_label.py @@ -12,6 +12,11 @@ from pyhealth.calib.base_classes import SetPredictor from pyhealth.calib.predictionset.base_conformal import _query_weighted_quantile +from pyhealth.calib.predictionset.scores import ( + SUPPORTED_SCORE_TYPES, + all_class_conformity_scores, + true_class_conformity_scores, +) from pyhealth.calib.utils import extract_embeddings, prepare_numpy_dataset from pyhealth.models import BaseModel @@ -33,6 +38,12 @@ class NeighborhoodLabel(SetPredictor): k_neighbors: Number of nearest calibration neighbors. Default 50. lambda_L: Temperature for exponential weights; smaller => more localization. Default 100.0. + score_type: Conformity score to use: "threshold" (default, + conformity score = p(true class), Sadinle, Lei, and Wasserman + 2019) or "aps" (Adaptive Prediction Sets, Romano, Sesia, and + Candes 2020). See :mod:`pyhealth.calib.predictionset.scores`. + random_state: Optional int seed for the RNG used by + score_type="aps". Ignored for score_type="threshold". debug: If True, process fewer samples for faster iteration. Examples: @@ -61,6 +72,11 @@ class NeighborhoodLabel(SetPredictor): ... y_true, y_prob, metrics=["accuracy", "miscoverage_ps"], ... y_predset=extra["y_predset"] ... ) + >>> + >>> # Use APS instead of the default threshold score + >>> ncp_aps = NeighborhoodLabel( + ... model=model, alpha=0.1, k_neighbors=50, score_type="aps") + >>> ncp_aps.calibrate(cal_dataset=cal_ds, cal_embeddings=cal_embeddings) """ def __init__( @@ -69,6 +85,8 @@ def __init__( alpha: float, k_neighbors: int = 50, lambda_L: float = 100.0, + score_type: str = "threshold", + random_state: int | None = None, debug: bool = False, **kwargs, ) -> None: @@ -78,6 +96,11 @@ def __init__( raise NotImplementedError( "NeighborhoodLabel only supports multiclass classification" ) + if score_type not in SUPPORTED_SCORE_TYPES: + raise ValueError( + f"Unknown score_type: {score_type!r}. Supported: " + f"{SUPPORTED_SCORE_TYPES}." + ) self.mode = self.model.mode @@ -87,6 +110,8 @@ def __init__( self.device = model.device self.debug = debug + self.score_type = score_type + self.rng = np.random.default_rng(random_state) if not (0.0 < alpha < 1.0): raise ValueError(f"alpha must be in (0, 1), got {alpha!r}") @@ -148,7 +173,9 @@ def calibrate( f"cal_dataset size {N}" ) - conformity_scores = y_prob[np.arange(N), y_true] + conformity_scores = true_class_conformity_scores( + y_prob, y_true, score_type=self.score_type, rng=self.rng + ) k = min(self.k_neighbors, N) self._nn = NearestNeighbors(n_neighbors=k, metric="euclidean").fit( @@ -223,7 +250,15 @@ def forward(self, **kwargs) -> Dict[str, torch.Tensor]: ) if pred["y_prob"].ndim > 1: th = th.view(-1, *([1] * (pred["y_prob"].ndim - 1))) - y_predset = pred["y_prob"] >= th + + y_prob_np = pred["y_prob"].detach().cpu().numpy() + conformity_scores = all_class_conformity_scores( + y_prob_np, score_type=self.score_type, rng=self.rng + ) + conformity_scores = torch.as_tensor( + conformity_scores, device=pred["y_prob"].device, dtype=pred["y_prob"].dtype + ) + y_predset = conformity_scores >= th # if threshold is high, include at least argmax empty = y_predset.sum(dim=1) == 0 if empty.any(): diff --git a/pyhealth/calib/predictionset/covariate/covariate_label.py b/pyhealth/calib/predictionset/covariate/covariate_label.py index e0d66ee90..3482e4e91 100644 --- a/pyhealth/calib/predictionset/covariate/covariate_label.py +++ b/pyhealth/calib/predictionset/covariate/covariate_label.py @@ -30,6 +30,11 @@ from pyhealth.calib.base_classes import SetPredictor from pyhealth.calib.calibration.kcal.kde import RBFKernelMean +from pyhealth.calib.predictionset.scores import ( + SUPPORTED_SCORE_TYPES, + all_class_conformity_scores, + true_class_conformity_scores, +) from pyhealth.calib.utils import prepare_numpy_dataset from pyhealth.datasets import get_dataloader from pyhealth.models import BaseModel @@ -263,6 +268,12 @@ class CovariateLabel(SetPredictor): distribution. Should be a callable that takes embeddings (numpy array) and returns density estimates. Used for KDE-based likelihood ratio weighting (CoDrug approach). + score_type: Conformity score to use: "threshold" (default, + conformity score = p(true class), Sadinle, Lei, and Wasserman + 2019) or "aps" (Adaptive Prediction Sets, Romano, Sesia, and + Candes 2020). See :mod:`pyhealth.calib.predictionset.scores`. + random_state: Optional int seed for the RNG used by + score_type="aps". Ignored for score_type="threshold". debug: Whether to use debug mode (processes fewer samples for faster iteration) @@ -330,6 +341,12 @@ class CovariateLabel(SetPredictor): >>> custom_weights = compute_custom_weights(val_data, test_data) >>> cal_model = CovariateLabel(model, alpha=0.1) >>> cal_model.calibrate(cal_dataset=val_data, cal_weights=custom_weights) + + **Example 3: APS instead of the default threshold score** + + >>> cal_model_aps = CovariateLabel(model, alpha=0.1, score_type="aps") + >>> cal_model_aps.calibrate(cal_dataset=val_data, + ... cal_embeddings=cal_embs, test_embeddings=test_embs) """ def __init__( @@ -338,6 +355,8 @@ def __init__( alpha: Union[float, np.ndarray], kde_test: Optional[Callable] = None, kde_cal: Optional[Callable] = None, + score_type: str = "threshold", + random_state: int | None = None, debug: bool = False, **kwargs, ) -> None: @@ -347,6 +366,11 @@ def __init__( raise NotImplementedError( "CovariateLabel only supports multiclass classification" ) + if score_type not in SUPPORTED_SCORE_TYPES: + raise ValueError( + f"Unknown score_type: {score_type!r}. Supported: " + f"{SUPPORTED_SCORE_TYPES}." + ) self.mode = self.model.mode @@ -357,6 +381,8 @@ def __init__( self.device = model.device self.debug = debug + self.score_type = score_type + self.rng = np.random.default_rng(random_state) # Store alpha if not isinstance(alpha, float): @@ -485,8 +511,10 @@ def calibrate( # Keep weights un-normalized here self._sum_cal_weights = np.sum(likelihood_ratios) - # Extract conformity scores (probabilities of true class) - conformity_scores = y_prob[np.arange(N), y_true] + # Extract conformity scores (higher = more conforming) + conformity_scores = true_class_conformity_scores( + y_prob, y_true, score_type=self.score_type, rng=self.rng + ) # Compute weighted quantile thresholds if isinstance(self.alpha, float): @@ -523,8 +551,15 @@ def forward(self, **kwargs) -> Dict[str, torch.Tensor]: """ pred = self.model(**kwargs) - # Construct prediction set by thresholding probabilities - pred["y_predset"] = pred["y_prob"] > self.t + # Construct prediction set by thresholding conformity scores + y_prob = pred["y_prob"].detach().cpu().numpy() + conformity_scores = all_class_conformity_scores( + y_prob, score_type=self.score_type, rng=self.rng + ) + conformity_scores = torch.as_tensor( + conformity_scores, device=pred["y_prob"].device, dtype=pred["y_prob"].dtype + ) + pred["y_predset"] = conformity_scores > self.t return pred diff --git a/pyhealth/calib/predictionset/label.py b/pyhealth/calib/predictionset/label.py index 8ff87d070..b77934a28 100644 --- a/pyhealth/calib/predictionset/label.py +++ b/pyhealth/calib/predictionset/label.py @@ -9,14 +9,17 @@ """ -from typing import Dict, Union - import numpy as np import torch from torch.utils.data import Subset from pyhealth.calib.base_classes import SetPredictor from pyhealth.calib.predictionset.base_conformal import _query_quantile +from pyhealth.calib.predictionset.scores import ( + SUPPORTED_SCORE_TYPES, + all_class_nc_scores, + true_class_nc_scores, +) from pyhealth.calib.utils import prepare_numpy_dataset from pyhealth.models import BaseModel @@ -42,7 +45,16 @@ class LABEL(SetPredictor): :param model: A trained base model. :type model: BaseModel :param alpha: Target mis-coverage rate(s). - :type alpha: Union[float, np.ndarray] + :type alpha: float | np.ndarray + :param score_type: Nonconformity score to use: "threshold" (default, + the LAC score from Sadinle, Lei, and Wasserman 2019, NC score + = 1 - p(true class)) or "aps" (Adaptive Prediction Sets, Romano, + Sesia, and Candes 2020). See + :mod:`pyhealth.calib.predictionset.scores` for the exact formulas. + :type score_type: str + :param random_state: Optional int seed for the RNG used by + score_type="aps". Ignored for score_type="threshold". + :type random_state: int | None Examples: >>> from pyhealth.datasets import ISRUCDataset, split_by_patient, get_dataloader @@ -70,20 +82,37 @@ class LABEL(SetPredictor): ... y_predset=extra_output['y_predset']) ... ) {'accuracy': 0.709843241966832, 'miscoverage_ps': array([0.1499847 , 0.29997638, 0.14993964, 0.14994704, 0.14999252])} + >>> + >>> # Use APS instead of the default threshold score + >>> cal_model_aps = LABEL(model, 0.15, score_type="aps", random_state=0) + >>> cal_model_aps.calibrate(cal_dataset=test_data) """ def __init__( - self, model: BaseModel, alpha: Union[float, np.ndarray], debug=False, **kwargs + self, + model: BaseModel, + alpha: float | np.ndarray, + score_type: str = "threshold", + random_state: int | None = None, + debug=False, + **kwargs, ) -> None: super().__init__(model, **kwargs) if model.mode != "multiclass": raise NotImplementedError() + if score_type not in SUPPORTED_SCORE_TYPES: + raise ValueError( + f"Unknown score_type: {score_type!r}. Supported: " + f"{SUPPORTED_SCORE_TYPES}." + ) self.mode = self.model.mode # multiclass for param in model.parameters(): param.requires_grad = False self.model.eval() self.device = model.device self.debug = debug + self.score_type = score_type + self.rng = np.random.default_rng(random_state) if not isinstance(alpha, float): alpha = np.asarray(alpha) @@ -104,31 +133,37 @@ def calibrate(self, cal_dataset: Subset): y_true = cal_dataset["y_true"] N, K = cal_dataset["y_prob"].shape - # NC scores: 1 - p(true class); higher = less conforming + # NC scores: higher = less conforming + nc_scores = true_class_nc_scores( + y_prob, y_true, score_type=self.score_type, rng=self.rng + ) if isinstance(self.alpha, float): - t = _query_quantile( - 1.0 - y_prob[np.arange(N), y_true], self.alpha - ) + t = _query_quantile(nc_scores, self.alpha) else: t = [ - _query_quantile( - 1.0 - y_prob[y_true == k, k], self.alpha[k] - ) + _query_quantile(nc_scores[y_true == k], self.alpha[k]) for k in range(K) ] self.t = torch.tensor(t, device=self.device) - def forward(self, **kwargs) -> Dict[str, torch.Tensor]: + def forward(self, **kwargs) -> dict[str, torch.Tensor]: """Forward propagation (just like the original model). :return: A dictionary with all results from the base model, with the following updates: y_predset: a bool tensor representing the prediction for each class. - :rtype: Dict[str, torch.Tensor] + :rtype: dict[str, torch.Tensor] """ pred = self.model(**kwargs) - # Include class y if its NC score (1 - p(y)) <= NC threshold - pred["y_predset"] = (1.0 - pred["y_prob"]) <= self.t + y_prob = pred["y_prob"].detach().cpu().numpy() + nc_scores = all_class_nc_scores( + y_prob, score_type=self.score_type, rng=self.rng + ) + nc_scores = torch.as_tensor( + nc_scores, device=pred["y_prob"].device, dtype=pred["y_prob"].dtype + ) + # Include class y if its NC score <= NC threshold + pred["y_predset"] = nc_scores <= self.t return pred if __name__ == "__main__": diff --git a/pyhealth/calib/predictionset/scores.py b/pyhealth/calib/predictionset/scores.py new file mode 100644 index 000000000..ae4172d1d --- /dev/null +++ b/pyhealth/calib/predictionset/scores.py @@ -0,0 +1,221 @@ +"""Shared, pluggable conformity/nonconformity score functions. + +This module separates the *score* used by a conformal-prediction-set method +from the *calibration/thresholding procedure* it's plugged into. These are +two independent axes: the choice of score ("threshold"/LAC vs "aps") does +not depend on how the resulting scores get turned into a threshold (marginal +quantile, per-class quantile, per-cluster quantile, weighted quantile for +covariate shift, or localized weighted quantile for neighborhood methods). + +Supported score types: + + - "threshold" (a.k.a. LAC, Sadinle, Lei, and Wasserman 2019): the score + for class k is simply based on the model's predicted probability for + k. Simple, but not adaptive to how "peaked" or "flat" the predicted + distribution is. + + - "aps" (Adaptive Prediction Sets, Romano, Sesia, and Candes 2020; + score definition as restated in Angelopoulos, Bates, Malik, and + Jordan 2021, "Uncertainty Sets for Image Classifiers using Conformal + Prediction", Algorithm 2): the score for class k is the cumulative + sum of predicted probabilities for all classes ranked strictly above + k, plus a Uniform(0,1)-randomized fraction of k's own probability:: + + E(x, k) = sum_{j : pi(x,j) > pi(x,k)} pi(x,j) + U * pi(x,k) + + where pi(x, ·) are the model's predicted class probabilities and + U ~ Uniform(0,1) is drawn once per example and reused across every + candidate class k for that example (so the resulting prediction sets + are "nested": the set of included classes is always a prefix of the + classes sorted by decreasing probability). This adapts the set size + to the model's confidence for each individual input, which the + "threshold" score does not. + +Both scores are computed here in *nonconformity* convention (higher = less +conforming, i.e. 1 minus a probability-like quantity) since that's the +convention BaseConformal/LABEL/ClusterLabel use internally. A *conformity* +(higher = more conforming) variant is also provided for CovariateLabel/ +NeighborhoodLabel, which use the opposite sign convention internally; it is +simply `1 - nonconformity`, preserving the same ranking of examples either +way. +""" + +import numpy as np + +__all__ = [ + "SUPPORTED_SCORE_TYPES", + "all_class_conformity_scores", + "all_class_nc_scores", + "true_class_conformity_scores", + "true_class_nc_scores", +] + +SUPPORTED_SCORE_TYPES = ("threshold", "aps") + + +def _validate_score_type(score_type: str) -> None: + if score_type not in SUPPORTED_SCORE_TYPES: + raise ValueError( + f"Unknown score_type: {score_type!r}. Supported: " + f"{SUPPORTED_SCORE_TYPES}." + ) + + +def _aps_all_class_nc_scores( + y_prob: np.ndarray, + rng: np.random.Generator, + randomize: bool, +) -> np.ndarray: + """Computes the APS nonconformity score for every class, every row. + + E(x, k) = [sum of predicted probabilities for classes ranked strictly + above k] + U * p(x, k), with U ~ Uniform(0,1) drawn once per row and + reused across all classes in that row (Angelopoulos et al. 2021, + Algorithm 2/3, with the regularization term lambda=0, i.e. plain APS + rather than RAPS). + + Args: + y_prob: Predicted probabilities, shape (N, K). + rng: Random generator used to draw the per-row U ~ Uniform(0,1) + tie-breaking/adaptivity term. + randomize: If False, uses U=1 for every row (the conservative, + non-randomized variant: ties are broken by always including the + full probability mass of a class's own rank). If True (the + variant the APS/RAPS papers use for their reported results), + draws a genuine U ~ Uniform(0,1) per row. + + Returns: + Nonconformity scores of shape (N, K); higher means less conforming. + """ + n = y_prob.shape[0] + # Ties in probability are broken randomly by perturbing the sort key + # infinitesimally, per the APS paper's note that "label-ordering ties + # should be broken randomly" when probabilities aren't all distinct. + tie_break = rng.uniform(0.0, 1e-12, size=y_prob.shape) + order = np.argsort(-(y_prob + tie_break), axis=1) # descending, per row + sorted_probs = np.take_along_axis(y_prob, order, axis=1) + cumsum = np.cumsum(sorted_probs, axis=1) + # Sum of all classes ranked strictly above each rank r (0-indexed): + # cumsum up to and including r, minus r's own probability. + cumsum_excl_own = cumsum - sorted_probs + + if randomize: + u = rng.uniform(0.0, 1.0, size=(n, 1)) + else: + u = np.ones((n, 1)) + + sorted_scores = cumsum_excl_own + u * sorted_probs # (N, K), sorted order + + # Undo the sort to get back to original class-index order. + inverse_order = np.argsort(order, axis=1) + scores = np.take_along_axis(sorted_scores, inverse_order, axis=1) + return scores + + +def all_class_nc_scores( + y_prob: np.ndarray, + score_type: str = "threshold", + rng: np.random.Generator | None = None, + randomize: bool = True, +) -> np.ndarray: + """Nonconformity score (higher = less conforming) for every class. + + Args: + y_prob: Predicted probabilities, shape (N, K). + score_type: "threshold" (Sadinle, Lei, and Wasserman 2019) or "aps" + (Romano, Sesia, and Candes 2020). Default "threshold". + rng: Random generator, required (and only used) if score_type="aps" + and randomize=True. + randomize: Whether to use the randomized ("exact coverage") variant + of APS. Ignored for score_type="threshold". + + Returns: + Nonconformity scores of shape (N, K). + + Examples: + >>> import numpy as np + >>> from pyhealth.calib.predictionset.scores import all_class_nc_scores + >>> y_prob = np.array([[0.7, 0.2, 0.1], [0.3, 0.5, 0.2]]) + >>> all_class_nc_scores(y_prob, score_type="threshold") + array([[0.3, 0.8, 0.9], + [0.7, 0.5, 0.8]]) + >>> rng = np.random.default_rng(0) + >>> scores = all_class_nc_scores(y_prob, score_type="aps", rng=rng) + >>> np.round(scores, 2) + array([[0.42, 0.82, 0.96], + [0.72, 0.36, 0.95]]) + """ + _validate_score_type(score_type) + if score_type == "threshold": + return 1.0 - y_prob + # score_type == "aps" + if rng is None: + rng = np.random.default_rng() + return _aps_all_class_nc_scores(y_prob, rng, randomize) + + +def all_class_conformity_scores( + y_prob: np.ndarray, + score_type: str = "threshold", + rng: np.random.Generator | None = None, + randomize: bool = True, +) -> np.ndarray: + """Conformity score (higher = more conforming) for every class. + + Equivalent to ``1 - all_class_nc_scores(...)``: same ranking of + examples, just the sign convention used by CovariateLabel and + NeighborhoodLabel (which threshold with ``score >= t`` rather than + ``nc_score <= t``). + + Examples: + >>> import numpy as np + >>> from pyhealth.calib.predictionset.scores import all_class_conformity_scores + >>> y_prob = np.array([[0.7, 0.2, 0.1], [0.3, 0.5, 0.2]]) + >>> all_class_conformity_scores(y_prob, score_type="threshold") + array([[0.7, 0.2, 0.1], + [0.3, 0.5, 0.2]]) + """ + return 1.0 - all_class_nc_scores(y_prob, score_type, rng, randomize) + + +def true_class_nc_scores( + y_prob: np.ndarray, + y_true: np.ndarray, + score_type: str = "threshold", + rng: np.random.Generator | None = None, + randomize: bool = True, +) -> np.ndarray: + """Nonconformity score of the true class only, shape (N,). Used during + calibration, where only the true label's score is needed. + + Examples: + >>> import numpy as np + >>> from pyhealth.calib.predictionset.scores import true_class_nc_scores + >>> y_prob = np.array([[0.7, 0.2, 0.1], [0.3, 0.5, 0.2]]) + >>> y_true = np.array([0, 1]) + >>> true_class_nc_scores(y_prob, y_true, score_type="threshold") + array([0.3, 0.5]) + """ + scores = all_class_nc_scores(y_prob, score_type, rng, randomize) + n = len(y_true) + return scores[np.arange(n), y_true] + + +def true_class_conformity_scores( + y_prob: np.ndarray, + y_true: np.ndarray, + score_type: str = "threshold", + rng: np.random.Generator | None = None, + randomize: bool = True, +) -> np.ndarray: + """Conformity score of the true class only, shape (N,). + + Examples: + >>> import numpy as np + >>> from pyhealth.calib.predictionset.scores import true_class_conformity_scores + >>> y_prob = np.array([[0.7, 0.2, 0.1], [0.3, 0.5, 0.2]]) + >>> y_true = np.array([0, 1]) + >>> true_class_conformity_scores(y_prob, y_true, score_type="threshold") + array([0.7, 0.5]) + """ + return 1.0 - true_class_nc_scores(y_prob, y_true, score_type, rng, randomize) diff --git a/tests/core/test_cluster_label.py b/tests/core/test_cluster_label.py index 3b63ba7ed..ca2ab382d 100644 --- a/tests/core/test_cluster_label.py +++ b/tests/core/test_cluster_label.py @@ -377,6 +377,40 @@ def test_prediction_sets_nonempty(self): torch.all(set_sizes > 0), "Some prediction sets are empty" ) + def test_score_type_aps_runs_end_to_end(self): + """score_type='aps' should calibrate and produce non-empty, + correctly-typed prediction sets, just like the default 'threshold'.""" + cluster_model = ClusterLabel( + model=self.model, + alpha=0.3, + n_clusters=2, + random_state=42, + score_type="aps", + ) + + train_indices = [0, 1, 2, 3, 4, 5] + cal_indices = [6, 7, 8, 9, 10, 11] + train_dataset = self.dataset.subset(train_indices) + cal_dataset = self.dataset.subset(cal_indices) + + train_embeddings = self._get_embeddings(train_dataset) + cal_embeddings = self._get_embeddings(cal_dataset) + + cluster_model.calibrate( + cal_dataset=cal_dataset, + train_embeddings=train_embeddings, + cal_embeddings=cal_embeddings, + ) + + test_loader = get_dataloader(self.dataset, batch_size=2, shuffle=False) + with torch.no_grad(): + for data_batch in test_loader: + output = cluster_model(**data_batch) + self.assertEqual(output["y_predset"].dtype, torch.bool) + self.assertEqual(output["y_predset"].shape, output["y_prob"].shape) + set_sizes = output["y_predset"].sum(dim=1) + self.assertTrue(torch.all(set_sizes > 0)) + def test_calibrate_requires_train_embeddings(self): """Test that calibrate requires train_embeddings.""" cluster_model = ClusterLabel( diff --git a/tests/core/test_covariate_label.py b/tests/core/test_covariate_label.py index 14c38cd46..6f2bfd04a 100644 --- a/tests/core/test_covariate_label.py +++ b/tests/core/test_covariate_label.py @@ -307,6 +307,39 @@ def test_prediction_sets_nonempty(self): torch.all(set_sizes > 0), "Some prediction sets are empty" ) + def test_score_type_aps_runs_end_to_end(self): + """score_type='aps' should calibrate and produce non-empty, + correctly-typed prediction sets, just like the default 'threshold'.""" + cal_model = CovariateLabel( + model=self.model, + alpha=0.3, + kde_test=self.kde_test, + kde_cal=self.kde_cal, + score_type="aps", + random_state=42, + ) + + cal_indices = [0, 1, 2, 3] + cal_dataset = self.dataset.subset(cal_indices) + cal_embeddings = self._get_embeddings(cal_dataset) + test_embeddings = self._get_embeddings(self.dataset) + + cal_model.calibrate( + cal_dataset=cal_dataset, + cal_embeddings=cal_embeddings, + test_embeddings=test_embeddings, + ) + + test_indices = [4, 5] + test_dataset = self.dataset.subset(test_indices) + test_loader = get_dataloader(test_dataset, batch_size=2, shuffle=False) + + with torch.no_grad(): + for data_batch in test_loader: + output = cal_model(**data_batch) + self.assertEqual(output["y_predset"].dtype, torch.bool) + self.assertEqual(output["y_predset"].shape, output["y_prob"].shape) + def test_weighted_quantile_function(self): """Test the weighted quantile helper function.""" from pyhealth.calib.predictionset.covariate.covariate_label import ( diff --git a/tests/core/test_neighborhood_label.py b/tests/core/test_neighborhood_label.py index b33f4c4b0..0c812d141 100644 --- a/tests/core/test_neighborhood_label.py +++ b/tests/core/test_neighborhood_label.py @@ -132,6 +132,28 @@ def test_prediction_sets_nonempty_batch(self): set_sizes = out["y_predset"].sum(dim=1) self.assertTrue(torch.all(set_sizes > 0), "Prediction sets should be non-empty") + def test_score_type_aps_runs_end_to_end(self): + """score_type='aps' should calibrate and produce non-empty, + correctly-typed prediction sets, just like the default 'threshold' + (NeighborhoodLabel always guarantees non-empty sets via its own + argmax fallback, independent of score_type).""" + ncp = NeighborhoodLabel( + model=self.model, alpha=0.3, k_neighbors=2, lambda_L=100.0, + score_type="aps", random_state=42, + ) + cal_dataset = self.dataset.subset([2, 3, 4, 5]) + cal_emb = self._get_embeddings(cal_dataset) + ncp.calibrate(cal_dataset=cal_dataset, cal_embeddings=cal_emb) + + loader = get_dataloader(self.dataset, batch_size=2, shuffle=False) + with torch.no_grad(): + for batch in loader: + out = ncp(**batch) + self.assertEqual(out["y_predset"].dtype, torch.bool) + self.assertEqual(out["y_predset"].shape, out["y_prob"].shape) + set_sizes = out["y_predset"].sum(dim=1) + self.assertTrue(torch.all(set_sizes > 0)) + def test_calibrate_without_embeddings_extracts(self): ncp = NeighborhoodLabel(model=self.model, alpha=0.1, k_neighbors=2) cal_dataset = self.dataset.subset([3, 4, 5]) diff --git a/tests/core/test_scores.py b/tests/core/test_scores.py new file mode 100644 index 000000000..e6af15c69 --- /dev/null +++ b/tests/core/test_scores.py @@ -0,0 +1,163 @@ +"""Tests for pyhealth.calib.predictionset.scores: the shared score module +implementing both the "threshold" (LAC) and "aps" (Adaptive Prediction +Sets, Romano/Sesia/Candes 2020) nonconformity/conformity scores. +""" + +import unittest + +import numpy as np + +from pyhealth.calib.predictionset.scores import ( + SUPPORTED_SCORE_TYPES, + all_class_conformity_scores, + all_class_nc_scores, + true_class_conformity_scores, + true_class_nc_scores, +) + + +class TestScoresThreshold(unittest.TestCase): + """"threshold" is just 1 - p (and its complement), regardless of rng.""" + + def setUp(self): + self.y_prob = np.array([[0.7, 0.2, 0.1], [0.3, 0.5, 0.2]]) + self.y_true = np.array([0, 1]) + + def test_all_class_nc_scores(self): + np.testing.assert_allclose( + all_class_nc_scores(self.y_prob, score_type="threshold"), + 1.0 - self.y_prob, + ) + + def test_all_class_conformity_scores(self): + np.testing.assert_allclose( + all_class_conformity_scores(self.y_prob, score_type="threshold"), + self.y_prob, + ) + + def test_true_class_nc_scores(self): + np.testing.assert_allclose( + true_class_nc_scores(self.y_prob, self.y_true, score_type="threshold"), + [0.3, 0.5], + ) + + def test_true_class_conformity_scores(self): + np.testing.assert_allclose( + true_class_conformity_scores(self.y_prob, self.y_true, score_type="threshold"), + [0.7, 0.5], + ) + + def test_default_score_type_is_threshold(self): + """Backward compatibility: omitting score_type must match the old, + hardcoded 1 - p behavior every caller used before score_type + existed.""" + np.testing.assert_allclose( + all_class_nc_scores(self.y_prob), + 1.0 - self.y_prob, + ) + + +class TestScoresAPS(unittest.TestCase): + """Verify the APS score formula: E(x,k) = [sum of probs ranked above k] + + U * p(k), and its structural properties.""" + + def test_non_randomized_matches_hand_computation(self): + """With randomize=False (U=1), APS collapses to the cumulative sum + of sorted probabilities -- hand-computable exactly.""" + y_prob = np.array([[0.5, 0.3, 0.15, 0.05]]) + rng = np.random.default_rng(0) + scores = all_class_nc_scores(y_prob, score_type="aps", rng=rng, randomize=False) + # sorted descending: 0.5, 0.3, 0.15, 0.05 -> cumsum 0.5, 0.8, 0.95, 1.0 + np.testing.assert_allclose(scores, [[0.5, 0.8, 0.95, 1.0]]) + + def test_scores_bounded_in_unit_interval(self): + rng = np.random.default_rng(1) + n, k = 50, 6 + logits = rng.normal(size=(n, k)) + y_prob = np.exp(logits) / np.exp(logits).sum(1, keepdims=True) + scores = all_class_nc_scores(y_prob, score_type="aps", rng=rng) + self.assertTrue(np.all(scores >= 0.0)) + self.assertTrue(np.all(scores <= 1.0)) + + def test_higher_probability_class_has_lower_or_equal_nc_score(self): + """APS nonconformity score must be monotonically non-decreasing as + predicted probability decreases (higher-probability classes are + included in smaller/first-formed sets).""" + rng = np.random.default_rng(2) + y_prob = np.array([[0.6, 0.25, 0.1, 0.05]]) + scores = all_class_nc_scores(y_prob, score_type="aps", rng=rng, randomize=False)[0] + order = np.argsort(-y_prob[0]) + sorted_scores = scores[order] + self.assertTrue(np.all(np.diff(sorted_scores) >= -1e-12)) + + def test_reproducible_with_seeded_rng(self): + y_prob = np.array([[0.4, 0.35, 0.25]]) + s1 = all_class_nc_scores(y_prob, score_type="aps", rng=np.random.default_rng(42)) + s2 = all_class_nc_scores(y_prob, score_type="aps", rng=np.random.default_rng(42)) + np.testing.assert_allclose(s1, s2) + + def test_nc_and_conformity_are_complementary(self): + rng = np.random.default_rng(3) + y_prob = np.array([[0.5, 0.3, 0.2]]) + nc = all_class_nc_scores(y_prob, score_type="aps", rng=np.random.default_rng(3)) + conf = all_class_conformity_scores(y_prob, score_type="aps", rng=np.random.default_rng(3)) + np.testing.assert_allclose(nc, 1.0 - conf) + + def test_true_class_score_matches_all_class_indexing(self): + rng = np.random.default_rng(4) + y_prob = np.array([[0.5, 0.3, 0.2], [0.1, 0.6, 0.3]]) + y_true = np.array([1, 2]) + all_scores = all_class_nc_scores(y_prob, score_type="aps", rng=np.random.default_rng(4)) + true_scores = true_class_nc_scores(y_prob, y_true, score_type="aps", rng=np.random.default_rng(4)) + np.testing.assert_allclose(true_scores, all_scores[np.arange(2), y_true]) + + +class TestScoresCoverage(unittest.TestCase): + """The core statistical property: both score types must achieve + approximately the target marginal coverage under split conformal + calibration, for both marginal and class-conditional targets.""" + + def _query_quantile(self, nc_scores, alpha): + nc_scores = np.sort(nc_scores) + n = len(nc_scores) + loc = int(np.ceil((1 - alpha) * (n + 1))) - 1 + if loc >= n: + return np.inf + return float(nc_scores[loc]) + + def test_marginal_coverage_threshold_and_aps(self): + rng = np.random.default_rng(5) + n, k = 4000, 5 + logits = rng.normal(size=(n, k)) * 2 + y_prob = np.exp(logits) / np.exp(logits).sum(1, keepdims=True) + y_true = np.array([rng.choice(k, p=y_prob[i]) for i in range(n)]) + cal, test = slice(0, n // 2), slice(n // 2, n) + alpha = 0.1 + + for score_type in SUPPORTED_SCORE_TYPES: + cal_rng = np.random.default_rng(6) + nc_cal = true_class_nc_scores( + y_prob[cal], y_true[cal], score_type=score_type, rng=cal_rng + ) + t = self._query_quantile(nc_cal, alpha) + test_rng = np.random.default_rng(7) + nc_test = all_class_nc_scores(y_prob[test], score_type=score_type, rng=test_rng) + predset = nc_test <= t + covered = predset[np.arange(n // 2), y_true[test]] + coverage = covered.mean() + # Allow generous slack for finite-sample noise at N=2000. + self.assertGreaterEqual( + coverage, 1 - alpha - 0.05, + f"{score_type} marginal coverage {coverage:.3f} too far below target", + ) + + +class TestScoresValidation(unittest.TestCase): + def test_unknown_score_type_raises(self): + y_prob = np.array([[0.5, 0.5]]) + with self.assertRaises(ValueError): + all_class_nc_scores(y_prob, score_type="not_a_real_score_type") + + +if __name__ == "__main__": + unittest.main() From a7852f48be7de36b50fd36c387883f925d1b8830 Mon Sep 17 00:00:00 2001 From: William Pang Date: Wed, 26 Aug 2026 10:21:14 -0700 Subject: [PATCH 30/61] Stop emitting fake missing-event placeholders (9782aca) --- pyhealth/processors/stagenet_processor.py | 42 +++-- pyhealth/processors/time_image_processor.py | 15 +- .../processors/tuple_time_text_processor.py | 24 +-- pyhealth/tasks/multimodal_mimic4.py | 149 ++++++------------ tests/core/test_stagenet_processor.py | 13 +- tests/test_tuple_time_text_processor.py | 8 +- 6 files changed, 112 insertions(+), 139 deletions(-) diff --git a/pyhealth/processors/stagenet_processor.py b/pyhealth/processors/stagenet_processor.py index 604376ec1..1f9b78893 100644 --- a/pyhealth/processors/stagenet_processor.py +++ b/pyhealth/processors/stagenet_processor.py @@ -86,7 +86,7 @@ def fit(self, samples: Iterable[Dict[str, Any]], field: str) -> None: if len(first_elem) > 0 and isinstance(first_elem[0], str): # Case 2: [["A", "B"], ["C"], ...] self._is_nested = True - break + break # Build vocabulary for codes and find max nested length max_inner_len = 0 @@ -178,9 +178,9 @@ def process( def _encode_codes(self, codes: List[str]) -> torch.Tensor: """Encode flat code list to indices.""" - # Handle empty code list - return single padding token + # Handle empty code list — zero events, not a fake pad token. if len(codes) == 0: - return torch.tensor([self.code_vocab[""]], dtype=torch.long) + return torch.zeros((0,), dtype=torch.long) indices = [] for code in codes: @@ -198,10 +198,9 @@ def _encode_nested_codes(self, nested_codes: List[List[str]]) -> torch.Tensor: assert self._max_nested_len is not None, "Max nested length must be set during fit()" # Handle empty nested codes (no visits/events) - # Return single padding token with shape (1, max_len) if len(nested_codes) == 0: - pad_token = self.code_vocab[""] - return torch.tensor([[pad_token] * self._max_nested_len], dtype=torch.long) + max_len = self._max_nested_len if self._max_nested_len is not None else 1 + return torch.zeros((0, max_len), dtype=torch.long) encoded_sequences = [] # Use global max length determined during fit @@ -345,9 +344,10 @@ class StageNetTensorProcessor(TemporalFeatureProcessor): >>> time.shape # (3,) """ - def __init__(self): + def __init__(self, forward_fill: bool = True): self._size = None # Feature dimension (set during fit) self._is_nested = None + self.forward_fill = forward_fill def fit(self, samples: Iterable[Dict[str, Any]], field: str) -> None: """Determine input structure. @@ -370,13 +370,14 @@ def fit(self, samples: Iterable[Dict[str, Any]], field: str) -> None: # Flat numeric: [1.5, 2.0, ...] self._is_nested = False self._size = 1 + break elif isinstance(first_elem, list): if len(first_elem) > 0: if isinstance(first_elem[0], (int, float)): # Nested numerics: [[1.0, 2.0], [3.0, 4.0]] self._is_nested = True self._size = len(first_elem) - break + break def process( self, value: Tuple[Optional[List], List] @@ -395,14 +396,31 @@ def process( """ # Unpack tuple: (time, values) time_data, value_data = value + value_data = list(value_data or []) - # Convert to numpy for easier imputation handling import numpy as np + if len(value_data) == 0: + n_feat = self._size if self._size is not None else 1 + nested = True if self._is_nested is None else self._is_nested + if nested: + value_tensor = torch.zeros((0, n_feat), dtype=torch.float) + else: + value_tensor = torch.zeros((0,), dtype=torch.float) + time_tensor = ( + torch.zeros((0,), dtype=torch.float) if time_data is not None else None + ) + return time_tensor, value_tensor + + # Convert to numpy for easier imputation handling value_array = np.array(value_data, dtype=float) - # Apply forward-fill imputation - if value_array.ndim == 1: + # Observation-mask fields must preserve false after a true observation; + # forward-filling a 0/1 mask would turn later missing labs into observed + # values. Ordinary numeric time-series retain the historical behaviour. + if not self.forward_fill: + value_array = np.nan_to_num(value_array, nan=0.0, posinf=0.0, neginf=0.0) + elif value_array.ndim == 1: # Flat numeric: [1.5, 2.0, nan, 3.0, ...] last_value = 0.0 for i in range(len(value_array)): @@ -499,4 +517,4 @@ def __repr__(self): return ( f"StageNetTensorProcessor(is_nested={self._is_nested}, " f"feature_dim={self._size})" - ) + ) \ No newline at end of file diff --git a/pyhealth/processors/time_image_processor.py b/pyhealth/processors/time_image_processor.py index 205782c7f..449fff729 100644 --- a/pyhealth/processors/time_image_processor.py +++ b/pyhealth/processors/time_image_processor.py @@ -279,7 +279,6 @@ def process( Raises: ValueError: If image_paths and time_diffs have different lengths. - ValueError: If image_paths is empty. FileNotFoundError: If any image file does not exist. """ image_paths, time_diffs = value @@ -291,7 +290,19 @@ def process( f"match." ) if len(image_paths) == 0: - raise ValueError("image_paths must be non-empty.") + if self.n_channels is not None: + c = self.n_channels + elif self.mode == "L": + c = 1 + elif self.mode == "RGBA": + c = 4 + else: + c = 3 + images = torch.zeros( + (0, c, self.image_size, self.image_size), dtype=torch.float32 + ) + timestamps = torch.zeros((0,), dtype=torch.float32) + return images, timestamps, "image" paired = sorted(zip(time_diffs, image_paths), key=lambda x: x[0]) diff --git a/pyhealth/processors/tuple_time_text_processor.py b/pyhealth/processors/tuple_time_text_processor.py index cb1fe99a0..7f1fe6a6f 100644 --- a/pyhealth/processors/tuple_time_text_processor.py +++ b/pyhealth/processors/tuple_time_text_processor.py @@ -5,7 +5,6 @@ from . import register_processor logger = logging.getLogger(__name__) -_MISSING_TEXT_TOKEN = "[MISSING_TEXT]" @register_processor("tuple_time_text") class TupleTimeTextProcessor(TemporalFeatureProcessor): @@ -22,7 +21,7 @@ def __init__( self, type_tag: str = "note", tokenizer_model: Optional[str] = None, - max_length: int = 128, + max_length: int = 512, padding: bool = True, truncation: bool = True, ): @@ -32,8 +31,9 @@ def __init__( type_tag: Modality identifier for automatic routing. Default: "note" tokenizer_model: Name or path of the HuggingFace tokenizer to use. If None, texts are returned as raw strings. Default: None - max_length: Maximum sequence length for tokenization. Default: 128 - padding: Whether to pad sequences to max_length. Default: True + max_length: Maximum sequence length for tokenization. Default: 512 + padding: Whether to pad sequences to the longest note in the sample. + Default: True truncation: Whether to truncate sequences to max_length. Default: True """ super().__init__() @@ -109,21 +109,21 @@ def process(self, value: Tuple[List[str], List[float]]) -> Union[Tuple[List[str] cleaned_texts.append(text) cleaned_times.append(t) - # Fast tokenizer path crashes on empty batches; force a single - # missingness token when all notes are empty/malformed. - if len(cleaned_texts) == 0: - cleaned_texts = [_MISSING_TEXT_TOKEN] - cleaned_times = [0.0] - texts = cleaned_texts time_diffs = cleaned_times time_tensor = torch.tensor(time_diffs, dtype=torch.float32) if self.tokenizer is not None: - # Tokenize the list of texts + # Fast tokenizers crash on tokenizer([]). Build empty tensors + # ourselves so a patient with no notes is zero events, not a + # fake "[MISSING_TEXT]" row whose BERT embedding is a constant + # the classifier can use as a mortality feature. + if len(texts) == 0: + empty = torch.zeros((0, 1), dtype=torch.long) + return empty, empty.clone(), empty.clone(), time_tensor, self.type_tag encoded = self.tokenizer( texts, - padding="max_length" if self.padding else False, + padding="longest" if self.padding else False, truncation=self.truncation, max_length=self.max_length, return_tensors="pt" diff --git a/pyhealth/tasks/multimodal_mimic4.py b/pyhealth/tasks/multimodal_mimic4.py index c3b1308e9..768c75819 100644 --- a/pyhealth/tasks/multimodal_mimic4.py +++ b/pyhealth/tasks/multimodal_mimic4.py @@ -83,6 +83,9 @@ def __init__( window_hours: Optional[float] = None, ): self.window_hours = window_hours + # Task cache key is uuid5 over {**vars(task), schemas}. Bump when + # emitted data changes so leaky caches cannot be reused. + self.emitted_data_version = 1 @staticmethod def _clean_text(text: Optional[str]) -> Optional[str]: @@ -148,6 +151,24 @@ def _compute_effective_window( return effective_start, effective_end + def _admission_window_end( + self, + admission_time: datetime, + admission_dischtime: datetime, + ) -> datetime: + """End of the observation window for one admission. + + Callers previously passed ``admission_dischtime`` directly, so + ``window_hours`` was inert and labs were collected through discharge. + For a mortality label that reads the outcome. Re-anchor per admission + and clamp to discharge so a later stay cannot inherit the first + admission's window. + """ + if self.window_hours is None: + return admission_dischtime + end = admission_time + timedelta(hours=self.window_hours) + return min(end, admission_dischtime) if admission_dischtime else end + def _build_admissions_to_process(self, patient: Any) -> Tuple[List[Any], int]: """Build admissions to process and derive mortality label. @@ -207,8 +228,8 @@ def _collect_labs( Returns: Tuple of (lab_times, lab_values, lab_masks). ``lab_masks`` is a parallel boolean tensor where ``True`` means observed and ``False`` - means imputed with 0.0. Falls back to a single missing placeholder - row when no valid lab events are found. + means imputed with 0.0. Returns empty lists when no valid lab + events are found; do not invent a placeholder row. """ try: import polars as pl @@ -266,17 +287,6 @@ def _collect_labs( ) lab_values.append(lab_vector) lab_masks.append(lab_mask) - else: # If missing lab for a given admission - lab_values.append( - [self.MISSING_FLOAT_TOKEN] * len(self.LAB_CATEGORY_NAMES) - ) - lab_masks.append([False] * len(self.LAB_CATEGORY_NAMES)) - lab_times.append(self.MISSING_FLOAT_TOKEN) - - if len(lab_values) == 0: # If missing lab for ALL admissions - lab_values.append([self.MISSING_FLOAT_TOKEN] * len(self.LAB_CATEGORY_NAMES)) - lab_masks.append([False] * len(self.LAB_CATEGORY_NAMES)) - lab_times.append(self.MISSING_FLOAT_TOKEN) return lab_times, lab_values, lab_masks def _collect_notes( @@ -306,9 +316,8 @@ def _collect_notes( with no matching sections are dropped entirely. Returns: - Tuple of (texts, hours_from_admission). Falls back to - ``([MISSING_TEXT_TOKEN], [MISSING_FLOAT_TOKEN])`` when the events - list is empty. + Tuple of (texts, hours_from_admission). Empty lists when the + events list is empty; do not invent a placeholder note. """ notes = patient.get_events( event_type=note_event_type, @@ -371,7 +380,7 @@ class ICDLabsMIMIC4(BaseMultimodalMIMIC4Task): input_schema: Dict[str, Union[str, Tuple[str, Dict]]] = { "icd_codes": ("stagenet", {"padding": PADDING}), "labs": ("stagenet_tensor", {}), - "labs_mask": ("stagenet_tensor", {}), + "labs_mask": ("stagenet_tensor", {"forward_fill": False}), } output_schema: Dict[str, str] = {"mortality": "binary"} @@ -420,32 +429,20 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: ) all_icd_codes.append(visit_icd_codes) all_icd_times.append(time_from_previous) - else: - all_icd_codes.append([self.MISSING_TEXT_TOKEN]) - all_icd_times.append(self.MISSING_FLOAT_TOKEN) previous_admission_time = admission_time lab_times, lab_values, lab_masks = self._collect_labs( patient=patient, admission_time=admission_time, - end_time=admission_dischtime, + end_time=self._admission_window_end( + admission_time, admission_dischtime + ), ) all_lab_times.extend(lab_times) all_lab_values.extend(lab_values) all_lab_masks.extend(lab_masks) - if len(all_lab_values) == 0: - all_lab_values.append( - [self.MISSING_FLOAT_TOKEN] * len(self.LAB_CATEGORY_NAMES) - ) - all_lab_masks.append([False] * len(self.LAB_CATEGORY_NAMES)) - all_lab_times.append(self.MISSING_FLOAT_TOKEN) - - if len(all_icd_codes) == 0: - all_icd_codes.append([self.MISSING_TEXT_TOKEN]) - all_icd_times.append(self.MISSING_FLOAT_TOKEN) - single_patient_longitudinal_record = { "patient_id": patient.patient_id, "icd_codes": (all_icd_times, all_icd_codes), @@ -502,10 +499,11 @@ class NotesLabsMIMIC4(BaseMultimodalMIMIC4Task): { "tokenizer_model": "emilyalsentzer/Bio_ClinicalBERT", "type_tag": "note", + "max_length": 512, }, ), "labs": ("stagenet_tensor", {}), - "labs_mask": ("stagenet_tensor", {}), + "labs_mask": ("stagenet_tensor", {"forward_fill": False}), } input_schema: Dict[str, Union[str, Tuple[str, Dict]]] = _BASE_INPUT_SCHEMA @@ -574,12 +572,8 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: all_note_texts.extend(note_texts) all_note_times.extend(note_times) - # Labs within the observation window - lab_end = ( - effective_end - if self.window_hours is not None - else admission_dischtime - ) + # Labs within the observation window of THIS admission. + lab_end = self._admission_window_end(admission_time, admission_dischtime) lab_times, lab_values, lab_masks = self._collect_labs( patient=patient, admission_time=admission_time, @@ -618,22 +612,8 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: if visit_icd_codes: all_icd_codes.append(visit_icd_codes) all_icd_times.append(time_from_previous) - else: - all_icd_codes.append([self.MISSING_TEXT_TOKEN]) - all_icd_times.append(self.MISSING_FLOAT_TOKEN) previous_admission_time = admission_time - if not all_lab_values: - all_lab_values.append( - [self.MISSING_FLOAT_TOKEN] * len(self.LAB_CATEGORY_NAMES) - ) - all_lab_masks.append([False] * len(self.LAB_CATEGORY_NAMES)) - all_lab_times.append(self.MISSING_FLOAT_TOKEN) - - if not all_note_texts: - all_note_texts = [self.MISSING_TEXT_TOKEN] - all_note_times = [self.MISSING_FLOAT_TOKEN] - record: Dict[str, Any] = { "patient_id": patient.patient_id, "admission_note_times": (all_note_texts, all_note_times), @@ -645,9 +625,6 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: } if self.include_icd: - if not all_icd_codes: - all_icd_codes.append([self.MISSING_TEXT_TOKEN]) - all_icd_times.append(self.MISSING_FLOAT_TOKEN) record["icd_codes"] = (all_icd_times, all_icd_codes) return [record] @@ -691,10 +668,11 @@ class NotesLabsCXRMIMIC4(BaseMultimodalMIMIC4Task): { "tokenizer_model": "emilyalsentzer/Bio_ClinicalBERT", "type_tag": "note", + "max_length": 512, }, ), "labs": ("stagenet_tensor", {}), - "labs_mask": ("stagenet_tensor", {}), + "labs_mask": ("stagenet_tensor", {"forward_fill": False}), "cxr_image_times": ( "time_image", { @@ -778,12 +756,8 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: all_note_texts.extend(note_texts) all_note_times.extend(note_times) - # Labs within the observation window - lab_end = ( - effective_end - if self.window_hours is not None - else admission_dischtime - ) + # Labs within the observation window of THIS admission. + lab_end = self._admission_window_end(admission_time, admission_dischtime) lab_times, lab_values, lab_masks = self._collect_labs( patient=patient, admission_time=admission_time, @@ -841,27 +815,8 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: if visit_icd_codes: all_icd_codes.append(visit_icd_codes) all_icd_times.append(time_from_previous) - else: - all_icd_codes.append([self.MISSING_TEXT_TOKEN]) - all_icd_times.append(self.MISSING_FLOAT_TOKEN) previous_admission_time = admission_time - if not all_lab_values: - all_lab_values.append( - [self.MISSING_FLOAT_TOKEN] * len(self.LAB_CATEGORY_NAMES) - ) - all_lab_masks.append([False] * len(self.LAB_CATEGORY_NAMES)) - all_lab_times.append(self.MISSING_FLOAT_TOKEN) - - if not all_note_texts: - all_note_texts = [self.MISSING_TEXT_TOKEN] - all_note_times = [self.MISSING_FLOAT_TOKEN] - - # time_image processor expects at least one path/time pair. - if len(all_cxr_paths) == 0: - all_cxr_paths = [self.MISSING_TEXT_TOKEN] - all_cxr_times = [self.MISSING_FLOAT_TOKEN] - record: Dict[str, Any] = { "patient_id": patient.patient_id, "admission_note_times": (all_note_texts, all_note_times), @@ -874,9 +829,6 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: } if self.include_icd: - if not all_icd_codes: - all_icd_codes.append([self.MISSING_TEXT_TOKEN]) - all_icd_times.append(self.MISSING_FLOAT_TOKEN) record["icd_codes"] = (all_icd_times, all_icd_codes) return [record] @@ -903,13 +855,12 @@ class LabsMIMIC4(BaseMultimodalMIMIC4Task): input_schema: ClassVar[Dict] = { "labs": ("stagenet_tensor", {}), - "labs_mask": ("stagenet_tensor", {}), + "labs_mask": ("stagenet_tensor", {"forward_fill": False}), } output_schema: ClassVar[Dict] = {"mortality": "binary"} def __init__(self, window_hours: Optional[float] = 24) -> None: - super().__init__() - self.window_hours = window_hours + super().__init__(window_hours=window_hours) def __call__(self, patient: Any) -> List[Dict[str, Any]]: # type: ignore[override] admissions_to_process, mortality_label = self._build_admissions_to_process( @@ -941,19 +892,14 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: # type: ignore[overri lab_times, lab_values, lab_masks = self._collect_labs( patient=patient, admission_time=admission_time, - end_time=admission_dischtime, + end_time=self._admission_window_end( + admission_time, admission_dischtime + ), ) all_lab_times.extend(lab_times) all_lab_values.extend(lab_values) all_lab_masks.extend(lab_masks) - if len(all_lab_values) == 0: - all_lab_values.append( - [self.MISSING_FLOAT_TOKEN] * len(self.LAB_CATEGORY_NAMES) - ) - all_lab_masks.append([False] * len(self.LAB_CATEGORY_NAMES)) - all_lab_times.append(self.MISSING_FLOAT_TOKEN) - single_patient_longitudinal_record = { "patient_id": patient.patient_id, "labs": (all_lab_times, all_lab_values), @@ -1026,9 +972,9 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: # type: ignore[overri if admission_dischtime < admission_time: admission_dischtime = admission_time - admission_end = admission_dischtime - if effective_end is not None and effective_end < admission_end: - admission_end = effective_end + admission_end = self._admission_window_end( + admission_time, admission_dischtime + ) # CXR metadata is filtered by timestamp; this includes StudyTime. metadata_events = patient.get_events( @@ -1048,11 +994,6 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: # type: ignore[overri except AttributeError: continue - # time_image processor expects at least one path/time pair. - if len(all_cxr_paths) == 0: - all_cxr_paths = [self.MISSING_TEXT_TOKEN] - all_cxr_times = [self.MISSING_FLOAT_TOKEN] - single_patient_longitudinal_record = { "patient_id": patient.patient_id, "cxr_image_times": (all_cxr_paths, all_cxr_times), diff --git a/tests/core/test_stagenet_processor.py b/tests/core/test_stagenet_processor.py index 6e217dc7d..52bbc24f6 100644 --- a/tests/core/test_stagenet_processor.py +++ b/tests/core/test_stagenet_processor.py @@ -199,9 +199,8 @@ def test_empty_codes_flat(self): time, values = processor.process((None, [])) - # Should return single padding token - self.assertEqual(values.shape, (1,)) - self.assertEqual(values[0].item(), processor.code_vocab[""]) + # Should return zero events, not a fake pad token + self.assertEqual(values.shape, (0,)) def test_empty_codes_nested(self): """Test processing empty nested codes.""" @@ -211,10 +210,8 @@ def test_empty_codes_nested(self): time, values = processor.process((None, [])) - # Should return single row of padding tokens - self.assertEqual(values.shape, (1, 2)) - self.assertEqual(values[0, 0].item(), processor.code_vocab[""]) - self.assertEqual(values[0, 1].item(), processor.code_vocab[""]) + # Should return zero events, not a fake pad row + self.assertEqual(values.shape, (0, 2)) def test_vocab_size_method(self): """Test vocab_size() returns correct size.""" @@ -462,4 +459,4 @@ def test_vocab_size_for_embedding_layer(self): if __name__ == "__main__": - unittest.main() + unittest.main() \ No newline at end of file diff --git a/tests/test_tuple_time_text_processor.py b/tests/test_tuple_time_text_processor.py index 7a8dfd5b5..f93235937 100644 --- a/tests/test_tuple_time_text_processor.py +++ b/tests/test_tuple_time_text_processor.py @@ -25,7 +25,13 @@ def test_tuple_time_text_processor(): assert torch.equal(time_tensor, torch.tensor([0.0, 24.0, 72.0])) assert tag == "clinical_note" + # Empty input is zero events, not a fake "[MISSING_TEXT]" token. + empty_texts, empty_time, empty_tag = processor.process(([], [])) + assert empty_texts == [] + assert empty_time.shape == (0,) + assert empty_tag == "clinical_note" + # Test registration from pyhealth.processors import get_processor ProcessorClass = get_processor("tuple_time_text") - assert ProcessorClass is TupleTimeTextProcessor + assert ProcessorClass is TupleTimeTextProcessor \ No newline at end of file From d2e66983eb31bfb3b9fb99a937137f6dad72b01f Mon Sep 17 00:00:00 2001 From: William Pang Date: Wed, 26 Aug 2026 10:40:01 -0700 Subject: [PATCH 31/61] Keep frozen text encoders in eval when Trainer calls train()(869ac8e) --- pyhealth/models/embedding/unified.py | 157 +++++++++++++++++++++++++-- 1 file changed, 148 insertions(+), 9 deletions(-) diff --git a/pyhealth/models/embedding/unified.py b/pyhealth/models/embedding/unified.py index 30f83a76d..d3236211c 100644 --- a/pyhealth/models/embedding/unified.py +++ b/pyhealth/models/embedding/unified.py @@ -46,9 +46,11 @@ import math import warnings +from contextlib import nullcontext from typing import Any, Optional import torch +import torch.nn.functional as F import torch.nn as nn from ...processors.base_processor import ModalityType, TemporalFeatureProcessor @@ -211,6 +213,8 @@ def __init__( image_pool: str = "mean", field_embeddings: Optional[dict[str, Any]] = None, freeze_text_encoder: bool = False, + normalize_content: bool = True, + numeric_standardizers: Optional[dict[str, Any]] = None, ): super().__init__() if image_pool != "mean": @@ -219,7 +223,12 @@ def __init__( ) self._embedding_dim = embedding_dim self._freeze_text_encoder = freeze_text_encoder + self._frozen_text_fields: set[str] = set() self.image_pool = image_pool + self.normalize_content = normalize_content + # Statistics live in buffers, so they travel in state_dict. A checkpoint + # therefore applies at inference the same transform it trained under. + self.numeric_standardizers = nn.ModuleDict(numeric_standardizers or {}) _field_embeddings = field_embeddings or {} self.encoders: nn.ModuleDict = nn.ModuleDict() @@ -347,6 +356,7 @@ def _set_projection( if freeze: for p in pre_built.transformer.parameters(): p.requires_grad = False + self._frozen_text_fields.add(field_name) pre_dim = getattr(pre_built, "embedding_dim", embedding_dim) _set_projection(pre_dim, pre_built.fc) return @@ -358,6 +368,7 @@ def _set_projection( if freeze: for p in bert.parameters(): p.requires_grad = False + self._frozen_text_fields.add(field_name) self.encoders[field_name] = bert hidden = bert.config.hidden_size if hidden != embedding_dim: @@ -431,6 +442,18 @@ def _build_numeric_encoder( def embedding_dim(self) -> int: return self._embedding_dim + def train(self, mode: bool = True) -> "UnifiedMultimodalEmbeddingModel": + """Keep a frozen text encoder in eval mode. + + ``nn.Module.train()`` would enable dropout inside the encoder. Its + output would then change between passes even though every weight has + ``requires_grad=False``. + """ + super().train(mode) + for field_name in self._frozen_text_fields: + self.encoders[field_name].eval() + return self + # ── Forward ─────────────────────────────────────────────────────────────── def forward( @@ -462,9 +485,22 @@ def forward( all_types: list[torch.Tensor] = [] for field_name, feat_dict in inputs.items(): + if field_name.endswith("_mask") and field_name[: -len("_mask")] in inputs: + # Observation-flag sibling consumed by the standardiser; not a + # modality of its own. Encoding it would duplicate every lab + # timestamp with a 0/1 vector. + continue value = feat_dict["value"] # (B, N_i, ...) or (B, S, F) time = feat_dict["time"] # (B, N_i) + # Three different masks meet here and must not be conflated. + # mask token level, from the processor schema; this is the + # attention mask a text encoder needs. + # pad_mask event level, from the collator; which slots are real + # events rather than batch padding. + # {field}_mask a separate FIELD meaning "was this value + # observed", consumed by the standardiser below. mask = feat_dict.get("mask") + pad_mask = feat_dict.get("pad_mask") if time is None: # Fallback: treat every event as occurring at t=0 @@ -523,11 +559,38 @@ def forward( ) elif modality == ModalityType.TEXT: + # Collate pads note slots to the longest sample in the batch. + # Running BERT on those empty rows is what OOM'd batch-32 + # notes_labs on a 48 GB GPU (~B*N=full pad width, L=512). b, n, l = value.shape - flat_ids = value.view(b * n, l) - flat_mask = mask.view(b * n, l) if mask is not None else None - out = encoder(input_ids=flat_ids, attention_mask=flat_mask) - cls_emb = out.last_hidden_state[:, 0, :] # (B*N, H) + flat_ids = value.reshape(b * n, l) + flat_attn = mask.reshape(b * n, l) if mask is not None else None + if pad_mask is not None: + valid = pad_mask.reshape(b * n).bool() + elif flat_attn is not None: + valid = flat_attn.any(dim=-1) + else: + valid = torch.ones( + b * n, dtype=torch.bool, device=value.device + ) + hidden = encoder.config.hidden_size + cls_emb = value.new_zeros( + (b * n, hidden), dtype=next(encoder.parameters()).dtype + ) + if valid.any(): + encode_kwargs = {"input_ids": flat_ids[valid]} + if flat_attn is not None: + encode_kwargs["attention_mask"] = flat_attn[valid] + ctx = ( + torch.no_grad() + if field_name in self._frozen_text_fields + else nullcontext() + ) + with ctx: + out = encoder(**encode_kwargs) + h = out.last_hidden_state[:, 0, :] + cls_emb = cls_emb.to(dtype=h.dtype) + cls_emb[valid] = h if field_name in self.projections: cls_emb = self.projections[field_name](cls_emb) emb = cls_emb.view(b, n, -1) # (B, N, E') @@ -535,15 +598,67 @@ def forward( elif modality == ModalityType.IMAGE: # encoder = Sequential(PatchEmbedding, _MeanPool) → (B*N, E') b, n, c, h, w = value.shape - flat_imgs = value.view(b * n, c, h, w) - img_emb = encoder(flat_imgs) # (B*N, E') + flat_imgs = value.reshape(b * n, c, h, w) + if pad_mask is not None: + valid = pad_mask.reshape(b * n).bool() + else: + valid = flat_imgs.reshape(b * n, -1).abs().sum(dim=-1) > 0 + if valid.any(): + img_valid = encoder(flat_imgs[valid]) + img_emb = img_valid.new_zeros( + (b * n, img_valid.shape[-1]) + ) + img_emb[valid] = img_valid + else: + img_emb = value.new_zeros((b * n, self._embedding_dim)) emb = img_emb.view(b, n, -1) # (B, N, E') else: # NUMERIC / SIGNAL + # Standardise BEFORE the projection. The projection mixes the + # features, so a transform after it cannot correct a feature + # whose physical unit gives it 300 times the magnitude of + # another. + standardizer = ( + self.numeric_standardizers[field_name] + if field_name in self.numeric_standardizers + else None + ) + if standardizer is not None: + # Observation flags live in the sibling ``{field}_mask`` + # FIELD, not in this field's dict. Reading the padding mask + # here would tell the standardiser that every real event + # was measured, which is exactly the distinction the + # standardiser exists to preserve. + sibling = inputs.get(f"{field_name}_mask") + obs = sibling["value"] if isinstance(sibling, dict) else None + if obs is None: + raise ValueError( + f"The standardiser for {field_name!r} needs a paired " + f"{field_name}_mask field in the batch." + ) + if obs.shape != value.shape: + raise ValueError( + f"{field_name}_mask has shape {tuple(obs.shape)}, " + f"which does not match {field_name} " + f"{tuple(value.shape)}." + ) + value = standardizer(value, obs.bool()) emb = encoder(value) # (B, T, E') # ── Build event-level validity mask ─────────────────────────── - if mask is None: + if pad_mask is not None: + # The collator is authoritative about batch padding. + event_mask = pad_mask.to(emb.device).float() + if event_mask.shape[1] != emb.shape[1]: + # A nested CODE field was flattened to (B, S*C); repeat the + # event flags along the same axis. + repeat = emb.shape[1] // event_mask.shape[1] + event_mask = ( + event_mask.unsqueeze(-1) + .expand(-1, -1, repeat) + .reshape(emb.shape[0], -1) + ) + elif mask is None: event_mask = torch.ones(emb.shape[:2], device=emb.device) else: if mask.dim() > time.dim(): @@ -570,7 +685,19 @@ def forward( cat_types = torch.cat(all_types, dim=1) # (B, S_total) # ── Sort by time ────────────────────────────────────────────────── - sort_idx = cat_time.argsort(dim=1) + # Padding carries time 0.0, so a plain ascending sort places it BEFORE + # every real event. Three consumers then read it: RNNLayer packs the + # first ``mask.sum()`` steps, ``get_last_visit`` indexes + # ``mask.sum() - 1``, and TransformerLayer takes position 0 as its CLS + # vector. Push invalid slots past every real one to keep the sequence + # left-aligned, which is what all three assume. + # + # Stable, because the key is heavily tied: all padding shares time 0.0 + # and events from one admission share offsets. An unstable sort makes + # event order differ between torch builds and between CPU and CUDA, + # silently changing RNN and Mamba outputs. + sort_key = cat_time.masked_fill(~cat_mask.bool(), float("inf")) + sort_idx = sort_key.argsort(dim=1, stable=True) cat_emb = cat_emb.gather(1, sort_idx.unsqueeze(-1).expand_as(cat_emb)) cat_time = cat_time.gather(1, sort_idx) cat_mask = cat_mask.gather(1, sort_idx) @@ -579,7 +706,19 @@ def forward( # ── Add time + type embeddings ──────────────────────────────────── time_emb = self.time_embed(cat_time) # (B, S_total, E') type_emb = self.type_embedding(cat_types) # (B, S_total, E') - final = cat_emb + time_emb + type_emb # (B, S_total, E') + if self.normalize_content: + # Put the content term on the scale of the additive terms. Without + # this the sum is decided by whichever modality has the larger + # magnitude, which is an accident of feature scaling and not a + # modelling decision. Measured at embedding_dim=128: text content + # norm 3.2, raw laboratory content norm 761.4, time and type + # together 13. F.layer_norm without weight or bias adds NO + # parameters, so an existing checkpoint still loads. + cat_emb = F.layer_norm(cat_emb, (cat_emb.shape[-1],)) + final = cat_emb + time_emb + type_emb + # Zero the padded slots so a consumer that ignores the mask, such as a + # mean pool, still cannot pick them up. + final = final * cat_mask.unsqueeze(-1).to(final.dtype) # (B, S_total, E') return { "sequence": final, # (B, S_total, E') From 412d509ff7b73e6cc4d2b6b6a67fc39e57a1e40d Mon Sep 17 00:00:00 2001 From: William Pang Date: Wed, 26 Aug 2026 12:54:32 -0700 Subject: [PATCH 32/61] Cache frozen [CLS] embeddings keyed on real tokens, not padded rows --- pyhealth/models/embedding/unified.py | 110 ++++++++++++++++++++++++--- 1 file changed, 99 insertions(+), 11 deletions(-) diff --git a/pyhealth/models/embedding/unified.py b/pyhealth/models/embedding/unified.py index d3236211c..3f786bc95 100644 --- a/pyhealth/models/embedding/unified.py +++ b/pyhealth/models/embedding/unified.py @@ -214,6 +214,8 @@ def __init__( field_embeddings: Optional[dict[str, Any]] = None, freeze_text_encoder: bool = False, normalize_content: bool = True, + cache_frozen_text: bool = True, + max_frozen_text_cache: int = 200_000, numeric_standardizers: Optional[dict[str, Any]] = None, ): super().__init__() @@ -224,6 +226,9 @@ def __init__( self._embedding_dim = embedding_dim self._freeze_text_encoder = freeze_text_encoder self._frozen_text_fields: set[str] = set() + self.cache_frozen_text = cache_frozen_text + self.max_frozen_text_cache = max_frozen_text_cache + self._frozen_text_cache: dict[str, dict[int, torch.Tensor]] = {} self.image_pool = image_pool self.normalize_content = normalize_content # Statistics live in buffers, so they travel in state_dict. A checkpoint @@ -442,12 +447,100 @@ def _build_numeric_encoder( def embedding_dim(self) -> int: return self._embedding_dim + def _encode_text_cls( + self, + field_name: str, + encoder: nn.Module, + flat_ids: torch.Tensor, + flat_mask: Optional[torch.Tensor], + ) -> torch.Tensor: + """Return the ``[CLS]`` vector for each row, from a cache when possible. + + A frozen encoder gives the same output for the same tokens, so a run of + 50 epochs would otherwise repeat the identical 110M-parameter forward + pass 50 times. + + The cache has three conditions. It is used only for a field in + ``_frozen_text_fields``, so a trainable encoder can never read it. The + key is the token identifiers under the attention mask, so a change of + tokenizer or truncation budget gives a different key. The cache has a + maximum size, and it recalculates a row when the cache is full. + + Key on the REAL tokens only. The collator pads each row to the widest + note in its batch, and batch composition changes every epoch because + the loader shuffles, so a key over the padded row gives the same note + a different key each epoch and the cache never hits. Measured on the + full-scale notes run: epoch time did not fall after epoch 1 + (3458s, 3936s, 4048s, 3835s) because every lookup missed. + """ + if flat_ids.shape[0] == 0: + hidden = encoder.config.hidden_size + return flat_ids.new_zeros( + (0, hidden), dtype=next(encoder.parameters()).dtype + ) + + if not (self.cache_frozen_text and field_name in self._frozen_text_fields): + ctx = torch.no_grad() if field_name in self._frozen_text_fields else nullcontext() + with ctx: + out = encoder(input_ids=flat_ids, attention_mask=flat_mask) + return out.last_hidden_state[:, 0, :] + + cache = self._frozen_text_cache.setdefault(field_name, {}) + ids_cpu = flat_ids.detach().cpu() + mask_cpu = ( + flat_mask.detach().cpu().to(torch.int8) + if flat_mask is not None + else torch.ones_like(ids_cpu, dtype=torch.int8) + ) + keys = [ + hash(tuple(i[m.bool()].tolist())) if m.any() else hash(tuple(i.tolist())) + for i, m in zip(ids_cpu, mask_cpu) + ] + + first_row_of_key: dict[int, int] = {} + for k, key in enumerate(keys): + if key not in cache and key not in first_row_of_key: + first_row_of_key[key] = k + missing = list(first_row_of_key.values()) + if missing: + index = torch.tensor(missing, device=flat_ids.device) + with torch.no_grad(): + out = encoder( + input_ids=flat_ids.index_select(0, index), + attention_mask=( + flat_mask.index_select(0, index) + if flat_mask is not None + else None + ), + ) + fresh = out.last_hidden_state[:, 0, :].detach() + for slot, row in zip(missing, fresh): + if len(cache) < self.max_frozen_text_cache: + cache[keys[slot]] = row.cpu() + + rows = [] + for k, key in enumerate(keys): + hit = cache.get(key) + if hit is None: + with torch.no_grad(): + out = encoder( + input_ids=flat_ids[k : k + 1], + attention_mask=( + flat_mask[k : k + 1] if flat_mask is not None else None + ), + ) + rows.append(out.last_hidden_state[0, 0, :].detach()) + else: + rows.append(hit.to(flat_ids.device)) + return torch.stack(rows).to(dtype=self.type_embedding.weight.dtype) + def train(self, mode: bool = True) -> "UnifiedMultimodalEmbeddingModel": """Keep a frozen text encoder in eval mode. ``nn.Module.train()`` would enable dropout inside the encoder. Its output would then change between passes even though every weight has - ``requires_grad=False``. + ``requires_grad=False``. That makes the cache incorrect, and it also + makes a frozen encoder give a different answer for the same input. """ super().train(mode) for field_name in self._frozen_text_fields: @@ -578,17 +671,12 @@ def forward( (b * n, hidden), dtype=next(encoder.parameters()).dtype ) if valid.any(): - encode_kwargs = {"input_ids": flat_ids[valid]} - if flat_attn is not None: - encode_kwargs["attention_mask"] = flat_attn[valid] - ctx = ( - torch.no_grad() - if field_name in self._frozen_text_fields - else nullcontext() + h = self._encode_text_cls( + field_name, + encoder, + flat_ids[valid], + flat_attn[valid] if flat_attn is not None else None, ) - with ctx: - out = encoder(**encode_kwargs) - h = out.last_hidden_state[:, 0, :] cls_emb = cls_emb.to(dtype=h.dtype) cls_emb[valid] = h if field_name in self.projections: From e767b1e4decb1563cf32b83351ede399f1f650e5 Mon Sep 17 00:00:00 2001 From: William Pang Date: Wed, 26 Aug 2026 13:06:18 -0700 Subject: [PATCH 33/61] Fill attention masks with dtype min and use fused SDPA --- pyhealth/models/transformer.py | 46 +++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/pyhealth/models/transformer.py b/pyhealth/models/transformer.py index 5bb2bdfe3..dc70b1310 100644 --- a/pyhealth/models/transformer.py +++ b/pyhealth/models/transformer.py @@ -7,6 +7,7 @@ from typing import Any, Dict, Optional, Tuple, Union, cast import torch +import torch.nn.functional as F from torch import nn from pyhealth.datasets import SampleDataset @@ -56,8 +57,10 @@ def forward( # avoiding a second masked_fill after softmax (saves one full # [B, H, S, S] boolean allocation and an extra copy). pad_mask = mask == 0 - scores = scores.masked_fill(pad_mask, -1e9) + scores = scores.masked_fill(pad_mask, torch.finfo(scores.dtype).min) p_attn = self.softmax(scores) + if mask is not None: + p_attn = p_attn.masked_fill(mask == 0, 0) if dropout is not None: p_attn = dropout(p_attn) @@ -150,19 +153,38 @@ def forward( ] # 2) Apply attention on all the projected vectors in batch. - if mask is not None: - mask = mask.unsqueeze(1) - x, attn = self.attention(query, key, value, mask=mask, dropout=self.dropout) - - if register_hook: - # Only store attn_map and hook during interpretability passes. - # Using .detach() gives an independent copy whose storage - # is NOT shared with the live graph, so the graph can be freed - # normally after .backward() without leaking GPU memory. + # Ordinary training uses fused SDPA. The explicit path stays behind + # register_hook=True for interpretability. + if not register_hook: + query_mask = None + attn_mask = None + if mask is not None: + valid = mask.bool() + if mask.dim() == 2: + query_mask = valid[:, None, :, None] + attn_mask = valid[:, None, None, :] + else: + query_mask = valid.any(dim=-1)[:, None, :, None] + attn_mask = valid.unsqueeze(1) + x = F.scaled_dot_product_attention( + query, + key, + value, + attn_mask=attn_mask, + dropout_p=self.dropout.p if self.training else 0.0, + ) + if query_mask is not None: + x = x * query_mask.to(x.dtype) + self.attn_map = None + self.attn_gradients = None + else: + if mask is not None: + mask = mask.unsqueeze(1) + x, attn = self.attention( + query, key, value, mask=mask, dropout=self.dropout + ) self.attn_map = attn.detach() attn.register_hook(self.save_attn_grad) - else: - self.attn_map = None # 3) "Concat" using a view and apply a final linear. x = x.transpose(1, 2).contiguous().view(batch_size, -1, self.h * self.d_k) From 92b306784f0c44dc9961853d8bc24c9b999b421a Mon Sep 17 00:00:00 2001 From: William Pang Date: Wed, 26 Aug 2026 13:15:56 -0700 Subject: [PATCH 34/61] Record batch padding and skip those slots in the unified sequence --- pyhealth/datasets/collate.py | 47 ++++++++++++-- pyhealth/datasets/utils.py | 75 +++++++++++++++++------ pyhealth/models/bottleneck_transformer.py | 4 ++ pyhealth/models/ehrmamba.py | 4 ++ pyhealth/models/jamba_ehr.py | 4 ++ pyhealth/models/rnn.py | 8 +++ pyhealth/models/transformer.py | 4 ++ 7 files changed, 124 insertions(+), 22 deletions(-) diff --git a/pyhealth/datasets/collate.py b/pyhealth/datasets/collate.py index 9e4c113c0..1267df193 100644 --- a/pyhealth/datasets/collate.py +++ b/pyhealth/datasets/collate.py @@ -17,14 +17,51 @@ from typing import Any import torch -from torch.nn.utils.rnn import pad_sequence +import torch.nn.functional as F + + +def _pad_stack(tensors: list[torch.Tensor]) -> torch.Tensor: + """Right-pad same-rank tensors to the per-dimension max, then stack. + + ``pad_sequence`` only pads dimension 0 and requires every trailing dimension + to already match. Tokenized notes are ``(n_notes, seq_len)`` and, once the + text processor pads to the longest note in a sample rather than to a fixed + ``max_length``, BOTH dimensions vary across samples. + """ + if len({t.dim() for t in tensors}) != 1: + raise ValueError("cannot pad tensors of differing rank") + target = [max(t.shape[d] for t in tensors) for d in range(tensors[0].dim())] + padded = [] + for t in tensors: + spec: list[int] = [] + for d in range(t.dim() - 1, -1, -1): + spec.extend([0, target[d] - t.shape[d]]) + padded.append(F.pad(t, spec) if any(spec) else t) + return torch.stack(padded) def _stack_or_pad(tensors: list[torch.Tensor]) -> torch.Tensor: - """Stack if all shapes match; pad along dim-0 otherwise.""" + """Stack if all shapes match; pad every ragged dimension otherwise.""" if all(t.shape == tensors[0].shape for t in tensors): return torch.stack(tensors) - return pad_sequence(tensors, batch_first=True) + return _pad_stack(tensors) + + +def _pad_mask(tensors: list[torch.Tensor]) -> torch.Tensor: + """Event-level validity for the tensor :func:`_stack_or_pad` just built. + + Batch padding is created here and nowhere else, so it has to be recorded + here. A padded slot carries value 0.0 and time 0.0, which is + indistinguishable from a real measurement taken at admission time, so a + model given no mask treats padding as data. + + This is deliberately NOT called ``mask``. A field may carry its own + ``{field}_mask`` meaning "was this value observed", which is a different + question from "is this slot real". + """ + lengths = torch.tensor([t.shape[0] for t in tensors]) + width = int(lengths.max()) + return torch.arange(width)[None, :] < lengths[:, None] def collate_temporal(batch: list[dict[str, Any]]) -> dict[str, Any]: @@ -62,6 +99,8 @@ def collate_temporal(batch: list[dict[str, Any]]) -> dict[str, Any]: sub_result[sub_key] = [None] * len(sub_vals) elif isinstance(sub_vals[0], torch.Tensor): sub_result[sub_key] = _stack_or_pad(sub_vals) + if sub_key == "time": + sub_result["pad_mask"] = _pad_mask(sub_vals) else: sub_result[sub_key] = sub_vals result[key] = sub_result @@ -75,4 +114,4 @@ def collate_temporal(batch: list[dict[str, Any]]) -> dict[str, Any]: else: result[key] = vals - return result + return result \ No newline at end of file diff --git a/pyhealth/datasets/utils.py b/pyhealth/datasets/utils.py index 0125ab4f9..b6607b5b8 100644 --- a/pyhealth/datasets/utils.py +++ b/pyhealth/datasets/utils.py @@ -7,11 +7,11 @@ import torch import litdata from dateutil.parser import parse as dateutil_parse -from torch.nn.utils.rnn import pad_sequence from torch.utils.data import DataLoader from pyhealth import BASE_CACHE_PATH from pyhealth.utils import create_directory +from pyhealth.datasets.collate import _pad_stack MODULE_CACHE_PATH = os.path.join(BASE_CACHE_PATH, "datasets") create_directory(MODULE_CACHE_PATH) @@ -251,6 +251,9 @@ def collate_fn_dict(batch: List[dict]) -> dict: return {key: [d[key] for d in batch] for key in batch[0]} +PAD_MASK_SUFFIX = "__pad_mask" + + def collate_fn_dict_with_padding(batch: List[dict]) -> dict: """Collates a batch of data into a dictionary with padding for tensor values. @@ -276,6 +279,7 @@ def collate_fn_dict_with_padding(batch: List[dict]) -> dict: transposed = list(zip(*values)) collated_elems = [] + event_lengths: Optional[List[int]] = None for elem_vals in transposed: first = elem_vals[0] @@ -286,17 +290,19 @@ def collate_fn_dict_with_padding(batch: List[dict]) -> dict: if all(v.shape == tensor_vals[0].shape for v in tensor_vals): collated_elems.append(torch.stack(tensor_vals)) else: - collated_elems.append( - pad_sequence( - tensor_vals, - batch_first=True, - padding_value=0, - ) - ) + if event_lengths is None: + event_lengths = [v.shape[0] for v in tensor_vals] + collated_elems.append(_pad_stack(tensor_vals)) else: collated_elems.append(list(elem_vals)) collated[key] = tuple(collated_elems) + if event_lengths is not None: + lengths = torch.tensor(event_lengths) + width = int(lengths.max()) + collated[f"{key}{PAD_MASK_SUFFIX}"] = ( + torch.arange(width)[None, :] < lengths[:, None] + ) # PyG Data objects (graph processor output) elif HAS_PYG and isinstance(values[0], PyGData): @@ -314,9 +320,7 @@ def collate_fn_dict_with_padding(batch: List[dict]) -> dict: # Scalars, treat as stackable collated[key] = torch.stack(values) elif values[0].dim() >= 1: - collated[key] = pad_sequence( - values, batch_first=True, padding_value=0 - ) + collated[key] = _pad_stack(values) else: raise ValueError(f"Unsupported tensor shape: {values[0].shape}") else: @@ -327,7 +331,13 @@ def collate_fn_dict_with_padding(batch: List[dict]) -> dict: def get_dataloader( - dataset: litdata.StreamingDataset, batch_size: int, shuffle: bool = False + dataset: litdata.StreamingDataset, + batch_size: int, + shuffle: bool = False, + num_workers: int = 0, + pin_memory: bool = False, + persistent_workers: bool = False, + prefetch_factor: Optional[int] = None, ) -> DataLoader: """Creates a DataLoader for a given dataset. @@ -335,18 +345,47 @@ def get_dataloader( dataset: The dataset to load data from. batch_size: The number of samples per batch. shuffle: Whether to shuffle the data at every epoch. + num_workers: Number of worker processes that load and collate batches. + pin_memory: Copy CPU tensors into page-locked memory before return. + persistent_workers: Keep the loader workers between epochs. This is valid + only when ``num_workers`` is more than 0. + prefetch_factor: Batches that each worker loads in advance. This is valid + only when ``num_workers`` is more than 0. ``None`` keeps the default + of PyTorch. Returns: A DataLoader instance for the dataset. """ dataset.set_shuffle(shuffle) - dataloader = DataLoader( - dataset, - batch_size=batch_size, - collate_fn=collate_fn_dict_with_padding, - ) + if num_workers < 0: + raise ValueError(f"num_workers must be non-negative, got {num_workers}.") + if persistent_workers and num_workers == 0: + raise ValueError("persistent_workers requires num_workers > 0.") + if prefetch_factor is not None and (num_workers == 0 or prefetch_factor <= 0): + raise ValueError( + "prefetch_factor must be positive and requires num_workers > 0." + ) - return dataloader + loader_kwargs = { + "dataset": dataset, + "batch_size": batch_size, + "collate_fn": collate_fn_dict_with_padding, + "num_workers": num_workers, + "pin_memory": pin_memory, + } + if num_workers > 0: + loader_kwargs["persistent_workers"] = persistent_workers + if prefetch_factor is not None: + loader_kwargs["prefetch_factor"] = prefetch_factor + + # StreamingDataLoader coordinates shard reads across workers. With a single + # process it adds no advantage, so keep the plain DataLoader there. + loader_class = ( + litdata.StreamingDataLoader + if isinstance(dataset, litdata.StreamingDataset) and num_workers > 0 + else DataLoader + ) + return loader_class(**loader_kwargs) def save_processors(sample_dataset, output_dir: str) -> Dict[str, str]: diff --git a/pyhealth/models/bottleneck_transformer.py b/pyhealth/models/bottleneck_transformer.py index 9d20f2549..c3b07d210 100644 --- a/pyhealth/models/bottleneck_transformer.py +++ b/pyhealth/models/bottleneck_transformer.py @@ -4,6 +4,7 @@ import torch.nn as nn from pyhealth.datasets import SampleDataset +from pyhealth.datasets.utils import PAD_MASK_SUFFIX from pyhealth.models import BaseModel from pyhealth.models.embedding import EmbeddingModel from pyhealth.models.embedding.unified import UnifiedMultimodalEmbeddingModel @@ -294,6 +295,9 @@ def _build_unified_inputs( field_dict["time"] = feature[schema.index("time")].to(self.device) if "mask" in schema: field_dict["mask"] = feature[schema.index("mask")].to(self.device) + pad_mask = kwargs.get(f"{field_name}{PAD_MASK_SUFFIX}") + if pad_mask is not None: + field_dict["pad_mask"] = pad_mask.to(self.device) inputs[field_name] = field_dict return inputs diff --git a/pyhealth/models/ehrmamba.py b/pyhealth/models/ehrmamba.py index afdbc8f35..aa5daedfa 100644 --- a/pyhealth/models/ehrmamba.py +++ b/pyhealth/models/ehrmamba.py @@ -4,6 +4,7 @@ from torch import nn from pyhealth.datasets import SampleDataset +from pyhealth.datasets.utils import PAD_MASK_SUFFIX from pyhealth.models import BaseModel from pyhealth.models.embedding import EmbeddingModel from pyhealth.models.embedding.unified import UnifiedMultimodalEmbeddingModel @@ -202,6 +203,9 @@ def _build_unified_inputs( field_dict["time"] = feature[schema.index("time")].to(self.device) if "mask" in schema: field_dict["mask"] = feature[schema.index("mask")].to(self.device) + pad_mask = kwargs.get(f"{field_name}{PAD_MASK_SUFFIX}") + if pad_mask is not None: + field_dict["pad_mask"] = pad_mask.to(self.device) inputs[field_name] = field_dict return inputs diff --git a/pyhealth/models/jamba_ehr.py b/pyhealth/models/jamba_ehr.py index 37d738f3f..a08879915 100644 --- a/pyhealth/models/jamba_ehr.py +++ b/pyhealth/models/jamba_ehr.py @@ -13,6 +13,7 @@ import torch.nn as nn from pyhealth.datasets import SampleDataset +from pyhealth.datasets.utils import PAD_MASK_SUFFIX from pyhealth.models import BaseModel from pyhealth.models.embedding import EmbeddingModel from pyhealth.models.embedding.unified import UnifiedMultimodalEmbeddingModel @@ -307,6 +308,9 @@ def _build_unified_inputs( field_dict["time"] = feature[schema.index("time")].to(self.device) if "mask" in schema: field_dict["mask"] = feature[schema.index("mask")].to(self.device) + pad_mask = kwargs.get(f"{field_name}{PAD_MASK_SUFFIX}") + if pad_mask is not None: + field_dict["pad_mask"] = pad_mask.to(self.device) inputs[field_name] = field_dict return inputs diff --git a/pyhealth/models/rnn.py b/pyhealth/models/rnn.py index 94fcea0ad..f68c368a7 100644 --- a/pyhealth/models/rnn.py +++ b/pyhealth/models/rnn.py @@ -5,6 +5,7 @@ import torch.nn.utils.rnn as rnn_utils from pyhealth.datasets import SampleDataset +from pyhealth.datasets.utils import PAD_MASK_SUFFIX from pyhealth.models import BaseModel from pyhealth.processors import ( DeepNestedFloatsProcessor, @@ -110,6 +111,10 @@ def forward( ) else: lengths = torch.sum(mask.int(), dim=-1).cpu() + # pack_padded_sequence rejects a zero length. Before batch padding + # was masked this was unreachable; a correct mask makes a sample + # with no valid event reachable, so clamp to 1. + lengths = torch.clamp(lengths, min=1) # Ensure tensor is contiguous for cuDNN compatibility x = x.contiguous() x = rnn_utils.pack_padded_sequence( @@ -259,6 +264,9 @@ def _build_unified_inputs( field_dict["time"] = feature[schema.index("time")].to(self.device) if "mask" in schema: field_dict["mask"] = feature[schema.index("mask")].to(self.device) + pad_mask = kwargs.get(f"{field_name}{PAD_MASK_SUFFIX}") + if pad_mask is not None: + field_dict["pad_mask"] = pad_mask.to(self.device) inputs[field_name] = field_dict return inputs diff --git a/pyhealth/models/transformer.py b/pyhealth/models/transformer.py index dc70b1310..9fc835df5 100644 --- a/pyhealth/models/transformer.py +++ b/pyhealth/models/transformer.py @@ -11,6 +11,7 @@ from torch import nn from pyhealth.datasets import SampleDataset +from pyhealth.datasets.utils import PAD_MASK_SUFFIX from pyhealth.models import BaseModel from pyhealth.models.embedding import EmbeddingModel from pyhealth.models.embedding.unified import UnifiedMultimodalEmbeddingModel @@ -513,6 +514,9 @@ def _build_unified_inputs( field_dict["time"] = feature[schema.index("time")].to(self.device) if "mask" in schema: field_dict["mask"] = feature[schema.index("mask")].to(self.device) + pad_mask = kwargs.get(f"{field_name}{PAD_MASK_SUFFIX}") + if pad_mask is not None: + field_dict["pad_mask"] = pad_mask.to(self.device) inputs[field_name] = field_dict return inputs From 114aeddfd8d77bcc64414c6bfd023496f944f1b4 Mon Sep 17 00:00:00 2001 From: William Pang Date: Wed, 26 Aug 2026 13:25:50 -0700 Subject: [PATCH 35/61] Thread collate pad_mask through the unified MLP path. --- pyhealth/models/mlp.py | 107 +++++++++++++++++++++++++++++++++-------- 1 file changed, 87 insertions(+), 20 deletions(-) diff --git a/pyhealth/models/mlp.py b/pyhealth/models/mlp.py index 299dc151e..e0d0cbd16 100644 --- a/pyhealth/models/mlp.py +++ b/pyhealth/models/mlp.py @@ -1,13 +1,15 @@ -from typing import Dict, cast +from typing import Any, Dict, Optional, cast import torch import torch.nn as nn from pyhealth.datasets import SampleDataset +from pyhealth.datasets.utils import PAD_MASK_SUFFIX from pyhealth.models import BaseModel from pyhealth.interpret.api import Interpretable from .embedding import EmbeddingModel +from .embedding.unified import UnifiedMultimodalEmbeddingModel class MLP(BaseModel, Interpretable): @@ -110,12 +112,14 @@ def __init__( hidden_dim: int = 128, n_layers: int = 2, activation: str = "relu", + unified_embedding: Optional[UnifiedMultimodalEmbeddingModel] = None, **kwargs, ): super(MLP, self).__init__(dataset) self.embedding_dim = embedding_dim self.hidden_dim = hidden_dim self.n_layers = n_layers + self._use_unified = unified_embedding is not None # validate kwargs for MLP layer if "input_size" in kwargs: @@ -126,9 +130,6 @@ def __init__( assert len(self.label_keys) == 1, "Only one label key is supported" self.label_key = self.label_keys[0] - # Use the EmbeddingModel to handle embedding logic - self.embedding_model = EmbeddingModel(dataset, embedding_dim) - # Set up activation function if activation == "relu": self.activation = nn.ReLU() @@ -143,18 +144,77 @@ def __init__( else: raise ValueError(f"Unsupported activation function {activation}") - # Create MLP layers for each feature - self.mlp = nn.ModuleDict() - for feature_key in self.feature_keys: - Modules = [] - Modules.append(nn.Linear(self.embedding_dim, self.hidden_dim)) - for _ in range(self.n_layers - 1): - Modules.append(self.activation) - Modules.append(nn.Linear(self.hidden_dim, self.hidden_dim)) - self.mlp[feature_key] = nn.Sequential(*Modules) - output_size = self.get_output_size() - self.fc = nn.Linear(len(self.feature_keys) * self.hidden_dim, output_size) + + if self._use_unified: + self.embedding_model = unified_embedding + modules = [nn.Linear(embedding_dim, hidden_dim)] + for _ in range(n_layers - 1): + modules.extend([self.activation, nn.Linear(hidden_dim, hidden_dim)]) + self.mlp = nn.ModuleDict({"unified": nn.Sequential(*modules)}) + self.fc = nn.Linear(hidden_dim, output_size) + else: + # Use the EmbeddingModel to handle embedding logic + self.embedding_model = EmbeddingModel(dataset, embedding_dim) + # Create MLP layers for each feature + self.mlp = nn.ModuleDict() + for feature_key in self.feature_keys: + modules = [nn.Linear(self.embedding_dim, self.hidden_dim)] + for _ in range(self.n_layers - 1): + modules.extend([self.activation, nn.Linear(self.hidden_dim, self.hidden_dim)]) + self.mlp[feature_key] = nn.Sequential(*modules) + self.fc = nn.Linear(len(self.feature_keys) * self.hidden_dim, output_size) + + def _build_unified_inputs( + self, kwargs: Dict[str, Any] + ) -> Dict[str, Dict[str, torch.Tensor]]: + """Build the inputs dict required by UnifiedMultimodalEmbeddingModel.""" + inputs: Dict[str, Dict[str, torch.Tensor]] = {} + for field_name in self.feature_keys: + feature = kwargs[field_name] + if isinstance(feature, torch.Tensor): + feature = (feature,) + schema = self.dataset.input_processors[field_name].schema() + field_dict: Dict[str, torch.Tensor] = {} + if "value" in schema: + field_dict["value"] = feature[schema.index("value")].to(self.device) + if "time" in schema: + field_dict["time"] = feature[schema.index("time")].to(self.device) + if "mask" in schema: + field_dict["mask"] = feature[schema.index("mask")].to(self.device) + pad_mask = kwargs.get(f"{field_name}{PAD_MASK_SUFFIX}") + if pad_mask is not None: + field_dict["pad_mask"] = pad_mask.to(self.device) + inputs[field_name] = field_dict + return inputs + + def _forward_unified(self, **kwargs: Any) -> Dict[str, torch.Tensor]: + """Forward pass in unified-embedding mode. + + Embeds all temporal fields jointly, mean-pools the event sequence, + applies a single MLP, and projects to label space. + """ + inputs = self._build_unified_inputs(kwargs) + out = self.embedding_model(inputs) + sequence = out["sequence"] # (B, S, E) + mask = out["mask"].float() # (B, S) + + # Masked mean-pool over the event sequence + x = (sequence * mask.unsqueeze(-1)).sum(dim=1) + x = x / mask.sum(dim=1, keepdim=True).clamp(min=1) # (B, E) + + x = self.mlp["unified"](x) # (B, hidden_dim) + logits = self.fc(x) + y_prob = self.prepare_y_prob(logits) + + results: Dict[str, torch.Tensor] = {"logit": logits, "y_prob": y_prob} + if self.label_key in kwargs: + y_true = cast(torch.Tensor, kwargs[self.label_key]).to(self.device) + results["loss"] = self.get_loss_function()(logits, y_true) + results["y_true"] = y_true + if kwargs.get("embed", False): + results["embed"] = x + return results @staticmethod def mean_pooling(x, mask): @@ -309,6 +369,11 @@ def forward( ) -> Dict[str, torch.Tensor]: """Forward propagation. + In **unified mode** (when ``unified_embedding`` was supplied at init) + the model jointly embeds all temporal fields, mean-pools the event + sequence, and processes it with a single MLP. Otherwise each field + is embedded and encoded independently. + Args: **kwargs: keyword arguments for the model. @@ -326,6 +391,9 @@ def forward( logit: the raw logits before activation. embed: (if embed=True in kwargs) the patient embedding. """ + if self._use_unified: + return self._forward_unified(**kwargs) + for feature_key in self.feature_keys: feature = kwargs[feature_key] @@ -355,10 +423,9 @@ def forward( batch_size, seq_len, inner_len = value.shape value = value.view(batch_size, seq_len * inner_len) if mask is not None: - mask = mask.to(self.device) - # Flatten mask properly if it exists - if mask.dim() == 3: - mask = mask.view(batch_size, seq_len * inner_len) + mask = mask.to(self.device) + if mask.dim() == 3: + mask = mask.view(batch_size, seq_len * inner_len) if mask is not None: mask = mask.to(self.device) @@ -441,4 +508,4 @@ def get_embedding_model(self) -> nn.Module | None: print(ret) # try loss backward - ret["loss"].backward() + ret["loss"].backward() \ No newline at end of file From c522d2e598903e510c2bd44d0ce4d2833b0373f5 Mon Sep 17 00:00:00 2001 From: William Pang Date: Wed, 26 Aug 2026 15:32:57 -0700 Subject: [PATCH 36/61] Give time embeddings a 10-year span and drop ICDLabsMIMIC4. --- pyhealth/models/embedding/unified.py | 49 +++++++++++++++++++--------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/pyhealth/models/embedding/unified.py b/pyhealth/models/embedding/unified.py index 3f786bc95..bf8050088 100644 --- a/pyhealth/models/embedding/unified.py +++ b/pyhealth/models/embedding/unified.py @@ -62,37 +62,54 @@ class SinusoidalTimeEmbedding(nn.Module): - """Continuous sinusoidal embedding for scalar time values (in hours). + """Multi-scale sinusoidal embedding for times in hours. - Identical in spirit to the positional encoding in "Attention is All You - Need" but operating on real-valued timestamps rather than integer positions. + Wavelengths are spaced geometrically from ``min_hours`` (within-stay + resolution) to ``max_hours`` (longitudinal span). The previous encoding + mapped ``t / 720 * 2π``, so every frequency wrapped every 30 days: a later + stay at +9606h produced the same embedding as +6h even after the task + stopped resetting the clock. Args: dim: Output embedding dimension (must be even). - max_hours: Maximum expected time value in hours. Values are normalised - to ``[0, 2π]`` before the sin/cos projection. Default 720 (30 days). + max_hours: Longest wavelength in hours. Default 87600 (10 years). + min_hours: Shortest wavelength in hours. Default 1.0. Shape: Input: ``(*, )`` float tensor of times in hours Output: ``(*, dim)`` """ - def __init__(self, dim: int, max_hours: float = 720.0): + def __init__( + self, + dim: int, + max_hours: float = 87600.0, + min_hours: float = 1.0, + ): super().__init__() assert dim % 2 == 0, f"dim must be even, got {dim}" + if min_hours <= 0 or max_hours <= min_hours: + raise ValueError( + f"need 0 < min_hours < max_hours, got min={min_hours}, max={max_hours}" + ) self.dim = dim - self.max_hours = max_hours + self.max_hours = float(max_hours) + self.min_hours = float(min_hours) half = dim // 2 - freqs = torch.exp( - -math.log(10000.0) * torch.arange(half, dtype=torch.float32) / (half - 1) + periods = torch.exp( + torch.linspace( + math.log(self.min_hours), + math.log(self.max_hours), + half, + dtype=torch.float32, + ) ) - self.register_buffer("freqs", freqs) # (dim//2,) + self.register_buffer("freqs", 2 * math.pi / periods) def forward(self, t: torch.Tensor) -> torch.Tensor: """:param t: ``(...,)`` float, times in hours.""" - t_norm = t / self.max_hours * 2 * math.pi # (...,) - args = t_norm.unsqueeze(-1) * self.freqs # (..., dim//2) - return torch.cat([args.sin(), args.cos()], dim=-1) # (..., dim) + args = t.unsqueeze(-1).to(dtype=self.freqs.dtype) * self.freqs + return torch.cat([args.sin(), args.cos()], dim=-1) class _MeanPool(nn.Module): @@ -159,8 +176,8 @@ class UnifiedMultimodalEmbeddingModel(nn.Module, BaseEmbeddingModel): ``dataset.input_processors`` directly. embedding_dim: Shared embedding dimension ``E'``. time_embedding: ``"sinusoidal"`` (default) or ``"learned"``. - max_time_hours: Normalisation constant for the time embedding. - Defaults to 720 h (30 days). + max_time_hours: Longest wavelength of the time embedding, in hours. + Defaults to 87600 (10 years). Shortest wavelength is 1 hour. image_size: Image size (H=W) assumed for IMAGE fields when using PatchEmbedding. Defaults to 224. image_channels: Number of input channels for IMAGE fields. Defaults to 3. @@ -206,7 +223,7 @@ def __init__( processors: dict[str, Any], embedding_dim: int = 128, time_embedding: str = "sinusoidal", - max_time_hours: float = 720.0, + max_time_hours: float = 87600.0, image_size: int = 224, image_channels: int = 3, patch_size: int = 16, From b75a8faea5abd7f0752a493250749df67d8d2ef0 Mon Sep 17 00:00:00 2001 From: William Pang Date: Wed, 26 Aug 2026 15:45:12 -0700 Subject: [PATCH 37/61] Point the old unified_embedding import at the package copy. --- pyhealth/models/embedding.py | 361 --------------------------- pyhealth/models/unified_embedding.py | 335 +------------------------ 2 files changed, 12 insertions(+), 684 deletions(-) delete mode 100644 pyhealth/models/embedding.py diff --git a/pyhealth/models/embedding.py b/pyhealth/models/embedding.py deleted file mode 100644 index 4232b2788..000000000 --- a/pyhealth/models/embedding.py +++ /dev/null @@ -1,361 +0,0 @@ -from __future__ import annotations - -from typing import Dict, Any, Optional, Union -import os - -import torch -import torch.nn as nn - -from ..datasets import SampleDataset -from ..processors import ( - MultiHotProcessor, - NestedFloatsProcessor, - NestedSequenceProcessor, - SequenceProcessor, - StageNetProcessor, - StageNetTensorProcessor, - TensorProcessor, - TimeseriesProcessor, - DeepNestedSequenceProcessor, - DeepNestedFloatsProcessor, -) -from .base_model import BaseModel - - -def _iter_text_vectors( - path: str, - embedding_dim: int, - wanted_tokens: set[str], - encoding: str = "utf-8", -) -> Dict[str, torch.Tensor]: - """Loads word vectors from a text file (e.g., GloVe) for a subset of tokens. - - Expected format: one token per line followed by embedding_dim floats. - - This function reads the file line-by-line and only retains vectors for - tokens present in `wanted_tokens`. - """ - - if not os.path.exists(path): - raise FileNotFoundError(f"pretrained embedding file not found: {path}") - - vectors: Dict[str, torch.Tensor] = {} - with open(path, "r", encoding=encoding) as f: - for line in f: - line = line.strip() - if not line: - continue - parts = line.split() - # token + embedding_dim values - if len(parts) < embedding_dim + 1: - continue - token = parts[0] - if token not in wanted_tokens: - continue - try: - vec = torch.tensor( - [float(x) for x in parts[1 : embedding_dim + 1]], - dtype=torch.float, - ) - except ValueError: - continue - vectors[token] = vec - return vectors - - -def init_embedding_with_pretrained( - embedding: nn.Embedding, - code_vocab: Dict[Any, int], - pretrained_path: str, - embedding_dim: int, - pad_token: str = "", - unk_token: str = "", - normalize: bool = False, - freeze: bool = False, -) -> int: - """Initializes an nn.Embedding from a pretrained text-vector file. - - Tokens not found in the pretrained file are left as the module's existing - random initialization. - - Returns: - int: number of tokens successfully initialized from the file. - """ - - # Build wanted token set (stringified) - vocab_tokens = {str(t) for t in code_vocab.keys()} - vectors = _iter_text_vectors(pretrained_path, embedding_dim, vocab_tokens) - - loaded = 0 - with torch.no_grad(): - for tok, idx in code_vocab.items(): - tok_s = str(tok) - if tok_s in vectors: - vec = vectors[tok_s] - if normalize: - vec = vec / (vec.norm(p=2) + 1e-12) - embedding.weight[idx].copy_(vec) - loaded += 1 - - # Ensure pad row is zero - if pad_token in code_vocab: - embedding.weight[code_vocab[pad_token]].zero_() - # If embedding has a padding_idx, keep it consistent - if embedding.padding_idx is not None: - embedding.weight[embedding.padding_idx].zero_() - - if freeze: - embedding.weight.requires_grad_(False) - - return loaded - - -class EmbeddingModel(BaseModel): - """ - EmbeddingModel is responsible for creating embedding layers for different types of input data. - - This model automatically creates appropriate embedding transformations based on the processor type: - - - SequenceProcessor: nn.Embedding - Input: (batch, seq_len) - Output: (batch, seq_len, embedding_dim) - - - NestedSequenceProcessor: nn.Embedding - Input: (batch, num_visits, max_codes_per_visit) - Output: (batch, num_visits, max_codes_per_visit, embedding_dim) - - - DeepNestedSequenceProcessor: nn.Embedding - Input: (batch, num_groups, num_visits, max_codes_per_visit) - Output: (batch, num_groups, num_visits, max_codes_per_visit, embedding_dim) - - - TimeseriesProcessor / NestedFloatsProcessor / DeepNestedFloatsProcessor / StageNetTensorProcessor: - nn.Linear over the last dimension - Input: (..., size) - Output: (..., embedding_dim) - - - TensorProcessor: nn.Linear (size inferred from first sample) - - - MultiHotProcessor: nn.Linear over multi-hot vector - """ - - def __init__( - self, - dataset: SampleDataset, - embedding_dim: int = 128, - pretrained_emb_path: Optional[Union[str, Dict[str, str]]] = None, - freeze_pretrained: bool = False, - normalize_pretrained: bool = False, - ): - super().__init__(dataset) - self.embedding_dim = embedding_dim - self.embedding_layers = nn.ModuleDict() - - for field_name, processor in self.dataset.input_processors.items(): - # Deep categorical: use special module that collapses last dim to embedding_dim - - # Regular categorical sequences -> nn.Embedding (adds embedding dim) - if isinstance( - processor, - ( - SequenceProcessor, - StageNetProcessor, - NestedSequenceProcessor, - DeepNestedSequenceProcessor, - ), - ): - vocab_size = len(processor.code_vocab) - - if isinstance( - processor, (NestedSequenceProcessor, DeepNestedSequenceProcessor) - ): - self.embedding_layers[field_name] = nn.Embedding( - num_embeddings=vocab_size, - embedding_dim=embedding_dim, - padding_idx=0, - ) - else: - self.embedding_layers[field_name] = nn.Embedding( - num_embeddings=vocab_size, - embedding_dim=embedding_dim, - padding_idx=0, - ) - - # Optional pretrained initialization (e.g., GloVe). - if pretrained_emb_path is not None: - if isinstance(pretrained_emb_path, str): - path = pretrained_emb_path - else: - path = pretrained_emb_path.get(field_name) - if path: - init_embedding_with_pretrained( - self.embedding_layers[field_name], - processor.code_vocab, - path, - embedding_dim=embedding_dim, - normalize=normalize_pretrained, - freeze=freeze_pretrained, - ) - - # Numeric features (including deep nested floats) -> nn.Linear over last dim - elif isinstance( - processor, - ( - TimeseriesProcessor, - StageNetTensorProcessor, - NestedFloatsProcessor, - DeepNestedFloatsProcessor, - ), - ): - # Assuming processor.size() returns the last-dim size - in_features = processor.size() - self.embedding_layers[field_name] = nn.Linear( - in_features=in_features, out_features=embedding_dim - ) - - elif isinstance(processor, TensorProcessor): - # Infer size from first sample - sample_tensor = None - for sample in dataset: - if field_name in sample: - sample_tensor = processor.process(sample[field_name]) - break - if sample_tensor is not None: - input_size = ( - sample_tensor.shape[-1] if sample_tensor.dim() > 0 else 1 - ) - self.embedding_layers[field_name] = nn.Linear( - in_features=input_size, out_features=embedding_dim - ) - - elif isinstance(processor, MultiHotProcessor): - num_categories = processor.size() - self.embedding_layers[field_name] = nn.Linear( - in_features=num_categories, out_features=embedding_dim - ) - - # Smart Processor (Token-based) -> Transformers - elif hasattr(processor, "is_token") and processor.is_token(): - try: - from transformers import AutoModel - except ImportError: - raise ImportError( - "Please install `transformers` to use token-based processors." - ) - - # Load the model - self.embedding_layers[field_name] = AutoModel.from_pretrained( - processor.tokenizer_model - ) - - # Check if we need projection - if ( - self.embedding_layers[field_name].config.hidden_size - != self.embedding_dim - ): - self.embedding_layers[f"{field_name}_proj"] = nn.Linear( - self.embedding_layers[field_name].config.hidden_size, - self.embedding_dim, - ) - - else: - print( - "Warning: No embedding created for field due to lack of compatible processor:", - field_name, - ) - - def forward( - self, - inputs: Dict[str, torch.Tensor], - masks: Dict[str, torch.Tensor] = None, - output_mask: bool = False, - ) -> ( - Dict[str, torch.Tensor] - | tuple[Dict[str, torch.Tensor], Dict[str, torch.Tensor]] - ): - - embedded: Dict[str, torch.Tensor] = {} - out_masks: Dict[str, torch.Tensor] = {} if output_mask else None - - for field_name, tensor in inputs.items(): - processor = self.dataset.input_processors.get(field_name, None) - - if field_name not in self.embedding_layers: - # No embedding layer -> passthrough - embedded[field_name] = tensor - continue - - # Check if it's a transformer model - layer = self.embedding_layers[field_name] - - # Check for transformers.PreTrainedModel (but without importing if possible, use class name check) - # or check if it has 'config' attribute - if hasattr(layer, "config") and hasattr(layer, "forward"): - # It's likely a transformer - tensor = tensor.to(self.device).long() # Ensure LongTensor for IDs - - mask = None - if masks is not None and field_name in masks: - mask = masks[field_name].to(self.device) - - # Handle 3D input (Batch, Num_Notes, Seq_Len) - is_3d = inputs[field_name].dim() == 3 - - if is_3d: - b, n, l = inputs[field_name].shape - tensor = tensor.view(b * n, l) - if mask is not None: - mask = mask.view(b * n, l) - - # Forward pass through transformer - output = layer(input_ids=tensor, attention_mask=mask) - x = output.last_hidden_state # (Batch, Seq, Hidden) - - if is_3d: - # If we had 3D input, we MUST pool the sequence dim (L) to get one vector per note - # Resulting shape: (B, N, H) - - # Pool L dim -> (B*N, H) using CLS token (index 0) - x = x[:, 0, :] - - # Check projections - if f"{field_name}_proj" in self.embedding_layers: - x = self.embedding_layers[f"{field_name}_proj"](x) - - x = x.view(b, n, -1) - - else: - # 2D input (Batch, Seq) -> (Batch, Seq, Hidden) - # No pooling, treating as sequence of tokens (word embeddings) - if f"{field_name}_proj" in self.embedding_layers: - x = self.embedding_layers[f"{field_name}_proj"](x) - - embedded[field_name] = x - - else: - # Standard layers - tensor = tensor.to(self.device) - embedded[field_name] = layer(tensor) - - if output_mask: - # Generate a mask for this field - # For transformers, we might already have a mask, or use pad token - if masks is not None and field_name in masks: - out_masks[field_name] = masks[field_name].to(self.device) - elif hasattr(processor, "code_vocab"): - pad_idx = processor.code_vocab.get("", 0) - out_masks[field_name] = tensor != pad_idx - else: - # Default mask generation (e.g. for simple linear layers where 0 might be padding?) - # Be careful changing this behavior. - # Previous code: - # masks[field_name] = (tensor != pad_idx) -> where pad_idx was 0 default - pad_idx = 0 - out_masks[field_name] = tensor != pad_idx - - if output_mask: - return embedded, out_masks - else: - return embedded - - def __repr__(self) -> str: - return f"EmbeddingModel(embedding_layers={self.embedding_layers})" diff --git a/pyhealth/models/unified_embedding.py b/pyhealth/models/unified_embedding.py index 014326b41..ba4a3edd4 100644 --- a/pyhealth/models/unified_embedding.py +++ b/pyhealth/models/unified_embedding.py @@ -1,327 +1,16 @@ -"""UnifiedMultimodalEmbeddingModel — temporally aligned multimodal embedding. +"""Deprecated import path for the unified multimodal encoder. -Takes K temporal features ( dict outputs from ``TemporalFeatureProcessor`` -subclasses ), embeds each event with a modality-specific encoder, then -interleaves all events on a shared timeline by sorting on timestamp and adding -sinusoidal time embeddings + learned modality-type embeddings. - -Output shape: ``(B, S_total, E')`` — a single sequence of events usable by -any downstream sequence model (Transformer, Mamba, RNN, …). - -Quickstart:: - - from pyhealth.models.unified_embedding import UnifiedMultimodalEmbeddingModel - from pyhealth.datasets.collate import collate_temporal - model = UnifiedMultimodalEmbeddingModel(dataset, embedding_dim=128) - # inside forward: - # inputs = {field: {"value": Tensor, "time": Tensor, ...}, ...} - out = model(inputs) - # out["sequence"]: (B, S_total, 128) - # out["mask"]: (B, S_total) — 1 = real event, 0 = padding - # out["time"]: (B, S_total) — hours from first event +Use ``pyhealth.models.embedding`` instead. This module re-exports the live +classes so older ``from pyhealth.models.unified_embedding import ...`` call +sites keep working against one implementation. """ -from __future__ import annotations - -import math -from typing import Any - -import torch -import torch.nn as nn - -from pyhealth.processors.base_processor import ModalityType, TemporalFeatureProcessor - - -# ── Helpers ─────────────────────────────────────────────────────────────────── - - -class SinusoidalTimeEmbedding(nn.Module): - """Continuous sinusoidal embedding for scalar time values (in hours). - - Identical in spirit to the positional encoding in "Attention is All You - Need" but operating on real-valued timestamps rather than integer positions. - - Args: - dim: Output embedding dimension (must be even). - max_hours: Maximum expected time value in hours. Values are normalised - to ``[0, 2π]`` before the sin/cos projection. Default 720 (30 days). - - Shape: - Input: ``(*, )`` float tensor of times in hours - Output: ``(*, dim)`` - """ - - def __init__(self, dim: int, max_hours: float = 720.0): - super().__init__() - assert dim % 2 == 0, f"dim must be even, got {dim}" - self.dim = dim - self.max_hours = max_hours - half = dim // 2 - freqs = torch.exp( - -math.log(10000.0) * torch.arange(half, dtype=torch.float32) / (half - 1) - ) - self.register_buffer("freqs", freqs) # (dim//2,) - - def forward(self, t: torch.Tensor) -> torch.Tensor: - """:param t: ``(...,)`` float, times in hours.""" - t_norm = t / self.max_hours * 2 * math.pi # (...,) - args = t_norm.unsqueeze(-1) * self.freqs # (..., dim//2) - return torch.cat([args.sin(), args.cos()], dim=-1) # (..., dim) - - -def _build_image_encoder(embedding_dim: int) -> nn.Module: - """Lightweight 5-layer CNN encoder: C × H × W → embedding_dim. - - Uses ``torchvision.models.resnet18`` pre-trained backbone, strips the - final FC layer, and adds a projection to ``embedding_dim``. Falls back to - a toy Conv-pool-flatten network if torchvision is not installed. - """ - try: - import torchvision.models as tv - - backbone = tv.resnet18(weights=None) - in_features = backbone.fc.in_features - backbone.fc = nn.Linear(in_features, embedding_dim) - return backbone - except ImportError: - # Minimal fallback: single conv → global avg pool → linear - return nn.Sequential( - nn.Conv2d(3, 32, 3, padding=1), - nn.ReLU(), - nn.AdaptiveAvgPool2d(1), - nn.Flatten(), - nn.Linear(32, embedding_dim), - ) - - -# ── Main model ─────────────────────────────────────────────────────────────── - - -class UnifiedMultimodalEmbeddingModel(nn.Module): - """Embed heterogeneous temporal features into a single aligned sequence. - - **All** input processors must be ``TemporalFeatureProcessor`` subclasses. - Non-temporal processors (e.g. ``SequenceProcessor``, ``MultiHotProcessor``) - are rejected with a clear error — use the existing ``EmbeddingModel`` for - those fields. - - Algorithm - --------- - For each temporal field: - - 1. Route ``inputs[field]["value"]`` through a modality-specific encoder → - ``(B, N_i, E')`` per-event embeddings. - 2. Retrieve ``inputs[field]["time"]`` → ``(B, N_i)`` timestamps (hours). - 3. (Optional) Retrieve ``inputs[field]["mask"]`` → ``(B, N_i, L)`` or - ``(B, N_i)`` attention mask; reduced to event-level ``(B, N_i)`` if - token-level. - - Then: - - 4. Concatenate across all fields → ``(B, S_total, E')``. - 5. Sort events along dim=1 by timestamp (ascending). - 6. Add ``SinusoidalTimeEmbedding(time)`` + ``type_embedding(modality_idx)``. - 7. Return ``{"sequence", "time", "mask", "type_ids"}``. - - Args: - processors: ``dict[field_name, TemporalFeatureProcessor]`` — the - processors for each temporal field in the dataset. Pass - ``dataset.input_processors`` directly. - embedding_dim: Shared embedding dimension ``E'``. - time_embedding: ``"sinusoidal"`` (default) or ``"learned"``. - max_time_hours: Normalisation constant for the time embedding. - Defaults to 720 h (30 days). - - Example:: - - model = UnifiedMultimodalEmbeddingModel( - processors=dataset.input_processors, - embedding_dim=128, - ) - # inputs: {field: {"value": Tensor, "time": Tensor, "mask": Tensor}} - out = model(inputs) - seq = out["sequence"] # (B, S_total, 128) - mask = out["mask"] # (B, S_total) float, 1=valid 0=pad - """ - - def __init__( - self, - processors: dict[str, Any], - embedding_dim: int = 128, - time_embedding: str = "sinusoidal", - max_time_hours: float = 720.0, - ): - super().__init__() - self.embedding_dim = embedding_dim - - self.encoders: nn.ModuleDict = nn.ModuleDict() - self.projections: nn.ModuleDict = nn.ModuleDict() - self.modality_types: dict[str, ModalityType] = {} - - for field_name, processor in processors.items(): - if not isinstance(processor, TemporalFeatureProcessor): - raise TypeError( - f"UnifiedMultimodalEmbeddingModel requires every input processor " - f"to be a TemporalFeatureProcessor subclass, but '{field_name}' " - f"uses {type(processor).__name__}. For non-temporal fields use " - f"the existing EmbeddingModel." - ) - - m = processor.modality() - self.modality_types[field_name] = m - - if m == ModalityType.CODE: - vocab_size = processor.value_dim() - self.encoders[field_name] = nn.Embedding( - vocab_size, embedding_dim, padding_idx=0 - ) - - elif m == ModalityType.TEXT: - if processor.is_token(): - from transformers import AutoModel - - bert = AutoModel.from_pretrained(processor.tokenizer_model) - self.encoders[field_name] = bert - hidden = bert.config.hidden_size - if hidden != embedding_dim: - self.projections[field_name] = nn.Linear(hidden, embedding_dim) - else: - raise ValueError( - f"TEXT processor '{field_name}' must use a tokenizer " - f"(set tokenizer_model=...) to be used with " - f"UnifiedMultimodalEmbeddingModel." - ) - - elif m == ModalityType.IMAGE: - self.encoders[field_name] = _build_image_encoder(embedding_dim) - - elif m in (ModalityType.NUMERIC, ModalityType.SIGNAL): - in_features = processor.value_dim() - self.encoders[field_name] = nn.Linear(in_features, embedding_dim) - - else: - raise NotImplementedError( - f"No encoder implemented for modality {m!r} (field '{field_name}')." - ) - - # Shared type embedding — one vector per unique modality in this dataset - unique_modalities = sorted(set(self.modality_types.values())) - self._modality_to_idx: dict[ModalityType, int] = { - mod: i for i, mod in enumerate(unique_modalities) - } - self.type_embedding = nn.Embedding(len(unique_modalities), embedding_dim) - - # Time embedding - if time_embedding == "sinusoidal": - self.time_embed = SinusoidalTimeEmbedding(embedding_dim, max_time_hours) - else: - raise NotImplementedError("Only 'sinusoidal' time embedding is implemented.") - - # ── Forward ─────────────────────────────────────────────────────────────── - - def forward( - self, - inputs: dict[str, dict[str, torch.Tensor]], - ) -> dict[str, torch.Tensor]: - """Encode and temporally align all temporal features. - - Args: - inputs: ``{field_name: {"value": Tensor, "time": Tensor, - "mask": Tensor (optional)}}`` - — one dict per temporal feature, exactly as produced by - ``collate_temporal``. - - Returns: - A dict with keys: - - * ``"sequence"`` — ``(B, S_total, E')`` temporally-sorted events - * ``"time"`` — ``(B, S_total)`` timestamps (hours) - * ``"mask"`` — ``(B, S_total)`` 1=real event, 0=padding - * ``"type_ids"`` — ``(B, S_total)`` modality index per event - """ - all_embeddings: list[torch.Tensor] = [] - all_times: list[torch.Tensor] = [] - all_masks: list[torch.Tensor] = [] - all_types: list[torch.Tensor] = [] - - for field_name, feat_dict in inputs.items(): - value = feat_dict["value"] # (B, N_i, ...) or (B, S, F) - time = feat_dict["time"] # (B, N_i) - mask = feat_dict.get("mask") - - if time is None: - # Fallback: treat every event as occurring at t=0 - time = torch.zeros(value.shape[:2], device=value.device) - - modality = self.modality_types[field_name] - encoder = self.encoders[field_name] - - # ── Encode ──────────────────────────────────────────────────── - if modality == ModalityType.CODE: - emb = encoder(value) # (B, S, E') - - elif modality == ModalityType.TEXT: - b, n, l = value.shape - flat_ids = value.view(b * n, l) - flat_mask = mask.view(b * n, l) if mask is not None else None - out = encoder(input_ids=flat_ids, attention_mask=flat_mask) - cls_emb = out.last_hidden_state[:, 0, :] # (B*N, H) - if field_name in self.projections: - cls_emb = self.projections[field_name](cls_emb) - emb = cls_emb.view(b, n, -1) # (B, N, E') - - elif modality == ModalityType.IMAGE: - b, n, c, h, w = value.shape - flat_imgs = value.view(b * n, c, h, w) - img_emb = encoder(flat_imgs) # (B*N, E') - emb = img_emb.view(b, n, -1) - - else: # NUMERIC / SIGNAL - emb = encoder(value) # (B, T, E') - - # ── Build event-level validity mask ─────────────────────────── - if mask is None: - event_mask = torch.ones(emb.shape[:2], device=emb.device) - else: - if mask.dim() > time.dim(): - # token-level (B, N, L) → event-level (B, N) - event_mask = (mask.sum(dim=-1) > 0).float() - else: - event_mask = mask.float() - - # ── Modality type indices ───────────────────────────────────── - type_idx = self._modality_to_idx[modality] - type_ids = torch.full( - emb.shape[:2], type_idx, dtype=torch.long, device=emb.device - ) - - all_embeddings.append(emb) - all_times.append(time) - all_masks.append(event_mask) - all_types.append(type_ids) - - # ── Concatenate across all fields ───────────────────────────────── - cat_emb = torch.cat(all_embeddings, dim=1) # (B, S_total, E') - cat_time = torch.cat(all_times, dim=1) # (B, S_total) - cat_mask = torch.cat(all_masks, dim=1) # (B, S_total) - cat_types = torch.cat(all_types, dim=1) # (B, S_total) - - # ── Sort by time ────────────────────────────────────────────────── - sort_idx = cat_time.argsort(dim=1) - cat_emb = cat_emb.gather( - 1, sort_idx.unsqueeze(-1).expand_as(cat_emb) - ) - cat_time = cat_time.gather(1, sort_idx) - cat_mask = cat_mask.gather(1, sort_idx) - cat_types = cat_types.gather(1, sort_idx) - # ── Add time + type embeddings ──────────────────────────────────── - time_emb = self.time_embed(cat_time) # (B, S_total, E') - type_emb = self.type_embedding(cat_types) # (B, S_total, E') - final = cat_emb + time_emb + type_emb # (B, S_total, E') +from pyhealth.models.embedding.unified import ( + SinusoidalTimeEmbedding, + UnifiedMultimodalEmbeddingModel, +) - return { - "sequence": final, # (B, S_total, E') - "time": cat_time, # (B, S_total) - "mask": cat_mask, # (B, S_total) - "type_ids": cat_types, # (B, S_total) - } +__all__ = [ + "SinusoidalTimeEmbedding", + "UnifiedMultimodalEmbeddingModel", +] \ No newline at end of file From 3e03d7bef14cd5481b6d70a7bd537d52cbe98972 Mon Sep 17 00:00:00 2001 From: William Pang Date: Wed, 26 Aug 2026 15:50:09 -0700 Subject: [PATCH 38/61] Refuse unknown amp_dtype instead of silently selecting fp16. --- pyhealth/trainer.py | 38 +++++++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/pyhealth/trainer.py b/pyhealth/trainer.py index 8a86cc709..6d7ef176f 100644 --- a/pyhealth/trainer.py +++ b/pyhealth/trainer.py @@ -20,6 +20,40 @@ logger = logging.getLogger(__name__) +_AMP_DTYPES = { + "bf16": torch.bfloat16, + "bfloat16": torch.bfloat16, + "fp16": torch.float16, + "float16": torch.float16, +} + + +def resolve_amp_dtype(amp_dtype: str, use_amp: bool = False) -> torch.dtype: + """Validate the mixed-precision dtype and return it. + + The previous expression was ``bfloat16 if amp_dtype == "bf16" else float16``, + so every other spelling silently selected fp16 and, through GradScaler, + changed gradient behaviour too. + """ + name = str(amp_dtype).lower() + if name not in _AMP_DTYPES: + raise ValueError( + f"amp_dtype must be one of {sorted(_AMP_DTYPES)}, got {amp_dtype!r}." + ) + dtype = _AMP_DTYPES[name] + if ( + use_amp + and dtype == torch.bfloat16 + and torch.cuda.is_available() + and not torch.cuda.is_bf16_supported() + ): + raise RuntimeError( + "bf16 mixed precision was requested, but this CUDA device does not " + "support bf16." + ) + return dtype + + def is_best(best_score: float, score: float, monitor_criterion: str) -> bool: if monitor_criterion == "max": return score > best_score @@ -168,9 +202,7 @@ def train( if optimizer_params is None: optimizer_params = {"lr": 1e-3} - _amp_dtype = ( - torch.bfloat16 if amp_dtype == "bf16" else torch.float16 - ) + _amp_dtype = resolve_amp_dtype(amp_dtype, use_amp=use_amp) # GradScaler only needed for fp16; bf16 has fp32 dynamic range scaler = ( torch.cuda.amp.GradScaler() From 7a4691b7ae190a731aba91cf5a55d97e61dcbdc5 Mon Sep 17 00:00:00 2001 From: William Pang Date: Wed, 26 Aug 2026 15:52:16 -0700 Subject: [PATCH 39/61] Point AMP autocast at the trainer device instead of hardcoding CUDA --- pyhealth/trainer.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pyhealth/trainer.py b/pyhealth/trainer.py index 6d7ef176f..2085221b3 100644 --- a/pyhealth/trainer.py +++ b/pyhealth/trainer.py @@ -54,6 +54,14 @@ def resolve_amp_dtype(amp_dtype: str, use_amp: bool = False) -> torch.dtype: return dtype +def autocast_device_type(device) -> str: + """Device type string for ``torch.autocast``, from the trainer device.""" + name = str(device).split(":", 1)[0].lower() + if name in ("cuda", "cpu", "mps"): + return name + return "cpu" + + def is_best(best_score: float, score: float, monitor_criterion: str) -> bool: if monitor_criterion == "max": return score > best_score @@ -277,7 +285,10 @@ def train( data = next(data_iterator) # forward (with optional AMP) if use_amp: - with torch.autocast(device_type="cuda", dtype=_amp_dtype): + with torch.autocast( + device_type=autocast_device_type(self.device), + dtype=_amp_dtype, + ): output = self.model(**data) loss = output["loss"] / accumulation_steps else: From 4b78f28e398820740f6286dabb56c6519011f3ef Mon Sep 17 00:00:00 2001 From: William Pang Date: Wed, 26 Aug 2026 15:58:55 -0700 Subject: [PATCH 40/61] Default to a full stay and stamp admission-context notes at admit. --- .../unified_embedding_e2e_mimic4.py | 10 +++++++++- pyhealth/tasks/multimodal_mimic4.py | 6 +++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/examples/mortality_prediction/unified_embedding_e2e_mimic4.py b/examples/mortality_prediction/unified_embedding_e2e_mimic4.py index d7182955a..ba7327ad3 100644 --- a/examples/mortality_prediction/unified_embedding_e2e_mimic4.py +++ b/examples/mortality_prediction/unified_embedding_e2e_mimic4.py @@ -431,7 +431,15 @@ def parse_args() -> argparse.Namespace: "--dev 5000 limits to 5000. Omit for full dataset." ), ) - parser.add_argument("--observation-window-hours", type=int, default=24) + parser.add_argument( + "--observation-window-hours", + type=int, + default=None, + help=( + "If set, collect labs/CXR/radiology only this many hours from each " + "admission. Default: full stay (through discharge)." + ), + ) parser.add_argument( "--freeze-encoder", action="store_true", diff --git a/pyhealth/tasks/multimodal_mimic4.py b/pyhealth/tasks/multimodal_mimic4.py index 768c75819..a2861fbea 100644 --- a/pyhealth/tasks/multimodal_mimic4.py +++ b/pyhealth/tasks/multimodal_mimic4.py @@ -484,7 +484,7 @@ class NotesLabsMIMIC4(BaseMultimodalMIMIC4Task): Args: window_hours: Hours from admission for lab collection. ``None`` - collects for the full admission span. Default: 24. + collects for the full admission span. Default: None. include_icd: When ``True``, collect discharge-coded ICD codes and add ``icd_codes`` to the sample dict / input schema. Default: ``False``. """ @@ -846,7 +846,7 @@ class LabsMIMIC4(BaseMultimodalMIMIC4Task): Args: window_hours: Hours from admission to collect lab measurements. - ``None`` collects for the full admission span. Default: 24. + ``None`` collects for the full admission span. Default: None. """ PADDING: int = 0 @@ -859,7 +859,7 @@ class LabsMIMIC4(BaseMultimodalMIMIC4Task): } output_schema: ClassVar[Dict] = {"mortality": "binary"} - def __init__(self, window_hours: Optional[float] = 24) -> None: + def __init__(self, window_hours: Optional[float] = None) -> None: super().__init__(window_hours=window_hours) def __call__(self, patient: Any) -> List[Dict[str, Any]]: # type: ignore[override] From 32ca3a6c7c6ced3a6af4ed22ef7ea4d38b758ce7 Mon Sep 17 00:00:00 2001 From: William Pang Date: Wed, 26 Aug 2026 16:48:01 -0700 Subject: [PATCH 41/61] Add scripts for lamba labs and update condor scripts --- .../condor/labs_notes/run_labs_notes_rnn.sh | 13 +- .../labs_notes_cxr/run_labs_notes_cxr_rnn.sh | 13 +- ...bs_notes_bottleneck_transformer_variant.py | 138 +++++++++++++++++ .../tmux_run_labs_notes_ehrmamba_variant.py | 136 +++++++++++++++++ .../tmux_run_labs_notes_jambaehr_variant.py | 142 ++++++++++++++++++ .../tmux_run_labs_notes_rnn_variant.py | 134 +++++++++++++++++ ...tmux_run_labs_notes_transformer_variant.py | 134 +++++++++++++++++ 7 files changed, 692 insertions(+), 18 deletions(-) create mode 100644 scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_bottleneck_transformer_variant.py create mode 100644 scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_ehrmamba_variant.py create mode 100644 scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_jambaehr_variant.py create mode 100644 scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_rnn_variant.py create mode 100644 scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_transformer_variant.py diff --git a/scripts/will/condor/labs_notes/run_labs_notes_rnn.sh b/scripts/will/condor/labs_notes/run_labs_notes_rnn.sh index 33661ff11..410a33b2c 100755 --- a/scripts/will/condor/labs_notes/run_labs_notes_rnn.sh +++ b/scripts/will/condor/labs_notes/run_labs_notes_rnn.sh @@ -21,21 +21,20 @@ CACHE_DIR="${CACHE_DIR:-/shared/eng/wp14/pyhealth_cache_labs_notes}" OUTPUT_DIR="${OUTPUT_DIR:-/home/wp14/output}" CONDA_SH="${CONDA_SH:-}" -DEV_MODE="${DEV_MODE:-1}" +DEV_MODE="${DEV_MODE:-0}" EMBEDDING_DIM="${EMBEDDING_DIM:-128}" HIDDEN_DIM="${HIDDEN_DIM:-128}" RNN_TYPE="${RNN_TYPE:-GRU}" RNN_LAYERS="${RNN_LAYERS:-2}" DROPOUT="${DROPOUT:-0.1}" -EPOCHS="${EPOCHS:-15}" +EPOCHS="${EPOCHS:-50}" BATCH_SIZE="${BATCH_SIZE:-32}" -LR="${LR:-1e-3}" +LR="${LR:-1e-4}" WEIGHT_DECAY="${WEIGHT_DECAY:-1e-5}" PATIENCE="${PATIENCE:-5}" NUM_WORKERS="${NUM_WORKERS:-4}" FREEZE_ENCODER="${FREEZE_ENCODER:-1}" -INCLUDE_VITALS="${INCLUDE_VITALS:-0}" -USE_AMP="${USE_AMP:-0}" +USE_AMP="${USE_AMP:-1}" AMP_DTYPE="${AMP_DTYPE:-bf16}" # Condor GPU cgroups can expose a truncated CUDA_VISIBLE_DEVICES UUID that @@ -168,10 +167,6 @@ if [[ "${FREEZE_ENCODER}" == "1" ]]; then COMMON+=(--freeze-encoder) fi -if [[ "${INCLUDE_VITALS}" == "1" ]]; then - COMMON+=(--include-vitals) -fi - if [[ "${USE_AMP}" == "1" ]]; then COMMON+=(--use-amp --amp-dtype "${AMP_DTYPE}") fi diff --git a/scripts/will/condor/labs_notes_cxr/run_labs_notes_cxr_rnn.sh b/scripts/will/condor/labs_notes_cxr/run_labs_notes_cxr_rnn.sh index aa9037ae7..23ad6a367 100755 --- a/scripts/will/condor/labs_notes_cxr/run_labs_notes_cxr_rnn.sh +++ b/scripts/will/condor/labs_notes_cxr/run_labs_notes_cxr_rnn.sh @@ -24,21 +24,20 @@ CACHE_DIR="${CACHE_DIR:-/shared/rsaas/wp14/pyhealth_cache_labs_notes_cxr}" OUTPUT_DIR="${OUTPUT_DIR:-/home/wp14/output}" CONDA_SH="${CONDA_SH:-}" -DEV_MODE="${DEV_MODE:-1}" +DEV_MODE="${DEV_MODE:-0}" EMBEDDING_DIM="${EMBEDDING_DIM:-128}" HIDDEN_DIM="${HIDDEN_DIM:-128}" RNN_TYPE="${RNN_TYPE:-GRU}" RNN_LAYERS="${RNN_LAYERS:-2}" DROPOUT="${DROPOUT:-0.1}" -EPOCHS="${EPOCHS:-15}" +EPOCHS="${EPOCHS:-50}" BATCH_SIZE="${BATCH_SIZE:-32}" -LR="${LR:-1e-3}" +LR="${LR:-1e-4}" WEIGHT_DECAY="${WEIGHT_DECAY:-1e-5}" PATIENCE="${PATIENCE:-5}" NUM_WORKERS="${NUM_WORKERS:-4}" FREEZE_ENCODER="${FREEZE_ENCODER:-1}" -INCLUDE_VITALS="${INCLUDE_VITALS:-0}" -USE_AMP="${USE_AMP:-0}" +USE_AMP="${USE_AMP:-1}" AMP_DTYPE="${AMP_DTYPE:-bf16}" # Condor GPU cgroups can expose a truncated CUDA_VISIBLE_DEVICES UUID that @@ -174,10 +173,6 @@ if [[ "${FREEZE_ENCODER}" == "1" ]]; then COMMON+=(--freeze-encoder) fi -if [[ "${INCLUDE_VITALS}" == "1" ]]; then - COMMON+=(--include-vitals) -fi - if [[ "${USE_AMP}" == "1" ]]; then COMMON+=(--use-amp --amp-dtype "${AMP_DTYPE}") fi diff --git a/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_bottleneck_transformer_variant.py b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_bottleneck_transformer_variant.py new file mode 100644 index 000000000..b91d975ff --- /dev/null +++ b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_bottleneck_transformer_variant.py @@ -0,0 +1,138 @@ +project_dir = "PyHealth" +seed = 12 +conda_env = "pyhealth2" +ehr_root = "/home/ubuntu/mimiciv-data/ehr" +note_root = "/home/ubuntu/mimiciv-data" +cache_dir = "/home/ubuntu/mimiciv-data/pyhealth_cache_labs_notes" +logs_dir = "/home/ubuntu/logs" +output_dir = "/home/ubuntu/output" +embedding_dim = 128 +hidden_dim = 128 +heads = 4 +num_layers = 2 +bottlenecks_n = 4 +fusion_startidx = 1 +dropout = 0.1 +use_amp = True +amp_dtype = "bf16" +epochs = 50 +batch_size = 32 +lr = 1e-4 +weight_decay = 1e-5 +patience = 5 +num_workers = 4 +freeze_encoder = True +dev = False +use_old_cache = False +use_wandb = True +wandb_project = "pyhealth-multimodal-labs-notes" +wandb_run_name = None # defaults to "{model}_seed{seed}" if unset +cuda_visible_devices = "0" +session_name = f"bottleneck_transformer_labs_notes_s{seed}" + +# ── Step 0a: Check which CUDA GPU is available ─────────────────────────────── +print(f""" +### STEP 0a: Check which CUDA GPU is available (pick an idle index for CUDA_VISIBLE_DEVICES) + +nvidia-smi --query-gpu=index,utilization.gpu,memory.used,memory.free --format=csv +""") + +# ── Step 0b: Clean logs and cache ──────────────────────────────────────────── +print(f""" +### STEP 0b (optional): Clean logs and cache + +rm -rf {logs_dir}/* +{'# cache preserved (use_old_cache=True)' if use_old_cache else f'rm -rf {cache_dir}/*'} +rm -rf {output_dir}/* +""") + +# ── Step 1: Start a tmux session ──────────────────────────────────────────── +print(f""" +### STEP 1: Start a tmux session (attached) + +tmux new-session -s {session_name} +""") + +# ── Step 2: Paste this into the session to run training ───────────────────── +print("\n" + "=" * 60 + "\n") + +log_dir = logs_dir +log_tag = f"bottleneck_transformer_labs_notes_s{seed}" + +flags = [ + f"--ehr-root {ehr_root}", + f"--note-root {note_root}", + f"--cache-dir {cache_dir}", + f"--task notes_labs{' --dev' if dev else ''}", + "--model bottleneck_transformer", + f"--embedding-dim {embedding_dim}", + f"--hidden-dim {hidden_dim}", + f"--heads {heads}", + f"--num-layers {num_layers}", + f"--bottlenecks-n {bottlenecks_n}", + f"--fusion-startidx {fusion_startidx}", + f"--dropout {dropout}", +] +if use_amp: + flags.append("--use-amp") + flags.append(f"--amp-dtype {amp_dtype}") +flags += [ + f"--epochs {epochs}", + f"--batch-size {batch_size}", + f"--lr {lr}", + f"--weight-decay {weight_decay}", + f"--patience {patience}", + f"--num-workers {num_workers}", +] +if freeze_encoder: + flags.append("--freeze-encoder") +if use_wandb: + flags.append("--wandb") + flags.append(f"--wandb-project {wandb_project}") + if wandb_run_name: + flags.append(f"--wandb-run-name {wandb_run_name}") +flags += [ + f"--seed {seed}", + f"--output-dir {output_dir}", +] +flags_block = " \\\n ".join(flags) + +print(f""" +### STEP 2: Paste this into the tmux session + +mkdir -p {log_dir} && +eval "$(conda shell.bash hook)" && +conda activate {conda_env} && +cd {project_dir} && +export PYTHONPATH={project_dir}:$PYTHONPATH && +export CUDA_VISIBLE_DEVICES={cuda_visible_devices} && +python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \\ + {flags_block} \\ + > >(tee {log_dir}/{log_tag}.out) \\ + 2> >(tee {log_dir}/{log_tag}.err >&2) +""") + +# ── Step 3: Detach / reattach / monitor ────────────────────────────────────── +print(f""" +### STEP 3: Detach without killing (Ctrl+b d), then reattach later with + +tmux attach -t {session_name} + +### To check on it later without attaching: + +tail -f {log_dir}/{log_tag}.out + +### To kill the session when done: + +tmux kill-session -t {session_name} +""") + +# ── Step 4: Confirm which note sections were used for filtering ───────────── +print(f""" +### STEP 4: Confirm which discharge-note AND radiology-note sections +### NotesLabsMIMIC4 filtered to (e.g. discharge: "chief complaint"; +### radiology: "indication", "impression"). Logged once at INFO level by +### pyhealth.tasks.multimodal_mimic4 when the task is constructed. + +grep "filtering discharge notes to sections" {log_dir}/{log_tag}.out +""") \ No newline at end of file diff --git a/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_ehrmamba_variant.py b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_ehrmamba_variant.py new file mode 100644 index 000000000..739281819 --- /dev/null +++ b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_ehrmamba_variant.py @@ -0,0 +1,136 @@ +project_dir = "PyHealth" +seed = 12 +conda_env = "pyhealth2" +ehr_root = "/home/ubuntu/mimiciv-data/ehr" +note_root = "/home/ubuntu/mimiciv-data" +cache_dir = "/home/ubuntu/mimiciv-data/pyhealth_cache_labs_notes" +logs_dir = "/home/ubuntu/logs" +output_dir = "/home/ubuntu/output" +embedding_dim = 128 +hidden_dim = 128 +num_layers = 2 +mamba_state_size = 16 +mamba_conv_kernel = 4 +dropout = 0.1 +use_amp = True +amp_dtype = "bf16" +epochs = 50 +batch_size = 32 +lr = 1e-4 +weight_decay = 1e-5 +patience = 5 +num_workers = 4 +freeze_encoder = True +dev = False +use_old_cache = False +use_wandb = True +wandb_project = "pyhealth-multimodal-labs-notes" +wandb_run_name = None # defaults to "{model}_seed{seed}" if unset +cuda_visible_devices = "0" +session_name = f"ehrmamba_labs_notes_s{seed}" + +# ── Step 0a: Check which CUDA GPU is available ─────────────────────────────── +print(f""" +### STEP 0a: Check which CUDA GPU is available (pick an idle index for CUDA_VISIBLE_DEVICES) + +nvidia-smi --query-gpu=index,utilization.gpu,memory.used,memory.free --format=csv +""") + +# ── Step 0b: Clean logs and cache ──────────────────────────────────────────── +print(f""" +### STEP 0b (optional): Clean logs and cache + +rm -rf {logs_dir}/* +{'# cache preserved (use_old_cache=True)' if use_old_cache else f'rm -rf {cache_dir}/*'} +rm -rf {output_dir}/* +""") + +# ── Step 1: Start a tmux session ──────────────────────────────────────────── +print(f""" +### STEP 1: Start a tmux session (attached) + +tmux new-session -s {session_name} +""") + +# ── Step 2: Paste this into the session to run training ───────────────────── +print("\n" + "=" * 60 + "\n") + +log_dir = logs_dir +log_tag = f"ehrmamba_labs_notes_s{seed}" + +flags = [ + f"--ehr-root {ehr_root}", + f"--note-root {note_root}", + f"--cache-dir {cache_dir}", + f"--task notes_labs{' --dev' if dev else ''}", + "--model ehrmamba", + f"--embedding-dim {embedding_dim}", + f"--hidden-dim {hidden_dim}", + f"--num-layers {num_layers}", + f"--mamba-state-size {mamba_state_size}", + f"--mamba-conv-kernel {mamba_conv_kernel}", + f"--dropout {dropout}", +] +if use_amp: + flags.append("--use-amp") + flags.append(f"--amp-dtype {amp_dtype}") +flags += [ + f"--epochs {epochs}", + f"--batch-size {batch_size}", + f"--lr {lr}", + f"--weight-decay {weight_decay}", + f"--patience {patience}", + f"--num-workers {num_workers}", +] +if freeze_encoder: + flags.append("--freeze-encoder") +if use_wandb: + flags.append("--wandb") + flags.append(f"--wandb-project {wandb_project}") + if wandb_run_name: + flags.append(f"--wandb-run-name {wandb_run_name}") +flags += [ + f"--seed {seed}", + f"--output-dir {output_dir}", +] +flags_block = " \\\n ".join(flags) + +print(f""" +### STEP 2: Paste this into the tmux session + +mkdir -p {log_dir} && +eval "$(conda shell.bash hook)" && +conda activate {conda_env} && +cd {project_dir} && +export PYTHONPATH={project_dir}:$PYTHONPATH && +export CUDA_VISIBLE_DEVICES={cuda_visible_devices} && +python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \\ + {flags_block} \\ + > >(tee {log_dir}/{log_tag}.out) \\ + 2> >(tee {log_dir}/{log_tag}.err >&2) +""") + +# ── Step 3: Detach / reattach / monitor ────────────────────────────────────── +print(f""" +### STEP 3: Detach without killing (Ctrl+b d), then reattach later with + +tmux attach -t {session_name} + +### To check on it later without attaching: + +tail -f {log_dir}/{log_tag}.out + +### To kill the session when done: + +tmux kill-session -t {session_name} +""") + +# ── Step 4: Confirm which note sections were used for filtering ───────────── +print(f""" +### STEP 4: Confirm which discharge-note AND radiology-note sections +### NotesLabsMIMIC4 filtered to (e.g. discharge: "chief complaint"; +### radiology: "indication", "impression"). Logged once at INFO level by +### pyhealth.tasks.multimodal_mimic4 when the task is constructed. + +grep "filtering discharge notes to sections" {log_dir}/{log_tag}.out +""") \ No newline at end of file diff --git a/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_jambaehr_variant.py b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_jambaehr_variant.py new file mode 100644 index 000000000..d7fc5df3e --- /dev/null +++ b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_jambaehr_variant.py @@ -0,0 +1,142 @@ +project_dir = "PyHealth" +seed = 12 +conda_env = "pyhealth2" +ehr_root = "/home/ubuntu/mimiciv-data/ehr" +note_root = "/home/ubuntu/mimiciv-data" +cache_dir = "/home/ubuntu/mimiciv-data/pyhealth_cache_labs_notes" +logs_dir = "/home/ubuntu/logs" +output_dir = "/home/ubuntu/output" +embedding_dim = 128 +hidden_dim = 128 +heads = 4 +num_layers = 2 +jamba_transformer_layers = 2 +jamba_mamba_layers = 6 +mamba_state_size = 16 +mamba_conv_kernel = 4 +dropout = 0.1 +use_amp = True +amp_dtype = "bf16" +epochs = 50 +batch_size = 32 +lr = 1e-4 +weight_decay = 1e-5 +patience = 5 +num_workers = 4 +freeze_encoder = True +dev = False +use_old_cache = False +use_wandb = True +wandb_project = "pyhealth-multimodal-labs-notes" +wandb_run_name = None # defaults to "{model}_seed{seed}" if unset +cuda_visible_devices = "0" +session_name = f"jambaehr_labs_notes_s{seed}_batchsize{batch_size}" + +# ── Step 0a: Check which CUDA GPU is available ─────────────────────────────── +print(f""" +### STEP 0a: Check which CUDA GPU is available (pick an idle index for CUDA_VISIBLE_DEVICES) + +nvidia-smi --query-gpu=index,utilization.gpu,memory.free --format=csv +""") + +# ── Step 0b: Clean logs and cache ──────────────────────────────────────────── +print(f""" +### STEP 0b (optional): Clean logs and cache + +rm -rf {logs_dir}/* +{'# cache preserved (use_old_cache=True)' if use_old_cache else f'rm -rf {cache_dir}/*'} +rm -rf {output_dir}/* +""") + +# ── Step 1: Start a tmux session ──────────────────────────────────────────── +print(f""" +### STEP 1: Start a tmux session (attached) + +tmux new-session -s {session_name} +""") + +# ── Step 2: Paste this into the session to run training ───────────────────── +print("\n" + "=" * 60 + "\n") + +log_dir = logs_dir +log_tag = f"jambaehr_labs_notes_s{seed}" + +flags = [ + f"--ehr-root {ehr_root}", + f"--note-root {note_root}", + f"--cache-dir {cache_dir}", + f"--task notes_labs{' --dev' if dev else ''}", + "--model jambaehr", + f"--embedding-dim {embedding_dim}", + f"--hidden-dim {hidden_dim}", + f"--heads {heads}", + f"--num-layers {num_layers}", + f"--jamba-transformer-layers {jamba_transformer_layers}", + f"--jamba-mamba-layers {jamba_mamba_layers}", + f"--mamba-state-size {mamba_state_size}", + f"--mamba-conv-kernel {mamba_conv_kernel}", + f"--dropout {dropout}", +] +if use_amp: + flags.append("--use-amp") + flags.append(f"--amp-dtype {amp_dtype}") +flags += [ + f"--epochs {epochs}", + f"--batch-size {batch_size}", + f"--lr {lr}", + f"--weight-decay {weight_decay}", + f"--patience {patience}", + f"--num-workers {num_workers}", +] +if freeze_encoder: + flags.append("--freeze-encoder") +if use_wandb: + flags.append("--wandb") + flags.append(f"--wandb-project {wandb_project}") + if wandb_run_name: + flags.append(f"--wandb-run-name {wandb_run_name}") +flags += [ + f"--seed {seed}", + f"--output-dir {output_dir}", +] +flags_block = " \\\n ".join(flags) + +print(f""" +### STEP 2: Paste this into the tmux session + +mkdir -p {log_dir} && +eval "$(conda shell.bash hook)" && +conda activate {conda_env} && +cd {project_dir} && +export PYTHONPATH={project_dir}:$PYTHONPATH && +export CUDA_VISIBLE_DEVICES={cuda_visible_devices} && +python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \\ + {flags_block} \\ + > >(tee {log_dir}/{log_tag}.out) \\ + 2> >(tee {log_dir}/{log_tag}.err >&2) +""") + +# ── Step 3: Detach / reattach / monitor ────────────────────────────────────── +print(f""" +### STEP 3: Detach without killing (Ctrl+b d), then reattach later with + +tmux attach -t {session_name} + +### To check on it later without attaching: + +tail -f {log_dir}/{log_tag}.out + +### To kill the session when done: + +tmux kill-session -t {session_name} +""") + +# ── Step 4: Confirm which note sections were used for filtering ───────────── +print(f""" +### STEP 4: Confirm which discharge-note AND radiology-note sections +### NotesLabsMIMIC4 filtered to (e.g. discharge: "chief complaint"; +### radiology: "indication", "impression"). Logged once at INFO level by +### pyhealth.tasks.multimodal_mimic4 when the task is constructed. + +grep "filtering discharge notes to sections" {log_dir}/{log_tag}.out +""") \ No newline at end of file diff --git a/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_rnn_variant.py b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_rnn_variant.py new file mode 100644 index 000000000..047970f73 --- /dev/null +++ b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_rnn_variant.py @@ -0,0 +1,134 @@ +project_dir = "PyHealth" +seed = 12 +conda_env = "pyhealth2" +ehr_root = "/home/ubuntu/mimiciv-data/ehr" +note_root = "/home/ubuntu/mimiciv-data" +cache_dir = "/home/ubuntu/mimiciv-data/pyhealth_cache_labs_notes" +logs_dir = "/home/ubuntu/logs" +output_dir = "/home/ubuntu/output" +embedding_dim = 128 +hidden_dim = 128 +rnn_type = "GRU" +rnn_layers = 2 +dropout = 0.1 +use_amp = True +amp_dtype = "bf16" +epochs = 50 +batch_size = 32 +lr = 1e-4 +weight_decay = 1e-5 +patience = 5 +num_workers = 4 +freeze_encoder = True +dev = False +use_old_cache = False +use_wandb = True +wandb_project = "pyhealth-multimodal-labs-notes" +wandb_run_name = None # defaults to "{model}_seed{seed}" if unset +cuda_visible_devices = "0" +session_name = f"rnn_labs_notes_s{seed}" + +# ── Step 0a: Check which CUDA GPU is available ─────────────────────────────── +print(f""" +### STEP 0a: Check which CUDA GPU is available (pick an idle index for CUDA_VISIBLE_DEVICES) + +nvidia-smi --query-gpu=index,utilization.gpu,memory.used,memory.free --format=csv +""") + +# ── Step 0b: Clean logs and cache ──────────────────────────────────────────── +print(f""" +### STEP 0b (optional): Clean logs and cache + +rm -rf {logs_dir}/* +{'# cache preserved (use_old_cache=True)' if use_old_cache else f'rm -rf {cache_dir}/*'} +rm -rf {output_dir}/* +""") + +# ── Step 1: Start a tmux session ──────────────────────────────────────────── +print(f""" +### STEP 1: Start a tmux session (attached) + +tmux new-session -s {session_name} +""") + +# ── Step 2: Paste this into the session to run training ───────────────────── +print("\n" + "=" * 60 + "\n") + +log_dir = logs_dir +log_tag = f"rnn_labs_notes_s{seed}" + +flags = [ + f"--ehr-root {ehr_root}", + f"--note-root {note_root}", + f"--cache-dir {cache_dir}", + f"--task notes_labs{' --dev' if dev else ''}", + "--model rnn", + f"--embedding-dim {embedding_dim}", + f"--hidden-dim {hidden_dim}", + f"--rnn-type {rnn_type}", + f"--rnn-layers {rnn_layers}", + f"--dropout {dropout}", +] +if use_amp: + flags.append("--use-amp") + flags.append(f"--amp-dtype {amp_dtype}") +flags += [ + f"--epochs {epochs}", + f"--batch-size {batch_size}", + f"--lr {lr}", + f"--weight-decay {weight_decay}", + f"--patience {patience}", + f"--num-workers {num_workers}", +] +if freeze_encoder: + flags.append("--freeze-encoder") +if use_wandb: + flags.append("--wandb") + flags.append(f"--wandb-project {wandb_project}") + if wandb_run_name: + flags.append(f"--wandb-run-name {wandb_run_name}") +flags += [ + f"--seed {seed}", + f"--output-dir {output_dir}", +] +flags_block = " \\\n ".join(flags) + +print(f""" +### STEP 2: Paste this into the tmux session + +mkdir -p {log_dir} && +eval "$(conda shell.bash hook)" && +conda activate {conda_env} && +cd {project_dir} && +export PYTHONPATH={project_dir}:$PYTHONPATH && +export CUDA_VISIBLE_DEVICES={cuda_visible_devices} && +python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \\ + {flags_block} \\ + > >(tee {log_dir}/{log_tag}.out) \\ + 2> >(tee {log_dir}/{log_tag}.err >&2) +""") + +# ── Step 3: Detach / reattach / monitor ────────────────────────────────────── +print(f""" +### STEP 3: Detach without killing (Ctrl+b d), then reattach later with + +tmux attach -t {session_name} + +### To check on it later without attaching: + +tail -f {log_dir}/{log_tag}.out + +### To kill the session when done: + +tmux kill-session -t {session_name} +""") + +# ── Step 4: Confirm which note sections were used for filtering ───────────── +print(f""" +### STEP 4: Confirm which discharge-note AND radiology-note sections +### NotesLabsMIMIC4 filtered to (e.g. discharge: "chief complaint"; +### radiology: "indication", "impression"). Logged once at INFO level by +### pyhealth.tasks.multimodal_mimic4 when the task is constructed. + +grep "filtering discharge notes to sections" {log_dir}/{log_tag}.out +""") \ No newline at end of file diff --git a/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_transformer_variant.py b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_transformer_variant.py new file mode 100644 index 000000000..bceb635e6 --- /dev/null +++ b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_transformer_variant.py @@ -0,0 +1,134 @@ +project_dir = "PyHealth" +seed = 12 +conda_env = "pyhealth2" +ehr_root = "/home/ubuntu/mimiciv-data/ehr" +note_root = "/home/ubuntu/mimiciv-data" +cache_dir = "/home/ubuntu/mimiciv-data/pyhealth_cache_labs_notes" +logs_dir = "/home/ubuntu/logs" +output_dir = "/home/ubuntu/output" +embedding_dim = 128 +hidden_dim = 128 +heads = 4 +num_layers = 2 +dropout = 0.1 +use_amp = True +amp_dtype = "bf16" +epochs = 50 +batch_size = 32 +lr = 1e-4 +weight_decay = 1e-5 +patience = 5 +num_workers = 4 +freeze_encoder = True +dev = False +use_old_cache = False +use_wandb = True +wandb_project = "pyhealth-multimodal-labs-notes" +wandb_run_name = None # defaults to "{model}_seed{seed}" if unset +cuda_visible_devices = "0" +session_name = f"transformer_labs_notes_s{seed}" + +# ── Step 0a: Check which CUDA GPU is available ─────────────────────────────── +print(f""" +### STEP 0a: Check which CUDA GPU is available (pick an idle index for CUDA_VISIBLE_DEVICES) + +nvidia-smi --query-gpu=index,utilization.gpu,memory.used,memory.free --format=csv +""") + +# ── Step 0b: Clean logs and cache ──────────────────────────────────────────── +print(f""" +### STEP 0b (optional): Clean logs and cache + +rm -rf {logs_dir}/* +{'# cache preserved (use_old_cache=True)' if use_old_cache else f'rm -rf {cache_dir}/*'} +rm -rf {output_dir}/* +""") + +# ── Step 1: Start a tmux session ──────────────────────────────────────────── +print(f""" +### STEP 1: Start a tmux session (attached) + +tmux new-session -s {session_name} +""") + +# ── Step 2: Paste this into the session to run training ───────────────────── +print("\n" + "=" * 60 + "\n") + +log_dir = logs_dir +log_tag = f"transformer_labs_notes_s{seed}" + +flags = [ + f"--ehr-root {ehr_root}", + f"--note-root {note_root}", + f"--cache-dir {cache_dir}", + f"--task notes_labs{' --dev' if dev else ''}", + "--model transformer", + f"--embedding-dim {embedding_dim}", + f"--hidden-dim {hidden_dim}", + f"--heads {heads}", + f"--num-layers {num_layers}", + f"--dropout {dropout}", +] +if use_amp: + flags.append("--use-amp") + flags.append(f"--amp-dtype {amp_dtype}") +flags += [ + f"--epochs {epochs}", + f"--batch-size {batch_size}", + f"--lr {lr}", + f"--weight-decay {weight_decay}", + f"--patience {patience}", + f"--num-workers {num_workers}", +] +if freeze_encoder: + flags.append("--freeze-encoder") +if use_wandb: + flags.append("--wandb") + flags.append(f"--wandb-project {wandb_project}") + if wandb_run_name: + flags.append(f"--wandb-run-name {wandb_run_name}") +flags += [ + f"--seed {seed}", + f"--output-dir {output_dir}", +] +flags_block = " \\\n ".join(flags) + +print(f""" +### STEP 2: Paste this into the tmux session + +mkdir -p {log_dir} && +eval "$(conda shell.bash hook)" && +conda activate {conda_env} && +cd {project_dir} && +export PYTHONPATH={project_dir}:$PYTHONPATH && +export CUDA_VISIBLE_DEVICES={cuda_visible_devices} && +python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \\ + {flags_block} \\ + > >(tee {log_dir}/{log_tag}.out) \\ + 2> >(tee {log_dir}/{log_tag}.err >&2) +""") + +# ── Step 3: Detach / reattach / monitor ────────────────────────────────────── +print(f""" +### STEP 3: Detach without killing (Ctrl+b d), then reattach later with + +tmux attach -t {session_name} + +### To check on it later without attaching: + +tail -f {log_dir}/{log_tag}.out + +### To kill the session when done: + +tmux kill-session -t {session_name} +""") + +# ── Step 4: Confirm which note sections were used for filtering ───────────── +print(f""" +### STEP 4: Confirm which discharge-note AND radiology-note sections +### NotesLabsMIMIC4 filtered to (e.g. discharge: "chief complaint"; +### radiology: "indication", "impression"). Logged once at INFO level by +### pyhealth.tasks.multimodal_mimic4 when the task is constructed. + +grep "filtering discharge notes to sections" {log_dir}/{log_tag}.out +""") \ No newline at end of file From b4f88f99d96a823486e49c35b30f30e1645accac Mon Sep 17 00:00:00 2001 From: William Pang Date: Thu, 27 Aug 2026 10:32:05 -0700 Subject: [PATCH 42/61] Create pyhealth2_environment.yml --- pyhealth2_environment.yml | 290 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 290 insertions(+) create mode 100644 pyhealth2_environment.yml diff --git a/pyhealth2_environment.yml b/pyhealth2_environment.yml new file mode 100644 index 000000000..b9920a0c9 --- /dev/null +++ b/pyhealth2_environment.yml @@ -0,0 +1,290 @@ +name: pyhealth2 +channels: + - defaults + - conda-forge +dependencies: + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - _python_abi3_support=1.0=hd8ed1ab_2 + - annotated-types=0.7.0=pyhd8ed1ab_1 + - anyio=4.14.0=pyhcf101f3_0 + - argon2-cffi=25.1.0=pyhd8ed1ab_0 + - argon2-cffi-bindings=25.1.0=py312h4c3975b_2 + - arrow=1.4.0=pyhcf101f3_0 + - asttokens=3.0.1=pyhd8ed1ab_0 + - async-lru=2.3.0=pyhcf101f3_0 + - attrs=26.1.0=pyhcf101f3_0 + - babel=2.18.0=pyhcf101f3_1 + - backports.zstd=1.6.0=py312h90b7ffd_0 + - beautifulsoup4=4.15.0=pyha770c72_0 + - bleach=6.4.0=pyhcf101f3_0 + - bleach-with-css=6.4.0=hac0b51c_0 + - brotli-python=1.2.0=py312hdb49522_1 + - bzip2=1.0.8=h5eee18b_6 + - c-ares=1.34.8=hb03c661_0 + - ca-certificates=2026.6.17=hbd8a1cb_0 + - cached-property=1.5.2=hd8ed1ab_1 + - cached_property=1.5.2=pyha770c72_1 + - cffi=1.17.1=py312h06ac9bb_0 + - charset-normalizer=3.4.7=pyhd8ed1ab_0 + - comm=0.2.3=pyhe01879c_0 + - cpython=3.12.13=py312hd8ed1ab_0 + - debugpy=1.8.21=py312h8285ef7_0 + - defusedxml=0.7.1=pyhd8ed1ab_0 + - exceptiongroup=1.3.1=pyhd8ed1ab_0 + - executing=2.2.1=pyhd8ed1ab_0 + - fqdn=1.5.1=pyhd8ed1ab_1 + - gitdb=4.0.12=pyhd8ed1ab_0 + - gitpython=3.1.50=pyhd8ed1ab_0 + - h11=0.16.0=pyhcf101f3_1 + - h2=4.3.0=pyhcf101f3_0 + - hpack=4.1.0=pyhd8ed1ab_0 + - htcondor=25.11.0=py312h7900ff3_2 + - htcondor-classads=25.11.0=h793e66c_2 + - htcondor-cli=25.11.0=py312h7900ff3_2 + - htcondor-utils=25.11.0=h8d23d0f_2 + - httpcore=1.0.9=pyh29332c3_0 + - httpx=0.28.1=pyhd8ed1ab_0 + - hyperframe=6.1.0=pyhd8ed1ab_0 + - icu=78.3=h33c6efd_0 + - importlib-metadata=9.0.0=pyhcf101f3_0 + - ipykernel=7.3.0=pyha191276_0 + - ipython=9.14.1=pyh53cf698_0 + - ipython_pygments_lexers=1.1.1=pyhd8ed1ab_0 + - isoduration=20.11.0=pyhd8ed1ab_1 + - jedi=0.19.2=pyhd8ed1ab_1 + - jinja2=3.1.6=pyhcf101f3_1 + - json5=0.15.0=pyhd8ed1ab_0 + - jsonpointer=3.1.1=pyhcf101f3_0 + - jsonschema=4.26.0=pyhcf101f3_0 + - jsonschema-specifications=2025.9.1=pyhcf101f3_0 + - jsonschema-with-format-nongpl=4.26.0=hcf101f3_0 + - jupyter-builder=1.0.2=pyhcf101f3_0 + - jupyter-lsp=2.3.1=pyhcf101f3_0 + - jupyter_client=8.9.1=pyhcf101f3_0 + - jupyter_core=5.9.1=pyhc90fa1f_0 + - jupyter_events=0.12.1=pyhcf101f3_0 + - jupyter_server=2.20.0=pyhcf101f3_0 + - jupyter_server_terminals=0.5.4=pyhcf101f3_0 + - jupyterlab=4.6.0=pyhd8ed1ab_0 + - jupyterlab_pygments=0.3.0=pyhd8ed1ab_2 + - jupyterlab_server=2.28.0=pyhcf101f3_0 + - keyutils=1.6.3=hb9d3cd8_0 + - krb5=1.22.2=hbde042b_1 + - lark=1.3.1=pyhd8ed1ab_0 + - ld_impl_linux-64=2.44=h9e0c5a2_3 + - libabseil=20260107.1=cxx17_h7b12aa8_0 + - libcap=2.77=hd0affe5_1 + - libcondor_utils=25.11.0=h4effbbe_2 + - libcurl=8.21.0=hae6b9f4_2 + - libdrm=2.4.125=hb03c661_1 + - libedit=3.1.20250104=pl5321h7949ede_0 + - libev=4.33=hd590300_2 + - libexpat=2.7.5=h7354ed3_0 + - libffi=3.4.4=h6a678d5_1 + - libgcc=15.2.0=h69a1729_7 + - libgcc-ng=15.2.0=h166f726_7 + - libgomp=15.2.0=h4751f2c_7 + - liblzma=5.8.2=hb03c661_0 + - libnghttp2=1.68.1=h877daf1_0 + - libnsl=2.0.1=hb9d3cd8_1 + - libpciaccess=0.18=hb9d3cd8_0 + - libpsl=0.22.0=h49b2146_1 + - libsodium=1.0.22=h280c20c_1 + - libsqlite=3.53.3=h0c1763c_0 + - libssh2=1.11.1=hcf80075_0 + - libstdcxx=15.2.0=h39759b7_7 + - libstdcxx-ng=15.2.0=hc03a8fd_7 + - libsystemd0=257.13=hd0affe5_0 + - libuuid=2.42.2=h5347b49_0 + - libxcb=1.17.0=h9b100fa_0 + - libxcrypt=4.4.36=hd590300_1 + - libzlib=1.3.2=h25fd6f3_2 + - markupsafe=3.0.3=py312h8a5da7c_1 + - matplotlib-inline=0.2.2=pyhd8ed1ab_0 + - mistune=3.3.2=pyhcf101f3_0 + - munge=0.5.16=h63a00c3_0 + - nbclient=0.11.0=pyhd8ed1ab_0 + - nbconvert-core=7.17.1=pyhcf101f3_0 + - nbformat=5.10.4=pyhd8ed1ab_1 + - ncurses=6.5=h7934f7d_0 + - nest-asyncio2=1.7.2=pyhcf101f3_0 + - notebook=7.6.0=pyhcf101f3_0 + - notebook-shim=0.2.4=pyhd8ed1ab_1 + - nvidia-ml-py=13.590.48=pyhd8ed1ab_0 + - nvitop=1.7.0=pyh707e725_0 + - nvtop=3.3.2=h5433ab2_0 + - openssl=3.6.3=h35e630c_0 + - overrides=7.7.0=pyhd8ed1ab_1 + - packaging=26.0=py312h06a4308_0 + - pandocfilters=1.5.0=pyhd8ed1ab_0 + - parso=0.8.7=pyhcf101f3_0 + - pcre2=10.47=haa7fec5_0 + - pexpect=4.9.0=pyhd8ed1ab_1 + - pip=26.0.1=pyhc872135_1 + - prometheus_client=0.25.0=pyhd8ed1ab_0 + - prompt-toolkit=3.0.52=pyha770c72_0 + - protobuf=6.33.5=py312ha7b3241_2 + - psutil=7.2.2=py312h5253ce2_0 + - pthread-stubs=0.3=h0ce48e5_1 + - ptyprocess=0.7.0=pyhd8ed1ab_1 + - pure_eval=0.2.3=pyhd8ed1ab_1 + - pycparser=3.0=pyhcf101f3_0 + - pygments=2.20.0=pyhd8ed1ab_0 + - pysocks=1.7.1=pyha55dd90_7 + - python=3.12.9=h9e4cc4f_1_cpython + - python-dateutil=2.9.0.post0=pyhe01879c_2 + - python-fastjsonschema=2.21.2=pyhe01879c_0 + - python-gil=3.12.13=hd8ed1ab_0 + - python-htcondor=25.11.0=py312h40fa4ac_2 + - python-json-logger=4.1.0=pyhd8ed1ab_0 + - python-tzdata=2026.2=pyhd8ed1ab_0 + - python_abi=3.12=3_cp312 + - pyyaml=6.0.3=py312h8a5da7c_1 + - pyzmq=27.1.0=py312hda471dd_3 + - readline=8.3=hc2a1206_0 + - referencing=0.37.0=pyhcf101f3_0 + - rfc3339-validator=0.1.4=pyhd8ed1ab_1 + - rfc3986-validator=0.1.1=pyh9f0ad1d_0 + - rfc3987-syntax=1.1.0=pyhe01879c_1 + - rpds-py=2026.5.1=py312h192e038_0 + - scitokens-cpp=1.4.0=h096d96b_0 + - send2trash=2.1.0=pyha191276_1 + - sentry-sdk=2.64.0=pyhd8ed1ab_0 + - setuptools=82.0.1=py312h06a4308_0 + - six=1.17.0=pyhe01879c_1 + - smmap=5.0.3=pyhcf101f3_1 + - sniffio=1.3.1=pyhd8ed1ab_2 + - soupsieve=2.8.4=pyhd8ed1ab_0 + - sqlite=3.51.2=h3e8d24a_0 + - stack_data=0.6.3=pyhd8ed1ab_1 + - terminado=0.18.1=pyhc90fa1f_1 + - tinycss2=1.4.0=pyhd8ed1ab_0 + - tk=8.6.15=h54e0aa7_0 + - tomli=2.4.1=pyhcf101f3_0 + - traitlets=5.15.1=pyhcf101f3_0 + - typing-extensions=4.15.0=h396c80c_0 + - typing-inspection=0.4.2=pyhcf101f3_2 + - typing_extensions=4.15.0=pyhcf101f3_0 + - typing_utils=0.1.0=pyhd8ed1ab_1 + - uri-template=1.3.0=pyhd8ed1ab_1 + - wandb=0.28.0=py312h868fb18_0 + - wcwidth=0.8.1=pyhd8ed1ab_0 + - webcolors=25.10.0=pyhd8ed1ab_0 + - webencodings=0.5.1=pyhd8ed1ab_3 + - websocket-client=1.9.0=pyhd8ed1ab_0 + - wheel=0.46.3=py312h06a4308_0 + - xorg-libx11=1.8.12=h9b100fa_1 + - xorg-libxau=1.0.12=h9b100fa_0 + - xorg-libxdmcp=1.1.5=h9b100fa_0 + - xorg-xorgproto=2024.1=h5eee18b_1 + - xz=5.8.2=h448239c_0 + - yaml=0.2.5=h280c20c_3 + - zeromq=4.3.5=h09e67af_11 + - zipp=4.1.0=pyhcf101f3_0 + - zlib=1.3.2=h25fd6f3_2 + - zstd=1.5.7=hb78ec9c_6 + - pip: + - accelerate==1.13.0 + - axial-positional-embedding==0.3.12 + - bokeh==3.9.0 + - boto3==1.42.88 + - botocore==1.42.88 + - certifi==2026.2.25 + - click==8.3.2 + - cloudpickle==3.1.2 + - colt5-attention==0.11.1 + - contourpy==1.3.3 + - cycler==0.12.1 + - dask==2025.11.0 + - decorator==5.2.1 + - distributed==2025.11.0 + - einops==0.8.2 + - filelock==3.25.2 + - fonttools==4.62.1 + - fsspec==2026.2.0 + - hf-xet==1.4.3 + - huggingface-hub==0.36.2 + - hyper-connections==0.4.9 + - idna==3.11 + - ipywidgets==8.1.8 + - jmespath==1.1.0 + - joblib==1.5.3 + - jupyterlab-widgets==3.0.16 + - kiwisolver==1.5.0 + - lazy-loader==0.5 + - lightning-utilities==0.15.3 + - linear-attention-transformer==0.19.1 + - linformer==0.2.3 + - litdata==0.2.61 + - littleutils==0.2.4 + - local-attention==1.11.2 + - locket==1.0.0 + - lz4==4.4.5 + - matplotlib==3.10.8 + - mne==1.10.2 + - more-itertools==10.8.0 + - mpmath==1.3.0 + - msgpack==1.1.2 + - narwhals==2.13.0 + - networkx==3.6.1 + - numpy==2.2.6 + - nvidia-cublas-cu12==12.6.4.1 + - nvidia-cuda-cupti-cu12==12.6.80 + - nvidia-cuda-nvrtc-cu12==12.6.77 + - nvidia-cuda-runtime-cu12==12.6.77 + - nvidia-cudnn-cu12==9.5.1.17 + - nvidia-cufft-cu12==11.3.0.4 + - nvidia-cufile-cu12==1.11.1.6 + - nvidia-curand-cu12==10.3.7.77 + - nvidia-cusolver-cu12==11.7.1.2 + - nvidia-cusparse-cu12==12.5.4.2 + - nvidia-cusparselt-cu12==0.6.3 + - nvidia-nccl-cu12==2.26.2 + - nvidia-nvjitlink-cu12==12.6.85 + - nvidia-nvtx-cu12==12.6.77 + - obstore==0.9.2 + - ogb==1.3.6 + - outdated==0.2.2 + - pandas==2.3.3 + - partd==1.4.2 + - peft==0.18.1 + - pillow==12.1.1 + - platformdirs==4.9.6 + - polars==1.35.2 + - polars-runtime-32==1.35.2 + - pooch==1.9.0 + - product-key-memory==0.3.0 + - pyarrow==22.0.0 + - pydantic==2.11.10 + - pydantic-core==2.33.2 + - pyhealth==2.0.0 + - pyparsing==3.3.2 + - pytz==2026.1.post1 + - rdkit==2026.3.1 + - regex==2026.4.4 + - requests==2.33.1 + - s3transfer==0.16.0 + - safetensors==0.7.0 + - scikit-learn==1.7.2 + - scipy==1.17.1 + - sortedcontainers==2.4.0 + - sympy==1.14.0 + - tblib==3.2.2 + - threadpoolctl==3.6.0 + - tifffile==2026.3.3 + - tokenizers==0.21.4 + - toolz==1.1.0 + - torch==2.7.1 + - torch-einops-utils==0.0.30 + - torchvision==0.22.1 + - tornado==6.5.5 + - tqdm==4.67.3 + - transformers==4.53.3 + - triton==3.3.1 + - tzdata==2026.1 + - urllib3==2.5.0 + - widgetsnbextension==4.0.15 + - xyzservices==2026.3.0 + - zict==3.0.0 +prefix: /home/wp14/miniconda3/envs/pyhealth2 From 731c8c6c380ffda5f8023fa70aa53635573a7f5f Mon Sep 17 00:00:00 2001 From: William Pang Date: Thu, 27 Aug 2026 10:54:44 -0700 Subject: [PATCH 43/61] Update pyhealth2_environment.yml --- pyhealth2_environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyhealth2_environment.yml b/pyhealth2_environment.yml index b9920a0c9..82e2a3fb7 100644 --- a/pyhealth2_environment.yml +++ b/pyhealth2_environment.yml @@ -245,6 +245,7 @@ dependencies: - nvidia-nvtx-cu12==12.6.77 - obstore==0.9.2 - ogb==1.3.6 + - orjson==3.11.9 - outdated==0.2.2 - pandas==2.3.3 - partd==1.4.2 From 3ae1a2ad6c8b2d0ec6883a895e59967dd037b212 Mon Sep 17 00:00:00 2001 From: Rian354 Date: Thu, 27 Aug 2026 17:58:44 -0400 Subject: [PATCH 44/61] Add rian/scripts: notes_labs seed-1 paper-cell launchers (MLP, JambaEHR, BottleneckTransformer) with results. Claude-Session: https://claude.ai/code/session_01U3rt6DmCQ2NiPmfwujzoje --- rian/scripts/README.md | 43 +++++++++++++++++++ ...notes_labs_bottleneck_transformer_seed1.sh | 23 ++++++++++ rian/scripts/run_notes_labs_jambaehr_seed1.sh | 23 ++++++++++ rian/scripts/run_notes_labs_mlp_seed1.sh | 22 ++++++++++ 4 files changed, 111 insertions(+) create mode 100644 rian/scripts/README.md create mode 100755 rian/scripts/run_notes_labs_bottleneck_transformer_seed1.sh create mode 100755 rian/scripts/run_notes_labs_jambaehr_seed1.sh create mode 100755 rian/scripts/run_notes_labs_mlp_seed1.sh diff --git a/rian/scripts/README.md b/rian/scripts/README.md new file mode 100644 index 000000000..51dba92fa --- /dev/null +++ b/rian/scripts/README.md @@ -0,0 +1,43 @@ +# rian/scripts — Tranche 1 paper cells (notes_labs, seed 1) + +One launcher per cell. Each mirrors the original `notes_labs × MLP × seed 1` cell +(`run_config.json` of that run), differing only in `--model` and its own flags. + +``` +GPU=2 bash rian/scripts/run_notes_labs_jambaehr_seed1.sh +GPU=3 bash rian/scripts/run_notes_labs_bottleneck_transformer_seed1.sh +GPU=0 bash rian/scripts/run_notes_labs_mlp_seed1.sh +``` + +Env overrides: `TREE` (checkout to run from), `GPU`, `EHR_ROOT`, `NOTE_ROOT`, `CACHE_DIR`. +Logs go to `$TREE/logs/.out`, outputs to `$TREE/output/tranche1_/`. +`PYTHONPATH` is pinned to `$TREE` because the `pyhealth2` env carries an editable install +that otherwise shadows the checkout. + +## Protocol (all cells) + +Full stay (admit → discharge), no ICD, empty sequences for missing modalities, frozen +Bio_ClinicalBERT with a 1e6-entry `[CLS]` cache, bf16 AMP, batch 32, lr 1e-4, dropout 0.1, +embedding/hidden 128, 50 epochs, patience 5, seed 1, split by patient with seed 1. +Tree = PR #1185 tip `8d4a4c9` + the "frozen `[CLS]` cache cap to 1e6" follow-up commit only. +All cells hit the same task cache (`NotesLabsMIMIC4_c447f3bb…`): 144,586 / 18,073 / 18,074 +patients, 852 test positives. Test metrics are sklearn `average_precision_score` / +`roc_auc_score` over `predictions_.csv` from the best-val checkpoint. + +## Results (sunlab-serv-03, one RTX A6000 per cell, 2026-08-27) + +| cell | test PR-AUC | test ROC-AUC | best val PR-AUC (epoch) | stopped at | epoch after warm-up | total train | +|---|---|---|---|---|---|---| +| MLP, 1e6 cache | 0.5662 | 0.9437 | 0.5931 (16) | 21 | 191 s | 1.5 h | +| MLP, 200k cache (original cell) | 0.5705 | 0.9386 | 0.5996 (22) | 27 | 4984 s | 38.9 h | +| JambaEHR (2 transformer + 6 mamba) | 0.8081 | 0.9751 | 0.8203 (1) | 6 | 372 s | 1.0 h | +| BottleneckTransformer (n=4, fusion start 1) | 0.6907 | 0.9608 | 0.6958 (8) | 13 | 224 s | 1.2 h | + +Notes: + +- The two MLP rows are the compute-matched cache comparison: same host, same GPU, + same split, same flags — only the cache cap differs. Epoch time after warm-up drops + 26×; metrics land within run-to-run noise (not bit-identical: early stopping fired at + a different epoch). +- Single seed. JambaEHR peaks at epoch 1; treat the gap to the other backbones as + provisional until multi-seed. diff --git a/rian/scripts/run_notes_labs_bottleneck_transformer_seed1.sh b/rian/scripts/run_notes_labs_bottleneck_transformer_seed1.sh new file mode 100755 index 000000000..23f1ea2f6 --- /dev/null +++ b/rian/scripts/run_notes_labs_bottleneck_transformer_seed1.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Paper cell: notes_labs x BottleneckTransformer x seed 1. +# Full stay, empty-sequence missingness, frozen BERT, bf16 AMP, 1e6-entry [CLS] cache. +# Tree = 8d4a4c9 + the frozen-cache-cap commit only (results-identical, faster epochs). +set -euo pipefail +TREE="${TREE:-/home/rianatri/ml4h-tranche1-cache1m}" +GPU="${GPU:-0}" +EHR_ROOT="${EHR_ROOT:-/shared/rsaas/physionet.org/files/mimiciv/2.2}" +NOTE_ROOT="${NOTE_ROOT:-/shared/rsaas/physionet.org/files/mimic-note}" +CACHE_DIR="${CACHE_DIR:-/home/rianatri/pyhealth_cache/tranche1_v4_notes_labs}" +NAME=notes_labs_bottleneck_transformer_seed1_cache1m +cd "$TREE"; mkdir -p logs output +PYTHONPATH="$TREE" CUDA_VISIBLE_DEVICES="$GPU" nohup python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \ + --task notes_labs --model bottleneck_transformer \ + --ehr-root "$EHR_ROOT" --note-root "$NOTE_ROOT" --cache-dir "$CACHE_DIR" \ + --output-dir "$TREE/output/tranche1_$NAME" \ + --embedding-dim 128 --hidden-dim 128 --batch-size 32 --lr 1e-4 --dropout 0.1 \ + --epochs 50 --patience 5 --seed 1 --use-amp --amp-dtype bf16 --freeze-encoder \ + --max-frozen-text-cache 1000000 \ + --heads 4 --num-layers 2 --bottlenecks-n 4 --fusion-startidx 1 \ + > "logs/$NAME.out" 2>&1 & +echo $! > "logs/$NAME.pid" +echo "launched $NAME pid $(cat logs/$NAME.pid) on GPU $GPU" diff --git a/rian/scripts/run_notes_labs_jambaehr_seed1.sh b/rian/scripts/run_notes_labs_jambaehr_seed1.sh new file mode 100755 index 000000000..9c1661b78 --- /dev/null +++ b/rian/scripts/run_notes_labs_jambaehr_seed1.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Paper cell: notes_labs x JambaEHR x seed 1. +# Full stay, empty-sequence missingness, frozen BERT, bf16 AMP, 1e6-entry [CLS] cache. +# Tree = 8d4a4c9 + the frozen-cache-cap commit only (results-identical, faster epochs). +set -euo pipefail +TREE="${TREE:-/home/rianatri/ml4h-tranche1-cache1m}" +GPU="${GPU:-0}" +EHR_ROOT="${EHR_ROOT:-/shared/rsaas/physionet.org/files/mimiciv/2.2}" +NOTE_ROOT="${NOTE_ROOT:-/shared/rsaas/physionet.org/files/mimic-note}" +CACHE_DIR="${CACHE_DIR:-/home/rianatri/pyhealth_cache/tranche1_v4_notes_labs}" +NAME=notes_labs_jambaehr_seed1_cache1m +cd "$TREE"; mkdir -p logs output +PYTHONPATH="$TREE" CUDA_VISIBLE_DEVICES="$GPU" nohup python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \ + --task notes_labs --model jambaehr \ + --ehr-root "$EHR_ROOT" --note-root "$NOTE_ROOT" --cache-dir "$CACHE_DIR" \ + --output-dir "$TREE/output/tranche1_$NAME" \ + --embedding-dim 128 --hidden-dim 128 --batch-size 32 --lr 1e-4 --dropout 0.1 \ + --epochs 50 --patience 5 --seed 1 --use-amp --amp-dtype bf16 --freeze-encoder \ + --max-frozen-text-cache 1000000 \ + --heads 4 --num-layers 2 --jamba-transformer-layers 2 --jamba-mamba-layers 6 \ + > "logs/$NAME.out" 2>&1 & +echo $! > "logs/$NAME.pid" +echo "launched $NAME pid $(cat logs/$NAME.pid) on GPU $GPU" diff --git a/rian/scripts/run_notes_labs_mlp_seed1.sh b/rian/scripts/run_notes_labs_mlp_seed1.sh new file mode 100755 index 000000000..744377113 --- /dev/null +++ b/rian/scripts/run_notes_labs_mlp_seed1.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Paper cell: notes_labs x MLP x seed 1 (cache-1M compute-matched rerun of the original 200k-cache cell). +# Full stay, empty-sequence missingness, frozen BERT, bf16 AMP, 1e6-entry [CLS] cache. +# Tree = 8d4a4c9 + the frozen-cache-cap commit only (results-identical, faster epochs). +set -euo pipefail +TREE="${TREE:-/home/rianatri/ml4h-tranche1-cache1m}" +GPU="${GPU:-0}" +EHR_ROOT="${EHR_ROOT:-/shared/rsaas/physionet.org/files/mimiciv/2.2}" +NOTE_ROOT="${NOTE_ROOT:-/shared/rsaas/physionet.org/files/mimic-note}" +CACHE_DIR="${CACHE_DIR:-/home/rianatri/pyhealth_cache/tranche1_v4_notes_labs}" +NAME=notes_labs_mlp_seed1_cache1m +cd "$TREE"; mkdir -p logs output +PYTHONPATH="$TREE" CUDA_VISIBLE_DEVICES="$GPU" nohup python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \ + --task notes_labs --model mlp \ + --ehr-root "$EHR_ROOT" --note-root "$NOTE_ROOT" --cache-dir "$CACHE_DIR" \ + --output-dir "$TREE/output/tranche1_$NAME" \ + --embedding-dim 128 --hidden-dim 128 --batch-size 32 --lr 1e-4 --dropout 0.1 \ + --epochs 50 --patience 5 --seed 1 --use-amp --amp-dtype bf16 --freeze-encoder \ + --max-frozen-text-cache 1000000 \ + > "logs/$NAME.out" 2>&1 & +echo $! > "logs/$NAME.pid" +echo "launched $NAME pid $(cat logs/$NAME.pid) on GPU $GPU" From 0d8907d1912097873bdbe20049298faa58570d38 Mon Sep 17 00:00:00 2001 From: Rian354 Date: Fri, 28 Aug 2026 05:32:13 -0400 Subject: [PATCH 45/61] rian/scripts: add cache-500k and unfrozen-BERT launchers; note the overflow-path fix. Claude-Session: https://claude.ai/code/session_01U3rt6DmCQ2NiPmfwujzoje --- rian/scripts/README.md | 20 +++++++++++++++ ...s_bottleneck_transformer_seed1_unfrozen.sh | 25 +++++++++++++++++++ .../run_notes_labs_jambaehr_seed1_unfrozen.sh | 25 +++++++++++++++++++ .../run_notes_labs_mlp_seed1_cache500k.sh | 21 ++++++++++++++++ .../run_notes_labs_mlp_seed1_unfrozen.sh | 24 ++++++++++++++++++ 5 files changed, 115 insertions(+) create mode 100755 rian/scripts/run_notes_labs_bottleneck_transformer_seed1_unfrozen.sh create mode 100755 rian/scripts/run_notes_labs_jambaehr_seed1_unfrozen.sh create mode 100755 rian/scripts/run_notes_labs_mlp_seed1_cache500k.sh create mode 100755 rian/scripts/run_notes_labs_mlp_seed1_unfrozen.sh diff --git a/rian/scripts/README.md b/rian/scripts/README.md index 51dba92fa..a99eb6366 100644 --- a/rian/scripts/README.md +++ b/rian/scripts/README.md @@ -41,3 +41,23 @@ Notes: a different epoch). - Single seed. JambaEHR peaks at epoch 1; treat the gap to the other backbones as provisional until multi-seed. +- Why 200k was slower than *no* cache: once the cap is reached, misses were encoded in + the batched pass, not inserted, then encoded again one row at a time by the assembly + loop. Fixed on the PR follow-ups ("Encode a frozen-cache miss once per forward, even + when the cache is full"). The 200k row above ran on the pre-fix code. + +## Cache-cap sweep and unfrozen BERT (launched 2026-08-28, results pending) + +| script | condition | tree | +|---|---|---| +| `run_notes_labs_mlp_seed1_cache500k.sh` | frozen BERT, 500k cap | same as 1M cells | +| `run_notes_labs_mlp_seed1_unfrozen.sh` | BERT trained end-to-end | 1M tree + gradient-checkpointing commit | +| `run_notes_labs_jambaehr_seed1_unfrozen.sh` | BERT trained end-to-end | same | +| `run_notes_labs_bottleneck_transformer_seed1_unfrozen.sh` | BERT trained end-to-end | same | + +The unfrozen launchers drop `--freeze-encoder` and pass `--text-grad-checkpoint-rows 256`: +a trainable BERT keeps every note row's activations for the backward pass and one fat batch +exceeded 47 GB at step 4; the flag turns on per-layer gradient checkpointing and chunks note +rows through the encoder (math unchanged, ~12.5 GB peak, epoch 0 ≈ 2.1 h vs 191 s frozen+cached). +Epoch-0 val PR-AUC, unfrozen vs frozen+1M: MLP 0.399 vs 0.498, Bottleneck 0.563 vs 0.609, +JambaEHR 0.739 vs 0.808 — same lr 1e-4 on all of BERT, single seed, first epoch only. diff --git a/rian/scripts/run_notes_labs_bottleneck_transformer_seed1_unfrozen.sh b/rian/scripts/run_notes_labs_bottleneck_transformer_seed1_unfrozen.sh new file mode 100755 index 000000000..632295891 --- /dev/null +++ b/rian/scripts/run_notes_labs_bottleneck_transformer_seed1_unfrozen.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Paper cell: notes_labs x BottleneckTransformer x seed 1 — BERT UNFROZEN (trained end-to-end), no [CLS] cache. +# Identical to run_notes_labs_bottleneck_transformer_seed1.sh except --freeze-encoder is dropped. +# The cache flag is kept for config parity (inert when the encoder trains). --text-grad-checkpoint-rows +# bounds activation memory (per-layer checkpointing + 256-row chunks); the math is unchanged. +# Tree = cache1m + the gradient-checkpointing commit. +set -euo pipefail +TREE="${TREE:-/home/rianatri/ml4h-tranche1-unfrozen}" +GPU="${GPU:-0}" +EHR_ROOT="${EHR_ROOT:-/shared/rsaas/physionet.org/files/mimiciv/2.2}" +NOTE_ROOT="${NOTE_ROOT:-/shared/rsaas/physionet.org/files/mimic-note}" +CACHE_DIR="${CACHE_DIR:-/home/rianatri/pyhealth_cache/tranche1_v4_notes_labs}" +NAME=notes_labs_bottleneck_transformer_seed1_unfrozen +cd "$TREE"; mkdir -p logs output +PYTHONPATH="$TREE" CUDA_VISIBLE_DEVICES="$GPU" nohup python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \ + --task notes_labs --model bottleneck_transformer \ + --ehr-root "$EHR_ROOT" --note-root "$NOTE_ROOT" --cache-dir "$CACHE_DIR" \ + --output-dir "$TREE/output/tranche1_$NAME" \ + --embedding-dim 128 --hidden-dim 128 --batch-size 32 --lr 1e-4 --dropout 0.1 \ + --epochs 50 --patience 5 --seed 1 --use-amp --amp-dtype bf16 \ + --max-frozen-text-cache 1000000 --text-grad-checkpoint-rows 256 \ + --heads 4 --num-layers 2 --bottlenecks-n 4 --fusion-startidx 1 \ + > "logs/$NAME.out" 2>&1 & +echo $! > "logs/$NAME.pid" +echo "launched $NAME pid $(cat logs/$NAME.pid) on GPU $GPU" diff --git a/rian/scripts/run_notes_labs_jambaehr_seed1_unfrozen.sh b/rian/scripts/run_notes_labs_jambaehr_seed1_unfrozen.sh new file mode 100755 index 000000000..2bc21fb34 --- /dev/null +++ b/rian/scripts/run_notes_labs_jambaehr_seed1_unfrozen.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Paper cell: notes_labs x JambaEHR x seed 1 — BERT UNFROZEN (trained end-to-end), no [CLS] cache. +# Identical to run_notes_labs_jambaehr_seed1.sh except --freeze-encoder is dropped. +# The cache flag is kept for config parity (inert when the encoder trains). --text-grad-checkpoint-rows +# bounds activation memory (per-layer checkpointing + 256-row chunks); the math is unchanged. +# Tree = cache1m + the gradient-checkpointing commit. +set -euo pipefail +TREE="${TREE:-/home/rianatri/ml4h-tranche1-unfrozen}" +GPU="${GPU:-0}" +EHR_ROOT="${EHR_ROOT:-/shared/rsaas/physionet.org/files/mimiciv/2.2}" +NOTE_ROOT="${NOTE_ROOT:-/shared/rsaas/physionet.org/files/mimic-note}" +CACHE_DIR="${CACHE_DIR:-/home/rianatri/pyhealth_cache/tranche1_v4_notes_labs}" +NAME=notes_labs_jambaehr_seed1_unfrozen +cd "$TREE"; mkdir -p logs output +PYTHONPATH="$TREE" CUDA_VISIBLE_DEVICES="$GPU" nohup python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \ + --task notes_labs --model jambaehr \ + --ehr-root "$EHR_ROOT" --note-root "$NOTE_ROOT" --cache-dir "$CACHE_DIR" \ + --output-dir "$TREE/output/tranche1_$NAME" \ + --embedding-dim 128 --hidden-dim 128 --batch-size 32 --lr 1e-4 --dropout 0.1 \ + --epochs 50 --patience 5 --seed 1 --use-amp --amp-dtype bf16 \ + --max-frozen-text-cache 1000000 --text-grad-checkpoint-rows 256 \ + --heads 4 --num-layers 2 --jamba-transformer-layers 2 --jamba-mamba-layers 6 \ + > "logs/$NAME.out" 2>&1 & +echo $! > "logs/$NAME.pid" +echo "launched $NAME pid $(cat logs/$NAME.pid) on GPU $GPU" diff --git a/rian/scripts/run_notes_labs_mlp_seed1_cache500k.sh b/rian/scripts/run_notes_labs_mlp_seed1_cache500k.sh new file mode 100755 index 000000000..b0f9f03f2 --- /dev/null +++ b/rian/scripts/run_notes_labs_mlp_seed1_cache500k.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Paper cell: notes_labs x MLP x seed 1 — 500k [CLS] cache (cap sweep point between 200k and 1M). +# Same tree and flags as run_notes_labs_mlp_seed1.sh; only --max-frozen-text-cache differs. +set -euo pipefail +TREE="${TREE:-/home/rianatri/ml4h-tranche1-cache1m}" +GPU="${GPU:-0}" +EHR_ROOT="${EHR_ROOT:-/shared/rsaas/physionet.org/files/mimiciv/2.2}" +NOTE_ROOT="${NOTE_ROOT:-/shared/rsaas/physionet.org/files/mimic-note}" +CACHE_DIR="${CACHE_DIR:-/home/rianatri/pyhealth_cache/tranche1_v4_notes_labs}" +NAME=notes_labs_mlp_seed1_cache500k +cd "$TREE"; mkdir -p logs output +PYTHONPATH="$TREE" CUDA_VISIBLE_DEVICES="$GPU" nohup python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \ + --task notes_labs --model mlp \ + --ehr-root "$EHR_ROOT" --note-root "$NOTE_ROOT" --cache-dir "$CACHE_DIR" \ + --output-dir "$TREE/output/tranche1_$NAME" \ + --embedding-dim 128 --hidden-dim 128 --batch-size 32 --lr 1e-4 --dropout 0.1 \ + --epochs 50 --patience 5 --seed 1 --use-amp --amp-dtype bf16 --freeze-encoder \ + --max-frozen-text-cache 500000 \ + > "logs/$NAME.out" 2>&1 & +echo $! > "logs/$NAME.pid" +echo "launched $NAME pid $(cat logs/$NAME.pid) on GPU $GPU" diff --git a/rian/scripts/run_notes_labs_mlp_seed1_unfrozen.sh b/rian/scripts/run_notes_labs_mlp_seed1_unfrozen.sh new file mode 100755 index 000000000..9ada38158 --- /dev/null +++ b/rian/scripts/run_notes_labs_mlp_seed1_unfrozen.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Paper cell: notes_labs x MLP x seed 1 — BERT UNFROZEN (trained end-to-end), no [CLS] cache. +# Identical to run_notes_labs_mlp_seed1.sh except --freeze-encoder is dropped. +# The cache flag is kept for config parity (inert when the encoder trains). --text-grad-checkpoint-rows +# bounds activation memory (per-layer checkpointing + 256-row chunks); the math is unchanged. +# Tree = cache1m + the gradient-checkpointing commit. +set -euo pipefail +TREE="${TREE:-/home/rianatri/ml4h-tranche1-unfrozen}" +GPU="${GPU:-0}" +EHR_ROOT="${EHR_ROOT:-/shared/rsaas/physionet.org/files/mimiciv/2.2}" +NOTE_ROOT="${NOTE_ROOT:-/shared/rsaas/physionet.org/files/mimic-note}" +CACHE_DIR="${CACHE_DIR:-/home/rianatri/pyhealth_cache/tranche1_v4_notes_labs}" +NAME=notes_labs_mlp_seed1_unfrozen +cd "$TREE"; mkdir -p logs output +PYTHONPATH="$TREE" CUDA_VISIBLE_DEVICES="$GPU" nohup python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \ + --task notes_labs --model mlp \ + --ehr-root "$EHR_ROOT" --note-root "$NOTE_ROOT" --cache-dir "$CACHE_DIR" \ + --output-dir "$TREE/output/tranche1_$NAME" \ + --embedding-dim 128 --hidden-dim 128 --batch-size 32 --lr 1e-4 --dropout 0.1 \ + --epochs 50 --patience 5 --seed 1 --use-amp --amp-dtype bf16 \ + --max-frozen-text-cache 1000000 --text-grad-checkpoint-rows 256 \ + > "logs/$NAME.out" 2>&1 & +echo $! > "logs/$NAME.pid" +echo "launched $NAME pid $(cat logs/$NAME.pid) on GPU $GPU" From deacd03cf9205cfa46dbc4461a404cb19276b860 Mon Sep 17 00:00:00 2001 From: William Pang Date: Fri, 28 Aug 2026 16:55:38 +0000 Subject: [PATCH 46/61] Add MLP in _build_model --- .../unified_embedding_e2e_mimic4.py | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/examples/mortality_prediction/unified_embedding_e2e_mimic4.py b/examples/mortality_prediction/unified_embedding_e2e_mimic4.py index ba7327ad3..ee2799a5f 100644 --- a/examples/mortality_prediction/unified_embedding_e2e_mimic4.py +++ b/examples/mortality_prediction/unified_embedding_e2e_mimic4.py @@ -54,7 +54,7 @@ split_by_patient, split_by_sample, ) -from pyhealth.models import RNN, Transformer, UnifiedMultimodalEmbeddingModel +from pyhealth.models import MLP, RNN, Transformer, UnifiedMultimodalEmbeddingModel from pyhealth.models.bottleneck_transformer import BottleneckTransformer from pyhealth.models.ehrmamba import EHRMamba from pyhealth.models.jamba_ehr import JambaEHR @@ -162,6 +162,15 @@ def _build_model(args: argparse.Namespace, sample_dataset: Any): freeze_text_encoder=args.freeze_encoder, ) + if args.model == "mlp": + return MLP( + dataset=sample_dataset, + embedding_dim=args.embedding_dim, + hidden_dim=args.hidden_dim, + n_layers=args.mlp_layers, + activation=args.mlp_activation, + unified_embedding=unified, + ) if args.model == "rnn": return RNN( dataset=sample_dataset, @@ -377,7 +386,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--model", type=str, - choices=["rnn", "transformer", "bottleneck_transformer", + choices=["mlp", "rnn", "transformer", "bottleneck_transformer", "ehrmamba", "jambaehr"], default="rnn", ) @@ -449,6 +458,14 @@ def parse_args() -> argparse.Namespace: "downstream backbone (RNN/Transformer head + projection layer). " ), ) + parser.add_argument("--mlp-layers", type=int, default=2) + parser.add_argument( + "--mlp-activation", + type=str, + default="relu", + choices=["relu", "tanh", "sigmoid", "leaky_relu", "elu"], + ) + parser.add_argument("--rnn-type", type=str, default="GRU") parser.add_argument("--rnn-layers", type=int, default=1) parser.add_argument("--bidirectional", action="store_true") From fb272d2ab3f04fcf614cd10adeed149ca49585da Mon Sep 17 00:00:00 2001 From: William Pang Date: Sun, 30 Aug 2026 12:56:25 -0700 Subject: [PATCH 47/61] Default the frozen [CLS] cache cap to 1e6 and log cache growth. --- .../unified_embedding_e2e_mimic4.py | 12 ++++ pyhealth/models/embedding/unified.py | 36 +++++++++-- rian/scripts/README.md | 63 ------------------- ...notes_labs_bottleneck_transformer_seed1.sh | 23 ------- ...s_bottleneck_transformer_seed1_unfrozen.sh | 25 -------- rian/scripts/run_notes_labs_jambaehr_seed1.sh | 23 ------- .../run_notes_labs_jambaehr_seed1_unfrozen.sh | 25 -------- rian/scripts/run_notes_labs_mlp_seed1.sh | 22 ------- .../run_notes_labs_mlp_seed1_cache500k.sh | 21 ------- .../run_notes_labs_mlp_seed1_unfrozen.sh | 24 ------- 10 files changed, 44 insertions(+), 230 deletions(-) delete mode 100644 rian/scripts/README.md delete mode 100755 rian/scripts/run_notes_labs_bottleneck_transformer_seed1.sh delete mode 100755 rian/scripts/run_notes_labs_bottleneck_transformer_seed1_unfrozen.sh delete mode 100755 rian/scripts/run_notes_labs_jambaehr_seed1.sh delete mode 100755 rian/scripts/run_notes_labs_jambaehr_seed1_unfrozen.sh delete mode 100755 rian/scripts/run_notes_labs_mlp_seed1.sh delete mode 100755 rian/scripts/run_notes_labs_mlp_seed1_cache500k.sh delete mode 100755 rian/scripts/run_notes_labs_mlp_seed1_unfrozen.sh diff --git a/examples/mortality_prediction/unified_embedding_e2e_mimic4.py b/examples/mortality_prediction/unified_embedding_e2e_mimic4.py index ee2799a5f..6dbd14f5d 100644 --- a/examples/mortality_prediction/unified_embedding_e2e_mimic4.py +++ b/examples/mortality_prediction/unified_embedding_e2e_mimic4.py @@ -160,6 +160,7 @@ def _build_model(args: argparse.Namespace, sample_dataset: Any): processors=sample_dataset.input_processors, embedding_dim=args.embedding_dim, freeze_text_encoder=args.freeze_encoder, + max_frozen_text_cache=args.max_frozen_text_cache, ) if args.model == "mlp": @@ -449,6 +450,17 @@ def parse_args() -> argparse.Namespace: "admission. Default: full stay (through discharge)." ), ) + parser.add_argument( + "--max-frozen-text-cache", + type=int, + default=1_000_000, + help=( + "Max unique frozen [CLS] vectors on CPU. Default 1e6 (~3 GB " + "fp32, ~8 GB with Python overhead). 0 means no cap. The cap is " + "a RAM fuse, not what makes the cache fast: speedup needs " + "cap >= unique notes. 200k is too small for full MIMIC." + ), + ) parser.add_argument( "--freeze-encoder", action="store_true", diff --git a/pyhealth/models/embedding/unified.py b/pyhealth/models/embedding/unified.py index bf8050088..9058e3ced 100644 --- a/pyhealth/models/embedding/unified.py +++ b/pyhealth/models/embedding/unified.py @@ -48,6 +48,7 @@ import warnings from contextlib import nullcontext from typing import Any, Optional +import logging import torch import torch.nn.functional as F @@ -57,6 +58,13 @@ from .base import BaseEmbeddingModel from .vision import PatchEmbedding +logger = logging.getLogger(__name__) + +# Frozen [CLS] is 768-d float32 on CPU. Python tensor objects add overhead; +# treat 8 KB/entry as a conservative host-RAM estimate when logging. +_CLS_BYTES = 768 * 4 +_CACHE_LOG_SIZES = {1, 10_000, 50_000, 100_000, 200_000, 500_000, 1_000_000, 2_000_000} + # ── Helpers ─────────────────────────────────────────────────────────────────── @@ -232,7 +240,7 @@ def __init__( freeze_text_encoder: bool = False, normalize_content: bool = True, cache_frozen_text: bool = True, - max_frozen_text_cache: int = 200_000, + max_frozen_text_cache: Optional[int] = 1_000_000, numeric_standardizers: Optional[dict[str, Any]] = None, ): super().__init__() @@ -464,6 +472,10 @@ def _build_numeric_encoder( def embedding_dim(self) -> int: return self._embedding_dim + @property + def frozen_text_cache_size(self) -> int: + return sum(len(c) for c in self._frozen_text_cache.values()) + def _encode_text_cls( self, field_name: str, @@ -480,8 +492,11 @@ def _encode_text_cls( The cache has three conditions. It is used only for a field in ``_frozen_text_fields``, so a trainable encoder can never read it. The key is the token identifiers under the attention mask, so a change of - tokenizer or truncation budget gives a different key. The cache has a - maximum size, and it recalculates a row when the cache is full. + tokenizer or truncation budget gives a different key. ``None`` or + ``<= 0`` means no size cap. Default is 1_000_000 (~3 GB of fp32 + ``[CLS]`` vectors): a RAM fuse, not a speedup knob. A smaller cap + than the unique-note count makes overflow notes re-run BERT every + epoch. Notes are never dropped from the sequence. Key on the REAL tokens only. The collator pads each row to the widest note in its batch, and batch composition changes every epoch because @@ -532,8 +547,21 @@ def _encode_text_cls( ) fresh = out.last_hidden_state[:, 0, :].detach() for slot, row in zip(missing, fresh): - if len(cache) < self.max_frozen_text_cache: + cap = self.max_frozen_text_cache + if cap is None or cap <= 0 or len(cache) < cap: cache[keys[slot]] = row.cpu() + n = len(cache) + if n in _CACHE_LOG_SIZES or (n > 0 and n % 250_000 == 0): + logger.info( + "frozen text cache field=%s size=%d " + "(~%.2f GB fp32 [CLS] tensors, ~%.2f GB with " + "Python overhead). Overflow is re-encoded next " + "forward; notes are never dropped.", + field_name, + n, + n * _CLS_BYTES / 1e9, + n * 8e3 / 1e9, + ) rows = [] for k, key in enumerate(keys): diff --git a/rian/scripts/README.md b/rian/scripts/README.md deleted file mode 100644 index a99eb6366..000000000 --- a/rian/scripts/README.md +++ /dev/null @@ -1,63 +0,0 @@ -# rian/scripts — Tranche 1 paper cells (notes_labs, seed 1) - -One launcher per cell. Each mirrors the original `notes_labs × MLP × seed 1` cell -(`run_config.json` of that run), differing only in `--model` and its own flags. - -``` -GPU=2 bash rian/scripts/run_notes_labs_jambaehr_seed1.sh -GPU=3 bash rian/scripts/run_notes_labs_bottleneck_transformer_seed1.sh -GPU=0 bash rian/scripts/run_notes_labs_mlp_seed1.sh -``` - -Env overrides: `TREE` (checkout to run from), `GPU`, `EHR_ROOT`, `NOTE_ROOT`, `CACHE_DIR`. -Logs go to `$TREE/logs/.out`, outputs to `$TREE/output/tranche1_/`. -`PYTHONPATH` is pinned to `$TREE` because the `pyhealth2` env carries an editable install -that otherwise shadows the checkout. - -## Protocol (all cells) - -Full stay (admit → discharge), no ICD, empty sequences for missing modalities, frozen -Bio_ClinicalBERT with a 1e6-entry `[CLS]` cache, bf16 AMP, batch 32, lr 1e-4, dropout 0.1, -embedding/hidden 128, 50 epochs, patience 5, seed 1, split by patient with seed 1. -Tree = PR #1185 tip `8d4a4c9` + the "frozen `[CLS]` cache cap to 1e6" follow-up commit only. -All cells hit the same task cache (`NotesLabsMIMIC4_c447f3bb…`): 144,586 / 18,073 / 18,074 -patients, 852 test positives. Test metrics are sklearn `average_precision_score` / -`roc_auc_score` over `predictions_.csv` from the best-val checkpoint. - -## Results (sunlab-serv-03, one RTX A6000 per cell, 2026-08-27) - -| cell | test PR-AUC | test ROC-AUC | best val PR-AUC (epoch) | stopped at | epoch after warm-up | total train | -|---|---|---|---|---|---|---| -| MLP, 1e6 cache | 0.5662 | 0.9437 | 0.5931 (16) | 21 | 191 s | 1.5 h | -| MLP, 200k cache (original cell) | 0.5705 | 0.9386 | 0.5996 (22) | 27 | 4984 s | 38.9 h | -| JambaEHR (2 transformer + 6 mamba) | 0.8081 | 0.9751 | 0.8203 (1) | 6 | 372 s | 1.0 h | -| BottleneckTransformer (n=4, fusion start 1) | 0.6907 | 0.9608 | 0.6958 (8) | 13 | 224 s | 1.2 h | - -Notes: - -- The two MLP rows are the compute-matched cache comparison: same host, same GPU, - same split, same flags — only the cache cap differs. Epoch time after warm-up drops - 26×; metrics land within run-to-run noise (not bit-identical: early stopping fired at - a different epoch). -- Single seed. JambaEHR peaks at epoch 1; treat the gap to the other backbones as - provisional until multi-seed. -- Why 200k was slower than *no* cache: once the cap is reached, misses were encoded in - the batched pass, not inserted, then encoded again one row at a time by the assembly - loop. Fixed on the PR follow-ups ("Encode a frozen-cache miss once per forward, even - when the cache is full"). The 200k row above ran on the pre-fix code. - -## Cache-cap sweep and unfrozen BERT (launched 2026-08-28, results pending) - -| script | condition | tree | -|---|---|---| -| `run_notes_labs_mlp_seed1_cache500k.sh` | frozen BERT, 500k cap | same as 1M cells | -| `run_notes_labs_mlp_seed1_unfrozen.sh` | BERT trained end-to-end | 1M tree + gradient-checkpointing commit | -| `run_notes_labs_jambaehr_seed1_unfrozen.sh` | BERT trained end-to-end | same | -| `run_notes_labs_bottleneck_transformer_seed1_unfrozen.sh` | BERT trained end-to-end | same | - -The unfrozen launchers drop `--freeze-encoder` and pass `--text-grad-checkpoint-rows 256`: -a trainable BERT keeps every note row's activations for the backward pass and one fat batch -exceeded 47 GB at step 4; the flag turns on per-layer gradient checkpointing and chunks note -rows through the encoder (math unchanged, ~12.5 GB peak, epoch 0 ≈ 2.1 h vs 191 s frozen+cached). -Epoch-0 val PR-AUC, unfrozen vs frozen+1M: MLP 0.399 vs 0.498, Bottleneck 0.563 vs 0.609, -JambaEHR 0.739 vs 0.808 — same lr 1e-4 on all of BERT, single seed, first epoch only. diff --git a/rian/scripts/run_notes_labs_bottleneck_transformer_seed1.sh b/rian/scripts/run_notes_labs_bottleneck_transformer_seed1.sh deleted file mode 100755 index 23f1ea2f6..000000000 --- a/rian/scripts/run_notes_labs_bottleneck_transformer_seed1.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -# Paper cell: notes_labs x BottleneckTransformer x seed 1. -# Full stay, empty-sequence missingness, frozen BERT, bf16 AMP, 1e6-entry [CLS] cache. -# Tree = 8d4a4c9 + the frozen-cache-cap commit only (results-identical, faster epochs). -set -euo pipefail -TREE="${TREE:-/home/rianatri/ml4h-tranche1-cache1m}" -GPU="${GPU:-0}" -EHR_ROOT="${EHR_ROOT:-/shared/rsaas/physionet.org/files/mimiciv/2.2}" -NOTE_ROOT="${NOTE_ROOT:-/shared/rsaas/physionet.org/files/mimic-note}" -CACHE_DIR="${CACHE_DIR:-/home/rianatri/pyhealth_cache/tranche1_v4_notes_labs}" -NAME=notes_labs_bottleneck_transformer_seed1_cache1m -cd "$TREE"; mkdir -p logs output -PYTHONPATH="$TREE" CUDA_VISIBLE_DEVICES="$GPU" nohup python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \ - --task notes_labs --model bottleneck_transformer \ - --ehr-root "$EHR_ROOT" --note-root "$NOTE_ROOT" --cache-dir "$CACHE_DIR" \ - --output-dir "$TREE/output/tranche1_$NAME" \ - --embedding-dim 128 --hidden-dim 128 --batch-size 32 --lr 1e-4 --dropout 0.1 \ - --epochs 50 --patience 5 --seed 1 --use-amp --amp-dtype bf16 --freeze-encoder \ - --max-frozen-text-cache 1000000 \ - --heads 4 --num-layers 2 --bottlenecks-n 4 --fusion-startidx 1 \ - > "logs/$NAME.out" 2>&1 & -echo $! > "logs/$NAME.pid" -echo "launched $NAME pid $(cat logs/$NAME.pid) on GPU $GPU" diff --git a/rian/scripts/run_notes_labs_bottleneck_transformer_seed1_unfrozen.sh b/rian/scripts/run_notes_labs_bottleneck_transformer_seed1_unfrozen.sh deleted file mode 100755 index 632295891..000000000 --- a/rian/scripts/run_notes_labs_bottleneck_transformer_seed1_unfrozen.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -# Paper cell: notes_labs x BottleneckTransformer x seed 1 — BERT UNFROZEN (trained end-to-end), no [CLS] cache. -# Identical to run_notes_labs_bottleneck_transformer_seed1.sh except --freeze-encoder is dropped. -# The cache flag is kept for config parity (inert when the encoder trains). --text-grad-checkpoint-rows -# bounds activation memory (per-layer checkpointing + 256-row chunks); the math is unchanged. -# Tree = cache1m + the gradient-checkpointing commit. -set -euo pipefail -TREE="${TREE:-/home/rianatri/ml4h-tranche1-unfrozen}" -GPU="${GPU:-0}" -EHR_ROOT="${EHR_ROOT:-/shared/rsaas/physionet.org/files/mimiciv/2.2}" -NOTE_ROOT="${NOTE_ROOT:-/shared/rsaas/physionet.org/files/mimic-note}" -CACHE_DIR="${CACHE_DIR:-/home/rianatri/pyhealth_cache/tranche1_v4_notes_labs}" -NAME=notes_labs_bottleneck_transformer_seed1_unfrozen -cd "$TREE"; mkdir -p logs output -PYTHONPATH="$TREE" CUDA_VISIBLE_DEVICES="$GPU" nohup python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \ - --task notes_labs --model bottleneck_transformer \ - --ehr-root "$EHR_ROOT" --note-root "$NOTE_ROOT" --cache-dir "$CACHE_DIR" \ - --output-dir "$TREE/output/tranche1_$NAME" \ - --embedding-dim 128 --hidden-dim 128 --batch-size 32 --lr 1e-4 --dropout 0.1 \ - --epochs 50 --patience 5 --seed 1 --use-amp --amp-dtype bf16 \ - --max-frozen-text-cache 1000000 --text-grad-checkpoint-rows 256 \ - --heads 4 --num-layers 2 --bottlenecks-n 4 --fusion-startidx 1 \ - > "logs/$NAME.out" 2>&1 & -echo $! > "logs/$NAME.pid" -echo "launched $NAME pid $(cat logs/$NAME.pid) on GPU $GPU" diff --git a/rian/scripts/run_notes_labs_jambaehr_seed1.sh b/rian/scripts/run_notes_labs_jambaehr_seed1.sh deleted file mode 100755 index 9c1661b78..000000000 --- a/rian/scripts/run_notes_labs_jambaehr_seed1.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -# Paper cell: notes_labs x JambaEHR x seed 1. -# Full stay, empty-sequence missingness, frozen BERT, bf16 AMP, 1e6-entry [CLS] cache. -# Tree = 8d4a4c9 + the frozen-cache-cap commit only (results-identical, faster epochs). -set -euo pipefail -TREE="${TREE:-/home/rianatri/ml4h-tranche1-cache1m}" -GPU="${GPU:-0}" -EHR_ROOT="${EHR_ROOT:-/shared/rsaas/physionet.org/files/mimiciv/2.2}" -NOTE_ROOT="${NOTE_ROOT:-/shared/rsaas/physionet.org/files/mimic-note}" -CACHE_DIR="${CACHE_DIR:-/home/rianatri/pyhealth_cache/tranche1_v4_notes_labs}" -NAME=notes_labs_jambaehr_seed1_cache1m -cd "$TREE"; mkdir -p logs output -PYTHONPATH="$TREE" CUDA_VISIBLE_DEVICES="$GPU" nohup python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \ - --task notes_labs --model jambaehr \ - --ehr-root "$EHR_ROOT" --note-root "$NOTE_ROOT" --cache-dir "$CACHE_DIR" \ - --output-dir "$TREE/output/tranche1_$NAME" \ - --embedding-dim 128 --hidden-dim 128 --batch-size 32 --lr 1e-4 --dropout 0.1 \ - --epochs 50 --patience 5 --seed 1 --use-amp --amp-dtype bf16 --freeze-encoder \ - --max-frozen-text-cache 1000000 \ - --heads 4 --num-layers 2 --jamba-transformer-layers 2 --jamba-mamba-layers 6 \ - > "logs/$NAME.out" 2>&1 & -echo $! > "logs/$NAME.pid" -echo "launched $NAME pid $(cat logs/$NAME.pid) on GPU $GPU" diff --git a/rian/scripts/run_notes_labs_jambaehr_seed1_unfrozen.sh b/rian/scripts/run_notes_labs_jambaehr_seed1_unfrozen.sh deleted file mode 100755 index 2bc21fb34..000000000 --- a/rian/scripts/run_notes_labs_jambaehr_seed1_unfrozen.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -# Paper cell: notes_labs x JambaEHR x seed 1 — BERT UNFROZEN (trained end-to-end), no [CLS] cache. -# Identical to run_notes_labs_jambaehr_seed1.sh except --freeze-encoder is dropped. -# The cache flag is kept for config parity (inert when the encoder trains). --text-grad-checkpoint-rows -# bounds activation memory (per-layer checkpointing + 256-row chunks); the math is unchanged. -# Tree = cache1m + the gradient-checkpointing commit. -set -euo pipefail -TREE="${TREE:-/home/rianatri/ml4h-tranche1-unfrozen}" -GPU="${GPU:-0}" -EHR_ROOT="${EHR_ROOT:-/shared/rsaas/physionet.org/files/mimiciv/2.2}" -NOTE_ROOT="${NOTE_ROOT:-/shared/rsaas/physionet.org/files/mimic-note}" -CACHE_DIR="${CACHE_DIR:-/home/rianatri/pyhealth_cache/tranche1_v4_notes_labs}" -NAME=notes_labs_jambaehr_seed1_unfrozen -cd "$TREE"; mkdir -p logs output -PYTHONPATH="$TREE" CUDA_VISIBLE_DEVICES="$GPU" nohup python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \ - --task notes_labs --model jambaehr \ - --ehr-root "$EHR_ROOT" --note-root "$NOTE_ROOT" --cache-dir "$CACHE_DIR" \ - --output-dir "$TREE/output/tranche1_$NAME" \ - --embedding-dim 128 --hidden-dim 128 --batch-size 32 --lr 1e-4 --dropout 0.1 \ - --epochs 50 --patience 5 --seed 1 --use-amp --amp-dtype bf16 \ - --max-frozen-text-cache 1000000 --text-grad-checkpoint-rows 256 \ - --heads 4 --num-layers 2 --jamba-transformer-layers 2 --jamba-mamba-layers 6 \ - > "logs/$NAME.out" 2>&1 & -echo $! > "logs/$NAME.pid" -echo "launched $NAME pid $(cat logs/$NAME.pid) on GPU $GPU" diff --git a/rian/scripts/run_notes_labs_mlp_seed1.sh b/rian/scripts/run_notes_labs_mlp_seed1.sh deleted file mode 100755 index 744377113..000000000 --- a/rian/scripts/run_notes_labs_mlp_seed1.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -# Paper cell: notes_labs x MLP x seed 1 (cache-1M compute-matched rerun of the original 200k-cache cell). -# Full stay, empty-sequence missingness, frozen BERT, bf16 AMP, 1e6-entry [CLS] cache. -# Tree = 8d4a4c9 + the frozen-cache-cap commit only (results-identical, faster epochs). -set -euo pipefail -TREE="${TREE:-/home/rianatri/ml4h-tranche1-cache1m}" -GPU="${GPU:-0}" -EHR_ROOT="${EHR_ROOT:-/shared/rsaas/physionet.org/files/mimiciv/2.2}" -NOTE_ROOT="${NOTE_ROOT:-/shared/rsaas/physionet.org/files/mimic-note}" -CACHE_DIR="${CACHE_DIR:-/home/rianatri/pyhealth_cache/tranche1_v4_notes_labs}" -NAME=notes_labs_mlp_seed1_cache1m -cd "$TREE"; mkdir -p logs output -PYTHONPATH="$TREE" CUDA_VISIBLE_DEVICES="$GPU" nohup python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \ - --task notes_labs --model mlp \ - --ehr-root "$EHR_ROOT" --note-root "$NOTE_ROOT" --cache-dir "$CACHE_DIR" \ - --output-dir "$TREE/output/tranche1_$NAME" \ - --embedding-dim 128 --hidden-dim 128 --batch-size 32 --lr 1e-4 --dropout 0.1 \ - --epochs 50 --patience 5 --seed 1 --use-amp --amp-dtype bf16 --freeze-encoder \ - --max-frozen-text-cache 1000000 \ - > "logs/$NAME.out" 2>&1 & -echo $! > "logs/$NAME.pid" -echo "launched $NAME pid $(cat logs/$NAME.pid) on GPU $GPU" diff --git a/rian/scripts/run_notes_labs_mlp_seed1_cache500k.sh b/rian/scripts/run_notes_labs_mlp_seed1_cache500k.sh deleted file mode 100755 index b0f9f03f2..000000000 --- a/rian/scripts/run_notes_labs_mlp_seed1_cache500k.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash -# Paper cell: notes_labs x MLP x seed 1 — 500k [CLS] cache (cap sweep point between 200k and 1M). -# Same tree and flags as run_notes_labs_mlp_seed1.sh; only --max-frozen-text-cache differs. -set -euo pipefail -TREE="${TREE:-/home/rianatri/ml4h-tranche1-cache1m}" -GPU="${GPU:-0}" -EHR_ROOT="${EHR_ROOT:-/shared/rsaas/physionet.org/files/mimiciv/2.2}" -NOTE_ROOT="${NOTE_ROOT:-/shared/rsaas/physionet.org/files/mimic-note}" -CACHE_DIR="${CACHE_DIR:-/home/rianatri/pyhealth_cache/tranche1_v4_notes_labs}" -NAME=notes_labs_mlp_seed1_cache500k -cd "$TREE"; mkdir -p logs output -PYTHONPATH="$TREE" CUDA_VISIBLE_DEVICES="$GPU" nohup python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \ - --task notes_labs --model mlp \ - --ehr-root "$EHR_ROOT" --note-root "$NOTE_ROOT" --cache-dir "$CACHE_DIR" \ - --output-dir "$TREE/output/tranche1_$NAME" \ - --embedding-dim 128 --hidden-dim 128 --batch-size 32 --lr 1e-4 --dropout 0.1 \ - --epochs 50 --patience 5 --seed 1 --use-amp --amp-dtype bf16 --freeze-encoder \ - --max-frozen-text-cache 500000 \ - > "logs/$NAME.out" 2>&1 & -echo $! > "logs/$NAME.pid" -echo "launched $NAME pid $(cat logs/$NAME.pid) on GPU $GPU" diff --git a/rian/scripts/run_notes_labs_mlp_seed1_unfrozen.sh b/rian/scripts/run_notes_labs_mlp_seed1_unfrozen.sh deleted file mode 100755 index 9ada38158..000000000 --- a/rian/scripts/run_notes_labs_mlp_seed1_unfrozen.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -# Paper cell: notes_labs x MLP x seed 1 — BERT UNFROZEN (trained end-to-end), no [CLS] cache. -# Identical to run_notes_labs_mlp_seed1.sh except --freeze-encoder is dropped. -# The cache flag is kept for config parity (inert when the encoder trains). --text-grad-checkpoint-rows -# bounds activation memory (per-layer checkpointing + 256-row chunks); the math is unchanged. -# Tree = cache1m + the gradient-checkpointing commit. -set -euo pipefail -TREE="${TREE:-/home/rianatri/ml4h-tranche1-unfrozen}" -GPU="${GPU:-0}" -EHR_ROOT="${EHR_ROOT:-/shared/rsaas/physionet.org/files/mimiciv/2.2}" -NOTE_ROOT="${NOTE_ROOT:-/shared/rsaas/physionet.org/files/mimic-note}" -CACHE_DIR="${CACHE_DIR:-/home/rianatri/pyhealth_cache/tranche1_v4_notes_labs}" -NAME=notes_labs_mlp_seed1_unfrozen -cd "$TREE"; mkdir -p logs output -PYTHONPATH="$TREE" CUDA_VISIBLE_DEVICES="$GPU" nohup python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \ - --task notes_labs --model mlp \ - --ehr-root "$EHR_ROOT" --note-root "$NOTE_ROOT" --cache-dir "$CACHE_DIR" \ - --output-dir "$TREE/output/tranche1_$NAME" \ - --embedding-dim 128 --hidden-dim 128 --batch-size 32 --lr 1e-4 --dropout 0.1 \ - --epochs 50 --patience 5 --seed 1 --use-amp --amp-dtype bf16 \ - --max-frozen-text-cache 1000000 --text-grad-checkpoint-rows 256 \ - > "logs/$NAME.out" 2>&1 & -echo $! > "logs/$NAME.pid" -echo "launched $NAME pid $(cat logs/$NAME.pid) on GPU $GPU" From 659610e7f0a96efe7966756d840da98b25105248 Mon Sep 17 00:00:00 2001 From: William Pang Date: Sun, 30 Aug 2026 13:07:16 -0700 Subject: [PATCH 48/61] Bound trainable text-encoder memory with gradient checkpointing and chunked note rows. --- .../unified_embedding_e2e_mimic4.py | 11 +++++++ pyhealth/models/embedding/unified.py | 30 ++++++++++++++++--- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/examples/mortality_prediction/unified_embedding_e2e_mimic4.py b/examples/mortality_prediction/unified_embedding_e2e_mimic4.py index 6dbd14f5d..085b54ae6 100644 --- a/examples/mortality_prediction/unified_embedding_e2e_mimic4.py +++ b/examples/mortality_prediction/unified_embedding_e2e_mimic4.py @@ -161,6 +161,7 @@ def _build_model(args: argparse.Namespace, sample_dataset: Any): embedding_dim=args.embedding_dim, freeze_text_encoder=args.freeze_encoder, max_frozen_text_cache=args.max_frozen_text_cache, + text_grad_checkpoint_rows=args.text_grad_checkpoint_rows, ) if args.model == "mlp": @@ -461,6 +462,16 @@ def parse_args() -> argparse.Namespace: "cap >= unique notes. 200k is too small for full MIMIC." ), ) + parser.add_argument( + "--text-grad-checkpoint-rows", + type=int, + default=0, + help=( + "Trainable text encoder only: enable gradient checkpointing and " + "run note rows through BERT in chunks of this size. Bounds " + "activation memory; the math is unchanged. 0 disables." + ), + ) parser.add_argument( "--freeze-encoder", action="store_true", diff --git a/pyhealth/models/embedding/unified.py b/pyhealth/models/embedding/unified.py index 9058e3ced..3a10d3641 100644 --- a/pyhealth/models/embedding/unified.py +++ b/pyhealth/models/embedding/unified.py @@ -52,6 +52,7 @@ import torch import torch.nn.functional as F +from torch.utils.checkpoint import checkpoint as _checkpoint import torch.nn as nn from ...processors.base_processor import ModalityType, TemporalFeatureProcessor @@ -241,6 +242,7 @@ def __init__( normalize_content: bool = True, cache_frozen_text: bool = True, max_frozen_text_cache: Optional[int] = 1_000_000, + text_grad_checkpoint_rows: int = 0, numeric_standardizers: Optional[dict[str, Any]] = None, ): super().__init__() @@ -253,6 +255,11 @@ def __init__( self._frozen_text_fields: set[str] = set() self.cache_frozen_text = cache_frozen_text self.max_frozen_text_cache = max_frozen_text_cache + # A trainable text encoder keeps every note row's activations for the + # backward pass; one batch with many long notes exceeds 47 GB. When > 0, + # the encoder uses per-layer gradient checkpointing and note rows go + # through it in chunks of this size under torch checkpoint. Same math. + self.text_grad_checkpoint_rows = text_grad_checkpoint_rows self._frozen_text_cache: dict[str, dict[int, torch.Tensor]] = {} self.image_pool = image_pool self.normalize_content = normalize_content @@ -399,6 +406,8 @@ def _set_projection( for p in bert.parameters(): p.requires_grad = False self._frozen_text_fields.add(field_name) + elif self.text_grad_checkpoint_rows > 0: + bert.gradient_checkpointing_enable() self.encoders[field_name] = bert hidden = bert.config.hidden_size if hidden != embedding_dim: @@ -512,10 +521,23 @@ def _encode_text_cls( ) if not (self.cache_frozen_text and field_name in self._frozen_text_fields): - ctx = torch.no_grad() if field_name in self._frozen_text_fields else nullcontext() - with ctx: - out = encoder(input_ids=flat_ids, attention_mask=flat_mask) - return out.last_hidden_state[:, 0, :] + frozen = field_name in self._frozen_text_fields + rows = self.text_grad_checkpoint_rows + if frozen or rows <= 0 or not torch.is_grad_enabled(): + ctx = torch.no_grad() if frozen else nullcontext() + with ctx: + out = encoder(input_ids=flat_ids, attention_mask=flat_mask) + return out.last_hidden_state[:, 0, :] + + def _cls(ids: torch.Tensor, mask: Optional[torch.Tensor]) -> torch.Tensor: + return encoder(input_ids=ids, attention_mask=mask).last_hidden_state[:, 0, :] + + outs = [] + for start in range(0, flat_ids.shape[0], rows): + ids = flat_ids[start : start + rows] + mask = flat_mask[start : start + rows] if flat_mask is not None else None + outs.append(_checkpoint(_cls, ids, mask, use_reentrant=False)) + return torch.cat(outs, dim=0) cache = self._frozen_text_cache.setdefault(field_name, {}) ids_cpu = flat_ids.detach().cpu() From 4e158a6b87ee636c70295f8c7be18d365a4fcde2 Mon Sep 17 00:00:00 2001 From: William Pang Date: Sun, 30 Aug 2026 13:08:31 -0700 Subject: [PATCH 49/61] Encode a frozen-cache miss once per forward, even when the cache is full. --- pyhealth/models/embedding/unified.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/pyhealth/models/embedding/unified.py b/pyhealth/models/embedding/unified.py index 3a10d3641..331d174b7 100644 --- a/pyhealth/models/embedding/unified.py +++ b/pyhealth/models/embedding/unified.py @@ -556,6 +556,9 @@ def _cls(ids: torch.Tensor, mask: Optional[torch.Tensor]) -> torch.Tensor: if key not in cache and key not in first_row_of_key: first_row_of_key[key] = k missing = list(first_row_of_key.values()) + # Rows encoded in this forward, kept even when the cache is full so a + # miss is never encoded twice in one call. + fresh_rows: dict[int, torch.Tensor] = {} if missing: index = torch.tensor(missing, device=flat_ids.device) with torch.no_grad(): @@ -569,6 +572,7 @@ def _cls(ids: torch.Tensor, mask: Optional[torch.Tensor]) -> torch.Tensor: ) fresh = out.last_hidden_state[:, 0, :].detach() for slot, row in zip(missing, fresh): + fresh_rows[keys[slot]] = row cap = self.max_frozen_text_cache if cap is None or cap <= 0 or len(cache) < cap: cache[keys[slot]] = row.cpu() @@ -586,17 +590,10 @@ def _cls(ids: torch.Tensor, mask: Optional[torch.Tensor]) -> torch.Tensor: ) rows = [] - for k, key in enumerate(keys): + for key in keys: hit = cache.get(key) if hit is None: - with torch.no_grad(): - out = encoder( - input_ids=flat_ids[k : k + 1], - attention_mask=( - flat_mask[k : k + 1] if flat_mask is not None else None - ), - ) - rows.append(out.last_hidden_state[0, 0, :].detach()) + rows.append(fresh_rows[key]) else: rows.append(hit.to(flat_ids.device)) return torch.stack(rows).to(dtype=self.type_embedding.weight.dtype) From be65739a7e810d4aa78239ae21fc24d957e9ace3 Mon Sep 17 00:00:00 2001 From: William Pang Date: Sun, 30 Aug 2026 13:57:42 -0700 Subject: [PATCH 50/61] Add Will's Run Scripts --- .../unified_embedding_e2e_mimic4.py | 177 ++++++++++++++---- ...bs_notes_bottleneck_transformer_variant.py | 2 - .../tmux_run_labs_notes_ehrmamba_variant.py | 6 +- .../tmux_run_labs_notes_jambaehr_variant.py | 6 +- .../tmux_run_labs_notes_mlp_variant.py | 134 +++++++++++++ .../tmux_run_labs_notes_rnn_variant.py | 6 +- ...tmux_run_labs_notes_transformer_variant.py | 6 +- 7 files changed, 282 insertions(+), 55 deletions(-) create mode 100644 scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_mlp_variant.py diff --git a/examples/mortality_prediction/unified_embedding_e2e_mimic4.py b/examples/mortality_prediction/unified_embedding_e2e_mimic4.py index 085b54ae6..75968787c 100644 --- a/examples/mortality_prediction/unified_embedding_e2e_mimic4.py +++ b/examples/mortality_prediction/unified_embedding_e2e_mimic4.py @@ -43,6 +43,8 @@ import argparse import csv +import logging +import warnings from pathlib import Path from typing import Any, Dict, Optional, Tuple @@ -66,6 +68,8 @@ from pyhealth.trainer import Trainer from pyhealth.utils import set_seed +logger = logging.getLogger(__name__) + class WandbLogger: @@ -148,14 +152,102 @@ def _build_task(args: argparse.Namespace): raise ValueError(f"Unknown task: {args.task}") -def _split_dataset(dataset: Any, seed: int) -> Tuple[Any, Any, Any]: +def _split_dataset( + dataset: Any, seed: int, allow_leaky_split: bool = False +) -> Tuple[Any, Any, Any]: + """Split by patient. Refuse the leaky by-sample fallback unless asked. + + The fallback puts a patient with several admissions in both train and test, + which inflates every metric, and it used to happen with no warning at all. + A run that cannot be split correctly should stop rather than quietly produce + numbers nobody can use. + """ train_ds, val_ds, test_ds = split_by_patient(dataset, [0.8, 0.1, 0.1], seed=seed) - if len(train_ds) == 0 or len(test_ds) == 0: - train_ds, val_ds, test_ds = split_by_sample(dataset, [0.8, 0.1, 0.1], seed=seed) - return train_ds, val_ds, test_ds + if len(train_ds) > 0 and len(test_ds) > 0: + return train_ds, val_ds, test_ds + + if not allow_leaky_split: + raise RuntimeError( + f"split_by_patient produced an empty split (train={len(train_ds)}, " + f"test={len(test_ds)}) on {len(dataset)} samples. The by-sample " + "fallback would put the same patient in train and test, so this " + "run is refused. Widen the cohort, or pass --allow-leaky-split if " + "you are running a smoke test and know the metrics are garbage." + ) + + warnings.warn( + "Falling back to split_by_sample at your request. The same patient may " + "now appear in train and test, so these metrics are optimistic and not " + "comparable to patient-split runs.", + RuntimeWarning, + stacklevel=2, + ) + return split_by_sample(dataset, [0.8, 0.1, 0.1], seed=seed) + + +# Architecture flags each backbone actually consumes. Anything else the parser +# accepts is inert for that model and is warned about at startup, so a launcher +# can never appear to set something that never reached the model. --dropout is +# absent for mlp on purpose: pyhealth.models.mlp.MLP has no dropout parameter, +# and it takes **kwargs, so passing one is swallowed rather than rejected. +_ARCH_FLAGS_USED: Dict[str, Tuple[str, ...]] = { + "mlp": ("embedding_dim", "hidden_dim", "mlp_layers", "mlp_activation"), + "rnn": ( + "embedding_dim", + "hidden_dim", + "dropout", + "rnn_type", + "rnn_layers", + "bidirectional", + ), + "transformer": ("embedding_dim", "dropout", "heads", "num_layers"), + "bottleneck_transformer": ( + "embedding_dim", + "dropout", + "heads", + "num_layers", + "bottlenecks_n", + "fusion_startidx", + ), + "ehrmamba": ( + "embedding_dim", + "dropout", + "num_layers", + "mamba_state_size", + "mamba_conv_kernel", + ), + "jambaehr": ( + "embedding_dim", + "dropout", + "heads", + "mamba_state_size", + "mamba_conv_kernel", + "jamba_transformer_layers", + "jamba_mamba_layers", + ), +} + +_ARCH_FLAGS_ALL: Tuple[str, ...] = tuple( + sorted({flag for flags in _ARCH_FLAGS_USED.values() for flag in flags}) +) + + +def _inert_arch_flags(model: str) -> list[str]: + """Architecture flags this backbone ignores, as CLI spellings.""" + used = set(_ARCH_FLAGS_USED[model]) + return ["--" + f.replace("_", "-") for f in _ARCH_FLAGS_ALL if f not in used] def _build_model(args: argparse.Namespace, sample_dataset: Any): + inert = _inert_arch_flags(args.model) + if inert: + logger.warning( + "%s ignores these flags: %s. Do not read them as settings that " + "took effect.", + args.model, + " ".join(inert), + ) + unified = UnifiedMultimodalEmbeddingModel( processors=sample_dataset.input_processors, embedding_dim=args.embedding_dim, @@ -269,7 +361,9 @@ def run(args: argparse.Namespace) -> Path: "Task produced zero samples. Check roots/tables or adjust settings." ) - train_ds, val_ds, test_ds = _split_dataset(sample_dataset, seed=args.seed) + train_ds, val_ds, test_ds = _split_dataset( + sample_dataset, seed=args.seed, allow_leaky_split=args.allow_leaky_split + ) model = _build_model(args, sample_dataset) @@ -306,27 +400,14 @@ def run(args: argparse.Namespace) -> Path: exp_name=exp_name, ) - # BottleneckTransformer is more fragile on full MIMIC-IV with no warmup. - # Use safer defaults unless explicitly overridden from CLI. + # Optimizer settings come from the CLI only. There used to be a per-model + # branch here that gave bottleneck_transformer max_grad_norm=0.5 and Adam + # eps=1e-6 while every other backbone got 1.0 and 1e-8, so a six-backbone + # table that reads as compute-matched was not. If a model needs different + # optimizer settings, the launcher has to say so. effective_lr = args.lr effective_max_grad_norm = args.max_grad_norm - optimizer_params = {} - - if args.model == "bottleneck_transformer": - if effective_lr is None: - effective_lr = 1e-4 - if effective_max_grad_norm is None: - effective_max_grad_norm = 0.5 - optimizer_params["eps"] = args.adam_eps if args.adam_eps is not None else 1e-6 - else: - if effective_lr is None: - effective_lr = 1e-4 - if effective_max_grad_norm is None: - effective_max_grad_norm = 1.0 - if args.adam_eps is not None: - optimizer_params["eps"] = args.adam_eps - - optimizer_params["lr"] = effective_lr + optimizer_params = {"lr": effective_lr, "eps": args.adam_eps} if args.epochs > 0 and len(train_ds) > 0: metrics_history = trainer.train( @@ -401,16 +482,23 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--lr", type=float, - default=None, - help="Learning rate. Default is 1e-4 for all models.", + default=1e-4, + help="Learning rate, same for every model.", ) parser.add_argument( "--adam-eps", type=float, - default=None, + default=1e-8, + help="Adam epsilon, same for every model (torch default).", + ) + parser.add_argument( + "--allow-leaky-split", + action="store_true", + default=False, help=( - "Adam epsilon. Default is model-specific: 1e-8 for non-BT models, " - "1e-6 for bottleneck_transformer." + "Permit the by-sample split fallback when the patient split is " + "empty. The same patient can then land in train and test. Smoke " + "tests only — the metrics are not usable." ), ) parser.add_argument("--weight-decay", type=float, default=0.0) @@ -422,10 +510,15 @@ def parse_args() -> argparse.Namespace: ) parser.add_argument( "--amp-dtype", + "--amp_dtype", + dest="amp_dtype", type=str, - default="bf16", + default=None, choices=["bf16", "fp16"], - help="AMP dtype when --use-amp is set. bf16 is more stable (default).", + help=( + "AMP dtype. Requires --use-amp; passing this alone is an error " + "rather than a silently fp32 run. Defaults to bf16 with --use-amp." + ), ) parser.add_argument("--num-workers", type=int, default=1) parser.add_argument("--seed", type=int, default=42) @@ -502,11 +595,8 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--max-grad-norm", type=float, - default=None, - help=( - "Gradient clipping max norm. Default is model-specific: None for " - "non-BT models, 0.5 for bottleneck_transformer." - ), + default=1.0, + help="Gradient clipping max norm, same for every model.", ) parser.add_argument( @@ -539,7 +629,20 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--jamba-mamba-layers", type=int, default=6, help="Number of Mamba (SSM) layers in JambaEHR.") - return parser.parse_args() + args = parser.parse_args() + + # The Tranche 1 flag list says --amp_dtype "bf16" and never mentions + # --use-amp, so following it literally used to give a silently fp32 run. + if args.amp_dtype is not None and not args.use_amp: + parser.error( + f"--amp-dtype {args.amp_dtype} was passed without --use-amp, so " + "mixed precision would be off and the run would be fp32 while the " + "config claimed otherwise. Pass --use-amp, or drop --amp-dtype." + ) + if args.amp_dtype is None: + args.amp_dtype = "bf16" + + return args if __name__ == "__main__": diff --git a/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_bottleneck_transformer_variant.py b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_bottleneck_transformer_variant.py index b91d975ff..c50d92de8 100644 --- a/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_bottleneck_transformer_variant.py +++ b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_bottleneck_transformer_variant.py @@ -7,7 +7,6 @@ logs_dir = "/home/ubuntu/logs" output_dir = "/home/ubuntu/output" embedding_dim = 128 -hidden_dim = 128 heads = 4 num_layers = 2 bottlenecks_n = 4 @@ -66,7 +65,6 @@ f"--task notes_labs{' --dev' if dev else ''}", "--model bottleneck_transformer", f"--embedding-dim {embedding_dim}", - f"--hidden-dim {hidden_dim}", f"--heads {heads}", f"--num-layers {num_layers}", f"--bottlenecks-n {bottlenecks_n}", diff --git a/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_ehrmamba_variant.py b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_ehrmamba_variant.py index 739281819..91fd7924d 100644 --- a/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_ehrmamba_variant.py +++ b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_ehrmamba_variant.py @@ -1,5 +1,5 @@ project_dir = "PyHealth" -seed = 12 +seed = 2 conda_env = "pyhealth2" ehr_root = "/home/ubuntu/mimiciv-data/ehr" note_root = "/home/ubuntu/mimiciv-data" @@ -17,14 +17,13 @@ epochs = 50 batch_size = 32 lr = 1e-4 -weight_decay = 1e-5 patience = 5 num_workers = 4 freeze_encoder = True dev = False use_old_cache = False use_wandb = True -wandb_project = "pyhealth-multimodal-labs-notes" +wandb_project = f"pyhealth-multimodal-labs-notes-seed-{seed}" wandb_run_name = None # defaults to "{model}_seed{seed}" if unset cuda_visible_devices = "0" session_name = f"ehrmamba_labs_notes_s{seed}" @@ -78,7 +77,6 @@ f"--epochs {epochs}", f"--batch-size {batch_size}", f"--lr {lr}", - f"--weight-decay {weight_decay}", f"--patience {patience}", f"--num-workers {num_workers}", ] diff --git a/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_jambaehr_variant.py b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_jambaehr_variant.py index d7fc5df3e..d07cb477b 100644 --- a/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_jambaehr_variant.py +++ b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_jambaehr_variant.py @@ -1,5 +1,5 @@ project_dir = "PyHealth" -seed = 12 +seed = 2 conda_env = "pyhealth2" ehr_root = "/home/ubuntu/mimiciv-data/ehr" note_root = "/home/ubuntu/mimiciv-data" @@ -20,14 +20,13 @@ epochs = 50 batch_size = 32 lr = 1e-4 -weight_decay = 1e-5 patience = 5 num_workers = 4 freeze_encoder = True dev = False use_old_cache = False use_wandb = True -wandb_project = "pyhealth-multimodal-labs-notes" +wandb_project = f"pyhealth-multimodal-labs-notes-seed-{seed}" wandb_run_name = None # defaults to "{model}_seed{seed}" if unset cuda_visible_devices = "0" session_name = f"jambaehr_labs_notes_s{seed}_batchsize{batch_size}" @@ -84,7 +83,6 @@ f"--epochs {epochs}", f"--batch-size {batch_size}", f"--lr {lr}", - f"--weight-decay {weight_decay}", f"--patience {patience}", f"--num-workers {num_workers}", ] diff --git a/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_mlp_variant.py b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_mlp_variant.py new file mode 100644 index 000000000..09dafec26 --- /dev/null +++ b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_mlp_variant.py @@ -0,0 +1,134 @@ +project_dir = "PyHealth" +seed = 2 +conda_env = "pyhealth2" +ehr_root = "/home/ubuntu/mimiciv-data/ehr" +note_root = "/home/ubuntu/mimiciv-data" +cache_dir = "/home/ubuntu/mimiciv-data/pyhealth_cache_labs_notes" +logs_dir = "/home/ubuntu/logs" +output_dir = "/home/ubuntu/output" +embedding_dim = 128 +hidden_dim = 128 +mlp_layers = 2 +mlp_activation = "relu" +dropout = 0.1 +use_amp = True +amp_dtype = "bf16" +epochs = 50 +batch_size = 32 +lr = 1e-4 +patience = 5 +num_workers = 4 +freeze_encoder = True +max_frozen_text_cache = 1000000 +dev = False +use_old_cache = False +use_wandb = True +wandb_project = f"pyhealth-multimodal-labs-notes-seed-{seed}" +wandb_run_name = None # defaults to "{model}_seed{seed}" if unset +cuda_visible_devices = "0" +session_name = f"mlp_labs_notes_s{seed}" + +# ── Step 0a: Check which CUDA GPU is available ─────────────────────────────── +print(f""" +### STEP 0a: Check which CUDA GPU is available (pick an idle index for CUDA_VISIBLE_DEVICES) + +nvidia-smi --query-gpu=index,utilization.gpu,memory.used,memory.free --format=csv +""") + +# ── Step 0b: Clean logs and cache ──────────────────────────────────────────── +print(f""" +### STEP 0b (optional): Clean logs and cache + +rm -rf {logs_dir}/* +{'# cache preserved (use_old_cache=True)' if use_old_cache else f'rm -rf {cache_dir}/*'} +rm -rf {output_dir}/* +""") + +# ── Step 1: Start a tmux session ──────────────────────────────────────────── +print(f""" +### STEP 1: Start a tmux session (attached) + +tmux new-session -s {session_name} +""") + +# ── Step 2: Paste this into the session to run training ───────────────────── +print("\n" + "=" * 60 + "\n") + +log_dir = logs_dir +log_tag = f"mlp_labs_notes_s{seed}" + +flags = [ + f"--ehr-root {ehr_root}", + f"--note-root {note_root}", + f"--cache-dir {cache_dir}", + f"--task notes_labs{' --dev' if dev else ''}", + "--model mlp", + f"--embedding-dim {embedding_dim}", + f"--hidden-dim {hidden_dim}", + f"--mlp-layers {mlp_layers}", + f"--mlp-activation {mlp_activation}", + f"--dropout {dropout}", +] +if use_amp: + flags.append("--use-amp") + flags.append(f"--amp-dtype {amp_dtype}") +flags += [ + f"--epochs {epochs}", + f"--batch-size {batch_size}", + f"--lr {lr}", + f"--patience {patience}", + f"--num-workers {num_workers}", +] +if freeze_encoder: + flags.append("--freeze-encoder") + flags.append(f"--max-frozen-text-cache {max_frozen_text_cache}") +if use_wandb: + flags.append("--wandb") + flags.append(f"--wandb-project {wandb_project}") + if wandb_run_name: + flags.append(f"--wandb-run-name {wandb_run_name}") +flags += [ + f"--seed {seed}", + f"--output-dir {output_dir}", +] +flags_block = " \\\n ".join(flags) + +print(f""" +### STEP 2: Paste this into the tmux session + +mkdir -p {log_dir} && +eval "$(conda shell.bash hook)" && +conda activate {conda_env} && +cd {project_dir} && +export PYTHONPATH={project_dir}:$PYTHONPATH && +export CUDA_VISIBLE_DEVICES={cuda_visible_devices} && +python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \\ + {flags_block} \\ + > >(tee {log_dir}/{log_tag}.out) \\ + 2> >(tee {log_dir}/{log_tag}.err >&2) +""") + +# ── Step 3: Detach / reattach / monitor ────────────────────────────────────── +print(f""" +### STEP 3: Detach without killing (Ctrl+b d), then reattach later with + +tmux attach -t {session_name} + +### To check on it later without attaching: + +tail -f {log_dir}/{log_tag}.out + +### To kill the session when done: + +tmux kill-session -t {session_name} +""") + +# ── Step 4: Confirm which note sections were used for filtering ───────────── +print(f""" +### STEP 4: Confirm which discharge-note AND radiology-note sections +### NotesLabsMIMIC4 filtered to (e.g. discharge: "chief complaint"; +### radiology: "indication", "impression"). Logged once at INFO level by +### pyhealth.tasks.multimodal_mimic4 when the task is constructed. + +grep "filtering discharge notes to sections" {log_dir}/{log_tag}.out +""") \ No newline at end of file diff --git a/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_rnn_variant.py b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_rnn_variant.py index 047970f73..f0ec7574d 100644 --- a/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_rnn_variant.py +++ b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_rnn_variant.py @@ -1,5 +1,5 @@ project_dir = "PyHealth" -seed = 12 +seed = 2 conda_env = "pyhealth2" ehr_root = "/home/ubuntu/mimiciv-data/ehr" note_root = "/home/ubuntu/mimiciv-data" @@ -16,14 +16,13 @@ epochs = 50 batch_size = 32 lr = 1e-4 -weight_decay = 1e-5 patience = 5 num_workers = 4 freeze_encoder = True dev = False use_old_cache = False use_wandb = True -wandb_project = "pyhealth-multimodal-labs-notes" +wandb_project = f"pyhealth-multimodal-labs-notes-seed-{seed}" wandb_run_name = None # defaults to "{model}_seed{seed}" if unset cuda_visible_devices = "0" session_name = f"rnn_labs_notes_s{seed}" @@ -76,7 +75,6 @@ f"--epochs {epochs}", f"--batch-size {batch_size}", f"--lr {lr}", - f"--weight-decay {weight_decay}", f"--patience {patience}", f"--num-workers {num_workers}", ] diff --git a/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_transformer_variant.py b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_transformer_variant.py index bceb635e6..672fe86c8 100644 --- a/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_transformer_variant.py +++ b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_transformer_variant.py @@ -1,5 +1,5 @@ project_dir = "PyHealth" -seed = 12 +seed = 2 conda_env = "pyhealth2" ehr_root = "/home/ubuntu/mimiciv-data/ehr" note_root = "/home/ubuntu/mimiciv-data" @@ -16,14 +16,13 @@ epochs = 50 batch_size = 32 lr = 1e-4 -weight_decay = 1e-5 patience = 5 num_workers = 4 freeze_encoder = True dev = False use_old_cache = False use_wandb = True -wandb_project = "pyhealth-multimodal-labs-notes" +wandb_project = f"pyhealth-multimodal-labs-notes-seed-{seed}" wandb_run_name = None # defaults to "{model}_seed{seed}" if unset cuda_visible_devices = "0" session_name = f"transformer_labs_notes_s{seed}" @@ -76,7 +75,6 @@ f"--epochs {epochs}", f"--batch-size {batch_size}", f"--lr {lr}", - f"--weight-decay {weight_decay}", f"--patience {patience}", f"--num-workers {num_workers}", ] From ef7430c0008cf06be5bc86be0fbe956b7b1e9f71 Mon Sep 17 00:00:00 2001 From: William Pang Date: Wed, 2 Sep 2026 04:04:22 +0000 Subject: [PATCH 51/61] Update pyhealth environment file and will run scripts --- pyhealth2_environment.yml | 1 + .../tmux_run_labs_notes_bottleneck_transformer_variant.py | 4 ++-- .../labs_notes/tmux_run_labs_notes_jambaehr_variant.py | 2 +- .../lambda_labs/labs_notes/tmux_run_labs_notes_mlp_variant.py | 2 -- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/pyhealth2_environment.yml b/pyhealth2_environment.yml index 82e2a3fb7..9e8ab5154 100644 --- a/pyhealth2_environment.yml +++ b/pyhealth2_environment.yml @@ -161,6 +161,7 @@ dependencies: - terminado=0.18.1=pyhc90fa1f_1 - tinycss2=1.4.0=pyhd8ed1ab_0 - tk=8.6.15=h54e0aa7_0 + - tmux - tomli=2.4.1=pyhcf101f3_0 - traitlets=5.15.1=pyhcf101f3_0 - typing-extensions=4.15.0=h396c80c_0 diff --git a/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_bottleneck_transformer_variant.py b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_bottleneck_transformer_variant.py index c50d92de8..192e9f097 100644 --- a/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_bottleneck_transformer_variant.py +++ b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_bottleneck_transformer_variant.py @@ -1,5 +1,5 @@ project_dir = "PyHealth" -seed = 12 +seed = 2 conda_env = "pyhealth2" ehr_root = "/home/ubuntu/mimiciv-data/ehr" note_root = "/home/ubuntu/mimiciv-data" @@ -24,7 +24,7 @@ dev = False use_old_cache = False use_wandb = True -wandb_project = "pyhealth-multimodal-labs-notes" +wandb_project = f"pyhealth-multimodal-labs-notes-seed-{seed}" wandb_run_name = None # defaults to "{model}_seed{seed}" if unset cuda_visible_devices = "0" session_name = f"bottleneck_transformer_labs_notes_s{seed}" diff --git a/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_jambaehr_variant.py b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_jambaehr_variant.py index d07cb477b..df2aa7cee 100644 --- a/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_jambaehr_variant.py +++ b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_jambaehr_variant.py @@ -29,7 +29,7 @@ wandb_project = f"pyhealth-multimodal-labs-notes-seed-{seed}" wandb_run_name = None # defaults to "{model}_seed{seed}" if unset cuda_visible_devices = "0" -session_name = f"jambaehr_labs_notes_s{seed}_batchsize{batch_size}" +session_name = f"jambaehr_labs_notes_s{seed}" # ── Step 0a: Check which CUDA GPU is available ─────────────────────────────── print(f""" diff --git a/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_mlp_variant.py b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_mlp_variant.py index 09dafec26..b9e91df5e 100644 --- a/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_mlp_variant.py +++ b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_mlp_variant.py @@ -19,7 +19,6 @@ patience = 5 num_workers = 4 freeze_encoder = True -max_frozen_text_cache = 1000000 dev = False use_old_cache = False use_wandb = True @@ -81,7 +80,6 @@ ] if freeze_encoder: flags.append("--freeze-encoder") - flags.append(f"--max-frozen-text-cache {max_frozen_text_cache}") if use_wandb: flags.append("--wandb") flags.append(f"--wandb-project {wandb_project}") From f8fefc0577a9a47284e54ff80490f5870892ae80 Mon Sep 17 00:00:00 2001 From: William Pang Date: Wed, 2 Sep 2026 15:16:27 +0000 Subject: [PATCH 52/61] Avoid ambiguous -1 reshape when a batch has no note or image slots. Replace the trailing -1 in .view()/.reshape() calls for the text and image branches with the known embedding dim (or an explicit empty mask), since torch can't infer -1 for a 0-element tensor and raises "cannot reshape tensor of 0 elements ... dimension size -1 is ambiguous" when a batch has zero note or image slots. --- pyhealth/models/embedding/unified.py | 29 +++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/pyhealth/models/embedding/unified.py b/pyhealth/models/embedding/unified.py index 331d174b7..22368698b 100644 --- a/pyhealth/models/embedding/unified.py +++ b/pyhealth/models/embedding/unified.py @@ -745,25 +745,40 @@ def forward( cls_emb[valid] = h if field_name in self.projections: cls_emb = self.projections[field_name](cls_emb) - emb = cls_emb.view(b, n, -1) # (B, N, E') + # cls_emb.shape[-1] instead of -1: when b*n == 0 (no note + # slots in this batch), the -1 is ambiguous for a 0-element + # tensor and torch raises "cannot reshape tensor of 0 + # elements ... because the unspecified dimension size -1 + # can be any value and is ambiguous". + emb = cls_emb.view(b, n, cls_emb.shape[-1]) # (B, N, E') elif modality == ModalityType.IMAGE: # encoder = Sequential(PatchEmbedding, _MeanPool) → (B*N, E') b, n, c, h, w = value.shape flat_imgs = value.reshape(b * n, c, h, w) - if pad_mask is not None: + if b * n == 0: + # No image slots in this batch (e.g. no sample has a CXR + # image). flat_imgs.reshape(0, -1) is ambiguous since -1 + # can be any value when the tensor already has 0 elements. + valid = torch.zeros(b * n, dtype=torch.bool, device=value.device) + elif pad_mask is not None: valid = pad_mask.reshape(b * n).bool() else: valid = flat_imgs.reshape(b * n, -1).abs().sum(dim=-1) > 0 if valid.any(): img_valid = encoder(flat_imgs[valid]) - img_emb = img_valid.new_zeros( - (b * n, img_valid.shape[-1]) - ) + emb_dim = img_valid.shape[-1] + img_emb = img_valid.new_zeros((b * n, emb_dim)) img_emb[valid] = img_valid else: - img_emb = value.new_zeros((b * n, self._embedding_dim)) - emb = img_emb.view(b, n, -1) # (B, N, E') + emb_dim = self._embedding_dim + img_emb = value.new_zeros((b * n, emb_dim)) + # emb_dim instead of -1: when b*n == 0 (no image slots in + # this batch), the -1 is ambiguous for a 0-element tensor + # and torch raises "cannot reshape tensor of 0 elements + # ... because the unspecified dimension size -1 can be any + # value and is ambiguous". + emb = img_emb.view(b, n, emb_dim) # (B, N, E') else: # NUMERIC / SIGNAL # Standardise BEFORE the projection. The projection mixes the From fc7050a3671a9ad4641642b441f9d6181b2e4e54 Mon Sep 17 00:00:00 2001 From: William Pang Date: Thu, 3 Sep 2026 13:29:08 +0000 Subject: [PATCH 53/61] Create Will's CXR Labs Notes variant scripts --- ...otes_cxr_bottleneck_transformer_variant.py | 142 +++++++++++++++++ ...mux_run_labs_notes_cxr_ehrmamba_variant.py | 140 +++++++++++++++++ ...mux_run_labs_notes_cxr_jambaehr_variant.py | 146 ++++++++++++++++++ .../tmux_run_labs_notes_cxr_mlp_variant.py | 138 +++++++++++++++++ .../tmux_run_labs_notes_cxr_rnn_variant.py | 138 +++++++++++++++++ ..._run_labs_notes_cxr_transformer_variant.py | 138 +++++++++++++++++ 6 files changed, 842 insertions(+) create mode 100644 scripts/will/lambda_labs/labs_notes_cxr/tmux_run_labs_notes_cxr_bottleneck_transformer_variant.py create mode 100644 scripts/will/lambda_labs/labs_notes_cxr/tmux_run_labs_notes_cxr_ehrmamba_variant.py create mode 100644 scripts/will/lambda_labs/labs_notes_cxr/tmux_run_labs_notes_cxr_jambaehr_variant.py create mode 100644 scripts/will/lambda_labs/labs_notes_cxr/tmux_run_labs_notes_cxr_mlp_variant.py create mode 100644 scripts/will/lambda_labs/labs_notes_cxr/tmux_run_labs_notes_cxr_rnn_variant.py create mode 100644 scripts/will/lambda_labs/labs_notes_cxr/tmux_run_labs_notes_cxr_transformer_variant.py diff --git a/scripts/will/lambda_labs/labs_notes_cxr/tmux_run_labs_notes_cxr_bottleneck_transformer_variant.py b/scripts/will/lambda_labs/labs_notes_cxr/tmux_run_labs_notes_cxr_bottleneck_transformer_variant.py new file mode 100644 index 000000000..8bbcee6aa --- /dev/null +++ b/scripts/will/lambda_labs/labs_notes_cxr/tmux_run_labs_notes_cxr_bottleneck_transformer_variant.py @@ -0,0 +1,142 @@ +project_dir = "PyHealth" +seed = 2 +conda_env = "pyhealth2" +ehr_root = "/home/ubuntu/mimiciv-data/ehr" +note_root = "/home/ubuntu/mimiciv-data" +cxr_root = "/home/ubuntu/mimiciv-data/CXR-jpg" +cxr_variant = "default" +cache_dir = "/home/ubuntu/mimiciv-data/pyhealth_cache_labs_notes_cxr" +logs_dir = "/home/ubuntu/logs" +output_dir = "/home/ubuntu/output" +embedding_dim = 128 +heads = 4 +num_layers = 2 +bottlenecks_n = 4 +fusion_startidx = 1 +dropout = 0.1 +use_amp = True +amp_dtype = "bf16" +epochs = 50 +batch_size = 32 +lr = 1e-4 +weight_decay = 1e-5 +patience = 5 +num_workers = 4 +freeze_encoder = True +dev = False +use_old_cache = False +use_wandb = True +wandb_project = f"pyhealth-multimodal-labs-notes-cxr-seed-{seed}" +wandb_run_name = None # defaults to "{model}_seed{seed}" if unset +cuda_visible_devices = "0" +session_name = f"bottleneck_transformer_labs_notes_cxr_s{seed}" + +# ── Step 0a: Check which CUDA GPU is available ─────────────────────────────── +print(f""" +### STEP 0a: Check which CUDA GPU is available (pick an idle index for CUDA_VISIBLE_DEVICES) + +nvidia-smi --query-gpu=index,utilization.gpu,memory.used,memory.free --format=csv +""") + +# ── Step 0b: Clean logs and cache ──────────────────────────────────────────── +print(f""" +### STEP 0b (optional): Clean logs and cache + +rm -rf {logs_dir}/* +{'# cache preserved (use_old_cache=True)' if use_old_cache else f'rm -rf {cache_dir}/*'} +rm -rf {output_dir}/* +""") + +# ── Step 1: Start a tmux session ──────────────────────────────────────────── +print(f""" +### STEP 1: Start a tmux session (attached) + +tmux new-session -s {session_name} +""") + +# ── Step 2: Paste this into the session to run training ───────────────────── +print("\n" + "=" * 60 + "\n") + +log_dir = logs_dir +log_tag = f"bottleneck_transformer_labs_notes_cxr_s{seed}" + +flags = [ + f"--ehr-root {ehr_root}", + f"--note-root {note_root}", + f"--cxr-root {cxr_root}", + f"--cxr-variant {cxr_variant}", + f"--cache-dir {cache_dir}", + f"--task notes_labs_cxr{' --dev' if dev else ''}", + "--model bottleneck_transformer", + f"--embedding-dim {embedding_dim}", + f"--heads {heads}", + f"--num-layers {num_layers}", + f"--bottlenecks-n {bottlenecks_n}", + f"--fusion-startidx {fusion_startidx}", + f"--dropout {dropout}", +] +if use_amp: + flags.append("--use-amp") + flags.append(f"--amp-dtype {amp_dtype}") +flags += [ + f"--epochs {epochs}", + f"--batch-size {batch_size}", + f"--lr {lr}", + f"--weight-decay {weight_decay}", + f"--patience {patience}", + f"--num-workers {num_workers}", +] +if freeze_encoder: + flags.append("--freeze-encoder") +if use_wandb: + flags.append("--wandb") + flags.append(f"--wandb-project {wandb_project}") + if wandb_run_name: + flags.append(f"--wandb-run-name {wandb_run_name}") +flags += [ + f"--seed {seed}", + f"--output-dir {output_dir}", +] +flags_block = " \\\n ".join(flags) + +print(f""" +### STEP 2: Paste this into the tmux session + +mkdir -p {log_dir} && +eval "$(conda shell.bash hook)" && +conda activate {conda_env} && +cd {project_dir} && +export PYTHONPATH={project_dir}:$PYTHONPATH && +export CUDA_VISIBLE_DEVICES={cuda_visible_devices} && +python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \\ + {flags_block} \\ + > >(tee {log_dir}/{log_tag}.out) \\ + 2> >(tee {log_dir}/{log_tag}.err >&2) +""") + +# ── Step 3: Detach / reattach / monitor ────────────────────────────────────── +print(f""" +### STEP 3: Detach without killing (Ctrl+b d), then reattach later with + +tmux attach -t {session_name} + +### To check on it later without attaching: + +tail -f {log_dir}/{log_tag}.out + +### To kill the session when done: + +tmux kill-session -t {session_name} +""") + +# ── Step 4: Confirm which note sections + CXR variant were used ───────────── +print(f""" +### STEP 4: Confirm which discharge-note AND radiology-note sections +### NotesLabsCXRMIMIC4 filtered to (e.g. discharge: "chief complaint"; +### radiology: "indication", "impression"), and which CXR variant was loaded. +### Both logged once at INFO level by pyhealth.tasks.multimodal_mimic4 / +### pyhealth.datasets.MIMIC4Dataset when the task/dataset are constructed. + +grep "filtering discharge notes to sections" {log_dir}/{log_tag}.out +grep -i "cxr" {log_dir}/{log_tag}.out | head -20 +""") diff --git a/scripts/will/lambda_labs/labs_notes_cxr/tmux_run_labs_notes_cxr_ehrmamba_variant.py b/scripts/will/lambda_labs/labs_notes_cxr/tmux_run_labs_notes_cxr_ehrmamba_variant.py new file mode 100644 index 000000000..ddd90a5c1 --- /dev/null +++ b/scripts/will/lambda_labs/labs_notes_cxr/tmux_run_labs_notes_cxr_ehrmamba_variant.py @@ -0,0 +1,140 @@ +project_dir = "PyHealth" +seed = 2 +conda_env = "pyhealth2" +ehr_root = "/home/ubuntu/mimiciv-data/ehr" +note_root = "/home/ubuntu/mimiciv-data" +cxr_root = "/home/ubuntu/mimiciv-data/CXR-jpg" +cxr_variant = "default" +cache_dir = "/home/ubuntu/mimiciv-data/pyhealth_cache_labs_notes_cxr" +logs_dir = "/home/ubuntu/logs" +output_dir = "/home/ubuntu/output" +embedding_dim = 128 +hidden_dim = 128 +num_layers = 2 +mamba_state_size = 16 +mamba_conv_kernel = 4 +dropout = 0.1 +use_amp = True +amp_dtype = "bf16" +epochs = 50 +batch_size = 32 +lr = 1e-4 +patience = 5 +num_workers = 4 +freeze_encoder = True +dev = False +use_old_cache = False +use_wandb = True +wandb_project = f"pyhealth-multimodal-labs-notes-cxr-seed-{seed}" +wandb_run_name = None # defaults to "{model}_seed{seed}" if unset +cuda_visible_devices = "0" +session_name = f"ehrmamba_labs_notes_cxr_s{seed}" + +# ── Step 0a: Check which CUDA GPU is available ─────────────────────────────── +print(f""" +### STEP 0a: Check which CUDA GPU is available (pick an idle index for CUDA_VISIBLE_DEVICES) + +nvidia-smi --query-gpu=index,utilization.gpu,memory.used,memory.free --format=csv +""") + +# ── Step 0b: Clean logs and cache ──────────────────────────────────────────── +print(f""" +### STEP 0b (optional): Clean logs and cache + +rm -rf {logs_dir}/* +{'# cache preserved (use_old_cache=True)' if use_old_cache else f'rm -rf {cache_dir}/*'} +rm -rf {output_dir}/* +""") + +# ── Step 1: Start a tmux session ──────────────────────────────────────────── +print(f""" +### STEP 1: Start a tmux session (attached) + +tmux new-session -s {session_name} +""") + +# ── Step 2: Paste this into the session to run training ───────────────────── +print("\n" + "=" * 60 + "\n") + +log_dir = logs_dir +log_tag = f"ehrmamba_labs_notes_cxr_s{seed}" + +flags = [ + f"--ehr-root {ehr_root}", + f"--note-root {note_root}", + f"--cxr-root {cxr_root}", + f"--cxr-variant {cxr_variant}", + f"--cache-dir {cache_dir}", + f"--task notes_labs_cxr{' --dev' if dev else ''}", + "--model ehrmamba", + f"--embedding-dim {embedding_dim}", + f"--hidden-dim {hidden_dim}", + f"--num-layers {num_layers}", + f"--mamba-state-size {mamba_state_size}", + f"--mamba-conv-kernel {mamba_conv_kernel}", + f"--dropout {dropout}", +] +if use_amp: + flags.append("--use-amp") + flags.append(f"--amp-dtype {amp_dtype}") +flags += [ + f"--epochs {epochs}", + f"--batch-size {batch_size}", + f"--lr {lr}", + f"--patience {patience}", + f"--num-workers {num_workers}", +] +if freeze_encoder: + flags.append("--freeze-encoder") +if use_wandb: + flags.append("--wandb") + flags.append(f"--wandb-project {wandb_project}") + if wandb_run_name: + flags.append(f"--wandb-run-name {wandb_run_name}") +flags += [ + f"--seed {seed}", + f"--output-dir {output_dir}", +] +flags_block = " \\\n ".join(flags) + +print(f""" +### STEP 2: Paste this into the tmux session + +mkdir -p {log_dir} && +eval "$(conda shell.bash hook)" && +conda activate {conda_env} && +cd {project_dir} && +export PYTHONPATH={project_dir}:$PYTHONPATH && +export CUDA_VISIBLE_DEVICES={cuda_visible_devices} && +python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \\ + {flags_block} \\ + > >(tee {log_dir}/{log_tag}.out) \\ + 2> >(tee {log_dir}/{log_tag}.err >&2) +""") + +# ── Step 3: Detach / reattach / monitor ────────────────────────────────────── +print(f""" +### STEP 3: Detach without killing (Ctrl+b d), then reattach later with + +tmux attach -t {session_name} + +### To check on it later without attaching: + +tail -f {log_dir}/{log_tag}.out + +### To kill the session when done: + +tmux kill-session -t {session_name} +""") + +# ── Step 4: Confirm which note sections + CXR variant were used ───────────── +print(f""" +### STEP 4: Confirm which discharge-note AND radiology-note sections +### NotesLabsCXRMIMIC4 filtered to (e.g. discharge: "chief complaint"; +### radiology: "indication", "impression"), and which CXR variant was loaded. +### Both logged once at INFO level by pyhealth.tasks.multimodal_mimic4 / +### pyhealth.datasets.MIMIC4Dataset when the task/dataset are constructed. + +grep "filtering discharge notes to sections" {log_dir}/{log_tag}.out +grep -i "cxr" {log_dir}/{log_tag}.out | head -20 +""") diff --git a/scripts/will/lambda_labs/labs_notes_cxr/tmux_run_labs_notes_cxr_jambaehr_variant.py b/scripts/will/lambda_labs/labs_notes_cxr/tmux_run_labs_notes_cxr_jambaehr_variant.py new file mode 100644 index 000000000..5e20dab40 --- /dev/null +++ b/scripts/will/lambda_labs/labs_notes_cxr/tmux_run_labs_notes_cxr_jambaehr_variant.py @@ -0,0 +1,146 @@ +project_dir = "PyHealth" +seed = 2 +conda_env = "pyhealth2" +ehr_root = "/home/ubuntu/mimiciv-data/ehr" +note_root = "/home/ubuntu/mimiciv-data" +cxr_root = "/home/ubuntu/mimiciv-data/CXR-jpg" +cxr_variant = "default" +cache_dir = "/home/ubuntu/mimiciv-data/pyhealth_cache_labs_notes_cxr" +logs_dir = "/home/ubuntu/logs" +output_dir = "/home/ubuntu/output" +embedding_dim = 128 +hidden_dim = 128 +heads = 4 +num_layers = 2 +jamba_transformer_layers = 2 +jamba_mamba_layers = 6 +mamba_state_size = 16 +mamba_conv_kernel = 4 +dropout = 0.1 +use_amp = True +amp_dtype = "bf16" +epochs = 50 +batch_size = 32 +lr = 1e-4 +patience = 5 +num_workers = 4 +freeze_encoder = True +dev = False +use_old_cache = False +use_wandb = True +wandb_project = f"pyhealth-multimodal-labs-notes-cxr-seed-{seed}" +wandb_run_name = None # defaults to "{model}_seed{seed}" if unset +cuda_visible_devices = "0" +session_name = f"jambaehr_labs_notes_cxr_s{seed}" + +# ── Step 0a: Check which CUDA GPU is available ─────────────────────────────── +print(f""" +### STEP 0a: Check which CUDA GPU is available (pick an idle index for CUDA_VISIBLE_DEVICES) + +nvidia-smi --query-gpu=index,utilization.gpu,memory.free --format=csv +""") + +# ── Step 0b: Clean logs and cache ──────────────────────────────────────────── +print(f""" +### STEP 0b (optional): Clean logs and cache + +rm -rf {logs_dir}/* +{'# cache preserved (use_old_cache=True)' if use_old_cache else f'rm -rf {cache_dir}/*'} +rm -rf {output_dir}/* +""") + +# ── Step 1: Start a tmux session ──────────────────────────────────────────── +print(f""" +### STEP 1: Start a tmux session (attached) + +tmux new-session -s {session_name} +""") + +# ── Step 2: Paste this into the session to run training ───────────────────── +print("\n" + "=" * 60 + "\n") + +log_dir = logs_dir +log_tag = f"jambaehr_labs_notes_cxr_s{seed}" + +flags = [ + f"--ehr-root {ehr_root}", + f"--note-root {note_root}", + f"--cxr-root {cxr_root}", + f"--cxr-variant {cxr_variant}", + f"--cache-dir {cache_dir}", + f"--task notes_labs_cxr{' --dev' if dev else ''}", + "--model jambaehr", + f"--embedding-dim {embedding_dim}", + f"--hidden-dim {hidden_dim}", + f"--heads {heads}", + f"--num-layers {num_layers}", + f"--jamba-transformer-layers {jamba_transformer_layers}", + f"--jamba-mamba-layers {jamba_mamba_layers}", + f"--mamba-state-size {mamba_state_size}", + f"--mamba-conv-kernel {mamba_conv_kernel}", + f"--dropout {dropout}", +] +if use_amp: + flags.append("--use-amp") + flags.append(f"--amp-dtype {amp_dtype}") +flags += [ + f"--epochs {epochs}", + f"--batch-size {batch_size}", + f"--lr {lr}", + f"--patience {patience}", + f"--num-workers {num_workers}", +] +if freeze_encoder: + flags.append("--freeze-encoder") +if use_wandb: + flags.append("--wandb") + flags.append(f"--wandb-project {wandb_project}") + if wandb_run_name: + flags.append(f"--wandb-run-name {wandb_run_name}") +flags += [ + f"--seed {seed}", + f"--output-dir {output_dir}", +] +flags_block = " \\\n ".join(flags) + +print(f""" +### STEP 2: Paste this into the tmux session + +mkdir -p {log_dir} && +eval "$(conda shell.bash hook)" && +conda activate {conda_env} && +cd {project_dir} && +export PYTHONPATH={project_dir}:$PYTHONPATH && +export CUDA_VISIBLE_DEVICES={cuda_visible_devices} && +python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \\ + {flags_block} \\ + > >(tee {log_dir}/{log_tag}.out) \\ + 2> >(tee {log_dir}/{log_tag}.err >&2) +""") + +# ── Step 3: Detach / reattach / monitor ────────────────────────────────────── +print(f""" +### STEP 3: Detach without killing (Ctrl+b d), then reattach later with + +tmux attach -t {session_name} + +### To check on it later without attaching: + +tail -f {log_dir}/{log_tag}.out + +### To kill the session when done: + +tmux kill-session -t {session_name} +""") + +# ── Step 4: Confirm which note sections + CXR variant were used ───────────── +print(f""" +### STEP 4: Confirm which discharge-note AND radiology-note sections +### NotesLabsCXRMIMIC4 filtered to (e.g. discharge: "chief complaint"; +### radiology: "indication", "impression"), and which CXR variant was loaded. +### Both logged once at INFO level by pyhealth.tasks.multimodal_mimic4 / +### pyhealth.datasets.MIMIC4Dataset when the task/dataset are constructed. + +grep "filtering discharge notes to sections" {log_dir}/{log_tag}.out +grep -i "cxr" {log_dir}/{log_tag}.out | head -20 +""") diff --git a/scripts/will/lambda_labs/labs_notes_cxr/tmux_run_labs_notes_cxr_mlp_variant.py b/scripts/will/lambda_labs/labs_notes_cxr/tmux_run_labs_notes_cxr_mlp_variant.py new file mode 100644 index 000000000..1a857b949 --- /dev/null +++ b/scripts/will/lambda_labs/labs_notes_cxr/tmux_run_labs_notes_cxr_mlp_variant.py @@ -0,0 +1,138 @@ +project_dir = "PyHealth" +seed = 2 +conda_env = "pyhealth2" +ehr_root = "/home/ubuntu/mimiciv-data/ehr" +note_root = "/home/ubuntu/mimiciv-data" +cxr_root = "/home/ubuntu/mimiciv-data/CXR-jpg" +cxr_variant = "default" +cache_dir = "/home/ubuntu/mimiciv-data/pyhealth_cache_labs_notes_cxr" +logs_dir = "/home/ubuntu/logs" +output_dir = "/home/ubuntu/output" +embedding_dim = 128 +hidden_dim = 128 +mlp_layers = 2 +mlp_activation = "relu" +dropout = 0.1 +use_amp = True +amp_dtype = "bf16" +epochs = 50 +batch_size = 32 +lr = 1e-4 +patience = 5 +num_workers = 4 +freeze_encoder = True +dev = False +use_old_cache = False +use_wandb = True +wandb_project = f"pyhealth-multimodal-labs-notes-cxr-seed-{seed}" +wandb_run_name = None # defaults to "{model}_seed{seed}" if unset +cuda_visible_devices = "0" +session_name = f"mlp_labs_notes_cxr_s{seed}" + +# ── Step 0a: Check which CUDA GPU is available ─────────────────────────────── +print(f""" +### STEP 0a: Check which CUDA GPU is available (pick an idle index for CUDA_VISIBLE_DEVICES) + +nvidia-smi --query-gpu=index,utilization.gpu,memory.used,memory.free --format=csv +""") + +# ── Step 0b: Clean logs and cache ──────────────────────────────────────────── +print(f""" +### STEP 0b (optional): Clean logs and cache + +rm -rf {logs_dir}/* +{'# cache preserved (use_old_cache=True)' if use_old_cache else f'rm -rf {cache_dir}/*'} +rm -rf {output_dir}/* +""") + +# ── Step 1: Start a tmux session ──────────────────────────────────────────── +print(f""" +### STEP 1: Start a tmux session (attached) + +tmux new-session -s {session_name} +""") + +# ── Step 2: Paste this into the session to run training ───────────────────── +print("\n" + "=" * 60 + "\n") + +log_dir = logs_dir +log_tag = f"mlp_labs_notes_cxr_s{seed}" + +flags = [ + f"--ehr-root {ehr_root}", + f"--note-root {note_root}", + f"--cxr-root {cxr_root}", + f"--cxr-variant {cxr_variant}", + f"--cache-dir {cache_dir}", + f"--task notes_labs_cxr{' --dev' if dev else ''}", + "--model mlp", + f"--embedding-dim {embedding_dim}", + f"--hidden-dim {hidden_dim}", + f"--mlp-layers {mlp_layers}", + f"--mlp-activation {mlp_activation}", + f"--dropout {dropout}", +] +if use_amp: + flags.append("--use-amp") + flags.append(f"--amp-dtype {amp_dtype}") +flags += [ + f"--epochs {epochs}", + f"--batch-size {batch_size}", + f"--lr {lr}", + f"--patience {patience}", + f"--num-workers {num_workers}", +] +if freeze_encoder: + flags.append("--freeze-encoder") +if use_wandb: + flags.append("--wandb") + flags.append(f"--wandb-project {wandb_project}") + if wandb_run_name: + flags.append(f"--wandb-run-name {wandb_run_name}") +flags += [ + f"--seed {seed}", + f"--output-dir {output_dir}", +] +flags_block = " \\\n ".join(flags) + +print(f""" +### STEP 2: Paste this into the tmux session + +mkdir -p {log_dir} && +eval "$(conda shell.bash hook)" && +conda activate {conda_env} && +cd {project_dir} && +export PYTHONPATH={project_dir}:$PYTHONPATH && +export CUDA_VISIBLE_DEVICES={cuda_visible_devices} && +python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \\ + {flags_block} \\ + > >(tee {log_dir}/{log_tag}.out) \\ + 2> >(tee {log_dir}/{log_tag}.err >&2) +""") + +# ── Step 3: Detach / reattach / monitor ────────────────────────────────────── +print(f""" +### STEP 3: Detach without killing (Ctrl+b d), then reattach later with + +tmux attach -t {session_name} + +### To check on it later without attaching: + +tail -f {log_dir}/{log_tag}.out + +### To kill the session when done: + +tmux kill-session -t {session_name} +""") + +# ── Step 4: Confirm which note sections + CXR variant were used ───────────── +print(f""" +### STEP 4: Confirm which discharge-note AND radiology-note sections +### NotesLabsCXRMIMIC4 filtered to (e.g. discharge: "chief complaint"; +### radiology: "indication", "impression"), and which CXR variant was loaded. +### Both logged once at INFO level by pyhealth.tasks.multimodal_mimic4 / +### pyhealth.datasets.MIMIC4Dataset when the task/dataset are constructed. + +grep "filtering discharge notes to sections" {log_dir}/{log_tag}.out +grep -i "cxr" {log_dir}/{log_tag}.out | head -20 +""") diff --git a/scripts/will/lambda_labs/labs_notes_cxr/tmux_run_labs_notes_cxr_rnn_variant.py b/scripts/will/lambda_labs/labs_notes_cxr/tmux_run_labs_notes_cxr_rnn_variant.py new file mode 100644 index 000000000..904eddf83 --- /dev/null +++ b/scripts/will/lambda_labs/labs_notes_cxr/tmux_run_labs_notes_cxr_rnn_variant.py @@ -0,0 +1,138 @@ +project_dir = "PyHealth" +seed = 2 +conda_env = "pyhealth2" +ehr_root = "/home/ubuntu/mimiciv-data/ehr" +note_root = "/home/ubuntu/mimiciv-data" +cxr_root = "/home/ubuntu/mimiciv-data/CXR-jpg" +cxr_variant = "default" +cache_dir = "/home/ubuntu/mimiciv-data/pyhealth_cache_labs_notes_cxr" +logs_dir = "/home/ubuntu/logs" +output_dir = "/home/ubuntu/output" +embedding_dim = 128 +hidden_dim = 128 +rnn_type = "GRU" +rnn_layers = 2 +dropout = 0.1 +use_amp = True +amp_dtype = "bf16" +epochs = 50 +batch_size = 32 +lr = 1e-4 +patience = 5 +num_workers = 4 +freeze_encoder = True +dev = False +use_old_cache = False +use_wandb = True +wandb_project = f"pyhealth-multimodal-labs-notes-cxr-seed-{seed}" +wandb_run_name = None # defaults to "{model}_seed{seed}" if unset +cuda_visible_devices = "0" +session_name = f"rnn_labs_notes_cxr_s{seed}" + +# ── Step 0a: Check which CUDA GPU is available ─────────────────────────────── +print(f""" +### STEP 0a: Check which CUDA GPU is available (pick an idle index for CUDA_VISIBLE_DEVICES) + +nvidia-smi --query-gpu=index,utilization.gpu,memory.used,memory.free --format=csv +""") + +# ── Step 0b: Clean logs and cache ──────────────────────────────────────────── +print(f""" +### STEP 0b (optional): Clean logs and cache + +rm -rf {logs_dir}/* +{'# cache preserved (use_old_cache=True)' if use_old_cache else f'rm -rf {cache_dir}/*'} +rm -rf {output_dir}/* +""") + +# ── Step 1: Start a tmux session ──────────────────────────────────────────── +print(f""" +### STEP 1: Start a tmux session (attached) + +tmux new-session -s {session_name} +""") + +# ── Step 2: Paste this into the session to run training ───────────────────── +print("\n" + "=" * 60 + "\n") + +log_dir = logs_dir +log_tag = f"rnn_labs_notes_cxr_s{seed}" + +flags = [ + f"--ehr-root {ehr_root}", + f"--note-root {note_root}", + f"--cxr-root {cxr_root}", + f"--cxr-variant {cxr_variant}", + f"--cache-dir {cache_dir}", + f"--task notes_labs_cxr{' --dev' if dev else ''}", + "--model rnn", + f"--embedding-dim {embedding_dim}", + f"--hidden-dim {hidden_dim}", + f"--rnn-type {rnn_type}", + f"--rnn-layers {rnn_layers}", + f"--dropout {dropout}", +] +if use_amp: + flags.append("--use-amp") + flags.append(f"--amp-dtype {amp_dtype}") +flags += [ + f"--epochs {epochs}", + f"--batch-size {batch_size}", + f"--lr {lr}", + f"--patience {patience}", + f"--num-workers {num_workers}", +] +if freeze_encoder: + flags.append("--freeze-encoder") +if use_wandb: + flags.append("--wandb") + flags.append(f"--wandb-project {wandb_project}") + if wandb_run_name: + flags.append(f"--wandb-run-name {wandb_run_name}") +flags += [ + f"--seed {seed}", + f"--output-dir {output_dir}", +] +flags_block = " \\\n ".join(flags) + +print(f""" +### STEP 2: Paste this into the tmux session + +mkdir -p {log_dir} && +eval "$(conda shell.bash hook)" && +conda activate {conda_env} && +cd {project_dir} && +export PYTHONPATH={project_dir}:$PYTHONPATH && +export CUDA_VISIBLE_DEVICES={cuda_visible_devices} && +python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \\ + {flags_block} \\ + > >(tee {log_dir}/{log_tag}.out) \\ + 2> >(tee {log_dir}/{log_tag}.err >&2) +""") + +# ── Step 3: Detach / reattach / monitor ────────────────────────────────────── +print(f""" +### STEP 3: Detach without killing (Ctrl+b d), then reattach later with + +tmux attach -t {session_name} + +### To check on it later without attaching: + +tail -f {log_dir}/{log_tag}.out + +### To kill the session when done: + +tmux kill-session -t {session_name} +""") + +# ── Step 4: Confirm which note sections + CXR variant were used ───────────── +print(f""" +### STEP 4: Confirm which discharge-note AND radiology-note sections +### NotesLabsCXRMIMIC4 filtered to (e.g. discharge: "chief complaint"; +### radiology: "indication", "impression"), and which CXR variant was loaded. +### Both logged once at INFO level by pyhealth.tasks.multimodal_mimic4 / +### pyhealth.datasets.MIMIC4Dataset when the task/dataset are constructed. + +grep "filtering discharge notes to sections" {log_dir}/{log_tag}.out +grep -i "cxr" {log_dir}/{log_tag}.out | head -20 +""") diff --git a/scripts/will/lambda_labs/labs_notes_cxr/tmux_run_labs_notes_cxr_transformer_variant.py b/scripts/will/lambda_labs/labs_notes_cxr/tmux_run_labs_notes_cxr_transformer_variant.py new file mode 100644 index 000000000..edbb2ba88 --- /dev/null +++ b/scripts/will/lambda_labs/labs_notes_cxr/tmux_run_labs_notes_cxr_transformer_variant.py @@ -0,0 +1,138 @@ +project_dir = "PyHealth" +seed = 2 +conda_env = "pyhealth2" +ehr_root = "/home/ubuntu/mimiciv-data/ehr" +note_root = "/home/ubuntu/mimiciv-data" +cxr_root = "/home/ubuntu/mimiciv-data/CXR-jpg" +cxr_variant = "default" +cache_dir = "/home/ubuntu/mimiciv-data/pyhealth_cache_labs_notes_cxr" +logs_dir = "/home/ubuntu/logs" +output_dir = "/home/ubuntu/output" +embedding_dim = 128 +hidden_dim = 128 +heads = 4 +num_layers = 2 +dropout = 0.1 +use_amp = True +amp_dtype = "bf16" +epochs = 50 +batch_size = 32 +lr = 1e-4 +patience = 5 +num_workers = 4 +freeze_encoder = True +dev = False +use_old_cache = False +use_wandb = True +wandb_project = f"pyhealth-multimodal-labs-notes-cxr-seed-{seed}" +wandb_run_name = None # defaults to "{model}_seed{seed}" if unset +cuda_visible_devices = "0" +session_name = f"transformer_labs_notes_cxr_s{seed}" + +# ── Step 0a: Check which CUDA GPU is available ─────────────────────────────── +print(f""" +### STEP 0a: Check which CUDA GPU is available (pick an idle index for CUDA_VISIBLE_DEVICES) + +nvidia-smi --query-gpu=index,utilization.gpu,memory.used,memory.free --format=csv +""") + +# ── Step 0b: Clean logs and cache ──────────────────────────────────────────── +print(f""" +### STEP 0b (optional): Clean logs and cache + +rm -rf {logs_dir}/* +{'# cache preserved (use_old_cache=True)' if use_old_cache else f'rm -rf {cache_dir}/*'} +rm -rf {output_dir}/* +""") + +# ── Step 1: Start a tmux session ──────────────────────────────────────────── +print(f""" +### STEP 1: Start a tmux session (attached) + +tmux new-session -s {session_name} +""") + +# ── Step 2: Paste this into the session to run training ───────────────────── +print("\n" + "=" * 60 + "\n") + +log_dir = logs_dir +log_tag = f"transformer_labs_notes_cxr_s{seed}" + +flags = [ + f"--ehr-root {ehr_root}", + f"--note-root {note_root}", + f"--cxr-root {cxr_root}", + f"--cxr-variant {cxr_variant}", + f"--cache-dir {cache_dir}", + f"--task notes_labs_cxr{' --dev' if dev else ''}", + "--model transformer", + f"--embedding-dim {embedding_dim}", + f"--hidden-dim {hidden_dim}", + f"--heads {heads}", + f"--num-layers {num_layers}", + f"--dropout {dropout}", +] +if use_amp: + flags.append("--use-amp") + flags.append(f"--amp-dtype {amp_dtype}") +flags += [ + f"--epochs {epochs}", + f"--batch-size {batch_size}", + f"--lr {lr}", + f"--patience {patience}", + f"--num-workers {num_workers}", +] +if freeze_encoder: + flags.append("--freeze-encoder") +if use_wandb: + flags.append("--wandb") + flags.append(f"--wandb-project {wandb_project}") + if wandb_run_name: + flags.append(f"--wandb-run-name {wandb_run_name}") +flags += [ + f"--seed {seed}", + f"--output-dir {output_dir}", +] +flags_block = " \\\n ".join(flags) + +print(f""" +### STEP 2: Paste this into the tmux session + +mkdir -p {log_dir} && +eval "$(conda shell.bash hook)" && +conda activate {conda_env} && +cd {project_dir} && +export PYTHONPATH={project_dir}:$PYTHONPATH && +export CUDA_VISIBLE_DEVICES={cuda_visible_devices} && +python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \\ + {flags_block} \\ + > >(tee {log_dir}/{log_tag}.out) \\ + 2> >(tee {log_dir}/{log_tag}.err >&2) +""") + +# ── Step 3: Detach / reattach / monitor ────────────────────────────────────── +print(f""" +### STEP 3: Detach without killing (Ctrl+b d), then reattach later with + +tmux attach -t {session_name} + +### To check on it later without attaching: + +tail -f {log_dir}/{log_tag}.out + +### To kill the session when done: + +tmux kill-session -t {session_name} +""") + +# ── Step 4: Confirm which note sections + CXR variant were used ───────────── +print(f""" +### STEP 4: Confirm which discharge-note AND radiology-note sections +### NotesLabsCXRMIMIC4 filtered to (e.g. discharge: "chief complaint"; +### radiology: "indication", "impression"), and which CXR variant was loaded. +### Both logged once at INFO level by pyhealth.tasks.multimodal_mimic4 / +### pyhealth.datasets.MIMIC4Dataset when the task/dataset are constructed. + +grep "filtering discharge notes to sections" {log_dir}/{log_tag}.out +grep -i "cxr" {log_dir}/{log_tag}.out | head -20 +""") From de7c37bee83dadd45d16f0e96f654c136f022fdd Mon Sep 17 00:00:00 2001 From: William Pang Date: Fri, 4 Sep 2026 17:59:42 +0000 Subject: [PATCH 54/61] Add labs only scripts --- ...abs_only_bottleneck_transformer_variant.py | 121 +++++++++++++++++ .../tmux_run_labs_only_ehrmamba_variant.py | 119 +++++++++++++++++ .../tmux_run_labs_only_jambaehr_variant.py | 125 ++++++++++++++++++ .../tmux_run_labs_only_mlp_variant.py | 117 ++++++++++++++++ .../tmux_run_labs_only_rnn_variant.py | 117 ++++++++++++++++ .../tmux_run_labs_only_transformer_variant.py | 117 ++++++++++++++++ 6 files changed, 716 insertions(+) create mode 100644 scripts/will/lambda_labs/labs_only/tmux_run_labs_only_bottleneck_transformer_variant.py create mode 100644 scripts/will/lambda_labs/labs_only/tmux_run_labs_only_ehrmamba_variant.py create mode 100644 scripts/will/lambda_labs/labs_only/tmux_run_labs_only_jambaehr_variant.py create mode 100644 scripts/will/lambda_labs/labs_only/tmux_run_labs_only_mlp_variant.py create mode 100644 scripts/will/lambda_labs/labs_only/tmux_run_labs_only_rnn_variant.py create mode 100644 scripts/will/lambda_labs/labs_only/tmux_run_labs_only_transformer_variant.py diff --git a/scripts/will/lambda_labs/labs_only/tmux_run_labs_only_bottleneck_transformer_variant.py b/scripts/will/lambda_labs/labs_only/tmux_run_labs_only_bottleneck_transformer_variant.py new file mode 100644 index 000000000..939199cbd --- /dev/null +++ b/scripts/will/lambda_labs/labs_only/tmux_run_labs_only_bottleneck_transformer_variant.py @@ -0,0 +1,121 @@ +project_dir = "PyHealth" +seed = 2 +conda_env = "pyhealth2" +ehr_root = "/home/ubuntu/mimiciv-data/ehr" +cache_dir = "/home/ubuntu/mimiciv-data/pyhealth_cache_labs" +logs_dir = "/home/ubuntu/logs" +output_dir = "/home/ubuntu/output" +embedding_dim = 128 +heads = 4 +num_layers = 2 +bottlenecks_n = 4 +fusion_startidx = 1 +dropout = 0.1 +use_amp = True +amp_dtype = "bf16" +epochs = 50 +batch_size = 32 +lr = 1e-4 +weight_decay = 1e-5 +patience = 5 +num_workers = 4 +dev = False +use_old_cache = False +use_wandb = True +wandb_project = f"pyhealth-multimodal-labs-seed-{seed}" +wandb_run_name = None # defaults to "{model}_seed{seed}" if unset +cuda_visible_devices = "0" +session_name = f"bottleneck_transformer_labs_s{seed}" + +# ── Step 0a: Check which CUDA GPU is available ─────────────────────────────── +print(f""" +### STEP 0a: Check which CUDA GPU is available (pick an idle index for CUDA_VISIBLE_DEVICES) + +nvidia-smi --query-gpu=index,utilization.gpu,memory.used,memory.free --format=csv +""") + +# ── Step 0b: Clean logs and cache ──────────────────────────────────────────── +print(f""" +### STEP 0b (optional): Clean logs and cache + +rm -rf {logs_dir}/* +{'# cache preserved (use_old_cache=True)' if use_old_cache else f'rm -rf {cache_dir}/*'} +rm -rf {output_dir}/* +""") + +# ── Step 1: Start a tmux session ──────────────────────────────────────────── +print(f""" +### STEP 1: Start a tmux session (attached) + +tmux new-session -s {session_name} +""") + +# ── Step 2: Paste this into the session to run training ───────────────────── +print("\n" + "=" * 60 + "\n") + +log_dir = logs_dir +log_tag = f"bottleneck_transformer_labs_s{seed}" + +flags = [ + f"--ehr-root {ehr_root}", + f"--cache-dir {cache_dir}", + f"--task labs{' --dev' if dev else ''}", + "--model bottleneck_transformer", + f"--embedding-dim {embedding_dim}", + f"--heads {heads}", + f"--num-layers {num_layers}", + f"--bottlenecks-n {bottlenecks_n}", + f"--fusion-startidx {fusion_startidx}", + f"--dropout {dropout}", +] +if use_amp: + flags.append("--use-amp") + flags.append(f"--amp-dtype {amp_dtype}") +flags += [ + f"--epochs {epochs}", + f"--batch-size {batch_size}", + f"--lr {lr}", + f"--weight-decay {weight_decay}", + f"--patience {patience}", + f"--num-workers {num_workers}", +] +if use_wandb: + flags.append("--wandb") + flags.append(f"--wandb-project {wandb_project}") + if wandb_run_name: + flags.append(f"--wandb-run-name {wandb_run_name}") +flags += [ + f"--seed {seed}", + f"--output-dir {output_dir}", +] +flags_block = " \\\n ".join(flags) + +print(f""" +### STEP 2: Paste this into the tmux session + +mkdir -p {log_dir} && +eval "$(conda shell.bash hook)" && +conda activate {conda_env} && +cd {project_dir} && +export PYTHONPATH={project_dir}:$PYTHONPATH && +export CUDA_VISIBLE_DEVICES={cuda_visible_devices} && +python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \\ + {flags_block} \\ + > >(tee {log_dir}/{log_tag}.out) \\ + 2> >(tee {log_dir}/{log_tag}.err >&2) +""") + +# ── Step 3: Detach / reattach / monitor ────────────────────────────────────── +print(f""" +### STEP 3: Detach without killing (Ctrl+b d), then reattach later with + +tmux attach -t {session_name} + +### To check on it later without attaching: + +tail -f {log_dir}/{log_tag}.out + +### To kill the session when done: + +tmux kill-session -t {session_name} +""") diff --git a/scripts/will/lambda_labs/labs_only/tmux_run_labs_only_ehrmamba_variant.py b/scripts/will/lambda_labs/labs_only/tmux_run_labs_only_ehrmamba_variant.py new file mode 100644 index 000000000..a2cbd20fe --- /dev/null +++ b/scripts/will/lambda_labs/labs_only/tmux_run_labs_only_ehrmamba_variant.py @@ -0,0 +1,119 @@ +project_dir = "PyHealth" +seed = 2 +conda_env = "pyhealth2" +ehr_root = "/home/ubuntu/mimiciv-data/ehr" +cache_dir = "/home/ubuntu/mimiciv-data/pyhealth_cache_labs" +logs_dir = "/home/ubuntu/logs" +output_dir = "/home/ubuntu/output" +embedding_dim = 128 +hidden_dim = 128 +num_layers = 2 +mamba_state_size = 16 +mamba_conv_kernel = 4 +dropout = 0.1 +use_amp = True +amp_dtype = "bf16" +epochs = 50 +batch_size = 32 +lr = 1e-4 +patience = 5 +num_workers = 4 +dev = False +use_old_cache = False +use_wandb = True +wandb_project = f"pyhealth-multimodal-labs-seed-{seed}" +wandb_run_name = None # defaults to "{model}_seed{seed}" if unset +cuda_visible_devices = "0" +session_name = f"ehrmamba_labs_s{seed}" + +# ── Step 0a: Check which CUDA GPU is available ─────────────────────────────── +print(f""" +### STEP 0a: Check which CUDA GPU is available (pick an idle index for CUDA_VISIBLE_DEVICES) + +nvidia-smi --query-gpu=index,utilization.gpu,memory.used,memory.free --format=csv +""") + +# ── Step 0b: Clean logs and cache ──────────────────────────────────────────── +print(f""" +### STEP 0b (optional): Clean logs and cache + +rm -rf {logs_dir}/* +{'# cache preserved (use_old_cache=True)' if use_old_cache else f'rm -rf {cache_dir}/*'} +rm -rf {output_dir}/* +""") + +# ── Step 1: Start a tmux session ──────────────────────────────────────────── +print(f""" +### STEP 1: Start a tmux session (attached) + +tmux new-session -s {session_name} +""") + +# ── Step 2: Paste this into the session to run training ───────────────────── +print("\n" + "=" * 60 + "\n") + +log_dir = logs_dir +log_tag = f"ehrmamba_labs_s{seed}" + +flags = [ + f"--ehr-root {ehr_root}", + f"--cache-dir {cache_dir}", + f"--task labs{' --dev' if dev else ''}", + "--model ehrmamba", + f"--embedding-dim {embedding_dim}", + f"--hidden-dim {hidden_dim}", + f"--num-layers {num_layers}", + f"--mamba-state-size {mamba_state_size}", + f"--mamba-conv-kernel {mamba_conv_kernel}", + f"--dropout {dropout}", +] +if use_amp: + flags.append("--use-amp") + flags.append(f"--amp-dtype {amp_dtype}") +flags += [ + f"--epochs {epochs}", + f"--batch-size {batch_size}", + f"--lr {lr}", + f"--patience {patience}", + f"--num-workers {num_workers}", +] +if use_wandb: + flags.append("--wandb") + flags.append(f"--wandb-project {wandb_project}") + if wandb_run_name: + flags.append(f"--wandb-run-name {wandb_run_name}") +flags += [ + f"--seed {seed}", + f"--output-dir {output_dir}", +] +flags_block = " \\\n ".join(flags) + +print(f""" +### STEP 2: Paste this into the tmux session + +mkdir -p {log_dir} && +eval "$(conda shell.bash hook)" && +conda activate {conda_env} && +cd {project_dir} && +export PYTHONPATH={project_dir}:$PYTHONPATH && +export CUDA_VISIBLE_DEVICES={cuda_visible_devices} && +python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \\ + {flags_block} \\ + > >(tee {log_dir}/{log_tag}.out) \\ + 2> >(tee {log_dir}/{log_tag}.err >&2) +""") + +# ── Step 3: Detach / reattach / monitor ────────────────────────────────────── +print(f""" +### STEP 3: Detach without killing (Ctrl+b d), then reattach later with + +tmux attach -t {session_name} + +### To check on it later without attaching: + +tail -f {log_dir}/{log_tag}.out + +### To kill the session when done: + +tmux kill-session -t {session_name} +""") diff --git a/scripts/will/lambda_labs/labs_only/tmux_run_labs_only_jambaehr_variant.py b/scripts/will/lambda_labs/labs_only/tmux_run_labs_only_jambaehr_variant.py new file mode 100644 index 000000000..3e7e6fb21 --- /dev/null +++ b/scripts/will/lambda_labs/labs_only/tmux_run_labs_only_jambaehr_variant.py @@ -0,0 +1,125 @@ +project_dir = "PyHealth" +seed = 2 +conda_env = "pyhealth2" +ehr_root = "/home/ubuntu/mimiciv-data/ehr" +cache_dir = "/home/ubuntu/mimiciv-data/pyhealth_cache_labs" +logs_dir = "/home/ubuntu/logs" +output_dir = "/home/ubuntu/output" +embedding_dim = 128 +hidden_dim = 128 +heads = 4 +num_layers = 2 +jamba_transformer_layers = 2 +jamba_mamba_layers = 6 +mamba_state_size = 16 +mamba_conv_kernel = 4 +dropout = 0.1 +use_amp = True +amp_dtype = "bf16" +epochs = 50 +batch_size = 32 +lr = 1e-4 +patience = 5 +num_workers = 4 +dev = False +use_old_cache = False +use_wandb = True +wandb_project = f"pyhealth-multimodal-labs-seed-{seed}" +wandb_run_name = None # defaults to "{model}_seed{seed}" if unset +cuda_visible_devices = "0" +session_name = f"jambaehr_labs_s{seed}" + +# ── Step 0a: Check which CUDA GPU is available ─────────────────────────────── +print(f""" +### STEP 0a: Check which CUDA GPU is available (pick an idle index for CUDA_VISIBLE_DEVICES) + +nvidia-smi --query-gpu=index,utilization.gpu,memory.free --format=csv +""") + +# ── Step 0b: Clean logs and cache ──────────────────────────────────────────── +print(f""" +### STEP 0b (optional): Clean logs and cache + +rm -rf {logs_dir}/* +{'# cache preserved (use_old_cache=True)' if use_old_cache else f'rm -rf {cache_dir}/*'} +rm -rf {output_dir}/* +""") + +# ── Step 1: Start a tmux session ──────────────────────────────────────────── +print(f""" +### STEP 1: Start a tmux session (attached) + +tmux new-session -s {session_name} +""") + +# ── Step 2: Paste this into the session to run training ───────────────────── +print("\n" + "=" * 60 + "\n") + +log_dir = logs_dir +log_tag = f"jambaehr_labs_s{seed}" + +flags = [ + f"--ehr-root {ehr_root}", + f"--cache-dir {cache_dir}", + f"--task labs{' --dev' if dev else ''}", + "--model jambaehr", + f"--embedding-dim {embedding_dim}", + f"--hidden-dim {hidden_dim}", + f"--heads {heads}", + f"--num-layers {num_layers}", + f"--jamba-transformer-layers {jamba_transformer_layers}", + f"--jamba-mamba-layers {jamba_mamba_layers}", + f"--mamba-state-size {mamba_state_size}", + f"--mamba-conv-kernel {mamba_conv_kernel}", + f"--dropout {dropout}", +] +if use_amp: + flags.append("--use-amp") + flags.append(f"--amp-dtype {amp_dtype}") +flags += [ + f"--epochs {epochs}", + f"--batch-size {batch_size}", + f"--lr {lr}", + f"--patience {patience}", + f"--num-workers {num_workers}", +] +if use_wandb: + flags.append("--wandb") + flags.append(f"--wandb-project {wandb_project}") + if wandb_run_name: + flags.append(f"--wandb-run-name {wandb_run_name}") +flags += [ + f"--seed {seed}", + f"--output-dir {output_dir}", +] +flags_block = " \\\n ".join(flags) + +print(f""" +### STEP 2: Paste this into the tmux session + +mkdir -p {log_dir} && +eval "$(conda shell.bash hook)" && +conda activate {conda_env} && +cd {project_dir} && +export PYTHONPATH={project_dir}:$PYTHONPATH && +export CUDA_VISIBLE_DEVICES={cuda_visible_devices} && +python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \\ + {flags_block} \\ + > >(tee {log_dir}/{log_tag}.out) \\ + 2> >(tee {log_dir}/{log_tag}.err >&2) +""") + +# ── Step 3: Detach / reattach / monitor ────────────────────────────────────── +print(f""" +### STEP 3: Detach without killing (Ctrl+b d), then reattach later with + +tmux attach -t {session_name} + +### To check on it later without attaching: + +tail -f {log_dir}/{log_tag}.out + +### To kill the session when done: + +tmux kill-session -t {session_name} +""") diff --git a/scripts/will/lambda_labs/labs_only/tmux_run_labs_only_mlp_variant.py b/scripts/will/lambda_labs/labs_only/tmux_run_labs_only_mlp_variant.py new file mode 100644 index 000000000..c79f1fc96 --- /dev/null +++ b/scripts/will/lambda_labs/labs_only/tmux_run_labs_only_mlp_variant.py @@ -0,0 +1,117 @@ +project_dir = "PyHealth" +seed = 2 +conda_env = "pyhealth2" +ehr_root = "/home/ubuntu/mimiciv-data/ehr" +cache_dir = "/home/ubuntu/mimiciv-data/pyhealth_cache_labs" +logs_dir = "/home/ubuntu/logs" +output_dir = "/home/ubuntu/output" +embedding_dim = 128 +hidden_dim = 128 +mlp_layers = 2 +mlp_activation = "relu" +dropout = 0.1 +use_amp = True +amp_dtype = "bf16" +epochs = 50 +batch_size = 32 +lr = 1e-4 +patience = 5 +num_workers = 4 +dev = False +use_old_cache = False +use_wandb = True +wandb_project = f"pyhealth-multimodal-labs-seed-{seed}" +wandb_run_name = None # defaults to "{model}_seed{seed}" if unset +cuda_visible_devices = "0" +session_name = f"mlp_labs_s{seed}" + +# ── Step 0a: Check which CUDA GPU is available ─────────────────────────────── +print(f""" +### STEP 0a: Check which CUDA GPU is available (pick an idle index for CUDA_VISIBLE_DEVICES) + +nvidia-smi --query-gpu=index,utilization.gpu,memory.used,memory.free --format=csv +""") + +# ── Step 0b: Clean logs and cache ──────────────────────────────────────────── +print(f""" +### STEP 0b (optional): Clean logs and cache + +rm -rf {logs_dir}/* +{'# cache preserved (use_old_cache=True)' if use_old_cache else f'rm -rf {cache_dir}/*'} +rm -rf {output_dir}/* +""") + +# ── Step 1: Start a tmux session ──────────────────────────────────────────── +print(f""" +### STEP 1: Start a tmux session (attached) + +tmux new-session -s {session_name} +""") + +# ── Step 2: Paste this into the session to run training ───────────────────── +print("\n" + "=" * 60 + "\n") + +log_dir = logs_dir +log_tag = f"mlp_labs_s{seed}" + +flags = [ + f"--ehr-root {ehr_root}", + f"--cache-dir {cache_dir}", + f"--task labs{' --dev' if dev else ''}", + "--model mlp", + f"--embedding-dim {embedding_dim}", + f"--hidden-dim {hidden_dim}", + f"--mlp-layers {mlp_layers}", + f"--mlp-activation {mlp_activation}", + f"--dropout {dropout}", +] +if use_amp: + flags.append("--use-amp") + flags.append(f"--amp-dtype {amp_dtype}") +flags += [ + f"--epochs {epochs}", + f"--batch-size {batch_size}", + f"--lr {lr}", + f"--patience {patience}", + f"--num-workers {num_workers}", +] +if use_wandb: + flags.append("--wandb") + flags.append(f"--wandb-project {wandb_project}") + if wandb_run_name: + flags.append(f"--wandb-run-name {wandb_run_name}") +flags += [ + f"--seed {seed}", + f"--output-dir {output_dir}", +] +flags_block = " \\\n ".join(flags) + +print(f""" +### STEP 2: Paste this into the tmux session + +mkdir -p {log_dir} && +eval "$(conda shell.bash hook)" && +conda activate {conda_env} && +cd {project_dir} && +export PYTHONPATH={project_dir}:$PYTHONPATH && +export CUDA_VISIBLE_DEVICES={cuda_visible_devices} && +python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \\ + {flags_block} \\ + > >(tee {log_dir}/{log_tag}.out) \\ + 2> >(tee {log_dir}/{log_tag}.err >&2) +""") + +# ── Step 3: Detach / reattach / monitor ────────────────────────────────────── +print(f""" +### STEP 3: Detach without killing (Ctrl+b d), then reattach later with + +tmux attach -t {session_name} + +### To check on it later without attaching: + +tail -f {log_dir}/{log_tag}.out + +### To kill the session when done: + +tmux kill-session -t {session_name} +""") diff --git a/scripts/will/lambda_labs/labs_only/tmux_run_labs_only_rnn_variant.py b/scripts/will/lambda_labs/labs_only/tmux_run_labs_only_rnn_variant.py new file mode 100644 index 000000000..164c773f2 --- /dev/null +++ b/scripts/will/lambda_labs/labs_only/tmux_run_labs_only_rnn_variant.py @@ -0,0 +1,117 @@ +project_dir = "PyHealth" +seed = 2 +conda_env = "pyhealth2" +ehr_root = "/home/ubuntu/mimiciv-data/ehr" +cache_dir = "/home/ubuntu/mimiciv-data/pyhealth_cache_labs" +logs_dir = "/home/ubuntu/logs" +output_dir = "/home/ubuntu/output" +embedding_dim = 128 +hidden_dim = 128 +rnn_type = "GRU" +rnn_layers = 2 +dropout = 0.1 +use_amp = True +amp_dtype = "bf16" +epochs = 50 +batch_size = 32 +lr = 1e-4 +patience = 5 +num_workers = 4 +dev = False +use_old_cache = False +use_wandb = True +wandb_project = f"pyhealth-multimodal-labs-seed-{seed}" +wandb_run_name = None # defaults to "{model}_seed{seed}" if unset +cuda_visible_devices = "0" +session_name = f"rnn_labs_s{seed}" + +# ── Step 0a: Check which CUDA GPU is available ─────────────────────────────── +print(f""" +### STEP 0a: Check which CUDA GPU is available (pick an idle index for CUDA_VISIBLE_DEVICES) + +nvidia-smi --query-gpu=index,utilization.gpu,memory.used,memory.free --format=csv +""") + +# ── Step 0b: Clean logs and cache ──────────────────────────────────────────── +print(f""" +### STEP 0b (optional): Clean logs and cache + +rm -rf {logs_dir}/* +{'# cache preserved (use_old_cache=True)' if use_old_cache else f'rm -rf {cache_dir}/*'} +rm -rf {output_dir}/* +""") + +# ── Step 1: Start a tmux session ──────────────────────────────────────────── +print(f""" +### STEP 1: Start a tmux session (attached) + +tmux new-session -s {session_name} +""") + +# ── Step 2: Paste this into the session to run training ───────────────────── +print("\n" + "=" * 60 + "\n") + +log_dir = logs_dir +log_tag = f"rnn_labs_s{seed}" + +flags = [ + f"--ehr-root {ehr_root}", + f"--cache-dir {cache_dir}", + f"--task labs{' --dev' if dev else ''}", + "--model rnn", + f"--embedding-dim {embedding_dim}", + f"--hidden-dim {hidden_dim}", + f"--rnn-type {rnn_type}", + f"--rnn-layers {rnn_layers}", + f"--dropout {dropout}", +] +if use_amp: + flags.append("--use-amp") + flags.append(f"--amp-dtype {amp_dtype}") +flags += [ + f"--epochs {epochs}", + f"--batch-size {batch_size}", + f"--lr {lr}", + f"--patience {patience}", + f"--num-workers {num_workers}", +] +if use_wandb: + flags.append("--wandb") + flags.append(f"--wandb-project {wandb_project}") + if wandb_run_name: + flags.append(f"--wandb-run-name {wandb_run_name}") +flags += [ + f"--seed {seed}", + f"--output-dir {output_dir}", +] +flags_block = " \\\n ".join(flags) + +print(f""" +### STEP 2: Paste this into the tmux session + +mkdir -p {log_dir} && +eval "$(conda shell.bash hook)" && +conda activate {conda_env} && +cd {project_dir} && +export PYTHONPATH={project_dir}:$PYTHONPATH && +export CUDA_VISIBLE_DEVICES={cuda_visible_devices} && +python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \\ + {flags_block} \\ + > >(tee {log_dir}/{log_tag}.out) \\ + 2> >(tee {log_dir}/{log_tag}.err >&2) +""") + +# ── Step 3: Detach / reattach / monitor ────────────────────────────────────── +print(f""" +### STEP 3: Detach without killing (Ctrl+b d), then reattach later with + +tmux attach -t {session_name} + +### To check on it later without attaching: + +tail -f {log_dir}/{log_tag}.out + +### To kill the session when done: + +tmux kill-session -t {session_name} +""") diff --git a/scripts/will/lambda_labs/labs_only/tmux_run_labs_only_transformer_variant.py b/scripts/will/lambda_labs/labs_only/tmux_run_labs_only_transformer_variant.py new file mode 100644 index 000000000..aeb341b1d --- /dev/null +++ b/scripts/will/lambda_labs/labs_only/tmux_run_labs_only_transformer_variant.py @@ -0,0 +1,117 @@ +project_dir = "PyHealth" +seed = 2 +conda_env = "pyhealth2" +ehr_root = "/home/ubuntu/mimiciv-data/ehr" +cache_dir = "/home/ubuntu/mimiciv-data/pyhealth_cache_labs" +logs_dir = "/home/ubuntu/logs" +output_dir = "/home/ubuntu/output" +embedding_dim = 128 +hidden_dim = 128 +heads = 4 +num_layers = 2 +dropout = 0.1 +use_amp = True +amp_dtype = "bf16" +epochs = 50 +batch_size = 32 +lr = 1e-4 +patience = 5 +num_workers = 4 +dev = False +use_old_cache = False +use_wandb = True +wandb_project = f"pyhealth-multimodal-labs-seed-{seed}" +wandb_run_name = None # defaults to "{model}_seed{seed}" if unset +cuda_visible_devices = "0" +session_name = f"transformer_labs_s{seed}" + +# ── Step 0a: Check which CUDA GPU is available ─────────────────────────────── +print(f""" +### STEP 0a: Check which CUDA GPU is available (pick an idle index for CUDA_VISIBLE_DEVICES) + +nvidia-smi --query-gpu=index,utilization.gpu,memory.used,memory.free --format=csv +""") + +# ── Step 0b: Clean logs and cache ──────────────────────────────────────────── +print(f""" +### STEP 0b (optional): Clean logs and cache + +rm -rf {logs_dir}/* +{'# cache preserved (use_old_cache=True)' if use_old_cache else f'rm -rf {cache_dir}/*'} +rm -rf {output_dir}/* +""") + +# ── Step 1: Start a tmux session ──────────────────────────────────────────── +print(f""" +### STEP 1: Start a tmux session (attached) + +tmux new-session -s {session_name} +""") + +# ── Step 2: Paste this into the session to run training ───────────────────── +print("\n" + "=" * 60 + "\n") + +log_dir = logs_dir +log_tag = f"transformer_labs_s{seed}" + +flags = [ + f"--ehr-root {ehr_root}", + f"--cache-dir {cache_dir}", + f"--task labs{' --dev' if dev else ''}", + "--model transformer", + f"--embedding-dim {embedding_dim}", + f"--hidden-dim {hidden_dim}", + f"--heads {heads}", + f"--num-layers {num_layers}", + f"--dropout {dropout}", +] +if use_amp: + flags.append("--use-amp") + flags.append(f"--amp-dtype {amp_dtype}") +flags += [ + f"--epochs {epochs}", + f"--batch-size {batch_size}", + f"--lr {lr}", + f"--patience {patience}", + f"--num-workers {num_workers}", +] +if use_wandb: + flags.append("--wandb") + flags.append(f"--wandb-project {wandb_project}") + if wandb_run_name: + flags.append(f"--wandb-run-name {wandb_run_name}") +flags += [ + f"--seed {seed}", + f"--output-dir {output_dir}", +] +flags_block = " \\\n ".join(flags) + +print(f""" +### STEP 2: Paste this into the tmux session + +mkdir -p {log_dir} && +eval "$(conda shell.bash hook)" && +conda activate {conda_env} && +cd {project_dir} && +export PYTHONPATH={project_dir}:$PYTHONPATH && +export CUDA_VISIBLE_DEVICES={cuda_visible_devices} && +python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \\ + {flags_block} \\ + > >(tee {log_dir}/{log_tag}.out) \\ + 2> >(tee {log_dir}/{log_tag}.err >&2) +""") + +# ── Step 3: Detach / reattach / monitor ────────────────────────────────────── +print(f""" +### STEP 3: Detach without killing (Ctrl+b d), then reattach later with + +tmux attach -t {session_name} + +### To check on it later without attaching: + +tail -f {log_dir}/{log_tag}.out + +### To kill the session when done: + +tmux kill-session -t {session_name} +""") From cdeb3e045ebf4eab3659eb331718e30053340189 Mon Sep 17 00:00:00 2001 From: William Pang Date: Fri, 4 Sep 2026 18:33:49 +0000 Subject: [PATCH 55/61] Unify event timestamps in pyhealth/tasks/multimodal_mimic4.py onto a single per-sample clock. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, time offsets were computed inconsistently across modalities within a multi-admission sample: - ICD codes: hours since the previous admission (a delta, reset to 0 for the first admission). - Labs, notes, CXR: hours since that admission's own start (reset to 0 at every admission). Since labs, admission_note_times, and cxr_image_times concatenate events from every admission into one sequence per patient, the per-admission reset caused values from different admissions to collide — e.g., a lab drawn 6h into stay 2 sorted identically to one drawn 6h into stay 1, even though the two are actually days apart. --- pyhealth/tasks/multimodal_mimic4.py | 105 ++++++++++++++-------------- 1 file changed, 51 insertions(+), 54 deletions(-) diff --git a/pyhealth/tasks/multimodal_mimic4.py b/pyhealth/tasks/multimodal_mimic4.py index a2861fbea..aa1d588b5 100644 --- a/pyhealth/tasks/multimodal_mimic4.py +++ b/pyhealth/tasks/multimodal_mimic4.py @@ -83,9 +83,6 @@ def __init__( window_hours: Optional[float] = None, ): self.window_hours = window_hours - # Task cache key is uuid5 over {**vars(task), schemas}. Bump when - # emitted data changes so leaky caches cannot be reused. - self.emitted_data_version = 1 @staticmethod def _clean_text(text: Optional[str]) -> Optional[str]: @@ -124,6 +121,16 @@ def _parse_datetime(value: Any) -> Optional[datetime]: def _to_hours(delta_seconds: float) -> float: return delta_seconds / 3600.0 + @classmethod + def _hours_since(cls, timestamp: datetime, origin: datetime) -> float: + """Hours from ``origin`` to ``timestamp``. + + Collection windows stay per admission. The value written onto the + unified timeline is hours from the first stay in this sample, so a + later stay at +6h does not sort with the first stay at +6h. + """ + return cls._to_hours((timestamp - origin).total_seconds()) + def _compute_effective_window( self, admissions_to_process: List[Any], @@ -217,13 +224,16 @@ def _collect_labs( patient: Any, admission_time: datetime, end_time: datetime, + time_origin: datetime, ) -> Tuple[List[float], List[List[float]], List[List[bool]]]: """Collect lab values and observation masks for one admission. Args: patient: Patient object. - admission_time: Start of the window; times are relative to this. + admission_time: Start of this stay's collection window. end_time: End of the window (inclusive). + time_origin: First stay in this sample. Event times are hours + from here, not from ``admission_time``. Returns: Tuple of (lab_times, lab_values, lab_masks). ``lab_masks`` is a @@ -282,9 +292,7 @@ def _collect_labs( break lab_vector.append(category_value) lab_mask.append(observed) - lab_times.append( - self._to_hours((lab_ts - admission_time).total_seconds()) - ) + lab_times.append(self._hours_since(lab_ts, time_origin)) lab_values.append(lab_vector) lab_masks.append(lab_mask) return lab_times, lab_values, lab_masks @@ -299,6 +307,7 @@ def _collect_notes( end_time: Optional[datetime] = None, section_headers: Optional[List[str]] = None, fallback_to_full_note: bool = False, + time_origin: Optional[datetime] = None, ) -> Tuple[List[str], List[float]]: """Collect notes of a given type for one admission. @@ -306,7 +315,9 @@ def _collect_notes( patient: Patient object. note_event_type: Event type string (e.g. "discharge", "radiology"). hadm_id: Admission ID to filter by. - admission_time: Admission start time; used to compute time offsets. + admission_time: This stay's admit time (unused for the timeline + once ``time_origin`` is set; kept so existing call sites that + pass it positionally stay valid). start_time: Optional start of the time window. end_time: Optional end of the time window. section_headers: When provided, extract only these named sections @@ -316,8 +327,8 @@ def _collect_notes( with no matching sections are dropped entirely. Returns: - Tuple of (texts, hours_from_admission). Empty lists when the - events list is empty; do not invent a placeholder note. + Tuple of (texts, hours from the sample's first stay). Empty lists + when the events list is empty; do not invent a placeholder note. """ notes = patient.get_events( event_type=note_event_type, @@ -340,11 +351,9 @@ def _collect_notes( elif not fallback_to_full_note: continue - time_from_admission = self._to_hours( - (note.timestamp - admission_time).total_seconds() - ) + origin = time_origin if time_origin is not None else admission_time texts.append(note_text) - note_times.append(time_from_admission) + note_times.append(self._hours_since(note.timestamp, origin)) except ( AttributeError ): # note object is missing .text or .timestamp attribute (e.g. malformed note) @@ -358,8 +367,8 @@ class ICDLabsMIMIC4(BaseMultimodalMIMIC4Task): A notes-free structured-EHR task that uses only: - - **ICD codes**: diagnosis and procedure codes per admission, processed by - ``StageNetProcessor`` with inter-admission time offsets. + - **ICD codes**: diagnosis and procedure codes per admission, placed on + the same hours-from-first-stay timeline as labs. - **Lab values**: 10-dimensional lab vectors (one per lab category) at each measurement timestamp, processed by ``StageNetTensorProcessor``. @@ -399,13 +408,13 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: effective_start, effective_end = self._compute_effective_window( admissions_to_process ) + time_origin = admissions_to_process[0].timestamp all_icd_codes: List[List[str]] = [] all_icd_times: List[float] = [] all_lab_values: List[List[float]] = [] all_lab_masks: List[List[bool]] = [] all_lab_times: List[float] = [] - previous_admission_time = None for admission in admissions_to_process: admission_time = admission.timestamp @@ -421,16 +430,8 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: visit_icd_codes = self._collect_icd_codes(patient, admission.hadm_id) if visit_icd_codes: - if previous_admission_time is None: - time_from_previous = 0.0 - else: - time_from_previous = self._to_hours( - (admission_time - previous_admission_time).total_seconds() - ) all_icd_codes.append(visit_icd_codes) - all_icd_times.append(time_from_previous) - - previous_admission_time = admission_time + all_icd_times.append(self._hours_since(admission_time, time_origin)) lab_times, lab_values, lab_masks = self._collect_labs( patient=patient, @@ -438,6 +439,7 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: end_time=self._admission_window_end( admission_time, admission_dischtime ), + time_origin=time_origin, ) all_lab_times.extend(lab_times) all_lab_values.extend(lab_values) @@ -480,7 +482,8 @@ class NotesLabsMIMIC4(BaseMultimodalMIMIC4Task): labs: 10-dim lab vectors at each measurement timestamp. labs_mask: Boolean observation mask parallel to ``labs``. icd_codes: (only when ``include_icd=True``) Diagnosis + procedure codes - per admission with inter-admission time offsets. + per admission, on the same hours-from-first-stay timeline as + ``labs``. Args: window_hours: Hours from admission for lab collection. ``None`` @@ -540,6 +543,7 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: effective_start, effective_end = self._compute_effective_window( admissions_to_process ) + time_origin = admissions_to_process[0].timestamp all_note_texts: List[str] = [] all_note_times: List[float] = [] @@ -548,7 +552,6 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: all_lab_times: List[float] = [] all_icd_codes: List[List[str]] = [] all_icd_times: List[float] = [] - previous_admission_time = None for admission in admissions_to_process: admission_time = admission.timestamp @@ -568,6 +571,7 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: admission.hadm_id, admission_time, section_headers=self.DISCHARGE_CLINICAL_HEADERS, + time_origin=time_origin, ) all_note_texts.extend(note_texts) all_note_times.extend(note_times) @@ -578,6 +582,7 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: patient=patient, admission_time=admission_time, end_time=lab_end, + time_origin=time_origin, ) all_lab_times.extend(lab_times) all_lab_values.extend(lab_values) @@ -596,23 +601,18 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: start_time=admission_time, end_time=lab_end, section_headers=self.RADIOLOGY_CLINICAL_HEADERS, + time_origin=time_origin, ) all_note_texts.extend(radiology_texts) all_note_times.extend(radiology_times) if self.include_icd: visit_icd_codes = self._collect_icd_codes(patient, admission.hadm_id) - time_from_previous = ( - 0.0 - if previous_admission_time is None - else self._to_hours( - (admission_time - previous_admission_time).total_seconds() - ) - ) if visit_icd_codes: all_icd_codes.append(visit_icd_codes) - all_icd_times.append(time_from_previous) - previous_admission_time = admission_time + all_icd_times.append( + self._hours_since(admission_time, time_origin) + ) record: Dict[str, Any] = { "patient_id": patient.patient_id, @@ -649,7 +649,8 @@ class NotesLabsCXRMIMIC4(BaseMultimodalMIMIC4Task): cxr_image_times: In-window CXR image paths at their exam-relative timestamp. icd_codes: (only when ``include_icd=True``) Diagnosis + procedure codes - per admission with inter-admission time offsets. + per admission, on the same hours-from-first-stay timeline as + ``labs``. Args: window_hours: Hours from admission for lab/CXR collection. @@ -717,6 +718,7 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: effective_start, effective_end = self._compute_effective_window( admissions_to_process ) + time_origin = admissions_to_process[0].timestamp all_note_texts: List[str] = [] all_note_times: List[float] = [] @@ -727,7 +729,6 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: all_icd_times: List[float] = [] all_cxr_paths: List[str] = [] all_cxr_times: List[float] = [] - previous_admission_time = None for admission in admissions_to_process: admission_time = admission.timestamp @@ -752,6 +753,7 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: admission.hadm_id, admission_time, section_headers=self.DISCHARGE_CLINICAL_HEADERS, + time_origin=time_origin, ) all_note_texts.extend(note_texts) all_note_times.extend(note_times) @@ -762,6 +764,7 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: patient=patient, admission_time=admission_time, end_time=lab_end, + time_origin=time_origin, ) all_lab_times.extend(lab_times) all_lab_values.extend(lab_values) @@ -780,6 +783,7 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: start_time=admission_time, end_time=lab_end, section_headers=self.RADIOLOGY_CLINICAL_HEADERS, + time_origin=time_origin, ) all_note_texts.extend(radiology_texts) all_note_times.extend(radiology_times) @@ -796,26 +800,18 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: if event.image_path: all_cxr_paths.append(event.image_path) all_cxr_times.append( - self._to_hours( - (event.timestamp - admission_time).total_seconds() - ) + self._hours_since(event.timestamp, time_origin) ) except AttributeError: continue if self.include_icd: visit_icd_codes = self._collect_icd_codes(patient, admission.hadm_id) - time_from_previous = ( - 0.0 - if previous_admission_time is None - else self._to_hours( - (admission_time - previous_admission_time).total_seconds() - ) - ) if visit_icd_codes: all_icd_codes.append(visit_icd_codes) - all_icd_times.append(time_from_previous) - previous_admission_time = admission_time + all_icd_times.append( + self._hours_since(admission_time, time_origin) + ) record: Dict[str, Any] = { "patient_id": patient.patient_id, @@ -872,6 +868,7 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: # type: ignore[overri effective_start, effective_end = self._compute_effective_window( admissions_to_process ) + time_origin = admissions_to_process[0].timestamp all_lab_times: List[float] = [] all_lab_values: List[List[float]] = [] @@ -895,6 +892,7 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: # type: ignore[overri end_time=self._admission_window_end( admission_time, admission_dischtime ), + time_origin=time_origin, ) all_lab_times.extend(lab_times) all_lab_values.extend(lab_values) @@ -951,6 +949,7 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: # type: ignore[overri effective_start, effective_end = self._compute_effective_window( admissions_to_process ) + time_origin = admissions_to_process[0].timestamp all_cxr_paths: List[str] = [] all_cxr_times: List[float] = [] @@ -987,9 +986,7 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: # type: ignore[overri if event.image_path: all_cxr_paths.append(event.image_path) all_cxr_times.append( - self._to_hours( - (event.timestamp - admission_time).total_seconds() - ) + self._hours_since(event.timestamp, time_origin) ) except AttributeError: continue From d45e3ccd08f065e865bbfec9b2ab3538caadb679 Mon Sep 17 00:00:00 2001 From: Rian354 Date: Fri, 4 Sep 2026 15:58:23 -0400 Subject: [PATCH 56/61] Wire lab z-scoring, run provenance, and a named eval split. Five small changes on top of cdeb3e0, plus the paper launchers. 1. lab_standardizer.py + wiring. Per-feature z-score fit on the training split only; missing values stay missing. UnifiedMultimodalEmbeddingModel already accepted numeric_standardizers, so this is only the fit + the hand-off. --no-lab-standardization runs raw labs as an ablation, so the default is a choice you can turn off rather than a commitment. Measured on EHRMamba/labs+notes/seed 1: 0.7473 without, 0.8011 with. 2. write_run_config. metrics_history.json records what a run scored but not the conditions that produced it, and it records resolved values rather than raw flags -- that distinction is what surfaced a per-model optimizer override where run_config stored adam_eps: null while the optimizer used 1e-6. Also records source_sha256 so a table can be shown to come from one build. 3. eval_split. The inference loader fell back test-or-val-or-train, so a run without a test split reported TRAINING metrics as test with nothing saying so. The split is now named, warned about, and recorded. 4. exp_name includes the task. It was {model}_seed{seed}, so labs and notes_labs at one seed wrote to the same directory and the second run silently destroyed the first. This matters immediately: the plan is 36 paired cells. 5. Restore emitted_data_version. cdeb3e0 removed it. It is part of vars(task), which is what the task-cache uuid5 key is built from, so without it a cache built before an emitted-data change is silently reused -- and cdeb3e0 changes every event timestamp, which is exactly when the bump is needed. Deliberately not included: the time_origin fix. cdeb3e0 already does it, and by inspection it is identical to ours (same _hours_since helper, same anchor on admissions_to_process[0].timestamp). No need to revert it. scripts/paper: common.sh holds the protocol; will.sh and rian.sh add only the data roots and the CPU tuning for their machine. rian.sh pins OMP threads and uses loader workers because those nodes run several cells at once -- unpinned, four concurrent cells put ~800 threads on 128 cores and epoch time went 191s to 8600s with the GPUs at 0-1%. will.sh keeps num_workers=4 and no pinning. Verified on a dev split before and after: output dir goes mlp_seed1 -> notes_labs_mlp_seed1, run_config.json appears with eval_split=test, and the fitted mean/std/count buffers land in the checkpoint with --no-lab- standardization correctly removing them. --- .../unified_embedding_e2e_mimic4.py | 50 ++++++- pyhealth/processors/__init__.py | 7 + pyhealth/processors/lab_standardizer.py | 130 ++++++++++++++++++ pyhealth/tasks/multimodal_mimic4.py | 6 + pyhealth/utils.py | 76 +++++++++- scripts/paper/common.sh | 47 +++++++ scripts/paper/rian.sh | 18 +++ scripts/paper/will.sh | 8 ++ tests/test_p2_lab_standardizer.py | 109 +++++++++++++++ 9 files changed, 445 insertions(+), 6 deletions(-) create mode 100644 pyhealth/processors/lab_standardizer.py create mode 100644 scripts/paper/common.sh create mode 100755 scripts/paper/rian.sh create mode 100755 scripts/paper/will.sh create mode 100644 tests/test_p2_lab_standardizer.py diff --git a/examples/mortality_prediction/unified_embedding_e2e_mimic4.py b/examples/mortality_prediction/unified_embedding_e2e_mimic4.py index 75968787c..0736cb0e0 100644 --- a/examples/mortality_prediction/unified_embedding_e2e_mimic4.py +++ b/examples/mortality_prediction/unified_embedding_e2e_mimic4.py @@ -65,8 +65,9 @@ NotesLabsCXRMIMIC4, NotesLabsMIMIC4, ) +from pyhealth.processors import fit_lab_standardizer from pyhealth.trainer import Trainer -from pyhealth.utils import set_seed +from pyhealth.utils import set_seed, write_run_config logger = logging.getLogger(__name__) @@ -238,7 +239,11 @@ def _inert_arch_flags(model: str) -> list[str]: return ["--" + f.replace("_", "-") for f in _ARCH_FLAGS_ALL if f not in used] -def _build_model(args: argparse.Namespace, sample_dataset: Any): +def _build_model( + args: argparse.Namespace, + sample_dataset: Any, + numeric_standardizers: Optional[dict[str, Any]] = None, +): inert = _inert_arch_flags(args.model) if inert: logger.warning( @@ -252,6 +257,7 @@ def _build_model(args: argparse.Namespace, sample_dataset: Any): processors=sample_dataset.input_processors, embedding_dim=args.embedding_dim, freeze_text_encoder=args.freeze_encoder, + numeric_standardizers=numeric_standardizers, max_frozen_text_cache=args.max_frozen_text_cache, text_grad_checkpoint_rows=args.text_grad_checkpoint_rows, ) @@ -365,7 +371,13 @@ def run(args: argparse.Namespace) -> Path: sample_dataset, seed=args.seed, allow_leaky_split=args.allow_leaky_split ) - model = _build_model(args, sample_dataset) + # Lab z-scores, fit on the training split only. Missing values stay + # missing; --no-lab-standardization runs raw labs as an ablation. + numeric_standardizers: dict[str, Any] = {} + if "labs" in sample_dataset.input_processors and not args.no_lab_standardization: + numeric_standardizers["labs"] = fit_lab_standardizer(train_ds) + + model = _build_model(args, sample_dataset, numeric_standardizers) train_loader = get_dataloader(train_ds, batch_size=args.batch_size, shuffle=True) val_loader = ( @@ -379,7 +391,7 @@ def run(args: argparse.Namespace) -> Path: else None ) - exp_name = f"{args.model}_seed{args.seed}" + exp_name = f"{args.task}_{args.model}_seed{args.seed}" output_dir = Path(args.output_dir) wandb_logger = WandbLogger( @@ -430,11 +442,33 @@ def run(args: argparse.Namespace) -> Path: test_scores = trainer.evaluate(test_loader) wandb_logger.log({f"test_{k}": v for k, v in test_scores.items()}) - inference_loader = test_loader or val_loader or train_loader + if test_loader is not None: + inference_loader, eval_split = test_loader, "test" + elif val_loader is not None: + inference_loader, eval_split = val_loader, "val" + warnings.warn("No test split; predictions come from VAL.", RuntimeWarning) + else: + inference_loader, eval_split = train_loader, "train" + warnings.warn( + "No test or val split; predictions come from TRAIN and are held-in.", + RuntimeWarning, + ) y_true, y_prob, _, patient_ids = trainer.inference( inference_loader, return_patient_ids=True ) + write_run_config( + str(output_dir / exp_name), + { + **vars(args), + "eval_split": eval_split, + "lab_standardization": bool(numeric_standardizers), + "n_train": len(train_ds), + "n_val": len(val_ds), + "n_test": len(test_ds), + }, + ) + output_csv = output_dir / exp_name / f"predictions_{args.model}.csv" _write_predictions(output_csv, patient_ids, y_true, y_prob) @@ -501,6 +535,12 @@ def parse_args() -> argparse.Namespace: "tests only — the metrics are not usable." ), ) + parser.add_argument( + "--no-lab-standardization", + action="store_true", + default=False, + help="Disable train-split lab z-scoring (raw-lab ablation).", + ) parser.add_argument("--weight-decay", type=float, default=0.0) parser.add_argument("--device", type=str, default=None) parser.add_argument( diff --git a/pyhealth/processors/__init__.py b/pyhealth/processors/__init__.py index 4568a5ece..d7df193ae 100644 --- a/pyhealth/processors/__init__.py +++ b/pyhealth/processors/__init__.py @@ -82,4 +82,11 @@ def get_processor(name: str): "TupleTimeTextProcessor", "CehrProcessor", "ConceptVocab", + "LabStandardizer", + "fit_lab_standardizer", + "", ] +from .lab_standardizer import ( + LabStandardizer, + fit_lab_standardizer, +) diff --git a/pyhealth/processors/lab_standardizer.py b/pyhealth/processors/lab_standardizer.py new file mode 100644 index 000000000..0e193c453 --- /dev/null +++ b/pyhealth/processors/lab_standardizer.py @@ -0,0 +1,130 @@ +"""Train-split-only z-scoring for masked temporal laboratory values. + +Labs and their observation mask are separate temporal fields. Only rows whose +mask is true are fitted: zero-filled / forward-filled missing values must never +affect a lab's mean or variance. +""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import Any, Optional + +import torch +from torch import nn + + +def _indices(dataset: Any) -> Optional[list[int]]: + """Explicit sample indices for this split, or None for a plain iterable. + + ``SampleDataset`` subclasses ``litdata.StreamingDataset``, whose ``__iter__`` + and ``__len__`` are sharded by ``WORLD_SIZE``: under torchrun, iterating + would silently fit on 1/WORLD_SIZE of the train split. Indexing is not + sharded, and ``region_of_interest`` is the only unsharded description of + what the split holds. (``patient_to_index`` is not usable — ``subset()`` + copies it unchanged, so after ``split_by_patient`` it still indexes the + parent and raises "index ... didn't find a match within the chunk + intervals".) + """ + roi = getattr(dataset, "region_of_interest", None) + return list(range(sum(end - start for start, end in roi))) if roi else None + + +class LabStandardizer(nn.Module): + """Per-feature z-score with persistent train-only statistics. + + ``mean``/``std``/``observed_count`` are buffers, so they travel in the + model ``state_dict`` and a checkpoint transforms serving inputs exactly as + it did at training time. + """ + + def __init__( + self, + mean: torch.Tensor, + std: torch.Tensor, + observed_count: torch.Tensor, + ) -> None: + super().__init__() + if (std <= 0).any() or not torch.isfinite(mean).all(): + raise ValueError("Lab statistics must be finite with positive std.") + self.register_buffer("mean", mean.detach().to(torch.float32).clone()) + self.register_buffer("std", std.detach().to(torch.float32).clone()) + self.register_buffer("observed_count", observed_count.detach().clone()) + + @property + def feature_dim(self) -> int: + return int(self.mean.numel()) + + @classmethod + def fit( + cls, + samples: Iterable[dict[str, Any]], + *, + value_field: str = "labs", + observation_mask_field: Optional[str] = None, + ) -> "LabStandardizer": + """Fit on observed, finite values of the supplied (already split) data.""" + mask_field = observation_mask_field or f"{value_field}_mask" + idx = _indices(samples) + stream = samples if idx is None else (samples[i] for i in idx) + + count = total = total_sq = None + for sample in stream: + if value_field not in sample or mask_field not in sample: + continue + v = sample[value_field] + m = sample[mask_field] + v = v[1] if isinstance(v, (tuple, list)) else v + m = m[1] if isinstance(m, (tuple, list)) else m + v = torch.as_tensor(v, dtype=torch.float64) + m = torch.as_tensor(m).bool() & torch.isfinite(v) + if v.ndim == 1: + v, m = v.unsqueeze(0), m.unsqueeze(0) + if count is None: + z = torch.zeros(v.shape[-1], dtype=torch.float64) + count, total, total_sq = z.clone(), z.clone(), z.clone() + obs = torch.where(m, v, torch.zeros_like(v)) + count += m.sum(0).to(torch.float64) + total += obs.sum(0) + total_sq += (obs * obs).sum(0) + + if count is None: + raise ValueError( + f"No samples carried both {value_field!r} and {mask_field!r}." + ) + + seen = count > 0 + mean = torch.where(seen, total / count.clamp(min=1), torch.zeros_like(total)) + var = torch.where( + seen, + (total_sq / count.clamp(min=1) - mean.square()).clamp_min(0), + torch.ones_like(total), + ) + # A constant train feature maps to zero; unit std keeps that finite. + std = torch.where(var > 0, var.sqrt(), torch.ones_like(var)) + return cls(mean.to(torch.float32), std.to(torch.float32), count) + + def forward( + self, values: torch.Tensor, observed_mask: torch.Tensor + ) -> torch.Tensor: + """Z-score observed values; missing or unfittable features map to zero. + + Deliberately not clipped: there is no universally valid physiological + range for these MIMIC category aggregates, so values outside train + support stay as large finite z-scores and remain auditable. + """ + if values.shape[-1] != self.feature_dim: + raise ValueError( + f"Expected {self.feature_dim} lab features, got {values.shape[-1]}." + ) + values = values.to(dtype=self.mean.dtype) + observed = observed_mask.bool() & torch.isfinite(values) + z = (values - self.mean) / self.std + return torch.where(observed & (self.observed_count > 0), z, torch.zeros_like(z)) + + +def fit_lab_standardizer( + train_dataset: Iterable[dict[str, Any]], **kwargs: Any +) -> LabStandardizer: + """Fit on the training split only. Train-only is enforced by the caller.""" + return LabStandardizer.fit(train_dataset, **kwargs) diff --git a/pyhealth/tasks/multimodal_mimic4.py b/pyhealth/tasks/multimodal_mimic4.py index aa1d588b5..20dbef61c 100644 --- a/pyhealth/tasks/multimodal_mimic4.py +++ b/pyhealth/tasks/multimodal_mimic4.py @@ -83,6 +83,12 @@ def __init__( window_hours: Optional[float] = None, ): self.window_hours = window_hours + # Part of vars(task), which is what the task-cache uuid5 key is built + # from. Without it, a cache built before an emitted-data change is + # silently reused. Bump whenever the emitted data changes. + # 2: event times are hours from the first stay in the sample, not + # reset per admission. + self.emitted_data_version = 2 @staticmethod def _clean_text(text: Optional[str]) -> Optional[str]: diff --git a/pyhealth/utils.py b/pyhealth/utils.py index b4af8980a..b46c66ac4 100644 --- a/pyhealth/utils.py +++ b/pyhealth/utils.py @@ -1,4 +1,6 @@ import json +import hashlib +import subprocess import os import pickle import random @@ -65,4 +67,76 @@ def set_env(**environ): yield finally: os.environ.clear() - os.environ.update(old_environ) \ No newline at end of file + os.environ.update(old_environ) + + +def _jsonable(value): + if isinstance(value, (str, int, float, bool)) or value is None: + return value + if isinstance(value, (list, tuple)): + return [_jsonable(v) for v in value] + if isinstance(value, dict): + return {str(k): _jsonable(v) for k, v in value.items()} + return str(value) + +def _git_revision(): + repo = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + try: + commit = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=repo, stderr=subprocess.DEVNULL + ).decode().strip() + dirty = bool(subprocess.check_output( + ["git", "status", "--porcelain"], cwd=repo, stderr=subprocess.DEVNULL + ).decode().strip()) + return {"commit": commit, "dirty": dirty} + except Exception: + return {"commit": None, "dirty": None} + +def _source_digest(): + """Hash the package source so code identity survives a non-git deploy. + + Cluster runs typically execute from an unpacked tarball rather than a + clone, so the git lookup returns nothing exactly where provenance matters + most. Hashing the sources keeps "which code produced this result" + answerable either way. + """ + package = os.path.dirname(os.path.abspath(__file__)) + digest = hashlib.sha256() + try: + for root, dirs, files in os.walk(package): + dirs[:] = sorted(d for d in dirs if d != "__pycache__") + for name in sorted(files): + if not name.endswith(".py"): + continue + path = os.path.join(root, name) + digest.update(os.path.relpath(path, package).encode()) + with open(path, "rb") as f: + digest.update(f.read()) + return digest.hexdigest() + except OSError: + return None + +def write_run_config(exp_path, config): + """Persist the resolved run configuration next to the run's metrics. + + ``metrics_history.json`` records what a run scored but not the conditions + that produced it. Record the resolved settings, not the raw flags, so + derived conditions (lr, split mode, eval split) are recoverable. + """ + record = { + "config": {str(k): _jsonable(v) for k, v in config.items()}, + "git": _git_revision(), + "source_sha256": _source_digest(), + "torch": torch.__version__, + } + os.makedirs(exp_path, exist_ok=True) + path = os.path.join(exp_path, "run_config.json") + tmp = f"{path}.tmp.{os.getpid()}" + try: + with open(tmp, "w") as f: + json.dump(record, f, indent=2, sort_keys=True) + os.replace(tmp, path) + finally: + if os.path.exists(tmp): + os.remove(tmp) + return path diff --git a/scripts/paper/common.sh b/scripts/paper/common.sh new file mode 100644 index 000000000..ac6491011 --- /dev/null +++ b/scripts/paper/common.sh @@ -0,0 +1,47 @@ +# Shared Tranche 1 protocol. Sourced by rian.sh / will.sh, which add only the +# data roots and the CPU tuning appropriate to their machine. +# +# TASK=labs_notes MODEL=ehrmamba SEED=3 bash scripts/paper/rian.sh +# +# Env: TASK MODEL SEED TREE EHR_ROOT NOTE_ROOT CXR_ROOT CACHE_DIR OUT GPU + +TASK="${TASK:?set TASK=labs|labs_notes|labs_notes_cxr}" +MODEL="${MODEL:?set MODEL=mlp|rnn|transformer|bottleneck_transformer|ehrmamba|jambaehr}" +SEED="${SEED:?set SEED}" +TREE="${TREE:-$HOME/PyHealth}" +CACHE_DIR="${CACHE_DIR:-$HOME/pyhealth_cache/$TASK}" +OUT="${OUT:-$TREE/output}" + +case "$TASK" in + labs) TASK_FLAG=labs; ROOTS=() ;; + labs_notes) TASK_FLAG=notes_labs; ROOTS=(--note-root "$NOTE_ROOT") ;; + labs_notes_cxr) TASK_FLAG=notes_labs_cxr; ROOTS=(--note-root "$NOTE_ROOT" --cxr-root "$CXR_ROOT" --cxr-variant sunlab) ;; + *) echo "unknown TASK=$TASK" >&2; exit 2 ;; +esac + +case "$MODEL" in + mlp) ARCH=(--mlp-layers 2 --mlp-activation relu) ;; + rnn) ARCH=(--rnn-type GRU --rnn-layers 1) ;; + transformer) ARCH=(--heads 4 --num-layers 2) ;; + bottleneck_transformer) ARCH=(--heads 4 --num-layers 2 --bottlenecks-n 4 --fusion-startidx 1) ;; + ehrmamba) ARCH=(--num-layers 2 --mamba-state-size 16 --mamba-conv-kernel 4) ;; + jambaehr) ARCH=(--heads 4 --jamba-transformer-layers 2 --jamba-mamba-layers 6) ;; + *) echo "unknown MODEL=$MODEL" >&2; exit 2 ;; +esac + +# Identical for every cell: the Tranche 1 protocol. +PROTOCOL=(--embedding-dim 128 --hidden-dim 128 --dropout 0.1 + --batch-size 32 --lr 1e-4 --epochs 50 --patience 5 + --use-amp --amp-dtype bf16 --freeze-encoder) + +launch () { # any extra flags are passed through + cd "$TREE"; export PYTHONPATH="$TREE" + mkdir -p logs "$OUT" + [[ -n "${GPU:-}" ]] && export CUDA_VISIBLE_DEVICES="$GPU" + python examples/mortality_prediction/unified_embedding_e2e_mimic4.py \ + --task "$TASK_FLAG" --model "$MODEL" --seed "$SEED" \ + --ehr-root "$EHR_ROOT" "${ROOTS[@]}" \ + --cache-dir "$CACHE_DIR" --output-dir "$OUT" \ + "${PROTOCOL[@]}" "${ARCH[@]}" "$@" \ + 2>&1 | tee "logs/${TASK}_${MODEL}_seed${SEED}.out" +} diff --git a/scripts/paper/rian.sh b/scripts/paper/rian.sh new file mode 100755 index 000000000..4d42736df --- /dev/null +++ b/scripts/paper/rian.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Rian's cells: seeds 1,3,5 on labs and labs_notes, plus all of labs_notes_cxr. +# +# Same protocol as will.sh; only the data roots and CPU tuning differ. These +# nodes are shared and run several cells at once, and torch defaults to 64 +# intra-op + 128 inter-op threads with nothing pinning them: four unpinned +# cells put ~800 threads on 128 cores and epoch time went 191s -> 8600s with +# the GPUs at 0-1%. Keep THREADS x concurrent_cells under the node's cores. +set -euo pipefail +EHR_ROOT="${EHR_ROOT:-/shared/rsaas/physionet.org/files/mimiciv/2.2}" +NOTE_ROOT="${NOTE_ROOT:-/shared/rsaas/physionet.org/files/mimic-note}" +CXR_ROOT="${CXR_ROOT:-/shared/rsaas/physionet.org/files/MIMIC-CXR}" +THREADS="${THREADS:-8}" +export OMP_NUM_THREADS="$THREADS" MKL_NUM_THREADS="$THREADS" +export OPENBLAS_NUM_THREADS="$THREADS" NUMEXPR_NUM_THREADS="$THREADS" +source "$(dirname "$(readlink -f "$0")")/common.sh" +launch --num-workers "${NUM_WORKERS:-8}" \ + --loader-num-workers "${LOADER_WORKERS:-4}" --persistent-workers diff --git a/scripts/paper/will.sh b/scripts/paper/will.sh new file mode 100755 index 000000000..97d1ec8f8 --- /dev/null +++ b/scripts/paper/will.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Will's cells: seeds 2 and 4 on labs and labs_notes. +set -euo pipefail +EHR_ROOT="${EHR_ROOT:-/home/ubuntu/mimiciv-data/ehr}" +NOTE_ROOT="${NOTE_ROOT:-/home/ubuntu/mimiciv-data}" +CXR_ROOT="${CXR_ROOT:-/home/ubuntu/mimiciv-data/CXR-jpg}" +source "$(dirname "$(readlink -f "$0")")/common.sh" +launch --num-workers "${NUM_WORKERS:-4}" diff --git a/tests/test_p2_lab_standardizer.py b/tests/test_p2_lab_standardizer.py new file mode 100644 index 000000000..2b3e574a4 --- /dev/null +++ b/tests/test_p2_lab_standardizer.py @@ -0,0 +1,109 @@ +"""Proof that lab z-scores fit on observed train rows, not a WORLD_SIZE shard. + +``SampleDataset`` subclasses ``litdata.StreamingDataset``. Under ``torchrun``, +``WORLD_SIZE`` is set before ``torch.distributed`` is initialised, so +``__len__`` / ``__iter__`` silently yield 1/N of the train split (the same +shard on every rank). Measured on real litdata with 20 samples: ``len()`` +reports 5 under ``WORLD_SIZE=4`` while ``region_of_interest`` still sums to 20. + +Fitting padded 0.0 as if it were a measurement also moves sodium's mean from +140 to 105. ``patient_to_index`` is unusable after ``subset()``: it still +holds parent indices and raised ``ValueError: index 237 didn't find a match +within the chunk intervals``. + +Repro:: + + PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 PYTHONPATH=. \\ + python -m pytest tests/test_p2_lab_standardizer.py -q +""" + +from __future__ import annotations + +import os +import unittest +from unittest import mock + +import torch + + +def _lab_samples(n: int = 40): + torch.manual_seed(0) + return [ + { + "labs": torch.stack( + [140.0 + torch.randn(1) * 4, 1.0 + torch.randn(1) * 0.2] + ).view(1, 2), + "labs_mask": torch.ones(1, 2, dtype=torch.bool), + } + for _ in range(n) + ] + + +class TestP2LabStandardizer(unittest.TestCase): + def test_fit_ignores_padded_zeros(self): + from pyhealth.processors import fit_lab_standardizer + + samples = [ + { + "labs": torch.tensor([[140.0, 1.0], [0.0, 0.0]]), + "labs_mask": torch.tensor([[True, True], [False, False]]), + }, + { + "labs": torch.tensor([[142.0, 1.2], [138.0, 0.8]]), + "labs_mask": torch.tensor([[True, True], [True, True]]), + }, + ] + standardizer = fit_lab_standardizer(samples) + # Observed sodium 140, 142, 138. Mean 140, not 105 from the padded 0.0. + self.assertAlmostEqual(standardizer.mean[0].item(), 140.0, places=4) + + def test_unobserved_slot_maps_to_zero(self): + from pyhealth.processors import fit_lab_standardizer + + standardizer = fit_lab_standardizer(_lab_samples()) + values = torch.tensor([[[140.0, 1.0], [0.0, 0.0]]]) + observed = torch.tensor([[[True, True], [False, False]]]) + out = standardizer(values, observed) + self.assertEqual(out[0, 1].abs().sum().item(), 0.0) + self.assertTrue(torch.isfinite(out).all()) + + def test_world_size_does_not_shrink_the_fit(self): + from pyhealth.processors import fit_lab_standardizer + + samples = _lab_samples(40) + + class _ShardedByWorldSize: + def __init__(self, records): + self._records = records + self.region_of_interest = [(0, len(records))] + + def _visible(self): + world = int(os.environ.get("WORLD_SIZE", "1")) + return self._records[: len(self._records) // world] + + def __len__(self): + return len(self._visible()) + + def __iter__(self): + return iter(self._visible()) + + def __getitem__(self, index): + return self._records[index] + + dataset = _ShardedByWorldSize(samples) + single = fit_lab_standardizer(dataset) + with mock.patch.dict(os.environ, {"WORLD_SIZE": "4"}): + sharded = fit_lab_standardizer(dataset) + self.assertTrue(torch.allclose(single.mean, sharded.mean)) + self.assertTrue(torch.allclose(single.std, sharded.std)) + self.assertEqual( + int(single.observed_count.sum()), int(sharded.observed_count.sum()) + ) + + def test_statistics_travel_in_the_state_dict(self): + from pyhealth.processors import fit_lab_standardizer + + standardizer = fit_lab_standardizer(_lab_samples()) + keys = set(standardizer.state_dict()) + self.assertTrue({"mean", "std"} <= keys) + self.assertEqual(tuple(standardizer.state_dict()["mean"].shape), (2,)) From 6ad4726f45f0ddb98ead20dd47aa343994e520dc Mon Sep 17 00:00:00 2001 From: William Pang Date: Fri, 4 Sep 2026 20:13:58 +0000 Subject: [PATCH 57/61] Revert "Unify event timestamps in pyhealth/tasks/multimodal_mimic4.py onto a single per-sample clock." This reverts commit cdeb3e045ebf4eab3659eb331718e30053340189. --- pyhealth/tasks/multimodal_mimic4.py | 105 ++++++++++++++-------------- 1 file changed, 54 insertions(+), 51 deletions(-) diff --git a/pyhealth/tasks/multimodal_mimic4.py b/pyhealth/tasks/multimodal_mimic4.py index aa1d588b5..a2861fbea 100644 --- a/pyhealth/tasks/multimodal_mimic4.py +++ b/pyhealth/tasks/multimodal_mimic4.py @@ -83,6 +83,9 @@ def __init__( window_hours: Optional[float] = None, ): self.window_hours = window_hours + # Task cache key is uuid5 over {**vars(task), schemas}. Bump when + # emitted data changes so leaky caches cannot be reused. + self.emitted_data_version = 1 @staticmethod def _clean_text(text: Optional[str]) -> Optional[str]: @@ -121,16 +124,6 @@ def _parse_datetime(value: Any) -> Optional[datetime]: def _to_hours(delta_seconds: float) -> float: return delta_seconds / 3600.0 - @classmethod - def _hours_since(cls, timestamp: datetime, origin: datetime) -> float: - """Hours from ``origin`` to ``timestamp``. - - Collection windows stay per admission. The value written onto the - unified timeline is hours from the first stay in this sample, so a - later stay at +6h does not sort with the first stay at +6h. - """ - return cls._to_hours((timestamp - origin).total_seconds()) - def _compute_effective_window( self, admissions_to_process: List[Any], @@ -224,16 +217,13 @@ def _collect_labs( patient: Any, admission_time: datetime, end_time: datetime, - time_origin: datetime, ) -> Tuple[List[float], List[List[float]], List[List[bool]]]: """Collect lab values and observation masks for one admission. Args: patient: Patient object. - admission_time: Start of this stay's collection window. + admission_time: Start of the window; times are relative to this. end_time: End of the window (inclusive). - time_origin: First stay in this sample. Event times are hours - from here, not from ``admission_time``. Returns: Tuple of (lab_times, lab_values, lab_masks). ``lab_masks`` is a @@ -292,7 +282,9 @@ def _collect_labs( break lab_vector.append(category_value) lab_mask.append(observed) - lab_times.append(self._hours_since(lab_ts, time_origin)) + lab_times.append( + self._to_hours((lab_ts - admission_time).total_seconds()) + ) lab_values.append(lab_vector) lab_masks.append(lab_mask) return lab_times, lab_values, lab_masks @@ -307,7 +299,6 @@ def _collect_notes( end_time: Optional[datetime] = None, section_headers: Optional[List[str]] = None, fallback_to_full_note: bool = False, - time_origin: Optional[datetime] = None, ) -> Tuple[List[str], List[float]]: """Collect notes of a given type for one admission. @@ -315,9 +306,7 @@ def _collect_notes( patient: Patient object. note_event_type: Event type string (e.g. "discharge", "radiology"). hadm_id: Admission ID to filter by. - admission_time: This stay's admit time (unused for the timeline - once ``time_origin`` is set; kept so existing call sites that - pass it positionally stay valid). + admission_time: Admission start time; used to compute time offsets. start_time: Optional start of the time window. end_time: Optional end of the time window. section_headers: When provided, extract only these named sections @@ -327,8 +316,8 @@ def _collect_notes( with no matching sections are dropped entirely. Returns: - Tuple of (texts, hours from the sample's first stay). Empty lists - when the events list is empty; do not invent a placeholder note. + Tuple of (texts, hours_from_admission). Empty lists when the + events list is empty; do not invent a placeholder note. """ notes = patient.get_events( event_type=note_event_type, @@ -351,9 +340,11 @@ def _collect_notes( elif not fallback_to_full_note: continue - origin = time_origin if time_origin is not None else admission_time + time_from_admission = self._to_hours( + (note.timestamp - admission_time).total_seconds() + ) texts.append(note_text) - note_times.append(self._hours_since(note.timestamp, origin)) + note_times.append(time_from_admission) except ( AttributeError ): # note object is missing .text or .timestamp attribute (e.g. malformed note) @@ -367,8 +358,8 @@ class ICDLabsMIMIC4(BaseMultimodalMIMIC4Task): A notes-free structured-EHR task that uses only: - - **ICD codes**: diagnosis and procedure codes per admission, placed on - the same hours-from-first-stay timeline as labs. + - **ICD codes**: diagnosis and procedure codes per admission, processed by + ``StageNetProcessor`` with inter-admission time offsets. - **Lab values**: 10-dimensional lab vectors (one per lab category) at each measurement timestamp, processed by ``StageNetTensorProcessor``. @@ -408,13 +399,13 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: effective_start, effective_end = self._compute_effective_window( admissions_to_process ) - time_origin = admissions_to_process[0].timestamp all_icd_codes: List[List[str]] = [] all_icd_times: List[float] = [] all_lab_values: List[List[float]] = [] all_lab_masks: List[List[bool]] = [] all_lab_times: List[float] = [] + previous_admission_time = None for admission in admissions_to_process: admission_time = admission.timestamp @@ -430,8 +421,16 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: visit_icd_codes = self._collect_icd_codes(patient, admission.hadm_id) if visit_icd_codes: + if previous_admission_time is None: + time_from_previous = 0.0 + else: + time_from_previous = self._to_hours( + (admission_time - previous_admission_time).total_seconds() + ) all_icd_codes.append(visit_icd_codes) - all_icd_times.append(self._hours_since(admission_time, time_origin)) + all_icd_times.append(time_from_previous) + + previous_admission_time = admission_time lab_times, lab_values, lab_masks = self._collect_labs( patient=patient, @@ -439,7 +438,6 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: end_time=self._admission_window_end( admission_time, admission_dischtime ), - time_origin=time_origin, ) all_lab_times.extend(lab_times) all_lab_values.extend(lab_values) @@ -482,8 +480,7 @@ class NotesLabsMIMIC4(BaseMultimodalMIMIC4Task): labs: 10-dim lab vectors at each measurement timestamp. labs_mask: Boolean observation mask parallel to ``labs``. icd_codes: (only when ``include_icd=True``) Diagnosis + procedure codes - per admission, on the same hours-from-first-stay timeline as - ``labs``. + per admission with inter-admission time offsets. Args: window_hours: Hours from admission for lab collection. ``None`` @@ -543,7 +540,6 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: effective_start, effective_end = self._compute_effective_window( admissions_to_process ) - time_origin = admissions_to_process[0].timestamp all_note_texts: List[str] = [] all_note_times: List[float] = [] @@ -552,6 +548,7 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: all_lab_times: List[float] = [] all_icd_codes: List[List[str]] = [] all_icd_times: List[float] = [] + previous_admission_time = None for admission in admissions_to_process: admission_time = admission.timestamp @@ -571,7 +568,6 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: admission.hadm_id, admission_time, section_headers=self.DISCHARGE_CLINICAL_HEADERS, - time_origin=time_origin, ) all_note_texts.extend(note_texts) all_note_times.extend(note_times) @@ -582,7 +578,6 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: patient=patient, admission_time=admission_time, end_time=lab_end, - time_origin=time_origin, ) all_lab_times.extend(lab_times) all_lab_values.extend(lab_values) @@ -601,18 +596,23 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: start_time=admission_time, end_time=lab_end, section_headers=self.RADIOLOGY_CLINICAL_HEADERS, - time_origin=time_origin, ) all_note_texts.extend(radiology_texts) all_note_times.extend(radiology_times) if self.include_icd: visit_icd_codes = self._collect_icd_codes(patient, admission.hadm_id) + time_from_previous = ( + 0.0 + if previous_admission_time is None + else self._to_hours( + (admission_time - previous_admission_time).total_seconds() + ) + ) if visit_icd_codes: all_icd_codes.append(visit_icd_codes) - all_icd_times.append( - self._hours_since(admission_time, time_origin) - ) + all_icd_times.append(time_from_previous) + previous_admission_time = admission_time record: Dict[str, Any] = { "patient_id": patient.patient_id, @@ -649,8 +649,7 @@ class NotesLabsCXRMIMIC4(BaseMultimodalMIMIC4Task): cxr_image_times: In-window CXR image paths at their exam-relative timestamp. icd_codes: (only when ``include_icd=True``) Diagnosis + procedure codes - per admission, on the same hours-from-first-stay timeline as - ``labs``. + per admission with inter-admission time offsets. Args: window_hours: Hours from admission for lab/CXR collection. @@ -718,7 +717,6 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: effective_start, effective_end = self._compute_effective_window( admissions_to_process ) - time_origin = admissions_to_process[0].timestamp all_note_texts: List[str] = [] all_note_times: List[float] = [] @@ -729,6 +727,7 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: all_icd_times: List[float] = [] all_cxr_paths: List[str] = [] all_cxr_times: List[float] = [] + previous_admission_time = None for admission in admissions_to_process: admission_time = admission.timestamp @@ -753,7 +752,6 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: admission.hadm_id, admission_time, section_headers=self.DISCHARGE_CLINICAL_HEADERS, - time_origin=time_origin, ) all_note_texts.extend(note_texts) all_note_times.extend(note_times) @@ -764,7 +762,6 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: patient=patient, admission_time=admission_time, end_time=lab_end, - time_origin=time_origin, ) all_lab_times.extend(lab_times) all_lab_values.extend(lab_values) @@ -783,7 +780,6 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: start_time=admission_time, end_time=lab_end, section_headers=self.RADIOLOGY_CLINICAL_HEADERS, - time_origin=time_origin, ) all_note_texts.extend(radiology_texts) all_note_times.extend(radiology_times) @@ -800,18 +796,26 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: if event.image_path: all_cxr_paths.append(event.image_path) all_cxr_times.append( - self._hours_since(event.timestamp, time_origin) + self._to_hours( + (event.timestamp - admission_time).total_seconds() + ) ) except AttributeError: continue if self.include_icd: visit_icd_codes = self._collect_icd_codes(patient, admission.hadm_id) + time_from_previous = ( + 0.0 + if previous_admission_time is None + else self._to_hours( + (admission_time - previous_admission_time).total_seconds() + ) + ) if visit_icd_codes: all_icd_codes.append(visit_icd_codes) - all_icd_times.append( - self._hours_since(admission_time, time_origin) - ) + all_icd_times.append(time_from_previous) + previous_admission_time = admission_time record: Dict[str, Any] = { "patient_id": patient.patient_id, @@ -868,7 +872,6 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: # type: ignore[overri effective_start, effective_end = self._compute_effective_window( admissions_to_process ) - time_origin = admissions_to_process[0].timestamp all_lab_times: List[float] = [] all_lab_values: List[List[float]] = [] @@ -892,7 +895,6 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: # type: ignore[overri end_time=self._admission_window_end( admission_time, admission_dischtime ), - time_origin=time_origin, ) all_lab_times.extend(lab_times) all_lab_values.extend(lab_values) @@ -949,7 +951,6 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: # type: ignore[overri effective_start, effective_end = self._compute_effective_window( admissions_to_process ) - time_origin = admissions_to_process[0].timestamp all_cxr_paths: List[str] = [] all_cxr_times: List[float] = [] @@ -986,7 +987,9 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: # type: ignore[overri if event.image_path: all_cxr_paths.append(event.image_path) all_cxr_times.append( - self._hours_since(event.timestamp, time_origin) + self._to_hours( + (event.timestamp - admission_time).total_seconds() + ) ) except AttributeError: continue From 2c9319950813a8533e15ab2297a16c9346accdce Mon Sep 17 00:00:00 2001 From: William Pang Date: Fri, 4 Sep 2026 20:19:04 +0000 Subject: [PATCH 58/61] Revert "Revert "Unify event timestamps in pyhealth/tasks/multimodal_mimic4.py onto a single per-sample clock."" This reverts commit 6ad4726f45f0ddb98ead20dd47aa343994e520dc. --- pyhealth/tasks/multimodal_mimic4.py | 105 ++++++++++++++-------------- 1 file changed, 51 insertions(+), 54 deletions(-) diff --git a/pyhealth/tasks/multimodal_mimic4.py b/pyhealth/tasks/multimodal_mimic4.py index a2861fbea..aa1d588b5 100644 --- a/pyhealth/tasks/multimodal_mimic4.py +++ b/pyhealth/tasks/multimodal_mimic4.py @@ -83,9 +83,6 @@ def __init__( window_hours: Optional[float] = None, ): self.window_hours = window_hours - # Task cache key is uuid5 over {**vars(task), schemas}. Bump when - # emitted data changes so leaky caches cannot be reused. - self.emitted_data_version = 1 @staticmethod def _clean_text(text: Optional[str]) -> Optional[str]: @@ -124,6 +121,16 @@ def _parse_datetime(value: Any) -> Optional[datetime]: def _to_hours(delta_seconds: float) -> float: return delta_seconds / 3600.0 + @classmethod + def _hours_since(cls, timestamp: datetime, origin: datetime) -> float: + """Hours from ``origin`` to ``timestamp``. + + Collection windows stay per admission. The value written onto the + unified timeline is hours from the first stay in this sample, so a + later stay at +6h does not sort with the first stay at +6h. + """ + return cls._to_hours((timestamp - origin).total_seconds()) + def _compute_effective_window( self, admissions_to_process: List[Any], @@ -217,13 +224,16 @@ def _collect_labs( patient: Any, admission_time: datetime, end_time: datetime, + time_origin: datetime, ) -> Tuple[List[float], List[List[float]], List[List[bool]]]: """Collect lab values and observation masks for one admission. Args: patient: Patient object. - admission_time: Start of the window; times are relative to this. + admission_time: Start of this stay's collection window. end_time: End of the window (inclusive). + time_origin: First stay in this sample. Event times are hours + from here, not from ``admission_time``. Returns: Tuple of (lab_times, lab_values, lab_masks). ``lab_masks`` is a @@ -282,9 +292,7 @@ def _collect_labs( break lab_vector.append(category_value) lab_mask.append(observed) - lab_times.append( - self._to_hours((lab_ts - admission_time).total_seconds()) - ) + lab_times.append(self._hours_since(lab_ts, time_origin)) lab_values.append(lab_vector) lab_masks.append(lab_mask) return lab_times, lab_values, lab_masks @@ -299,6 +307,7 @@ def _collect_notes( end_time: Optional[datetime] = None, section_headers: Optional[List[str]] = None, fallback_to_full_note: bool = False, + time_origin: Optional[datetime] = None, ) -> Tuple[List[str], List[float]]: """Collect notes of a given type for one admission. @@ -306,7 +315,9 @@ def _collect_notes( patient: Patient object. note_event_type: Event type string (e.g. "discharge", "radiology"). hadm_id: Admission ID to filter by. - admission_time: Admission start time; used to compute time offsets. + admission_time: This stay's admit time (unused for the timeline + once ``time_origin`` is set; kept so existing call sites that + pass it positionally stay valid). start_time: Optional start of the time window. end_time: Optional end of the time window. section_headers: When provided, extract only these named sections @@ -316,8 +327,8 @@ def _collect_notes( with no matching sections are dropped entirely. Returns: - Tuple of (texts, hours_from_admission). Empty lists when the - events list is empty; do not invent a placeholder note. + Tuple of (texts, hours from the sample's first stay). Empty lists + when the events list is empty; do not invent a placeholder note. """ notes = patient.get_events( event_type=note_event_type, @@ -340,11 +351,9 @@ def _collect_notes( elif not fallback_to_full_note: continue - time_from_admission = self._to_hours( - (note.timestamp - admission_time).total_seconds() - ) + origin = time_origin if time_origin is not None else admission_time texts.append(note_text) - note_times.append(time_from_admission) + note_times.append(self._hours_since(note.timestamp, origin)) except ( AttributeError ): # note object is missing .text or .timestamp attribute (e.g. malformed note) @@ -358,8 +367,8 @@ class ICDLabsMIMIC4(BaseMultimodalMIMIC4Task): A notes-free structured-EHR task that uses only: - - **ICD codes**: diagnosis and procedure codes per admission, processed by - ``StageNetProcessor`` with inter-admission time offsets. + - **ICD codes**: diagnosis and procedure codes per admission, placed on + the same hours-from-first-stay timeline as labs. - **Lab values**: 10-dimensional lab vectors (one per lab category) at each measurement timestamp, processed by ``StageNetTensorProcessor``. @@ -399,13 +408,13 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: effective_start, effective_end = self._compute_effective_window( admissions_to_process ) + time_origin = admissions_to_process[0].timestamp all_icd_codes: List[List[str]] = [] all_icd_times: List[float] = [] all_lab_values: List[List[float]] = [] all_lab_masks: List[List[bool]] = [] all_lab_times: List[float] = [] - previous_admission_time = None for admission in admissions_to_process: admission_time = admission.timestamp @@ -421,16 +430,8 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: visit_icd_codes = self._collect_icd_codes(patient, admission.hadm_id) if visit_icd_codes: - if previous_admission_time is None: - time_from_previous = 0.0 - else: - time_from_previous = self._to_hours( - (admission_time - previous_admission_time).total_seconds() - ) all_icd_codes.append(visit_icd_codes) - all_icd_times.append(time_from_previous) - - previous_admission_time = admission_time + all_icd_times.append(self._hours_since(admission_time, time_origin)) lab_times, lab_values, lab_masks = self._collect_labs( patient=patient, @@ -438,6 +439,7 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: end_time=self._admission_window_end( admission_time, admission_dischtime ), + time_origin=time_origin, ) all_lab_times.extend(lab_times) all_lab_values.extend(lab_values) @@ -480,7 +482,8 @@ class NotesLabsMIMIC4(BaseMultimodalMIMIC4Task): labs: 10-dim lab vectors at each measurement timestamp. labs_mask: Boolean observation mask parallel to ``labs``. icd_codes: (only when ``include_icd=True``) Diagnosis + procedure codes - per admission with inter-admission time offsets. + per admission, on the same hours-from-first-stay timeline as + ``labs``. Args: window_hours: Hours from admission for lab collection. ``None`` @@ -540,6 +543,7 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: effective_start, effective_end = self._compute_effective_window( admissions_to_process ) + time_origin = admissions_to_process[0].timestamp all_note_texts: List[str] = [] all_note_times: List[float] = [] @@ -548,7 +552,6 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: all_lab_times: List[float] = [] all_icd_codes: List[List[str]] = [] all_icd_times: List[float] = [] - previous_admission_time = None for admission in admissions_to_process: admission_time = admission.timestamp @@ -568,6 +571,7 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: admission.hadm_id, admission_time, section_headers=self.DISCHARGE_CLINICAL_HEADERS, + time_origin=time_origin, ) all_note_texts.extend(note_texts) all_note_times.extend(note_times) @@ -578,6 +582,7 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: patient=patient, admission_time=admission_time, end_time=lab_end, + time_origin=time_origin, ) all_lab_times.extend(lab_times) all_lab_values.extend(lab_values) @@ -596,23 +601,18 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: start_time=admission_time, end_time=lab_end, section_headers=self.RADIOLOGY_CLINICAL_HEADERS, + time_origin=time_origin, ) all_note_texts.extend(radiology_texts) all_note_times.extend(radiology_times) if self.include_icd: visit_icd_codes = self._collect_icd_codes(patient, admission.hadm_id) - time_from_previous = ( - 0.0 - if previous_admission_time is None - else self._to_hours( - (admission_time - previous_admission_time).total_seconds() - ) - ) if visit_icd_codes: all_icd_codes.append(visit_icd_codes) - all_icd_times.append(time_from_previous) - previous_admission_time = admission_time + all_icd_times.append( + self._hours_since(admission_time, time_origin) + ) record: Dict[str, Any] = { "patient_id": patient.patient_id, @@ -649,7 +649,8 @@ class NotesLabsCXRMIMIC4(BaseMultimodalMIMIC4Task): cxr_image_times: In-window CXR image paths at their exam-relative timestamp. icd_codes: (only when ``include_icd=True``) Diagnosis + procedure codes - per admission with inter-admission time offsets. + per admission, on the same hours-from-first-stay timeline as + ``labs``. Args: window_hours: Hours from admission for lab/CXR collection. @@ -717,6 +718,7 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: effective_start, effective_end = self._compute_effective_window( admissions_to_process ) + time_origin = admissions_to_process[0].timestamp all_note_texts: List[str] = [] all_note_times: List[float] = [] @@ -727,7 +729,6 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: all_icd_times: List[float] = [] all_cxr_paths: List[str] = [] all_cxr_times: List[float] = [] - previous_admission_time = None for admission in admissions_to_process: admission_time = admission.timestamp @@ -752,6 +753,7 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: admission.hadm_id, admission_time, section_headers=self.DISCHARGE_CLINICAL_HEADERS, + time_origin=time_origin, ) all_note_texts.extend(note_texts) all_note_times.extend(note_times) @@ -762,6 +764,7 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: patient=patient, admission_time=admission_time, end_time=lab_end, + time_origin=time_origin, ) all_lab_times.extend(lab_times) all_lab_values.extend(lab_values) @@ -780,6 +783,7 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: start_time=admission_time, end_time=lab_end, section_headers=self.RADIOLOGY_CLINICAL_HEADERS, + time_origin=time_origin, ) all_note_texts.extend(radiology_texts) all_note_times.extend(radiology_times) @@ -796,26 +800,18 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: if event.image_path: all_cxr_paths.append(event.image_path) all_cxr_times.append( - self._to_hours( - (event.timestamp - admission_time).total_seconds() - ) + self._hours_since(event.timestamp, time_origin) ) except AttributeError: continue if self.include_icd: visit_icd_codes = self._collect_icd_codes(patient, admission.hadm_id) - time_from_previous = ( - 0.0 - if previous_admission_time is None - else self._to_hours( - (admission_time - previous_admission_time).total_seconds() - ) - ) if visit_icd_codes: all_icd_codes.append(visit_icd_codes) - all_icd_times.append(time_from_previous) - previous_admission_time = admission_time + all_icd_times.append( + self._hours_since(admission_time, time_origin) + ) record: Dict[str, Any] = { "patient_id": patient.patient_id, @@ -872,6 +868,7 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: # type: ignore[overri effective_start, effective_end = self._compute_effective_window( admissions_to_process ) + time_origin = admissions_to_process[0].timestamp all_lab_times: List[float] = [] all_lab_values: List[List[float]] = [] @@ -895,6 +892,7 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: # type: ignore[overri end_time=self._admission_window_end( admission_time, admission_dischtime ), + time_origin=time_origin, ) all_lab_times.extend(lab_times) all_lab_values.extend(lab_values) @@ -951,6 +949,7 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: # type: ignore[overri effective_start, effective_end = self._compute_effective_window( admissions_to_process ) + time_origin = admissions_to_process[0].timestamp all_cxr_paths: List[str] = [] all_cxr_times: List[float] = [] @@ -987,9 +986,7 @@ def __call__(self, patient: Any) -> List[Dict[str, Any]]: # type: ignore[overri if event.image_path: all_cxr_paths.append(event.image_path) all_cxr_times.append( - self._to_hours( - (event.timestamp - admission_time).total_seconds() - ) + self._hours_since(event.timestamp, time_origin) ) except AttributeError: continue From c526a51960f5067b6373fdb9b0b1f80052bfd790 Mon Sep 17 00:00:00 2001 From: Rian354 Date: Fri, 4 Sep 2026 16:42:12 -0500 Subject: [PATCH 59/61] paper/rian.sh: drop flags the runner does not define, pass extras through --loader-num-workers and --persistent-workers do not exist on this branch, so every rian cell died at argparse before doing any work. This runner has no dataloader-worker control at all: --num-workers feeds the dataset build only, and thread pinning is what actually keeps concurrent cells off each other. Also forward "$@" so callers can add flags (--wandb, --observation-window-hours) without editing the launcher. --- scripts/paper/rian.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/paper/rian.sh b/scripts/paper/rian.sh index 4d42736df..e7570e7d7 100755 --- a/scripts/paper/rian.sh +++ b/scripts/paper/rian.sh @@ -6,6 +6,10 @@ # intra-op + 128 inter-op threads with nothing pinning them: four unpinned # cells put ~800 threads on 128 cores and epoch time went 191s -> 8600s with # the GPUs at 0-1%. Keep THREADS x concurrent_cells under the node's cores. +# +# THREADS is the whole CPU story here: this runner's --num-workers feeds the +# dataset build only, and it has no dataloader-worker flag, so pinning the +# BLAS/OMP thread pools is what keeps concurrent cells off each other. set -euo pipefail EHR_ROOT="${EHR_ROOT:-/shared/rsaas/physionet.org/files/mimiciv/2.2}" NOTE_ROOT="${NOTE_ROOT:-/shared/rsaas/physionet.org/files/mimic-note}" @@ -14,5 +18,4 @@ THREADS="${THREADS:-8}" export OMP_NUM_THREADS="$THREADS" MKL_NUM_THREADS="$THREADS" export OPENBLAS_NUM_THREADS="$THREADS" NUMEXPR_NUM_THREADS="$THREADS" source "$(dirname "$(readlink -f "$0")")/common.sh" -launch --num-workers "${NUM_WORKERS:-8}" \ - --loader-num-workers "${LOADER_WORKERS:-4}" --persistent-workers +launch --num-workers "${NUM_WORKERS:-8}" "$@" From f4b3707238898ccb051ed8d2e28c4961d54d851b Mon Sep 17 00:00:00 2001 From: Rian354 Date: Fri, 4 Sep 2026 19:19:20 -0400 Subject: [PATCH 60/61] Persist test metrics, log CPU usage, and make run identity window-aware Four small fixes found while running the Tranche 1 sweep. 1. Test evaluation was gated on wandb. `if wandb_logger.enabled and test_loader is not None` meant a run without --wandb never computed test metrics at all -- not merely unlogged, never calculated. Ungated. 2. Test metrics are now written to test_metrics.json. metrics_history.json carries validation only and log.txt has no test lines, so the numbers that go in a paper previously lived nowhere on disk: only in stdout and W&B, recoverable afterwards only by re-scoring predictions_*.csv by hand. 3. Per-epoch CPU accounting alongside the existing VRAM and epoch_time_s: train_cpu_seconds and train_cpu_util_pct. Counts dataloader workers, since self-only time badly understates a data-loading-bound run. psutil is already present via wandb, with a resource fallback. 4. exp_name and the W&B run name now include the observation window. An observation-window arm is a different experiment from the full-stay run at the same task/model/seed, but both resolved to the same name -- so they shared an output directory and collided in W&B. Runs also now set W&B group (arm) and job_type (backbone) so a many-cell sweep is navigable. Also: create_directory used `if not exists: makedirs`, which two processes importing pyhealth for the first time can both pass, leaving one to die on FileExistsError. Seen on a shared cluster home with two concurrent jobs. --- .../unified_embedding_e2e_mimic4.py | 37 ++++++++++++++-- pyhealth/trainer.py | 44 +++++++++++++++++++ pyhealth/utils.py | 7 ++- 3 files changed, 83 insertions(+), 5 deletions(-) diff --git a/examples/mortality_prediction/unified_embedding_e2e_mimic4.py b/examples/mortality_prediction/unified_embedding_e2e_mimic4.py index 0736cb0e0..059604e65 100644 --- a/examples/mortality_prediction/unified_embedding_e2e_mimic4.py +++ b/examples/mortality_prediction/unified_embedding_e2e_mimic4.py @@ -43,6 +43,7 @@ import argparse import csv +import json import logging import warnings from pathlib import Path @@ -82,6 +83,8 @@ def __init__( run_name: str, tags: list[str], config: Dict[str, Any], + group: Optional[str] = None, + job_type: Optional[str] = None, ) -> None: self.enabled = enabled self._run = None @@ -94,6 +97,8 @@ def __init__( name=run_name, tags=tags, config=config, + group=group, + job_type=job_type, ) def log(self, data: Dict[str, Any], step: Optional[int] = None) -> None: @@ -391,7 +396,15 @@ def run(args: argparse.Namespace) -> Path: else None ) - exp_name = f"{args.task}_{args.model}_seed{args.seed}" + # The window belongs in the name: an observation-window arm is a different + # experiment from the full-stay run at the same task/model/seed, and without + # the suffix the two share an output directory and a W&B run name. + window_suffix = ( + f"_w{int(args.observation_window_hours)}" + if args.observation_window_hours + else "" + ) + exp_name = f"{args.task}_{args.model}_seed{args.seed}{window_suffix}" output_dir = Path(args.output_dir) wandb_logger = WandbLogger( @@ -401,6 +414,10 @@ def run(args: argparse.Namespace) -> Path: run_name=args.wandb_run_name or exp_name, tags=args.wandb_tags.split(",") if args.wandb_tags else [args.task, args.model], config=vars(args), + # Group by arm and split by backbone so a many-cell sweep is navigable + # instead of one flat list of runs. + group=f"{args.task}{window_suffix}", + job_type=args.model, ) trainer = Trainer( @@ -438,9 +455,15 @@ def run(args: argparse.Namespace) -> Path: for epoch_record in metrics_history: wandb_logger.log(epoch_record, step=epoch_record["epoch"]) - if wandb_logger.enabled and test_loader is not None: + # Test evaluation must not depend on the logger. This was gated on + # wandb_logger.enabled, so a run without --wandb never computed test + # metrics at all -- they were not merely unlogged, they were never + # calculated. + test_scores = None + if test_loader is not None: test_scores = trainer.evaluate(test_loader) - wandb_logger.log({f"test_{k}": v for k, v in test_scores.items()}) + if wandb_logger.enabled: + wandb_logger.log({f"test_{k}": v for k, v in test_scores.items()}) if test_loader is not None: inference_loader, eval_split = test_loader, "test" @@ -469,6 +492,14 @@ def run(args: argparse.Namespace) -> Path: }, ) + # metrics_history.json carries validation only, so without this the test + # numbers that go in the paper live nowhere on disk -- only in stdout and + # W&B, and are recoverable afterwards only by re-scoring predictions. + if test_scores is not None: + test_path = output_dir / exp_name / "test_metrics.json" + with open(test_path, "w") as handle: + json.dump({"eval_split": eval_split, **test_scores}, handle, indent=2) + output_csv = output_dir / exp_name / f"predictions_{args.model}.csv" _write_predictions(output_csv, patient_ids, y_true, y_prob) diff --git a/pyhealth/trainer.py b/pyhealth/trainer.py index 2085221b3..f67bcbc84 100644 --- a/pyhealth/trainer.py +++ b/pyhealth/trainer.py @@ -90,6 +90,38 @@ def _vram_stats(device: str) -> Dict[str, float]: return {"vram_allocated_mb": allocated, "vram_peak_mb": peak} +def _cpu_seconds() -> Optional[float]: + """Cumulative CPU seconds for this process and its dataloader workers. + + Self-only time badly understates a run whose cost is data loading, since + workers are separate processes. psutil is already available via wandb; the + resource fallback only counts children that have been reaped, so it reads + low while persistent workers are still alive. + """ + try: + import psutil + + proc = psutil.Process() + times = proc.cpu_times() + total = times.user + times.system + for child in proc.children(recursive=True): + try: + ctimes = child.cpu_times() + total += ctimes.user + ctimes.system + except (psutil.NoSuchProcess, psutil.AccessDenied): + continue + return total + except Exception: + try: + import resource + + me = resource.getrusage(resource.RUSAGE_SELF) + kids = resource.getrusage(resource.RUSAGE_CHILDREN) + return me.ru_utime + me.ru_stime + kids.ru_utime + kids.ru_stime + except Exception: + return None + + def get_metrics_fn(mode: str) -> Callable: if mode == "binary": return binary_metrics_fn @@ -270,6 +302,7 @@ def train( if torch.cuda.is_available() and str(self.device).startswith("cuda"): torch.cuda.reset_peak_memory_stats(self.device) epoch_start = time.perf_counter() + cpu_start = _cpu_seconds() # batch training loop logger.info("") for step_idx in trange( @@ -336,6 +369,16 @@ def train( epoch_time = time.perf_counter() - epoch_start vram = _vram_stats(self.device) + cpu = {} + cpu_end = _cpu_seconds() + if cpu_start is not None and cpu_end is not None: + cpu_s = max(cpu_end - cpu_start, 0.0) + cpu = { + "cpu_seconds": round(cpu_s, 2), + # >100% means several cores busy, which is the normal case + # with dataloader workers. + "cpu_util_pct": round(100.0 * cpu_s / max(epoch_time, 1e-9), 1), + } epochs_done = epoch + 1 epochs_left = epochs - epochs_done @@ -372,6 +415,7 @@ def train( "epoch_time_s": round(epoch_time, 3), "skipped_steps": epoch_skipped_steps, **{f"train_{k}": v for k, v in vram.items()}, + **{f"train_{k}": v for k, v in cpu.items()}, } # validation diff --git a/pyhealth/utils.py b/pyhealth/utils.py index b46c66ac4..e3653afbc 100644 --- a/pyhealth/utils.py +++ b/pyhealth/utils.py @@ -23,8 +23,11 @@ def set_seed(seed): def create_directory(directory): - if not os.path.exists(directory): - os.makedirs(directory) + # exist_ok, not a prior exists() check: two processes importing pyhealth + # for the first time both pass the check and one loses the makedirs race + # with FileExistsError. Seen on a shared cluster home, where two concurrent + # jobs both tried to create ~/.cache/pyhealth/medcode/. + os.makedirs(directory, exist_ok=True) def load_pickle(filename): From a2e49df7f19eb0fc34d4d771574b09ca428017e8 Mon Sep 17 00:00:00 2001 From: William Pang Date: Sat, 5 Sep 2026 17:19:16 +0000 Subject: [PATCH 61/61] Update Will's scripts for Wandb naming --- .../tmux_run_labs_notes_bottleneck_transformer_variant.py | 2 +- .../labs_notes/tmux_run_labs_notes_ehrmamba_variant.py | 2 +- .../labs_notes/tmux_run_labs_notes_jambaehr_variant.py | 2 +- .../lambda_labs/labs_notes/tmux_run_labs_notes_mlp_variant.py | 2 +- .../lambda_labs/labs_notes/tmux_run_labs_notes_rnn_variant.py | 2 +- .../labs_notes/tmux_run_labs_notes_transformer_variant.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_bottleneck_transformer_variant.py b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_bottleneck_transformer_variant.py index 192e9f097..8c8d0d242 100644 --- a/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_bottleneck_transformer_variant.py +++ b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_bottleneck_transformer_variant.py @@ -24,7 +24,7 @@ dev = False use_old_cache = False use_wandb = True -wandb_project = f"pyhealth-multimodal-labs-notes-seed-{seed}" +wandb_project = f"ml4h-tranche1-pyhealth-multimodal-labs-notes-seed-{seed}" wandb_run_name = None # defaults to "{model}_seed{seed}" if unset cuda_visible_devices = "0" session_name = f"bottleneck_transformer_labs_notes_s{seed}" diff --git a/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_ehrmamba_variant.py b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_ehrmamba_variant.py index 91fd7924d..f7504ebd7 100644 --- a/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_ehrmamba_variant.py +++ b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_ehrmamba_variant.py @@ -23,7 +23,7 @@ dev = False use_old_cache = False use_wandb = True -wandb_project = f"pyhealth-multimodal-labs-notes-seed-{seed}" +wandb_project = f"ml4h-tranche1-pyhealth-multimodal-labs-notes-seed-{seed}" wandb_run_name = None # defaults to "{model}_seed{seed}" if unset cuda_visible_devices = "0" session_name = f"ehrmamba_labs_notes_s{seed}" diff --git a/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_jambaehr_variant.py b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_jambaehr_variant.py index df2aa7cee..222d4e22f 100644 --- a/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_jambaehr_variant.py +++ b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_jambaehr_variant.py @@ -26,7 +26,7 @@ dev = False use_old_cache = False use_wandb = True -wandb_project = f"pyhealth-multimodal-labs-notes-seed-{seed}" +wandb_project = f"ml4h-tranche1-pyhealth-multimodal-labs-notes-seed-{seed}" wandb_run_name = None # defaults to "{model}_seed{seed}" if unset cuda_visible_devices = "0" session_name = f"jambaehr_labs_notes_s{seed}" diff --git a/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_mlp_variant.py b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_mlp_variant.py index b9e91df5e..b50a83366 100644 --- a/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_mlp_variant.py +++ b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_mlp_variant.py @@ -22,7 +22,7 @@ dev = False use_old_cache = False use_wandb = True -wandb_project = f"pyhealth-multimodal-labs-notes-seed-{seed}" +wandb_project = f"ml4h-tranche1-pyhealth-multimodal-labs-notes-seed-{seed}" wandb_run_name = None # defaults to "{model}_seed{seed}" if unset cuda_visible_devices = "0" session_name = f"mlp_labs_notes_s{seed}" diff --git a/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_rnn_variant.py b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_rnn_variant.py index f0ec7574d..b85a9f705 100644 --- a/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_rnn_variant.py +++ b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_rnn_variant.py @@ -22,7 +22,7 @@ dev = False use_old_cache = False use_wandb = True -wandb_project = f"pyhealth-multimodal-labs-notes-seed-{seed}" +wandb_project = f"ml4h-tranche1-pyhealth-multimodal-labs-notes-seed-{seed}" wandb_run_name = None # defaults to "{model}_seed{seed}" if unset cuda_visible_devices = "0" session_name = f"rnn_labs_notes_s{seed}" diff --git a/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_transformer_variant.py b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_transformer_variant.py index 672fe86c8..2911aa723 100644 --- a/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_transformer_variant.py +++ b/scripts/will/lambda_labs/labs_notes/tmux_run_labs_notes_transformer_variant.py @@ -22,7 +22,7 @@ dev = False use_old_cache = False use_wandb = True -wandb_project = f"pyhealth-multimodal-labs-notes-seed-{seed}" +wandb_project = f"ml4h-tranche1-pyhealth-multimodal-labs-notes-seed-{seed}" wandb_run_name = None # defaults to "{model}_seed{seed}" if unset cuda_visible_devices = "0" session_name = f"transformer_labs_notes_s{seed}"