Add plant-data reconciliation and Bayesian digital twin notebook

.github/workflows/data-reconciliation-bayesian-twin-validation.yml

@@ -0,0 +1,167 @@
+name: Data reconciliation and Bayesian twin validation
+
+on:
+  pull_request:
+    paths:
+      - notebooks/process/data_reconciliation_bayesian_digital_twin.ipynb
+      - scripts/generate_data_reconciliation_bayesian_twin_notebook.py
+      - notebooks/examples_of_NeqSim_in_Colab.ipynb
+      - README.md
+      - notebooks/maintenance_ledger/data_reconciliation_bayesian_twin_20260901.json
+      - notebooks/notebook_maintenance_ledger.json
+      - .github/workflows/data-reconciliation-bayesian-twin-validation.yml
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  source-build-execute-render:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+
+    steps:
+      - name: Check out NeqSim-Colab
+        uses: actions/checkout@v7
+
+      - name: Check out the reviewed NeqSim master snapshot
+        uses: actions/checkout@v7
+        with:
+          repository: equinor/neqsim
+          ref: fdf6b227b4240589ebbfb900527b37e667f3efc8
+          fetch-depth: 1
+          path: neqsim-source
+
+      - name: Set up Java 17
+        uses: actions/setup-java@v5
+        with:
+          distribution: temurin
+          java-version: "17"
+          cache: maven
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v7
+        with:
+          python-version: "3.12"
+
+      - name: Build the reviewed NeqSim source snapshot
+        working-directory: neqsim-source
+        run: ./mvnw -q -DskipTests -Dmaven.javadoc.skip=true package
+
+      - name: Record the source-built runtime
+        shell: bash
+        run: |
+          SOURCE_JAR=$(find "$GITHUB_WORKSPACE/neqsim-source/target" -maxdepth 1 -type f -name 'neqsim-*.jar' | grep -Ev '(sources|javadoc|tests|original)' | sort | head -n 1)
+          test -n "$SOURCE_JAR"
+          SOURCE_COMMIT=$(git -C "$GITHUB_WORKSPACE/neqsim-source" rev-parse HEAD)
+          test "$SOURCE_COMMIT" = "fdf6b227b4240589ebbfb900527b37e667f3efc8"
+          SOURCE_SHA256=$(sha256sum "$SOURCE_JAR" | awk '{print $1}')
+          echo "NEQSIM_SOURCE_ROOT=$GITHUB_WORKSPACE/neqsim-source" >> "$GITHUB_ENV"
+          echo "NEQSIM_SOURCE_JAR=$SOURCE_JAR" >> "$GITHUB_ENV"
+          printf 'commit=%s\njar_sha256=%s\njar_name=%s\n' \
+            "$SOURCE_COMMIT" "$SOURCE_SHA256" "$(basename "$SOURCE_JAR")" \
+            > "$GITHUB_WORKSPACE/neqsim-source-build-metadata.txt"
+          cat "$GITHUB_WORKSPACE/neqsim-source-build-metadata.txt"
+
+      - name: Create a clean validation environment
+        run: |
+          python scripts/bootstrap_neqsim_validation_env.py \
+            --venv /tmp/neqsim-data-reconciliation-validation \
+            --wheelhouse-root /tmp/neqsim-validation-wheelhouse \
+            --neqsim-version 3.18.0 \
+            --refresh \
+            --extra-package numpy==2.5.2 \
+            --extra-package pandas==3.0.5 \
+            --extra-package matplotlib==3.10.8 \
+            --extra-package scipy==1.17.0 \
+            --extra-package nbformat==5.11.1 \
+            --extra-package nbconvert==7.17.1
+
+      - name: Validate and run the notebook generator
+        run: |
+          /tmp/neqsim-data-reconciliation-validation/bin/python -m py_compile \
+            scripts/generate_data_reconciliation_bayesian_twin_notebook.py
+          /tmp/neqsim-data-reconciliation-validation/bin/python \
+            scripts/generate_data_reconciliation_bayesian_twin_notebook.py
+
+      - name: Execute the notebook from top to bottom
+        env:
+          IPYTHONDIR: /tmp/ipython-data-reconciliation
+          MPLCONFIGDIR: /tmp/matplotlib-data-reconciliation
+          NEQSIM_JVM_AUTOSTART: "0"
+          NEQSIM_NOTEBOOK_FIGURE_DIR: ${{ github.workspace }}/validation-artifacts/data-reconciliation-bayesian-twin
+        run: |
+          /tmp/neqsim-data-reconciliation-validation/bin/python \
+            scripts/execute_notebook_inprocess.py \
+            notebooks/process/data_reconciliation_bayesian_digital_twin.ipynb
+
+      - name: Run notebook integrity checks
+        run: |
+          /tmp/neqsim-data-reconciliation-validation/bin/python scripts/check_notebook.py \
+            notebooks/process/data_reconciliation_bayesian_digital_twin.ipynb \
+            --require-main-source
+
+      - name: Verify retained execution evidence
+        run: |
+          /tmp/neqsim-data-reconciliation-validation/bin/python - <<'PY'
+          from pathlib import Path
+          import nbformat
+
+          path = Path(
+              "notebooks/process/"
+              "data_reconciliation_bayesian_digital_twin.ipynb"
+          )
+          notebook = nbformat.read(path, as_version=4)
+          code_cells = [
+              cell for cell in notebook.cells if cell.cell_type == "code"
+          ]
+          errors = []
+          stderr = []
+          figures = 0
+          for cell in code_cells:
+              for output in cell.get("outputs", []):
+                  if output.output_type == "error":
+                      errors.append(output)
+                  if output.output_type == "stream" and output.name == "stderr":
+                      stderr.append(output.text)
+                  if (
+                      output.output_type in ("display_data", "execute_result")
+                      and "image/png" in output.get("data", {})
+                  ):
+                      figures += 1
+
+          notebook_text = str(notebook)
+          assert len(code_cells) == 27
+          assert [cell.execution_count for cell in code_cells] == list(range(1, 28))
+          assert not errors
+          assert not stderr
+          assert figures == 9
+          assert "Validation passed: 27 / 27 named checks." in notebook_text
+          assert "fdf6b227b4240589ebbfb900527b37e667f3efc8" in notebook_text
+          assert "https://github.com/equinor/neqsim/issues/3393" in notebook_text
+          figure_dir = Path(
+              "validation-artifacts/data-reconciliation-bayesian-twin"
+          )
+          assert len(list(figure_dir.glob("*.png"))) == 9
+          print("Retained-output, provenance, and figure checks passed.")
+          PY
+
+      - name: Render the executed notebook to HTML
+        run: |
+          /tmp/neqsim-data-reconciliation-validation/bin/jupyter nbconvert \
+            --to html \
+            --output data_reconciliation_bayesian_digital_twin.html \
+            --output-dir validation-artifacts/data-reconciliation-bayesian-twin \
+            notebooks/process/data_reconciliation_bayesian_digital_twin.ipynb
+
+      - name: Upload source runtime and static fallbacks
+        uses: actions/upload-artifact@v4
+        with:
+          name: data-reconciliation-bayesian-twin-source-executed
+          path: |
+            ${{ env.NEQSIM_SOURCE_JAR }}
+            neqsim-source-build-metadata.txt
+            notebooks/process/data_reconciliation_bayesian_digital_twin.ipynb
+            validation-artifacts/data-reconciliation-bayesian-twin/*.html
+            validation-artifacts/data-reconciliation-bayesian-twin/*.png
+          retention-days: 7

README.md

@@ -11,6 +11,7 @@ Advanced notebooks use the released Python distribution only as the JPype bridge
 
 * [LNG process simulation and benchmark comparison](notebooks/process/LNG_Process_Benchmark_Comparison.ipynb) – Run closed-loop SMR, C3MR, DMR, and nitrogen-expander models with common KPIs, literature checks, and an exchanger grid-convergence study.
 * [IoT and Industry 4.0 with NeqSim](notebooks/AI/IoT_and_Industry4.0_with_NeqSim.ipynb) – Build an instrumented digital twin, stream dynamic simulation data, and explore Industry 4.0 workflows backed by NeqSim measurements.
+* [Plant-data reconciliation and a Bayesian digital twin](notebooks/process/data_reconciliation_bayesian_digital_twin.ipynb) – Qualify historian windows, reconcile redundant meters, isolate gross errors, calibrate compressor efficiency, validate a Bayesian posterior, and propagate uncertainty to an operating decision.
 * [Seismic acquisition to RMS-ready subsurface inputs](notebooks/reservoir/seismic_to_rms_input_workflow.ipynb) – Calculate CMP moveout and stacking, interpret public Reek seismic and wells, validate horizons and faults, screen seismic attributes, and export a checked RMS import package.
 * [RMS-origin reservoir to OPM Flow, ERT, and NeqSim](notebooks/reservoir/rms_to_opm_flow_agent_ert.ipynb) – Audit public Reek ROFF exports, demonstrate blocking and property spreading, run OPM Flow and ERT, and define a governed RMS-agent contract.
 

notebooks/examples_of_NeqSim_in_Colab.ipynb

@@ -461,6 +461,7 @@
         "## Innovative technologies combining AI technologies and NeqSim\n",
         "* [Real-Time process monitoring for operational safety and compliance](AI/Real_Time_process_monitoring_for_operational_safety_and_compliance.ipynb)\n",
         "* [NeqSim and Data Analytics using Seeq](AI/NeqSim_and_Seeq.ipynb)\n",
+        "* [Plant-data reconciliation and a Bayesian digital twin](process/data_reconciliation_bayesian_digital_twin.ipynb): qualify historian windows, close redundant mass balances with native weighted least squares, detect and isolate gross sensor errors, calibrate compressor efficiency, validate an independent Bayesian posterior, test held-out predictions, and propagate uncertainty to power decisions.\n",
         "* Optimization of process parameters to maximize efficiency and production\n",
         "* Anomaly detection to detect unusual patterns or deviations in process data\n",
         "* Integration of different data sources to analyze data from different sources (sensor data, simulation data, weather data and external environmental data) to provide a more holistic understanding of processes\n",

notebooks/maintenance_ledger/data_reconciliation_bayesian_twin_20260901.json

@@ -0,0 +1,107 @@
+{
+  "schema_version": 1,
+  "shard": "data-reconciliation-bayesian-twin-20260901",
+  "updated_at": "2026-09-01T15:17:29Z",
+  "notebooks": [
+    {
+      "path": "notebooks/process/data_reconciliation_bayesian_digital_twin.ipynb",
+      "verified_date": "2026-09-01",
+      "verified_at_utc": "2026-09-01T15:17:29Z",
+      "neqsim_version": "Python bridge 3.18.0 with Java runtime built from equinor/neqsim master commit fdf6b227b4240589ebbfb900527b37e667f3efc8",
+      "neqsim_commit": "fdf6b227b4240589ebbfb900527b37e667f3efc8",
+      "neqsim_jar_sha256": "070494e0b9488e1c2d1f2ba75c212466346ee6c07eed950001fa5360daecfbd0",
+      "python_version": "3.12.13",
+      "java_version": "17.0.20",
+      "numpy_version": "2.5.2",
+      "pandas_version": "3.0.5",
+      "matplotlib_version": "3.10.8",
+      "scipy_version": "1.17.0",
+      "nbformat_version": "5.11.1",
+      "nbconvert_version": "7.17.1",
+      "execution_status": "passed",
+      "execution_method": "Clean isolated Python 3.12 process; all 27 code cells executed top-to-bottom against the source-built NeqSim master JAR with retained stdout, tables, JSON, and inline figures.",
+      "code_cells": 27,
+      "substantive_code_cells": 27,
+      "markdown_cells": 30,
+      "git_blob_sha1": "6f117a2f7911311568114264114d8e3c4c9d25ca",
+      "notebook_bytes": 1039574,
+      "publication": {
+        "branch": "codex/data-reconciliation-bayesian-digital-twin",
+        "mode": "focused draft pull request",
+        "files": [
+          "notebooks/process/data_reconciliation_bayesian_digital_twin.ipynb",
+          "scripts/generate_data_reconciliation_bayesian_twin_notebook.py",
+          ".github/workflows/data-reconciliation-bayesian-twin-validation.yml",
+          "notebooks/maintenance_ledger/data_reconciliation_bayesian_twin_20260901.json",
+          "notebooks/notebook_maintenance_ledger.json",
+          "notebooks/examples_of_NeqSim_in_Colab.ipynb",
+          "README.md"
+        ]
+      },
+      "data_scope": {
+        "type": "deterministic synthetic teaching data",
+        "base_seed": 20260901,
+        "derived_seeds": "Base seed plus one for calibration noise and plus two for posterior sampling.",
+        "model_basis": "SRK with classic mixing rule",
+        "integrity": "Synthetic truth, measurement uncertainties, random seed, resolved NeqSim commit, and source-JAR SHA-256 are retained in the notebook outputs."
+      },
+      "capabilities_demonstrated": [
+        "native Cao-Rhinehart steady-state qualification for historian-style signals",
+        "native uncertainty-weighted linear data reconciliation with an independent NumPy cross-check",
+        "global chi-square testing, normalized-residual ranking, and gross-error isolation",
+        "virtual-meter reconstruction after exclusion of a demonstrably suspect measurement",
+        "native NeqSim batch calibration of compressor polytropic efficiency",
+        "NeqSim-generated response surfaces with off-grid emulator validation",
+        "sequential Bayesian inference with prior, posterior, and credible interval",
+        "held-out posterior-predictive validation",
+        "propagation of parameter uncertainty to compressor-power decisions",
+        "machine-readable JSON handoff with provenance, validation, and limitations"
+      ],
+      "engineering_validation": {
+        "named_assertions_passed": 27,
+        "assertions_failed": 0,
+        "repository_checker_errors": 0,
+        "repository_checker_warnings": 0,
+        "retained_png_figures": 9,
+        "display_equations": 5,
+        "stderr_streams": 0,
+        "execution_errors": 0
+      },
+      "result_summary": {
+        "first_all_tag_steady_sample": 46,
+        "normal_chi_square": 0.2904018361751536,
+        "maximum_post_balance_residual_kg_h": 7.275957614183426e-12,
+        "gross_error_candidate": "separator_gas",
+        "raw_candidate_error_kg_h": 565.0,
+        "virtual_meter_error_kg_h": 34.38566047148197,
+        "synthetic_efficiency_truth": 0.78,
+        "native_batch_efficiency": 0.7806156712444108,
+        "bayesian_posterior_mean": 0.780598291398739,
+        "bayesian_95_interval": [
+          0.7777569776605775,
+          0.7833774383972046
+        ],
+        "holdout_rmse_K": 0.36818662979379674,
+        "holdout_interval_coverage": 1.0,
+        "probability_below_1_70_MW": 0.7652397196576916,
+        "validation_checks_passed": 27,
+        "validation_checks_total": 27
+      },
+      "rendered_visual_validation": {
+        "renderer": "nbconvert 7.17.1 HTML with MathJax source and nine retained Matplotlib PNG outputs",
+        "display_equations_inspected": 5,
+        "figures_inspected": 9,
+        "result": "passed",
+        "notes": "Every retained figure was inspected at original resolution for titles, units, legends, thresholds, uncertainty bands, clipping, overlap, and consistency with stored numerical results. All display equations use Colab-safe compact dollar delimiters."
+      },
+      "documentation_impact": "Adds the planned replacement for the retired synthetic-data-generation notebook to the AI/digital-twin catalog and README featured list. It complements the existing online-simulation, model-versus-measurement, IoT, and machine-learning notebooks with measurement trust, inference, and uncertainty governance.",
+      "known_upstream_issue": {
+        "repository": "equinor/neqsim",
+        "number": 3393,
+        "url": "https://github.com/equinor/neqsim/issues/3393",
+        "impact": "The core data-reconciliation example uses obsolete API calls; this notebook uses and validates the current signatures."
+      },
+      "issue_handling": "Opened equinor/neqsim issue #3393 with the failing calls, current replacements, environment, and acceptance criteria. The new notebook links the issue and independently validates the corrected API usage."
+    }
+  ]
+}

notebooks/notebook_maintenance_ledger.json

@@ -1,7 +1,7 @@
 {
  "schema_version": 3,
- "updated_at": "2026-09-01T06:41:37Z",
-  "active_notebook_count": 296,
+ "updated_at": "2026-09-01T15:17:29Z",
+ "active_notebook_count": 297,
  "shards_glob": "maintenance_ledger/*.json",
  "retired_notebooks": [
   {