fix: use random IVs for stored keys (put/get), matching the Dart SDK

.github/dependabot.yml

@@ -21,3 +21,14 @@ updates:
       pip:
         patterns:
           - "*"
+  # Dart interop test helper (pubspec + lockfile)
+  - package-ecosystem: "pub"
+    directory: "/test/interop"
+    schedule:
+      interval: "daily"
+    cooldown:
+      default-days: 7
+    groups:
+      pub:
+        patterns:
+          - "*"

.github/workflows/interop.yml

@@ -0,0 +1,74 @@
+# Opt-in cross-SDK IV interop test (Python at_client <-> Dart at_client).
+# Manual only (workflow_dispatch): starts the ephemeral environment container,
+# onboards two test atSigns via CRAM, and runs test/interop_test.py with AT_INTEROP=1.
+name: interop (Python <-> Dart)
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: interop-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  interop:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    env:
+      # Run the SDK from source (deps installed from poetry.lock; package not built).
+      PYTHONPATH: ${{ github.workspace }}
+    services:
+      ee:
+        image: atsigncompany/ephemeral
+        env:
+          DNS_FQDN: vip.ve.atsign.zone
+          FIRST_PORT: 2500
+        ports:
+          - 64:64
+          - 2500-2540:2500-2540
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+
+      - name: Resolve the EE FQDN to localhost (matches the cert CN)
+        run: echo "127.0.0.1 vip.ve.atsign.zone" | sudo tee -a /etc/hosts
+
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: '3.14'
+      - uses: dart-lang/setup-dart@65eb853c7ba17dde3be364c3d2858773e7144260 # v1.7.2
+      - name: Install uv
+        uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
+
+      - name: Install SDK dependencies (from poetry.lock)
+        run: |
+          sudo apt-get install -y python3-poetry-plugin-export
+          uv venv
+          . .venv/bin/activate
+          poetry export --format requirements.txt --output requirements.txt
+          uv pip install --require-hashes -r requirements.txt
+          echo "$PWD/.venv/bin" >> "$GITHUB_PATH"
+
+      - name: Wait for the EE root, then onboard @alpha and @bravo (CRAM)
+        env:
+          HOME: /tmp/eehome
+        run: |
+          mkdir -p "$HOME/.atsign/keys"
+          sleep 30   # let the root register the atServers
+          python test/interop/onboard.py @alpha @bravo
+
+      - name: dart pub get (interop helper)
+        run: dart pub get
+        working-directory: test/interop
+
+      - name: Run interop test
+        env:
+          HOME: /tmp/eehome
+          AT_INTEROP: '1'
+          AT_ROOT: vip.ve.atsign.zone:64
+          AT_ROOT_DOMAIN: vip.ve.atsign.zone
+        run: |
+          rm -rf /tmp/eehome/.atsign/storage   # clean Dart local state (keeps .atKeys)
+          python -m unittest discover -s test -p 'interop_test.py' -v

at_client/atclient.py

@@ -21,6 +21,11 @@
 from .common.keys import AtKey, Keys, SharedKey, PrivateHiddenKey, PublicKey, SelfKey
 from .util.authutil import AuthUtil
 
+# The 16-byte all-zero IV used before random IVs existed. Matches the Dart SDK's
+# AtChopsUtil.generateIVLegacy() / getIV(null); used to decrypt legacy data (no ivNonce).
+LEGACY_IV = b"\x00" * 16
+
+
 class AtClient(ABC):
     def __init__(self, atsign:AtSign, root_address:Address=Address("root.atsign.org", 64), secondary_address:Address=None, queue:Queue=None, verbose:bool = False):
         self.atsign = atsign
@@ -174,6 +179,13 @@ def get_encryption_key_shared_by_other(self, shared_key: SharedKey):
 
 
     def put(self, key, value):
+        # Generate the IV once here, before dispatching to the per-type encryptor —
+        # mirroring the Dart SDK's AtClientImpl._putInternal, which sets a random ivNonce
+        # for every put ahead of encryption. This guarantees every encrypted put gets a
+        # random IV regardless of key type; the per-type _put_* methods keep a `?=`
+        # backstop so direct calls are safe too. Public keys aren't encrypted, so no IV.
+        if isinstance(key, (SharedKey, SelfKey)) and key.metadata.iv_nonce is None:
+            key.metadata.iv_nonce = EncryptionUtil.generate_iv_nonce()
         if isinstance(key, SharedKey):
             return self._put_shared_key(key, value)
         elif isinstance(key, SelfKey):
@@ -186,8 +198,16 @@ def put(self, key, value):
     def _put_self_key(self, key: SelfKey, value: str):
         key.metadata.data_signature = EncryptionUtil.sign_sha256_rsa(value, self.keys[KeysUtil.encryption_private_key_name])
 
+        # Generate a random IV per self key too (stored as ivNonce in metadata). Dart's
+        # current SelfKeyEncryption does NOT do this (it uses the zero IV) — that's a
+        # security gap on the Dart side; doing it here is interop-safe because get falls
+        # back to the legacy zero IV when ivNonce is absent, and Dart's self decrypt
+        # already honors ivNonce when present.
+        if key.metadata.iv_nonce is None:
+            key.metadata.iv_nonce = EncryptionUtil.generate_iv_nonce()
+        self_key = self.keys[KeysUtil.self_encryption_key_name]
         try:
-            cipher_text = EncryptionUtil.aes_encrypt_from_base64(value, self.keys[KeysUtil.self_encryption_key_name])
+            cipher_text = EncryptionUtil.aes_encrypt_from_base64(value, self_key, key.metadata.iv_nonce)
         except Exception as e:
             raise AtEncryptionException(f"Failed to encrypt value with self encryption key - {e}")
 
@@ -218,7 +238,11 @@ def _put_shared_key(self, key: SharedKey, value: str):
             share_to_encryption_key = self.get_encryption_key_shared_by_me(key)
 
             what = "encrypt value with shared encryption key"
-            cipher_text = EncryptionUtil.aes_encrypt_from_base64(value, share_to_encryption_key)
+            # Match the Dart SDK: shared keys always get a random IV, persisted as
+            # ivNonce in the key metadata (which is serialized into the update command).
+            if key.metadata.iv_nonce is None:
+                key.metadata.iv_nonce = EncryptionUtil.generate_iv_nonce()
+            cipher_text = EncryptionUtil.aes_encrypt_from_base64(value, share_to_encryption_key, key.metadata.iv_nonce)
         except Exception as e:
             raise AtEncryptionException(f"Failed to {what} - {e}")
 
@@ -255,6 +279,17 @@ def get_lookup_response(self, command: str):
         return fetched
 
 
+    @staticmethod
+    def _iv_from_fetched(fetched) -> bytes:
+        """Read the IV from a fetched (all) lookup response's metadata.
+
+        Returns the ivNonce bytes if present, else the legacy all-zero IV — matching
+        the Dart SDK's `metadata.ivNonce != null ? fromBase64 : generateIVLegacy()`.
+        """
+        meta = fetched.get("metaData") or {}
+        iv_b64 = meta.get("ivNonce")
+        return base64.b64decode(iv_b64) if iv_b64 else LEGACY_IV
+
     def _get_self_key(self, key: SelfKey):
         command = LlookupVerbBuilder().with_at_key(key, LlookupVerbBuilder.Type.ALL).build()
 
@@ -263,8 +298,9 @@ def _get_self_key(self, key: SelfKey):
         decrypted_value = None
         encrypted_value = fetched["data"]
         self_encryption_key = self.keys[KeysUtil.self_encryption_key_name]
+        iv = self._iv_from_fetched(fetched)
         try:
-            decrypted_value = EncryptionUtil.aes_decrypt_from_base64(encrypted_value, self_encryption_key)
+            decrypted_value = EncryptionUtil.aes_decrypt_from_base64(encrypted_value, self_encryption_key, iv)
         except Exception as e:
             raise AtDecryptionException(f"Failed to {command} - {e}")
 
@@ -296,38 +332,31 @@ def _get_shared_key(self, key: SharedKey):
     def _get_shared_by_me_with_other(self, shared_key: SharedKey):
         share_encryption_key = self.get_encryption_key_shared_by_me(shared_key)
 
-        raw_response = None
-        command = "llookup:" + str(shared_key)
-        try:
-            raw_response = self.secondary_connection.execute_command(command, True)
-        except (AtKeyNotFoundException, AtInternalServerException) as e: raise e
-        except AtSecondaryConnectException as e:
-            raise AtSecondaryConnectException(f"Failed to execute {command} - {e}")
+        # Use llookup:all so we get the metadata (ivNonce) alongside the value.
+        command = "llookup:all:" + str(shared_key)
+        fetched = self.get_lookup_response(command)
+        iv = self._iv_from_fetched(fetched)
 
         try:
-            return EncryptionUtil.aes_decrypt_from_base64(raw_response.get_raw_data_response(), share_encryption_key)
+            return EncryptionUtil.aes_decrypt_from_base64(fetched["data"], share_encryption_key, iv)
         except Exception as e:
             raise AtDecryptionException(f"Failed to decrypt value with shared encryption key - {e}")
 
     def _get_shared_by_other_with_me(self, shared_key:SharedKey):
-        what = None
         share_encryption_key = self.get_encryption_key_shared_by_other(shared_key)
 
-        raw_response = None
-        command = "lookup:" + shared_key.name
+        # Use lookup:all so we get the metadata (ivNonce) alongside the value.
+        command = "lookup:all:" + shared_key.name
         if shared_key.get_namespace() is not None and shared_key.get_namespace():
             command += "." + shared_key.get_namespace()
         command += str(shared_key.shared_by)
-        try:
-            raw_response = self.secondary_connection.execute_command(command, True)
-        except Exception as e:
-            raise AtSecondaryConnectException(f"Failed to execute {command} - {e}")
+        fetched = self.get_lookup_response(command)
+        iv = self._iv_from_fetched(fetched)
 
-        what = "decrypt value with shared encryption key"
         try:
-            return EncryptionUtil.aes_decrypt_from_base64(raw_response.get_raw_data_response(), share_encryption_key)
+            return EncryptionUtil.aes_decrypt_from_base64(fetched["data"], share_encryption_key, iv)
         except Exception as e:
-            raise AtDecryptionException(f"Failed to {what} - {e}")
+            raise AtDecryptionException(f"Failed to decrypt value with shared encryption key - {e}")
 
     def delete(self, key):
         if isinstance(key, SharedKey) or isinstance(key, SelfKey) or isinstance(key, PublicKey):

at_client/common/metadata.py

@@ -68,8 +68,11 @@ def from_json(json_str):
         metadata.shared_key_enc = data.get('sharedKeyEnc')
         metadata.pub_key_cs = data.get('pubKeyCS')
         metadata.encoding = data.get('encoding')
-        metadata.iv_nonce = data.get('ivNonce')
-
+        # ivNonce travels as base64 on the wire; keep it as raw bytes internally so it
+        # round-trips with generate_iv_nonce()/__str__ and is usable directly as an IV.
+        _iv = data.get('ivNonce')
+        metadata.iv_nonce = binascii.a2b_base64(_iv) if _iv else None
+
         return metadata
 
     @staticmethod
@@ -100,7 +103,9 @@ def from_dict(data_dict):
         metadata.shared_key_enc = data_dict.get('sharedKeyEnc')
         metadata.pub_key_cs = data_dict.get('pubKeyCS')
         metadata.encoding = data_dict.get('encoding')
-        metadata.iv_nonce = data_dict.get('ivNonce')
+        # ivNonce travels as base64; keep raw bytes internally (see from_json).
+        _iv = data_dict.get('ivNonce')
+        metadata.iv_nonce = binascii.a2b_base64(_iv) if _iv else None
 
         return metadata
 

at_client/util/verbbuilder.py

@@ -181,6 +181,7 @@ def set_metadata(self, metadata):
         self.shared_key_enc = metadata.shared_key_enc
         self.pub_key_cs = metadata.pub_key_cs
         self.encoding = metadata.encoding
+        self.iv_nonce = metadata.iv_nonce
         return self
 
     def with_at_key(self, at_key, value):
@@ -228,6 +229,7 @@ def _build_metadata_str(self):
         metadata.shared_key_enc = self.shared_key_enc
         metadata.pub_key_cs = self.pub_key_cs
         metadata.encoding = self.encoding
+        metadata.iv_nonce = getattr(self, "iv_nonce", None)
         return str(metadata)
 
 class LlookupVerbBuilder:

test/interop/.gitignore

@@ -0,0 +1,2 @@
+# Dart tool-generated; regenerated by `dart pub get`. pubspec.lock IS committed.
+.dart_tool/

test/interop/README.md

@@ -0,0 +1,45 @@
+# Cross-SDK IV interop test
+
+`test/interop_test.py` verifies that this Python SDK and the Dart
+reference `at_client` interoperate for random-IV / `ivNonce` shared-key
+encryption, in both directions:
+
+- Python `put` shared -> Dart `get` shared (Python's random IV read by Dart)
+- Dart `put` shared -> Python `get` shared (Dart's random IV read by Python)
+
+This directory holds the Dart helper (`bin/iv_interop.dart` +
+`pubspec.yaml`) the test shells out to.
+
+## Guarded - off by default
+
+`interop_test.py` is discovered by the normal `unittest` run but **skips**
+unless `AT_INTEROP=1` and a Dart SDK is on PATH, so it never affects
+standard/fork CI.
+
+## Run locally
+
+Prerequisites: Dart SDK; an atServer reachable (e.g. the ephemeral
+environment); two onboarded atSigns with `.atKeys` under
+`$HOME/.atsign/keys`.
+
+```bash
+# EE example: @alpha and @bravo onboarded, keys in /tmp/eehome/.atsign/keys
+HOME=/tmp/eehome AT_INTEROP=1 \
+  AT_ROOT=vip.ve.atsign.zone:64 AT_ROOT_DOMAIN=vip.ve.atsign.zone \
+  python -m unittest discover -s test -p 'interop_test.py' -v
+```
+
+Env (all optional, EE-friendly defaults): `AT_INTEROP_ATSIGN1`
+(`@alpha`), `AT_INTEROP_ATSIGN2` (`@bravo`), `AT_ROOT`
+(`vip.ve.atsign.zone:64`), `AT_ROOT_DOMAIN` (host of `AT_ROOT`).
+
+> Re-running against a **recreated** EE? Clear the Dart client's local
+> storage first (`rm -rf $HOME/.atsign/storage`) - it caches keys from the
+> previous atServer and will otherwise fail to decrypt after re-onboarding.
+> A fresh CI runner never hits this.
+
+## CI
+
+An opt-in workflow is in `.github/workflows/interop.yml` (manual
+`workflow_dispatch`): it starts the ephemeral environment, onboards two
+atSigns, installs dependencies, and runs this test with `AT_INTEROP=1`.

test/interop/bin/iv_interop.dart

@@ -0,0 +1,83 @@
+// Interop helper using the Dart reference at_client. Put/get self and shared keys
+// so a Python peer (at_python fix/put-get-random-iv) can read/write them, proving
+// the Python IV/ivNonce behavior is wire-compatible with Dart.
+//
+//   dart run bin/iv_interop.dart --atsign @alpha --root-domain vip.ve.atsign.zone \
+//     --op put-shared --key demo --value hi --shared-with @bravo
+import 'dart:io';
+
+import 'package:at_client/at_client.dart';
+import 'package:at_cli_commons/at_cli_commons.dart';
+
+const String ns = 'itest';
+
+/// Dart at_client is local-first: put writes locally then syncs to the server, and
+/// get of self/own keys reads locally. For cross-SDK interop we must push after a
+/// put and pull before a get, so wait until the client is in sync with the server.
+Future<void> waitInSync(AtClient atClient, {int timeoutSec = 30}) async {
+  final sync = atClient.syncService;
+  for (var i = 0; i < timeoutSec; i++) {
+    sync.sync();
+    try {
+      if (await sync.isInSync()) return;
+    } catch (_) {}
+    await Future.delayed(const Duration(seconds: 1));
+  }
+}
+
+Future<void> main(List<String> args) async {
+  final parser = CLIBase.createArgsParser(namespace: ns)
+    ..addOption('op', mandatory: true)
+    ..addOption('key', mandatory: true)
+    ..addOption('value', defaultsTo: '')
+    ..addOption('shared-with');
+  final cli = await CLIBase.fromCommandLineArgs(args, parser: parser, namespace: ns);
+  final atClient = cli.atClient;
+  final me = atClient.getCurrentAtSign()!;
+  final a = parser.parse(args);
+  final op = a['op'] as String;
+  final keyName = a['key'] as String;
+  final value = a['value'] as String;
+  final sw = a['shared-with'] as String?;
+
+  switch (op) {
+    case 'put-self':
+      final k = AtKey()
+        ..key = keyName
+        ..namespace = ns
+        ..sharedBy = me;
+      await atClient.put(k, value);
+      await waitInSync(atClient); // push to server
+      stdout.writeln('OK');
+      break;
+    case 'get-self':
+      await waitInSync(atClient); // pull peer's writes
+      final k = AtKey()
+        ..key = keyName
+        ..namespace = ns
+        ..sharedBy = me;
+      final r = await atClient.get(k);
+      stdout.writeln('VALUE:${r.value}');
+      break;
+    case 'put-shared':
+      final k = AtKey()
+        ..key = keyName
+        ..namespace = ns
+        ..sharedBy = me
+        ..sharedWith = sw;
+      await atClient.put(k, value);
+      await waitInSync(atClient); // push to server
+      stdout.writeln('OK');
+      break;
+    case 'get-shared':
+      final k = AtKey()
+        ..key = keyName
+        ..namespace = ns
+        ..sharedBy = sw
+        ..sharedWith = me;
+      final r = await atClient.get(k);
+      stdout.writeln('VALUE:${r.value}');
+      break;
+  }
+  exit(0);
+}