Add HiSuite 14 PMS split AES-GCM support

README.md

@@ -9,6 +9,19 @@ The `kobackupdec` is a Python3 script aimed to decrypt Huawei *HiSuite* or *KoBa
 
 On 1.1.2021 the script will get its _end of life_ status. It was needed two years ago to overcome issues for some Huawei devices' forensics acquisitions. Now commercial forensics solutions include the very same capabilities, and much more: there are no more reasons to maintain it. We've got messages from guys using this script to manage theirs backups: we do not recommend it, and we did not write it for this reason. Anyhow we're happy some of you did find it useful, and we thank you for the feedback. We shared it to the community, trying to give back something: if someone has any interest in maintaining it, please let us know so we can include a link to the project. 
 
+
+## HiSuite PMS split application data
+
+This version also recognizes newer HiSuite/LocalBackup application-data
+packages stored as `<package>.tar.0`, `<package>.tar.1`, ... when the matching
+`BackupFileModuleInfo` entry has `isPmsSplit=true`.
+
+Support was validated against a HiSuite/LocalBackup 14.7.0.240 backup
+(`backupVersion=31`). Each split part uses PBKDF2-HMAC-SHA256 (10,000
+iterations, 32-byte key) and AES-256-GCM. The final 16 bytes of each part are
+the GCM authentication tag. With `-e`, the reconstructed TAR is extracted to
+`data/data/<package>/`; without `-e`, it is saved as `data/data/<package>.tar`.
+
 ## Usage
 
 The script *assumes* that backups are encrypted with a user-provided password. Actually it does not support the HiSuite _self_ generated password, when the user does not provide its own.

kobackupdec.py

@@ -51,8 +51,10 @@
 import os
 import os.path
 import pathlib
+import shutil
 import sys
 import tarfile
+import tempfile
 import xml.dom.minidom
 
 from Crypto.Cipher import AES
@@ -80,6 +82,7 @@ def __init__(self, type_name):
         self._path = None
         self._records_num = None
         self._copy_file_path = None
+        self._is_pms_split = False
 
     @property
     def type_name(self):
@@ -145,6 +148,14 @@ def path(self, value_string):
         else:
             logging.error('empty file path!')
 
+    @property
+    def is_pms_split(self):
+        return self._is_pms_split
+
+    @is_pms_split.setter
+    def is_pms_split(self, value_bool):
+        self._is_pms_split = bool(value_bool)
+
     def do_check(self):
         if self._name and (self._encMsgV3 or self._iv):
             return True
@@ -170,6 +181,8 @@ class Decryptor:
 
     count = 5000
     dklen = 32
+    pms_split_count = 10000
+    pms_split_tag_len = 16
     chunk_size = 1024*1024*64
 
     def __init__(self, password):
@@ -317,6 +330,32 @@ def decrypt_package(self, dec_material, data):
         decryptor = AES.new(key, mode=AES.MODE_CTR, counter=counter_obj)
         return decryptor.decrypt(data)
 
+    def decrypt_pms_split_part(self, dec_material, data):
+        '''Decrypt one HiSuite PMS split TAR part using AES-256-GCM.'''
+        if not self._good:
+            logging.warning('well, it is hard to decrypt with a wrong key.')
+        if not dec_material.encMsgV3:
+            logging.error('cannot decrypt PMS split with an empty encMsgV3!')
+            return None
+        if len(data) <= Decryptor.pms_split_tag_len:
+            logging.error('PMS split part is too short!')
+            return None
+
+        salt = dec_material.encMsgV3[:32]
+        nonce = dec_material.encMsgV3[32:]
+        key = PBKDF2(self._bkey, salt, Decryptor.dklen,
+                     Decryptor.pms_split_count, Decryptor.prf,
+                     hmac_hash_module=None)
+        ciphertext = data[:-Decryptor.pms_split_tag_len]
+        tag = data[-Decryptor.pms_split_tag_len:]
+        decryptor = AES.new(key, mode=AES.MODE_GCM, nonce=nonce,
+                            mac_len=Decryptor.pms_split_tag_len)
+        try:
+            return decryptor.decrypt_and_verify(ciphertext, tag)
+        except ValueError:
+            logging.error('PMS split GCM authentication failed!')
+            return None
+
     def decrypt_large_package(self, dec_material, entry):
         if not self._good:
             logging.warning('well, it is hard to decrypt with a wrong key.')
@@ -541,6 +580,8 @@ def xml_get_column_value(xml_node):
                 column_value = str(child.getAttribute('String'))
             elif child.hasAttribute('Integer'):
                 column_value = int(child.getAttribute('Integer'))
+            elif child.hasAttribute('Boolean'):
+                column_value = child.getAttribute('Boolean').lower() == 'true'
             elif child.hasAttribute('Null'):
                 column_value = None
             else:
@@ -578,6 +619,8 @@ def parse_backup_file_module_info(xml_entry):
             decm.name = xml_get_column_value(entry)
         elif tag_name == 'copyFilePath':
             decm.copy_file_path = xml_get_column_value(entry)
+        elif tag_name == 'isPmsSplit':
+            decm.is_pms_split = xml_get_column_value(entry)
         elif tag_name == 'checkMsgV3':
             # [TBR][TODO] Reverse this double sized checkMsgV3.
             pass
@@ -678,6 +721,100 @@ def tar_extract_win(tar_obj, dest_dir):
             except FileNotFoundError:
                 logging.warning('unable to extract %s', dest_file)
 
+
+# --- PMS split application TARs ---------------------------------------------
+
+def pms_split_package_name(entry):
+    "Return package name for <package>.tar.<number>, else None."
+    marker = '.tar.'
+    if marker not in entry.name:
+        return None
+    package_name, part_number = entry.name.rsplit(marker, 1)
+    if not package_name or not part_number.isdigit():
+        return None
+    return package_name
+
+
+def pms_split_part_number(entry):
+    "Return the numeric suffix of a PMS split TAR part."
+    return int(entry.name.rsplit('.tar.', 1)[1])
+
+
+def decrypt_pms_split_packages(decrypt_info, path_in, path_out, expandtar):
+    "Decrypt PMS split application TARs found anywhere below path_in."
+    data_app_dir = path_out.absolute().joinpath('data/data')
+    groups = {}
+
+    for entry in path_in.glob('**/*.tar.*'):
+        if not entry.is_file():
+            continue
+        package_name = pms_split_package_name(entry)
+        if package_name is None:
+            continue
+        dec_material = decrypt_info.get_decrypt_material(
+            package_name, DecryptInfo.info_type.FILE)
+        if not dec_material or not dec_material.is_pms_split:
+            continue
+        groups.setdefault((entry.parent, package_name), []).append(entry)
+
+    handled = set()
+    for (_, package_name), entries in groups.items():
+        entries.sort(key=pms_split_part_number)
+        part_numbers = [pms_split_part_number(entry) for entry in entries]
+        expected = list(range(len(entries)))
+        if part_numbers != expected:
+            logging.error(
+                'PMS split package %s has missing/out-of-order parts: %s',
+                package_name, part_numbers)
+            continue
+
+        dec_material = decrypt_info.get_decrypt_material(
+            package_name, DecryptInfo.info_type.FILE)
+        logging.info('Decrypting PMS split package %s (%d parts)',
+                     package_name, len(entries))
+
+        success = True
+        with tempfile.TemporaryFile() as clear_tar:
+            for entry in entries:
+                logging.debug('Decrypting PMS split part %s', entry)
+                cleartext = decrypt_info.decryptor.decrypt_pms_split_part(
+                    dec_material, entry.read_bytes())
+                if cleartext is None:
+                    logging.error('Unable to decrypt PMS split part %s', entry)
+                    success = False
+                    break
+                clear_tar.write(cleartext)
+
+            if not success:
+                continue
+
+            clear_tar.seek(0)
+            if expandtar:
+                dest_dir = data_app_dir.joinpath(package_name)
+                dest_dir.mkdir(0o755, parents=True, exist_ok=True)
+                try:
+                    with tarfile.open(fileobj=clear_tar, mode='r:*') as tar_data:
+                        if os.name == 'nt':
+                            tar_extract_win(tar_data, dest_dir)
+                        else:
+                            tar_data.extractall(path=dest_dir)
+                except tarfile.TarError:
+                    logging.error(
+                        'Decrypted PMS split package %s is not a TAR',
+                        package_name)
+                    continue
+            else:
+                dest_file = data_app_dir.joinpath(package_name + '.tar')
+                dest_file.parent.mkdir(0o755, parents=True, exist_ok=True)
+                with dest_file.open('wb') as output_fd:
+                    shutil.copyfileobj(clear_tar, output_fd)
+
+        handled.update(entries)
+        logging.info('PMS split package %s decrypted successfully',
+                     package_name)
+
+    return handled
+
 # --- decrypt_entry -----------------------------------------------------------
 
 def decrypt_entry(decrypt_info, entry, type_info, search=False):
@@ -707,7 +844,8 @@ def decrypt_large_entry(decrypt_info, entry, type_info, search=False):
 
 # --- decrypt_files_in_root ---------------------------------------------------
 
-def decrypt_files_in_root(decrypt_info, path_in, path_out, expandtar):
+def decrypt_files_in_root(decrypt_info, path_in, path_out, expandtar,
+                          handled=None):
 
     data_apk_dir = path_out.absolute().joinpath('data/app')
     data_app_dir = path_out.absolute().joinpath('data/data')
@@ -717,6 +855,8 @@ def decrypt_files_in_root(decrypt_info, path_in, path_out, expandtar):
     for entry in path_in.glob('*'):
         if entry.is_dir():
             continue
+        if handled and entry in handled:
+            continue
         cleartext = None
         extension = entry.suffix.lower()
 
@@ -777,7 +917,8 @@ def decrypt_files_in_root(decrypt_info, path_in, path_out, expandtar):
 
 # --- decrypt_files_in_folder -------------------------------------------------
 
-def decrypt_files_in_folder(decrypt_info, folder, path_out, expandtar):
+def decrypt_files_in_folder(decrypt_info, folder, path_out, expandtar,
+                            handled=None):
 
     folder_to_media_type = {'movies': 'video', 'pictures': 'photo',
                             'audios': 'audio', }
@@ -789,11 +930,16 @@ def decrypt_files_in_folder(decrypt_info, folder, path_out, expandtar):
     # needed to decrypt .enc files... Not tested for side effects.
     xml_files = folder.glob('*.xml')
     for entry in xml_files:
+        # info.xml is backup/module metadata, not a Multimedia XML file.
+        if entry.name.lower() == 'info.xml':
+            continue
         parse_generic_xml(entry, decrypt_info)
 
     for entry in folder.glob('**/*'):
         if entry.is_dir():
             continue
+        if handled and entry in handled:
+            continue
 
         logging.info('working on [%s]', entry.name)
         extension = entry.suffix.lower()
@@ -882,11 +1028,16 @@ def decrypt_backup(password, path_in, path_out, expandtar):
 
     logging.debug(decrypt_info.dump())
 
-    decrypt_files_in_root(decrypt_info, path_in, path_out, expandtar)
+    handled_pms = decrypt_pms_split_packages(decrypt_info, path_in, path_out,
+                                             expandtar)
+
+    decrypt_files_in_root(decrypt_info, path_in, path_out, expandtar,
+                          handled_pms)
 
     for entry in path_in.glob('*'):
         if entry.is_dir():
-            decrypt_files_in_folder(decrypt_info, entry, path_out, expandtar)
+            decrypt_files_in_folder(decrypt_info, entry, path_out, expandtar,
+                                    handled_pms)
 
 # --- decrypt_media -----------------------------------------------------------
 

tests/test_pms_split.py

@@ -0,0 +1,91 @@
+import io
+import tarfile
+import unittest
+
+from Crypto.Cipher import AES
+from Crypto.Hash import SHA256
+from Crypto.Hash import HMAC
+from Crypto.Protocol.KDF import PBKDF2
+
+import kobackupdec
+
+
+class PmsSplitTests(unittest.TestCase):
+
+    PASSWORD = b'test-password-1234-AA'
+    SALT = bytes(range(32))
+    NONCE = bytes(range(16))
+
+    @staticmethod
+    def _prf(password, salt):
+        return HMAC.new(password, salt, SHA256).digest()
+
+    def _encrypt_part(self, cleartext):
+        key = PBKDF2(self.PASSWORD, self.SALT, 32, 10000,
+                     self._prf, hmac_hash_module=None)
+        cipher = AES.new(key, AES.MODE_GCM, nonce=self.NONCE, mac_len=16)
+        ciphertext, tag = cipher.encrypt_and_digest(cleartext)
+        return ciphertext + tag
+
+    def _make_tar(self):
+        output = io.BytesIO()
+        with tarfile.open(fileobj=output, mode='w') as tar:
+            payloads = {
+                'cache/example.txt': b'cache data',
+                'files/games/com.mojang/minecraftWorlds/test/level.dat':
+                    b'level data',
+            }
+            for name, data in payloads.items():
+                info = tarfile.TarInfo(name)
+                info.size = len(data)
+                tar.addfile(info, io.BytesIO(data))
+        return output.getvalue()
+
+    def _material(self):
+        material = kobackupdec.DecryptMaterial('BackupFileModuleInfo')
+        material.name = 'com.example.app'
+        material.encMsgV3 = (self.SALT + self.NONCE).hex()
+        material.is_pms_split = True
+        return material
+
+    def _decryptor(self, password=None):
+        decryptor = kobackupdec.Decryptor(password or self.PASSWORD)
+        decryptor.type_attch = 3
+        decryptor.crypto_init()
+        return decryptor
+
+    def test_decrypt_pms_split_parts_reconstructs_tar(self):
+        clear_tar = self._make_tar()
+        split = len(clear_tar) // 3
+        clear_parts = [
+            clear_tar[:split],
+            clear_tar[split:2 * split],
+            clear_tar[2 * split:],
+        ]
+        encrypted_parts = [self._encrypt_part(part) for part in clear_parts]
+
+        decryptor = self._decryptor()
+        material = self._material()
+        reconstructed = b''.join(
+            decryptor.decrypt_pms_split_part(material, part)
+            for part in encrypted_parts)
+
+        self.assertEqual(clear_tar, reconstructed)
+        with tarfile.open(fileobj=io.BytesIO(reconstructed), mode='r:*') as tar:
+            self.assertIn('cache/example.txt', tar.getnames())
+
+    def test_wrong_password_fails_gcm_authentication(self):
+        encrypted = self._encrypt_part(b'test payload')
+        decryptor = self._decryptor(b'wrong-password')
+        self.assertIsNone(
+            decryptor.decrypt_pms_split_part(self._material(), encrypted))
+
+    def test_boolean_xml_value(self):
+        document = kobackupdec.xml.dom.minidom.parseString(
+            '<column><value Boolean="true" /></column>')
+        self.assertTrue(
+            kobackupdec.xml_get_column_value(document.documentElement))
+
+
+if __name__ == '__main__':
+    unittest.main()