diff --git a/src/borg/archive.py b/src/borg/archive.py index 4af2d47f18..f6d53f86b7 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -2075,6 +2075,9 @@ def rebuild_archives( ): """Analyze and rebuild archives, expecting some damage and trying to make stuff consistent again.""" + missing_chunk_size: dict = {} # chunk_id -> chunk size in bytes + missing_chunk_refs: defaultdict = defaultdict(lambda: defaultdict(set)) # chunk_id -> {path: {archive_name}} + def add_callback(chunk): id_ = self.key.id_hash(chunk) cdata = self.repo_objs.format(id_, {}, chunk, ro_type=ROBJ_ARCHIVE_STREAM) @@ -2091,16 +2094,22 @@ def add_reference(id_, size, cdata): self.chunks.update_pack_info(pack_results) def verify_file_chunks(archive_name, item): - """Verifies that all file chunks are present. Missing file chunks will be logged.""" + """Verify that all file chunks are present. + + Record each missing chunk's size in missing_chunk_size and, in missing_chunk_refs, the + file path and archive it occurs in. Log each missing chunk at debug level. + """ offset = 0 for chunk in item.chunks: chunk_id, size = chunk if chunk_id not in self.chunks: - logger.error( + logger.debug( "{}: {}: Missing file chunk detected (Byte {}-{}, Chunk {}).".format( archive_name, item.path, offset, offset + size, bin_to_hex(chunk_id) ) ) + missing_chunk_size[chunk_id] = size + missing_chunk_refs[chunk_id][item.path].add(archive_name) self.error_found = True offset += size if "size" in item: @@ -2114,6 +2123,17 @@ def verify_file_chunks(archive_name, item): ) ) + def report_missing_chunks(): + """Log the missing chunks, each with its size and the files and archives referencing it.""" + if not missing_chunk_refs: + return + logger.error("The following chunks are missing in the repository:") + for chunk_id, refs in missing_chunk_refs.items(): + logger.error(f"- Chunk {bin_to_hex(chunk_id)}, {missing_chunk_size[chunk_id]:,} bytes") + for path in sorted(refs): + archive_names = ", ".join(sorted(refs[path])) + logger.error(f" - {path}: {archive_names}") + def robust_iterator(archive): """Iterates through all archive items @@ -2271,6 +2291,7 @@ def valid_item(obj): if archive_id != new_archive_id: self.manifest.archives.delete_by_id(archive_id) pi.finish() + report_missing_chunks() def finish(self): if self.repair: diff --git a/src/borg/testsuite/archiver/check_cmd_test.py b/src/borg/testsuite/archiver/check_cmd_test.py index 1e91ea84ba..57bf103e41 100644 --- a/src/borg/testsuite/archiver/check_cmd_test.py +++ b/src/borg/testsuite/archiver/check_cmd_test.py @@ -169,9 +169,13 @@ def test_missing_file_chunk(archivers, request): pytest.fail("should not happen") # convert 'fail' output = cmd(archiver, "check", exit_code=1) - assert "Missing file chunk detected" in output + assert "The following chunks are missing in the repository:" in output + assert bin_to_hex(killed_chunk.id) in output + assert src_file in output output = cmd(archiver, "check", "--repair", exit_code=0) - assert "Missing file chunk detected" in output # repair is not changing anything, just reporting. + # repair is not changing anything, just reporting. + assert "The following chunks are missing in the repository:" in output + assert bin_to_hex(killed_chunk.id) in output # check does not modify the chunks list. for archive_name in ("archive1", "archive2"): @@ -191,7 +195,7 @@ def test_missing_file_chunk(archivers, request): # check should not complain anymore about missing chunks: output = cmd(archiver, "check", "-v", "--repair", exit_code=0) - assert "Missing file chunk detected" not in output + assert "The following chunks are missing in the repository:" not in output def test_missing_archive_item_chunk(archivers, request): @@ -474,11 +478,14 @@ def test_verify_data(archivers, request, init_args): # repair will find the defect chunk and remove it output = cmd(archiver, "check", "--repair", "--verify-data", exit_code=0) assert f"{bin_to_hex(chunk.id)}, integrity error" in output - assert f"{src_file}: Missing file chunk detected" in output + assert "The following chunks are missing in the repository:" in output + assert bin_to_hex(chunk.id) in output + assert src_file in output # run with --verify-data again, it will notice the missing chunk. output = cmd(archiver, "check", "--archives-only", "--verify-data", exit_code=1) - assert f"{src_file}: Missing file chunk detected" in output + assert "The following chunks are missing in the repository:" in output + assert bin_to_hex(chunk.id) in output @pytest.mark.parametrize("init_args", [["--encryption=aes256-ocb"], ["--encryption", "none"]]) @@ -506,12 +513,15 @@ def test_corrupted_file_chunk(archivers, request, init_args): # repair: the defect chunk will be removed. output = cmd(archiver, "check", "--repair", "--verify-data", exit_code=0) assert f"{bin_to_hex(chunk.id)}, integrity error" in output - assert f"{src_file}: Missing file chunk detected" in output + assert "The following chunks are missing in the repository:" in output + assert bin_to_hex(chunk.id) in output + assert src_file in output # run normal check again cmd(archiver, "check", "--repository-only", exit_code=0) output = cmd(archiver, "check", "--archives-only", exit_code=1) - assert f"{src_file}: Missing file chunk detected" in output + assert "The following chunks are missing in the repository:" in output + assert src_file in output @pytest.mark.skip(