diff --git a/docker_squash/v2_image.py b/docker_squash/v2_image.py index 861b0bd..9eb1d31 100644 --- a/docker_squash/v2_image.py +++ b/docker_squash/v2_image.py @@ -1,3 +1,4 @@ +import gzip import hashlib import json import os @@ -52,6 +53,11 @@ def _squash(self): self.diff_ids = self._generate_diff_ids() self.chain_ids = self._generate_chain_ids(self.diff_ids) + if self.layer_paths_to_squash: + # Must run after the diff ID is computed: the diff ID is the digest + # of the uncompressed tar. + self._compress_squashed_layer() + metadata = self._generate_image_metadata() image_id = self._write_image_metadata(metadata) @@ -239,6 +245,26 @@ def _generate_diff_ids(self): return diff_ids + def _compress_squashed_layer(self): + """ + Replace the squashed layer.tar with its gzip. + + The containerd image store keeps a loaded layer as given and pushes it + as is, so a raw tar is pushed uncompressed. The daemon derives the + compressed digest itself and the config's diff ID still refers to the + uncompressed tar, so no other metadata changes. The mtime is fixed to + keep the output reproducible. + """ + compressed = self.squashed_tar + ".gz" + + with open(self.squashed_tar, "rb") as src, open(compressed, "wb") as raw: + with gzip.GzipFile( + filename="", mode="wb", fileobj=raw, compresslevel=9, mtime=0 + ) as dst: + shutil.copyfileobj(src, dst) + + os.replace(compressed, self.squashed_tar) + def _compute_sha256(self, layer_tar): sha256 = hashlib.sha256() diff --git a/tests/test_integ_squash.py b/tests/test_integ_squash.py index eda83f4..25a41f8 100644 --- a/tests/test_integ_squash.py +++ b/tests/test_integ_squash.py @@ -1,10 +1,13 @@ import codecs +import gzip +import hashlib import json import logging import os import re import shutil import tarfile +import tempfile import unittest import uuid from io import BytesIO @@ -746,6 +749,34 @@ def test_should_squash_every_layer(self): with self.SquashedImage(image) as squashed_image: self.assertEqual(len(squashed_image.layers), 1) + # https://github.com/goldmann/docker-squash/issues/260 + def test_squashed_layer_is_gzipped_and_keeps_uncompressed_diff_id(self): + dockerfile = """ + FROM %s + RUN touch /tmp/test1 + RUN touch /tmp/test2 + """ % TestIntegSquash.BUSYBOX_IMAGE + + out_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, out_dir, True) + out_tar = os.path.join(out_dir, "squashed.tar") + + with self.Image(dockerfile) as image: + with self.SquashedImage(image, output_path=out_tar): + with tarfile.open(out_tar, mode="r") as tar: + manifest = json.load(tar.extractfile("manifest.json"))[0] + config = json.load(tar.extractfile(manifest["Config"])) + layer = tar.extractfile(manifest["Layers"][-1]).read() + + self.assertEqual(layer[:2], b"\x1f\x8b") + uncompressed = gzip.decompress(layer) + self.assertEqual( + config["rootfs"]["diff_ids"][-1], + "sha256:%s" % hashlib.sha256(uncompressed).hexdigest(), + ) + with tarfile.open(fileobj=BytesIO(uncompressed)) as layer_tar: + self.assertIn("tmp/test1", layer_tar.getnames()) + # https://github.com/goldmann/docker-scripts/issues/44 def test_remove_tmp_dir_after_failure(self): self.caplog.set_level(logging.DEBUG, logger="cekit") diff --git a/tests/test_unit_load_image.py b/tests/test_unit_load_image.py index 8b26fb0..c98cdae 100644 --- a/tests/test_unit_load_image.py +++ b/tests/test_unit_load_image.py @@ -1,3 +1,4 @@ +import gzip import os import shutil import tempfile @@ -82,3 +83,39 @@ def test_moves_legacy_layer_directories(self): self.image._move_layers(["abc"], self.src, self.dest) self.assertTrue(os.path.isfile(os.path.join(self.dest, "abc", "layer.tar"))) + + +class TestCompressSquashedLayer(unittest.TestCase): + def setUp(self): + self.root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.root, True) + self.image = V2Image(mock.Mock(), mock.Mock(), "whatever", None) + self.image.squashed_tar = os.path.join(self.root, "layer.tar") + self.content = b"a" * 100000 + + with open(self.image.squashed_tar, "wb") as f: + f.write(self.content) + + def test_replaces_layer_with_its_gzip(self): + self.image._compress_squashed_layer() + + with open(self.image.squashed_tar, "rb") as f: + data = f.read() + + self.assertEqual(data[:2], b"\x1f\x8b") + self.assertEqual(gzip.decompress(data), self.content) + self.assertLess(len(data), len(self.content)) + self.assertEqual(os.listdir(self.root), ["layer.tar"]) + + def test_output_is_reproducible(self): + self.image._compress_squashed_layer() + with open(self.image.squashed_tar, "rb") as f: + first = f.read() + + with open(self.image.squashed_tar, "wb") as f: + f.write(self.content) + self.image._compress_squashed_layer() + with open(self.image.squashed_tar, "rb") as f: + second = f.read() + + self.assertEqual(first, second)