Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions docker_squash/v2_image.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import gzip
import hashlib
import json
import os
Expand Down Expand Up @@ -52,6 +53,11 @@ def _squash(self):
self.diff_ids = self._generate_diff_ids()
self.chain_ids = self._generate_chain_ids(self.diff_ids)

if self.layer_paths_to_squash:
# Must run after the diff ID is computed: the diff ID is the digest
# of the uncompressed tar.
self._compress_squashed_layer()

metadata = self._generate_image_metadata()
image_id = self._write_image_metadata(metadata)

Expand Down Expand Up @@ -239,6 +245,26 @@ def _generate_diff_ids(self):

return diff_ids

def _compress_squashed_layer(self):
"""
Replace the squashed layer.tar with its gzip.

The containerd image store keeps a loaded layer as given and pushes it
as is, so a raw tar is pushed uncompressed. The daemon derives the
compressed digest itself and the config's diff ID still refers to the
uncompressed tar, so no other metadata changes. The mtime is fixed to
keep the output reproducible.
"""
compressed = self.squashed_tar + ".gz"

with open(self.squashed_tar, "rb") as src, open(compressed, "wb") as raw:
with gzip.GzipFile(
filename="", mode="wb", fileobj=raw, compresslevel=9, mtime=0
) as dst:
shutil.copyfileobj(src, dst)

os.replace(compressed, self.squashed_tar)

def _compute_sha256(self, layer_tar):
sha256 = hashlib.sha256()

Expand Down
31 changes: 31 additions & 0 deletions tests/test_integ_squash.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
import codecs
import gzip
import hashlib
import json
import logging
import os
import re
import shutil
import tarfile
import tempfile
import unittest
import uuid
from io import BytesIO
Expand Down Expand Up @@ -746,6 +749,34 @@ def test_should_squash_every_layer(self):
with self.SquashedImage(image) as squashed_image:
self.assertEqual(len(squashed_image.layers), 1)

# https://github.com/goldmann/docker-squash/issues/260
def test_squashed_layer_is_gzipped_and_keeps_uncompressed_diff_id(self):
dockerfile = """
FROM %s
RUN touch /tmp/test1
RUN touch /tmp/test2
""" % TestIntegSquash.BUSYBOX_IMAGE

out_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, out_dir, True)
out_tar = os.path.join(out_dir, "squashed.tar")

with self.Image(dockerfile) as image:
with self.SquashedImage(image, output_path=out_tar):
with tarfile.open(out_tar, mode="r") as tar:
manifest = json.load(tar.extractfile("manifest.json"))[0]
config = json.load(tar.extractfile(manifest["Config"]))
layer = tar.extractfile(manifest["Layers"][-1]).read()

self.assertEqual(layer[:2], b"\x1f\x8b")
uncompressed = gzip.decompress(layer)
self.assertEqual(
config["rootfs"]["diff_ids"][-1],
"sha256:%s" % hashlib.sha256(uncompressed).hexdigest(),
)
with tarfile.open(fileobj=BytesIO(uncompressed)) as layer_tar:
self.assertIn("tmp/test1", layer_tar.getnames())

# https://github.com/goldmann/docker-scripts/issues/44
def test_remove_tmp_dir_after_failure(self):
self.caplog.set_level(logging.DEBUG, logger="cekit")
Expand Down
37 changes: 37 additions & 0 deletions tests/test_unit_load_image.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import gzip
import os
import shutil
import tempfile
Expand Down Expand Up @@ -82,3 +83,39 @@ def test_moves_legacy_layer_directories(self):
self.image._move_layers(["abc"], self.src, self.dest)

self.assertTrue(os.path.isfile(os.path.join(self.dest, "abc", "layer.tar")))


class TestCompressSquashedLayer(unittest.TestCase):
def setUp(self):
self.root = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, self.root, True)
self.image = V2Image(mock.Mock(), mock.Mock(), "whatever", None)
self.image.squashed_tar = os.path.join(self.root, "layer.tar")
self.content = b"a" * 100000

with open(self.image.squashed_tar, "wb") as f:
f.write(self.content)

def test_replaces_layer_with_its_gzip(self):
self.image._compress_squashed_layer()

with open(self.image.squashed_tar, "rb") as f:
data = f.read()

self.assertEqual(data[:2], b"\x1f\x8b")
self.assertEqual(gzip.decompress(data), self.content)
self.assertLess(len(data), len(self.content))
self.assertEqual(os.listdir(self.root), ["layer.tar"])

def test_output_is_reproducible(self):
self.image._compress_squashed_layer()
with open(self.image.squashed_tar, "rb") as f:
first = f.read()

with open(self.image.squashed_tar, "wb") as f:
f.write(self.content)
self.image._compress_squashed_layer()
with open(self.image.squashed_tar, "rb") as f:
second = f.read()

self.assertEqual(first, second)
Loading