Skip to content

Commit 4c0c626

Browse files
smirasmira
committed
feat: use zstd compression in place of xz
Initramfs and kernel are compressed with zstd. Extensions are compressed with zstd for Talos 1.8+. Signed-off-by: Andrey Smirnov <andrey.smirnov@siderolabs.com>
1 parent 98906ed commit 4c0c626

20 files changed

Lines changed: 202 additions & 297 deletions

File tree

‎.drone.jsonnet‎

Lines changed: 28 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -368,17 +368,10 @@ local ExtensionsStep(with_e2e=true) =
368368

369369
// generates the extension list patch manifest
370370
local extensions_patch_manifest = Step(
371-
'extensions-patch-manifest',
372-
with_make=false,
373-
environment=creds_env_vars,
371+
'installer-with-extensions',
372+
environment={ IMAGE_REGISTRY: local_registry },
374373
depends_on=[
375374
extensions_artifacts,
376-
],
377-
extra_commands=[
378-
// create a patch file to pass to the downstream build
379-
// ignore nvidia extensions, testing nvidia extensions needs a machine with nvidia graphics card
380-
'jq -R < _out/extensions-metadata | jq -s -f hack/test/extensions/extension-patch-filter.jq | yq eval ".[] | split_doc" -P > _out/extensions-patch.yaml',
381-
'cat _out/extensions-patch.yaml',
382375
]
383376
);
384377

@@ -1056,35 +1049,35 @@ local release = {
10561049
files: [
10571050
'_out/akamai-amd64.raw.gz',
10581051
'_out/akamai-arm64.raw.gz',
1059-
'_out/aws-amd64.raw.xz',
1060-
'_out/aws-arm64.raw.xz',
1061-
'_out/azure-amd64.vhd.xz',
1062-
'_out/azure-arm64.vhd.xz',
1052+
'_out/aws-amd64.raw.zst',
1053+
'_out/aws-arm64.raw.zst',
1054+
'_out/azure-amd64.vhd.zst',
1055+
'_out/azure-arm64.vhd.zst',
10631056
'_out/cloud-images.json',
10641057
'_out/digital-ocean-amd64.raw.gz',
10651058
'_out/digital-ocean-arm64.raw.gz',
1066-
'_out/exoscale-amd64.qcow2.xz',
1067-
'_out/exoscale-arm64.qcow2.xz',
1059+
'_out/exoscale-amd64.qcow2.zst',
1060+
'_out/exoscale-arm64.qcow2.zst',
10681061
'_out/gcp-amd64.raw.tar.gz',
10691062
'_out/gcp-arm64.raw.tar.gz',
1070-
'_out/hcloud-amd64.raw.xz',
1071-
'_out/hcloud-arm64.raw.xz',
1072-
'_out/initramfs-amd64.xz',
1073-
'_out/initramfs-arm64.xz',
1063+
'_out/hcloud-amd64.raw.zst',
1064+
'_out/hcloud-arm64.raw.zst',
1065+
'_out/initramfs-amd64.zst',
1066+
'_out/initramfs-arm64.zst',
10741067
'_out/metal-amd64.iso',
10751068
'_out/metal-arm64.iso',
1076-
'_out/metal-amd64.raw.xz',
1077-
'_out/metal-arm64.raw.xz',
1078-
'_out/nocloud-amd64.raw.xz',
1079-
'_out/nocloud-arm64.raw.xz',
1080-
'_out/opennebula-amd64.raw.xz',
1081-
'_out/opennebula-arm64.raw.xz',
1082-
'_out/openstack-amd64.raw.xz',
1083-
'_out/openstack-arm64.raw.xz',
1084-
'_out/oracle-amd64.qcow2.xz',
1085-
'_out/oracle-arm64.qcow2.xz',
1086-
'_out/scaleway-amd64.raw.xz',
1087-
'_out/scaleway-arm64.raw.xz',
1069+
'_out/metal-amd64.raw.zst',
1070+
'_out/metal-arm64.raw.zst',
1071+
'_out/nocloud-amd64.raw.zst',
1072+
'_out/nocloud-arm64.raw.zst',
1073+
'_out/opennebula-amd64.raw.zst',
1074+
'_out/opennebula-arm64.raw.zst',
1075+
'_out/openstack-amd64.raw.zst',
1076+
'_out/openstack-arm64.raw.zst',
1077+
'_out/oracle-amd64.qcow2.zst',
1078+
'_out/oracle-arm64.qcow2.zst',
1079+
'_out/scaleway-amd64.raw.zst',
1080+
'_out/scaleway-arm64.raw.zst',
10881081
'_out/sd-boot-amd64.efi',
10891082
'_out/sd-boot-arm64.efi',
10901083
'_out/sd-stub-amd64.efi',
@@ -1099,14 +1092,14 @@ local release = {
10991092
'_out/talosctl-linux-arm64',
11001093
'_out/talosctl-linux-armv7',
11011094
'_out/talosctl-windows-amd64.exe',
1102-
'_out/upcloud-amd64.raw.xz',
1103-
'_out/upcloud-arm64.raw.xz',
1095+
'_out/upcloud-amd64.raw.zst',
1096+
'_out/upcloud-arm64.raw.zst',
11041097
'_out/vmware-amd64.ova',
11051098
'_out/vmware-arm64.ova',
11061099
'_out/vmlinuz-amd64',
11071100
'_out/vmlinuz-arm64',
1108-
'_out/vultr-amd64.raw.xz',
1109-
'_out/vultr-arm64.raw.xz',
1101+
'_out/vultr-amd64.raw.zst',
1102+
'_out/vultr-arm64.raw.zst',
11101103
],
11111104
checksum: ['sha256', 'sha512'],
11121105
},

‎Dockerfile‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -688,14 +688,16 @@ RUN find /rootfs -print0 \
688688
| xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}"
689689

690690
FROM rootfs-base-arm64 AS rootfs-squashfs-arm64
691+
ARG ZSTD_COMPRESSION_LEVEL
691692
RUN find /rootfs -print0 \
692693
| xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}"
693-
RUN mksquashfs /rootfs /rootfs.sqsh -all-root -noappend -comp xz -Xdict-size 100% -no-progress
694+
RUN mksquashfs /rootfs /rootfs.sqsh -all-root -noappend -comp zstd -Xcompression-level ${ZSTD_COMPRESSION_LEVEL} -no-progress
694695

695696
FROM rootfs-base-amd64 AS rootfs-squashfs-amd64
697+
ARG ZSTD_COMPRESSION_LEVEL
696698
RUN find /rootfs -print0 \
697699
| xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}"
698-
RUN mksquashfs /rootfs /rootfs.sqsh -all-root -noappend -comp xz -Xdict-size 100% -no-progress
700+
RUN mksquashfs /rootfs /rootfs.sqsh -all-root -noappend -comp zstd -Xcompression-level ${ZSTD_COMPRESSION_LEVEL} -no-progress
699701

700702
FROM scratch AS squashfs-arm64
701703
COPY --from=rootfs-squashfs-arm64 /rootfs.sqsh /
@@ -710,6 +712,7 @@ COPY --from=rootfs-base /rootfs /
710712

711713
FROM build AS initramfs-archive-arm64
712714
WORKDIR /initramfs
715+
ARG ZSTD_COMPRESSION_LEVEL
713716
COPY --from=squashfs-arm64 /rootfs.sqsh .
714717
COPY --from=init-build-arm64 /init .
715718
RUN find . -print0 \
@@ -718,11 +721,12 @@ RUN set -o pipefail \
718721
&& find . 2>/dev/null \
719722
| LC_ALL=c sort \
720723
| cpio --reproducible -H newc -o \
721-
| xz -v -C crc32 -0 -e -T 0 -z \
724+
| zstd -c -T0 -${ZSTD_COMPRESSION_LEVEL} \
722725
> /initramfs.xz
723726

724727
FROM build AS initramfs-archive-amd64
725728
WORKDIR /initramfs
729+
ARG ZSTD_COMPRESSION_LEVEL
726730
COPY --from=squashfs-amd64 /rootfs.sqsh .
727731
COPY --from=init-build-amd64 /init .
728732
RUN find . -print0 \
@@ -731,7 +735,7 @@ RUN set -o pipefail \
731735
&& find . 2>/dev/null \
732736
| LC_ALL=c sort \
733737
| cpio --reproducible -H newc -o \
734-
| xz -v -C crc32 -0 -e -T 0 -z \
738+
| zstd -c -T0 -${ZSTD_COMPRESSION_LEVEL} \
735739
> /initramfs.xz
736740

737741
FROM initramfs-archive-${TARGETARCH} AS initramfs-archive
@@ -800,7 +804,8 @@ RUN apk add --no-cache --update --no-scripts \
800804
util-linux \
801805
xfsprogs \
802806
xorriso \
803-
xz
807+
xz \
808+
zstd
804809
ARG TARGETARCH
805810
ENV TARGETARCH ${TARGETARCH}
806811
COPY --from=installer-build /installer /bin/installer

‎Makefile‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,13 @@ DOCKER_LOGIN_ENABLED ?= true
1313
NAME = Talos
1414

1515
CLOUD_IMAGES_EXTRA_ARGS ?= ""
16+
ZSTD_COMPRESSION_LEVEL ?= 18
1617

1718
ARTIFACTS := _out
1819
TOOLS ?= ghcr.io/siderolabs/tools:v1.8.0-alpha.0
1920

2021
PKGS_PREFIX ?= ghcr.io/siderolabs
21-
PKGS ?= v1.8.0-alpha.0-7-g718a7da
22+
PKGS ?= v1.8.0-alpha.0-8-gca6249b
2223
EXTRAS ?= v1.8.0-alpha.0
2324

2425
PKG_FHS ?= $(PKGS_PREFIX)/fhs:$(PKGS)
@@ -202,6 +203,7 @@ COMMON_ARGS += --build-arg=PKG_RASPBERYPI_FIRMWARE=$(PKG_RASPBERYPI_FIRMWARE)
202203
COMMON_ARGS += --build-arg=PKG_KERNEL=$(PKG_KERNEL)
203204
COMMON_ARGS += --build-arg=PKG_TALOSCTL_CNI_BUNDLE_INSTALL=$(PKG_TALOSCTL_CNI_BUNDLE_INSTALL)
204205
COMMON_ARGS += --build-arg=ABBREV_TAG=$(ABBREV_TAG)
206+
COMMON_ARGS += --build-arg=ZSTD_COMPRESSION_LEVEL=$(ZSTD_COMPRESSION_LEVEL)
205207

206208
CI_ARGS ?=
207209

@@ -524,6 +526,12 @@ provision-tests-track-%:
524526
REGISTRY=$(IMAGE_REGISTRY) \
525527
ARTIFACTS=$(ARTIFACTS)
526528

529+
installer-with-extensions: $(ARTIFACTS)/extensions-metadata
530+
$(MAKE) image-installer \
531+
IMAGER_ARGS="--base-installer-image=$(REGISTRY_AND_USERNAME)/installer:$(IMAGE_TAG) $(shell cat $(ARTIFACTS)/extensions-metadata | grep -vE 'tailscale|xen-guest-agent|nvidia' | xargs -n 1 echo --system-extension-image)"
532+
crane push $(ARTIFACTS)/installer-amd64.tar $(REGISTRY_AND_USERNAME)/installer:$(IMAGE_TAG)-amd64-extensions
533+
echo -n "$(REGISTRY_AND_USERNAME)/installer:$(IMAGE_TAG)-amd64-extensions" | jq -Rs -f hack/test/extensions/extension-patch-filter.jq | yq eval ".[] | split_doc" -P > $(ARTIFACTS)/extensions-patch.yaml
534+
527535
# Assets for releases
528536

529537
.PHONY: $(ARTIFACTS)/$(TALOS_RELEASE)

‎cmd/installer/pkg/install/extensions.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111

1212
"github.com/siderolabs/talos/pkg/imager/extensions"
1313
"github.com/siderolabs/talos/pkg/machinery/constants"
14+
"github.com/siderolabs/talos/pkg/machinery/imager/quirks"
1415
)
1516

1617
func (i *Installer) installExtensions() error {
@@ -19,6 +20,7 @@ func (i *Installer) installExtensions() error {
1920
Arch: i.options.Arch,
2021
ExtensionTreePath: constants.SystemExtensionsPath,
2122
Printf: log.Printf,
23+
Quirks: quirks.New(i.options.Version),
2224
}
2325

2426
return builder.Build()

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ require (
9292
github.com/jeromer/syslogparser v1.1.0
9393
github.com/jsimonetti/rtnetlink v1.4.1
9494
github.com/jxskiss/base62 v1.1.0
95+
github.com/klauspost/compress v1.17.7
9596
github.com/klauspost/cpuid/v2 v2.2.7
9697
github.com/linode/go-metadata v0.2.0
9798
github.com/martinlindhe/base36 v1.1.1
@@ -267,7 +268,6 @@ require (
267268
github.com/josharian/intern v1.0.0 // indirect
268269
github.com/josharian/native v1.1.0 // indirect
269270
github.com/json-iterator/go v1.1.12 // indirect
270-
github.com/klauspost/compress v1.17.7 // indirect
271271
github.com/kylelemons/godebug v1.1.0 // indirect
272272
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
273273
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect

0 commit comments

Comments
 (0)