flatpak-build-bundle: Add --oci-layer-compress=zlib

Add an option to build OCI bundles with zstd compressed layers.

gzip is kept as the default for maximum compatibility:

Ecosystem support:

 distribution/distribution: no explicit support, but works
 quay.io: sinc 2021
 Amazon ECR: supported
 pulp_container: since 2022
 flatpak: since first-OCI supporting version
 tardiff: since first version

app/flatpak-builtins-build-bundle.c

@@ -49,6 +49,7 @@ static gboolean opt_runtime = FALSE;
 static char **opt_gpg_file;
 static gboolean opt_oci = FALSE;
 static gboolean opt_oci_use_labels = TRUE; // Unused now
+static char *opt_oci_layer_compress;
 static char **opt_gpg_key_ids;
 static char *opt_gpg_homedir;
 static char *opt_from_commit;
@@ -65,6 +66,7 @@ static GOptionEntry options[] = {
  { "oci", 0, 0, G_OPTION_ARG_NONE, &opt_oci, N_("Export oci image instead of flatpak bundle"), NULL },
  // This is not used anymore as it is the default, but accept it if old code uses it
  { "oci-use-labels", 0, G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_NONE, &opt_oci_use_labels, NULL, NULL },
+ { "oci-layer-compress", 0, 0, G_OPTION_ARG_STRING, &opt_oci_layer_compress, N_("How to compress OCI image layers (default: gzip)"), "gzip|zstd" },
  { NULL }
 };
 
@@ -455,6 +457,7 @@ generate_labels (FlatpakOciDescriptor *layer_desc,
 static gboolean
 build_oci (OstreeRepo *repo, const char *commit_checksum, GFile *dir,
  const char *name, const char *ref_str,
+ FlatpakOciWriteLayerFlags write_layer_flags,
  GCancellable *cancellable, GError **error)
 {
  g_autoptr(GFile) root = NULL;
@@ -498,7 +501,7 @@ build_oci (OstreeRepo *repo, const char *commit_checksum, GFile *dir,
  if (registry == NULL)
  return FALSE;
 
- layer_writer = flatpak_oci_registry_write_layer (registry, cancellable, error);
+ layer_writer = flatpak_oci_registry_write_layer (registry, write_layer_flags, cancellable, error);
  if (layer_writer == NULL)
  return FALSE;
 
@@ -678,7 +681,19 @@ flatpak_builtin_build_bundle (int argc, char **argv, GCancellable *cancellable,
 
  if (opt_oci)
  {
- if (!build_oci (repo, commit_checksum, file, name, full_branch, cancellable, error))
+ FlatpakOciWriteLayerFlags write_layer_flags;
+
+ if (opt_oci_layer_compress == NULL)
+ opt_oci_layer_compress = "gzip";
+
+ if (strcmp(opt_oci_layer_compress, "gzip") == 0)
+ write_layer_flags = FLATPAK_OCI_WRITE_LAYER_FLAGS_NONE;
+ else if (strcmp(opt_oci_layer_compress, "zstd") == 0)
+ write_layer_flags = FLATPAK_OCI_WRITE_LAYER_FLAGS_ZSTD;
+ else
+ return usage_error (context, _("--oci-layer-compress value must be gzip or zstd"), error);
+
+ if (!build_oci (repo, commit_checksum, file, name, full_branch, write_layer_flags, cancellable, error))
  return FALSE;
  }
  else

common/Makefile.am.inc

@@ -202,6 +202,8 @@ libflatpak_common_la_SOURCES = \
  common/flatpak-uri.c \
  common/flatpak-prune.c \
  common/flatpak-prune-private.h \
+ common/flatpak-zstd-compressor.c \
+ common/flatpak-zstd-compressor-private.h \
  common/flatpak-zstd-decompressor.c \
  common/flatpak-zstd-decompressor-private.h \
  common/valgrind-private.h \

common/flatpak-json-oci-private.h

@@ -29,8 +29,8 @@ G_BEGIN_DECLS
 #define FLATPAK_OCI_MEDIA_TYPE_IMAGE_MANIFEST "application/vnd.oci.image.manifest.v1+json"
 #define FLATPAK_DOCKER_MEDIA_TYPE_IMAGE_MANIFEST2 "application/vnd.docker.distribution.manifest.v2+json"
 #define FLATPAK_OCI_MEDIA_TYPE_IMAGE_INDEX "application/vnd.oci.image.index.v1+json"
-#define FLATPAK_OCI_MEDIA_TYPE_IMAGE_LAYER "application/vnd.oci.image.layer.v1.tar+gzip"
-#define FLATPAK_OCI_MEDIA_TYPE_IMAGE_LAYER_NONDISTRIBUTABLE "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip"
+#define FLATPAK_OCI_MEDIA_TYPE_IMAGE_LAYER_GZIP "application/vnd.oci.image.layer.v1.tar+gzip"
+#define FLATPAK_OCI_MEDIA_TYPE_IMAGE_LAYER_ZSTD "application/vnd.oci.image.layer.v1.tar+zstd"
 #define FLATPAK_OCI_MEDIA_TYPE_IMAGE_CONFIG "application/vnd.oci.image.config.v1+json"
 #define FLATPAK_DOCKER_MEDIA_TYPE_IMAGE_IMAGE_CONFIG "application/vnd.docker.container.image.v1+json"
 

common/flatpak-oci-registry-private.h

@@ -119,9 +119,16 @@ FlatpakOciImage * flatpak_oci_registry_load_image_config (FlatpakOciRegistr
  gsize *out_size,
  GCancellable *cancellable,
  GError **error);
-FlatpakOciLayerWriter *flatpak_oci_registry_write_layer (FlatpakOciRegistry *self,
- GCancellable *cancellable,
- GError **error);
+
+typedef enum {
+ FLATPAK_OCI_WRITE_LAYER_FLAGS_NONE = 0,
+ FLATPAK_OCI_WRITE_LAYER_FLAGS_ZSTD = 1 << 0,
+} FlatpakOciWriteLayerFlags;
+
+FlatpakOciLayerWriter *flatpak_oci_registry_write_layer (FlatpakOciRegistry *self,
+ FlatpakOciWriteLayerFlags flags,
+ GCancellable *cancellable,
+ GError **error);
 
 int flatpak_oci_registry_apply_delta (FlatpakOciRegistry *self,
  int delta_fd,

common/flatpak-oci-registry.c

@@ -32,6 +32,7 @@
 #include "flatpak-utils-private.h"
 #include "flatpak-uri-private.h"
 #include "flatpak-dir-private.h"
+#include "flatpak-zstd-compressor-private.h"
 #include "flatpak-zstd-decompressor-private.h"
 
 #define MAX_JSON_SIZE (1024 * 1024)
@@ -1285,12 +1286,13 @@ struct FlatpakOciLayerWriter
 {
  GObject parent;
 
- FlatpakOciRegistry *registry;
+ FlatpakOciRegistry *registry;
+ FlatpakOciWriteLayerFlags flags;
 
  GChecksum *uncompressed_checksum;
  GChecksum *compressed_checksum;
  struct archive *archive;
- GZlibCompressor *compressor;
+ GConverter  *compressor;
  guint64 uncompressed_size;
  guint64 compressed_size;
  GLnxTmpfile tmpf;
@@ -1389,7 +1391,7 @@ flatpak_oci_layer_writer_compress (FlatpakOciLayerWriter *self,
 
  do
  {
- res = g_converter_convert (G_CONVERTER (self->compressor),
+ res = g_converter_convert (self->compressor,
  buffer, length,
  compressed_buffer, sizeof (compressed_buffer),
  flags, &bytes_read, &bytes_written,
@@ -1457,9 +1459,10 @@ flatpak_oci_layer_writer_close_cb (struct archive *archive,
 }
 
 FlatpakOciLayerWriter *
-flatpak_oci_registry_write_layer (FlatpakOciRegistry *self,
- GCancellable *cancellable,
- GError **error)
+flatpak_oci_registry_write_layer (FlatpakOciRegistry *self,
+ FlatpakOciWriteLayerFlags flags,
+ GCancellable *cancellable,
+ GError **error)
 {
  g_autoptr(FlatpakOciLayerWriter) oci_layer_writer = NULL;
  g_autoptr(FlatpakAutoArchiveWrite) a = NULL;
@@ -1476,6 +1479,7 @@ flatpak_oci_registry_write_layer (FlatpakOciRegistry *self,
 
  oci_layer_writer = g_object_new (FLATPAK_TYPE_OCI_LAYER_WRITER, NULL);
  oci_layer_writer->registry = g_object_ref (self);
+ oci_layer_writer->flags = flags;
 
  if (!glnx_open_tmpfile_linkable_at (self->dfd,
  "blobs/sha256",
@@ -1513,7 +1517,32 @@ flatpak_oci_registry_write_layer (FlatpakOciRegistry *self,
  /* Transfer ownership of the tmpfile */
  oci_layer_writer->tmpf = tmpf;
  tmpf.initialized = 0;
- oci_layer_writer->compressor = g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_GZIP, -1);
+
+ if ((flags & FLATPAK_OCI_WRITE_LAYER_FLAGS_ZSTD) != 0)
+ {
+ /*
+ * For the Fedora Flatpak Runtime:
+ *
+ * gzip -6 (default) 83s 712 MiB
+ * zlib-ng -6 38s 741 MiB (bsdtar internal)
+ * zstd -3 9s 670 MiB
+ * zstd -6 22s 627 MiB
+ * zstd -9 34s 584 MiB
+ *
+ * So, even -9 is 240% faster, while producing a 18% smaller result.
+ */
+#ifdef HAVE_ZSTD
+ oci_layer_writer->compressor = (GConverter *)flatpak_zstd_compressor_new (9);
+#else
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED,
+ _("Flatpak was compiled without zstd support");
+ return NULL;
+#endif
+ }
+ else
+ {
+ oci_layer_writer->compressor = (GConverter *)g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_GZIP, -1);
+ }
 
  return g_steal_pointer (&oci_layer_writer);
 }
@@ -1545,8 +1574,14 @@ flatpak_oci_layer_writer_close (FlatpakOciLayerWriter *self,
  if (res_out != NULL)
  {
  g_autofree char *digest = g_strdup_printf ("sha256:%s", g_checksum_get_string (self->compressed_checksum));
+ const char *media_type;
+
+ if ((self->flags & FLATPAK_OCI_WRITE_LAYER_FLAGS_ZSTD) != 0)
+ media_type = FLATPAK_OCI_MEDIA_TYPE_IMAGE_LAYER_ZSTD;
+ else
+ media_type = FLATPAK_OCI_MEDIA_TYPE_IMAGE_LAYER_GZIP;
 
- *res_out = flatpak_oci_descriptor_new (FLATPAK_OCI_MEDIA_TYPE_IMAGE_LAYER, digest, self->compressed_size);
+ *res_out = flatpak_oci_descriptor_new (media_type, digest, self->compressed_size);
  }
 
  return TRUE;

common/flatpak-zstd-compressor-private.h

@@ -0,0 +1,51 @@
+/* Copyright (C) 2023 Red Hat, Inc.
+ *
+ * SPDX-License-Identifier: LGPL-2.1-or-later
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General
+ * Public License along with this library; if not, see <http:https://www.gnu.org/licenses/>.
+ *
+ * Based on gzlibcompressor.h:
+ * Author: Alexander Larsson <[email protected]>
+ * Author: Owen Taylor <[email protected]>
+ */
+
+#ifndef __FLATPAK_ZSTD_COMPRESSOR_H__
+#define __FLATPAK_ZSTD_COMPRESSOR_H__
+
+#include <gio/gio.h>
+
+G_BEGIN_DECLS
+
+#define FLATPAK_TYPE_ZSTD_COMPRESSOR (flatpak_zstd_compressor_get_type ())
+#define FLATPAK_ZSTD_COMPRESSOR(o) (G_TYPE_CHECK_INSTANCE_CAST ((o), FLATPAK_TYPE_ZSTD_COMPRESSOR, FlatpakZstdCompressor))
+#define FLATPAK_ZSTD_COMPRESSOR_CLASS(k) (G_TYPE_CHECK_CLASS_CAST((k), FLATPAK_TYPE_ZSTD_COMPRESSOR, FlatpakZstdCompressorClass))
+#define FLATPAK_IS_ZSTD_COMPRESSOR(o) (G_TYPE_CHECK_INSTANCE_TYPE ((o), FLATPAK_TYPE_ZSTD_COMPRESSOR))
+#define FLATPAK_IS_ZSTD_COMPRESSOR_CLASS(k) (G_TYPE_CHECK_CLASS_TYPE ((k), FLATPAK_TYPE_ZSTD_COMPRESSOR))
+#define FLATPAK_ZSTD_COMPRESSOR_GET_CLASS(o) (G_TYPE_INSTANCE_GET_CLASS ((o), FLATPAK_TYPE_ZSTD_COMPRESSOR, FlatpakZstdCompressorClass))
+
+typedef struct _FlatpakZstdCompressor FlatpakZstdCompressor;
+typedef struct _FlatpakZstdCompressorClass FlatpakZstdCompressorClass;
+
+struct _FlatpakZstdCompressorClass
+{
+ GObjectClass parent_class;
+};
+
+GType flatpak_zstd_compressor_get_type (void) G_GNUC_CONST;
+
+FlatpakZstdCompressor *flatpak_zstd_compressor_new (int level);
+
+G_END_DECLS
+
+#endif /* __FLATPAK_ZSTD_COMPRESSOR_H__ */