[WIP]: build tags

.github/workflows/job-build.yml

@@ -104,7 +104,12 @@ jobs:
         name: "Run: make binaries with custom BUILDTAGS"
         run: |
           set -eux
-          # no_ipfs: make sure it does not incur any IPFS-related dependency
+          # Disabling all optional features and ensuring this does not incur any of the related dependencies
           go mod vendor
-          rm -rf vendor/github.com/ipfs vendor/github.com/multiformats
-          BUILDTAGS=no_ipfs make binaries
+          rm -rf \
+            vendor/github.com/ipfs \
+            vendor/github.com/multiformats \
+            vendor/github.com/containerd/nydus-snapshotter \
+            vendor/github.com/containerd/stargz-snapshotter \
+            vendor/github.com/containerd/accelerated-container-image
+          BUILDTAGS=no_ipfs,no_nydus,no_esgz,no_obd make binaries

.yamllint

@@ -2,6 +2,9 @@
 
 extends: default
 
+ignore:
+  - _
+
 rules:
   indentation:
     spaces: 2

BUILDING.md

@@ -20,3 +20,6 @@ BUILDTAGS=no_ipfs make
 
 The following build tags are supported:
 * `no_ipfs` (since v2.1.3): Disable IPFS
+* `no_nydus` (since v2.1.4): Disable Nydus
+* `no_esgz` (ibid): Disable Estargz
+* `no_obd` (ibid): Disable OverlayBD

Makefile

@@ -27,6 +27,9 @@ DOCKER ?= docker
 GO ?= go
 GOOS ?= $(shell $(GO) env GOOS)
 GOARCH ?= $(shell $(GO) env GOARCH)
+# The following tags can be used to disable some features:
+# BUILDTAGS ?= no_nydus no_esgz no_obd no_ipfs
+BUILDTAGS ?=
 ifeq ($(GOOS),windows)
 	BIN_EXT := .exe
 endif
@@ -46,9 +49,6 @@ LINT_COMMIT_RANGE ?= main..HEAD
 GO_BUILD_LDFLAGS ?= -s -w
 GO_BUILD_FLAGS ?=
 
-BUILDTAGS ?=
-GO_TAGS=$(if $(BUILDTAGS),-tags "$(strip $(BUILDTAGS))",)
-
 ##########################
 # Helpers
 ##########################
@@ -57,7 +57,7 @@ ifdef VERBOSE
 	VERBOSE_FLAG_LONG := --verbose
 endif
 
-export GO_BUILD=CGO_ENABLED=0 GOOS=$(GOOS) $(GO) -C $(MAKEFILE_DIR) build $(GO_TAGS) -ldflags "$(GO_BUILD_LDFLAGS) $(VERBOSE_FLAG) -X $(PACKAGE)/pkg/version.Version=$(VERSION) -X $(PACKAGE)/pkg/version.Revision=$(REVISION)"
+export GO_BUILD=CGO_ENABLED=0 GOOS=$(GOOS) $(GO) -C $(MAKEFILE_DIR) build -tags "$(BUILDTAGS)" -ldflags "$(GO_BUILD_LDFLAGS) $(VERBOSE_FLAG) -X $(PACKAGE)/pkg/version.Version=$(VERSION) -X $(PACKAGE)/pkg/version.Revision=$(REVISION)"
 
 ifndef NO_COLOR
     NC := \033[0m
@@ -131,7 +131,7 @@ clean:
 lint-go:
 	$(call title, $@: $(GOOS))
 	@cd $(MAKEFILE_DIR) \
-		&& golangci-lint run $(VERBOSE_FLAG_LONG) ./...
+		&& golangci-lint run --build-tags "$(BUILDTAGS)" $(VERBOSE_FLAG_LONG) ./...
 	$(call footer, $@)
 
 lint-go-all:
@@ -140,7 +140,8 @@ lint-go-all:
 		&& GOOS=linux make lint-go \
 		&& GOOS=windows make lint-go \
 		&& GOOS=freebsd make lint-go \
-		&& GOOS=darwin make lint-go
+		&& GOOS=darwin make lint-go \
+		&& GOOS=linux BUILDTAGS=no_nydus,no_esgz,no_obd,no_ipfs make lint-go
 	$(call footer, $@)
 
 lint-yaml:
@@ -194,7 +195,7 @@ lint-licenses-all:
 fix-go:
 	$(call title, $@: $(GOOS))
 	@cd $(MAKEFILE_DIR) \
-		&& golangci-lint run --fix
+		&& golangci-lint run --build-tags "$(BUILDTAGS)" --fix
 	$(call footer, $@)
 
 fix-go-all:
@@ -203,7 +204,8 @@ fix-go-all:
 		&& GOOS=linux make fix-go \
 		&& GOOS=windows make fix-go \
 		&& GOOS=freebsd make fix-go \
-		&& GOOS=darwin make fix-go
+		&& GOOS=darwin make fix-go \
+		&& GOOS=linux BUILDTAGS=no_nydus,no_esgz,no_obd,no_ipfs make fix-go
 	$(call footer, $@)
 
 fix-mod:

mod/tigron/.yamllint

@@ -2,6 +2,9 @@
 
 extends: default
 
+ignore:
+  - _
+
 rules:
   indentation:
     spaces: 2

pkg/cmd/image/convert.go

@@ -22,25 +22,16 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"os"
 	"strings"
 
-	"github.com/klauspost/compress/zstd"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 
-	overlaybdconvert "github.com/containerd/accelerated-container-image/pkg/convertor"
 	containerd "github.com/containerd/containerd/v2/client"
 	"github.com/containerd/containerd/v2/core/content"
 	"github.com/containerd/containerd/v2/core/images"
 	"github.com/containerd/containerd/v2/core/images/converter"
 	"github.com/containerd/containerd/v2/core/images/converter/uncompress"
 	"github.com/containerd/log"
-	nydusconvert "github.com/containerd/nydus-snapshotter/pkg/converter"
-	"github.com/containerd/stargz-snapshotter/estargz"
-	estargzconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz"
-	estargzexternaltocconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz/externaltoc"
-	zstdchunkedconvert "github.com/containerd/stargz-snapshotter/nativeconverter/zstdchunked"
-	"github.com/containerd/stargz-snapshotter/recorder"
 
 	"github.com/containerd/nerdctl/v2/pkg/api/types"
 	"github.com/containerd/nerdctl/v2/pkg/clientutil"
@@ -114,61 +105,33 @@ func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRa
 			return errors.New("options --estargz, --zstdchunked, --overlaybd, --nydus and --soci lead to conflict, only one of them can be used")
 		}
 
-		var convertFunc converter.ConvertFunc
+		var convertOpt converter.Opt
 		var convertType string
 		switch {
 		case estargz:
-			convertFunc, finalize, err = getESGZConverter(options)
-			if err != nil {
-				return err
-			}
 			convertType = "estargz"
-		case zstd:
-			convertFunc, err = getZstdConverter(options)
+			convertOpt, finalize, err = converterutil.ESGZConvertOpt(options.EstargzOptions, options.GOptions.Experimental)
 			if err != nil {
 				return err
 			}
+		case zstd:
 			convertType = "zstd"
+			convertOpt, err = converterutil.ZstdConvertOpt(options.ZstdOptions)
 		case zstdchunked:
-			convertFunc, err = getZstdchunkedConverter(options)
-			if err != nil {
-				return err
-			}
 			convertType = "zstdchunked"
+			convertOpt, err = converterutil.ESGZZstdChunkedConvertOpt(options.ZstdChunkedOptions, options.GOptions.Experimental)
 		case overlaybd:
-			obdOpts, err := getOBDConvertOpts(options)
-			if err != nil {
-				return err
-			}
-			obdOpts = append(obdOpts, overlaybdconvert.WithClient(client))
-			obdOpts = append(obdOpts, overlaybdconvert.WithImageRef(srcRef))
-			convertFunc = overlaybdconvert.IndexConvertFunc(obdOpts...)
-			convertOpts = append(convertOpts, converter.WithIndexConvertFunc(convertFunc))
 			convertType = "overlaybd"
+			convertOpt, err = converterutil.OverlayBDConvertOpt(options.OverlaybdOptions, client, srcRef)
 		case nydus:
-			nydusOpts, err := getNydusConvertOpts(options)
+			convertType = "nydus"
+			var defaultWorkDir string
+			defaultWorkDir, err = clientutil.DataStore(options.GOptions.DataRoot, options.GOptions.Address)
 			if err != nil {
 				return err
 			}
-			convertHooks := converter.ConvertHooks{
-				PostConvertHook: nydusconvert.ConvertHookFunc(nydusconvert.MergeOption{
-					WorkDir:          nydusOpts.WorkDir,
-					BuilderPath:      nydusOpts.BuilderPath,
-					FsVersion:        nydusOpts.FsVersion,
-					ChunkDictPath:    nydusOpts.ChunkDictPath,
-					PrefetchPatterns: nydusOpts.PrefetchPatterns,
-					OCI:              true,
-				}),
-			}
-			convertOpts = append(convertOpts, converter.WithIndexConvertFunc(
-				converter.IndexConvertFuncWithHook(
-					nydusconvert.LayerConvertFunc(*nydusOpts),
-					true,
-					platMC,
-					convertHooks,
-				)),
-			)
-			convertType = "nydus"
+
+			convertOpt, err = converterutil.NydusConvertOpt(options.NydusOptions, platMC, defaultWorkDir)
 		case soci:
 			// Convert image to SOCI format
 			convertedRef, err := snapshotterutil.ConvertSociIndexV2(ctx, client, srcRef, targetRef, options.GOptions, options.Platforms, options.SociOptions)
@@ -181,16 +144,20 @@ func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRa
 			return printConvertedImage(options.Stdout, options, res)
 		}
 
-		if convertType != "overlaybd" {
-			convertOpts = append(convertOpts, converter.WithLayerConvertFunc(convertFunc))
+		if err != nil {
+			return err
 		}
+
+		convertOpts = append(convertOpts, convertOpt)
+
 		if !options.Oci {
 			if nydus || overlaybd {
 				log.G(ctx).Warnf("option --%s should be used in conjunction with --oci, forcibly enabling on oci mediatype for %s conversion", convertType, convertType)
 			} else {
 				log.G(ctx).Warnf("option --%s should be used in conjunction with --oci", convertType)
 			}
 		}
+
 		if options.Uncompress {
 			return fmt.Errorf("option --%s conflicts with --uncompress", convertType)
 		}
@@ -232,140 +199,6 @@ func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRa
 	return printConvertedImage(options.Stdout, options, res)
 }
 
-func getESGZConverter(options types.ImageConvertOptions) (convertFunc converter.ConvertFunc, finalize func(ctx context.Context, cs content.Store, ref string, desc *ocispec.Descriptor) (*images.Image, error), _ error) {
-	if options.EstargzExternalToc && !options.GOptions.Experimental {
-		return nil, nil, fmt.Errorf("estargz-external-toc requires experimental mode to be enabled")
-	}
-	if options.EstargzKeepDiffID && !options.GOptions.Experimental {
-		return nil, nil, fmt.Errorf("option --estargz-keep-diff-id must be specified with --estargz-external-toc")
-	}
-	if options.EstargzExternalToc {
-		if !options.EstargzKeepDiffID {
-			esgzOpts, err := getESGZConvertOpts(options)
-			if err != nil {
-				return nil, nil, err
-			}
-			convertFunc, finalize = estargzexternaltocconvert.LayerConvertFunc(esgzOpts, options.EstargzCompressionLevel)
-		} else {
-			convertFunc, finalize = estargzexternaltocconvert.LayerConvertLossLessFunc(estargzexternaltocconvert.LayerConvertLossLessConfig{
-				CompressionLevel: options.EstargzCompressionLevel,
-				ChunkSize:        options.EstargzChunkSize,
-				MinChunkSize:     options.EstargzMinChunkSize,
-			})
-		}
-	} else {
-		esgzOpts, err := getESGZConvertOpts(options)
-		if err != nil {
-			return nil, nil, err
-		}
-		convertFunc = estargzconvert.LayerConvertFunc(esgzOpts...)
-	}
-	return convertFunc, finalize, nil
-}
-
-func getESGZConvertOpts(options types.ImageConvertOptions) ([]estargz.Option, error) {
-
-	esgzOpts := []estargz.Option{
-		estargz.WithCompressionLevel(options.EstargzCompressionLevel),
-		estargz.WithChunkSize(options.EstargzChunkSize),
-		estargz.WithMinChunkSize(options.EstargzMinChunkSize),
-	}
-
-	if options.EstargzRecordIn != "" {
-		if !options.GOptions.Experimental {
-			return nil, fmt.Errorf("estargz-record-in requires experimental mode to be enabled")
-		}
-
-		log.L.Warn("--estargz-record-in flag is experimental and subject to change")
-		paths, err := readPathsFromRecordFile(options.EstargzRecordIn)
-		if err != nil {
-			return nil, err
-		}
-		esgzOpts = append(esgzOpts, estargz.WithPrioritizedFiles(paths))
-		var ignored []string
-		esgzOpts = append(esgzOpts, estargz.WithAllowPrioritizeNotFound(&ignored))
-	}
-	return esgzOpts, nil
-}
-
-func getZstdConverter(options types.ImageConvertOptions) (converter.ConvertFunc, error) {
-	return converterutil.ZstdLayerConvertFunc(options)
-}
-
-func getZstdchunkedConverter(options types.ImageConvertOptions) (converter.ConvertFunc, error) {
-
-	esgzOpts := []estargz.Option{
-		estargz.WithChunkSize(options.ZstdChunkedChunkSize),
-	}
-
-	if options.ZstdChunkedRecordIn != "" {
-		if !options.GOptions.Experimental {
-			return nil, fmt.Errorf("zstdchunked-record-in requires experimental mode to be enabled")
-		}
-
-		log.L.Warn("--zstdchunked-record-in flag is experimental and subject to change")
-		paths, err := readPathsFromRecordFile(options.ZstdChunkedRecordIn)
-		if err != nil {
-			return nil, err
-		}
-		esgzOpts = append(esgzOpts, estargz.WithPrioritizedFiles(paths))
-		var ignored []string
-		esgzOpts = append(esgzOpts, estargz.WithAllowPrioritizeNotFound(&ignored))
-	}
-	return zstdchunkedconvert.LayerConvertFuncWithCompressionLevel(zstd.EncoderLevelFromZstd(options.ZstdChunkedCompressionLevel), esgzOpts...), nil
-}
-
-func getNydusConvertOpts(options types.ImageConvertOptions) (*nydusconvert.PackOption, error) {
-	workDir := options.NydusWorkDir
-	if workDir == "" {
-		var err error
-		workDir, err = clientutil.DataStore(options.GOptions.DataRoot, options.GOptions.Address)
-		if err != nil {
-			return nil, err
-		}
-	}
-	return &nydusconvert.PackOption{
-		BuilderPath: options.NydusBuilderPath,
-		// the path will finally be used is <NERDCTL_DATA_ROOT>/nydus-converter-<hash>,
-		// for example: /var/lib/nerdctl/1935db59/nydus-converter-3269662176/,
-		// and it will be deleted after the conversion
-		WorkDir:          workDir,
-		PrefetchPatterns: options.NydusPrefetchPatterns,
-		Compressor:       options.NydusCompressor,
-		FsVersion:        "6",
-	}, nil
-}
-
-func getOBDConvertOpts(options types.ImageConvertOptions) ([]overlaybdconvert.Option, error) {
-	obdOpts := []overlaybdconvert.Option{
-		overlaybdconvert.WithFsType(options.OverlayFsType),
-		overlaybdconvert.WithDbstr(options.OverlaydbDBStr),
-	}
-	return obdOpts, nil
-}
-
-func readPathsFromRecordFile(filename string) ([]string, error) {
-	r, err := os.Open(filename)
-	if err != nil {
-		return nil, err
-	}
-	defer r.Close()
-	dec := json.NewDecoder(r)
-	var paths []string
-	added := make(map[string]struct{})
-	for dec.More() {
-		var e recorder.Entry
-		if err := dec.Decode(&e); err != nil {
-			return nil, err
-		}
-		if _, ok := added[e.Path]; !ok {
-			paths = append(paths, e.Path)
-			added[e.Path] = struct{}{}
-		}
-	}
-	return paths, nil
-}
-
 func printConvertedImage(stdout io.Writer, options types.ImageConvertOptions, img converterutil.ConvertedImageInfo) error {
 	switch options.Format {
 	case "json":

pkg/features/errors.go

@@ -0,0 +1,10 @@
+package features
+
+import "errors"
+
+var (
+	ErrIPFSSupportMissing      = errors.New("ipfs support has been disabled by the distributor of this build")
+	ErrESGZSupportMissing      = errors.New("estargz support has been disabled by the distributor of this build (eg: build tag `no_esgz`)")
+	ErrOverlayBDSupportMissing = errors.New("overlaybd support has been disabled by the distributor of this build (eg: build tag `no_obd`)")
+	ErrNydusSupportMissing     = errors.New("nydus support has been disabled by the distributor of this build (eg: build tag `no_nydus`)")
+)

pkg/imgutil/converter/errors.go

@@ -0,0 +1,12 @@
+package converter
+
+import (
+	"errors"
+)
+
+var (
+	ErrZstdInRequiresExperimental   = errors.New("option --zstdchunked-record-in requires experimental mode to be enabled")
+	ErrESGZTocRequiresExperimental  = errors.New("option --estargz-external-toc requires experimental mode to be enabled")
+	ErrESGZDiffRequiresExperimental = errors.New("option --estargz-keep-diff-id requires experimental mode to be enabled")
+	ErrESGSInRequiresExperimental   = errors.New("option --estargz-record-in requires experimental mode to be enabled")
+)