[deckhouse-cli] mirror: add packages mirroring support

internal/layout.go

@@ -36,6 +36,12 @@ import "path"
 //	<root>/modules/<module-name>:<version>                    - Module main image
 //	<root>/modules/<module-name>/release:<channel>            - Module release channel metadata
 //	<root>/modules/<module-name>/extra/<extra-name>:<version> - Module extra images
+//
+// Packages (same shape as modules, but under packages/ with a version/ release segment):
+//
+//	<root>/packages/<package-name>:<version>                    - Package main image
+//	<root>/packages/<package-name>/version:<channel>            - Package version channel metadata
+//	<root>/packages/<package-name>/extra/<extra-name>:<version> - Package extra images
 const (
 	InstallSegment           = "install"
 	InstallStandaloneSegment = "install-standalone"
@@ -45,6 +51,10 @@ const (
 	ModulesReleaseSegment = "release"
 	ModulesExtraSegment   = "extra"
 
+	PackagesSegment        = "packages"
+	PackagesVersionSegment = "version"
+	PackagesExtraSegment   = "extra"
+
 	InstallerSegment = "installer"
 
 	SecuritySegment = "security"
@@ -65,6 +75,10 @@ var pathByMirrorType = map[MirrorType]string{
 	MirrorTypeModules:                "",                    // Module main image at root of module dir
 	MirrorTypeModulesReleaseChannels: ModulesReleaseSegment, // modules/<name>/release
 
+	// Package paths are relative to packages/<package-name>/ directory
+	MirrorTypePackages:                "",                     // Package main image at root of package dir
+	MirrorTypePackagesVersionChannels: PackagesVersionSegment, // packages/<name>/version
+
 	MirrorTypeSecurity:                   SecuritySegment,
 	MirrorTypeSecurityTrivyDBSegment:     path.Join(SecuritySegment, SecurityTrivyDBSegment),
 	MirrorTypeSecurityTrivyBDUSegment:    path.Join(SecuritySegment, SecurityTrivyBDUSegment),

internal/mirror.go

@@ -30,4 +30,6 @@ const (
 	MirrorTypeSecurityTrivyBDUSegment
 	MirrorTypeSecurityTrivyJavaDBSegment
 	MirrorTypeSecurityTrivyChecksSegment
+	MirrorTypePackages
+	MirrorTypePackagesVersionChannels
 )

internal/mirror/cmd/pull/flags/flags.go

@@ -58,6 +58,9 @@ var (
 	ModulesWhitelist  []string
 	ModulesBlacklist  []string
 
+	PackagesWhitelist []string
+	PackagesBlacklist []string
+
 	SourceRegistryRepo     = EnterpriseEditionRepo // Fallback to EE if nothing was given as source.
 	SourceRegistryLogin    string
 	SourceRegistryPassword string
@@ -70,6 +73,7 @@ var (
 	NoPlatform      bool
 	NoSecurityDB    bool
 	NoModules       bool
+	NoPackages      bool
 	NoInstaller     bool
 	OnlyExtraImages bool
 	SkipVexImages   bool
@@ -200,6 +204,20 @@ module-name@=v1.3.0+stable → exact tag match: include only v1.3.0 and and publ
 		"/modules",
 		"Suffix to append to source repo path to locate modules.",
 	)
+	flagSet.StringArrayVar(
+		&PackagesWhitelist,
+		"include-package",
+		nil,
+		`Whitelist specific packages for downloading. Use one flag per each package. Disables blacklisting by --exclude-package.
+
+Packages are mirrored exactly like modules (same name@version constraint dialect), but live under the packages/ registry segment with their release metadata under packages/<name>/version.`,
+	)
+	flagSet.StringArrayVar(
+		&PackagesBlacklist,
+		"exclude-package",
+		nil,
+		`Blacklist specific packages from downloading. Format is "package-name[@version]". Use one flag per each package. Overridden by use of --include-package.`,
+	)
 	flagSet.Int64VarP(
 		&ImagesBundleChunkSizeGB,
 		"images-bundle-chunk-size",
@@ -249,6 +267,12 @@ module-name@=v1.3.0+stable → exact tag match: include only v1.3.0 and and publ
 		false,
 		"Do not pull Deckhouse modules into bundle.",
 	)
+	flagSet.BoolVar(
+		&NoPackages,
+		"no-packages",
+		false,
+		"Do not pull Deckhouse packages into bundle.",
+	)
 	flagSet.BoolVar(
 		&NoInstaller,
 		"no-installer",

internal/mirror/cmd/pull/pull.go

@@ -99,6 +99,7 @@ func NewCommand() *cobra.Command {
 
 	pullflags.AddFlags(pullCmd.Flags())
 	pullCmd.MarkFlagsMutuallyExclusive("include-module", "exclude-module")
+	pullCmd.MarkFlagsMutuallyExclusive("include-package", "exclude-package")
 	pullCmd.MarkFlagsMutuallyExclusive("include-platform", "deckhouse-tag")
 	pullCmd.MarkFlagsMutuallyExclusive("include-platform", "since-version")
 	pullCmd.MarkFlagsMutuallyExclusive("proxy-registry", "deckhouse-tag")
@@ -171,6 +172,7 @@ func buildPullParams(logger params.Logger) *params.PullParams {
 		SkipPlatform:          pullflags.NoPlatform,
 		SkipSecurityDatabases: pullflags.NoSecurityDB,
 		SkipModules:           pullflags.NoModules,
+		SkipPackages:          pullflags.NoPackages,
 		SkipInstaller:         pullflags.NoInstaller,
 		OnlyExtraImages:       pullflags.OnlyExtraImages,
 		IgnoreSuspend:         pullflags.IgnoreSuspend,
@@ -285,6 +287,12 @@ func (p *Puller) Execute(ctx context.Context) error {
 		return err
 	}
 
+	// Create package filter from CLI flags
+	packageFilter, err := p.createPackageFilter()
+	if err != nil {
+		return err
+	}
+
 	svc := mirror.NewPullService(
 		registryservice.NewService(c, edition, logger),
 		pullflags.TempDir,
@@ -293,13 +301,15 @@ func (p *Puller) Execute(ctx context.Context) error {
 			SkipPlatform:       pullflags.NoPlatform,
 			SkipSecurity:       pullflags.NoSecurityDB,
 			SkipModules:        pullflags.NoModules,
+			SkipPackages:       pullflags.NoPackages,
 			SkipVexImages:      pullflags.SkipVexImages,
 			SkipInstaller:      pullflags.NoInstaller,
 			InstallerTag:       pullflags.InstallerTag,
 			OnlyExtraImages:    pullflags.OnlyExtraImages,
 			IgnoreSuspend:      pullflags.IgnoreSuspend,
 			PlatformConstraint: pullflags.PlatformConstraint,
 			ModuleFilter:       filter,
+			PackageFilter:      packageFilter,
 			BundleDir:          pullflags.ImagesBundlePath,
 			BundleChunkSize:    pullflags.ImagesBundleChunkSizeGB * 1000 * 1000 * 1000,
 			Timeout:            pullflags.MirrorTimeout,
@@ -412,6 +422,27 @@ func (p *Puller) createModuleFilter() (*modules.Filter, error) {
 	return filter, nil
 }
 
+// createPackageFilter creates the appropriate package filter based on whitelist/blacklist.
+// Packages reuse the modules filter because selection logic (names + semver
+// constraints) is identical.
+func (p *Puller) createPackageFilter() (*modules.Filter, error) {
+	if pullflags.PackagesWhitelist != nil {
+		filter, err := modules.NewFilter(pullflags.PackagesWhitelist, modules.FilterTypeWhitelist)
+		if err != nil {
+			return nil, fmt.Errorf("Prepare package filter: %w", err)
+		}
+
+		return filter, nil
+	}
+
+	filter, err := modules.NewFilter(pullflags.PackagesBlacklist, modules.FilterTypeBlacklist)
+	if err != nil {
+		return nil, fmt.Errorf("Prepare package filter: %w", err)
+	}
+
+	return filter, nil
+}
+
 // computeGOSTDigests computes GOST digests for the bundle if enabled
 func (p *Puller) computeGOSTDigests() error {
 	if !pullflags.DoGOSTDigest {

internal/mirror/cmd/pull/pull_test.go

@@ -1262,6 +1262,7 @@ func TestPullFunction(t *testing.T) {
 	originalNoPlatform := pullflags.NoPlatform
 	originalNoSecurityDB := pullflags.NoSecurityDB
 	originalNoModules := pullflags.NoModules
+	originalNoPackages := pullflags.NoPackages
 	originalNoInstaller := pullflags.NoInstaller
 
 	defer func() {
@@ -1271,6 +1272,7 @@ func TestPullFunction(t *testing.T) {
 		pullflags.NoPlatform = originalNoPlatform
 		pullflags.NoSecurityDB = originalNoSecurityDB
 		pullflags.NoModules = originalNoModules
+		pullflags.NoPackages = originalNoPackages
 		pullflags.NoInstaller = originalNoInstaller
 	}()
 
@@ -1281,6 +1283,7 @@ func TestPullFunction(t *testing.T) {
 	pullflags.NoPlatform = true
 	pullflags.NoSecurityDB = true
 	pullflags.NoModules = true
+	pullflags.NoPackages = true
 	pullflags.NoInstaller = true
 
 	cmd := &cobra.Command{}