resolving conflicts

Signed-off-by: Avinash Singh <avinashsingh.rcoem@gmail.com>

Author: avinashsingh77

.github/workflows/ci.yml

@@ -20,12 +20,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8    # v6.0.1
         with:
           submodules: recursive
 
       - name: Install Go
-        uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00
+        uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c    # v6.1.0
         with:
           go-version-file: go.mod
 

.github/workflows/codeql-analysis.yml

@@ -28,15 +28,15 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8    # v6.0.1
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@1b168cd39490f61582a9beae412bb7057a6b2c4e
+        uses: github/codeql-action/init@5d4e8d1aca955e8d8589aabd499c5cae939e33c7    # v4.31.9
         with:
           languages: ${{ matrix.language }}
 
       - name: Autobuild
-        uses: github/codeql-action/autobuild@1b168cd39490f61582a9beae412bb7057a6b2c4e
+        uses: github/codeql-action/autobuild@5d4e8d1aca955e8d8589aabd499c5cae939e33c7    # v4.31.9
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@1b168cd39490f61582a9beae412bb7057a6b2c4e
+        uses: github/codeql-action/analyze@5d4e8d1aca955e8d8589aabd499c5cae939e33c7    # v4.31.9

.github/workflows/e2e.yml

@@ -33,12 +33,12 @@ jobs:
       PACKAGE_DIR: modctl-test-package
     steps:
       - name: Checkout code
-        uses: actions/checkout@v6
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8    # v6.0.1
         with:
           submodules: recursive
 
       - name: Install Go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c    # v6.1.0
         with:
           go-version-file: go.mod
           cache-dependency-path: go.sum
@@ -49,7 +49,7 @@ jobs:
 
       - name: Cache Package
         id: cache-package
-        uses: actions/cache@v5
+        uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb    # v5.0.1
         with:
           path: ${{ env.PACKAGE_DIR }}
           key: modctl-test-packages
@@ -79,7 +79,7 @@ jobs:
           go build -tags "static system_libgit2 enable_libgit2"
 
       - name: Upload modctl
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f    # v6.0.0
         with:
           name: modctl-artifact
           path: modctl
@@ -90,15 +90,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Cache model
-        uses: actions/cache@v5
+        uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb    # v5.0.1
         id: cache-model
         with:
           path: ${{ env.MODEL_DIR }}
           key: ${{env.MODEL_DIR}}-${{ env.MODEL }}
 
       - name: Set up Python
         if: steps.cache-model.outputs.cache-hit != 'true'
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548    # v6.1.0
         with:
           python-version: "3.10"
 
@@ -110,7 +110,7 @@ jobs:
           hf download ${{ env.MODEL }} --local-dir ${{ env.MODEL_DIR }}
 
       - name: Upload model
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f    # v6.0.0
         with:
           name: model-artifact
           path: ${{ env.MODEL_DIR }}
@@ -122,12 +122,12 @@ jobs:
     needs: [build-modctl, download-model]
     steps:
       - name: Download modctl artifact
-        uses: actions/download-artifact@v7
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131    # v7.0.0
         with:
           name: modctl-artifact
           path: modctl
       - name: Download model artifact
-        uses: actions/download-artifact@v7
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131    # v7.0.0
         with:
           name: model-artifact
           path: ${{ env.MODEL_DIR }}

.github/workflows/lint.yml

@@ -16,7 +16,7 @@ jobs:
     timeout-minutes: 30
     steps:
       - name: Checkout code
-        uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8    # v6.0.1
         with:
           fetch-depth: '0'
 
@@ -39,7 +39,7 @@ jobs:
             LIBGIT2_SYS_USE_PKG_CONFIG: "1"
 
       - name: Golangci lint
-        uses: golangci/golangci-lint-action@v9.2.0
+        uses: golangci/golangci-lint-action@1e7e51e771db61008b38414a730f564565cf7c20    # v9.2.0
         with:
           version: v2.4
           args: --verbose --timeout=10m

.github/workflows/release.yaml

@@ -27,12 +27,12 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v6
+      uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8    # v6.0.1
       with:
         fetch-depth: 0
 
     - name: Set up Go
-      uses: actions/setup-go@v6
+      uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c    # v6.1.0
       with:
         go-version: '1.24' # Adjust to your Go version
 
@@ -125,7 +125,7 @@ jobs:
         nfpm pkg --packager rpm --config hack/nfpm.yaml --target dist/modctl-${TAG}-${{ matrix.goos }}-${{ matrix.goarch }}.rpm
 
     - name: Upload artifacts
-      uses: actions/upload-artifact@v6
+      uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f    # v6.0.0
       with:
         name: modctl-${{ matrix.goos }}-${{ matrix.goarch }}
         path: dist/
@@ -135,7 +135,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Download all artifacts
-      uses: actions/download-artifact@v7
+      uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131    # v7.0.0
       with:
         path: artifacts
 
@@ -145,7 +145,7 @@ jobs:
           find . -type f \( -name "modctl-*.tar.gz" -o -name "modctl-*.deb" -o -name "modctl-*.rpm" \) -exec shasum -a 256 {} \; > ../checksums.txt
 
     - name: Create draft release
-      uses: softprops/action-gh-release@v2
+      uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b    # v2.5.0
       with:
         draft: true
         files: |

cmd/attach.go

@@ -51,6 +51,7 @@ func init() {
 	flags := attachCmd.Flags()
 	flags.StringVarP(&attachConfig.Source, "source", "s", "", "source model artifact name")
 	flags.StringVarP(&attachConfig.Target, "target", "t", "", "target model artifact name")
+	flags.StringVarP(&attachConfig.DestinationDir, "destination-dir", "d", "", "destination directory for the attached file should be specified as a relative path; by default, it will match the original directory of the attachment")
 	flags.BoolVarP(&attachConfig.OutputRemote, "output-remote", "", false, "turning on this flag will output model artifact to remote registry directly")
 	flags.BoolVarP(&attachConfig.PlainHTTP, "plain-http", "", false, "turning on this flag will use plain HTTP instead of HTTPS")
 	flags.BoolVarP(&attachConfig.Insecure, "insecure", "", false, "turning on this flag will disable TLS verification")

cmd/modelfile/generate.go

@@ -26,20 +26,65 @@ import (
 
 	configmodelfile "github.com/modelpack/modctl/pkg/config/modelfile"
 	"github.com/modelpack/modctl/pkg/modelfile"
+	"github.com/modelpack/modctl/pkg/modelprovider"
 )
 
 var generateConfig = configmodelfile.NewGenerateConfig()
 
 // generateCmd represents the modelfile tools command for generating modelfile.
 var generateCmd = &cobra.Command{
-	Use:                "generate [flags] <path>",
-	Short:              "A command line tool for generating modelfile in the workspace, the workspace must be a directory including model files and model configuration files",
-	Args:               cobra.ExactArgs(1),
+	Use:   "generate [flags] [<path>]",
+	Short: "Generate a modelfile from a local workspace or remote model provider",
+	Long: `Generate a modelfile from either a local directory containing model files or by downloading a model from a supported provider.
+
+The workspace must be a directory including model files and model configuration files.
+Alternatively, use --model-url to download a model from a supported provider (e.g., HuggingFace, ModelScope).
+
+For short-form URLs (owner/repo), you must explicitly specify the provider using --provider flag.
+Full URLs with domain names will auto-detect the provider.`,
+	Example: `  # Generate from local directory
+  modctl modelfile generate ./my-model-dir
+
+  # Generate from Hugging Face using full URL (auto-detects provider)
+  modctl modelfile generate --model-url https://huggingface.co/meta-llama/Llama-2-7b-hf
+
+  # Generate from Hugging Face using short form (requires --provider)
+  modctl modelfile generate --model-url meta-llama/Llama-2-7b-hf --provider huggingface
+
+  # Generate from ModelScope using full URL (auto-detects provider)
+  modctl modelfile generate --model-url https://modelscope.cn/models/qwen/Qwen-7B
+
+  # Generate from ModelScope using short form (requires --provider)
+  modctl modelfile generate --model-url qwen/Qwen-7B --provider modelscope
+
+  # Generate with custom download directory
+  modctl modelfile generate --model-url meta-llama/Llama-2-7b-hf --provider huggingface --download-dir $HOME/models
+
+  # Generate with custom output path
+  modctl modelfile generate ./my-model-dir --output ./output/modelfile.yaml
+
+  # Generate with metadata overrides
+  modctl modelfile generate ./my-model-dir --name my-custom-model --family llama3`,
+	Args:               cobra.MaximumNArgs(1),
 	DisableAutoGenTag:  true,
 	SilenceUsage:       true,
 	FParseErrWhitelist: cobra.FParseErrWhitelist{UnknownFlags: true},
 	RunE: func(cmd *cobra.Command, args []string) error {
-		if err := generateConfig.Convert(args[0]); err != nil {
+		// If model-url is provided, path is optional
+		workspace := "."
+		if len(args) > 0 {
+			workspace = args[0]
+		}
+
+		// Validate that either path or model-url is provided
+		if generateConfig.ModelURL != "" && len(args) > 0 {
+			return fmt.Errorf("the <path> argument and the --model-url flag are mutually exclusive")
+		}
+		if generateConfig.ModelURL == "" && len(args) == 0 {
+			return fmt.Errorf("either a <path> argument or the --model-url flag must be provided")
+		}
+
+		if err := generateConfig.Convert(workspace); err != nil {
 			return err
 		}
 
@@ -64,6 +109,9 @@ func init() {
 	flags.StringVarP(&generateConfig.Output, "output", "O", ".", "specify the output path of modelfilem, must be a directory")
 	flags.BoolVar(&generateConfig.IgnoreUnrecognizedFileTypes, "ignore-unrecognized-file-types", false, "ignore the unrecognized file types in the workspace")
 	flags.BoolVar(&generateConfig.Overwrite, "overwrite", false, "overwrite the existing modelfile")
+	flags.StringVar(&generateConfig.ModelURL, "model-url", "", "download model from a supported provider (full URL or short-form with --provider)")
+	flags.StringVarP(&generateConfig.Provider, "provider", "p", "", "explicitly specify the provider for short-form URLs (huggingface, modelscope)")
+	flags.StringVar(&generateConfig.DownloadDir, "download-dir", "", "custom directory for downloading models (default: system temp directory)")
 	flags.StringArrayVar(&generateConfig.ExcludePatterns, "exclude", []string{}, "specify glob patterns to exclude files/directories (e.g. *.log, checkpoints/*)")
 
 	// Mark the ignore-unrecognized-file-types flag as deprecated and hidden
@@ -76,7 +124,65 @@ func init() {
 }
 
 // runGenerate runs the generate modelfile.
-func runGenerate(_ context.Context) error {
+func runGenerate(ctx context.Context) error {
+	// If model URL is provided, download the model first
+	if generateConfig.ModelURL != "" {
+		fmt.Printf("Model URL provided: %s\n", generateConfig.ModelURL)
+
+		// Get the appropriate provider for this URL
+		registry := modelprovider.GetRegistry()
+		provider, err := registry.SelectProvider(generateConfig.ModelURL, generateConfig.Provider)
+		if err != nil {
+			return fmt.Errorf("failed to select provider: %w", err)
+		}
+
+		fmt.Printf("Using provider: %s\n", provider.Name())
+
+		// Check if user is authenticated with the provider
+		if err := provider.CheckAuth(); err != nil {
+			return fmt.Errorf("%s authentication check failed: %w", provider.Name(), err)
+		}
+
+		// Determine the download directory
+		var downloadDir string
+		var cleanupDir bool
+
+		if generateConfig.DownloadDir != "" {
+			// Use user-specified directory
+			downloadDir = generateConfig.DownloadDir
+			cleanupDir = false
+
+			// Create the directory if it doesn't exist
+			if err := os.MkdirAll(downloadDir, 0755); err != nil {
+				return fmt.Errorf("failed to create download directory: %w", err)
+			}
+			fmt.Printf("Using custom download directory: %s\n", downloadDir)
+		} else {
+			// Create a temporary directory for downloading the model
+			tmpDir, err := os.MkdirTemp("", "modctl-model-downloads-*")
+			if err != nil {
+				return fmt.Errorf("failed to create temporary directory: %w", err)
+			}
+			downloadDir = tmpDir
+			cleanupDir = true
+		}
+
+		// Clean up the directory only if it was a temporary directory
+		if cleanupDir {
+			defer os.RemoveAll(downloadDir)
+		}
+
+		// Download the model
+		downloadPath, err := provider.DownloadModel(ctx, generateConfig.ModelURL, downloadDir)
+		if err != nil {
+			return fmt.Errorf("failed to download model from %s: %w", provider.Name(), err)
+		}
+
+		// Update workspace to the downloaded model path
+		generateConfig.Workspace = downloadPath
+		fmt.Printf("Using downloaded model at: %s\n", downloadPath)
+	}
+
 	fmt.Printf("Generating modelfile for %s\n", generateConfig.Workspace)
 	modelfile, err := modelfile.NewModelfileByWorkspace(generateConfig.Workspace, generateConfig)
 	if err != nil {

cmd/upload.go

@@ -53,6 +53,7 @@ func init() {
 	flags.BoolVarP(&uploadConfig.PlainHTTP, "plain-http", "", false, "turning on this flag will use plain HTTP instead of HTTPS")
 	flags.BoolVarP(&uploadConfig.Insecure, "insecure", "", false, "turning on this flag will disable TLS verification")
 	flags.BoolVar(&uploadConfig.Raw, "raw", true, "turning on this flag will upload model artifact layer in raw format")
+	flags.StringVar(&uploadConfig.DestinationDir, "destination-dir", "", "destination directory for the uploaded file should be specified as a relative path; by default, it will match the original directory of the uploaded file")
 
 	if err := viper.BindPFlags(flags); err != nil {
 		panic(fmt.Errorf("bind cache list flags to viper: %w", err))

go.mod

@@ -3,7 +3,7 @@ module github.com/modelpack/modctl
 go 1.24.1
 
 require (
-	d7y.io/api/v2 v2.1.94
+	d7y.io/api/v2 v2.2.1
 	github.com/antgroup/hugescm v0.18.3
 	github.com/avast/retry-go/v4 v4.7.0
 	github.com/distribution/distribution/v3 v3.0.0
@@ -21,7 +21,7 @@ require (
 	github.com/spf13/cobra v1.10.2
 	github.com/spf13/viper v1.21.0
 	github.com/stretchr/testify v1.11.1
-	github.com/vbauerster/mpb/v8 v8.11.2
+	github.com/vbauerster/mpb/v8 v8.11.3
 	golang.org/x/crypto v0.45.0
 	golang.org/x/sync v0.18.0
 	golang.org/x/sys v0.39.0

go.sum

@@ -1,5 +1,5 @@
-d7y.io/api/v2 v2.1.94 h1:4IY/KFhj2tVvRisIPWqiFI2s4xOQTAlJtkZTodNucpQ=
-d7y.io/api/v2 v2.1.94/go.mod h1:TtW9UE0CebRB/CWIEbWfRnljmpKf/mNoe2qIUO+PoP0=
+d7y.io/api/v2 v2.2.1 h1:po39adZpbGn6sIl9vR9E1fVbOLim5SOP/3Tt0g8Ii58=
+d7y.io/api/v2 v2.2.1/go.mod h1:TtW9UE0CebRB/CWIEbWfRnljmpKf/mNoe2qIUO+PoP0=
 dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
 dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg=
@@ -245,8 +245,8 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
-github.com/vbauerster/mpb/v8 v8.11.2 h1:OqLoHznUVU7SKS/WV+1dB5/hm20YLheYupiHhL5+M1Y=
-github.com/vbauerster/mpb/v8 v8.11.2/go.mod h1:mEB/M353al1a7wMUNtiymmPsEkGlJgeJmtlbY5adCJ8=
+github.com/vbauerster/mpb/v8 v8.11.3 h1:iniBmO4ySXCl4gVdmJpgrtormH5uvjpxcx/dMyVU9Jw=
+github.com/vbauerster/mpb/v8 v8.11.3/go.mod h1:n9M7WbP0NFjpgKS5XdEC3tMRgZTNM/xtC8zWGkiMuy0=
 github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
 github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
 github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY=