Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docker/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ ARG TARGETARCH
SHELL ["/bin/sh", "-o", "pipefail", "-c"]

# Install runtime dependencies for git operations and TLS
RUN apk add --no-cache ca-certificates curl git git-daemon tzdata zstd && \
RUN apk add --no-cache ca-certificates curl git git-daemon git-lfs tzdata zstd && \
addgroup -g 1000 cachew && \
adduser -D -u 1000 -G cachew cachew

Expand Down
4 changes: 3 additions & 1 deletion internal/gitclone/command.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,9 @@ import (
"github.com/alecthomas/errors"
)

func (r *Repository) gitCommand(ctx context.Context, args ...string) (*exec.Cmd, error) {
// GitCommand returns a git subprocess configured with repository-scoped
// authentication and any per-URL git config overrides disabled.
func (r *Repository) GitCommand(ctx context.Context, args ...string) (*exec.Cmd, error) {
repoURL := r.upstreamURL
var token string
if r.credentialProvider != nil && strings.Contains(repoURL, "github.com") {
Expand Down
6 changes: 3 additions & 3 deletions internal/gitclone/command_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ func TestGitCommand(t *testing.T) {
credentialProvider: nil,
}

cmd, err := repo.gitCommand(ctx, "version")
cmd, err := repo.GitCommand(ctx, "version")
assert.NoError(t, err)

assert.NotZero(t, cmd)
Expand All @@ -70,7 +70,7 @@ func TestGitCommandWithEmptyURL(t *testing.T) {
credentialProvider: nil,
}

cmd, err := repo.gitCommand(ctx, "version")
cmd, err := repo.GitCommand(ctx, "version")
assert.NoError(t, err)

assert.NotZero(t, cmd)
Expand Down Expand Up @@ -124,7 +124,7 @@ func TestGitCommandWithCredentialProvider(t *testing.T) {
},
}

cmd, err := repo.gitCommand(ctx, "version")
cmd, err := repo.GitCommand(ctx, "version")
assert.NoError(t, err)
assert.NotZero(t, cmd)

Expand Down
7 changes: 3 additions & 4 deletions internal/gitclone/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -502,7 +502,7 @@ func (r *Repository) executeClone(ctx context.Context) error {
r.upstreamURL, r.path,
}

cmd, err := r.gitCommand(cloneCtx, args...)
cmd, err := r.GitCommand(cloneCtx, args...)
if err != nil {
return errors.Wrap(err, "create git command")
}
Expand Down Expand Up @@ -583,8 +583,7 @@ func (r *Repository) fetchInternal(ctx context.Context, timeout time.Duration, e
}
args = append(args, "fetch", "--prune", "--prune-tags")

// #nosec G204 - r.path is controlled by us
cmd, err := r.gitCommand(fetchCtx, args...)
cmd, err := r.GitCommand(fetchCtx, args...)
if err != nil {
return errors.Wrap(err, "create git command")
}
Expand Down Expand Up @@ -682,7 +681,7 @@ func (r *Repository) GetLocalRefs(ctx context.Context) (map[string]string, error

func (r *Repository) GetUpstreamRefs(ctx context.Context) (map[string]string, error) {
// #nosec G204 - r.upstreamURL is controlled by us
cmd, err := r.gitCommand(ctx, "ls-remote", r.upstreamURL)
cmd, err := r.GitCommand(ctx, "ls-remote", r.upstreamURL)
if err != nil {
return nil, errors.Wrap(err, "create git command")
}
Expand Down
37 changes: 30 additions & 7 deletions internal/snapshot/snapshot.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,31 +25,54 @@ import (
// Exclude patterns use tar's --exclude syntax.
// threads controls zstd parallelism; 0 uses all available CPU cores.
func Create(ctx context.Context, remote cache.Cache, key cache.Key, directory string, ttl time.Duration, excludePatterns []string, threads int) error {
return CreatePaths(ctx, remote, key, directory, filepath.Base(directory), []string{"."}, ttl, excludePatterns, threads)
}

// CreatePaths archives named paths within baseDir using tar with zstd compression,
// then uploads the resulting archive to the cache.
//
// The archive preserves all file permissions, ownership, and symlinks.
// Each entry in includePaths is archived relative to baseDir and must exist.
// This allows callers to archive either an entire directory with "." or a
// specific subtree such as "lfs" while preserving that relative path prefix.
// Exclude patterns use tar's --exclude syntax.
// threads controls zstd parallelism; 0 uses all available CPU cores.
func CreatePaths(ctx context.Context, remote cache.Cache, key cache.Key, baseDir, archiveName string, includePaths []string, ttl time.Duration, excludePatterns []string, threads int) error {
if threads <= 0 {
threads = runtime.NumCPU()
}

// Verify directory exists
if info, err := os.Stat(directory); err != nil {
return errors.Wrap(err, "failed to stat directory")
if len(includePaths) == 0 {
return errors.New("includePaths must not be empty")
}

if info, err := os.Stat(baseDir); err != nil {
return errors.Wrap(err, "failed to stat base directory")
} else if !info.IsDir() {
return errors.Errorf("not a directory: %s", directory)
return errors.Errorf("not a directory: %s", baseDir)
}
for _, path := range includePaths {
targetPath := filepath.Join(baseDir, path)
if _, err := os.Stat(targetPath); err != nil {
return errors.Wrapf(err, "failed to stat include path %q", path)
}
}

headers := make(http.Header)
headers.Set("Content-Type", "application/zstd")
headers.Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", filepath.Base(directory)+".tar.zst"))
headers.Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", archiveName+".tar.zst"))

wc, err := remote.Create(ctx, key, headers, ttl)
if err != nil {
return errors.Wrap(err, "failed to create object")
}

tarArgs := []string{"-cpf", "-", "-C", directory}
tarArgs := []string{"-cpf", "-", "-C", baseDir}
for _, pattern := range excludePatterns {
tarArgs = append(tarArgs, "--exclude", pattern)
}
tarArgs = append(tarArgs, ".")
tarArgs = append(tarArgs, "--")
tarArgs = append(tarArgs, includePaths...)

tarCmd := exec.CommandContext(ctx, "tar", tarArgs...)
zstdCmd := exec.CommandContext(ctx, "zstd", "-c", fmt.Sprintf("-T%d", threads)) //nolint:gosec // threads is a validated integer, not user input
Expand Down
15 changes: 15 additions & 0 deletions internal/strategy/git/git.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ type Config struct {
SnapshotInterval time.Duration `hcl:"snapshot-interval,optional" help:"How often to generate tar.zstd workstation snapshots. 0 disables snapshots." default:"0"`
MirrorSnapshotInterval time.Duration `hcl:"mirror-snapshot-interval,optional" help:"How often to generate mirror snapshots for pod bootstrap. 0 uses snapshot-interval. Defaults to 2h." default:"2h"`
RepackInterval time.Duration `hcl:"repack-interval,optional" help:"How often to run full repack. 0 disables." default:"0"`
LFSSnapshotEnabled bool `hcl:"lfs-snapshot-enabled,optional" help:"When true, also generate a separate LFS object snapshot at /git/{repo}/lfs-snapshot.tar.zst on each snapshot interval. Requires git-lfs and a configured GitHub App." default:"false"`
// ServerURL is embedded as remote.origin.url in snapshots so git pull goes through cachew.
ServerURL string `hcl:"server-url,optional" help:"Base URL of this cachew instance, embedded in snapshot remote URLs." default:"${CACHEW_URL}"`
ZstdThreads int `hcl:"zstd-threads,optional" help:"Threads for zstd compression/decompression (0 = all CPU cores)." default:"0"`
Expand Down Expand Up @@ -151,6 +152,9 @@ func New(

if s.config.SnapshotInterval > 0 {
s.scheduleSnapshotJobs(repo)
if s.config.LFSSnapshotEnabled {
s.scheduleLFSSnapshotJobs(repo)
}
}
if s.config.RepackInterval > 0 {
s.scheduleRepackJobs(repo)
Expand Down Expand Up @@ -219,6 +223,11 @@ func (s *Strategy) handleRequest(w http.ResponseWriter, r *http.Request) {
return
}

if strings.HasSuffix(pathValue, "/lfs-snapshot.tar.zst") {
s.handleLFSSnapshotRequest(w, r, host, pathValue)
return
}

service := r.URL.Query().Get("service")
isReceivePack := service == "git-receive-pack" || strings.HasSuffix(pathValue, "/git-receive-pack")

Expand Down Expand Up @@ -497,6 +506,9 @@ func (s *Strategy) startClone(ctx context.Context, repo *gitclone.Repository) {

if s.config.SnapshotInterval > 0 {
s.scheduleSnapshotJobs(repo)
if s.config.LFSSnapshotEnabled {
s.scheduleLFSSnapshotJobs(repo)
}
}
if s.config.RepackInterval > 0 {
s.scheduleRepackJobs(repo)
Expand Down Expand Up @@ -524,6 +536,9 @@ func (s *Strategy) startClone(ctx context.Context, repo *gitclone.Repository) {

if s.config.SnapshotInterval > 0 {
s.scheduleSnapshotJobs(repo)
if s.config.LFSSnapshotEnabled {
s.scheduleLFSSnapshotJobs(repo)
}
}
if s.config.RepackInterval > 0 {
s.scheduleRepackJobs(repo)
Expand Down
Loading