Read STDIN in catp, optimize PrepareLine

.github/workflows/release-assets.yml

@@ -14,7 +14,7 @@ env:
 jobs:
   build:
     name: Upload Release Assets
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
       - name: Install Go
         uses: actions/setup-go@v5

.github/workflows/test-unit.yml

@@ -80,8 +80,9 @@ jobs:
           curl -sLO https://github.com/vearutop/gocovdiff/releases/download/v1.4.2/linux_amd64.tar.gz && tar xf linux_amd64.tar.gz && rm linux_amd64.tar.gz
           gocovdiff_hash=$(git hash-object ./gocovdiff)
           [ "$gocovdiff_hash" == "c37862c73a677e5a9c069470287823ab5bbf0244" ] || (echo "::error::unexpected hash for gocovdiff, possible tampering: $gocovdiff_hash" && exit 1)
-          git fetch origin master ${{ github.event.pull_request.base.sha }}
-          REP=$(./gocovdiff -mod github.com/$GITHUB_REPOSITORY -cov unit.coverprofile -gha-annotations gha-unit.txt -delta-cov-file delta-cov-unit.txt -target-delta-cov ${TARGET_DELTA_COV})
+          # Fetch PR diff from GitHub API.
+          curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -H "Accept: application/vnd.github.v3.diff" https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }} > pull_request.diff
+          REP=$(./gocovdiff -diff pull_request.diff -mod github.com/$GITHUB_REPOSITORY -cov unit.coverprofile -gha-annotations gha-unit.txt -delta-cov-file delta-cov-unit.txt -target-delta-cov ${TARGET_DELTA_COV})
           echo "${REP}"
           cat gha-unit.txt
           DIFF=$(test -e unit-base.txt && ./gocovdiff -mod github.com/$GITHUB_REPOSITORY -func-cov unit.txt -func-base-cov unit-base.txt || echo "Missing base coverage file")

cmd/catp/README.md

@@ -23,6 +23,7 @@ catp dev, go1.22rc1 CGO_ZSTD
 catp prints contents of files to STDOUT or dir/file output,
 while printing current progress status to STDERR.
 It can decompress data from .gz and .zst files.
+Use dash (-) as PATH to read STDIN.
 
 Usage of catp:
 catp [OPTIONS] PATH ...
@@ -37,7 +38,7 @@ catp [OPTIONS] PATH ...
     	files will be written to out dir with original base names
     	disables output flag
   -output string
-    	output to file instead of STDOUT
+    	output to file (can have .gz or .zst ext for compression) instead of STDOUT
   -parallel int
     	number of parallel readers if multiple files are provided
     	lines from different files will go to output simultaneously (out of order of files, but in order of lines in each file)

cmd/catp/catp/app.go

@@ -61,6 +61,32 @@
 
 	hasOptions bool
 	options    Options
+
+	hasCompression bool
+}
+
+// humanReadableBytes converts bytes to a human-readable string (TB, GB, MB, KB, or bytes).
+func humanReadableBytes(bytes int64) string {
+	const (
+		Byte  = 1
+		KByte = Byte * 1024
+		MByte = KByte * 1024
+		GByte = MByte * 1024
+		TByte = GByte * 1024
+	)
+
+	switch {
+	case bytes >= TByte:
+		return fmt.Sprintf("%.2f TB", float64(bytes)/float64(TByte))
+	case bytes >= GByte:
+		return fmt.Sprintf("%.2f GB", float64(bytes)/float64(GByte))
+	case bytes >= MByte:
+		return fmt.Sprintf("%.2f MB", float64(bytes)/float64(MByte))
+	case bytes >= KByte:
+		return fmt.Sprintf("%.2f KB", float64(bytes)/float64(KByte))
+	default:
+		return fmt.Sprintf("%d B", bytes)
+	}
 }
 
 // st renders Status as a string.
@@ -98,17 +124,29 @@
 		}
 	} else {
 		if s.LinesCompleted != 0 {
-			res = fmt.Sprintf("%s: %.1f%% bytes read, %d lines processed, %.1f l/s, %.1f MB/s, elapsed %s, remaining %s",
-				s.Task, s.DonePercent, s.LinesCompleted, s.SpeedLPS, s.SpeedMBPS,
-				s.Elapsed.Round(10*time.Millisecond).String(), s.Remaining.String())
+			if r.totalBytes == -1 { // STDIN
+				res = fmt.Sprintf("%s read, %d lines processed, %.1f l/s, %.1f MB/s, elapsed %s",
+					humanReadableBytes(s.BytesCompleted), s.LinesCompleted, s.SpeedLPS, s.SpeedMBPS,
+					s.Elapsed.Round(10*time.Millisecond).String())
+			} else {
+				res = fmt.Sprintf("%s: %.1f%% bytes read, %d lines processed, %.1f l/s, %.1f MB/s, elapsed %s, remaining %s",
+					s.Task, s.DonePercent, s.LinesCompleted, s.SpeedLPS, s.SpeedMBPS,
+					s.Elapsed.Round(10*time.Millisecond).String(), s.Remaining.String())
+			}
 		} else {
-			res = fmt.Sprintf("%s: %.1f%% bytes read, %.1f MB/s, elapsed %s, remaining %s",
-				s.Task, s.DonePercent, s.SpeedMBPS,
-				s.Elapsed.Round(10*time.Millisecond).String(), s.Remaining.String())
+			if r.totalBytes == -1 {
+				res = fmt.Sprintf("%s read, %.1f MB/s, elapsed %s",
+					humanReadableBytes(s.BytesCompleted), s.SpeedMBPS,
+					s.Elapsed.Round(10*time.Millisecond).String())
+			} else {
+				res = fmt.Sprintf("%s: %.1f%% bytes read, %.1f MB/s, elapsed %s, remaining %s",
+					s.Task, s.DonePercent, s.SpeedMBPS,
+					s.Elapsed.Round(10*time.Millisecond).String(), s.Remaining.String())
+			}
 		}
 	}
 
-	if currentBytesUncompressed > currentBytes {
+	if currentBytesUncompressed > currentBytes && r.hasCompression {
 		lastBytesUncompressed := atomic.LoadInt64(&r.lastBytesUncompressed)
 		lastStatusTime := atomic.LoadInt64(&r.lastStatusTime)
 		now := time.Now().Unix()
@@ -158,6 +196,7 @@
 	s.Buffer(make([]byte, 64*1024), 10*1024*1024)
 
 	lines := 0
+	buf := make([]byte, 64*1024)
 
 	for s.Scan() {
 		lines++
@@ -175,7 +214,8 @@
 
 		if r.hasOptions {
 			if r.options.PrepareLine != nil {
-				line = r.options.PrepareLine(filename, lines, line)
+				buf = buf[:0]
+				line = r.options.PrepareLine(filename, lines, line, &buf)
 			}
 
 			if line == nil {
@@ -265,26 +305,32 @@
 }
 
 func (r *runner) cat(filename string) (err error) { //nolint:gocyclo
-	file, err := os.Open(filename) //nolint:gosec
-	if err != nil {
-		return err
-	}
+	var rd io.Reader
 
-	defer func() {
-		if clErr := file.Close(); clErr != nil && err == nil {
-			err = clErr
+	if filename == "-" {
+		rd = os.Stdin
+	} else {
+		file, err := os.Open(filename) //nolint:gosec
+		if err != nil {
+			return err
 		}
-	}()
 
-	rd := io.Reader(file)
+		defer func() {
+			if clErr := file.Close(); clErr != nil && err == nil {
+				err = clErr
+			}
+		}()
+
+		rd = io.Reader(file)
+	}
 
 	if !r.noProgress {
-		cr := progress.NewCountingReader(file)
+		cr := progress.NewCountingReader(rd)
 		cr.SetBytes(&r.currentBytes)
 		cr.SetLines(nil)
 
 		if r.parallel <= 1 {
-			cr = progress.NewCountingReader(file)
+			cr = progress.NewCountingReader(rd)
 			cr.SetLines(nil)
 			r.currentFile = cr
 			r.currentTotal = r.sizes[filename]
@@ -425,7 +471,8 @@
 // Options allows behavior customisations.
 type Options struct {
 	// PrepareLine is invoked for every line, if result is nil, line is skipped.
-	PrepareLine func(filename string, lineNr int, line []byte) []byte
+	// You can use buf to avoid allocations for a result, and change its capacity if needed.
+	PrepareLine func(filename string, lineNr int, line []byte, buf *[]byte) []byte
 }
 
 // Main is the entry point for catp CLI tool.
@@ -437,14 +484,6 @@
 
 	r := &runner{}
 
-	if len(options) > 0 {
-		r.hasOptions = true
-
-		for _, opt := range options {
-			opt(&r.options)
-		}
-	}
-
 	flag.Var(&pass, "pass", "filter matching, may contain multiple AND patterns separated by ^,\n"+
 		"if filter matches, line is passed to the output (unless filtered out by -skip)\n"+
 		"each -pass value is added with OR logic,\n"+
@@ -476,7 +515,8 @@
 		fmt.Println()
 		fmt.Println("catp prints contents of files to STDOUT or dir/file output, \n" +
 			"while printing current progress status to STDERR. \n" +
-			"It can decompress data from .gz and .zst files.")
+			"It can decompress data from .gz and .zst files.\n" +
+			"Use dash (-) as PATH to read STDIN.")
 		fmt.Println()
 		fmt.Println("Usage of catp:")
 		fmt.Println("catp [OPTIONS] PATH ...")
@@ -502,6 +542,14 @@
 		defer pprof.StopCPUProfile()
 	}
 
+	if len(options) > 0 {
+		r.hasOptions = true
+
+		for _, opt := range options {
+			opt(&r.options)
+		}
+	}
+
 	if *output != "" && r.outDir == "" { //nolint:nestif
 		fn := *output
 
@@ -597,35 +645,51 @@
 
 	var files []string
 
-	for _, f := range flag.Args() {
-		glob, err := filepath.Glob(f)
-		if err != nil {
-			return err
-		}
+	args := flag.Args()
 
-		for _, f := range glob {
-			alreadyThere := false
+	if len(args) == 1 && args[0] == "-" {
+		files = append(files, "-") // STDIN
+	} else {
+		for _, f := range args {
+			glob, err := filepath.Glob(f)
+			if err != nil {
+				return err
+			}
 
-			for _, e := range files {
-				if e == f {
-					alreadyThere = true
+			for _, f := range glob {
+				alreadyThere := false
 
-					break
+				for _, e := range files {
+					if e == f {
+						alreadyThere = true
+
+						break
+					}
 				}
-			}
 
-			if !alreadyThere {
-				files = append(files, f)
+				if !alreadyThere {
+					files = append(files, f)
+				}
 			}
 		}
 	}
 
 	for _, fn := range files {
+		if fn == "-" {
+			r.totalBytes = -1
+
+			continue
+		}
+
 		st, err := os.Stat(fn)
 		if err != nil {
 			return fmt.Errorf("failed to read file stats %s: %w", fn, err)
 		}
 
+		if strings.HasSuffix(fn, ".zst") || strings.HasSuffix(fn, ".gz") {
+			r.hasCompression = true
+		}
+
 		r.totalBytes += st.Size()
 		r.sizes[fn] = st.Size()
 	}

go.mod

@@ -3,8 +3,8 @@ module github.com/bool64/progress
 go 1.22
 
 require (
-	github.com/DataDog/zstd v1.5.6
-	github.com/bool64/dev v0.2.39
+	github.com/DataDog/zstd v1.5.7
+	github.com/bool64/dev v0.2.40
 	github.com/klauspost/compress v1.18.0
 	github.com/klauspost/pgzip v1.2.6
 )

go.sum

@@ -1,7 +1,7 @@
-github.com/DataDog/zstd v1.5.6 h1:LbEglqepa/ipmmQJUDnSsfvA8e8IStVcGaFWDuxvGOY=
-github.com/DataDog/zstd v1.5.6/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
-github.com/bool64/dev v0.2.39 h1:kP8DnMGlWXhGYJEZE/J0l/gVBdbuhoPGL+MJG4QbofE=
-github.com/bool64/dev v0.2.39/go.mod h1:iJbh1y/HkunEPhgebWRNcs8wfGq7sjvJ6W5iabL8ACg=
+github.com/DataDog/zstd v1.5.7 h1:ybO8RBeh29qrxIhCA9E8gKY6xfONU9T6G6aP9DTKfLE=
+github.com/DataDog/zstd v1.5.7/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
+github.com/bool64/dev v0.2.40 h1:LUSD+Aq+WB3KwVntqXstevJ0wB12ig1bEgoG8ZafsZU=
+github.com/bool64/dev v0.2.40/go.mod h1:iJbh1y/HkunEPhgebWRNcs8wfGq7sjvJ6W5iabL8ACg=
 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
 github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
 github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU=