Organize --help

Makefile

@@ -4,7 +4,7 @@ zannotate:
 	cd cmd/zannotate && \
 	go build -o zannotate && \
 	cd - && \
-	mv cmd/zannotate/zannotate .
+	cp cmd/zannotate/zannotate .
 
 clean:
 	rm -f zannotate

censys.go

@@ -56,8 +56,10 @@ func (a *CensysAnnotatorFactory) IsEnabled() bool {
 	return a.Enabled
 }
 
+func (a *CensysAnnotatorFactory) GroupName() string { return "Censys" }
+
 func (a *CensysAnnotatorFactory) AddFlags(flags *flag.FlagSet) {
-	flags.BoolVar(&a.Enabled, "censys", false, "censys internet intelligence")
+	flags.BoolVar(&a.Enabled, "censys", false, "annotate with censys internet intelligence")
 	flags.StringVar(&a.personalToken, "censys-pat", "", "censys API personal access token (PAT)")
 	flags.IntVar(&a.Threads, "censys-threads", 1, "how many enrichment threads to use. Note that free plan only allows 1 concurrent API request at a time")
 }

cmd/zannotate/main.go

@@ -16,6 +16,8 @@ package main
 
 import (
 	"flag"
+	"fmt"
+	"maps"
 	"os"
 	"slices"
 
@@ -28,11 +30,9 @@ func main() {
 
 	var conf zannotate.GlobalConf
 	flags := flag.NewFlagSet("flags", flag.ExitOnError)
-	flags.StringVar(&conf.InputFilePath, "input-file", "-", "ip addresses to read")
+	flags.StringVar(&conf.InputFilePath, "input-file", "-", "ip addresses to read, use '-' for std in")
 	flags.StringVar(&conf.InputFileType, "input-file-type", "ips", "ips, csv, json")
 	flags.StringVar(&conf.OutputFilePath, "output-file", "-", "where should JSON output be saved, defaults to stdout")
-	flags.StringVar(&conf.MetadataFilePath, "metadata-file", "",
-		"where should JSON metadata be saved")
 	flags.StringVar(&conf.LogFilePath, "log-file", "", "where should logs be saved, defaults to stderr")
 	flags.StringVar(&conf.StatusUpdatesFilePath, "status-file", "", "where should per-second status updates be saved, defaults to stderr")
 	flags.IntVar(&conf.Verbosity, "verbosity", 3, "log verbosity: 1 (lowest)--5 (highest)")
@@ -42,10 +42,7 @@ func main() {
 	// encode/decode threads
 	flags.IntVar(&conf.InputDecodeThreads, "input-decode-threads", 3, "number of golang processes to decode input data (e.g., json)")
 	flags.IntVar(&conf.OutputEncodeThreads, "output-encode-threads", 3, "number of golang processes to encode output data (e.g., json)")
-	// add the flags defined by each of the annotation modules
-	for _, annotator := range zannotate.Annotators {
-		annotator.AddFlags(flags)
-	}
+	prepareUsageString(flags) // Constructs the --help string
 	// parse
 	err := flags.Parse(os.Args[1:])
 	if err != nil {
@@ -95,3 +92,49 @@ func main() {
 	// perform annotation
 	zannotate.DoAnnotation(&conf)
 }
+
+func prepareUsageString(flags *flag.FlagSet) {
+	// Build grouped flag help: snapshot flags before each annotator registers its own.
+	type flagGroup struct {
+		name  string
+		flags []string
+	}
+	snapshot := func() map[string]bool {
+		seen := make(map[string]bool)
+		flags.VisitAll(func(f *flag.Flag) { seen[f.Name] = true })
+		return seen
+	}
+	groups := make([]flagGroup, 0, 1+len(zannotate.Annotators)) // Global options + each annotator
+	// Collect the global flags registered above.
+	globalFlags := snapshot()
+	globalNames := slices.Collect(maps.Keys(globalFlags))
+	groups = append(groups, flagGroup{"Global", globalNames})
+	// add the flags defined by each of the annotation modules
+	for _, annotator := range zannotate.Annotators {
+		pre := snapshot()
+		annotator.AddFlags(flags)
+		var added []string
+		flags.VisitAll(func(f *flag.Flag) {
+			if !pre[f.Name] {
+				added = append(added, f.Name)
+			}
+		})
+		if len(added) > 0 {
+			groups = append(groups, flagGroup{annotator.GroupName(), added})
+		}
+	}
+	flags.Usage = func() {
+		fmt.Fprintf(os.Stderr, "Usage: %s [global options] <-module [module-options]>...\n", os.Args[0])
+		fmt.Fprintf(os.Stderr, "At least one annotation module must be specified (e.g. -geoip2, -rdns, -routing).\n\n")
+		for _, g := range groups {
+			fmt.Fprintf(os.Stderr, "\n%s Options:\n", g.name)
+			// Create a temporary FlagSet containing only this group's flags so we
+			// can delegate formatting to PrintDefaults.
+			tmp := flag.NewFlagSet("", flag.ContinueOnError)
+			for _, name := range g.flags {
+				tmp.Var(flags.Lookup(name).Value, name, flags.Lookup(name).Usage)
+			}
+			tmp.PrintDefaults()
+		}
+	}
+}

conf.go

@@ -31,6 +31,7 @@ type Annotator interface {
 type AnnotatorFactory interface {
 	Initialize(c *GlobalConf) error
 	AddFlags(flags *flag.FlagSet)
+	GroupName() string
 	GetWorkers() int
 	IsEnabled() bool
 	MakeAnnotator(i int) Annotator
@@ -48,7 +49,6 @@ type GlobalConf struct {
 	InputFilePath             string
 	InputFileType             string
 	OutputFilePath            string
-	MetadataFilePath          string
 	LogFilePath               string
 	StatusUpdatesFilePath     string
 	Verbosity                 int

geoip.go

@@ -92,6 +92,8 @@ type GeoIP2Annotator struct {
 }
 
 // GeoIP2 Annotator Factory (Global)
+func (a *GeoIP2AnnotatorFactory) GroupName() string { return "GeoIP2" }
+
 func (a *GeoIP2AnnotatorFactory) AddFlags(flags *flag.FlagSet) {
 	flags.BoolVar(&a.Enabled, "geoip2", false, "annotate with Maxmind GeoIP2/GeoLite data")
 	flags.StringVar(&a.Path, "geoip2-database", "",

geoipasn.go

@@ -41,11 +41,13 @@ type GeoIPASNAnnotator struct {
 	Id      int
 }
 
+func (fact *GeoIPASNAnnotatorFactory) GroupName() string { return "GeoASN" }
+
 func (fact *GeoIPASNAnnotatorFactory) AddFlags(flags *flag.FlagSet) {
 	flags.BoolVar(&fact.Enabled, "geoasn", false, "annotate with Maxmind GeoLite/GeoIP ASN data")
 	flags.StringVar(&fact.Path, "geoasn-database", "", "path to Maxmind ASN database")
 	flags.StringVar(&fact.Mode, "geoasn-mode", "mmap", "how to open database: 'mmap' or 'memory'")
-	flags.IntVar(&fact.Threads, "geoasn-threads", 5, "how many geoASN processing threads to use")
+	flags.IntVar(&fact.Threads, "geoasn-threads", 5, "how many processing threads to use")
 }
 
 func (fact *GeoIPASNAnnotatorFactory) IsEnabled() bool {

greynoise_psychic.go

@@ -71,8 +71,10 @@ func (a *GreyNoiseAnnotatorFactory) IsEnabled() bool {
 	return a.Enabled
 }
 
+func (a *GreyNoiseAnnotatorFactory) GroupName() string { return "GreyNoise" }
+
 func (a *GreyNoiseAnnotatorFactory) AddFlags(flags *flag.FlagSet) {
-	flags.BoolVar(&a.Enabled, "greynoise", false, "greynoise psychic data intelligence")
+	flags.BoolVar(&a.Enabled, "greynoise", false, "annotate with greynoise psychic data intelligence")
 	flags.StringVar(&a.DBPath, "greynoise-database", "", "path to greynoise psychic .mmdb file")
 	flags.IntVar(&a.Threads, "greynoise-threads", 2, "how many enrichment threads to use")
 }

ipinfo.go

@@ -236,11 +236,13 @@ func (a *IPInfoAnnotatorFactory) IsEnabled() bool {
 	return a.Enabled
 }
 
+func (a *IPInfoAnnotatorFactory) GroupName() string { return "IPInfo" }
+
 func (a *IPInfoAnnotatorFactory) AddFlags(flags *flag.FlagSet) {
 	flags.BoolVar(&a.Enabled, "ipinfo", false, "annotate with IPInfo.io data using a local MaxMind DB file")
 	flags.StringVar(&a.DatabaseFilePath, "ipinfo-database", "", "path to MaxMind DB data file for IPInfo.io annotation")
 	// On a quick benchmark of 1M IPs using a local DB file on a M2 Macbook Air, 1 thread vs. 10 threads were about the same speed, annotating about 212k IPs/second.
-	flags.IntVar(&a.Threads, "ipinfo-threads", 1, "how many ipinfo annotator threads")
+	flags.IntVar(&a.Threads, "ipinfo-threads", 1, "how many annotator threads")
 }
 
 // IPInfo Annotator (Per-Worker)

rdap.go

@@ -35,9 +35,11 @@ type RDAPAnnotator struct {
 }
 
 // RDAP Annotator Factory (Global)
+func (a *RDAPAnnotatorFactory) GroupName() string { return "RDAP" }
+
 func (a *RDAPAnnotatorFactory) AddFlags(flags *flag.FlagSet) {
 	flags.BoolVar(&a.Enabled, "rdap", false, "annotate with RDAP (successor to WHOIS) lookup")
-	flags.IntVar(&a.Threads, "rdap-threads", 5, "how many rdap processing threads to use")
+	flags.IntVar(&a.Threads, "rdap-threads", 5, "how many processing threads to use")
 	flags.IntVar(&a.Timeout, "rdap-timeout", 5, "RDAP query timeout in seconds")
 }
 

rdns.go

@@ -46,6 +46,8 @@ type RDNSAnnotator struct {
 
 // RDNS Annotator Factory (Global)
 
+func (a *RDNSAnnotatorFactory) GroupName() string { return "Reverse DNS" }
+
 func (a *RDNSAnnotatorFactory) MakeAnnotator(i int) Annotator {
 	var v RDNSAnnotator
 	v.Factory = a
@@ -99,9 +101,9 @@ func (a *RDNSAnnotatorFactory) IsEnabled() bool {
 
 func (a *RDNSAnnotatorFactory) AddFlags(flags *flag.FlagSet) {
 	// Reverse DNS Lookup
-	flags.BoolVar(&a.Enabled, "rdns", false, "reverse dns lookup")
+	flags.BoolVar(&a.Enabled, "rdns", false, "annotate with reverse dns lookup")
 	flags.StringVar(&a.RawResolvers, "rdns-dns-servers", "", "list of DNS servers to use for DNS lookups, comma-separated IP list. If empty, will use system defaults")
-	flags.IntVar(&a.Threads, "rdns-threads", 100, "how many reverse dns threads")
+	flags.IntVar(&a.Threads, "rdns-threads", 100, "how many annotation threads to use")
 	flags.IntVar(&a.timeoutSecs, "rdns-timeout", 2, "timeout for each rdns query, in seconds")
 }
 

routing.go

@@ -39,13 +39,15 @@ type RoutingAnnotator struct {
 }
 
 // Routing Annotator Factory (Global)
+func (a *RoutingAnnotatorFactory) GroupName() string { return "Routing/BGP" }
+
 func (a *RoutingAnnotatorFactory) AddFlags(flags *flag.FlagSet) {
 	flags.BoolVar(&a.Enabled, "routing", false, "annotate with origin AS lookup")
 	flags.StringVar(&a.RoutingTablePath, "routing-mrt-file", "",
 		"path to MRT TABLE_DUMPv2 file")
 	flags.StringVar(&a.ASNamesPath, "routing-as-names", "", "path to as names file")
 	flags.IntVar(&a.Threads, "routing-threads", 5,
-		"how many routing processing threads to use")
+		"how many processing threads to use")
 }
 
 func (a *RoutingAnnotatorFactory) IsEnabled() bool {

spur.go

@@ -74,8 +74,10 @@ func (a *SpurAnnotatorFactory) IsEnabled() bool {
 	return a.Enabled
 }
 
+func (a *SpurAnnotatorFactory) GroupName() string { return "Spur" }
+
 func (a *SpurAnnotatorFactory) AddFlags(flags *flag.FlagSet) {
-	flags.BoolVar(&a.Enabled, "spur", false, "enrich with Spur's threat intelligence data")
+	flags.BoolVar(&a.Enabled, "spur", false, "annotate with Spur's threat intelligence data")
 	flags.IntVar(&a.Threads, "spur-threads", 100, "how many threads to use for Spur lookups")
 	flags.IntVar(&a.timeoutSecs, "spur-timeout", 2, "timeout for each Spur query, in seconds")
 }