Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -26,4 +26,7 @@ haystack

benchmark_results*

gosec-report.json
gosec-report.json

# Scratch directory for testing
scratch/
3 changes: 1 addition & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,10 @@ RUN CGO_ENABLED=0 go build \
-o haystack \
./cmd/haystack

FROM scratch
FROM cgr.dev/chainguard/static:latest

COPY --from=builder /build/haystack /haystack

# scratch doesn't have mkdir, so WORKDIR creates the directory
WORKDIR /data

EXPOSE 1337/udp
Expand Down
9 changes: 6 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,11 @@ go install github.com/nomasters/haystack@latest
# Default configuration (localhost:1337)
haystack serve

# Custom host and port
haystack serve -H 0.0.0.0 -p 9000
# Custom address
haystack serve -a 0.0.0.0:9000

# Using environment variables
HAYSTACK_ADDR=0.0.0.0:9000 haystack serve
```

### Client Operations
Expand Down Expand Up @@ -161,7 +164,7 @@ docker run -p 1337:1337/udp nomasters/haystack:main serve
docker run -p 1337:1337/udp nomasters/haystack:v0.1.0 serve

# Run with custom configuration
docker run -p 1337:1337/udp nomasters/haystack:v0.1.0 serve -H 0.0.0.0 -p 1337
docker run -p 1337:1337/udp -e HAYSTACK_ADDR=0.0.0.0:1337 nomasters/haystack:v0.1.0 serve
```

#### Available Tags
Expand Down
11 changes: 6 additions & 5 deletions client/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -110,14 +110,14 @@ func (c *Client) SetBytes(ctx context.Context, data []byte) error {
}
defer c.connPool.Put(conn)

// Set write timeout
// Set timeout (using SetDeadline like GET does)
if deadline, ok := ctx.Deadline(); ok {
if err := conn.SetWriteDeadline(deadline); err != nil {
c.logger.Errorf("Failed to set write deadline: %v", err)
if err := conn.SetDeadline(deadline); err != nil {
c.logger.Errorf("Failed to set deadline: %v", err)
}
} else {
if err := conn.SetWriteDeadline(time.Now().Add(c.writeTimeout)); err != nil {
c.logger.Errorf("Failed to set write timeout: %v", err)
if err := conn.SetDeadline(time.Now().Add(c.writeTimeout)); err != nil {
c.logger.Errorf("Failed to set timeout: %v", err)
}
}

Expand Down Expand Up @@ -300,6 +300,7 @@ func (p *connectionPool) MarkBad(conn net.Conn) {

// createConn creates a new connection.
func (p *connectionPool) createConn() (*pooledConn, error) {
// Use regular Dial for connected UDP socket
conn, err := net.Dial("udp", p.address)
if err != nil {
return nil, err
Expand Down
3 changes: 3 additions & 0 deletions client/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ func TestClient_SetAndGet(t *testing.T) {
retrievedNeedle, err := client.Get(ctx, hash)
if err != nil {
t.Errorf("GET operation failed: %v", err)
return // Exit early if GET failed
}

if retrievedNeedle.Hash() != testNeedle.Hash() {
Expand Down Expand Up @@ -723,6 +724,8 @@ func (m *mockLogger) Error(v ...any) {}
func (m *mockLogger) Errorf(format string, v ...any) {}
func (m *mockLogger) Info(v ...any) {}
func (m *mockLogger) Infof(format string, v ...any) {}
func (m *mockLogger) Debug(v ...any) {}
func (m *mockLogger) Debugf(format string, v ...any) {}

func TestPool_OverflowAndClosedScenarios(t *testing.T) {
serverAddr := startTestServer(t)
Expand Down
16 changes: 15 additions & 1 deletion cmd/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -142,8 +142,22 @@ EXAMPLES:
os.Exit(1)
}

// Output hash based on format
// Verify the SET by immediately doing a GET
// This ensures the packet was transmitted and stored
hash := n.Hash()
verifyCtx, verifyCancel := context.WithTimeout(context.Background(), 5*time.Second)
defer verifyCancel()

verifiedNeedle, err := c.Get(verifyCtx, hash)
if err != nil {
// SET might have succeeded but verification failed
// Still output the hash but warn the user
fmt.Fprintf(os.Stderr, "Warning: Could not verify SET operation (GET failed): %v\n", err)
} else if verifiedNeedle.Hash() != hash {
fmt.Fprintf(os.Stderr, "Warning: SET verification returned different hash\n")
}

// Output hash based on format
hashHex := hex.EncodeToString(hash[:])
switch *format {
case "hex":
Expand Down
79 changes: 60 additions & 19 deletions cmd/serve.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,12 @@ func runServe(args []string) {
fs := flag.NewFlagSet("serve", flag.ExitOnError)

// Define flags
port := fs.String("port", "1337", "Port for the server listener")
fs.StringVar(port, "p", "1337", "Port for the server listener (shorthand)")
host := fs.String("host", "", "Hostname of server listener")
storageType := fs.String("storage", "memory", "Storage backend: memory or mmap")
dataDir := fs.String("data-dir", "./data", "Data directory for mmap storage")
quiet := fs.Bool("quiet", false, "Disable logging output")
addr := fs.String("addr", getAddr(), "Server address (host:port)")
fs.StringVar(addr, "a", getAddr(), "Server address (host:port) (shorthand)")
storageType := fs.String("storage", getStorage(), "Storage backend: memory or mmap")
dataDir := fs.String("data-dir", getDataDir(), "Data directory for mmap storage")
logLevel := fs.String("log-level", getLogLevel(), "Log level: debug, info, error, or silent")
quiet := fs.Bool("quiet", false, "Disable logging output (same as --log-level=silent)")
fs.BoolVar(quiet, "q", false, "Disable logging output (shorthand)")
help := fs.Bool("help", false, "Show server command help")
fs.BoolVar(help, "h", false, "Show server command help (shorthand)")
Expand All @@ -38,18 +38,24 @@ USAGE:
haystack serve [options]

OPTIONS:
-p, --port <port> Port for the server listener (default: 1337)
--host <host> Hostname of server listener (default: "")
--storage <type> Storage backend: memory or mmap (default: memory)
--data-dir <path> Data directory for mmap storage (default: ./data)
-q, --quiet Disable logging output
-a, --addr <addr> Server address (host:port) (default: %s)
--storage <type> Storage backend: memory or mmap (default: %s)
--data-dir <path> Data directory for mmap storage (default: %s)
--log-level <level> Log level: debug, info, error, or silent (default: %s)
-q, --quiet Disable logging output (same as --log-level=silent)
-h, --help Show this help message

ENVIRONMENT VARIABLES:
HAYSTACK_ADDR Server address (overridden by --addr)
HAYSTACK_STORAGE Storage backend (overridden by --storage)
HAYSTACK_DATA_DIR Data directory (overridden by --data-dir)
HAYSTACK_LOG_LEVEL Log level (overridden by --log-level)

DESCRIPTION:
Server mode is used to run long-lived haystack servers.
Memory storage keeps data in RAM only.
MMAP storage persists data to disk using memory-mapped files.
`)
`, getAddr(), getStorage(), getDataDir(), getLogLevel())
}

// Parse flags
Expand All @@ -63,16 +69,19 @@ DESCRIPTION:
return
}

// Build address
addr := *host + ":" + *port
// Determine effective log level
effectiveLogLevel := *logLevel
if *quiet {
effectiveLogLevel = "silent"
}

// Set up logger based on quiet flag
// Set up logger based on log level
var log logger.Logger
if *quiet {
if effectiveLogLevel == "silent" {
log = logger.NewNoOp()
} else {
log = logger.New()
fmt.Printf("listening on: %s\n", addr)
log = logger.NewWithLevel(effectiveLogLevel)
fmt.Printf("listening on: %s (log level: %s)\n", *addr, effectiveLogLevel)
}

// Create storage backend based on type
Expand Down Expand Up @@ -116,7 +125,7 @@ DESCRIPTION:

// Handle graceful shutdown
go func() {
if err := srv.ListenAndServe(addr); err != nil {
if err := srv.ListenAndServe(*addr); err != nil {
log.Fatalf("Server error: %v", err)
}
}()
Expand Down Expand Up @@ -147,3 +156,35 @@ DESCRIPTION:
fmt.Println("Server stopped")
}
}

// getAddr returns the server address from environment or default
func getAddr() string {
if addr := os.Getenv("HAYSTACK_ADDR"); addr != "" {
return addr
}
return ":1337"
}

// getStorage returns the storage type from environment or default
func getStorage() string {
if storage := os.Getenv("HAYSTACK_STORAGE"); storage != "" {
return storage
}
return "memory"
}

// getDataDir returns the data directory from environment or default
func getDataDir() string {
if dataDir := os.Getenv("HAYSTACK_DATA_DIR"); dataDir != "" {
return dataDir
}
return "./data"
}

// getLogLevel returns the log level from environment or default
func getLogLevel() string {
if level := os.Getenv("HAYSTACK_LOG_LEVEL"); level != "" {
return level
}
return "info"
}
5 changes: 5 additions & 0 deletions examples/deployments/fly.io/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# Generated files
fly.toml.generated

# Fly.io state
.fly/
Loading