Fix crash bugs, add CSV output, and remove exposed credentials

.env.example

@@ -0,0 +1,6 @@
+PORT=
+
+INFLUXDB_SERVER_URL=
+INFLUXDB_AUTH_TOKEN=
+INFLUXDB_ORG=
+INFLUXDB_BUCKET=frog_fleet

.github/workflows/secret-scan.yml

@@ -0,0 +1,16 @@
+name: Secret scan
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  gitleaks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: gitleaks/gitleaks-action@v2
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -1,2 +1,3 @@
 .idea/
-api
+api
+.env

Dockerfile

@@ -0,0 +1,10 @@
+FROM golang:1.22-alpine AS builder
+WORKDIR /app
+COPY go.mod go.sum ./
+RUN go mod download
+COPY . .
+RUN CGO_ENABLED=0 GOOS=linux go build -o api .
+
+FROM gcr.io/distroless/static-debian12
+COPY --from=builder /app/api /api
+ENTRYPOINT ["/api"]

README.md

@@ -1,38 +1,103 @@
-# Ribbit Network API (WIP)
+# Ribbit Network API
 
-A public API for global CO2 measurements, powered by the Ribbit Network.
+A public API for global CO2 measurements, powered by the [Ribbit Network](https://ribbitnetwork.org) — an open-source network of citizen-operated CO2 sensors.
 
-## Example
+## Endpoints
+
+### `GET /`
+
+Health check. Returns `🐸`.
+
+---
+
+### `GET /data`
+
+Returns CO2, temperature, humidity, and location measurements from the sensor network for a given time range.
+
+#### Query parameters
+
+| Parameter  | Required | Description |
+|------------|----------|-------------|
+| `start`    | yes      | Start of time range (RFC 3339, e.g. `2024-01-01T00:00:00Z`) |
+| `stop`     | no       | End of time range (RFC 3339). Omit to query through the present. |
+| `hosts`    | no       | Comma-separated list of sensor IDs to filter by |
+| `fields`   | no       | Comma-separated list of fields to return. Available fields: `co2`, `lat`, `lon`, `humidity`, `baro_pressure`, `baro_temperature`, `alt`. Omit to return all fields. |
+| `interval` | no       | Aggregate readings into windows of this duration (e.g. `5m`, `1h`). Uses mean aggregation. Omit for raw data. |
+| `format`   | no       | Set to `csv` to receive a CSV response instead of JSON. |
+
+You can also request CSV by sending `Accept: text/csv`.
+
+#### JSON response (default)
 
 ```
-GET /data?start=1970-01-01T00:00:00Z&stop=1970-01-01T00:00:10Z
-    &hosts=00000000000000000000000000000000,11111111111111111111111111111111
-    &fields=co2,lat,lon
-    &interval=5m
+GET /data?start=2024-01-01T00:00:00Z&stop=2024-01-02T00:00:00Z&fields=co2,lat,lon&interval=1h
 ```
 
 ```json
 {
     "data": [
         {
-            "host": "00000000000000000000000000000000",
-            "time": "1970-01-01T00:00:00.000000",
-            "co2": 0.0000000000000,
-            "lat": 0.00,
-            "lon": 0.00
+            "time": "2024-01-01T00:00:00Z",
+            "host": "a3f2...",
+            "co2": 412.5,
+            "lat": 37.77,
+            "lon": -122.41
         },
         ...
     ]
 }
 ```
 
-## Running the API
+#### CSV response
+
+```
+GET /data?start=2024-01-01T00:00:00Z&fields=co2,lat,lon&format=csv
+```
+
+```
+time,host,co2,lat,lon,humidity,baro_pressure,baro_temperature,alt
+2024-01-01T00:00:00Z,a3f2...,412.5,37.77,-122.41,,,, 
+...
+```
+
+Or with a header:
+
+```sh
+curl -H "Accept: text/csv" "https://<host>/data?start=2024-01-01T00:00:00Z"
+```
+
+## Running locally
+
+**Prerequisites:** [Go](https://go.dev/doc/install) 1.17+
+
+1. Clone the repo:
+   ```sh
+   git clone https://github.com/Ribbit-Network/api && cd api
+   ```
+
+2. Copy the example env file and fill in your InfluxDB credentials:
+   ```sh
+   cp .env.example .env
+   ```
+
+3. Run:
+   ```sh
+   go run main.go
+   ```
+
+The API will be available at `http://localhost:<PORT>`.
+
+## Environment variables
 
-1. Install the latest version of Go: https://go.dev/doc/install
-2. Fork and clone the API
-3. Run the API locally: `go run main.go`
+| Variable              | Description |
+|-----------------------|-------------|
+| `PORT`                | Port to listen on (e.g. `8080`) |
+| `INFLUXDB_SERVER_URL` | InfluxDB Cloud instance URL |
+| `INFLUXDB_AUTH_TOKEN` | InfluxDB API token (use a read-only token in production) |
+| `INFLUXDB_ORG`        | InfluxDB organization name or email |
+| `INFLUXDB_BUCKET`     | InfluxDB bucket name (`frog_fleet`) |
 
 ## Contributing
 
 Feel free to open an issue or PR!
-You can also join the developer discord [here](https://discord.com/invite/vq8PkDb2TC).
+You can also join the developer Discord [here](https://discord.com/invite/vq8PkDb2TC).

fly.toml

@@ -0,0 +1,16 @@
+[app]
+name = "ribbit-api"
+primary_region = "sjc"
+
+[build]
+
+[http_service]
+internal_port = 8080
+force_https = true
+auto_stop_machines = false
+auto_start_machines = true
+min_machines_running = 1
+
+[[vm]]
+size = "shared-cpu-1x"
+memory = "256mb"

internal/data/data.go

@@ -1,7 +1,9 @@
 package data
 
 import (
+	"encoding/csv"
 	"encoding/json"
+	"fmt"
 	"log"
 	"net/http"
 	"reflect"
@@ -24,15 +26,17 @@ type Data struct {
 	Temperature float64 `json:"baro_temperature,omitempty"`
 }
 
+var csvHeaders = []string{"time", "host", "co2", "lat", "lon", "humidity", "baro_pressure", "baro_temperature", "alt"}
+
 func Handle(w http.ResponseWriter, r *http.Request) {
 	if r.Method != http.MethodGet {
-		w.WriteHeader(http.StatusMethodNotAllowed)
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
 		return
 	}
 
 	q, err := NewQuery(r.URL.Query())
 	if err != nil {
-		w.WriteHeader(http.StatusBadRequest)
+		http.Error(w, err.Error(), http.StatusBadRequest)
 		return
 	}
 	log.Println(q)
@@ -42,7 +46,7 @@ func Handle(w http.ResponseWriter, r *http.Request) {
 
 	res, err := db.Query(q)
 	if err != nil {
-		w.WriteHeader(http.StatusInternalServerError)
+		http.Error(w, "query failed", http.StatusInternalServerError)
 		return
 	}
 	defer res.Close()
@@ -58,32 +62,76 @@ func Handle(w http.ResponseWriter, r *http.Request) {
 			continue
 		}
 
-		time := rec.Time()
-		host := rec.ValueByKey("host").(string)
+		t := rec.Time()
+		host, ok := rec.ValueByKey("host").(string)
+		if !ok {
+			continue
+		}
 
-		key := time.String() + host
+		key := t.String() + host
 		if _, ok := points[key]; !ok {
-			points[key] = &Data{Time: time, Host: host}
+			points[key] = &Data{Time: t, Host: host}
 		}
 
-		val := reflect.ValueOf(rec.Value())
-		reflect.ValueOf(points[key]).Elem().Field(idx).Set(val)
+		v := rec.Value()
+		if v == nil {
+			continue
+		}
+		val := reflect.ValueOf(v)
+		field := reflect.ValueOf(points[key]).Elem().Field(idx)
+		if val.Type().AssignableTo(field.Type()) {
+			field.Set(val)
+		}
 	}
 	if res.Err() != nil {
-		w.WriteHeader(http.StatusInternalServerError)
+		http.Error(w, "query result error", http.StatusInternalServerError)
 		return
 	}
 
-	data := &struct {
+	data := getValues(points)
+
+	wantsCSV := r.URL.Query().Get("format") == "csv" ||
+		strings.Contains(r.Header.Get("Accept"), "text/csv")
+
+	if wantsCSV {
+		writeCSV(w, data)
+	} else {
+		writeJSON(w, data)
+	}
+}
+
+func writeJSON(w http.ResponseWriter, data []*Data) {
+	w.Header().Set("Content-Type", "application/json")
+	payload := &struct {
 		Data []*Data `json:"data"`
-	}{
-		Data: getValues(points),
+	}{Data: data}
+	if err := json.NewEncoder(w).Encode(payload); err != nil {
+		log.Println("json encode error:", err)
 	}
+}
 
-	if err := json.NewEncoder(w).Encode(data); err != nil {
-		w.WriteHeader(http.StatusInternalServerError)
-		return
+func writeCSV(w http.ResponseWriter, data []*Data) {
+	w.Header().Set("Content-Type", "text/csv")
+	w.Header().Set("Content-Disposition", "attachment; filename=ribbit-data.csv")
+
+	cw := csv.NewWriter(w)
+	_ = cw.Write(csvHeaders)
+
+	for _, d := range data {
+		_ = cw.Write([]string{
+			d.Time.UTC().Format(time.RFC3339),
+			d.Host,
+			fmt.Sprintf("%g", d.CO2),
+			fmt.Sprintf("%g", d.Latitude),
+			fmt.Sprintf("%g", d.Longitude),
+			fmt.Sprintf("%g", d.Humidity),
+			fmt.Sprintf("%g", d.Pressure),
+			fmt.Sprintf("%g", d.Temperature),
+			fmt.Sprintf("%g", d.Altitude),
+		})
 	}
+
+	cw.Flush()
 }
 
 func getIndexByField() map[string]int {

main.go

@@ -12,13 +12,17 @@ import (
 
 func main() {
 	if err := godotenv.Load(); err != nil {
-		log.Fatal(err)
+		log.Println("no .env file, relying on environment variables")
 	}
 
 	http.HandleFunc("/", handle)
 	http.HandleFunc("/data", data.Handle)
 
-	addr := fmt.Sprintf(":%s", os.Getenv("PORT"))
+	port := os.Getenv("PORT")
+	if port == "" {
+		port = "8080"
+	}
+	addr := fmt.Sprintf(":%s", port)
 
 	log.Println("API running at http://localhost" + addr)
 	if err := http.ListenAndServe(addr, nil); err != nil {