migrate k8s to helm + elasticsearch to loki

.gitignore

@@ -259,3 +259,6 @@ __marimo__/
 
 # Streamlit
 .streamlit/secrets.toml
+
+# Helm charts
+deployments/helm/fafnir/charts/*.tgz

Makefile

@@ -1,10 +1,10 @@
 .PHONY: default help lint \
 		docker-run-auth-service docker-run-user-service docker-run-security-service docker-run-stock-service docker-run-api-gateway \
-        docker-prod docker-stats docker-run docker-build docker-start docker-pause docker-stop docker-status \
+        docker-prod docker-prod-build docker-stats docker-run docker-build docker-start docker-pause docker-stop docker-status \
         docker-logs docker-nats docker-rm-volumes docker-prune docker-clean docker-reset \
         migrate-up migrate-down migrate-status migrate-create \
         generate seed \
-        kube-start kube-stop kube-deploy kube-delete kube-reset \
+        kube-start kube-stop kube-delete kube-delete-pod kube-deploy kube-reset \
         kube-status kube-nodes kube-pods kube-svc kube-deployments kube-logs \
         kube-forward kube-tunnel \
         locust
@@ -28,6 +28,9 @@ lint:
 	@cd src/stock-service && golangci-lint run ./...
 	@echo "Linting shared..."
 	@cd src/shared && golangci-lint run ./...
+	@echo "Linting CLI tools..."
+	@cd tools/cli/seedctl && golangci-lint run ./...
+	@cd tools/cli/logctl && golangci-lint run ./...
 
 # ------------------------------
 # Docker Service Operations
@@ -55,14 +58,17 @@ docker-api-gateway:
 docker-prod: docker-clean
 	./tools/scripts/docker.sh prod
 
+docker-prod-build:
+	./tools/scripts/docker.sh build-prod
+
 docker-stats:
 	./tools/scripts/docker.sh stats
 
 docker-run:
 	./tools/scripts/docker.sh run $(monitoring)
 
 docker-build:
-	./tools/scripts/docker.sh build
+	./tools/scripts/docker.sh build $(monitoring)
 
 docker-start:
 	./tools/scripts/docker.sh start
@@ -137,11 +143,26 @@ kube-start:
 kube-stop:
 	minikube stop -p fafnir-cluster
 
+kube-delete:
+	minikube delete -p fafnir-cluster
+
+kube-uninstall:
+	./tools/scripts/k8s.sh uninstall
+
+kube-secrets:
+	./tools/scripts/k8s.sh secrets
+
+kube-docker:
+	./tools/scripts/k8s.sh docker
+
 kube-deploy:
-	./tools/scripts/k8s.sh deploy $(pod)
+	./tools/scripts/k8s.sh deploy
 
-kube-delete:
-	./tools/scripts/k8s.sh delete
+kube-upgrade:
+	./tools/scripts/k8s.sh upgrade
+
+kube-delete-pod:
+	./tools/scripts/k8s.sh delete $(pod)
 
 kube-reset:
 	./tools/scripts/k8s.sh reset $(pod)

README.md

@@ -34,5 +34,7 @@ In no particular order:
 - [ ] Add unit, integration, end-to-end and load/stress tests.
   - [X] Perform load testing using Locust to simulate concurrent users ([issue #8](https://github.com/andrearcaina/fafnir/issues/8)).
 - [X] Explore Kubernetes local implementation ([issue #5](https://github.com/andrearcaina/fafnir/issues/5)).
-- [X] Explore centralized logging CLI with Elasticsearch for Minikube ([issue #6](https://github.com/andrearcaina/fafnir/issues/6)).
+  - [X] Use Helm for Kubernetes package management and manifests ([issue #29](https://github.com/andrearcaina/fafnir/issues/29)).
+- [X] Explore centralizing logging with Elasticsearch ([issue #6](https://github.com/andrearcaina/fafnir/issues/6)).
+  - [X] Migrate to utilizing [Loki](https://grafana.com/oss/loki/) for both Minikube and docker-compose ([issue #29](https://github.com/andrearcaina/fafnir/issues/29)).
 - [ ] Implement a CI/CD pipeline for automated testing and Docker builds.

deployments/docker/compose.dev.yml

@@ -1,9 +1,9 @@
 services:
     db:
-        image: fafnir-postgres-db
         build:
             context: ../../
             dockerfile: build/docker/postgres.Dockerfile
+        container_name: fafnir-postgres-db
         restart: always
         ports:
             - "5432:5432"
@@ -145,8 +145,9 @@ services:
             - STOCK_DB=${STOCK_DB}
             - DB_HOST=${DB_HOST_DOCKER}
             - DB_PORT=${DB_PORT}
-            - REDIS_HOST=redis
-            - REDIS_PORT=6379
+            - REDIS_HOST=${REDIS_HOST_DOCKER}
+            - REDIS_PORT=${REDIS_PORT}
+            - REDIS_PASSWORD=${REDIS_PASSWORD}
             - FMP_API_KEY=${FMP_API_KEY}
         volumes:
             - ../../src/stock-service:/app/src/stock-service:cached

deployments/docker/monitoring.yml

@@ -33,6 +33,29 @@ services:
     depends_on:
       - prometheus
 
+
+  loki:
+    image: grafana/loki:latest
+    container_name: fafnir-loki
+    ports:
+      - "3100:3100"
+    command: -config.file=/etc/loki/local-config.yaml
+    networks:
+      - fafnir-network
+
+  promtail:
+    image: grafana/promtail:latest
+    container_name: fafnir-promtail
+    volumes:
+      - ../../infra/monitoring/promtail-config.yml:/etc/promtail/config.yml
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+    command: -config.file=/etc/promtail/config.yml
+    networks:
+      - fafnir-network
+    depends_on:
+      - loki
+
 volumes:
   prom_data:
     driver: local

deployments/helm/fafnir/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

deployments/helm/fafnir/Chart.lock

@@ -0,0 +1,18 @@
+dependencies:
+- name: postgresql
+  repository: https://charts.bitnami.com/bitnami
+  version: 16.2.0
+- name: redis
+  repository: https://charts.bitnami.com/bitnami
+  version: 20.2.0
+- name: nats
+  repository: https://nats-io.github.io/k8s/helm/charts/
+  version: 1.2.4
+- name: loki
+  repository: https://grafana.github.io/helm-charts
+  version: 6.2.0
+- name: promtail
+  repository: https://grafana.github.io/helm-charts
+  version: 6.15.3
+digest: sha256:2234ab09828598f9ab1bf3c902230331a0743a507b697dd6c9ab214f271e2199
+generated: "2026-01-21T22:28:01.017425098-05:00"

deployments/helm/fafnir/Chart.yaml

@@ -0,0 +1,40 @@
+apiVersion: v2
+name: fafnir
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.0.0"
+dependencies:
+  - name: postgresql
+    version: 16.2.0
+    repository: https://charts.bitnami.com/bitnami
+  - name: redis
+    version: 20.2.0
+    repository: https://charts.bitnami.com/bitnami
+  - name: nats
+    version: 1.2.4
+    repository: https://nats-io.github.io/k8s/helm/charts/
+  - name: loki
+    version: 6.2.0
+    repository: https://grafana.github.io/helm-charts
+  - name: promtail
+    version: 6.15.3
+    repository: https://grafana.github.io/helm-charts

deployments/helm/fafnir/templates/_helpers.tpl

@@ -0,0 +1,23 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "fafnir.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+*/}}
+{{- define "fafnir.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}

deployments/helm/fafnir/templates/deployments-services.yaml

@@ -0,0 +1,31 @@
+{{- range $name, $svc := .Values.services }}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ $name }}
+  labels:
+    app: {{ $name }}
+    chart: {{ $.Chart.Name }}
+    release: {{ $.Release.Name }}
+spec:
+  replicas: {{ $svc.replicas | default 1 }}
+  selector:
+    matchLabels:
+      app: {{ $name }}
+  template:
+    metadata:
+      labels:
+        app: {{ $name }}
+    spec:
+      containers:
+        - name: {{ $name }}
+          image: {{ $svc.image }}
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: {{ $svc.containerPort }}
+          env:
+            {{- if $svc.env }}
+            {{- toYaml $svc.env | nindent 12 }}
+            {{- end }}
+{{- end }}

deployments/helm/fafnir/templates/postgres-init.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: postgres-init
+  namespace: {{ .Release.Namespace }}
+data:
+  init-dbs.sql: |
+    CREATE DATABASE auth_db;
+    CREATE DATABASE user_db;
+    CREATE DATABASE security_db;
+    CREATE DATABASE stock_db;
+    CREATE DATABASE order_db;
+    CREATE DATABASE portfolio_db;
+

deployments/helm/fafnir/templates/services-services.yaml

@@ -0,0 +1,17 @@
+{{- range $name, $svc := .Values.services }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ $name }}
+  labels:
+    app: {{ $name }}
+spec:
+  type: {{ $svc.type | default "ClusterIP" }}
+  selector:
+    app: {{ $name }}
+  ports:
+    - protocol: TCP
+      port: {{ $svc.servicePort }}
+      targetPort: {{ $svc.containerPort }}
+{{- end }}