Airflow metrics

ansible/deploy-airflow.yml

@@ -6,6 +6,7 @@
   roles:
     - nginx
     - dehydrated
+    - prometheus_statsd_exporter
     - oonidata_airflow
   vars:
     airflow_public_fqdn: "airflow.prod.ooni.io"

ansible/group_vars/airflow/vars.yml

@@ -10,3 +10,11 @@ airflow_fernet_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/airflo
 airflow_webserver_secret_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/airflow_webserver_secret_key', profile='oonidevops_user_prod') }}"
 airflow_executor: "LocalExecutor"
 airflow_database_conn: "postgresql+psycopg2://airflow:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/airflow_postgresql_password', profile='oonidevops_user_prod') }}@ooni-tier0-postgres.c7mgscca82no.eu-central-1.rds.amazonaws.com/airflow"
+
+# StatsD metrics, sent to the local prometheus statsd_exporter
+# These variables come from the airflow role used by this playbook
+# https://github.com/ooni/airflow-role/blob/5ba757e17c3fd63a4dc794a74ad1912f624d84d7/defaults/main/airflow-cfg.yml#L85-L95
+airflow_statsd_on: True
+airflow_statsd_host: localhost
+airflow_statsd_port: 8125
+airflow_statsd_prefix: airflow

ansible/roles/oonidata_airflow/tasks/main.yml

@@ -2,7 +2,7 @@
   ansible.builtin.apt:
     name:
       - build-essential
-      - python-dev
+      - python3-dev
       - g++
     state: latest
     update_cache: yes
@@ -71,6 +71,7 @@
     airflow_extra_packages:
       - postgres
       - virtualenv
+      - statsd
     airflow_services:
       airflow_webserver:
         service_name: airflow-webserver

ansible/roles/oonidata_airflow/templates/nginx-airflow.j2

@@ -20,6 +20,18 @@ server {
 
     add_header Access-Control-Allow-Origin *;
 
+    ## Prometheus statsd_exporter metrics
+    location /metrics/statsd_exporter {
+        auth_basic           "Administrator's Area";
+        auth_basic_user_file /etc/ooni/prometheus_passwd;
+
+        proxy_pass http://127.0.0.1:9102/metrics;
+
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header Host $host;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    }
+
     ## Airflow reverse proxy
     location / {
         proxy_pass http://127.0.0.1:8080;

ansible/roles/prometheus/templates/prometheus.yml

@@ -429,4 +429,14 @@ scrape_configs:
         replacement: "/$2"
         target_label: "__metrics_path__"
         action: "replace"
+
+  - job_name: "airflow"
+    scheme: https
+    metrics_path: /metrics/statsd_exporter
+    basic_auth:
+      username: 'prom'
+      password: '{{ prometheus_metrics_password }}'
+    static_configs:
+      - targets:
+        - airflow.prod.ooni.io
 ...

ansible/roles/prometheus_statsd_exporter/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+statsd_exporter_filename: "statsd_exporter-0.28.0.linux-amd64"
+statsd_exporter_download_url: "https://github.com/prometheus/statsd_exporter/releases/download/v0.28.0/{{statsd_exporter_filename}}.tar.gz"
+statsd_exporter_bin_path: /usr/local/bin/statsd_exporter
+statsd_exporter_statsd_listen_udp: ':8125'
+statsd_exporter_web_listen_address: '127.0.0.1:9102'

ansible/roles/prometheus_statsd_exporter/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: restart statsd_exporter
+  ansible.builtin.systemd_service:
+    name: statsd_exporter
+    state: restarted

ansible/roles/prometheus_statsd_exporter/tasks/main.yml

@@ -0,0 +1,56 @@
+---
+- name: Create statsd_exporter group
+  ansible.builtin.group:
+    name: statsd_exporter
+    state: present
+  become: true
+
+- name: Create statsd_exporter user
+  ansible.builtin.user:
+    name: statsd_exporter
+    group: statsd_exporter
+    shell: /sbin/nologin
+    system: true
+    createhome: false
+  become: true
+
+- name: Check current statsd_exporter version
+  ansible.builtin.command: "{{ statsd_exporter_bin_path }} --version"
+  failed_when: false
+  changed_when: false
+  register: exporter_installed
+
+# We can't install it with apt-get, use the binary from github
+- name: Download and unarchive statsd_exporter
+  ansible.builtin.unarchive:
+    src: "{{ statsd_exporter_download_url }}"
+    dest: /tmp/
+    remote_src: true
+    mode: "0755"
+  when: exporter_installed.rc != 0 # return code (rc) is != 0 when not installed
+  become: true
+
+- name: Move statsd_exporter binary into place
+  ansible.builtin.copy:
+    src: "/tmp/{{ statsd_exporter_filename }}/statsd_exporter"
+    dest: "{{ statsd_exporter_bin_path }}"
+    mode: "0755"
+    remote_src: true
+  when: exporter_installed.rc != 0
+  notify: restart statsd_exporter
+  become: true
+
+- name: Create statsd_exporter systemd unit
+  ansible.builtin.template:
+    src: statsd_exporter.service
+    dest: /etc/systemd/system/statsd_exporter.service
+    mode: "0644"
+  become: true
+  notify: restart statsd_exporter
+
+- name: Ensure statsd_exporter is running and enabled
+  ansible.builtin.service:
+    name: statsd_exporter
+    state: started
+    enabled: true
+  become: true

ansible/roles/prometheus_statsd_exporter/templates/statsd_exporter.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=Prometheus StatsD Exporter
+After=network.target
+
+[Service]
+TimeoutStartSec=0
+User=statsd_exporter
+ExecStart={{ statsd_exporter_bin_path }} \
+  --statsd.listen-udp={{ statsd_exporter_statsd_listen_udp }} \
+  --web.listen-address={{ statsd_exporter_web_listen_address }}
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target