initial commit

.gitignore

@@ -0,0 +1,2 @@
+prometheus.yaml
+recording_rules.yaml

README.md

@@ -0,0 +1,11 @@
+# Grafana + Prometheus testbed for the Open Code Quest leaderboard
+
+## Installation
+
+```sh
+sudo ./run.sh
+```
+
+## Usage
+
+Open http://localhost:3000/ and login with admin/admin

podman-compose.yaml

@@ -0,0 +1,45 @@
+version: '3.8'
+
+networks:
+  monitoring: {}
+
+volumes:
+  prometheus_data: {}
+  grafana_data: {}
+
+services:
+  grafana:
+    image: docker.io/grafana/grafana-oss:latest
+    container_name: grafana
+    restart: unless-stopped
+    volumes:
+      - grafana_data:/var/lib/grafana
+    environment:
+    - GF_LOG_LEVEL=debug
+    ports:
+      - 3000:3000
+    expose:
+      - 3000
+    networks:
+      - monitoring
+
+  prometheus:
+    image: docker.io/prom/prometheus:latest
+    container_name: prometheus
+    restart: unless-stopped
+    volumes:
+      - ./prometheus/prometheus.yaml:/etc/prometheus/prometheus.yaml:z
+      - ./prometheus/recording_rules.yaml:/etc/prometheus/recording_rules.yaml:z
+      - prometheus_data:/prometheus
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yaml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--web.console.libraries=/etc/prometheus/console_libraries'
+      - '--web.console.templates=/etc/prometheus/consoles'
+      - '--web.enable-lifecycle'
+    expose:
+      - 9090
+    ports:
+      - 9090:9090
+    networks:
+      - monitoring

prometheus/prometheus.yaml.template

@@ -0,0 +1,12 @@
+global:
+  scrape_interval: ${INTERVAL}
+  evaluation_interval: ${INTERVAL}
+
+scrape_configs:
+  - job_name: 'prometheus'
+    scrape_interval: ${INTERVAL}
+    static_configs:
+      - targets: ['localhost:9090']
+
+rule_files:
+  - "/etc/prometheus/recording_rules.yaml"

prometheus/recording_rules.yaml.template

@@ -0,0 +1,157 @@
+groups:
+  - name: opencodequest_base
+    rules:
+    - record: opencodequest_users
+      expr: |
+        sum(
+          timestamp(
+            label_replace(up{instance="localhost:9090"}, "user", "superman", "","")
+            or 
+            label_replace(up{instance="localhost:9090"}, "user", "invisibleman", "","")
+            or
+            label_replace(up{instance="localhost:9090"}, "user", "catwoman", "","")
+            or
+            label_replace(up{instance="localhost:9090"}, "user", "batman", "","")
+          ) >= bool ${TS_USERS_JOIN}
+        ) by (user)
+    - record: opencodequest_devspaces_workspaces
+      expr: |
+        sum(
+          timestamp(
+            label_replace(up{instance="localhost:9090"}, "user", "superman", "","")
+            or 
+            label_replace(up{instance="localhost:9090"}, "user", "invisibleman", "","")
+            or
+            label_replace(up{instance="localhost:9090"}, "user", "catwoman", "","")
+            or
+            label_replace(up{instance="localhost:9090"}, "user", "batman", "","")
+          ) >= bool ${TS_DEVSPACES}
+        ) by (user)
+    - record: opencodequest_leaderboard_hero:prod
+      expr: |
+        sum(
+          timestamp(label_replace(up{instance="localhost:9090"}, "user", "superman", "","")) >= bool ${TS_USER_SUPERMAN_HERO}
+          or 
+          timestamp(label_replace(up{instance="localhost:9090"}, "user", "catwoman", "","")) >= bool ${TS_USER_CATWOMAN_HERO}
+          or
+          timestamp(label_replace(up{instance="localhost:9090"}, "user", "invisibleman", "","")) >= bool ${TS_USER_INVISIBLEMAN_HERO}
+          or
+          timestamp(label_replace(up{instance="localhost:9090"}, "user", "batman", "","")) >= bool ${TS_USER_BATMAN_HERO}
+        ) by (user)
+    - record: opencodequest_leaderboard_villain:prod
+      expr: |
+        sum(
+          timestamp(label_replace(up{instance="localhost:9090"}, "user", "superman", "","")) >= bool ${TS_USER_SUPERMAN_VILLAIN}
+          or 
+          timestamp(label_replace(up{instance="localhost:9090"}, "user", "catwoman", "","")) >= bool ${TS_USER_CATWOMAN_VILLAIN}
+          or
+          timestamp(label_replace(up{instance="localhost:9090"}, "user", "invisibleman", "","")) >= bool ${TS_USER_INVISIBLEMAN_VILLAIN}
+          or
+          timestamp(label_replace(up{instance="localhost:9090"}, "user", "batman", "","")) >= bool ${TS_USER_BATMAN_VILLAIN}
+        ) by (user)
+    - record: opencodequest_leaderboard_fight:prod
+      expr: |
+        sum(
+          timestamp(label_replace(up{instance="localhost:9090"}, "user", "superman", "","")) >= bool ${TS_USER_SUPERMAN_FIGHT}
+          or 
+          timestamp(label_replace(up{instance="localhost:9090"}, "user", "catwoman", "","")) >= bool ${TS_USER_CATWOMAN_FIGHT}
+          or
+          timestamp(label_replace(up{instance="localhost:9090"}, "user", "invisibleman", "","")) >= bool ${TS_USER_INVISIBLEMAN_FIGHT}
+          or
+          timestamp(label_replace(up{instance="localhost:9090"}, "user", "batman", "","")) >= bool ${TS_USER_BATMAN_FIGHT}
+        ) by (user)
+    - record: opencodequest_leaderboard_fight_call:prod
+      expr: |
+        sum(
+          timestamp(label_replace(up{instance="localhost:9090"}, "user", "superman", "","")) >= bool ${TS_USER_SUPERMAN_FIGHT_CALL}
+          or 
+          timestamp(label_replace(up{instance="localhost:9090"}, "user", "catwoman", "","")) >= bool ${TS_USER_CATWOMAN_FIGHT_CALL}
+          or
+          timestamp(label_replace(up{instance="localhost:9090"}, "user", "invisibleman", "","")) >= bool ${TS_USER_INVISIBLEMAN_FIGHT_CALL}
+          or
+          timestamp(label_replace(up{instance="localhost:9090"}, "user", "batman", "","")) >= bool ${TS_USER_BATMAN_FIGHT_CALL}
+        ) by (user) 
+  - name: opencodequest_step1
+    #query_offset: ${OFFSET_STEP1}s
+    rules:
+    - record: opencodequest_leaderboard_hero_onetime_bonus:prod
+      expr: |
+        (increase(opencodequest_leaderboard_hero:prod[10s]) >= bool 0.5)
+        *
+        (
+          55
+          +
+          sum(
+              (
+                (${TS_EXERCISE_HERO} + ${OFFSET_STEP1})
+                -
+                timestamp(
+                  label_replace(up{instance="localhost:9090"}, "user", "superman", "","")
+                  or 
+                  label_replace(up{instance="localhost:9090"}, "user", "invisibleman", "","")
+                  or
+                  label_replace(up{instance="localhost:9090"}, "user", "catwoman", "","")
+                  or
+                  label_replace(up{instance="localhost:9090"}, "user", "batman", "","")
+                )
+              ) / 5
+          ) by (user)
+        )
+    - record: opencodequest_leaderboard_villain_onetime_bonus:prod
+      expr: |
+        (increase(opencodequest_leaderboard_villain:prod[10s]) >= bool 0.5)
+        *
+        (
+          25
+          +
+          sum(
+              (
+                (${TS_EXERCISE_VILLAIN} + ${OFFSET_STEP1})
+                -
+                timestamp(
+                  label_replace(up{instance="localhost:9090"}, "user", "superman", "","")
+                  or 
+                  label_replace(up{instance="localhost:9090"}, "user", "invisibleman", "","")
+                  or
+                  label_replace(up{instance="localhost:9090"}, "user", "catwoman", "","")
+                  or
+                  label_replace(up{instance="localhost:9090"}, "user", "batman", "","")
+                )
+              ) / 5
+          ) by (user)
+        )
+    - record: opencodequest_leaderboard_fight_onetime_bonus:prod
+      expr: |
+        (increase(opencodequest_leaderboard_fight:prod[10s]) >= bool 0.5)
+        *
+        (
+          29
+          +
+          sum(
+              (
+                (${TS_EXERCISE_VILLAIN} + ${OFFSET_STEP1})
+                -
+                timestamp(
+                  label_replace(up{instance="localhost:9090"}, "user", "superman", "","")
+                  or 
+                  label_replace(up{instance="localhost:9090"}, "user", "invisibleman", "","")
+                  or
+                  label_replace(up{instance="localhost:9090"}, "user", "catwoman", "","")
+                  or
+                  label_replace(up{instance="localhost:9090"}, "user", "batman", "","")
+                )
+              ) / 5
+          ) by (user)
+        )
+  - name: opencodequest_step2
+    #query_offset: ${OFFSET_STEP2}s
+    rules:
+    - record: opencodequest_leaderboard_hero_lifetime_bonus:prod
+      expr: |
+        sum_over_time(opencodequest_leaderboard_hero_onetime_bonus:prod[1h])
+    - record: opencodequest_leaderboard_villain_lifetime_bonus:prod
+      expr: |
+        sum_over_time(opencodequest_leaderboard_villain_onetime_bonus:prod[1h])
+    - record: opencodequest_leaderboard_fight_lifetime_bonus:prod
+      expr: |
+        sum_over_time(opencodequest_leaderboard_fight_onetime_bonus:prod[1h])

run.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+set -Eeuo pipefail
+
+# Prometheus settings
+export SCALE="1" # 1 for seconds or 60 for minutes
+export INTERVAL="$((SCALE * 5))s"
+export OFFSET_STEP1="$((SCALE * 5))"
+export OFFSET_STEP2="$((SCALE * 10))"
+
+# Set the timings
+export TS_NOW="$EPOCHSECONDS"
+export TS_USERS_JOIN="$(( TS_NOW + SCALE * 10 ))"
+export TS_DEVSPACES="$(( TS_NOW + SCALE * 20 ))"
+
+# Superman is on time
+export TS_USER_SUPERMAN_HERO="$(( TS_NOW + SCALE * 55 ))"
+export TS_USER_SUPERMAN_VILLAIN="$(( TS_NOW + SCALE * 85 ))"
+export TS_USER_SUPERMAN_FIGHT="$(( TS_NOW + SCALE * 130 ))"
+export TS_USER_SUPERMAN_FIGHT_CALL="$(( TS_NOW + SCALE * 135 ))"
+
+# Catwoman is early
+export TS_USER_CATWOMAN_HERO="$(( TS_NOW + SCALE * 50 ))"
+export TS_USER_CATWOMAN_VILLAIN="$(( TS_NOW + SCALE * 75 ))"
+export TS_USER_CATWOMAN_FIGHT="$(( TS_NOW + SCALE * 115 ))"
+export TS_USER_CATWOMAN_FIGHT_CALL="$(( TS_NOW + SCALE * 120 ))"
+
+# Invisible Man is late
+export TS_USER_INVISIBLEMAN_HERO="$(( TS_NOW + SCALE * 60 ))"
+export TS_USER_INVISIBLEMAN_VILLAIN="$(( TS_NOW + SCALE * 95 ))"
+export TS_USER_INVISIBLEMAN_FIGHT="$(( TS_NOW + SCALE * 145 ))"
+export TS_USER_INVISIBLEMAN_FIGHT_CALL="$(( TS_NOW + SCALE * 150 ))"
+
+# Batman is late but will catch up
+export TS_USER_BATMAN_HERO="$(( TS_NOW + SCALE * 65 ))"
+export TS_USER_BATMAN_VILLAIN="$(( TS_NOW + SCALE * 70 ))"
+export TS_USER_BATMAN_FIGHT="$(( TS_NOW + SCALE * 100 ))"
+export TS_USER_BATMAN_FIGHT_CALL="$(( TS_NOW + SCALE * 110 ))"
+
+# Expected timings for the exercises
+export TS_EXERCISE_HERO="$(( TS_NOW + SCALE * 55 ))"
+export TS_EXERCISE_VILLAIN="$(( TS_NOW + SCALE * 85 ))"
+export TS_EXERCISE_FIGHT="$(( TS_NOW + SCALE * 130 ))"
+export TS_EXERCISE_FIGHT_CALL="$(( TS_NOW + SCALE * 135 ))"
+
+# Process the template
+envsubst < prometheus/prometheus.yaml.template > prometheus/prometheus.yaml
+envsubst < prometheus/recording_rules.yaml.template > prometheus/recording_rules.yaml
+
+# Reload recording rules and remove all data
+sudo podman stop prometheus || true
+sudo podman rm prometheus || true
+sudo podman volume rm prometheus-grafana-testbed_prometheus_data || true
+sudo podman-compose up -d
+sudo podman logs -f prometheus