From 05b1b04513e260b8e4f062d492e8241907223c7b Mon Sep 17 00:00:00 2001 From: David Rotermund Date: Tue, 27 Oct 2026 13:43:32 +0000 Subject: [PATCH] Statistics with Grafana/Prometheus with sensor based on Overleaf instance statistics- #178 --- tools/reference_stack_compose/MONITORING.md | 141 + tools/reference_stack_compose/compose.yaml | 79 + tools/reference_stack_compose/down_grafana.sh | 2 + .../down_instance-sensor.sh | 2 + .../down_mongo-probe.sh | 2 + .../down_mongodb-exporter-probe.sh | 2 + .../down_mongodb-exporter.sh | 2 + .../down_node-exporter.sh | 2 + .../down_prometheus.sh | 2 + .../down_redis-exporter-probe.sh | 2 + .../down_redis-exporter.sh | 2 + tools/reference_stack_compose/env.sample | 90 +- tools/reference_stack_compose/exec_grafana.sh | 1 + .../exec_instance-sensor.sh | 1 + .../exec_mongo-probe.sh | 1 + .../exec_mongodb-exporter-probe.sh | 1 + .../exec_node-exporter.sh | 1 + .../exec_prometheus.sh | 1 + .../exec_redis-exporter-probe.sh | 1 + .../exec_redis-exporter.sh | 1 + .../grafana/.gitignore | 1 + .../grafana/compose.yaml | 54 + .../grafana/dashboards/overview.json | 2441 +++++++++++++++++ .../provisioning/dashboards/ollitex.yml | 14 + .../provisioning/datasources/prometheus.yml | 12 + .../instance-sensor/Dockerfile | 20 + .../instance-sensor/compose.yaml | 52 + .../instance-sensor/package.json | 12 + .../instance-sensor/sensor.mjs | 214 ++ tools/reference_stack_compose/logs_grafana.sh | 2 + .../logs_instance-sensor.sh | 2 + .../logs_mongo-probe.sh | 2 + .../logs_mongodb-exporter-probe.sh | 2 + .../logs_mongodb-exporter.sh | 2 + .../logs_node-exporter.sh | 2 + .../logs_prometheus.sh | 2 + .../logs_redis-exporter-probe.sh | 2 + .../logs_redis-exporter.sh | 2 + .../mongo-probe/compose.yaml | 26 + .../mongodb-exporter-probe/compose.yaml | 22 + .../mongodb-exporter/compose.yaml | 37 + .../reference_stack_compose/nginx/nginx.conf | 20 + .../node-exporter/compose.yaml | 29 + .../prometheus/.gitignore | 1 + .../prometheus/compose.yaml | 34 + .../prometheus/prometheus.yml | 67 + .../redis-exporter-probe/compose.yaml | 22 + .../redis-exporter/compose.yaml | 28 + tools/reference_stack_compose/up_grafana.sh | 2 + .../up_instance-sensor.sh | 2 + .../reference_stack_compose/up_mongo-probe.sh | 2 + .../up_mongodb-exporter-probe.sh | 2 + .../up_mongodb-exporter.sh | 2 + .../up_node-exporter.sh | 2 + .../reference_stack_compose/up_prometheus.sh | 2 + .../up_redis-exporter-probe.sh | 2 + .../up_redis-exporter.sh | 2 + 57 files changed, 3456 insertions(+), 22 deletions(-) create mode 100644 tools/reference_stack_compose/MONITORING.md create mode 100644 tools/reference_stack_compose/down_grafana.sh create mode 100644 tools/reference_stack_compose/down_instance-sensor.sh create mode 100644 tools/reference_stack_compose/down_mongo-probe.sh create mode 100644 tools/reference_stack_compose/down_mongodb-exporter-probe.sh create mode 100644 tools/reference_stack_compose/down_mongodb-exporter.sh create mode 100644 tools/reference_stack_compose/down_node-exporter.sh create mode 100644 tools/reference_stack_compose/down_prometheus.sh create mode 100644 tools/reference_stack_compose/down_redis-exporter-probe.sh create mode 100644 tools/reference_stack_compose/down_redis-exporter.sh create mode 100755 tools/reference_stack_compose/exec_grafana.sh create mode 100755 tools/reference_stack_compose/exec_instance-sensor.sh create mode 100755 tools/reference_stack_compose/exec_mongo-probe.sh create mode 100755 tools/reference_stack_compose/exec_mongodb-exporter-probe.sh create mode 100755 tools/reference_stack_compose/exec_node-exporter.sh create mode 100755 tools/reference_stack_compose/exec_prometheus.sh create mode 100755 tools/reference_stack_compose/exec_redis-exporter-probe.sh create mode 100755 tools/reference_stack_compose/exec_redis-exporter.sh create mode 100644 tools/reference_stack_compose/grafana/.gitignore create mode 100644 tools/reference_stack_compose/grafana/compose.yaml create mode 100644 tools/reference_stack_compose/grafana/dashboards/overview.json create mode 100644 tools/reference_stack_compose/grafana/provisioning/dashboards/ollitex.yml create mode 100644 tools/reference_stack_compose/grafana/provisioning/datasources/prometheus.yml create mode 100644 tools/reference_stack_compose/instance-sensor/Dockerfile create mode 100644 tools/reference_stack_compose/instance-sensor/compose.yaml create mode 100644 tools/reference_stack_compose/instance-sensor/package.json create mode 100644 tools/reference_stack_compose/instance-sensor/sensor.mjs create mode 100755 tools/reference_stack_compose/logs_grafana.sh create mode 100755 tools/reference_stack_compose/logs_instance-sensor.sh create mode 100755 tools/reference_stack_compose/logs_mongo-probe.sh create mode 100755 tools/reference_stack_compose/logs_mongodb-exporter-probe.sh create mode 100755 tools/reference_stack_compose/logs_mongodb-exporter.sh create mode 100755 tools/reference_stack_compose/logs_node-exporter.sh create mode 100755 tools/reference_stack_compose/logs_prometheus.sh create mode 100755 tools/reference_stack_compose/logs_redis-exporter-probe.sh create mode 100755 tools/reference_stack_compose/logs_redis-exporter.sh create mode 100644 tools/reference_stack_compose/mongo-probe/compose.yaml create mode 100644 tools/reference_stack_compose/mongodb-exporter-probe/compose.yaml create mode 100644 tools/reference_stack_compose/mongodb-exporter/compose.yaml create mode 100644 tools/reference_stack_compose/node-exporter/compose.yaml create mode 100644 tools/reference_stack_compose/prometheus/.gitignore create mode 100644 tools/reference_stack_compose/prometheus/compose.yaml create mode 100644 tools/reference_stack_compose/prometheus/prometheus.yml create mode 100644 tools/reference_stack_compose/redis-exporter-probe/compose.yaml create mode 100644 tools/reference_stack_compose/redis-exporter/compose.yaml create mode 100644 tools/reference_stack_compose/up_grafana.sh create mode 100644 tools/reference_stack_compose/up_instance-sensor.sh create mode 100644 tools/reference_stack_compose/up_mongo-probe.sh create mode 100644 tools/reference_stack_compose/up_mongodb-exporter-probe.sh create mode 100644 tools/reference_stack_compose/up_mongodb-exporter.sh create mode 100644 tools/reference_stack_compose/up_node-exporter.sh create mode 100644 tools/reference_stack_compose/up_prometheus.sh create mode 100644 tools/reference_stack_compose/up_redis-exporter-probe.sh create mode 100644 tools/reference_stack_compose/up_redis-exporter.sh diff --git a/tools/reference_stack_compose/MONITORING.md b/tools/reference_stack_compose/MONITORING.md new file mode 100644 index 0000000000..8382882bb1 --- /dev/null +++ b/tools/reference_stack_compose/MONITORING.md @@ -0,0 +1,141 @@ +# overleaf-cep monitoring (Prometheus + Grafana + exporters) + +Stack in `/data_1/docker/compose` (modeled on the OlliTeX toolkit compose from +`/data_1/image_mining/the_diff/overleaf/toolkit/compose.yaml`), all of it served +through the existing nginx facade. Container/service names are unprefixed +(monitoring-only, added 2026-10-27; renamed from `ollitex-*` on 2026-10-27). + +## Services (all on the shared external network `overleaf-network`) + +| service | role | +|--------------------------------|----------------------------------------------------------------------| +| `prometheus` | TSDB, 15d retention, `127.0.0.1:9090` (SSH-tunnel/admin only) | +| `grafana` | UI, **no host port** — served under `https:///grafana/` | +| `mongodb-exporter` | percona exporter for the stack Mongo (sharelatex db) | +| `mongodb-exporter-probe` | busybox spider: /metrics must answer (exporter image has no shell) | +| `redis-exporter` | oliver006 exporter for the stack Redis | +| `redis-exporter-probe` | busybox spider for the redis exporter /metrics | +| `mongo-probe` | mongosh ping — protocol-level Mongo liveness (stack's own image) | +| `node-exporter` | host CPU/mem/fs signals (reads host `/`, `/proc`, `/sys` ro) | +| `instance-sensor` | **PR yu-i-i/overleaf-cep#178 sensors as live gauges** (see below) | + +The repo's own helpers `down_grafana.sh` / `exec_grafana.sh` now address this +`grafana` service directly. + +## What is measured + +1. **Overleaf app sensors** — the sharelatex image ships `@overleaf/metrics` + on the web process (`overleaf:3000/metrics`, `app="web-api"`): + `num_active_users` (live editor users), `timer_http_request` (summaries with + quantiles), `mongo_command_started`, `nodejs_eventloop_lag_*`, GC/heap, + `security_http_auth_check_credentials`, `editor_join_project`, + `tpds_add_doc`, `lock_mongoTransaction_*`. (In-image side processes like + compile `:4000` bind 127.0.0.1 inside their container — not scrapable.) +2. **Instance stats — PR #178, integrated into Prometheus/Grafana instead of + the admin page** (`instance-sensor`, port 9464): + - users: total + active(24h) — optional standard/guest split via + `INSTANCE_STATS_SEGMENTATION=true` (PR#176 rule: `'guest-user'` in + `users.adminRoles`) + - projects: total + active(24h, `lastUpdated`) + - files: total docs + - storage: mongodb `dbStats.totalSize`, filestore `du` of + `seaweedfs-volume/data` (ro), redis RAM (`used_memory`) + redis disk + (AOF size) + Computed fresh on every scrape (no hourly Mongo snapshot state). +3. **MongoDB** — connections, per-db data/storage size, oplog free space. +4. **Redis** — memory, clients, command rate, keyspace hit/miss, AOF. +5. **Host** — CPU, memory, root fs, load, uptime, network. +6. **Filestore (SeaweedFS master `:9327`, native prometheus)** — leader, + under-replicated volumes, in-flight uploads/downloads. +7. **Scrape self-health** — `up` per job + the monitoring stack itself. + +## URLs / access + +- Grafana: `https://psintern.neuro.uni-bremen.de/grafana/` + login: `admin` / (see top section of `.env`, `GRAFANA_ADMIN_USER/PASSWORD`; + unset → Grafana fallback `admin`/`admin`) +- Prometheus: `http://127.0.0.1:9090` (local only — SSH tunnel from a laptop) +- Dashboard: folder **Overleaf** → **Overleaf — instance overview** + (file-provisioned; pinned datasource uid `prometheus`, + `grafana/provisioning/datasources/prometheus.yml`) + +## Git hygiene + +Data directories are git-ignored per folder: `grafana/.gitignore` → +`grafana-data`, `prometheus/.gitignore` → `prometheus-data` (verified with +`git check-ignore`). `.env` itself is in the root `.gitignore`. Keep new +runtime dirs out of git by adding one line to the folder `.gitignore` as you +add them. + +## SMTP + +Grafana reuses the stack's Overleaf relay — **no separate Grafana mail vars**; +`grafana/compose.yaml` reads the existing Overleaf settings as a single +source of truth: + +| Grafana setting | taken from | +|---|--------------------------------| +| `GF_SMTP_ENABLED=true` | fixed | +| `GF_SMTP_HOST` | `OVERLEAF_EMAIL_SMTP_HOST`:`OVERLEAF_EMAIL_SMTP_PORT` | +| `GF_SMTP_USER` | `OVERLEAF_EMAIL_SMTP_USER` | +| `GF_SMTP_PASSWORD` | `OVERLEAF_EMAIL_PASSWORD` | +| `GF_SMTP_FROM_ADDRESS` | `OVERLEAF_EMAIL_FROM_ADDRESS` | +| `GF_SMTP_FROM_NAME` | `"Overleaf Monitoring"` (display label only) | +| `GF_SMTP_SKIP_VERIFY=false` | fixed | + +Used for Grafana's reset-password mail (verified live: sent via the relay, +server responded `200 {"message":"Email sent"}`). Grafana 11's alerting has no +built-in SMTP contact point — SMTP covers account e-mail; alert routing would +need integrations (webhook/Slack/…). + +**Gotcha (learned 2026-10-27): the admin account's email must be set.** +Grafana creates the admin user with email `admin@localhost`; the reset-mail +endpoint `POST /api/user/password/send-reset-email {"userOrEmail": …}` matches +users by that email and answers `200 "Email sent"` *even when nobody matches* +(anti user-enumeration) — so it silently sends nothing. Fixed on two layers: +1. **self-healing on fresh data**: `GF_SECURITY_ADMIN_EMAIL` in + `grafana/compose.yaml` = `${OVERLEAF_ADMIN_EMAIL:-overleaf@uni-bremen.de}` — + honored only when Grafana *creates* the admin (i.e. after a grafana-data + wipe); it never rewrites an existing user row. +2. **existing install**: `update user set email='overleaf@uni-bremen.de' + where login='admin'` in `grafana-data/grafana.db` (grafana stopped). +Verify with `select email from user where login='admin'`. The working relay +itself was proven with a raw `smtplib.SMTP_SSL` test (TLS-465 + auth + RCPT all OK). + +**After a grafana-data wipe** (fresh admin creation): login = +`GRAFANA_ADMIN_USER`/`GRAFANA_ADMIN_PASSWORD` from `.env`; e-mail = `GF_SECURITY_ADMIN_EMAIL` +(default `overleaf@uni-bremen.de`); dashboards re-provision automatically +(file provider); admin UI settings/annotations are lost. + +To disable SMTP: set `GF_SMTP_ENABLED: "false"` in `grafana/compose.yaml` + +`docker compose up -d grafana`. + +## Ops notes / gotchas (learned while bringing this up) + +- **percona/mongodb_exporter 0.43 ships most collectors OFF by default** + (a default run answers only `mongodb_up` + go/process metrics). Working + flags for this stack are in `mongodb-exporter/compose.yaml`: + `--collector.diagnosticdata --collector.dbstats --collector.dbstatsfreestorage + --collector.replicasetstatus` (verified against this mongo 9.0). +- Bind-mount ownership: `prometheus/prometheus-data` must be owned by uid + **65534**, `grafana/grafana-data` by uid **472** (both images run + unprivileged; docker creates fresh bind-mount dirs as root). Pre-create + + `chown` if the dirs are ever deleted. +- Grafana subpath: `GF_SERVER_SERVE_FROM_SUB_PATH=true` + + `GF_SERVER_ROOT_URL=https:///grafana/` — nginx serves `/grafana/` + **without stripping** the prefix (see `nginx/nginx.conf`). + After editing `nginx.conf`: `docker exec nginx nginx -s reload` (conf is + bind-mounted, not baked in). +- Grafana 11 login via API: `POST /grafana/login` with **JSON** + `{user, password}` (the old `POST /api/login` is gone; form-encoded POSTs + return `form-auth.invalid`). +- Healthchecks: compose interpolates `${VAR}` inside healthcheck strings at + load time — `MONGO_HOSTNAME`/`MONGO_PORT` therefore live in `.env`. +- The `instance-sensor` image is a local build: + `docker build -t overleaf-instance-sensor:local ./instance-sensor`. + +## Adding/retiring + +Everything opt-in per service; remove a block from the root `compose.yaml` +and `docker compose down` to retire one. Image pins: `.env` +(commented block in the monitoring section). diff --git a/tools/reference_stack_compose/compose.yaml b/tools/reference_stack_compose/compose.yaml index ee88d5cd4e..8d7a24795d 100644 --- a/tools/reference_stack_compose/compose.yaml +++ b/tools/reference_stack_compose/compose.yaml @@ -87,6 +87,83 @@ services: wakapi: condition: service_healthy + mongo-probe: + extends: + file: ./mongo-probe/compose.yaml + service: mongo-probe + depends_on: + mongo: + condition: service_healthy + + mongodb-exporter: + extends: + file: ./mongodb-exporter/compose.yaml + service: mongodb-exporter + depends_on: + mongo: + condition: service_healthy + + mongodb-exporter-probe: + extends: + file: ./mongodb-exporter-probe/compose.yaml + service: mongodb-exporter-probe + depends_on: + mongodb-exporter: + condition: service_healthy + + redis-exporter: + extends: + file: ./redis-exporter/compose.yaml + service: redis-exporter + depends_on: + redis: + condition: service_healthy + + redis-exporter-probe: + extends: + file: ./redis-exporter-probe/compose.yaml + service: redis-exporter-probe + depends_on: + redis-exporter: + condition: service_healthy + + instance-sensor: + extends: + file: ./instance-sensor/compose.yaml + service: instance-sensor + depends_on: + mongo: + condition: service_healthy + redis: + condition: service_healthy + + node-exporter: + extends: + file: ./node-exporter/compose.yaml + service: node-exporter + + prometheus: + extends: + file: ./prometheus/compose.yaml + service: prometheus + depends_on: + mongodb-exporter: + condition: service_healthy + redis-exporter: + condition: service_healthy + instance-sensor: + condition: service_healthy + node-exporter: + condition: service_healthy + + grafana: + extends: + file: ./grafana/compose.yaml + service: grafana + depends_on: + prometheus: + condition: service_healthy + nginx: extends: file: ./nginx/compose.yaml @@ -96,6 +173,8 @@ services: condition: service_healthy wakapi-probe: condition: service_healthy + grafana: + condition: service_healthy networks: overleaf-network: diff --git a/tools/reference_stack_compose/down_grafana.sh b/tools/reference_stack_compose/down_grafana.sh new file mode 100644 index 0000000000..cbe52ecc37 --- /dev/null +++ b/tools/reference_stack_compose/down_grafana.sh @@ -0,0 +1,2 @@ +docker compose down grafana + diff --git a/tools/reference_stack_compose/down_instance-sensor.sh b/tools/reference_stack_compose/down_instance-sensor.sh new file mode 100644 index 0000000000..d1517133a8 --- /dev/null +++ b/tools/reference_stack_compose/down_instance-sensor.sh @@ -0,0 +1,2 @@ +docker compose down instance-sensor + diff --git a/tools/reference_stack_compose/down_mongo-probe.sh b/tools/reference_stack_compose/down_mongo-probe.sh new file mode 100644 index 0000000000..1f51273bd1 --- /dev/null +++ b/tools/reference_stack_compose/down_mongo-probe.sh @@ -0,0 +1,2 @@ +docker compose down mongo-probe + diff --git a/tools/reference_stack_compose/down_mongodb-exporter-probe.sh b/tools/reference_stack_compose/down_mongodb-exporter-probe.sh new file mode 100644 index 0000000000..b40fb26ffd --- /dev/null +++ b/tools/reference_stack_compose/down_mongodb-exporter-probe.sh @@ -0,0 +1,2 @@ +docker compose down mongodb-exporter-probe + diff --git a/tools/reference_stack_compose/down_mongodb-exporter.sh b/tools/reference_stack_compose/down_mongodb-exporter.sh new file mode 100644 index 0000000000..190de6ffa6 --- /dev/null +++ b/tools/reference_stack_compose/down_mongodb-exporter.sh @@ -0,0 +1,2 @@ +docker compose down mongodb-exporter + diff --git a/tools/reference_stack_compose/down_node-exporter.sh b/tools/reference_stack_compose/down_node-exporter.sh new file mode 100644 index 0000000000..ec057952b4 --- /dev/null +++ b/tools/reference_stack_compose/down_node-exporter.sh @@ -0,0 +1,2 @@ +docker compose down node-exporter + diff --git a/tools/reference_stack_compose/down_prometheus.sh b/tools/reference_stack_compose/down_prometheus.sh new file mode 100644 index 0000000000..4a2fc159b6 --- /dev/null +++ b/tools/reference_stack_compose/down_prometheus.sh @@ -0,0 +1,2 @@ +docker compose down prometheus + diff --git a/tools/reference_stack_compose/down_redis-exporter-probe.sh b/tools/reference_stack_compose/down_redis-exporter-probe.sh new file mode 100644 index 0000000000..8a4c6226ca --- /dev/null +++ b/tools/reference_stack_compose/down_redis-exporter-probe.sh @@ -0,0 +1,2 @@ +docker compose down redis-exporter-probe + diff --git a/tools/reference_stack_compose/down_redis-exporter.sh b/tools/reference_stack_compose/down_redis-exporter.sh new file mode 100644 index 0000000000..48d28b2ddd --- /dev/null +++ b/tools/reference_stack_compose/down_redis-exporter.sh @@ -0,0 +1,2 @@ +docker compose down redis-exporter + diff --git a/tools/reference_stack_compose/env.sample b/tools/reference_stack_compose/env.sample index dab07737f8..b217db59c0 100644 --- a/tools/reference_stack_compose/env.sample +++ b/tools/reference_stack_compose/env.sample @@ -13,83 +13,129 @@ # from .env into here. # ============================================================================ -# Image versions +################## +# Image versions # +################## OVERLEAF_VERSION=ext-ce-v6.3.1 MONGO_VERSION=9.0 REDIS_VERSION=8.10-alpine GIT_BRIDE_VERSION=6.3.1 -# Global +###################### +# Overleaf Global I. # +###################### # You need here the correct path (the compose.yaml is in /docker/compose_cep/overleaf) COMPOSE_DIRECTORY=/data_1/docker/compose TEX_LIVE_REPO=git.fb1.uni-bremen.de/davrot OVERLEAF_LEFT_FOOTER='[]' OVERLEAF_LOGIN_SUPPORT_TEXT="
University of Bremen Overleaf Wiki " - - COMPILES_HOST_DIR=${COMPOSE_DIRECTORY}/overleaf/data/data/compiles COMPILES_HOST_DIR_CACHE=${COMPOSE_DIRECTORY}/overleaf/data/data/cache CONVERSION_SANDBOX_HOST_DIR=${COMPOSE_DIRECTORY}/overleaf/data/tmp/conversions OVERLEAF_OIDC_IS_ADMIN_FIELD_VALUE=davrot@uni-bremen.de -# Nginx SSL -SSL_KEY_FILE=/etc/letsencrypt/live/psintern.neuro.uni-bremen.de/privkey.pem -SSL_CERT_FILE=/etc/letsencrypt/live/psintern.neuro.uni-bremen.de/fullchain.pem +########### +# Grafana # +########### +GRAFANA_ADMIN_USER= +GRAFANA_ADMIN_PASSWORD= +# admin account email (only applies when the admin user is first created — +# i.e. on a fresh grafana-data; Grafana has no API to retrofit it later) +GF_SECURITY_ADMIN_EMAIL= +# where the UI is reachable (nginx subpath; default fits this stack) +GRAFANA_ROOT_URL=https:///grafana/ -#S3 Buckets +# Monitoring image pins (bump + autoupdate on upgrade) +GRAFANA_IMAGE=grafana/grafana-oss:11.6.0 +PROMETHEUS_IMAGE=prom/prometheus:v2.53.5 +NODE_EXPORTER_IMAGE=prom/node-exporter:v1.8.2 + +# Grafana SMTP: no extra vars — grafana/compose.yaml reuses the Overleaf +# settings below (OVERLEAF_EMAIL_SMTP_HOST/PORT/USER, OVERLEAF_EMAIL_PASSWORD, +# OVERLEAF_EMAIL_FROM_ADDRESS) as a single source of truth. + +# instance-sensor (PR yu-i-i/overleaf-cep#178 sensors as live gauges): +# split active users into standard/guest (PR#176 rule) +INSTANCE_STATS_SEGMENTATION=false + +############# +# Nginx SSL # +############# +# If you use certbot +SSL_KEY_FILE=/etc/letsencrypt/live//privkey.pem +SSL_CERT_FILE=/etc/letsencrypt/live//fullchain.pem + +############## +# S3 Buckets # +############## # openssl rand -base64 30 | tr -dc 'a-zA-Z0-9' | head -c 20; echo AWS_ACCESS_KEY_ID= # openssl rand -base64 60 | tr -dc 'a-zA-Z0-9' | head -c 40; echo AWS_SECRET_ACCESS_KEY= -# Zotero +########## +# Zotero # +########## # (credentials from your Zotero account: Settings -> Advanced -> Credentials) ZOTERO_CLIENT_KEY= ZOTERO_CLIENT_SECRET= -# Github +########## +# Github # +########## # (from your GitHub App settings) GITHUB_SYNC_CLIENT_ID= GITHUB_SYNC_CLIENT_SECRET= +################### +# Overleaf SECRET # +################### # openssl rand -hex 32; echo OVERLEAF_INVITE_TOKEN_SECRET= +##################### +# Overleaf SSO OIDC # +##################### # Keycloak OpenID Connect Configuration -OVERLEAF_OIDC_ISSUER=https://sso.fb1.uni-bremen.de/sso/realms/master +OVERLEAF_OIDC_ISSUER=https:///sso/realms/master # (from your SSO realm / client registration) OVERLEAF_OIDC_CLIENT_ID= OVERLEAF_OIDC_CLIENT_SECRET= OVERLEAF_OIDC_IDENTITY_SERVICE_NAME='Log in with SSO FB1 Uni Bremen' +################## +# Overleaf eMail # +################## # eMail Account Configuration # (SMTP password for the user in OVERLEAF_EMAIL_SMTP_USER) -OVERLEAF_EMAIL_PASSWORD= -OVERLEAF_EMAIL_FROM_ADDRESS=overleaf@uni-bremen.de -OVERLEAF_EMAIL_SMTP_HOST=smtp.uni-bremen.de +OVERLEAF_EMAIL_PASSWORD= +OVERLEAF_EMAIL_FROM_ADDRESS= +OVERLEAF_EMAIL_SMTP_HOST= OVERLEAF_EMAIL_SMTP_PORT=465 OVERLEAF_EMAIL_SMTP_SECURE=true -OVERLEAF_EMAIL_SMTP_USER=overleaf +OVERLEAF_EMAIL_SMTP_USER= OVERLEAF_EMAIL_SMTP_IGNORE_TLS=false +####################### +# Overleaf Global II. # +####################### + # Other Overleaf Configurations Configuration -OVERLEAF_SITE_URL=https://psintern.neuro.uni-bremen.de -OVERLEAF_ADMIN_EMAIL=overleaf@uni-bremen.de -OVERLEAF_APP_NAME="University of Bremen -- Overleaf" -OVERLEAF_NAV_TITLE="University of Bremen -- Overleaf" -OVERLEAF_CUSTOM_EMAIL_FOOTER="University of Bremen -- Overleaf" +OVERLEAF_SITE_URL=https:// +OVERLEAF_ADMIN_EMAIL= +OVERLEAF_APP_NAME="Overleaf" +OVERLEAF_NAV_TITLE="Overleaf" +OVERLEAF_CUSTOM_EMAIL_FOOTER="Overleaf" ########## # Wakapi # ########## - # openssl rand -base64 30 | tr -dc "a-zA-Z0-9" | head -c 24; echo PG_PASSWORD= # HOST port for the wakapi web UI + API (3000 by convention; use 3001 if taken) WAKA_API_PORT=3000 - # image pins (bump + autoupdate on upgrade) WAKA_API_IMAGE=ghcr.io/muety/wakapi:2.18.2 POSTGRES_IMAGE=postgres:18-alpine diff --git a/tools/reference_stack_compose/exec_grafana.sh b/tools/reference_stack_compose/exec_grafana.sh new file mode 100755 index 0000000000..af0fa20096 --- /dev/null +++ b/tools/reference_stack_compose/exec_grafana.sh @@ -0,0 +1 @@ +docker exec -it grafana sh diff --git a/tools/reference_stack_compose/exec_instance-sensor.sh b/tools/reference_stack_compose/exec_instance-sensor.sh new file mode 100755 index 0000000000..44cca2434b --- /dev/null +++ b/tools/reference_stack_compose/exec_instance-sensor.sh @@ -0,0 +1 @@ +docker exec -it instance-sensor sh diff --git a/tools/reference_stack_compose/exec_mongo-probe.sh b/tools/reference_stack_compose/exec_mongo-probe.sh new file mode 100755 index 0000000000..77ae0e9025 --- /dev/null +++ b/tools/reference_stack_compose/exec_mongo-probe.sh @@ -0,0 +1 @@ +docker exec -it mongo-probe sh diff --git a/tools/reference_stack_compose/exec_mongodb-exporter-probe.sh b/tools/reference_stack_compose/exec_mongodb-exporter-probe.sh new file mode 100755 index 0000000000..8716249ee6 --- /dev/null +++ b/tools/reference_stack_compose/exec_mongodb-exporter-probe.sh @@ -0,0 +1 @@ +docker exec -it mongodb-exporter-probe sh diff --git a/tools/reference_stack_compose/exec_node-exporter.sh b/tools/reference_stack_compose/exec_node-exporter.sh new file mode 100755 index 0000000000..806b8a333b --- /dev/null +++ b/tools/reference_stack_compose/exec_node-exporter.sh @@ -0,0 +1 @@ +docker exec -it node-exporter sh diff --git a/tools/reference_stack_compose/exec_prometheus.sh b/tools/reference_stack_compose/exec_prometheus.sh new file mode 100755 index 0000000000..661a3106c8 --- /dev/null +++ b/tools/reference_stack_compose/exec_prometheus.sh @@ -0,0 +1 @@ +docker exec -it prometheus sh diff --git a/tools/reference_stack_compose/exec_redis-exporter-probe.sh b/tools/reference_stack_compose/exec_redis-exporter-probe.sh new file mode 100755 index 0000000000..da7cbe8654 --- /dev/null +++ b/tools/reference_stack_compose/exec_redis-exporter-probe.sh @@ -0,0 +1 @@ +docker exec -it redis-exporter-probe sh diff --git a/tools/reference_stack_compose/exec_redis-exporter.sh b/tools/reference_stack_compose/exec_redis-exporter.sh new file mode 100755 index 0000000000..8b3f1a8271 --- /dev/null +++ b/tools/reference_stack_compose/exec_redis-exporter.sh @@ -0,0 +1 @@ +docker exec -it redis-exporter sh diff --git a/tools/reference_stack_compose/grafana/.gitignore b/tools/reference_stack_compose/grafana/.gitignore new file mode 100644 index 0000000000..9e02e35444 --- /dev/null +++ b/tools/reference_stack_compose/grafana/.gitignore @@ -0,0 +1 @@ +grafana-data diff --git a/tools/reference_stack_compose/grafana/compose.yaml b/tools/reference_stack_compose/grafana/compose.yaml new file mode 100644 index 0000000000..8ec69fde88 --- /dev/null +++ b/tools/reference_stack_compose/grafana/compose.yaml @@ -0,0 +1,54 @@ +services: + grafana: + image: ${GRAFANA_IMAGE:-grafana/grafana-oss:11.6.0} + container_name: grafana + hostname: grafana + restart: unless-stopped + environment: + GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER} + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD} + # Admin ACCOUNT EMAIL — reuses the Overleaf admin mailbox (line ~61 of + # .env). NOTE: like the password, this env is honored ONLY when Grafana + # CREATES the admin user (fresh grafana-data); it never rewrites an + # existing user row. Default (Grafana shipped default) is + # admin@localhost — see MONITORING.md for the reset-mail gotcha. + GF_SECURITY_ADMIN_EMAIL: ${OVERLEAF_ADMIN_EMAIL:-overleaf@uni-bremen.de} + GF_ANALYTICS_REPORTING_ENABLED: "false" + GF_USERS_SIGNUP_ENABLED: "false" + # SMTP — reuses the stack's Overleaf relay settings (OVERLEAF_EMAIL_* in + # .env / env.sample): single source of truth, no Grafana-specific mail vars. + # GF_SMTP_FROM_NAME is the only grafana-local piece (display label). + GF_SMTP_ENABLED: "true" + GF_SMTP_HOST: ${OVERLEAF_EMAIL_SMTP_HOST}:${OVERLEAF_EMAIL_SMTP_PORT} + GF_SMTP_USER: ${OVERLEAF_EMAIL_SMTP_USER} + GF_SMTP_PASSWORD: ${OVERLEAF_EMAIL_PASSWORD} + GF_SMTP_FROM_ADDRESS: ${OVERLEAF_EMAIL_FROM_ADDRESS} + GF_SMTP_FROM_NAME: "Overleaf Monitoring" + GF_SMTP_SKIP_VERIFY: "false" + # AJ-3 style subpath serving (owner 2026-10-08, reference toolkit): + # URLs/route generation relative to the /grafana/ root + GF_SERVER_ROOT_URL: "${GRAFANA_ROOT_URL:-https://psintern.neuro.uni-bremen.de/grafana/}" + GF_SERVER_SERVE_FROM_SUB_PATH: "true" + # x-frame-options deny -> sameorigin (iframe embedding from the same + # origin stays possible; the fine-grained allowlist is the CSP below) + GF_SECURITY_ALLOW_EMBEDDING: "true" + GF_SECURITY_CSP_FRAME_ANCESTORS: "'self'" + volumes: + - ./provisioning:/etc/grafana/provisioning + - ./dashboards:/var/lib/grafana/dashboards + - ./grafana-data:/var/lib/grafana + expose: + - "3000" # nginx is the sole facade (see nginx/nginx.conf location /grafana/) + healthcheck: + # grafana-oss ships wget (verified on the reference stack) + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3000/api/health"] + interval: 15s + timeout: 5s + retries: 8 + start_period: 30s + networks: + - overleaf-network + +networks: + overleaf-network: + external: true diff --git a/tools/reference_stack_compose/grafana/dashboards/overview.json b/tools/reference_stack_compose/grafana/dashboards/overview.json new file mode 100644 index 0000000000..cb17829be4 --- /dev/null +++ b/tools/reference_stack_compose/grafana/dashboards/overview.json @@ -0,0 +1,2441 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "overleaf-cep instance overview: app sensors (sharelatex /metrics on overleaf:3000), PR yu-i-i/overleaf-cep#178 instance stats as live prometheus gauges, host, MongoDB, Redis, SeaweedFS filestore, scrape health. Datasource: pinned uid prometheus.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphPaper": { + "external": false + }, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Scrape health", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 6 + }, + { + "color": "green", + "value": 7 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "color": { + "mode": "thresholds" + }, + "graphMode": "none" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(up)", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Targets up (up==1)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 3, + "options": { + "color": { + "mode": "thresholds" + }, + "graphMode": "none" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "count(up)", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Total scrape targets", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "up", + "legendFormat": "{{job}}", + "refId": "A" + } + ], + "title": "up by job", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 5, + "panels": [], + "title": "Overleaf \u2014 app (web-api sensors, /metrics on overleaf:3000)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 0, + "y": 10 + }, + "id": 6, + "options": { + "color": { + "mode": "thresholds" + }, + "graphMode": "lines" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(num_active_users)", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Active users in editor", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 4, + "y": 10 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (method, path) (rate(timer_http_request_count[5m]))", + "legendFormat": "{{method}} {{path}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(timer_http_request_count[5m]))", + "legendFormat": "total", + "refId": "B" + } + ], + "title": "Requests (req/s)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 14, + "y": 10 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "1000 * sum(rate(timer_http_request_sum[5m])) / sum(rate(timer_http_request_count[5m]))", + "legendFormat": "avg all paths", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "1000 * timer_http_request{quantile='0.95'}", + "legendFormat": "p95 {{path}}", + "refId": "B" + } + ], + "title": "Request time (ms)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 18 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (method) (rate(mongo_command_started[5m]))", + "legendFormat": "{{method}}", + "refId": "A" + } + ], + "title": "MongoDB commands from app (ops/s)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 18 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "1000 * nodejs_eventloop_lag_p99_seconds", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Node.js event loop lag p99 (ms)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 18 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "nodejs_heap_size_used_bytes", + "legendFormat": "heap used", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "process_resident_memory_bytes", + "legendFormat": "rss", + "refId": "B" + } + ], + "title": "Node.js memory (bytes)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 26 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(editor_join_project[5m])) * 60", + "legendFormat": "editor joins/min", + "refId": "A" + } + ], + "title": "Editor joins (per min)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 26 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (status, path) (rate(security_http_auth_check_credentials{status!='pass'}[5m])) * 60", + "legendFormat": "{{status}} {{path}}", + "refId": "A" + } + ], + "title": "Auth check failures (per min)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 26 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(tpds_add_doc[5m])) * 60", + "legendFormat": "add_doc/min", + "refId": "A" + } + ], + "title": "Project doc ops (per min)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 15, + "panels": [], + "title": "Instance stats (PR yu-i-i/overleaf-cep#178 sensors, live)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 35 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "overleaf_instance_users_total", + "legendFormat": "total {{segment}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "overleaf_instance_users_active_24h", + "legendFormat": "active 24h {{segment}}", + "refId": "B" + } + ], + "title": "Users (total / active 24h)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 35 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "overleaf_instance_projects_total", + "legendFormat": "total", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "overleaf_instance_projects_active_24h", + "legendFormat": "active 24h", + "refId": "B" + } + ], + "title": "Projects (total / active 24h)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 35 + }, + "id": 18, + "options": { + "color": { + "mode": "thresholds" + }, + "graphMode": "lines" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(overleaf_instance_files_total)", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Document files (docs)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 43 + }, + "id": 19, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "overleaf_instance_storage_bytes", + "legendFormat": "{{component}}", + "refId": "A" + } + ], + "title": "Storage (bytes)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 43 + }, + "id": 20, + "options": { + "color": { + "mode": "thresholds" + }, + "graphMode": "none" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(overleaf_instance_sensor_up)", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Sensor up", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 51 + }, + "id": 21, + "panels": [], + "title": "Host (node_exporter)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 52 + }, + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{mode='idle'}[5m])))", + "legendFormat": "cpu", + "refId": "A" + } + ], + "title": "CPU usage (%)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 52 + }, + "id": 23, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)", + "legendFormat": "memory", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "100 * (node_memory_SwapTotal_bytes - node_memory_SwapFree_bytes) / node_memory_SwapTotal_bytes", + "legendFormat": "swap", + "refId": "B" + } + ], + "title": "Memory usage (%)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 52 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "100 * (1 - node_filesystem_avail_bytes{mountpoint='/'} / node_filesystem_size_bytes{mountpoint='/'})", + "legendFormat": "root fs", + "refId": "A" + } + ], + "title": "Root filesystem usage (%)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 60 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_load1", + "legendFormat": "load1", + "refId": "A" + } + ], + "title": "Load (1m)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "dtdurations" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 60 + }, + "id": 26, + "options": { + "color": { + "mode": "thresholds" + }, + "graphMode": "none" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "time() - node_boot_time_seconds", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 60 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(node_network_transmit_bytes_total[5m]))", + "legendFormat": "tx", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(node_network_receive_bytes_total[5m]))", + "legendFormat": "rx", + "refId": "B" + } + ], + "title": "Network I/O (bytes/s)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 69 + }, + "id": 28, + "panels": [], + "title": "MongoDB (percona exporter)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 0, + "y": 70 + }, + "id": 29, + "options": { + "color": { + "mode": "thresholds" + }, + "graphMode": "none" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "avg(mongodb_up)", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Mongo up", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 4, + "y": 70 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "mongodb_ss_connections{conn_type='active'}", + "legendFormat": "active", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "mongodb_ss_connections{conn_type='available'}", + "legendFormat": "available", + "refId": "B" + } + ], + "title": "Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 70 + }, + "id": 31, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (database) (mongodb_dbstats_dataSize)", + "legendFormat": "{{database}}", + "refId": "A" + } + ], + "title": "Database data size (bytes)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 78 + }, + "id": 32, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (database) (mongodb_dbstats_storageSize)", + "legendFormat": "{{database}}", + "refId": "A" + } + ], + "title": "Database storage size (bytes)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 78 + }, + "id": 33, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "100 * mongodb_oplog_stats_storageStats_freeStorageSize / mongodb_oplog_stats_storageStats_totalSize", + "legendFormat": "oplog free space", + "refId": "A" + } + ], + "title": "Oplog free space (%)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 86 + }, + "id": 34, + "panels": [], + "title": "Redis (oliver006 exporter)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 0, + "y": 87 + }, + "id": 35, + "options": { + "color": { + "mode": "thresholds" + }, + "graphMode": "none" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "avg(redis_up)", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Redis up", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 4, + "y": 87 + }, + "id": 36, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "redis_memory_used_bytes", + "legendFormat": "used memory", + "refId": "A" + } + ], + "title": "Memory used (bytes)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 14, + "y": 87 + }, + "id": 37, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "redis_connected_clients", + "legendFormat": "clients", + "refId": "A" + } + ], + "title": "Connected clients", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 95 + }, + "id": 38, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(redis_commands_processed_total[5m]))", + "legendFormat": "commands", + "refId": "A" + } + ], + "title": "Commands (ops/s)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 95 + }, + "id": 39, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(redis_keyspace_hits_total[5m]))", + "legendFormat": "hits", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(redis_keyspace_misses_total[5m]))", + "legendFormat": "misses", + "refId": "B" + } + ], + "title": "Keyspace hits / misses (per s)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 95 + }, + "id": 40, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "overleaf_instance_storage_bytes{component='redis_disk'}", + "legendFormat": "aof size (sensor)", + "refId": "A" + } + ], + "title": "Redis disk (AOF) size (bytes)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 103 + }, + "id": 41, + "panels": [], + "title": "Filestore (SeaweedFS master)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 0, + "y": 104 + }, + "id": 42, + "options": { + "color": { + "mode": "thresholds" + }, + "graphMode": "none" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "SeaweedFS_master_is_leader", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Master is leader", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 4, + "y": 104 + }, + "id": 43, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "SeaweedFS_master_under_replicated_volumes", + "legendFormat": "under-replicated", + "refId": "A" + } + ], + "title": "Under-replicated volumes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 14, + "y": 104 + }, + "id": 44, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "SeaweedFS_volumeServer_in_flight_upload_size", + "legendFormat": "in-flight upload", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "SeaweedFS_volumeServer_in_flight_download_size", + "legendFormat": "in-flight download", + "refId": "B" + } + ], + "title": "Volume server in-flight (bytes)", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "overleaf", + "ollitex", + "instance-stats" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Overleaf \u2014 instance overview", + "uid": "overleaf-overview", + "version": 1, + "weekStart": "" +} diff --git a/tools/reference_stack_compose/grafana/provisioning/dashboards/ollitex.yml b/tools/reference_stack_compose/grafana/provisioning/dashboards/ollitex.yml new file mode 100644 index 0000000000..f3f4c3027f --- /dev/null +++ b/tools/reference_stack_compose/grafana/provisioning/dashboards/ollitex.yml @@ -0,0 +1,14 @@ +# file-based dashboard provider (shared/deterministic: no in-UI editing +# required; dashboards ship with this compose tree) — reference: +# build-images/grafana/provisioning/dashboards/ollitex.yml +apiVersion: 1 +providers: + - name: overleaf + folder: Overleaf + type: file + disableDeletion: false + updateIntervalSeconds: 30 + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false diff --git a/tools/reference_stack_compose/grafana/provisioning/datasources/prometheus.yml b/tools/reference_stack_compose/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 0000000000..a5d3c907a6 --- /dev/null +++ b/tools/reference_stack_compose/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,12 @@ +# D22-style shared datasource provisioning (reference: build-images/grafana): +# pinned uid so every dashboard JSON can hard-reference it; url points at +# this stack's prometheus container name. +apiVersion: 1 +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false diff --git a/tools/reference_stack_compose/instance-sensor/Dockerfile b/tools/reference_stack_compose/instance-sensor/Dockerfile new file mode 100644 index 0000000000..29b8631e2f --- /dev/null +++ b/tools/reference_stack_compose/instance-sensor/Dockerfile @@ -0,0 +1,20 @@ +# overleaf-instance-sensor — self-contained prometheus gauge exporter for +# the PR yu-i-i/overleaf-cep#178 instance-statistics sensors. +FROM node:22-alpine + +WORKDIR /app +ENV NODE_ENV=production + +COPY package.json ./ +RUN npm install --omit=dev --no-audit --no-fund + +COPY sensor.mjs ./ +RUN chown -R node:node /app + +EXPOSE 9464 +USER node + +HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=5 \ + CMD wget -q --spider --timeout=3 http://127.0.0.1:9464/metrics || exit 1 + +ENTRYPOINT ["node", "sensor.mjs"] diff --git a/tools/reference_stack_compose/instance-sensor/compose.yaml b/tools/reference_stack_compose/instance-sensor/compose.yaml new file mode 100644 index 0000000000..53f835a254 --- /dev/null +++ b/tools/reference_stack_compose/instance-sensor/compose.yaml @@ -0,0 +1,52 @@ +# instance-sensor — the overleaf-cep "instance statistics" sensors, +# exposed LIVE as Prometheus gauges (scrape-on-read) instead of the +# hourly-cron-Mongo-snapshots of PR yu-i-i/overleaf-cep#178. Same sensor +# definitions, prometheus-native delivery: +# +# users : total + active (last 24h), split standard/guest when +# INSTANCE_STATS_SEGMENTATION != false (guest = user with +# 'guest-user' in adminRoles — the PR#176/PR#178 definition) +# projects : total + active (last 24h, projects.lastUpdated) +# files : total documents (docs collection) +# storage : mongodb (dbStats.totalSize), filestore (du of the SeaweedFS +# data dir — the S3 filestore content), redis RAM (used_memory) +# + redis disk (AOF current size) +# +# Self-contained node image (mongodb + ioredis drivers) — does NOT depend on +# the overleaf image layout. Read-only on its inputs. +services: + instance-sensor: + build: + context: . + dockerfile: Dockerfile + image: ${INSTANCE_SENSOR_IMAGE:-overleaf-instance-sensor:local} + container_name: instance-sensor + hostname: instance-sensor + restart: unless-stopped + environment: + MONGO_URL: "mongodb://${MONGO_HOSTNAME:-mongo}:${MONGO_PORT:-27017}/${MONGODB_DATABASE:-sharelatex}" + REDIS_ADDR: "redis://${REDIS_HOSTNAME:-redis}:${REDIS_PORT:-6379}" + # du() of the SeaweedFS volume data dir, mounted read-only below + # (mirror of PR#178's du-based overleaf_storage) + FILESTORE_DATA_DIR: /hostdata/filestore + # 'standard'/'guest' split (PR#178/PR#176): 'false' (default, per PR) + # or 'true' + SEGMENTATION: ${INSTANCE_STATS_SEGMENTATION:-false} + ACTIVE_WINDOW_HOURS: "24" + PORT: "9464" + expose: + - "9464" + volumes: + - ${COMPOSE_DIRECTORY:-/data_1/docker/compose}/seaweedfs-volume/data:/hostdata/filestore:ro + healthcheck: + test: ["CMD-SHELL", "wget -q --spider --timeout=5 http://127.0.0.1:9464/metrics >/dev/null 2>&1 || exit 1"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + networks: + - overleaf-network + +networks: + overleaf-network: + external: true diff --git a/tools/reference_stack_compose/instance-sensor/package.json b/tools/reference_stack_compose/instance-sensor/package.json new file mode 100644 index 0000000000..9d8eccbf95 --- /dev/null +++ b/tools/reference_stack_compose/instance-sensor/package.json @@ -0,0 +1,12 @@ +{ + "name": "overleaf-instance-sensor", + "version": "1.0.0", + "private": true, + "type": "module", + "description": "Prometheus gauges for the overleaf-cep instance statistics (PR yu-i-i/overleaf-cep#178, live instead of hourly Mongo snapshots)", + "engines": { "node": ">=18" }, + "dependencies": { + "ioredis": "^5.4.1", + "mongodb": "^6.10.0" + } +} diff --git a/tools/reference_stack_compose/instance-sensor/sensor.mjs b/tools/reference_stack_compose/instance-sensor/sensor.mjs new file mode 100644 index 0000000000..8b243352f6 --- /dev/null +++ b/tools/reference_stack_compose/instance-sensor/sensor.mjs @@ -0,0 +1,214 @@ +// overleaf-instance-sensor +// Live Prometheus gauges for the overleaf-cep instance statistics. +// +// Sensor definitions ported from PR yu-i-i/overleaf-cep#178 +// (services/web/modules/instance-stats/.../collectInstanceStats.mjs): +// * users : total count + active (lastActive >= now-24h) +// * projects : total count + active (lastUpdated >= now-24h) +// * files : docs collection count +// * storage : mongodb dbStats.totalSize, overleaf filestore (du of the +// data dir), redis RAM (used_memory) + redis disk (aof size) +// * optional standard/guest split: guest = 'guest-user' in user.adminRoles +// (the PR#176 segmentation; PR#178 env INSTANCE_STATS_USER_SEGMENTATION_ENABLED) +// +// Instead of the PR's hourly snapshot docs in Mongo, we compute on every +// scrape (cheap counts on this scale; fresh values, no retention state). +// Each probe is isolated: a failing source emits its error counter but the +// rest of the endpoint still answers. + +import http from 'node:http' +import { execFile } from 'node:child_process' +import { performance } from 'node:perf_hooks' +import { promisify } from 'node:util' +import { MongoClient } from 'mongodb' +import Redis from 'ioredis' + +const execFileAsync = promisify(execFile) + +const PORT = Number(process.env.PORT || 9464) +const MONGO_URL = process.env.MONGO_URL || 'mongodb://mongo:27017/sharelatex' +const REDIS_ADDR = process.env.REDIS_ADDR || 'redis://redis:6379' +const FILESTORE_DATA_DIR = process.env.FILESTORE_DATA_DIR || '/hostdata/filestore' +const SEGMENTATION = String(process.env.SEGMENTATION || 'false') === 'true' +const ACTIVE_WINDOW_MS = Number(process.env.ACTIVE_WINDOW_HOURS || 24) * 3600 * 1000 + +const GUEST_ADMIN_ROLE = 'guest-user' // PR#176/PR#178 guest definition + +const errors = new Map() // name -> failed probe count +let lastSuccessSeconds = 0 + +const guestUserFilter = () => ({ adminRoles: { $in: [GUEST_ADMIN_ROLE] } }) + +async function duBytes(path) { + // byte-exact du of a directory (same semantics as PR#178's duBytes()) + try { + const { stdout } = await execFileAsync('du', ['-sb', path]) + const m = String(stdout).match(/(\d+)/) + return m ? Number(m[1]) : 0 + } catch { + throw new Error(`du failed for ${path}`) + } +} + +function parseRedisInfoInt(info, key) { + const m = String(info || '').match(new RegExp(`^${key}:(\\d+)`, 'm')) + return m ? Number(m[1]) : 0 +} + +async function collect() { + const nowMs = Date.now() + const oneAgo = new Date(nowMs - ACTIVE_WINDOW_MS) + const out = [] + + let mongoOk = true + try { + const client = new MongoClient(MONGO_URL, { serverSelectionTimeoutMS: 5000 }) + await client.connect() + try { + const db = client.db() // db name comes from MONGO_URL + const users = db.collection('users') + const projects = db.collection('projects') + const docs = db.collection('docs') + + const totalUsers = await users.countDocuments() + const activeUsers = await users.countDocuments({ lastActive: { $gte: oneAgo } }) + out.push( + '# HELP overleaf_instance_users_total Total registered users (segment: all, plus standard/guest when segmentation is enabled).', + '# TYPE overleaf_instance_users_total gauge', + `overleaf_instance_users_total{segment="all"} ${totalUsers}`, + '# HELP overleaf_instance_users_active_24h Users active in the last 24 hours (lastActive >= now-24h).', + '# TYPE overleaf_instance_users_active_24h gauge', + `overleaf_instance_users_active_24h{segment="all"} ${activeUsers}`, + ) + + if (SEGMENTATION) { + const guestFilter = guestUserFilter() + const guestUsers = await users.countDocuments(guestFilter) + const activeGuest = await users.countDocuments({ + ...guestFilter, + lastActive: { $gte: oneAgo }, + }) + out.push( + `overleaf_instance_users_total{segment="standard"} ${totalUsers - guestUsers}`, + `overleaf_instance_users_total{segment="guest"} ${guestUsers}`, + `overleaf_instance_users_active_24h{segment="standard"} ${activeUsers - activeGuest}`, + `overleaf_instance_users_active_24h{segment="guest"} ${activeGuest}`, + ) + } + + const totalProjects = await projects.countDocuments() + const activeProjects = await projects.countDocuments({ lastUpdated: { $gte: oneAgo } }) + const totalFiles = await docs.countDocuments() + out.push( + '# HELP overleaf_instance_projects_total Total projects.', + '# TYPE overleaf_instance_projects_total gauge', + `overleaf_instance_projects_total ${totalProjects}`, + '# HELP overleaf_instance_projects_active_24h Projects updated in the last 24 hours (lastUpdated >= now-24h).', + '# TYPE overleaf_instance_projects_active_24h gauge', + `overleaf_instance_projects_active_24h ${activeProjects}`, + '# HELP overleaf_instance_files_total Total documents (files) in the docs collection.', + '# TYPE overleaf_instance_files_total gauge', + `overleaf_instance_files_total ${totalFiles}`, + ) + + // PR#178: mongodb_storage = dbStats.totalSize (bytes) + let mongoBytes = 0 + try { + const stats = await db.stats() + mongoBytes = Number(stats?.totalSize ?? 0) + } catch { + /* keep 0 */ + } + out.push( + '# HELP overleaf_instance_storage_bytes Storage used, bytes. component: mongodb | filestore | redis_ram | redis_disk.', + '# TYPE overleaf_instance_storage_bytes gauge', + `overleaf_instance_storage_bytes{component="mongodb"} ${mongoBytes}`, + ) + } finally { + await client.close() + } + } catch { + mongoOk = false + errors.set('mongo', (errors.get('mongo') || 0) + 1) + } + + // filestore size (SeaweedFS data dir, mounted ro) -------------------------- + let fileOk = true + try { + const bytes = await duBytes(FILESTORE_DATA_DIR) + out.push(`overleaf_instance_storage_bytes{component="filestore"} ${bytes}`) + } catch { + fileOk = false + errors.set('filestore', (errors.get('filestore') || 0) + 1) + } + + // redis storage: RAM (used_memory) + disk (aof_current_size) ---------------- + let redisOk = true + try { + const r = new Redis(REDIS_ADDR, { + maxRetriesPerRequest: 1, + connectTimeout: 5000, + lazyConnect: true, + }) + try { + const usedMemory = parseRedisInfoInt(await r.info('memory'), 'used_memory') + let aof = 0 + try { + aof = parseRedisInfoInt(await r.info('persistence'), 'aof_current_size') + } catch { + /* aof may be disabled — 0 is the true value */ + } + out.push( + `overleaf_instance_storage_bytes{component="redis_ram"} ${usedMemory}`, + `overleaf_instance_storage_bytes{component="redis_disk"} ${aof}`, + ) + } finally { + try { + r.disconnect() + } catch { + /* noop */ + } + } + } catch { + redisOk = false + errors.set('redis', (errors.get('redis') || 0) + 1) + } + + const up = mongoOk && fileOk && redisOk ? 1 : 0 + if (up === 1) lastSuccessSeconds = (Date.now() / 1000) | 0 + + out.push( + '# HELP overleaf_instance_sensor_up 1 when the last scrape computed all probes successfully.', + '# TYPE overleaf_instance_sensor_up gauge', + `overleaf_instance_sensor_up ${up}`, + '# HELP overleaf_instance_sensor_last_success_timestamp_seconds Unix time of the last fully successful scrape.', + '# TYPE overleaf_instance_sensor_last_success_timestamp_seconds gauge', + `overleaf_instance_sensor_last_success_timestamp_seconds ${lastSuccessSeconds}`, + '# HELP overleaf_instance_sensor_errors_total Failed probe count since start (name=mongo|filestore|redis).', + '# TYPE overleaf_instance_sensor_errors_total counter', + ) + for (const [name, n] of errors) { + out.push(`overleaf_instance_sensor_errors_total{name="${name}"} ${n}`) + } + return `${out.join('\n')}\n` +} + +const server = http.createServer(async (req, res) => { + if (req.url !== '/metrics') { + res.writeHead(404, { 'content-type': 'text/plain' }) + res.end('not found\n') + return + } + const t0 = performance.now() + const body = await collect() + const dt = (performance.now() - t0) / 1000 + res.writeHead(200, { + 'content-type': 'text/plain; version=0.0.4; charset=utf-8', + 'content-length': Buffer.byteLength(body), + }) + res.end(body) +}) + +server.listen(PORT, '0.0.0.0', () => { + console.log(`overleaf-instance-sensor listening on :${PORT}`) +}) diff --git a/tools/reference_stack_compose/logs_grafana.sh b/tools/reference_stack_compose/logs_grafana.sh new file mode 100755 index 0000000000..991d2114ee --- /dev/null +++ b/tools/reference_stack_compose/logs_grafana.sh @@ -0,0 +1,2 @@ +docker compose logs -f grafana + diff --git a/tools/reference_stack_compose/logs_instance-sensor.sh b/tools/reference_stack_compose/logs_instance-sensor.sh new file mode 100755 index 0000000000..246037641a --- /dev/null +++ b/tools/reference_stack_compose/logs_instance-sensor.sh @@ -0,0 +1,2 @@ +docker compose logs -f instance-sensor + diff --git a/tools/reference_stack_compose/logs_mongo-probe.sh b/tools/reference_stack_compose/logs_mongo-probe.sh new file mode 100755 index 0000000000..4a46122cc0 --- /dev/null +++ b/tools/reference_stack_compose/logs_mongo-probe.sh @@ -0,0 +1,2 @@ +docker compose logs -f mongo-probe + diff --git a/tools/reference_stack_compose/logs_mongodb-exporter-probe.sh b/tools/reference_stack_compose/logs_mongodb-exporter-probe.sh new file mode 100755 index 0000000000..f5a29406d4 --- /dev/null +++ b/tools/reference_stack_compose/logs_mongodb-exporter-probe.sh @@ -0,0 +1,2 @@ +docker compose logs -f mongodb-exporter-probe + diff --git a/tools/reference_stack_compose/logs_mongodb-exporter.sh b/tools/reference_stack_compose/logs_mongodb-exporter.sh new file mode 100755 index 0000000000..e5a3e6b4c6 --- /dev/null +++ b/tools/reference_stack_compose/logs_mongodb-exporter.sh @@ -0,0 +1,2 @@ +docker compose logs -f mongodb-exporter + diff --git a/tools/reference_stack_compose/logs_node-exporter.sh b/tools/reference_stack_compose/logs_node-exporter.sh new file mode 100755 index 0000000000..094757c73c --- /dev/null +++ b/tools/reference_stack_compose/logs_node-exporter.sh @@ -0,0 +1,2 @@ +docker compose logs -f node-exporter + diff --git a/tools/reference_stack_compose/logs_prometheus.sh b/tools/reference_stack_compose/logs_prometheus.sh new file mode 100755 index 0000000000..e44d4dc586 --- /dev/null +++ b/tools/reference_stack_compose/logs_prometheus.sh @@ -0,0 +1,2 @@ +docker compose logs -f prometheus + diff --git a/tools/reference_stack_compose/logs_redis-exporter-probe.sh b/tools/reference_stack_compose/logs_redis-exporter-probe.sh new file mode 100755 index 0000000000..40ee871e2a --- /dev/null +++ b/tools/reference_stack_compose/logs_redis-exporter-probe.sh @@ -0,0 +1,2 @@ +docker compose logs -f redis-exporter-probe + diff --git a/tools/reference_stack_compose/logs_redis-exporter.sh b/tools/reference_stack_compose/logs_redis-exporter.sh new file mode 100755 index 0000000000..ec20d82c7b --- /dev/null +++ b/tools/reference_stack_compose/logs_redis-exporter.sh @@ -0,0 +1,2 @@ +docker compose logs -f redis-exporter + diff --git a/tools/reference_stack_compose/mongo-probe/compose.yaml b/tools/reference_stack_compose/mongo-probe/compose.yaml new file mode 100644 index 0000000000..06197c4acb --- /dev/null +++ b/tools/reference_stack_compose/mongo-probe/compose.yaml @@ -0,0 +1,26 @@ +# mongo-probe — protocol-level liveness check of the stack Mongo. +# Reuses the stack's own mongo image: its mongosh is the canonical client. +# healthy == the db answers a real ping (same pattern as the OlliTeX +# toolkit mongo-probe). +services: + mongo-probe: + image: mongo:${MONGO_VERSION:-9.0} + container_name: mongo-probe + hostname: mongo-probe + restart: unless-stopped + command: ["sh", "-c", "sleep infinity"] + environment: + MONGO_HOSTNAME: ${MONGO_HOSTNAME:-mongo} + MONGO_PORT: ${MONGO_PORT:-27017} + healthcheck: + test: ["CMD-SHELL", "mongosh --quiet --host ${MONGO_HOSTNAME} --port ${MONGO_PORT} --eval 'db.adminCommand({ping:1})' >/dev/null 2>&1 || exit 1"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 20s + networks: + - overleaf-network + +networks: + overleaf-network: + external: true diff --git a/tools/reference_stack_compose/mongodb-exporter-probe/compose.yaml b/tools/reference_stack_compose/mongodb-exporter-probe/compose.yaml new file mode 100644 index 0000000000..29089b4ebc --- /dev/null +++ b/tools/reference_stack_compose/mongodb-exporter-probe/compose.yaml @@ -0,0 +1,22 @@ +# mongodb-exporter-probe — HTTP serving proof for the mongodb exporter. +# The exporter image has no shell, so its /metrics reachability is checked +# here (busybox spider). healthy == /metrics answers. +services: + mongodb-exporter-probe: + image: ${PROBE_IMAGE:-busybox:1.36} + container_name: mongodb-exporter-probe + hostname: mongodb-exporter-probe + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "wget -q --spider --timeout=5 http://mongodb-exporter:9216/metrics >/dev/null 2>&1 || exit 1"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + entrypoint: ["sh", "-c", "sleep infinity"] + networks: + - overleaf-network + +networks: + overleaf-network: + external: true diff --git a/tools/reference_stack_compose/mongodb-exporter/compose.yaml b/tools/reference_stack_compose/mongodb-exporter/compose.yaml new file mode 100644 index 0000000000..0581d2ef95 --- /dev/null +++ b/tools/reference_stack_compose/mongodb-exporter/compose.yaml @@ -0,0 +1,37 @@ +# mongodb-exporter — percona/mongodb_exporter for the stack Mongo. +# NOTE (verified 2026-10 against this stack's mongo:9.0 replset "overleaf"): +# percona 0.43 ships the rich collectors OFF by default (default run answers +# only mongodb_up + go/process self metrics). The --collector.* toggles are +# therefore part of the supported surface here — keep them. +services: + mongodb-exporter: + image: ${MONGODB_EXPORTER_IMAGE:-percona/mongodb_exporter:0.43.0} + container_name: mongodb-exporter + hostname: mongodb-exporter + restart: unless-stopped + command: + - "--mongodb.uri=mongodb://${MONGO_HOSTNAME:-mongo}:${MONGO_PORT:-27017}/${MONGODB_DATABASE:-sharelatex}" + # verified metric families: mongodb_ss_* (connections, oplog time, ...), + # mongodb_dbstats_dataSize/storageSize/collections per database, + # mongodb_replicasetstatus_*, mongodb_fs_* + - "--collector.diagnosticdata" + - "--collector.dbstats" + - "--collector.dbstatsfreestorage" + - "--collector.replicasetstatus" + expose: + - "9216" + # scratch image: single binary, no shell — liveness = the binary still + # starts. The HTTP SERVING proof lives in mongodb-exporter-probe + # (same proven pattern as wakapi-probe). + healthcheck: + test: ["CMD", "/mongodb_exporter", "--version"] + interval: 10s + timeout: 5s + retries: 6 + start_period: 15s + networks: + - overleaf-network + +networks: + overleaf-network: + external: true diff --git a/tools/reference_stack_compose/nginx/nginx.conf b/tools/reference_stack_compose/nginx/nginx.conf index abe111edb6..e741094f31 100644 --- a/tools/reference_stack_compose/nginx/nginx.conf +++ b/tools/reference_stack_compose/nginx/nginx.conf @@ -66,6 +66,26 @@ http { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_http_version 1.1; } + + # Bundled Grafana (monitoring UI) — same-origin under /grafana/. + # Unlike /wakapi the prefix is NOT stripped: grafana runs with + # GF_SERVER_SERVE_FROM_SUB_PATH=true + GF_SERVER_ROOT_URL ending in + # /grafana/, so it serves its routes at /grafana/... itself; the raw + # :3000 port is not published to the host (grafana compose). + location = /grafana { + return 301 /grafana/; + } + location /grafana/ { + proxy_pass http://grafana:3000; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_read_timeout 5m; + proxy_send_timeout 5m; + } } } diff --git a/tools/reference_stack_compose/node-exporter/compose.yaml b/tools/reference_stack_compose/node-exporter/compose.yaml new file mode 100644 index 0000000000..c1848e9d1a --- /dev/null +++ b/tools/reference_stack_compose/node-exporter/compose.yaml @@ -0,0 +1,29 @@ +# node-exporter — host-level signals (CPU, memory, filesystem, ...). +# Reads the HOST's /proc, / and /sys read-only into the container +# (same flags as the OlliTeX toolkit node-exporter). +services: + node-exporter: + image: ${NODE_EXPORTER_IMAGE:-prom/node-exporter:v1.9.1} + container_name: node-exporter + hostname: node-exporter + restart: unless-stopped + command: + - "--path.procfs=/host/proc" + - "--path.rootfs=/host" + - "--path.sysfs=/host/sys" + ports: + - "127.0.0.1:${NODE_EXPORTER_PORT:-9100}:9100" # host admin only (SSH tunnel) + volumes: + - /:/host:ro + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9100/metrics"] + start_period: 15s + interval: 30s + retries: 5 + timeout: 5s + networks: + - overleaf-network + +networks: + overleaf-network: + external: true diff --git a/tools/reference_stack_compose/prometheus/.gitignore b/tools/reference_stack_compose/prometheus/.gitignore new file mode 100644 index 0000000000..df0fa3fdd0 --- /dev/null +++ b/tools/reference_stack_compose/prometheus/.gitignore @@ -0,0 +1 @@ +prometheus-data diff --git a/tools/reference_stack_compose/prometheus/compose.yaml b/tools/reference_stack_compose/prometheus/compose.yaml new file mode 100644 index 0000000000..f3ddce2630 --- /dev/null +++ b/tools/reference_stack_compose/prometheus/compose.yaml @@ -0,0 +1,34 @@ +# prometheus — Prometheus for the overleaf-cep stack. +# Static scrape config (prometheus.yml, mounted ro) — all targets live on +# the shared `overleaf-network`, so no host port publishing is needed for +# scraping; the 127.0.0.1:9090 binding is for local admin/SSH-tunnel only. +# Retention 15d (reference: OlliTeX toolkit compose). +services: + prometheus: + image: ${PROMETHEUS_IMAGE:-prom/prometheus:v2.53.5} + container_name: prometheus + hostname: prometheus + restart: unless-stopped + entrypoint: ["/bin/prometheus"] + command: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + - "--storage.tsdb.retention.time=${PROMETHEUS_RETENTION:-15d}" + - "--web.enable-lifecycle" + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./prometheus-data:/prometheus + ports: + - "${PROMETHEUS_LISTEN_IP:-127.0.0.1}:${PROMETHEUS_PORT:-9090}:9090" + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9090/-/healthy"] + start_period: 20s + interval: 30s + retries: 5 + timeout: 5s + networks: + - overleaf-network + +networks: + overleaf-network: + external: true diff --git a/tools/reference_stack_compose/prometheus/prometheus.yml b/tools/reference_stack_compose/prometheus/prometheus.yml new file mode 100644 index 0000000000..05e149a953 --- /dev/null +++ b/tools/reference_stack_compose/prometheus/prometheus.yml @@ -0,0 +1,67 @@ +# overleaf-cep — Prometheus scrape config +# Modeled on the OlliTeX toolkit compose (reference: toolkit/compose.yaml + +# build-images/prometheus/prometheus.yml), adapted to THIS stack's container +# names. All targets are inside `overleaf-network` (Prometheus scrapes from +# inside the docker network; the nginx edge does not forward /metrics). +global: + scrape_interval: 15s + evaluation_interval: 15s + external_labels: + stack: overleaf-cep + +scrape_configs: + # --- the Overleaf app itself ------------------------------------------------ + # The sharelatex image ships the @overleaf/metrics sensors on the web + # process (libraries/metrics): num_active_users, timer_http_request + # (summaries with quantiles), mongo_command_*, nodejs_* (event loop lag, + # GC, heap), security_http_auth_check_credentials, editor_join_project, + # tpds_add_doc, lock_mongoTransaction_*, ... + # Verified live: 0.0.0.0:3000 inside the container, samples carry + # app="web-api",host="overleaf". (Other in-image services e.g. compile + # :4000 are 127.0.0.1-bound and therefore not network-scrapable.) + - job_name: overleaf-web + metrics_path: /metrics + static_configs: + - targets: ["overleaf:3000"] + labels: { app: web-api } + + # --- MongoDB (percona exporter) ---------------------------------------------- + - job_name: overleaf-mongodb + metrics_path: /metrics + static_configs: + - targets: ["mongodb-exporter:9216"] + + # --- Redis (oliver006 exporter) ------------------------------------------------ + - job_name: overleaf-redis + metrics_path: /metrics + static_configs: + - targets: ["redis-exporter:9121"] + + # --- Filestore (SeaweedFS master native prometheus, -metricsPort=9327) -------- + - job_name: overleaf-filestore + metrics_path: /metrics + static_configs: + - targets: ["seaweedfs-master:9327"] + labels: { component: master } + + # --- Instance stats sensor (PR yu-i-i/overleaf-cep#178, live as prometheus) --- + - job_name: overleaf-instance + metrics_path: /metrics + static_configs: + - targets: ["instance-sensor:9464"] + + # --- Host-level signals --------------------------------------------------------- + - job_name: node + metrics_path: /metrics + static_configs: + - targets: ["node-exporter:9100"] + + # --- The monitoring stack itself (self-up visibility in dashboards) ------------- + - job_name: prometheus + static_configs: + - targets: ["prometheus:9090"] + + - job_name: grafana + metrics_path: /metrics + static_configs: + - targets: ["grafana:3000"] diff --git a/tools/reference_stack_compose/redis-exporter-probe/compose.yaml b/tools/reference_stack_compose/redis-exporter-probe/compose.yaml new file mode 100644 index 0000000000..a3e6478c84 --- /dev/null +++ b/tools/reference_stack_compose/redis-exporter-probe/compose.yaml @@ -0,0 +1,22 @@ +# redis-exporter-probe — HTTP serving proof for the redis exporter. +# Same sidecar contract as mongodb-exporter-probe: +# busybox spider on the exporter's in-container /metrics (9121). +services: + redis-exporter-probe: + image: ${PROBE_IMAGE:-busybox:1.36} + container_name: redis-exporter-probe + hostname: redis-exporter-probe + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "wget -q --spider --timeout=5 http://redis-exporter:9121/metrics >/dev/null 2>&1 || exit 1"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + entrypoint: ["sh", "-c", "sleep infinity"] + networks: + - overleaf-network + +networks: + overleaf-network: + external: true diff --git a/tools/reference_stack_compose/redis-exporter/compose.yaml b/tools/reference_stack_compose/redis-exporter/compose.yaml new file mode 100644 index 0000000000..fae3b433b3 --- /dev/null +++ b/tools/reference_stack_compose/redis-exporter/compose.yaml @@ -0,0 +1,28 @@ +# redis-exporter — oliver006/redis_exporter for the stack Redis. +# Rich out of the box (verified live: redis_up, redis_memory_used_bytes, +# redis_connected_clients, redis_commands_processed_total, redis_aof_current_size, +# redis_keyspace_*_total, ...). +services: + redis-exporter: + image: ${REDIS_EXPORTER_IMAGE:-oliver006/redis_exporter:v1.58.0-alpine} + container_name: redis-exporter + hostname: redis-exporter + restart: unless-stopped + environment: + REDIS_ADDR: "redis://${REDIS_HOSTNAME:-redis}:${REDIS_PORT:-6379}" + expose: + - "9121" + # alpine edition has no wget/nc applets — liveness = the exporter binary + # itself still runs (image entrypoint is /redis_exporter). + healthcheck: + test: ["CMD", "/redis_exporter", "--version"] + interval: 10s + timeout: 5s + retries: 6 + start_period: 15s + networks: + - overleaf-network + +networks: + overleaf-network: + external: true diff --git a/tools/reference_stack_compose/up_grafana.sh b/tools/reference_stack_compose/up_grafana.sh new file mode 100644 index 0000000000..9a742385d5 --- /dev/null +++ b/tools/reference_stack_compose/up_grafana.sh @@ -0,0 +1,2 @@ +docker compose up -d grafana + diff --git a/tools/reference_stack_compose/up_instance-sensor.sh b/tools/reference_stack_compose/up_instance-sensor.sh new file mode 100644 index 0000000000..1296d393a3 --- /dev/null +++ b/tools/reference_stack_compose/up_instance-sensor.sh @@ -0,0 +1,2 @@ +docker compose up -d instance-sensor + diff --git a/tools/reference_stack_compose/up_mongo-probe.sh b/tools/reference_stack_compose/up_mongo-probe.sh new file mode 100644 index 0000000000..4e72ff5d49 --- /dev/null +++ b/tools/reference_stack_compose/up_mongo-probe.sh @@ -0,0 +1,2 @@ +docker compose up -d mongo-probe + diff --git a/tools/reference_stack_compose/up_mongodb-exporter-probe.sh b/tools/reference_stack_compose/up_mongodb-exporter-probe.sh new file mode 100644 index 0000000000..0dc5836340 --- /dev/null +++ b/tools/reference_stack_compose/up_mongodb-exporter-probe.sh @@ -0,0 +1,2 @@ +docker compose up -d mongodb-exporter-probe + diff --git a/tools/reference_stack_compose/up_mongodb-exporter.sh b/tools/reference_stack_compose/up_mongodb-exporter.sh new file mode 100644 index 0000000000..1315617284 --- /dev/null +++ b/tools/reference_stack_compose/up_mongodb-exporter.sh @@ -0,0 +1,2 @@ +docker compose up -d mongodb-exporter + diff --git a/tools/reference_stack_compose/up_node-exporter.sh b/tools/reference_stack_compose/up_node-exporter.sh new file mode 100644 index 0000000000..f7ef2ec409 --- /dev/null +++ b/tools/reference_stack_compose/up_node-exporter.sh @@ -0,0 +1,2 @@ +docker compose up -d node-exporter + diff --git a/tools/reference_stack_compose/up_prometheus.sh b/tools/reference_stack_compose/up_prometheus.sh new file mode 100644 index 0000000000..67ff5dbd3e --- /dev/null +++ b/tools/reference_stack_compose/up_prometheus.sh @@ -0,0 +1,2 @@ +docker compose up -d prometheus + diff --git a/tools/reference_stack_compose/up_redis-exporter-probe.sh b/tools/reference_stack_compose/up_redis-exporter-probe.sh new file mode 100644 index 0000000000..88cfc285e3 --- /dev/null +++ b/tools/reference_stack_compose/up_redis-exporter-probe.sh @@ -0,0 +1,2 @@ +docker compose up -d redis-exporter-probe + diff --git a/tools/reference_stack_compose/up_redis-exporter.sh b/tools/reference_stack_compose/up_redis-exporter.sh new file mode 100644 index 0000000000..f5a6bdc350 --- /dev/null +++ b/tools/reference_stack_compose/up_redis-exporter.sh @@ -0,0 +1,2 @@ +docker compose up -d redis-exporter +