Statistics with Grafana/Prometheus with sensor based on Overleaf instance statistics- #178

2026-11-12 21:23:45 +01:00 · 2026-10-27 13:43:32 +00:00
parent 6f2cf1c060
commit 05b1b04513
57 changed files with 3456 additions and 22 deletions
@@ -0,0 +1,214 @@
+// overleaf-instance-sensor
+// Live Prometheus gauges for the overleaf-cep instance statistics.
+//
+// Sensor definitions ported from PR yu-i-i/overleaf-cep#178
+// (services/web/modules/instance-stats/.../collectInstanceStats.mjs):
+//   * users     : total count + active (lastActive >= now-24h)
+//   * projects  : total count + active (lastUpdated >= now-24h)
+//   * files     : docs collection count
+//   * storage   : mongodb dbStats.totalSize, overleaf filestore (du of the
+//                 data dir), redis RAM (used_memory) + redis disk (aof size)
+//   * optional standard/guest split: guest = 'guest-user' in user.adminRoles
+//     (the PR#176 segmentation; PR#178 env INSTANCE_STATS_USER_SEGMENTATION_ENABLED)
+//
+// Instead of the PR's hourly snapshot docs in Mongo, we compute on every
+// scrape (cheap counts on this scale; fresh values, no retention state).
+// Each probe is isolated: a failing source emits its error counter but the
+// rest of the endpoint still answers.
+
+import http from 'node:http'
+import { execFile } from 'node:child_process'
+import { performance } from 'node:perf_hooks'
+import { promisify } from 'node:util'
+import { MongoClient } from 'mongodb'
+import Redis from 'ioredis'
+
+const execFileAsync = promisify(execFile)
+
+const PORT = Number(process.env.PORT || 9464)
+const MONGO_URL = process.env.MONGO_URL || 'mongodb://mongo:27017/sharelatex'
+const REDIS_ADDR = process.env.REDIS_ADDR || 'redis://redis:6379'
+const FILESTORE_DATA_DIR = process.env.FILESTORE_DATA_DIR || '/hostdata/filestore'
+const SEGMENTATION = String(process.env.SEGMENTATION || 'false') === 'true'
+const ACTIVE_WINDOW_MS = Number(process.env.ACTIVE_WINDOW_HOURS || 24) * 3600 * 1000
+
+const GUEST_ADMIN_ROLE = 'guest-user' // PR#176/PR#178 guest definition
+
+const errors = new Map() // name -> failed probe count
+let lastSuccessSeconds = 0
+
+const guestUserFilter = () => ({ adminRoles: { $in: [GUEST_ADMIN_ROLE] } })
+
+async function duBytes(path) {
+  // byte-exact du of a directory (same semantics as PR#178's duBytes())
+  try {
+    const { stdout } = await execFileAsync('du', ['-sb', path])
+    const m = String(stdout).match(/(\d+)/)
+    return m ? Number(m[1]) : 0
+  } catch {
+    throw new Error(`du failed for ${path}`)
+  }
+}
+
+function parseRedisInfoInt(info, key) {
+  const m = String(info || '').match(new RegExp(`^${key}:(\\d+)`, 'm'))
+  return m ? Number(m[1]) : 0
+}
+
+async function collect() {
+  const nowMs = Date.now()
+  const oneAgo = new Date(nowMs - ACTIVE_WINDOW_MS)
+  const out = []
+
+  let mongoOk = true
+  try {
+    const client = new MongoClient(MONGO_URL, { serverSelectionTimeoutMS: 5000 })
+    await client.connect()
+    try {
+      const db = client.db() // db name comes from MONGO_URL
+      const users = db.collection('users')
+      const projects = db.collection('projects')
+      const docs = db.collection('docs')
+
+      const totalUsers = await users.countDocuments()
+      const activeUsers = await users.countDocuments({ lastActive: { $gte: oneAgo } })
+      out.push(
+        '# HELP overleaf_instance_users_total Total registered users (segment: all, plus standard/guest when segmentation is enabled).',
+        '# TYPE overleaf_instance_users_total gauge',
+        `overleaf_instance_users_total{segment="all"} ${totalUsers}`,
+        '# HELP overleaf_instance_users_active_24h Users active in the last 24 hours (lastActive >= now-24h).',
+        '# TYPE overleaf_instance_users_active_24h gauge',
+        `overleaf_instance_users_active_24h{segment="all"} ${activeUsers}`,
+      )
+
+      if (SEGMENTATION) {
+        const guestFilter = guestUserFilter()
+        const guestUsers = await users.countDocuments(guestFilter)
+        const activeGuest = await users.countDocuments({
+          ...guestFilter,
+          lastActive: { $gte: oneAgo },
+        })
+        out.push(
+          `overleaf_instance_users_total{segment="standard"} ${totalUsers - guestUsers}`,
+          `overleaf_instance_users_total{segment="guest"} ${guestUsers}`,
+          `overleaf_instance_users_active_24h{segment="standard"} ${activeUsers - activeGuest}`,
+          `overleaf_instance_users_active_24h{segment="guest"} ${activeGuest}`,
+        )
+      }
+
+      const totalProjects = await projects.countDocuments()
+      const activeProjects = await projects.countDocuments({ lastUpdated: { $gte: oneAgo } })
+      const totalFiles = await docs.countDocuments()
+      out.push(
+        '# HELP overleaf_instance_projects_total Total projects.',
+        '# TYPE overleaf_instance_projects_total gauge',
+        `overleaf_instance_projects_total ${totalProjects}`,
+        '# HELP overleaf_instance_projects_active_24h Projects updated in the last 24 hours (lastUpdated >= now-24h).',
+        '# TYPE overleaf_instance_projects_active_24h gauge',
+        `overleaf_instance_projects_active_24h ${activeProjects}`,
+        '# HELP overleaf_instance_files_total Total documents (files) in the docs collection.',
+        '# TYPE overleaf_instance_files_total gauge',
+        `overleaf_instance_files_total ${totalFiles}`,
+      )
+
+      // PR#178: mongodb_storage = dbStats.totalSize (bytes)
+      let mongoBytes = 0
+      try {
+        const stats = await db.stats()
+        mongoBytes = Number(stats?.totalSize ?? 0)
+      } catch {
+        /* keep 0 */
+      }
+      out.push(
+        '# HELP overleaf_instance_storage_bytes Storage used, bytes. component: mongodb | filestore | redis_ram | redis_disk.',
+        '# TYPE overleaf_instance_storage_bytes gauge',
+        `overleaf_instance_storage_bytes{component="mongodb"} ${mongoBytes}`,
+      )
+    } finally {
+      await client.close()
+    }
+  } catch {
+    mongoOk = false
+    errors.set('mongo', (errors.get('mongo') || 0) + 1)
+  }
+
+  // filestore size (SeaweedFS data dir, mounted ro) --------------------------
+  let fileOk = true
+  try {
+    const bytes = await duBytes(FILESTORE_DATA_DIR)
+    out.push(`overleaf_instance_storage_bytes{component="filestore"} ${bytes}`)
+  } catch {
+    fileOk = false
+    errors.set('filestore', (errors.get('filestore') || 0) + 1)
+  }
+
+  // redis storage: RAM (used_memory) + disk (aof_current_size) ----------------
+  let redisOk = true
+  try {
+    const r = new Redis(REDIS_ADDR, {
+      maxRetriesPerRequest: 1,
+      connectTimeout: 5000,
+      lazyConnect: true,
+    })
+    try {
+      const usedMemory = parseRedisInfoInt(await r.info('memory'), 'used_memory')
+      let aof = 0
+      try {
+        aof = parseRedisInfoInt(await r.info('persistence'), 'aof_current_size')
+      } catch {
+        /* aof may be disabled — 0 is the true value */
+      }
+      out.push(
+        `overleaf_instance_storage_bytes{component="redis_ram"} ${usedMemory}`,
+        `overleaf_instance_storage_bytes{component="redis_disk"} ${aof}`,
+      )
+    } finally {
+      try {
+        r.disconnect()
+      } catch {
+        /* noop */
+      }
+    }
+  } catch {
+    redisOk = false
+    errors.set('redis', (errors.get('redis') || 0) + 1)
+  }
+
+  const up = mongoOk && fileOk && redisOk ? 1 : 0
+  if (up === 1) lastSuccessSeconds = (Date.now() / 1000) | 0
+
+  out.push(
+    '# HELP overleaf_instance_sensor_up 1 when the last scrape computed all probes successfully.',
+    '# TYPE overleaf_instance_sensor_up gauge',
+    `overleaf_instance_sensor_up ${up}`,
+    '# HELP overleaf_instance_sensor_last_success_timestamp_seconds Unix time of the last fully successful scrape.',
+    '# TYPE overleaf_instance_sensor_last_success_timestamp_seconds gauge',
+    `overleaf_instance_sensor_last_success_timestamp_seconds ${lastSuccessSeconds}`,
+    '# HELP overleaf_instance_sensor_errors_total Failed probe count since start (name=mongo|filestore|redis).',
+    '# TYPE overleaf_instance_sensor_errors_total counter',
+  )
+  for (const [name, n] of errors) {
+    out.push(`overleaf_instance_sensor_errors_total{name="${name}"} ${n}`)
+  }
+  return `${out.join('\n')}\n`
+}
+
+const server = http.createServer(async (req, res) => {
+  if (req.url !== '/metrics') {
+    res.writeHead(404, { 'content-type': 'text/plain' })
+    res.end('not found\n')
+    return
+  }
+  const t0 = performance.now()
+  const body = await collect()
+  const dt = (performance.now() - t0) / 1000
+  res.writeHead(200, {
+    'content-type': 'text/plain; version=0.0.4; charset=utf-8',
+    'content-length': Buffer.byteLength(body),
+  })
+  res.end(body)
+})
+
+server.listen(PORT, '0.0.0.0', () => {
+  console.log(`overleaf-instance-sensor listening on :${PORT}`)
+})