diff --git a/server-ce/Dockerfile b/server-ce/Dockerfile index daea67884f..68784c0a06 100644 --- a/server-ce/Dockerfile +++ b/server-ce/Dockerfile @@ -68,6 +68,8 @@ ADD server-ce/config/crontab-history /etc/cron.d/crontab-history RUN chmod 600 /etc/cron.d/crontab-history ADD server-ce/config/crontab-deletion /etc/cron.d/crontab-deletion RUN chmod 600 /etc/cron.d/crontab-deletion +ADD server-ce/config/crontab-instance-stats /etc/cron.d/crontab-instance-stats +RUN chmod 600 /etc/cron.d/crontab-instance-stats # Copy Phusion Image startup and shutdown scripts to their locations # ------------------------------------------------------------------ diff --git a/server-ce/config/crontab-instance-stats b/server-ce/config/crontab-instance-stats new file mode 100644 index 0000000000..3e1047a6ec --- /dev/null +++ b/server-ce/config/crontab-instance-stats @@ -0,0 +1,2 @@ +50 * * * * root bash /overleaf/cron/collect-instance-stats.sh >> /var/log/overleaf/cron-instance-stats.log 2>&1 + diff --git a/server-ce/cron/collect-instance-stats.sh b/server-ce/cron/collect-instance-stats.sh new file mode 100644 index 0000000000..30d93003ee --- /dev/null +++ b/server-ce/cron/collect-instance-stats.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -eu + +echo "--------------------------" +echo "Collect instance statistics" +echo "--------------------------" +date + +source /etc/container_environment.sh +source /etc/overleaf/env.sh + +# `@overleaf/settings` resolves `config/settings.defaults.*` relative to +# `process.cwd()`, so ensure we're running from `services/web`. +cd /overleaf/services/web + +node /overleaf/services/web/modules/instance-stats/app/src/collectInstanceStats.mjs + diff --git a/services/web/app/views/layout-react.pug b/services/web/app/views/layout-react.pug index 45accda5b1..0ea93694d8 100644 --- a/services/web/app/views/layout-react.pug +++ b/services/web/app/views/layout-react.pug @@ -15,6 +15,7 @@ block append meta - const canDisplaySplitTestMenu = hasFeature('saas') && hasNonAdminDomainCapability('view-split-test') - const canDisplaySurveyMenu = hasFeature('saas') && canDisplayAdminMenu && hasAdminCapability('manage-survey', false) - const canDisplayScriptLogMenu = hasFeature('saas') && hasAdminCapability('view-script-log', false) && canDisplayAdminMenu + - const canDisplayInstanceStats = settings.instanceStats && settings.instanceStats.enabled - const enableUpgradeButton = projectDashboardReact && usersBestSubscription && (usersBestSubscription.type === 'free' || usersBestSubscription.type === 'standalone-ai-add-on') - const showSignUpLink = hasFeature('registration-page') @@ -31,6 +32,7 @@ block append meta canDisplaySplitTestMenu, canDisplaySurveyMenu, canDisplayScriptLogMenu, + canDisplayInstanceStats, enableUpgradeButton, suppressNavbarRight: !!suppressNavbarRight, suppressNavContentLinks: !!suppressNavContentLinks, diff --git a/services/web/app/views/layout/navbar-marketing.pug b/services/web/app/views/layout/navbar-marketing.pug index 61693f105f..7814fcba98 100644 --- a/services/web/app/views/layout/navbar-marketing.pug +++ b/services/web/app/views/layout/navbar-marketing.pug @@ -68,6 +68,8 @@ nav.navbar.navbar-default.navbar-main.navbar-expand-lg( +dropdown-menu-link-item(href='/admin/user') Manage Users if canDisplayProjectUrlLookup +dropdown-menu-link-item(href='/admin/project') Manage Projects + if settings.instanceStats && settings.instanceStats.enabled + +dropdown-menu-link-item(href='/admin/instance-stats') Instance Statistics if canDisplayAdminRedirect +dropdown-menu-link-item(href=settings.adminUrl) Switch to Admin if canDisplaySplitTestMenu diff --git a/services/web/config/settings.defaults.js b/services/web/config/settings.defaults.js index 85457fea80..b730c2084e 100644 --- a/services/web/config/settings.defaults.js +++ b/services/web/config/settings.defaults.js @@ -1256,6 +1256,7 @@ module.exports = { 'authentication/oidc', 'admin-tools', // import after authentication 'registration-page', // import after authentication + 'instance-stats', 'template-gallery', 'git-bridge', 'git-sync/github', diff --git a/services/web/frontend/js/shared/components/navbar/admin-menu.tsx b/services/web/frontend/js/shared/components/navbar/admin-menu.tsx index d44ee6c1b3..202873a8c3 100644 --- a/services/web/frontend/js/shared/components/navbar/admin-menu.tsx +++ b/services/web/frontend/js/shared/components/navbar/admin-menu.tsx @@ -10,6 +10,7 @@ export default function AdminMenu({ canDisplaySplitTestMenu, canDisplaySurveyMenu, canDisplayScriptLogMenu, + canDisplayInstanceStats, adminUrl, }: Pick< DefaultNavbarMetadata, @@ -19,6 +20,7 @@ export default function AdminMenu({ | 'canDisplaySplitTestMenu' | 'canDisplaySurveyMenu' | 'canDisplayScriptLogMenu' + | 'canDisplayInstanceStats' | 'adminUrl' >) { const sendProjectListMB = useSendProjectListMB() @@ -68,6 +70,11 @@ export default function AdminMenu({ View Script Logs ) : null} + {canDisplayInstanceStats ? ( + + Instance Statistics + + ) : null} ) } diff --git a/services/web/frontend/js/shared/components/navbar/default-navbar.tsx b/services/web/frontend/js/shared/components/navbar/default-navbar.tsx index f6c8e196dc..70581cb122 100644 --- a/services/web/frontend/js/shared/components/navbar/default-navbar.tsx +++ b/services/web/frontend/js/shared/components/navbar/default-navbar.tsx @@ -30,6 +30,7 @@ function DefaultNavbar( canDisplaySplitTestMenu, canDisplaySurveyMenu, canDisplayScriptLogMenu, + canDisplayInstanceStats, enableUpgradeButton, suppressNavbarRight, suppressNavContentLinks, @@ -117,6 +118,7 @@ function DefaultNavbar( canDisplaySplitTestMenu={canDisplaySplitTestMenu} canDisplaySurveyMenu={canDisplaySurveyMenu} canDisplayScriptLogMenu={canDisplayScriptLogMenu} + canDisplayInstanceStats={canDisplayInstanceStats} adminUrl={adminUrl} /> ) : null} diff --git a/services/web/frontend/js/shared/components/types/default-navbar-metadata.ts b/services/web/frontend/js/shared/components/types/default-navbar-metadata.ts index bb59dd7923..99c688e6c0 100644 --- a/services/web/frontend/js/shared/components/types/default-navbar-metadata.ts +++ b/services/web/frontend/js/shared/components/types/default-navbar-metadata.ts @@ -13,6 +13,7 @@ export type DefaultNavbarMetadata = { canDisplaySplitTestMenu: boolean canDisplaySurveyMenu: boolean canDisplayScriptLogMenu: boolean + canDisplayInstanceStats: boolean enableUpgradeButton: boolean suppressNavbarRight: boolean suppressNavContentLinks: boolean diff --git a/services/web/modules/instance-stats/app/src/InstanceStatsController.mjs b/services/web/modules/instance-stats/app/src/InstanceStatsController.mjs new file mode 100644 index 0000000000..3bf893d926 --- /dev/null +++ b/services/web/modules/instance-stats/app/src/InstanceStatsController.mjs @@ -0,0 +1,95 @@ +import Path from 'node:path' +import { fileURLToPath } from 'node:url' +import logger from '@overleaf/logger' +import Settings from '@overleaf/settings' + +import { InstanceStat } from './models/InstanceStat.mjs' + +const __dirname = Path.dirname(fileURLToPath(import.meta.url)) + +const STAT_KEYS = [ + 'active_projects', + 'active_users', + 'user_count', + 'project_count', + 'file_count', + 'mongodb_storage', + 'overleaf_storage', + 'redis_storage', +] + +const WINDOWS = { + month: 30, + '6m': 180, + year: 365, +} + +function toUtcMidnight(date) { + return new Date( + Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()) + ) +} + +function validateWindow(window) { + if (window === 'all') return true + return Object.hasOwn(WINDOWS, window) +} + +async function page(req, res, next) { + try { + res.render(Path.resolve(__dirname, '../views/instance-stats'), { + title: 'Instance Statistics', + initialWindow: 'month', + }) + } catch (err) { + next(err) + } +} + +async function series(req, res, next) { + const metric = req.query.metric + const window = req.query.window || 'month' + + try { + if (typeof metric !== 'string' || !STAT_KEYS.includes(metric)) { + return res.status(400).json({ message: 'Invalid metric' }) + } + + if (typeof window !== 'string' || !validateWindow(window)) { + return res.status(400).json({ message: 'Invalid window' }) + } + + const retentionDays = Settings.instanceStats?.retentionDays || 365 + const now = new Date() + const cutoffDays = window === 'all' ? retentionDays : WINDOWS[window] + + const cutoffDate = toUtcMidnight( + new Date(now.getTime() - cutoffDays * 24 * 60 * 60 * 1000) + ) + + const docs = await InstanceStat.find( + { statKey: metric, day: { $gte: cutoffDate } }, + { _id: 0, day: 1, values: 1 } + ) + .sort({ day: 1 }) + .lean() + + return res.status(200).json({ + metric, + window, + points: docs.map(d => ({ + day: d.day.getTime(), + values: d.values, + })), + }) + } catch (err) { + logger.error({ err, metric, window }, 'Failed to fetch instance stats series') + next(err) + } +} + +export default { + page, + series, +} + diff --git a/services/web/modules/instance-stats/app/src/InstanceStatsRouter.mjs b/services/web/modules/instance-stats/app/src/InstanceStatsRouter.mjs new file mode 100644 index 0000000000..db8695e3f6 --- /dev/null +++ b/services/web/modules/instance-stats/app/src/InstanceStatsRouter.mjs @@ -0,0 +1,28 @@ +import logger from '@overleaf/logger' +import AuthorizationMiddleware from '../../../../app/src/Features/Authorization/AuthorizationMiddleware.mjs' +import InstanceStatsController from './InstanceStatsController.mjs' +import Settings from '@overleaf/settings' + +export default { + apply(webRouter) { + if (!Settings.instanceStats?.enabled) { + logger.info({ enabled: false }, 'Instance stats module disabled') + return + } + + logger.debug({}, 'Init instance-stats router') + + webRouter.get( + '/admin/instance-stats', + AuthorizationMiddleware.ensureUserIsSiteAdmin, + InstanceStatsController.page + ) + + webRouter.get( + '/admin/instance-stats/api/series', + AuthorizationMiddleware.ensureUserIsSiteAdmin, + InstanceStatsController.series + ) + }, +} + diff --git a/services/web/modules/instance-stats/app/src/clearInstanceStats.mjs b/services/web/modules/instance-stats/app/src/clearInstanceStats.mjs new file mode 100644 index 0000000000..b5e3b6c165 --- /dev/null +++ b/services/web/modules/instance-stats/app/src/clearInstanceStats.mjs @@ -0,0 +1,35 @@ +import logger from '@overleaf/logger' +import mongoose from '../../../../app/src/infrastructure/Mongoose.mjs' +import { InstanceStat } from './models/InstanceStat.mjs' + +function parseConfirmArg() { + const confirmArg = process.argv.find(arg => arg.startsWith('--confirm=')) + return confirmArg ? confirmArg.slice('--confirm='.length) : undefined +} + +async function main() { + await mongoose.connectionPromise + + const confirmation = + process.env.INSTANCE_STATS_CLEAR_CONFIRM || parseConfirmArg() + if (confirmation !== 'YES') { + throw new Error( + "Refusing to clear stats. Set INSTANCE_STATS_CLEAR_CONFIRM=YES or pass --confirm=YES." + ) + } + + const before = await InstanceStat.countDocuments({}) + const result = await InstanceStat.deleteMany({}) + + logger.info( + { before, deletedCount: result.deletedCount || 0 }, + 'Cleared instance statistics' + ) +} + +main() + .then(() => process.exit(0)) + .catch(err => { + logger.err({ err }, 'Failed clearing instance statistics') + process.exit(1) + }) diff --git a/services/web/modules/instance-stats/app/src/collectInstanceStats.mjs b/services/web/modules/instance-stats/app/src/collectInstanceStats.mjs new file mode 100644 index 0000000000..ed43d08116 --- /dev/null +++ b/services/web/modules/instance-stats/app/src/collectInstanceStats.mjs @@ -0,0 +1,175 @@ +import logger from '@overleaf/logger' +import Settings from '@overleaf/settings' +import mongoose from '../../../../app/src/infrastructure/Mongoose.mjs' +import { InstanceStat } from './models/InstanceStat.mjs' +import RedisWrapper from '../../../../app/src/infrastructure/RedisWrapper.mjs' +import { execFile } from 'node:child_process' +import { promisify } from 'node:util' + +const execFileAsync = promisify(execFile) + +/** Role string stored inside User.adminRoles (array of strings). */ +const GUEST_ADMIN_ROLE = 'guest-user' + +/** Mongo filter: user is a guest if their adminRoles list includes this role. */ +function guestUserFilter() { + return { adminRoles: { $in: [GUEST_ADMIN_ROLE] } } +} + +function isUserSegmentationEnabled() { + return Boolean(Settings.instanceStats?.userSegmentation?.enabled) +} + +function toUtcMidnight(date) { + return new Date( + Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()) + ) +} + +function parseFirstInt(output) { + const match = output?.match(/(\d+)/) + return match ? parseInt(match[1], 10) : 0 +} + +async function duBytes(path) { + // Prefer exact du output in bytes. Fall back to 0 on failure. + try { + const { stdout } = await execFileAsync('du', ['-b0s', path]) + return parseFirstInt(stdout) + } catch (err) { + logger.error({ err, path }, 'Failed to compute du bytes') + return 0 + } +} + +function parseRedisInfoInt(info, key) { + const re = new RegExp(`${key}:(\\d+)`) + const match = info?.match(re) + return match ? parseInt(match[1], 10) : 0 +} + +async function getRedisMemoryBytes() { + const rclient = RedisWrapper.client('instance_stats') + try { + // In ioredis, `info(section)` returns a string. + const info_memory = await rclient.info('memory') + const usedMemory = parseRedisInfoInt(info_memory, 'used_memory') + const info_aof = await rclient.info('persistence') + const usedDisk = parseRedisInfoInt(info_aof, 'aof_current_size') + + return { diskBytes: usedDisk, ramBytes: usedMemory } + } finally { + try { + await rclient.disconnect() + } catch {} + } +} + +async function upsertStat({ statKey, day, values, generatedAt }) { + await InstanceStat.updateOne( + { statKey, day }, + { $set: { values, generatedAt, day } }, + { upsert: true } + ) +} + +async function collectForDay({ day }) { + const now = new Date() + const oneDayAgo = new Date(now.getTime() - 24 * 60 * 60 * 1000) + const db = mongoose.connection.db + const segmentationEnabled = isUserSegmentationEnabled() + + const projects = db.collection('projects') + const users = db.collection('users') + const docs = db.collection('docs') + + const activeProjects = await projects.countDocuments({ + lastUpdated: { $gte: oneDayAgo }, + }) + + let activeUsersValues + let userCountValues + + if (segmentationEnabled) { + const guestMatch = guestUserFilter() + + const activeGuestUsers = await users.countDocuments({ + ...guestMatch, + lastActive: { $gte: oneDayAgo }, + }) + const activeUsersTotal = await users.countDocuments({ + lastActive: { $gte: oneDayAgo }, + }) + const activeStandardUsers = activeUsersTotal - activeGuestUsers + + const guestUsers = await users.countDocuments(guestMatch) + const totalUsers = await users.countDocuments() + const standardUsers = totalUsers - guestUsers + + // [standard, guest] — matches chart labels in instance-stats frontend config + activeUsersValues = [activeStandardUsers, activeGuestUsers] + userCountValues = [standardUsers, guestUsers] + } else { + const activeUsers = await users.countDocuments({ + lastActive: { $gte: oneDayAgo }, + }) + const totalUsers = await users.countDocuments() + + activeUsersValues = [activeUsers] + userCountValues = [totalUsers] + } + + const projectCount = await projects.countDocuments() + const fileCount = await docs.countDocuments() + + // Mongo storage: approximate with dbStats.totalSize (bytes). + // Note that this will be lower that when doing `du` for the mongo + // data folder, because that includes journaling files that we can not + // measure directly from mongo. + let mongodbStorageBytes = 0 + try { + const stats = await db.stats() + mongodbStorageBytes = stats?.totalSize ?? 0 + } catch (err) { + logger.error({ err }, 'Failed to query Mongo dbStats') + } + + const overleafStorageBytes = await duBytes('/var/lib/overleaf') + const { diskBytes: redisDiskBytes, ramBytes: redisRamBytes } = + await getRedisMemoryBytes() + + const generatedAt = new Date() + + await upsertStat({ statKey: 'active_projects', day, values: [activeProjects], generatedAt }) + await upsertStat({ statKey: 'active_users', day, values: activeUsersValues, generatedAt }) + await upsertStat({ statKey: 'user_count', day, values: userCountValues, generatedAt }) + await upsertStat({ statKey: 'project_count', day, values: [projectCount], generatedAt }) + await upsertStat({ statKey: 'file_count', day, values: [fileCount], generatedAt }) + await upsertStat({ statKey: 'mongodb_storage', day, values: [mongodbStorageBytes], generatedAt }) + await upsertStat({ statKey: 'overleaf_storage', day, values: [overleafStorageBytes], generatedAt }) + await upsertStat({ statKey: 'redis_storage', day, values: [redisDiskBytes, redisRamBytes], generatedAt }) +} + +async function main() { + if (!Settings.instanceStats?.enabled) { + logger.info({ enabled: false }, 'Instance stats collection disabled') + return + } + + await mongoose.connectionPromise + + const now = new Date() + const day = toUtcMidnight(now) + + logger.info({ day }, 'Collecting instance statistics') + await collectForDay({ day }) + logger.info({ day }, 'Collected instance statistics') +} + +main() + .then(() => process.exit(0)) + .catch(err => { + logger.err({ err }, 'Failed collecting instance statistics') + process.exit(1) + }) + diff --git a/services/web/modules/instance-stats/app/src/models/InstanceStat.mjs b/services/web/modules/instance-stats/app/src/models/InstanceStat.mjs new file mode 100644 index 0000000000..1bcea263a9 --- /dev/null +++ b/services/web/modules/instance-stats/app/src/models/InstanceStat.mjs @@ -0,0 +1,27 @@ +import mongoose from '../../../../../app/src/infrastructure/Mongoose.mjs' + +const { Schema } = mongoose + +const InstanceStatSchema = new Schema( + { + // Metric identifier + statKey: { type: String, required: true, index: true }, + // Day bucket (stored as a Date; comparisons should use UTC midnight) + day: { type: Date, required: true, index: true }, + // Values for the stacked bar series. + // For 1-series metrics: [y1] + // For 2-series metrics: [y1, y2] + values: { type: [Number], required: true, default: [] }, + generatedAt: { type: Date, required: true, default: () => new Date() }, + }, + { minimize: false } +) + +InstanceStatSchema.index({ statKey: 1, day: 1 }, { unique: true }) + +export const InstanceStat = mongoose.model( + 'InstanceStat', + InstanceStatSchema, + 'instanceStats' +) + diff --git a/services/web/modules/instance-stats/app/src/seedInstanceStats.mjs b/services/web/modules/instance-stats/app/src/seedInstanceStats.mjs new file mode 100644 index 0000000000..b548648ffc --- /dev/null +++ b/services/web/modules/instance-stats/app/src/seedInstanceStats.mjs @@ -0,0 +1,96 @@ +import logger from '@overleaf/logger' +import Settings from '@overleaf/settings' +import mongoose from '../../../../app/src/infrastructure/Mongoose.mjs' +import { InstanceStat } from './models/InstanceStat.mjs' + +const SEGMENTED_STAT_DEFS = { + active_projects: { base: [30], noise: [12], min: [0] }, + active_users: { base: [120, 80], noise: [25, 20], min: [0, 0] }, // [standard, guest] + user_count: { base: [900, 500], noise: [8, 6], min: [0, 0] }, // [standard, guest] + project_count: { base: [1600], noise: [6], min: [0] }, + file_count: { base: [12000], noise: [80], min: [0] }, + mongodb_storage: { base: [9.5e9], noise: [4.5e7], min: [0] }, // bytes + overleaf_storage: { base: [2.8e10], noise: [8.0e7], min: [0] }, // bytes + redis_storage: { base: [4.2e8, 1.9e8], noise: [1.2e7, 7e6], min: [0, 0] }, // bytes +} + +function toUtcMidnight(date) { + return new Date( + Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()) + ) +} + +function randomInRange(min, max) { + return Math.random() * (max - min) + min +} + +function generateValues(def, dayOffset) { + const season = Math.sin(dayOffset / 14) // gentle periodicity + return def.base.map((base, i) => { + const trend = base * (dayOffset * 0.0015) + const jitter = randomInRange(-def.noise[i], def.noise[i]) + const seasonal = season * def.noise[i] * 0.6 + return Math.max(def.min[i], Math.round(base + trend + jitter + seasonal)) + }) +} + +function getStatDefsForCurrentSegmentation() { + const segmentation = Settings.instanceStats?.userSegmentation + const segmentationEnabled = Boolean(segmentation?.enabled) + + if (segmentationEnabled) { + return SEGMENTED_STAT_DEFS + } + + return { + ...SEGMENTED_STAT_DEFS, + active_users: { base: [200], noise: [35], min: [0] }, + user_count: { base: [1400], noise: [10], min: [0] }, + } +} + +async function main() { + await mongoose.connectionPromise + + const days = Math.max(parseInt(process.env.INSTANCE_STATS_SEED_DAYS || '400', 10), 1) + // Default is true so we remove old seeded/real docs unless explicitly disabled. + const clear = process.env.INSTANCE_STATS_SEED_CLEAR !== 'false' + const now = new Date() + const statDefs = getStatDefsForCurrentSegmentation() + + if (clear) { + logger.info('Clearing existing instanceStats collection before seeding') + await InstanceStat.deleteMany({}) + } + + const ops = [] + for (let d = days - 1; d >= 0; d--) { + const day = toUtcMidnight(new Date(now.getTime() - d * 24 * 60 * 60 * 1000)) + for (const [statKey, def] of Object.entries(statDefs)) { + ops.push({ + updateOne: { + filter: { statKey, day }, + update: { + $set: { + values: generateValues(def, days - d), + generatedAt: new Date(), + }, + }, + upsert: true, + }, + }) + } + } + + logger.info({ days, docs: ops.length }, 'Seeding fake instance statistics') + await InstanceStat.bulkWrite(ops, { ordered: false }) + logger.info('Seed completed') +} + +main() + .then(() => process.exit(0)) + .catch(err => { + logger.err({ err }, 'Failed to seed instance statistics') + process.exit(1) + }) + diff --git a/services/web/modules/instance-stats/app/src/verifyInstanceStats.mjs b/services/web/modules/instance-stats/app/src/verifyInstanceStats.mjs new file mode 100644 index 0000000000..26eccf8625 --- /dev/null +++ b/services/web/modules/instance-stats/app/src/verifyInstanceStats.mjs @@ -0,0 +1,90 @@ +import logger from '@overleaf/logger' +import Settings from '@overleaf/settings' +import mongoose from '../../../../app/src/infrastructure/Mongoose.mjs' +import { InstanceStat } from './models/InstanceStat.mjs' + +const WINDOWS = { + month: 30, + '6m': 180, + year: 365, +} + +function toUtcMidnight(date) { + return new Date( + Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()) + ) +} + +function computeCutoff(window) { + const retentionDays = Settings.instanceStats?.retentionDays || 365 + const now = new Date() + const cutoffDays = window === 'all' ? retentionDays : WINDOWS[window] || 30 + return toUtcMidnight(new Date(now.getTime() - cutoffDays * 24 * 60 * 60 * 1000)) +} + +async function main() { + await mongoose.connectionPromise + const segmentationEnabled = Boolean( + Settings.instanceStats?.userSegmentation?.enabled + ) + + const statKeys = [ + 'active_projects', + 'active_users', + 'user_count', + 'project_count', + 'file_count', + 'mongodb_storage', + 'overleaf_storage', + 'redis_storage', + ] + + const windowsToCheck = ['month', '6m', 'year', 'all'] + + for (const statKey of statKeys) { + for (const window of windowsToCheck) { + const cutoff = computeCutoff(window) + const docs = await InstanceStat.find( + { statKey, day: { $gte: cutoff } }, + { _id: 0, values: 1 } + ) + .sort({ day: 1 }) + .limit(10) + .lean() + + if (docs.length === 0) { + logger.warn({ statKey, window }, 'No points in window (may be expected initially)') + continue + } + + const expectTwo = + statKey === 'redis_storage' || + (segmentationEnabled && + (statKey === 'active_users' || + statKey === 'user_count')) + for (const [i, d] of docs.entries()) { + const len = d.values?.length || 0 + if (expectTwo && len < 2) { + throw new Error( + `${statKey} expected 2-series values, but got len=${len} (example index ${i}, window=${window})` + ) + } + if (!expectTwo && len < 1) { + throw new Error( + `${statKey} expected 1-series values, but got len=${len} (example index ${i}, window=${window})` + ) + } + } + } + } + + logger.info('Instance stats verification completed') +} + +main() + .then(() => process.exit(0)) + .catch(err => { + logger.err({ err }, 'Instance stats verification failed') + process.exit(1) + }) + diff --git a/services/web/modules/instance-stats/app/views/instance-stats.pug b/services/web/modules/instance-stats/app/views/instance-stats.pug new file mode 100644 index 0000000000..cf00fcc405 --- /dev/null +++ b/services/web/modules/instance-stats/app/views/instance-stats.pug @@ -0,0 +1,57 @@ +extends ../../../../app/views/layout-marketing + +include ../../../../app/views/_mixins/bookmarkable_tabset.pug + +block entrypointVar + - entrypoint = 'modules/instance-stats/pages/instance-stats' + +block content + #main-content.content.content-alt + .container + .row + .col-sm-12 + .card#instance-stats-root + .card-body + .page-header + h1 Instance statistics + + .row.justify-content-end + .col-2 + label.form-label(for='timeFilter') Show: + select#timeFilter.form-select + option(value='month' selected) Last 1 month + option(value='6m') Last 6 months + option(value='year') Last 1 year + option(value='all') All + + hr + + // Charts (organized into tabs) + ol-tabs.ol-tabs(data-ol-bookmarkable-tabset) + .nav-tabs-container + ul.nav.nav-tabs.align-left(role='tablist') + +bookmarkable-tabset-header('user', 'User', true) + +bookmarkable-tabset-header('project', 'Project') + +bookmarkable-tabset-header('storage', 'Storage') + + .tab-content + .tab-pane.active(role='tabpanel' id='user') + .row(style='height:400px') + .col-md-6#active-users + .col-md-6#user-count + + .tab-pane(role='tabpanel' id='project') + .row(style='height:400px') + .col-md-4#active-projects + .col-md-4#project-count + .col-md-4#file-count + + .tab-pane(role='tabpanel' id='storage') + .row(style='height:400px') + .col-md-4#mongodb-storage + .col-md-4#overleaf-storage + .col-md-4#redis-storage + +block prepend foot-scripts + script(src='https://cdn.plot.ly/plotly-3.0.1.min.js' charset='utf-8' nonce=scriptNonce defer=deferScripts) + diff --git a/services/web/modules/instance-stats/frontend/js/features/instance-stats/api.ts b/services/web/modules/instance-stats/frontend/js/features/instance-stats/api.ts new file mode 100644 index 0000000000..f913dae51b --- /dev/null +++ b/services/web/modules/instance-stats/frontend/js/features/instance-stats/api.ts @@ -0,0 +1,29 @@ +import type { SeriesResponse, StatKey, WindowKey } from './types' + +function isSeriesResponse(value: unknown): value is SeriesResponse { + if (!value || typeof value !== 'object') return false + const asRecord = value as Record + return ( + typeof asRecord.metric === 'string' && + typeof asRecord.window === 'string' && + Array.isArray(asRecord.points) + ) +} + +export async function fetchSeries( + metric: StatKey, + window: WindowKey +): Promise { + const url = `/admin/instance-stats/api/series?metric=${encodeURIComponent(metric)}&window=${encodeURIComponent(window)}` + const res = await fetch(url, { headers: { Accept: 'application/json' } }) + if (!res.ok) { + const message = await res.text().catch(() => '') + throw new Error(`Failed to fetch ${metric} (${res.status}): ${message}`) + } + const json = await res.json() + if (!isSeriesResponse(json)) { + throw new Error(`Invalid response shape for metric ${metric}`) + } + return json +} + diff --git a/services/web/modules/instance-stats/frontend/js/features/instance-stats/config.ts b/services/web/modules/instance-stats/frontend/js/features/instance-stats/config.ts new file mode 100644 index 0000000000..f1df689b1d --- /dev/null +++ b/services/web/modules/instance-stats/frontend/js/features/instance-stats/config.ts @@ -0,0 +1,98 @@ +import type { StatConfig, WindowKey } from './types' + +const passThrough = (value: number) => value +const bytesToMB = (bytes: number) => bytes / (1024 * 1024) + +export const WINDOW_OPTIONS: Array<{ value: WindowKey; label: string }> = [ + { value: 'month', label: 'Last 1 month' }, + { value: '6m', label: 'Last 6 months' }, + { value: 'year', label: 'Last 1 year' }, + { value: 'all', label: 'All' }, +] + +export const STAT_CONFIG: StatConfig[] = [ + { + id: 'active-projects', + metric: 'active_projects', + tabId: 'project', + title: 'Active projects', + seriesCount: 1, + transform: passThrough, + colors: { y1: '#acbce2' }, + labels: {}, + }, + { + id: 'active-users', + metric: 'active_users', + tabId: 'user', + title: 'Active users', + seriesCount: 2, + transform: passThrough, + colors: { y1: '#d98c8c', y2: '#EDC9C9' }, + labels: { y1: 'Standard', y2: 'Guest' }, + }, + { + id: 'user-count', + metric: 'user_count', + tabId: 'user', + title: 'Users', + seriesCount: 2, + transform: passThrough, + colors: { y1: '#d98c8c', y2: '#EDC9C9' }, + labels: { y1: 'Standard', y2: 'Guest' }, + }, + { + id: 'project-count', + metric: 'project_count', + tabId: 'project', + title: 'Projects', + seriesCount: 1, + transform: passThrough, + colors: { y1: '#acbce2' }, + labels: {}, + }, + { + id: 'file-count', + metric: 'file_count', + tabId: 'project', + title: 'Files', + seriesCount: 1, + transform: passThrough, + colors: { y1: '#767676' }, + labels: {}, + }, + { + id: 'mongodb-storage', + metric: 'mongodb_storage', + tabId: 'storage', + title: 'Mongodb storage', + ylabel: 'MB', + seriesCount: 1, + transform: bytesToMB, + colors: { y1: '#767676' }, + labels: {}, + }, + { + id: 'overleaf-storage', + metric: 'overleaf_storage', + tabId: 'storage', + title: 'Overleaf storage', + ylabel: 'MB', + seriesCount: 1, + transform: bytesToMB, + colors: { y1: '#767676' }, + labels: {}, + }, + { + id: 'redis-storage', + metric: 'redis_storage', + tabId: 'storage', + title: 'Redis storage', + ylabel: 'MB', + seriesCount: 2, + transform: bytesToMB, + colors: { y1: '#767676', y2: '#ADADAD' }, + labels: { y1: 'Disk', y2: 'RAM' }, + }, +] + diff --git a/services/web/modules/instance-stats/frontend/js/features/instance-stats/render.ts b/services/web/modules/instance-stats/frontend/js/features/instance-stats/render.ts new file mode 100644 index 0000000000..5c835a42b5 --- /dev/null +++ b/services/web/modules/instance-stats/frontend/js/features/instance-stats/render.ts @@ -0,0 +1,119 @@ +import type { SeriesPoint, StatConfig } from './types' + +declare global { + interface Window { + Plotly?: { + newPlot: ( + id: string, + traces: unknown[], + layout: Record, + config: Record + ) => Promise + animate?: ( + id: string, + frameOrGroupNameOrAnimationDef: Record, + animationOpts: Record + ) => Promise + } + } +} + +function getSeriesCount(stat: StatConfig, points: SeriesPoint[]) { + const maxValuesLength = points.reduce( + (max, point) => Math.max(max, point.values.length), + 0 + ) + if (maxValuesLength >= 2) return 2 + if (maxValuesLength === 1) return 1 + return stat.seriesCount +} + +function buildTraces(stat: StatConfig, points: SeriesPoint[]) { + const x = points.map(point => point.day) + const seriesCount = getSeriesCount(stat, points) + const indexes = seriesCount === 2 ? [0, 1] : [0] + return indexes.map(index => { + const key = index === 0 ? 'y1' : 'y2' + const y = points.map(point => stat.transform(point.values[index] ?? 0)) + const label = seriesCount === 2 ? stat.labels[key] : undefined + const hovertemplate = + label != null + ? `%{x|%d %b %Y}
${label}: %{y}` + : '%{x|%d %b %Y}
%{y}' + return { + x, + y, + type: 'bar', + name: label, + showlegend: seriesCount === 2, + marker: { color: stat.colors[key] }, + hovertemplate, + } + }) +} + +function buildLayout(stat: StatConfig, points: SeriesPoint[]) { + const seriesCount = getSeriesCount(stat, points) + const indexes = seriesCount === 2 ? [0, 1] : [0] + const totals = points.map(point => + indexes.reduce((sum, index) => sum + (point.values[index] ?? 0), 0) + ) + const ymax = totals.length > 0 ? Math.max(...totals.map(stat.transform)) : 0 + return { + title: { text: stat.title }, + font: { family: 'Raleway, sans-serif' }, + bargap: 0.1, + xaxis: { + title: 'Time', + type: 'date', + tickmode: 'auto', + nticks: 8, + ticklabelmode: 'period', + hoverformat: '%d %b %Y', + automargin: true, + tickformatstops: [ + { dtickrange: [null, 24 * 60 * 60 * 1000], value: '%d %b' }, + { dtickrange: [24 * 60 * 60 * 1000, 'M1'], value: '%d %b' }, + { dtickrange: ['M1', 'M12'], value: '%b %Y' }, + { dtickrange: ['M12', null], value: '%Y' }, + ], + }, + yaxis: { + title: { text: stat.ylabel || '' }, + range: [0, ymax], + }, + barmode: 'stack', + } +} + +export async function renderChart(stat: StatConfig, points: SeriesPoint[]) { + if (!window.Plotly) { + throw new Error('Plotly is not available on window') + } + const traces = buildTraces(stat, points) + const layout = buildLayout(stat, points) + const initialTraces = traces.map(trace => ({ + ...trace, + y: (trace.y as number[]).map(() => 0), + })) + + await window.Plotly.newPlot(stat.id, initialTraces, layout, { + displayModeBar: false, + }) + + if (typeof window.Plotly.animate === 'function') { + return window.Plotly.animate( + stat.id, + { + data: traces.map(trace => ({ y: trace.y })), + }, + { + transition: { duration: 450, easing: 'cubic-in-out' }, + frame: { duration: 450, redraw: false }, + } + ) + } + + return undefined +} + diff --git a/services/web/modules/instance-stats/frontend/js/features/instance-stats/types.ts b/services/web/modules/instance-stats/frontend/js/features/instance-stats/types.ts new file mode 100644 index 0000000000..289fa5863e --- /dev/null +++ b/services/web/modules/instance-stats/frontend/js/features/instance-stats/types.ts @@ -0,0 +1,37 @@ +export type WindowKey = 'month' | '6m' | 'year' | 'all' + +export type TabId = 'user' | 'project' | 'storage' + +export type StatKey = + | 'active_projects' + | 'active_users' + | 'user_count' + | 'project_count' + | 'file_count' + | 'mongodb_storage' + | 'overleaf_storage' + | 'redis_storage' + +export interface SeriesPoint { + day: number + values: number[] +} + +export interface SeriesResponse { + metric: StatKey + window: WindowKey + points: SeriesPoint[] +} + +export interface StatConfig { + id: string + metric: StatKey + tabId: TabId + title: string + ylabel?: string + seriesCount: 1 | 2 + transform: (value: number) => number + colors: { y1: string; y2?: string } + labels: { y1?: string; y2?: string } +} + diff --git a/services/web/modules/instance-stats/frontend/js/pages/instance-stats.ts b/services/web/modules/instance-stats/frontend/js/pages/instance-stats.ts new file mode 100644 index 0000000000..d011480b2a --- /dev/null +++ b/services/web/modules/instance-stats/frontend/js/pages/instance-stats.ts @@ -0,0 +1,58 @@ +import { fetchSeries } from '../features/instance-stats/api' +import { STAT_CONFIG } from '../features/instance-stats/config' +import { renderChart } from '../features/instance-stats/render' +import type { TabId, WindowKey } from '../features/instance-stats/types' + +const WINDOW_VALUES: WindowKey[] = ['month', '6m', 'year', 'all'] + +function getWindowValue(select: HTMLSelectElement): WindowKey { + const value = select.value as WindowKey + if (WINDOW_VALUES.includes(value)) return value + return 'month' +} + +function getTabIdFromHash(): TabId { + const hash = window.location.hash?.replace(/^#/, '') || '' + if (hash === 'user' || hash === 'project' || hash === 'storage') return hash + return 'user' +} + +async function initInstanceStatsPage() { + const root = document.getElementById('instance-stats-root') + const select = document.getElementById('timeFilter') as HTMLSelectElement | null + if (!root || !select) return + + let requestId = 0 + + const renderWindow = async (window: WindowKey, tabId: TabId) => { + const myRequestId = ++requestId + const jobs = STAT_CONFIG.filter(stat => stat.tabId === tabId).map(async stat => { + const series = await fetchSeries(stat.metric, window) + if (myRequestId !== requestId) return + await renderChart(stat, series.points) + }) + await Promise.allSettled(jobs) + } + + const renderActiveTab = () => { + const tabId = getTabIdFromHash() + // Delay a tick so the bootstrap tab pane is visible before Plotly sizes + // the chart container. + setTimeout(() => void renderWindow(getWindowValue(select), tabId), 0) + } + + select.addEventListener('change', () => { + renderActiveTab() + }) + + window.addEventListener('hashchange', () => { + // Bookmarkable tabs update the hash first; wait a tick before rerender. + renderActiveTab() + }) + + // Initial render based on current hash. + renderActiveTab() +} + +void initInstanceStatsPage() + diff --git a/services/web/modules/instance-stats/index.mjs b/services/web/modules/instance-stats/index.mjs new file mode 100644 index 0000000000..02c9353e11 --- /dev/null +++ b/services/web/modules/instance-stats/index.mjs @@ -0,0 +1,25 @@ +import logger from '@overleaf/logger' +import Settings from '@overleaf/settings' + +let InstanceStatsModule = {} + +if (process.env.INSTANCE_STATS_ENABLED === 'true') { + logger.debug({}, 'Enabling instance-stats module') + + Settings.instanceStats = { + enabled: true, + retentionDays: parseInt(process.env.INSTANCE_STATS_RETENTION_DAYS, 10) || 365, + userSegmentation: { + // When enabled, active_users and user_count are split into [standard, guest] + // using users whose adminRoles array contains "guest-user". + enabled: process.env.INSTANCE_STATS_USER_SEGMENTATION_ENABLED === 'true', + }, + } + + const { default: InstanceStatsRouter } = await import('./app/src/InstanceStatsRouter.mjs') + InstanceStatsModule = { + router: InstanceStatsRouter, + } +} + +export default InstanceStatsModule