mirror of
https://github.com/yu-i-i/overleaf-cep.git
synced 2026-11-15 06:33:55 +01:00
Overleaf instance statistics (#178)
This commit is contained in:
Isaac Alonso Asensio
committed by
yu-i-i
yu-i-i
parent
21fbf6c5dc
commit
b8a622a390
@@ -68,6 +68,8 @@ ADD server-ce/config/crontab-history /etc/cron.d/crontab-history
|
||||
RUN chmod 600 /etc/cron.d/crontab-history
|
||||
ADD server-ce/config/crontab-deletion /etc/cron.d/crontab-deletion
|
||||
RUN chmod 600 /etc/cron.d/crontab-deletion
|
||||
ADD server-ce/config/crontab-instance-stats /etc/cron.d/crontab-instance-stats
|
||||
RUN chmod 600 /etc/cron.d/crontab-instance-stats
|
||||
|
||||
# Copy Phusion Image startup and shutdown scripts to their locations
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
50 * * * * root bash /overleaf/cron/collect-instance-stats.sh >> /var/log/overleaf/cron-instance-stats.log 2>&1
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -eu
|
||||
|
||||
echo "--------------------------"
|
||||
echo "Collect instance statistics"
|
||||
echo "--------------------------"
|
||||
date
|
||||
|
||||
source /etc/container_environment.sh
|
||||
source /etc/overleaf/env.sh
|
||||
|
||||
# `@overleaf/settings` resolves `config/settings.defaults.*` relative to
|
||||
# `process.cwd()`, so ensure we're running from `services/web`.
|
||||
cd /overleaf/services/web
|
||||
|
||||
node /overleaf/services/web/modules/instance-stats/app/src/collectInstanceStats.mjs
|
||||
|
||||
@@ -15,6 +15,7 @@ block append meta
|
||||
- const canDisplaySplitTestMenu = hasFeature('saas') && hasNonAdminDomainCapability('view-split-test')
|
||||
- const canDisplaySurveyMenu = hasFeature('saas') && canDisplayAdminMenu && hasAdminCapability('manage-survey', false)
|
||||
- const canDisplayScriptLogMenu = hasFeature('saas') && hasAdminCapability('view-script-log', false) && canDisplayAdminMenu
|
||||
- const canDisplayInstanceStats = settings.instanceStats && settings.instanceStats.enabled
|
||||
- const enableUpgradeButton = projectDashboardReact && usersBestSubscription && (usersBestSubscription.type === 'free' || usersBestSubscription.type === 'standalone-ai-add-on')
|
||||
- const showSignUpLink = hasFeature('registration-page')
|
||||
|
||||
@@ -31,6 +32,7 @@ block append meta
|
||||
canDisplaySplitTestMenu,
|
||||
canDisplaySurveyMenu,
|
||||
canDisplayScriptLogMenu,
|
||||
canDisplayInstanceStats,
|
||||
enableUpgradeButton,
|
||||
suppressNavbarRight: !!suppressNavbarRight,
|
||||
suppressNavContentLinks: !!suppressNavContentLinks,
|
||||
|
||||
@@ -68,6 +68,8 @@ nav.navbar.navbar-default.navbar-main.navbar-expand-lg(
|
||||
+dropdown-menu-link-item(href='/admin/user') Manage Users
|
||||
if canDisplayProjectUrlLookup
|
||||
+dropdown-menu-link-item(href='/admin/project') Manage Projects
|
||||
if settings.instanceStats && settings.instanceStats.enabled
|
||||
+dropdown-menu-link-item(href='/admin/instance-stats') Instance Statistics
|
||||
if canDisplayAdminRedirect
|
||||
+dropdown-menu-link-item(href=settings.adminUrl) Switch to Admin
|
||||
if canDisplaySplitTestMenu
|
||||
|
||||
@@ -1256,6 +1256,7 @@ module.exports = {
|
||||
'authentication/oidc',
|
||||
'admin-tools', // import after authentication
|
||||
'registration-page', // import after authentication
|
||||
'instance-stats',
|
||||
'template-gallery',
|
||||
'git-bridge',
|
||||
'git-sync/github',
|
||||
|
||||
@@ -10,6 +10,7 @@ export default function AdminMenu({
|
||||
canDisplaySplitTestMenu,
|
||||
canDisplaySurveyMenu,
|
||||
canDisplayScriptLogMenu,
|
||||
canDisplayInstanceStats,
|
||||
adminUrl,
|
||||
}: Pick<
|
||||
DefaultNavbarMetadata,
|
||||
@@ -19,6 +20,7 @@ export default function AdminMenu({
|
||||
| 'canDisplaySplitTestMenu'
|
||||
| 'canDisplaySurveyMenu'
|
||||
| 'canDisplayScriptLogMenu'
|
||||
| 'canDisplayInstanceStats'
|
||||
| 'adminUrl'
|
||||
>) {
|
||||
const sendProjectListMB = useSendProjectListMB()
|
||||
@@ -68,6 +70,11 @@ export default function AdminMenu({
|
||||
View Script Logs
|
||||
</NavDropdownLinkItem>
|
||||
) : null}
|
||||
{canDisplayInstanceStats ? (
|
||||
<NavDropdownLinkItem href="/admin/instance-stats">
|
||||
Instance Statistics
|
||||
</NavDropdownLinkItem>
|
||||
) : null}
|
||||
</NavDropdownMenu>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -30,6 +30,7 @@ function DefaultNavbar(
|
||||
canDisplaySplitTestMenu,
|
||||
canDisplaySurveyMenu,
|
||||
canDisplayScriptLogMenu,
|
||||
canDisplayInstanceStats,
|
||||
enableUpgradeButton,
|
||||
suppressNavbarRight,
|
||||
suppressNavContentLinks,
|
||||
@@ -117,6 +118,7 @@ function DefaultNavbar(
|
||||
canDisplaySplitTestMenu={canDisplaySplitTestMenu}
|
||||
canDisplaySurveyMenu={canDisplaySurveyMenu}
|
||||
canDisplayScriptLogMenu={canDisplayScriptLogMenu}
|
||||
canDisplayInstanceStats={canDisplayInstanceStats}
|
||||
adminUrl={adminUrl}
|
||||
/>
|
||||
) : null}
|
||||
|
||||
@@ -13,6 +13,7 @@ export type DefaultNavbarMetadata = {
|
||||
canDisplaySplitTestMenu: boolean
|
||||
canDisplaySurveyMenu: boolean
|
||||
canDisplayScriptLogMenu: boolean
|
||||
canDisplayInstanceStats: boolean
|
||||
enableUpgradeButton: boolean
|
||||
suppressNavbarRight: boolean
|
||||
suppressNavContentLinks: boolean
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
import Path from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import logger from '@overleaf/logger'
|
||||
import Settings from '@overleaf/settings'
|
||||
|
||||
import { InstanceStat } from './models/InstanceStat.mjs'
|
||||
|
||||
const __dirname = Path.dirname(fileURLToPath(import.meta.url))
|
||||
|
||||
const STAT_KEYS = [
|
||||
'active_projects',
|
||||
'active_users',
|
||||
'user_count',
|
||||
'project_count',
|
||||
'file_count',
|
||||
'mongodb_storage',
|
||||
'overleaf_storage',
|
||||
'redis_storage',
|
||||
]
|
||||
|
||||
const WINDOWS = {
|
||||
month: 30,
|
||||
'6m': 180,
|
||||
year: 365,
|
||||
}
|
||||
|
||||
function toUtcMidnight(date) {
|
||||
return new Date(
|
||||
Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate())
|
||||
)
|
||||
}
|
||||
|
||||
function validateWindow(window) {
|
||||
if (window === 'all') return true
|
||||
return Object.hasOwn(WINDOWS, window)
|
||||
}
|
||||
|
||||
async function page(req, res, next) {
|
||||
try {
|
||||
res.render(Path.resolve(__dirname, '../views/instance-stats'), {
|
||||
title: 'Instance Statistics',
|
||||
initialWindow: 'month',
|
||||
})
|
||||
} catch (err) {
|
||||
next(err)
|
||||
}
|
||||
}
|
||||
|
||||
async function series(req, res, next) {
|
||||
const metric = req.query.metric
|
||||
const window = req.query.window || 'month'
|
||||
|
||||
try {
|
||||
if (typeof metric !== 'string' || !STAT_KEYS.includes(metric)) {
|
||||
return res.status(400).json({ message: 'Invalid metric' })
|
||||
}
|
||||
|
||||
if (typeof window !== 'string' || !validateWindow(window)) {
|
||||
return res.status(400).json({ message: 'Invalid window' })
|
||||
}
|
||||
|
||||
const retentionDays = Settings.instanceStats?.retentionDays || 365
|
||||
const now = new Date()
|
||||
const cutoffDays = window === 'all' ? retentionDays : WINDOWS[window]
|
||||
|
||||
const cutoffDate = toUtcMidnight(
|
||||
new Date(now.getTime() - cutoffDays * 24 * 60 * 60 * 1000)
|
||||
)
|
||||
|
||||
const docs = await InstanceStat.find(
|
||||
{ statKey: metric, day: { $gte: cutoffDate } },
|
||||
{ _id: 0, day: 1, values: 1 }
|
||||
)
|
||||
.sort({ day: 1 })
|
||||
.lean()
|
||||
|
||||
return res.status(200).json({
|
||||
metric,
|
||||
window,
|
||||
points: docs.map(d => ({
|
||||
day: d.day.getTime(),
|
||||
values: d.values,
|
||||
})),
|
||||
})
|
||||
} catch (err) {
|
||||
logger.error({ err, metric, window }, 'Failed to fetch instance stats series')
|
||||
next(err)
|
||||
}
|
||||
}
|
||||
|
||||
export default {
|
||||
page,
|
||||
series,
|
||||
}
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
import logger from '@overleaf/logger'
|
||||
import AuthorizationMiddleware from '../../../../app/src/Features/Authorization/AuthorizationMiddleware.mjs'
|
||||
import InstanceStatsController from './InstanceStatsController.mjs'
|
||||
import Settings from '@overleaf/settings'
|
||||
|
||||
export default {
|
||||
apply(webRouter) {
|
||||
if (!Settings.instanceStats?.enabled) {
|
||||
logger.info({ enabled: false }, 'Instance stats module disabled')
|
||||
return
|
||||
}
|
||||
|
||||
logger.debug({}, 'Init instance-stats router')
|
||||
|
||||
webRouter.get(
|
||||
'/admin/instance-stats',
|
||||
AuthorizationMiddleware.ensureUserIsSiteAdmin,
|
||||
InstanceStatsController.page
|
||||
)
|
||||
|
||||
webRouter.get(
|
||||
'/admin/instance-stats/api/series',
|
||||
AuthorizationMiddleware.ensureUserIsSiteAdmin,
|
||||
InstanceStatsController.series
|
||||
)
|
||||
},
|
||||
}
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
import logger from '@overleaf/logger'
|
||||
import mongoose from '../../../../app/src/infrastructure/Mongoose.mjs'
|
||||
import { InstanceStat } from './models/InstanceStat.mjs'
|
||||
|
||||
function parseConfirmArg() {
|
||||
const confirmArg = process.argv.find(arg => arg.startsWith('--confirm='))
|
||||
return confirmArg ? confirmArg.slice('--confirm='.length) : undefined
|
||||
}
|
||||
|
||||
async function main() {
|
||||
await mongoose.connectionPromise
|
||||
|
||||
const confirmation =
|
||||
process.env.INSTANCE_STATS_CLEAR_CONFIRM || parseConfirmArg()
|
||||
if (confirmation !== 'YES') {
|
||||
throw new Error(
|
||||
"Refusing to clear stats. Set INSTANCE_STATS_CLEAR_CONFIRM=YES or pass --confirm=YES."
|
||||
)
|
||||
}
|
||||
|
||||
const before = await InstanceStat.countDocuments({})
|
||||
const result = await InstanceStat.deleteMany({})
|
||||
|
||||
logger.info(
|
||||
{ before, deletedCount: result.deletedCount || 0 },
|
||||
'Cleared instance statistics'
|
||||
)
|
||||
}
|
||||
|
||||
main()
|
||||
.then(() => process.exit(0))
|
||||
.catch(err => {
|
||||
logger.err({ err }, 'Failed clearing instance statistics')
|
||||
process.exit(1)
|
||||
})
|
||||
@@ -0,0 +1,175 @@
|
||||
import logger from '@overleaf/logger'
|
||||
import Settings from '@overleaf/settings'
|
||||
import mongoose from '../../../../app/src/infrastructure/Mongoose.mjs'
|
||||
import { InstanceStat } from './models/InstanceStat.mjs'
|
||||
import RedisWrapper from '../../../../app/src/infrastructure/RedisWrapper.mjs'
|
||||
import { execFile } from 'node:child_process'
|
||||
import { promisify } from 'node:util'
|
||||
|
||||
const execFileAsync = promisify(execFile)
|
||||
|
||||
/** Role string stored inside User.adminRoles (array of strings). */
|
||||
const GUEST_ADMIN_ROLE = 'guest-user'
|
||||
|
||||
/** Mongo filter: user is a guest if their adminRoles list includes this role. */
|
||||
function guestUserFilter() {
|
||||
return { adminRoles: { $in: [GUEST_ADMIN_ROLE] } }
|
||||
}
|
||||
|
||||
function isUserSegmentationEnabled() {
|
||||
return Boolean(Settings.instanceStats?.userSegmentation?.enabled)
|
||||
}
|
||||
|
||||
function toUtcMidnight(date) {
|
||||
return new Date(
|
||||
Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate())
|
||||
)
|
||||
}
|
||||
|
||||
function parseFirstInt(output) {
|
||||
const match = output?.match(/(\d+)/)
|
||||
return match ? parseInt(match[1], 10) : 0
|
||||
}
|
||||
|
||||
async function duBytes(path) {
|
||||
// Prefer exact du output in bytes. Fall back to 0 on failure.
|
||||
try {
|
||||
const { stdout } = await execFileAsync('du', ['-b0s', path])
|
||||
return parseFirstInt(stdout)
|
||||
} catch (err) {
|
||||
logger.error({ err, path }, 'Failed to compute du bytes')
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
function parseRedisInfoInt(info, key) {
|
||||
const re = new RegExp(`${key}:(\\d+)`)
|
||||
const match = info?.match(re)
|
||||
return match ? parseInt(match[1], 10) : 0
|
||||
}
|
||||
|
||||
async function getRedisMemoryBytes() {
|
||||
const rclient = RedisWrapper.client('instance_stats')
|
||||
try {
|
||||
// In ioredis, `info(section)` returns a string.
|
||||
const info_memory = await rclient.info('memory')
|
||||
const usedMemory = parseRedisInfoInt(info_memory, 'used_memory')
|
||||
const info_aof = await rclient.info('persistence')
|
||||
const usedDisk = parseRedisInfoInt(info_aof, 'aof_current_size')
|
||||
|
||||
return { diskBytes: usedDisk, ramBytes: usedMemory }
|
||||
} finally {
|
||||
try {
|
||||
await rclient.disconnect()
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
|
||||
async function upsertStat({ statKey, day, values, generatedAt }) {
|
||||
await InstanceStat.updateOne(
|
||||
{ statKey, day },
|
||||
{ $set: { values, generatedAt, day } },
|
||||
{ upsert: true }
|
||||
)
|
||||
}
|
||||
|
||||
async function collectForDay({ day }) {
|
||||
const now = new Date()
|
||||
const oneDayAgo = new Date(now.getTime() - 24 * 60 * 60 * 1000)
|
||||
const db = mongoose.connection.db
|
||||
const segmentationEnabled = isUserSegmentationEnabled()
|
||||
|
||||
const projects = db.collection('projects')
|
||||
const users = db.collection('users')
|
||||
const docs = db.collection('docs')
|
||||
|
||||
const activeProjects = await projects.countDocuments({
|
||||
lastUpdated: { $gte: oneDayAgo },
|
||||
})
|
||||
|
||||
let activeUsersValues
|
||||
let userCountValues
|
||||
|
||||
if (segmentationEnabled) {
|
||||
const guestMatch = guestUserFilter()
|
||||
|
||||
const activeGuestUsers = await users.countDocuments({
|
||||
...guestMatch,
|
||||
lastActive: { $gte: oneDayAgo },
|
||||
})
|
||||
const activeUsersTotal = await users.countDocuments({
|
||||
lastActive: { $gte: oneDayAgo },
|
||||
})
|
||||
const activeStandardUsers = activeUsersTotal - activeGuestUsers
|
||||
|
||||
const guestUsers = await users.countDocuments(guestMatch)
|
||||
const totalUsers = await users.countDocuments()
|
||||
const standardUsers = totalUsers - guestUsers
|
||||
|
||||
// [standard, guest] — matches chart labels in instance-stats frontend config
|
||||
activeUsersValues = [activeStandardUsers, activeGuestUsers]
|
||||
userCountValues = [standardUsers, guestUsers]
|
||||
} else {
|
||||
const activeUsers = await users.countDocuments({
|
||||
lastActive: { $gte: oneDayAgo },
|
||||
})
|
||||
const totalUsers = await users.countDocuments()
|
||||
|
||||
activeUsersValues = [activeUsers]
|
||||
userCountValues = [totalUsers]
|
||||
}
|
||||
|
||||
const projectCount = await projects.countDocuments()
|
||||
const fileCount = await docs.countDocuments()
|
||||
|
||||
// Mongo storage: approximate with dbStats.totalSize (bytes).
|
||||
// Note that this will be lower that when doing `du` for the mongo
|
||||
// data folder, because that includes journaling files that we can not
|
||||
// measure directly from mongo.
|
||||
let mongodbStorageBytes = 0
|
||||
try {
|
||||
const stats = await db.stats()
|
||||
mongodbStorageBytes = stats?.totalSize ?? 0
|
||||
} catch (err) {
|
||||
logger.error({ err }, 'Failed to query Mongo dbStats')
|
||||
}
|
||||
|
||||
const overleafStorageBytes = await duBytes('/var/lib/overleaf')
|
||||
const { diskBytes: redisDiskBytes, ramBytes: redisRamBytes } =
|
||||
await getRedisMemoryBytes()
|
||||
|
||||
const generatedAt = new Date()
|
||||
|
||||
await upsertStat({ statKey: 'active_projects', day, values: [activeProjects], generatedAt })
|
||||
await upsertStat({ statKey: 'active_users', day, values: activeUsersValues, generatedAt })
|
||||
await upsertStat({ statKey: 'user_count', day, values: userCountValues, generatedAt })
|
||||
await upsertStat({ statKey: 'project_count', day, values: [projectCount], generatedAt })
|
||||
await upsertStat({ statKey: 'file_count', day, values: [fileCount], generatedAt })
|
||||
await upsertStat({ statKey: 'mongodb_storage', day, values: [mongodbStorageBytes], generatedAt })
|
||||
await upsertStat({ statKey: 'overleaf_storage', day, values: [overleafStorageBytes], generatedAt })
|
||||
await upsertStat({ statKey: 'redis_storage', day, values: [redisDiskBytes, redisRamBytes], generatedAt })
|
||||
}
|
||||
|
||||
async function main() {
|
||||
if (!Settings.instanceStats?.enabled) {
|
||||
logger.info({ enabled: false }, 'Instance stats collection disabled')
|
||||
return
|
||||
}
|
||||
|
||||
await mongoose.connectionPromise
|
||||
|
||||
const now = new Date()
|
||||
const day = toUtcMidnight(now)
|
||||
|
||||
logger.info({ day }, 'Collecting instance statistics')
|
||||
await collectForDay({ day })
|
||||
logger.info({ day }, 'Collected instance statistics')
|
||||
}
|
||||
|
||||
main()
|
||||
.then(() => process.exit(0))
|
||||
.catch(err => {
|
||||
logger.err({ err }, 'Failed collecting instance statistics')
|
||||
process.exit(1)
|
||||
})
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import mongoose from '../../../../../app/src/infrastructure/Mongoose.mjs'
|
||||
|
||||
const { Schema } = mongoose
|
||||
|
||||
const InstanceStatSchema = new Schema(
|
||||
{
|
||||
// Metric identifier
|
||||
statKey: { type: String, required: true, index: true },
|
||||
// Day bucket (stored as a Date; comparisons should use UTC midnight)
|
||||
day: { type: Date, required: true, index: true },
|
||||
// Values for the stacked bar series.
|
||||
// For 1-series metrics: [y1]
|
||||
// For 2-series metrics: [y1, y2]
|
||||
values: { type: [Number], required: true, default: [] },
|
||||
generatedAt: { type: Date, required: true, default: () => new Date() },
|
||||
},
|
||||
{ minimize: false }
|
||||
)
|
||||
|
||||
InstanceStatSchema.index({ statKey: 1, day: 1 }, { unique: true })
|
||||
|
||||
export const InstanceStat = mongoose.model(
|
||||
'InstanceStat',
|
||||
InstanceStatSchema,
|
||||
'instanceStats'
|
||||
)
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
import logger from '@overleaf/logger'
|
||||
import Settings from '@overleaf/settings'
|
||||
import mongoose from '../../../../app/src/infrastructure/Mongoose.mjs'
|
||||
import { InstanceStat } from './models/InstanceStat.mjs'
|
||||
|
||||
const SEGMENTED_STAT_DEFS = {
|
||||
active_projects: { base: [30], noise: [12], min: [0] },
|
||||
active_users: { base: [120, 80], noise: [25, 20], min: [0, 0] }, // [standard, guest]
|
||||
user_count: { base: [900, 500], noise: [8, 6], min: [0, 0] }, // [standard, guest]
|
||||
project_count: { base: [1600], noise: [6], min: [0] },
|
||||
file_count: { base: [12000], noise: [80], min: [0] },
|
||||
mongodb_storage: { base: [9.5e9], noise: [4.5e7], min: [0] }, // bytes
|
||||
overleaf_storage: { base: [2.8e10], noise: [8.0e7], min: [0] }, // bytes
|
||||
redis_storage: { base: [4.2e8, 1.9e8], noise: [1.2e7, 7e6], min: [0, 0] }, // bytes
|
||||
}
|
||||
|
||||
function toUtcMidnight(date) {
|
||||
return new Date(
|
||||
Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate())
|
||||
)
|
||||
}
|
||||
|
||||
function randomInRange(min, max) {
|
||||
return Math.random() * (max - min) + min
|
||||
}
|
||||
|
||||
function generateValues(def, dayOffset) {
|
||||
const season = Math.sin(dayOffset / 14) // gentle periodicity
|
||||
return def.base.map((base, i) => {
|
||||
const trend = base * (dayOffset * 0.0015)
|
||||
const jitter = randomInRange(-def.noise[i], def.noise[i])
|
||||
const seasonal = season * def.noise[i] * 0.6
|
||||
return Math.max(def.min[i], Math.round(base + trend + jitter + seasonal))
|
||||
})
|
||||
}
|
||||
|
||||
function getStatDefsForCurrentSegmentation() {
|
||||
const segmentation = Settings.instanceStats?.userSegmentation
|
||||
const segmentationEnabled = Boolean(segmentation?.enabled)
|
||||
|
||||
if (segmentationEnabled) {
|
||||
return SEGMENTED_STAT_DEFS
|
||||
}
|
||||
|
||||
return {
|
||||
...SEGMENTED_STAT_DEFS,
|
||||
active_users: { base: [200], noise: [35], min: [0] },
|
||||
user_count: { base: [1400], noise: [10], min: [0] },
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
await mongoose.connectionPromise
|
||||
|
||||
const days = Math.max(parseInt(process.env.INSTANCE_STATS_SEED_DAYS || '400', 10), 1)
|
||||
// Default is true so we remove old seeded/real docs unless explicitly disabled.
|
||||
const clear = process.env.INSTANCE_STATS_SEED_CLEAR !== 'false'
|
||||
const now = new Date()
|
||||
const statDefs = getStatDefsForCurrentSegmentation()
|
||||
|
||||
if (clear) {
|
||||
logger.info('Clearing existing instanceStats collection before seeding')
|
||||
await InstanceStat.deleteMany({})
|
||||
}
|
||||
|
||||
const ops = []
|
||||
for (let d = days - 1; d >= 0; d--) {
|
||||
const day = toUtcMidnight(new Date(now.getTime() - d * 24 * 60 * 60 * 1000))
|
||||
for (const [statKey, def] of Object.entries(statDefs)) {
|
||||
ops.push({
|
||||
updateOne: {
|
||||
filter: { statKey, day },
|
||||
update: {
|
||||
$set: {
|
||||
values: generateValues(def, days - d),
|
||||
generatedAt: new Date(),
|
||||
},
|
||||
},
|
||||
upsert: true,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
logger.info({ days, docs: ops.length }, 'Seeding fake instance statistics')
|
||||
await InstanceStat.bulkWrite(ops, { ordered: false })
|
||||
logger.info('Seed completed')
|
||||
}
|
||||
|
||||
main()
|
||||
.then(() => process.exit(0))
|
||||
.catch(err => {
|
||||
logger.err({ err }, 'Failed to seed instance statistics')
|
||||
process.exit(1)
|
||||
})
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
import logger from '@overleaf/logger'
|
||||
import Settings from '@overleaf/settings'
|
||||
import mongoose from '../../../../app/src/infrastructure/Mongoose.mjs'
|
||||
import { InstanceStat } from './models/InstanceStat.mjs'
|
||||
|
||||
const WINDOWS = {
|
||||
month: 30,
|
||||
'6m': 180,
|
||||
year: 365,
|
||||
}
|
||||
|
||||
function toUtcMidnight(date) {
|
||||
return new Date(
|
||||
Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate())
|
||||
)
|
||||
}
|
||||
|
||||
function computeCutoff(window) {
|
||||
const retentionDays = Settings.instanceStats?.retentionDays || 365
|
||||
const now = new Date()
|
||||
const cutoffDays = window === 'all' ? retentionDays : WINDOWS[window] || 30
|
||||
return toUtcMidnight(new Date(now.getTime() - cutoffDays * 24 * 60 * 60 * 1000))
|
||||
}
|
||||
|
||||
async function main() {
|
||||
await mongoose.connectionPromise
|
||||
const segmentationEnabled = Boolean(
|
||||
Settings.instanceStats?.userSegmentation?.enabled
|
||||
)
|
||||
|
||||
const statKeys = [
|
||||
'active_projects',
|
||||
'active_users',
|
||||
'user_count',
|
||||
'project_count',
|
||||
'file_count',
|
||||
'mongodb_storage',
|
||||
'overleaf_storage',
|
||||
'redis_storage',
|
||||
]
|
||||
|
||||
const windowsToCheck = ['month', '6m', 'year', 'all']
|
||||
|
||||
for (const statKey of statKeys) {
|
||||
for (const window of windowsToCheck) {
|
||||
const cutoff = computeCutoff(window)
|
||||
const docs = await InstanceStat.find(
|
||||
{ statKey, day: { $gte: cutoff } },
|
||||
{ _id: 0, values: 1 }
|
||||
)
|
||||
.sort({ day: 1 })
|
||||
.limit(10)
|
||||
.lean()
|
||||
|
||||
if (docs.length === 0) {
|
||||
logger.warn({ statKey, window }, 'No points in window (may be expected initially)')
|
||||
continue
|
||||
}
|
||||
|
||||
const expectTwo =
|
||||
statKey === 'redis_storage' ||
|
||||
(segmentationEnabled &&
|
||||
(statKey === 'active_users' ||
|
||||
statKey === 'user_count'))
|
||||
for (const [i, d] of docs.entries()) {
|
||||
const len = d.values?.length || 0
|
||||
if (expectTwo && len < 2) {
|
||||
throw new Error(
|
||||
`${statKey} expected 2-series values, but got len=${len} (example index ${i}, window=${window})`
|
||||
)
|
||||
}
|
||||
if (!expectTwo && len < 1) {
|
||||
throw new Error(
|
||||
`${statKey} expected 1-series values, but got len=${len} (example index ${i}, window=${window})`
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logger.info('Instance stats verification completed')
|
||||
}
|
||||
|
||||
main()
|
||||
.then(() => process.exit(0))
|
||||
.catch(err => {
|
||||
logger.err({ err }, 'Instance stats verification failed')
|
||||
process.exit(1)
|
||||
})
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
extends ../../../../app/views/layout-marketing
|
||||
|
||||
include ../../../../app/views/_mixins/bookmarkable_tabset.pug
|
||||
|
||||
block entrypointVar
|
||||
- entrypoint = 'modules/instance-stats/pages/instance-stats'
|
||||
|
||||
block content
|
||||
#main-content.content.content-alt
|
||||
.container
|
||||
.row
|
||||
.col-sm-12
|
||||
.card#instance-stats-root
|
||||
.card-body
|
||||
.page-header
|
||||
h1 Instance statistics
|
||||
|
||||
.row.justify-content-end
|
||||
.col-2
|
||||
label.form-label(for='timeFilter') Show:
|
||||
select#timeFilter.form-select
|
||||
option(value='month' selected) Last 1 month
|
||||
option(value='6m') Last 6 months
|
||||
option(value='year') Last 1 year
|
||||
option(value='all') All
|
||||
|
||||
hr
|
||||
|
||||
// Charts (organized into tabs)
|
||||
ol-tabs.ol-tabs(data-ol-bookmarkable-tabset)
|
||||
.nav-tabs-container
|
||||
ul.nav.nav-tabs.align-left(role='tablist')
|
||||
+bookmarkable-tabset-header('user', 'User', true)
|
||||
+bookmarkable-tabset-header('project', 'Project')
|
||||
+bookmarkable-tabset-header('storage', 'Storage')
|
||||
|
||||
.tab-content
|
||||
.tab-pane.active(role='tabpanel' id='user')
|
||||
.row(style='height:400px')
|
||||
.col-md-6#active-users
|
||||
.col-md-6#user-count
|
||||
|
||||
.tab-pane(role='tabpanel' id='project')
|
||||
.row(style='height:400px')
|
||||
.col-md-4#active-projects
|
||||
.col-md-4#project-count
|
||||
.col-md-4#file-count
|
||||
|
||||
.tab-pane(role='tabpanel' id='storage')
|
||||
.row(style='height:400px')
|
||||
.col-md-4#mongodb-storage
|
||||
.col-md-4#overleaf-storage
|
||||
.col-md-4#redis-storage
|
||||
|
||||
block prepend foot-scripts
|
||||
script(src='https://cdn.plot.ly/plotly-3.0.1.min.js' charset='utf-8' nonce=scriptNonce defer=deferScripts)
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
import type { SeriesResponse, StatKey, WindowKey } from './types'
|
||||
|
||||
function isSeriesResponse(value: unknown): value is SeriesResponse {
|
||||
if (!value || typeof value !== 'object') return false
|
||||
const asRecord = value as Record<string, unknown>
|
||||
return (
|
||||
typeof asRecord.metric === 'string' &&
|
||||
typeof asRecord.window === 'string' &&
|
||||
Array.isArray(asRecord.points)
|
||||
)
|
||||
}
|
||||
|
||||
export async function fetchSeries(
|
||||
metric: StatKey,
|
||||
window: WindowKey
|
||||
): Promise<SeriesResponse> {
|
||||
const url = `/admin/instance-stats/api/series?metric=${encodeURIComponent(metric)}&window=${encodeURIComponent(window)}`
|
||||
const res = await fetch(url, { headers: { Accept: 'application/json' } })
|
||||
if (!res.ok) {
|
||||
const message = await res.text().catch(() => '')
|
||||
throw new Error(`Failed to fetch ${metric} (${res.status}): ${message}`)
|
||||
}
|
||||
const json = await res.json()
|
||||
if (!isSeriesResponse(json)) {
|
||||
throw new Error(`Invalid response shape for metric ${metric}`)
|
||||
}
|
||||
return json
|
||||
}
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
import type { StatConfig, WindowKey } from './types'
|
||||
|
||||
const passThrough = (value: number) => value
|
||||
const bytesToMB = (bytes: number) => bytes / (1024 * 1024)
|
||||
|
||||
export const WINDOW_OPTIONS: Array<{ value: WindowKey; label: string }> = [
|
||||
{ value: 'month', label: 'Last 1 month' },
|
||||
{ value: '6m', label: 'Last 6 months' },
|
||||
{ value: 'year', label: 'Last 1 year' },
|
||||
{ value: 'all', label: 'All' },
|
||||
]
|
||||
|
||||
export const STAT_CONFIG: StatConfig[] = [
|
||||
{
|
||||
id: 'active-projects',
|
||||
metric: 'active_projects',
|
||||
tabId: 'project',
|
||||
title: 'Active projects',
|
||||
seriesCount: 1,
|
||||
transform: passThrough,
|
||||
colors: { y1: '#acbce2' },
|
||||
labels: {},
|
||||
},
|
||||
{
|
||||
id: 'active-users',
|
||||
metric: 'active_users',
|
||||
tabId: 'user',
|
||||
title: 'Active users',
|
||||
seriesCount: 2,
|
||||
transform: passThrough,
|
||||
colors: { y1: '#d98c8c', y2: '#EDC9C9' },
|
||||
labels: { y1: 'Standard', y2: 'Guest' },
|
||||
},
|
||||
{
|
||||
id: 'user-count',
|
||||
metric: 'user_count',
|
||||
tabId: 'user',
|
||||
title: 'Users',
|
||||
seriesCount: 2,
|
||||
transform: passThrough,
|
||||
colors: { y1: '#d98c8c', y2: '#EDC9C9' },
|
||||
labels: { y1: 'Standard', y2: 'Guest' },
|
||||
},
|
||||
{
|
||||
id: 'project-count',
|
||||
metric: 'project_count',
|
||||
tabId: 'project',
|
||||
title: 'Projects',
|
||||
seriesCount: 1,
|
||||
transform: passThrough,
|
||||
colors: { y1: '#acbce2' },
|
||||
labels: {},
|
||||
},
|
||||
{
|
||||
id: 'file-count',
|
||||
metric: 'file_count',
|
||||
tabId: 'project',
|
||||
title: 'Files',
|
||||
seriesCount: 1,
|
||||
transform: passThrough,
|
||||
colors: { y1: '#767676' },
|
||||
labels: {},
|
||||
},
|
||||
{
|
||||
id: 'mongodb-storage',
|
||||
metric: 'mongodb_storage',
|
||||
tabId: 'storage',
|
||||
title: 'Mongodb storage',
|
||||
ylabel: 'MB',
|
||||
seriesCount: 1,
|
||||
transform: bytesToMB,
|
||||
colors: { y1: '#767676' },
|
||||
labels: {},
|
||||
},
|
||||
{
|
||||
id: 'overleaf-storage',
|
||||
metric: 'overleaf_storage',
|
||||
tabId: 'storage',
|
||||
title: 'Overleaf storage',
|
||||
ylabel: 'MB',
|
||||
seriesCount: 1,
|
||||
transform: bytesToMB,
|
||||
colors: { y1: '#767676' },
|
||||
labels: {},
|
||||
},
|
||||
{
|
||||
id: 'redis-storage',
|
||||
metric: 'redis_storage',
|
||||
tabId: 'storage',
|
||||
title: 'Redis storage',
|
||||
ylabel: 'MB',
|
||||
seriesCount: 2,
|
||||
transform: bytesToMB,
|
||||
colors: { y1: '#767676', y2: '#ADADAD' },
|
||||
labels: { y1: 'Disk', y2: 'RAM' },
|
||||
},
|
||||
]
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
import type { SeriesPoint, StatConfig } from './types'
|
||||
|
||||
declare global {
|
||||
interface Window {
|
||||
Plotly?: {
|
||||
newPlot: (
|
||||
id: string,
|
||||
traces: unknown[],
|
||||
layout: Record<string, unknown>,
|
||||
config: Record<string, unknown>
|
||||
) => Promise<unknown>
|
||||
animate?: (
|
||||
id: string,
|
||||
frameOrGroupNameOrAnimationDef: Record<string, unknown>,
|
||||
animationOpts: Record<string, unknown>
|
||||
) => Promise<unknown>
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function getSeriesCount(stat: StatConfig, points: SeriesPoint[]) {
|
||||
const maxValuesLength = points.reduce(
|
||||
(max, point) => Math.max(max, point.values.length),
|
||||
0
|
||||
)
|
||||
if (maxValuesLength >= 2) return 2
|
||||
if (maxValuesLength === 1) return 1
|
||||
return stat.seriesCount
|
||||
}
|
||||
|
||||
function buildTraces(stat: StatConfig, points: SeriesPoint[]) {
|
||||
const x = points.map(point => point.day)
|
||||
const seriesCount = getSeriesCount(stat, points)
|
||||
const indexes = seriesCount === 2 ? [0, 1] : [0]
|
||||
return indexes.map(index => {
|
||||
const key = index === 0 ? 'y1' : 'y2'
|
||||
const y = points.map(point => stat.transform(point.values[index] ?? 0))
|
||||
const label = seriesCount === 2 ? stat.labels[key] : undefined
|
||||
const hovertemplate =
|
||||
label != null
|
||||
? `%{x|%d %b %Y}<br>${label}: %{y}<extra></extra>`
|
||||
: '%{x|%d %b %Y}<br>%{y}<extra></extra>'
|
||||
return {
|
||||
x,
|
||||
y,
|
||||
type: 'bar',
|
||||
name: label,
|
||||
showlegend: seriesCount === 2,
|
||||
marker: { color: stat.colors[key] },
|
||||
hovertemplate,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function buildLayout(stat: StatConfig, points: SeriesPoint[]) {
|
||||
const seriesCount = getSeriesCount(stat, points)
|
||||
const indexes = seriesCount === 2 ? [0, 1] : [0]
|
||||
const totals = points.map(point =>
|
||||
indexes.reduce((sum, index) => sum + (point.values[index] ?? 0), 0)
|
||||
)
|
||||
const ymax = totals.length > 0 ? Math.max(...totals.map(stat.transform)) : 0
|
||||
return {
|
||||
title: { text: stat.title },
|
||||
font: { family: 'Raleway, sans-serif' },
|
||||
bargap: 0.1,
|
||||
xaxis: {
|
||||
title: 'Time',
|
||||
type: 'date',
|
||||
tickmode: 'auto',
|
||||
nticks: 8,
|
||||
ticklabelmode: 'period',
|
||||
hoverformat: '%d %b %Y',
|
||||
automargin: true,
|
||||
tickformatstops: [
|
||||
{ dtickrange: [null, 24 * 60 * 60 * 1000], value: '%d %b' },
|
||||
{ dtickrange: [24 * 60 * 60 * 1000, 'M1'], value: '%d %b' },
|
||||
{ dtickrange: ['M1', 'M12'], value: '%b %Y' },
|
||||
{ dtickrange: ['M12', null], value: '%Y' },
|
||||
],
|
||||
},
|
||||
yaxis: {
|
||||
title: { text: stat.ylabel || '' },
|
||||
range: [0, ymax],
|
||||
},
|
||||
barmode: 'stack',
|
||||
}
|
||||
}
|
||||
|
||||
export async function renderChart(stat: StatConfig, points: SeriesPoint[]) {
|
||||
if (!window.Plotly) {
|
||||
throw new Error('Plotly is not available on window')
|
||||
}
|
||||
const traces = buildTraces(stat, points)
|
||||
const layout = buildLayout(stat, points)
|
||||
const initialTraces = traces.map(trace => ({
|
||||
...trace,
|
||||
y: (trace.y as number[]).map(() => 0),
|
||||
}))
|
||||
|
||||
await window.Plotly.newPlot(stat.id, initialTraces, layout, {
|
||||
displayModeBar: false,
|
||||
})
|
||||
|
||||
if (typeof window.Plotly.animate === 'function') {
|
||||
return window.Plotly.animate(
|
||||
stat.id,
|
||||
{
|
||||
data: traces.map(trace => ({ y: trace.y })),
|
||||
},
|
||||
{
|
||||
transition: { duration: 450, easing: 'cubic-in-out' },
|
||||
frame: { duration: 450, redraw: false },
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
export type WindowKey = 'month' | '6m' | 'year' | 'all'
|
||||
|
||||
export type TabId = 'user' | 'project' | 'storage'
|
||||
|
||||
export type StatKey =
|
||||
| 'active_projects'
|
||||
| 'active_users'
|
||||
| 'user_count'
|
||||
| 'project_count'
|
||||
| 'file_count'
|
||||
| 'mongodb_storage'
|
||||
| 'overleaf_storage'
|
||||
| 'redis_storage'
|
||||
|
||||
export interface SeriesPoint {
|
||||
day: number
|
||||
values: number[]
|
||||
}
|
||||
|
||||
export interface SeriesResponse {
|
||||
metric: StatKey
|
||||
window: WindowKey
|
||||
points: SeriesPoint[]
|
||||
}
|
||||
|
||||
export interface StatConfig {
|
||||
id: string
|
||||
metric: StatKey
|
||||
tabId: TabId
|
||||
title: string
|
||||
ylabel?: string
|
||||
seriesCount: 1 | 2
|
||||
transform: (value: number) => number
|
||||
colors: { y1: string; y2?: string }
|
||||
labels: { y1?: string; y2?: string }
|
||||
}
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
import { fetchSeries } from '../features/instance-stats/api'
|
||||
import { STAT_CONFIG } from '../features/instance-stats/config'
|
||||
import { renderChart } from '../features/instance-stats/render'
|
||||
import type { TabId, WindowKey } from '../features/instance-stats/types'
|
||||
|
||||
const WINDOW_VALUES: WindowKey[] = ['month', '6m', 'year', 'all']
|
||||
|
||||
function getWindowValue(select: HTMLSelectElement): WindowKey {
|
||||
const value = select.value as WindowKey
|
||||
if (WINDOW_VALUES.includes(value)) return value
|
||||
return 'month'
|
||||
}
|
||||
|
||||
function getTabIdFromHash(): TabId {
|
||||
const hash = window.location.hash?.replace(/^#/, '') || ''
|
||||
if (hash === 'user' || hash === 'project' || hash === 'storage') return hash
|
||||
return 'user'
|
||||
}
|
||||
|
||||
async function initInstanceStatsPage() {
|
||||
const root = document.getElementById('instance-stats-root')
|
||||
const select = document.getElementById('timeFilter') as HTMLSelectElement | null
|
||||
if (!root || !select) return
|
||||
|
||||
let requestId = 0
|
||||
|
||||
const renderWindow = async (window: WindowKey, tabId: TabId) => {
|
||||
const myRequestId = ++requestId
|
||||
const jobs = STAT_CONFIG.filter(stat => stat.tabId === tabId).map(async stat => {
|
||||
const series = await fetchSeries(stat.metric, window)
|
||||
if (myRequestId !== requestId) return
|
||||
await renderChart(stat, series.points)
|
||||
})
|
||||
await Promise.allSettled(jobs)
|
||||
}
|
||||
|
||||
const renderActiveTab = () => {
|
||||
const tabId = getTabIdFromHash()
|
||||
// Delay a tick so the bootstrap tab pane is visible before Plotly sizes
|
||||
// the chart container.
|
||||
setTimeout(() => void renderWindow(getWindowValue(select), tabId), 0)
|
||||
}
|
||||
|
||||
select.addEventListener('change', () => {
|
||||
renderActiveTab()
|
||||
})
|
||||
|
||||
window.addEventListener('hashchange', () => {
|
||||
// Bookmarkable tabs update the hash first; wait a tick before rerender.
|
||||
renderActiveTab()
|
||||
})
|
||||
|
||||
// Initial render based on current hash.
|
||||
renderActiveTab()
|
||||
}
|
||||
|
||||
void initInstanceStatsPage()
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
import logger from '@overleaf/logger'
|
||||
import Settings from '@overleaf/settings'
|
||||
|
||||
let InstanceStatsModule = {}
|
||||
|
||||
if (process.env.INSTANCE_STATS_ENABLED === 'true') {
|
||||
logger.debug({}, 'Enabling instance-stats module')
|
||||
|
||||
Settings.instanceStats = {
|
||||
enabled: true,
|
||||
retentionDays: parseInt(process.env.INSTANCE_STATS_RETENTION_DAYS, 10) || 365,
|
||||
userSegmentation: {
|
||||
// When enabled, active_users and user_count are split into [standard, guest]
|
||||
// using users whose adminRoles array contains "guest-user".
|
||||
enabled: process.env.INSTANCE_STATS_USER_SEGMENTATION_ENABLED === 'true',
|
||||
},
|
||||
}
|
||||
|
||||
const { default: InstanceStatsRouter } = await import('./app/src/InstanceStatsRouter.mjs')
|
||||
InstanceStatsModule = {
|
||||
router: InstanceStatsRouter,
|
||||
}
|
||||
}
|
||||
|
||||
export default InstanceStatsModule
|
||||
Reference in New Issue
Block a user