diff --git a/services/web/app/src/Features/Analytics/AnalyticsManager.mjs b/services/web/app/src/Features/Analytics/AnalyticsManager.mjs index c3a2c4e09c..b763ae494b 100644 --- a/services/web/app/src/Features/Analytics/AnalyticsManager.mjs +++ b/services/web/app/src/Features/Analytics/AnalyticsManager.mjs @@ -28,6 +28,7 @@ const analyticsAccountMappingQueue = Queues.getQueue( 'analytics-account-mapping' ) const analyticsEmailChangeQueue = Queues.getQueue('analytics-email-change') +const analyticsPackageUsageQueue = Queues.getQueue('analytics-package-usage') const ONE_MINUTE_MS = 60 * 1000 @@ -97,6 +98,14 @@ function recordEventForSession(session, event, segmentation) { }) } +function emitPackageUsage(projectId, { documentClasses, packages }) { + analyticsPackageUsageQueue + .add('package-usage', { projectId, documentClasses, packages }) + .catch(err => { + logger.warn({ err, projectId }, 'Failed to emit package usage') + }) +} + async function setUserPropertyForUser(userId, propertyName, propertyValue) { if (_isAnalyticsDisabled() || _isSmokeTestUser(userId)) { return @@ -454,6 +463,7 @@ export default { recordEventForSession, recordEventForUser, recordEventForUserInBackground, + emitPackageUsage, setUserPropertyForUser, setUserPropertyForUserInBackground, setUserPropertyForSession, diff --git a/services/web/app/src/Features/Metadata/MetaController.mjs b/services/web/app/src/Features/Metadata/MetaController.mjs index 6d869c3c54..8bb1646107 100644 --- a/services/web/app/src/Features/Metadata/MetaController.mjs +++ b/services/web/app/src/Features/Metadata/MetaController.mjs @@ -3,6 +3,8 @@ import EditorRealTimeController from '../Editor/EditorRealTimeController.mjs' import MetaHandler from './MetaHandler.mjs' import logger from '@overleaf/logger' import { expressify } from '@overleaf/promise-utils' +import Analytics from '../Analytics/AnalyticsManager.mjs' +import SplitTestHandler from '../SplitTests/SplitTestHandler.mjs' async function getMetadata(req, res) { const { project_id: projectId } = req.params @@ -22,9 +24,37 @@ async function getMetadata(req, res) { ) } + const assignment = await SplitTestHandler.promises.getAssignment( + req, + res, + 'emit-package-usage-statistics' + ) + if (assignment.variant === 'enabled') { + sendAnalyticsEventForPackageUsage(projectId, projectMeta) + } + res.json({ projectId, projectMeta }) } +function sendAnalyticsEventForPackageUsage(projectId, projectMeta) { + const packagesSet = new Set() + const documentClassesSet = new Set() + for (const docMeta of Object.values(projectMeta)) { + if (docMeta.documentClass != null) { + documentClassesSet.add(docMeta.documentClass) + } + + for (const packageName of docMeta.packageNames) { + if (packageName.match(/^[a-zA-Z0-9-_]+$/)) { + packagesSet.add(packageName) + } + } + } + const packages = Array.from(packagesSet) + const documentClasses = Array.from(documentClassesSet) + Analytics.emitPackageUsage(projectId, { documentClasses, packages }) +} + async function broadcastMetadataForDoc(req, res) { const { project_id: projectId } = req.params const { doc_id: docId } = req.params diff --git a/services/web/app/src/Features/Metadata/MetaHandler.mjs b/services/web/app/src/Features/Metadata/MetaHandler.mjs index 491a5ca8b0..247ae2f04e 100644 --- a/services/web/app/src/Features/Metadata/MetaHandler.mjs +++ b/services/web/app/src/Features/Metadata/MetaHandler.mjs @@ -7,9 +7,16 @@ import { callbackify } from '@overleaf/promise-utils' * labels: string[] * packages: Record>, * packageNames: string[], + * documentClass: string | null * }} DocMeta */ +const LABEL_RE = /\\label{(.{0,80}?)}/g +const LABEL_OPTION_RE = /\blabel={?(.{0,80}?)[\s},\]]/g +const PACKAGE_RE = /^\\usepackage(?:\[.{0,80}?])?{(.{0,80}?)}/g +const REQ_PACKAGE_RE = /^\\RequirePackage(?:\[.{0,80}?])?{(.{0,80}?)}/g +const DOCUMENT_CLASS_RE = /^\\documentclass(?:\[.{0,80}?])?{(.{0,80}?)}/ + /** * @param {string[]} lines * @return {Promise} @@ -20,31 +27,34 @@ async function extractMetaFromDoc(lines) { labels: [], packages: {}, packageNames: [], + documentClass: null, } - const labelRe = /\\label{(.{0,80}?)}/g - const labelOptionRe = /\blabel={?(.{0,80}?)[\s},\]]/g - const packageRe = /^\\usepackage(?:\[.{0,80}?])?{(.{0,80}?)}/g - const reqPackageRe = /^\\RequirePackage(?:\[.{0,80}?])?{(.{0,80}?)}/g - for (const rawLine of lines) { const line = getNonCommentedContent(rawLine) - for (const label of lineMatches(labelRe, line)) { + for (const label of lineMatches(LABEL_RE, line)) { docMeta.labels.push(label) } - for (const label of lineMatches(labelOptionRe, line)) { + for (const label of lineMatches(LABEL_OPTION_RE, line)) { docMeta.labels.push(label) } - for (const pkg of lineMatches(packageRe, line, ',')) { + for (const pkg of lineMatches(PACKAGE_RE, line, ',')) { docMeta.packageNames.push(pkg) } - for (const pkg of lineMatches(reqPackageRe, line, ',')) { + for (const pkg of lineMatches(REQ_PACKAGE_RE, line, ',')) { docMeta.packageNames.push(pkg) } + + if (docMeta.documentClass == null) { + const match = line.match(DOCUMENT_CLASS_RE) + if (match != null) { + docMeta.documentClass = match[1] + } + } } for (const packageName of docMeta.packageNames) { diff --git a/services/web/app/src/infrastructure/Queues.mjs b/services/web/app/src/infrastructure/Queues.mjs index bea46766ca..02b1cd2266 100644 --- a/services/web/app/src/infrastructure/Queues.mjs +++ b/services/web/app/src/infrastructure/Queues.mjs @@ -30,6 +30,9 @@ const QUEUES_JOB_OPTIONS = { 'analytics-email-change': { removeOnFail: MAX_FAILED_JOBS_RETAINED_ANALYTICS, }, + 'analytics-package-usage': { + removeOnFail: MAX_FAILED_JOBS_RETAINED_ANALYTICS, + }, 'emails-onboarding': { removeOnFail: MAX_FAILED_JOBS_RETAINED, }, @@ -73,6 +76,7 @@ const ANALYTICS_QUEUES = [ 'analytics-email-change', 'analytics-events', 'analytics-editing-sessions', + 'analytics-package-usage', 'analytics-user-properties', 'analytics-user-exports', 'post-registration-analytics', diff --git a/services/web/test/unit/src/Analytics/AnalyticsManager.test.mjs b/services/web/test/unit/src/Analytics/AnalyticsManager.test.mjs index 805b425c58..a44690dd6d 100644 --- a/services/web/test/unit/src/Analytics/AnalyticsManager.test.mjs +++ b/services/web/test/unit/src/Analytics/AnalyticsManager.test.mjs @@ -50,6 +50,10 @@ describe('AnalyticsManager', function () { add: sinon.stub().resolves(), process: sinon.stub().resolves(), } + ctx.analyticsPackageUsageQueue = { + add: sinon.stub().resolves(), + process: sinon.stub().resolves(), + } ctx.Queues = { getQueue: queueName => { switch (queueName) { @@ -65,6 +69,8 @@ describe('AnalyticsManager', function () { return ctx.analyticsAccountMappingQueue case 'analytics-email-change': return ctx.analyticsEmailChangeQueue + case 'analytics-package-usage': + return ctx.analyticsPackageUsageQueue default: throw new Error('Unexpected queue name') } @@ -372,6 +378,8 @@ describe('AnalyticsManager', function () { return ctx.analyticsAccountMappingQueue case 'analytics-email-change': return ctx.analyticsEmailChangeQueue + case 'analytics-package-usage': + return ctx.analyticsPackageUsageQueue default: throw new Error('Unexpected queue name') } diff --git a/services/web/test/unit/src/Metadata/MetaController.test.mjs b/services/web/test/unit/src/Metadata/MetaController.test.mjs index 3f25c59cde..f8ebec7690 100644 --- a/services/web/test/unit/src/Metadata/MetaController.test.mjs +++ b/services/web/test/unit/src/Metadata/MetaController.test.mjs @@ -27,6 +27,11 @@ describe('MetaController', function () { default: ctx.MetaHandler, })) + vi.doMock( + '../../../../app/src/Features/Analytics/AnalyticsManager', + () => ({ default: {} }) + ) + ctx.MetadataController = (await import(modulePath)).default }) diff --git a/services/web/test/unit/src/Metadata/MetaHandler.test.mjs b/services/web/test/unit/src/Metadata/MetaHandler.test.mjs index 48d5cc51a4..aadf2ab99e 100644 --- a/services/web/test/unit/src/Metadata/MetaHandler.test.mjs +++ b/services/web/test/unit/src/Metadata/MetaHandler.test.mjs @@ -103,6 +103,7 @@ describe('MetaHandler', function () { baz: ctx.packageMapping.baz, }, packageNames: ['foo', 'bar', 'baz'], + documentClass: null, }) ctx.DocumentUpdaterHandler.promises.flushDocToMongo.should.be.calledWith( @@ -161,16 +162,19 @@ describe('MetaHandler', function () { labels: ['aaa'], packages: {}, packageNames: [], + documentClass: null, }, id_two: { labels: [], packages: {}, packageNames: [], + documentClass: null, }, id_three: { labels: ['bbb', 'ccc'], packages: {}, packageNames: [], + documentClass: null, }, id_four: { labels: [], @@ -185,6 +189,7 @@ describe('MetaHandler', function () { ], }, packageNames: ['baz', 'amsmath'], + documentClass: null, }, id_five: { labels: ['sec:intro'], @@ -213,6 +218,7 @@ describe('MetaHandler', function () { ], }, packageNames: ['foo', 'baz', 'hello'], + documentClass: null, }, })