Merge pull request #30366 from overleaf/mj-em-package-meta

[web] Add event for package usage

GitOrigin-RevId: e994becf01e7e4c8642cd1815ffe05907a5fd63c

services/web/app/src/Features/Analytics/AnalyticsManager.mjs

@@ -28,6 +28,7 @@ const analyticsAccountMappingQueue = Queues.getQueue(
   'analytics-account-mapping'
 )
 const analyticsEmailChangeQueue = Queues.getQueue('analytics-email-change')
+const analyticsPackageUsageQueue = Queues.getQueue('analytics-package-usage')
 
 const ONE_MINUTE_MS = 60 * 1000
 
@@ -97,6 +98,14 @@ function recordEventForSession(session, event, segmentation) {
   })
 }
 
+function emitPackageUsage(projectId, { documentClasses, packages }) {
+  analyticsPackageUsageQueue
+    .add('package-usage', { projectId, documentClasses, packages })
+    .catch(err => {
+      logger.warn({ err, projectId }, 'Failed to emit package usage')
+    })
+}
+
 async function setUserPropertyForUser(userId, propertyName, propertyValue) {
   if (_isAnalyticsDisabled() || _isSmokeTestUser(userId)) {
     return
@@ -454,6 +463,7 @@ export default {
   recordEventForSession,
   recordEventForUser,
   recordEventForUserInBackground,
+  emitPackageUsage,
   setUserPropertyForUser,
   setUserPropertyForUserInBackground,
   setUserPropertyForSession,

services/web/app/src/Features/Metadata/MetaController.mjs

@@ -3,6 +3,8 @@ import EditorRealTimeController from '../Editor/EditorRealTimeController.mjs'
 import MetaHandler from './MetaHandler.mjs'
 import logger from '@overleaf/logger'
 import { expressify } from '@overleaf/promise-utils'
+import Analytics from '../Analytics/AnalyticsManager.mjs'
+import SplitTestHandler from '../SplitTests/SplitTestHandler.mjs'
 
 async function getMetadata(req, res) {
   const { project_id: projectId } = req.params
@@ -22,9 +24,37 @@ async function getMetadata(req, res) {
     )
   }
 
+  const assignment = await SplitTestHandler.promises.getAssignment(
+    req,
+    res,
+    'emit-package-usage-statistics'
+  )
+  if (assignment.variant === 'enabled') {
+    sendAnalyticsEventForPackageUsage(projectId, projectMeta)
+  }
+
   res.json({ projectId, projectMeta })
 }
 
+function sendAnalyticsEventForPackageUsage(projectId, projectMeta) {
+  const packagesSet = new Set()
+  const documentClassesSet = new Set()
+  for (const docMeta of Object.values(projectMeta)) {
+    if (docMeta.documentClass != null) {
+      documentClassesSet.add(docMeta.documentClass)
+    }
+
+    for (const packageName of docMeta.packageNames) {
+      if (packageName.match(/^[a-zA-Z0-9-_]+$/)) {
+        packagesSet.add(packageName)
+      }
+    }
+  }
+  const packages = Array.from(packagesSet)
+  const documentClasses = Array.from(documentClassesSet)
+  Analytics.emitPackageUsage(projectId, { documentClasses, packages })
+}
+
 async function broadcastMetadataForDoc(req, res) {
   const { project_id: projectId } = req.params
   const { doc_id: docId } = req.params

services/web/app/src/Features/Metadata/MetaHandler.mjs

@@ -7,9 +7,16 @@ import { callbackify } from '@overleaf/promise-utils'
  *   labels: string[]
  *   packages: Record<string, Record<string, any>>,
  *   packageNames: string[],
+ *   documentClass: string | null
  * }} DocMeta
  */
 
+const LABEL_RE = /\\label{(.{0,80}?)}/g
+const LABEL_OPTION_RE = /\blabel={?(.{0,80}?)[\s},\]]/g
+const PACKAGE_RE = /^\\usepackage(?:\[.{0,80}?])?{(.{0,80}?)}/g
+const REQ_PACKAGE_RE = /^\\RequirePackage(?:\[.{0,80}?])?{(.{0,80}?)}/g
+const DOCUMENT_CLASS_RE = /^\\documentclass(?:\[.{0,80}?])?{(.{0,80}?)}/
+
 /**
  * @param {string[]} lines
  * @return {Promise<DocMeta>}
@@ -20,31 +27,34 @@ async function extractMetaFromDoc(lines) {
     labels: [],
     packages: {},
     packageNames: [],
+    documentClass: null,
   }
 
-  const labelRe = /\\label{(.{0,80}?)}/g
-  const labelOptionRe = /\blabel={?(.{0,80}?)[\s},\]]/g
-  const packageRe = /^\\usepackage(?:\[.{0,80}?])?{(.{0,80}?)}/g
-  const reqPackageRe = /^\\RequirePackage(?:\[.{0,80}?])?{(.{0,80}?)}/g
-
   for (const rawLine of lines) {
     const line = getNonCommentedContent(rawLine)
 
-    for (const label of lineMatches(labelRe, line)) {
+    for (const label of lineMatches(LABEL_RE, line)) {
       docMeta.labels.push(label)
     }
 
-    for (const label of lineMatches(labelOptionRe, line)) {
+    for (const label of lineMatches(LABEL_OPTION_RE, line)) {
       docMeta.labels.push(label)
     }
 
-    for (const pkg of lineMatches(packageRe, line, ',')) {
+    for (const pkg of lineMatches(PACKAGE_RE, line, ',')) {
       docMeta.packageNames.push(pkg)
     }
 
-    for (const pkg of lineMatches(reqPackageRe, line, ',')) {
+    for (const pkg of lineMatches(REQ_PACKAGE_RE, line, ',')) {
       docMeta.packageNames.push(pkg)
     }
+
+    if (docMeta.documentClass == null) {
+      const match = line.match(DOCUMENT_CLASS_RE)
+      if (match != null) {
+        docMeta.documentClass = match[1]
+      }
+    }
   }
 
   for (const packageName of docMeta.packageNames) {

services/web/app/src/infrastructure/Queues.mjs

@@ -30,6 +30,9 @@ const QUEUES_JOB_OPTIONS = {
   'analytics-email-change': {
     removeOnFail: MAX_FAILED_JOBS_RETAINED_ANALYTICS,
   },
+  'analytics-package-usage': {
+    removeOnFail: MAX_FAILED_JOBS_RETAINED_ANALYTICS,
+  },
   'emails-onboarding': {
     removeOnFail: MAX_FAILED_JOBS_RETAINED,
   },
@@ -73,6 +76,7 @@ const ANALYTICS_QUEUES = [
   'analytics-email-change',
   'analytics-events',
   'analytics-editing-sessions',
+  'analytics-package-usage',
   'analytics-user-properties',
   'analytics-user-exports',
   'post-registration-analytics',

services/web/test/unit/src/Analytics/AnalyticsManager.test.mjs

@@ -50,6 +50,10 @@ describe('AnalyticsManager', function () {
       add: sinon.stub().resolves(),
       process: sinon.stub().resolves(),
     }
+    ctx.analyticsPackageUsageQueue = {
+      add: sinon.stub().resolves(),
+      process: sinon.stub().resolves(),
+    }
     ctx.Queues = {
       getQueue: queueName => {
         switch (queueName) {
@@ -65,6 +69,8 @@ describe('AnalyticsManager', function () {
             return ctx.analyticsAccountMappingQueue
           case 'analytics-email-change':
             return ctx.analyticsEmailChangeQueue
+          case 'analytics-package-usage':
+            return ctx.analyticsPackageUsageQueue
           default:
             throw new Error('Unexpected queue name')
         }
@@ -372,6 +378,8 @@ describe('AnalyticsManager', function () {
                 return ctx.analyticsAccountMappingQueue
               case 'analytics-email-change':
                 return ctx.analyticsEmailChangeQueue
+              case 'analytics-package-usage':
+                return ctx.analyticsPackageUsageQueue
               default:
                 throw new Error('Unexpected queue name')
             }

services/web/test/unit/src/Metadata/MetaController.test.mjs

@@ -27,6 +27,11 @@ describe('MetaController', function () {
       default: ctx.MetaHandler,
     }))
 
+    vi.doMock(
+      '../../../../app/src/Features/Analytics/AnalyticsManager',
+      () => ({ default: {} })
+    )
+
     ctx.MetadataController = (await import(modulePath)).default
   })
 

services/web/test/unit/src/Metadata/MetaHandler.test.mjs

@@ -103,6 +103,7 @@ describe('MetaHandler', function () {
           baz: ctx.packageMapping.baz,
         },
         packageNames: ['foo', 'bar', 'baz'],
+        documentClass: null,
       })
 
       ctx.DocumentUpdaterHandler.promises.flushDocToMongo.should.be.calledWith(
@@ -161,16 +162,19 @@ describe('MetaHandler', function () {
           labels: ['aaa'],
           packages: {},
           packageNames: [],
+          documentClass: null,
         },
         id_two: {
           labels: [],
           packages: {},
           packageNames: [],
+          documentClass: null,
         },
         id_three: {
           labels: ['bbb', 'ccc'],
           packages: {},
           packageNames: [],
+          documentClass: null,
         },
         id_four: {
           labels: [],
@@ -185,6 +189,7 @@ describe('MetaHandler', function () {
             ],
           },
           packageNames: ['baz', 'amsmath'],
+          documentClass: null,
         },
         id_five: {
           labels: ['sec:intro'],
@@ -213,6 +218,7 @@ describe('MetaHandler', function () {
             ],
           },
           packageNames: ['foo', 'baz', 'hello'],
+          documentClass: null,
         },
       })