[web] speed up CI: bin-pack module acceptance tests (#35965)

GitOrigin-RevId: 92a33b4531d541c676018b3152c7abf74edd2eec

services/web/test/acceptance/getModuleTargets.js

@@ -1,13 +1,47 @@
 /* eslint-disable no-console */
 // silence settings module
 console.log = function () {}
+const fs = require('fs')
+const path = require('path')
 const Settings = require('@overleaf/settings')
 
 const MODULES = Settings.moduleImportSequence
-const TARGET = process.argv.slice(2).pop() || 'test_acceptance'
+const [TARGET = 'test_acceptance', shardIndexArg, shardTotalArg] =
+  process.argv.slice(2)
+
+let names = MODULES
+if (shardIndexArg && shardTotalArg) {
+  const shardIndex = parseInt(shardIndexArg, 10)
+  const shardTotal = parseInt(shardTotalArg, 10)
+  const timingsPath = path.join(__dirname, 'module-timings.json')
+  const timings = fs.existsSync(timingsPath)
+    ? JSON.parse(fs.readFileSync(timingsPath, 'utf8'))
+    : []
+  const msByName = new Map(timings.map(({ name, ms }) => [name, ms]))
+
+  // Greedy LPT (longest processing time first) bin-packing: sort modules by
+  // descending time, then drop each one into whichever shard currently has
+  // the smallest total, for a roughly time-fair split of work across shards.
+  const shards = Array.from({ length: shardTotal }, () => ({
+    totalMs: 0,
+    names: [],
+  }))
+  const sortedByMsDesc = MODULES.slice().sort(
+    (a, b) => (msByName.get(b) || 0) - (msByName.get(a) || 0)
+  )
+  for (const name of sortedByMsDesc) {
+    const shard = shards.reduce((lightest, s) =>
+      s.totalMs < lightest.totalMs ? s : lightest
+    )
+    shard.names.push(name)
+    shard.totalMs += msByName.get(name) || 0
+  }
+  names = shards[shardIndex - 1].names
+}
 
 if (TARGET === '--name-only') {
-  console.debug(MODULES.join('\n'))
+  console.debug(names.join('\n'))
 } else {
-  console.debug(MODULES.map(name => `modules/${name}/${TARGET}`).join('\n'))
+  const targets = names.map(name => `modules/${name}/${TARGET}`)
+  console.debug(targets.length > 0 ? targets.join('\n') : 'no_more_targets')
 }

services/web/test/acceptance/updateModuleTimings.js

@@ -0,0 +1,49 @@
+/* eslint-disable no-console */
+const fs = require('fs')
+const path = require('path')
+
+const ENTER_LEAVE_RE =
+  /^\[([^\]]+)] make\[\d+]: (Entering|Leaving) directory '.*\/modules\/([^/']+)'$/
+
+const files = process.argv.slice(2)
+if (files.length === 0) {
+  console.error(
+    'Usage: node updateModuleTimings.js <log-file> [<log-file> ...]'
+  )
+  process.exit(1)
+}
+
+const totalMsByName = new Map()
+const enteredAtByName = new Map()
+
+for (const file of files) {
+  const lines = fs.readFileSync(file, 'utf8').split('\n')
+  for (const line of lines) {
+    const match = line.match(ENTER_LEAVE_RE)
+    if (!match) continue
+    const [, timestamp, action, name] = match
+    const time = new Date(timestamp).getTime()
+    if (action === 'Entering') {
+      if (enteredAtByName.has(name)) {
+        throw new Error(
+          `module "${name}" entered again before leaving. Make sure to provide logs of the SaaS module stages.`
+        )
+      }
+      enteredAtByName.set(name, time)
+    } else if (enteredAtByName.has(name)) {
+      totalMsByName.set(
+        name,
+        (totalMsByName.get(name) || 0) + (time - enteredAtByName.get(name))
+      )
+      enteredAtByName.delete(name)
+    }
+  }
+}
+
+const result = Array.from(totalMsByName, ([name, ms]) => ({ name, ms })).sort(
+  (a, b) => a.name.localeCompare(b.name)
+)
+
+const outputPath = path.join(__dirname, 'module-timings.json')
+fs.writeFileSync(outputPath, JSON.stringify(result, null, 2) + '\n')
+console.log(`Wrote ${result.length} module timings to ${outputPath}`)