diff --git a/services/web/test/acceptance/getModuleTargets.js b/services/web/test/acceptance/getModuleTargets.js index 8e12482851..55afdb368b 100644 --- a/services/web/test/acceptance/getModuleTargets.js +++ b/services/web/test/acceptance/getModuleTargets.js @@ -1,13 +1,47 @@ /* eslint-disable no-console */ // silence settings module console.log = function () {} +const fs = require('fs') +const path = require('path') const Settings = require('@overleaf/settings') const MODULES = Settings.moduleImportSequence -const TARGET = process.argv.slice(2).pop() || 'test_acceptance' +const [TARGET = 'test_acceptance', shardIndexArg, shardTotalArg] = + process.argv.slice(2) + +let names = MODULES +if (shardIndexArg && shardTotalArg) { + const shardIndex = parseInt(shardIndexArg, 10) + const shardTotal = parseInt(shardTotalArg, 10) + const timingsPath = path.join(__dirname, 'module-timings.json') + const timings = fs.existsSync(timingsPath) + ? JSON.parse(fs.readFileSync(timingsPath, 'utf8')) + : [] + const msByName = new Map(timings.map(({ name, ms }) => [name, ms])) + + // Greedy LPT (longest processing time first) bin-packing: sort modules by + // descending time, then drop each one into whichever shard currently has + // the smallest total, for a roughly time-fair split of work across shards. + const shards = Array.from({ length: shardTotal }, () => ({ + totalMs: 0, + names: [], + })) + const sortedByMsDesc = MODULES.slice().sort( + (a, b) => (msByName.get(b) || 0) - (msByName.get(a) || 0) + ) + for (const name of sortedByMsDesc) { + const shard = shards.reduce((lightest, s) => + s.totalMs < lightest.totalMs ? s : lightest + ) + shard.names.push(name) + shard.totalMs += msByName.get(name) || 0 + } + names = shards[shardIndex - 1].names +} if (TARGET === '--name-only') { - console.debug(MODULES.join('\n')) + console.debug(names.join('\n')) } else { - console.debug(MODULES.map(name => `modules/${name}/${TARGET}`).join('\n')) + const targets = names.map(name => `modules/${name}/${TARGET}`) + console.debug(targets.length > 0 ? targets.join('\n') : 'no_more_targets') } diff --git a/services/web/test/acceptance/updateModuleTimings.js b/services/web/test/acceptance/updateModuleTimings.js new file mode 100644 index 0000000000..92d15a4617 --- /dev/null +++ b/services/web/test/acceptance/updateModuleTimings.js @@ -0,0 +1,49 @@ +/* eslint-disable no-console */ +const fs = require('fs') +const path = require('path') + +const ENTER_LEAVE_RE = + /^\[([^\]]+)] make\[\d+]: (Entering|Leaving) directory '.*\/modules\/([^/']+)'$/ + +const files = process.argv.slice(2) +if (files.length === 0) { + console.error( + 'Usage: node updateModuleTimings.js [ ...]' + ) + process.exit(1) +} + +const totalMsByName = new Map() +const enteredAtByName = new Map() + +for (const file of files) { + const lines = fs.readFileSync(file, 'utf8').split('\n') + for (const line of lines) { + const match = line.match(ENTER_LEAVE_RE) + if (!match) continue + const [, timestamp, action, name] = match + const time = new Date(timestamp).getTime() + if (action === 'Entering') { + if (enteredAtByName.has(name)) { + throw new Error( + `module "${name}" entered again before leaving. Make sure to provide logs of the SaaS module stages.` + ) + } + enteredAtByName.set(name, time) + } else if (enteredAtByName.has(name)) { + totalMsByName.set( + name, + (totalMsByName.get(name) || 0) + (time - enteredAtByName.get(name)) + ) + enteredAtByName.delete(name) + } + } +} + +const result = Array.from(totalMsByName, ([name, ms]) => ({ name, ms })).sort( + (a, b) => a.name.localeCompare(b.name) +) + +const outputPath = path.join(__dirname, 'module-timings.json') +fs.writeFileSync(outputPath, JSON.stringify(result, null, 2) + '\n') +console.log(`Wrote ${result.length} module timings to ${outputPath}`)