From 711d50a2f1a8bf373a6f948a40292f7943612913 Mon Sep 17 00:00:00 2001 From: Antoine Clausse Date: Mon, 29 Apr 2024 11:02:57 +0200 Subject: [PATCH] [web] Create script to update forgotten `featuresUpdatedAt` after the migration to 20s compile timeout (#18113) * Copy `migration_compile_timeout_60s_to_20s.js` script * Update `featuresUpdatedAt` * Add a comment about `featuresUpdatedAt` in migration_compile_timeout_60s_to_20s.js * Fix test on migration_compile_timeout_60s_to_20s.js * Fix: Include users having `featuresUpdatedAt` undefined in the update * Add test on `migration_compile_timeout_60s_to_20s_fixup_features_updated_at` GitOrigin-RevId: 4b2baf955a6a9f39bf9ce00b7839af551064c6cb --- .../migration_compile_timeout_60s_to_20s.js | 3 +- ...ut_60s_to_20s_fixup_features_updated_at.js | 65 ++++ .../src/MigrateUserFeatureTimeoutTests.js | 363 +++++++++++++----- 3 files changed, 333 insertions(+), 98 deletions(-) create mode 100644 services/web/scripts/migration_compile_timeout_60s_to_20s_fixup_features_updated_at.js diff --git a/services/web/scripts/migration_compile_timeout_60s_to_20s.js b/services/web/scripts/migration_compile_timeout_60s_to_20s.js index 1df2834b16..d58b58b60f 100644 --- a/services/web/scripts/migration_compile_timeout_60s_to_20s.js +++ b/services/web/scripts/migration_compile_timeout_60s_to_20s.js @@ -34,7 +34,8 @@ const main = async ({ COMMIT, SKIP_COUNT }) => { const nModified = await batchedUpdate( 'users', { 'features.compileTimeout': { $lte: 60, $ne: 20 } }, - { $set: { 'features.compileTimeout': 20 } } + // NOTE: Always update featuresUpdatedAt to ensure the user's features synced with BigQuery + { $set: { 'features.compileTimeout': 20, featuresUpdatedAt: new Date() } } ) console.log(`Updated ${nModified} records`) } diff --git a/services/web/scripts/migration_compile_timeout_60s_to_20s_fixup_features_updated_at.js b/services/web/scripts/migration_compile_timeout_60s_to_20s_fixup_features_updated_at.js new file mode 100644 index 0000000000..bc417477c1 --- /dev/null +++ b/services/web/scripts/migration_compile_timeout_60s_to_20s_fixup_features_updated_at.js @@ -0,0 +1,65 @@ +#!/usr/bin/env node + +const minimist = require('minimist') +const { + db, + READ_PREFERENCE_SECONDARY, + waitForDb, +} = require('../app/src/infrastructure/mongodb') +const { batchedUpdate } = require('./helpers/batchedUpdate') + +// A few seconds after the previous migration script was run +const FEATURES_UPDATED_AT = new Date('2024-04-16T12:41:00Z') + +const query = { + 'features.compileTimeout': 20, + $or: [ + { featuresUpdatedAt: { $exists: false } }, + { featuresUpdatedAt: { $lt: FEATURES_UPDATED_AT } }, + ], +} + +async function logCount() { + const usersToUpdate = await db.users.countDocuments(query, { + readPreference: READ_PREFERENCE_SECONDARY, + }) + console.log( + `Found ${usersToUpdate} users needing their featuresUpdatedAt updated` + ) +} + +const main = async ({ COMMIT, SKIP_COUNT }) => { + console.time('Script Duration') + + await waitForDb() + + if (!SKIP_COUNT) { + await logCount() + } + + if (COMMIT) { + const nModified = await batchedUpdate('users', query, { + $set: { featuresUpdatedAt: FEATURES_UPDATED_AT }, + }) + console.log(`Updated ${nModified} records`) + } + + console.timeEnd('Script Duration') +} + +const setup = () => { + const argv = minimist(process.argv.slice(2)) + const COMMIT = argv.commit !== undefined + const SKIP_COUNT = argv['skip-count'] !== undefined + if (!COMMIT) { + console.warn('Doing dry run. Add --commit to commit changes') + } + return { COMMIT, SKIP_COUNT } +} + +main(setup()) + .catch(err => { + console.error(err) + process.exit(1) + }) + .then(() => process.exit(0)) diff --git a/services/web/test/acceptance/src/MigrateUserFeatureTimeoutTests.js b/services/web/test/acceptance/src/MigrateUserFeatureTimeoutTests.js index a256ce8e78..60bd414892 100644 --- a/services/web/test/acceptance/src/MigrateUserFeatureTimeoutTests.js +++ b/services/web/test/acceptance/src/MigrateUserFeatureTimeoutTests.js @@ -17,118 +17,287 @@ async function runScript(args = []) { } } -describe('MigrateUserFeatureTimeoutTests', function () { - const usersInput = { - noFeatures: {}, - noFeatureTimeout: { features: {} }, - timeout10s: { features: { compileTimeout: 10, other: 'val1' }, bar: '1' }, - timeout20s: { features: { compileTimeout: 20, other: 'val2' }, bar: '2' }, - timeout30s: { features: { compileTimeout: 30, other: 'val3' }, bar: '3' }, - timeout60s: { features: { compileTimeout: 60, other: 'val4' }, bar: '4' }, - timeout120s: { features: { compileTimeout: 120, other: 'val5' }, bar: '5' }, - timeout180s: { features: { compileTimeout: 180, other: 'val6' }, bar: '6' }, +async function runFixupScript(args = []) { + try { + return await promisify(exec)( + [ + 'node', + 'scripts/migration_compile_timeout_60s_to_20s_fixup_features_updated_at.js', + ...args, + ].join(' ') + ) + } catch (error) { + logger.error({ error }, 'script failed') + throw error } +} - const usersKeys = Object.keys(usersInput) - const userIds = {} - - beforeEach('insert users', async function () { - const usersInsertedValues = await db.users.insertMany( - usersKeys.map(key => ({ - ...usersInput[key], - email: `${key}@example.com`, - })) - ) - usersKeys.forEach( - (key, index) => (userIds[key] = usersInsertedValues.insertedIds[index]) - ) - }) - - it('gives correct counts in dry mode', async function () { - const users = await db.users.find().toArray() - expect(users).to.have.lengthOf(usersKeys.length) - - const result = await runScript([]) - - expect(result.stderr).to.contain( - 'Doing dry run. Add --commit to commit changes' - ) - expect(result.stdout).to.contain( - 'Found 3 users with compileTimeout <= 60s && != 20s' - ) - expect(result.stdout).to.contain('Found 1 users with compileTimeout == 20s') - expect(result.stdout).not.to.contain('Updated') - - const usersAfter = await db.users.find().toArray() - - expect(usersAfter).to.deep.equal(users) - }) - - it("updates users compileTimeout when '--commit' is set", async function () { - const users = await db.users.find().toArray() - expect(users).to.have.lengthOf(usersKeys.length) - const result = await runScript(['--commit']) - - expect(result.stdout).to.contain( - 'Found 3 users with compileTimeout <= 60s && != 20s' - ) - expect(result.stdout).to.contain('Found 1 users with compileTimeout == 20s') - expect(result.stdout).to.contain('Updated 3 records') - - const usersAfter = await db.users.find().toArray() - - expect(usersAfter).to.deep.equal([ - { _id: userIds.noFeatures, email: 'noFeatures@example.com' }, - { - _id: userIds.noFeatureTimeout, - email: 'noFeatureTimeout@example.com', - features: {}, - }, - { - _id: userIds.timeout10s, - email: 'timeout10s@example.com', - features: { compileTimeout: 20, other: 'val1' }, +describe('MigrateUserFeatureTimeoutTests', function () { + describe('initial script', function () { + const usersInput = { + noFeatures: {}, + noFeatureTimeout: { features: {} }, + timeout10s: { + features: { compileTimeout: 10, other: 'val1' }, bar: '1', + featuresUpdatedAt: new Date('2020-01-01'), }, - { - _id: userIds.timeout20s, - email: 'timeout20s@example.com', - features: { compileTimeout: 20, other: 'val2' }, - bar: '2', - }, - { - _id: userIds.timeout30s, - email: 'timeout30s@example.com', - features: { compileTimeout: 20, other: 'val3' }, + timeout20s: { features: { compileTimeout: 20, other: 'val2' }, bar: '2' }, + timeout30s: { + features: { compileTimeout: 30, other: 'val3' }, bar: '3', + featuresUpdatedAt: new Date('2025-01-01'), }, - { - _id: userIds.timeout60s, - email: 'timeout60s@example.com', - features: { compileTimeout: 20, other: 'val4' }, + timeout60s: { + features: { compileTimeout: 60, other: 'val4' }, bar: '4', + featuresUpdatedAt: new Date(), }, - { - _id: userIds.timeout120s, - email: 'timeout120s@example.com', + timeout120s: { features: { compileTimeout: 120, other: 'val5' }, bar: '5', }, - { - _id: userIds.timeout180s, - email: 'timeout180s@example.com', + timeout180s: { features: { compileTimeout: 180, other: 'val6' }, bar: '6', + featuresUpdatedAt: new Date('2020-01-01'), }, - ]) + } - const result2 = await runScript([]) + const usersKeys = Object.keys(usersInput) + const userIds = {} - expect(result2.stdout).to.contain( - 'Found 0 users with compileTimeout <= 60s && != 20s' - ) - expect(result2.stdout).to.contain( - 'Found 4 users with compileTimeout == 20s' - ) + beforeEach('insert users', async function () { + const usersInsertedValues = await db.users.insertMany( + usersKeys.map(key => ({ + ...usersInput[key], + email: `${key}@example.com`, + })) + ) + usersKeys.forEach( + (key, index) => (userIds[key] = usersInsertedValues.insertedIds[index]) + ) + }) + + afterEach('clear users', async function () { + await db.users.deleteMany({}) + }) + + it('gives correct counts in dry mode', async function () { + const users = await db.users.find().toArray() + expect(users).to.have.lengthOf(usersKeys.length) + + const result = await runScript([]) + + expect(result.stderr).to.contain( + 'Doing dry run. Add --commit to commit changes' + ) + expect(result.stdout).to.contain( + 'Found 3 users with compileTimeout <= 60s && != 20s' + ) + expect(result.stdout).to.contain( + 'Found 1 users with compileTimeout == 20s' + ) + expect(result.stdout).not.to.contain('Updated') + + const usersAfter = await db.users.find().toArray() + + expect(usersAfter).to.deep.equal(users) + }) + + it("updates users compileTimeout when '--commit' is set", async function () { + const users = await db.users.find().toArray() + expect(users).to.have.lengthOf(usersKeys.length) + const result = await runScript(['--commit']) + + expect(result.stdout).to.contain( + 'Found 3 users with compileTimeout <= 60s && != 20s' + ) + expect(result.stdout).to.contain( + 'Found 1 users with compileTimeout == 20s' + ) + expect(result.stdout).to.contain('Updated 3 records') + + const usersAfter = await db.users.find().toArray() + + expect( + usersAfter.map(({ featuresUpdatedAt, ...rest }) => rest) + ).to.deep.equal([ + { _id: userIds.noFeatures, email: 'noFeatures@example.com' }, + { + _id: userIds.noFeatureTimeout, + email: 'noFeatureTimeout@example.com', + features: {}, + }, + { + _id: userIds.timeout10s, + email: 'timeout10s@example.com', + features: { compileTimeout: 20, other: 'val1' }, + bar: '1', + }, + { + _id: userIds.timeout20s, + email: 'timeout20s@example.com', + features: { compileTimeout: 20, other: 'val2' }, + bar: '2', + }, + { + _id: userIds.timeout30s, + email: 'timeout30s@example.com', + features: { compileTimeout: 20, other: 'val3' }, + bar: '3', + }, + { + _id: userIds.timeout60s, + email: 'timeout60s@example.com', + features: { compileTimeout: 20, other: 'val4' }, + bar: '4', + }, + { + _id: userIds.timeout120s, + email: 'timeout120s@example.com', + features: { compileTimeout: 120, other: 'val5' }, + bar: '5', + }, + { + _id: userIds.timeout180s, + email: 'timeout180s@example.com', + features: { compileTimeout: 180, other: 'val6' }, + bar: '6', + }, + ]) + + expect(usersAfter[0].featuresUpdatedAt).to.be.undefined + expect(usersAfter[1].featuresUpdatedAt).to.be.undefined + expect(usersAfter[2].featuresUpdatedAt).to.be.instanceOf(Date) + expect(usersAfter[3].featuresUpdatedAt).to.be.undefined // was already 20s + expect(usersAfter[4].featuresUpdatedAt).to.be.instanceOf(Date) + expect(usersAfter[5].featuresUpdatedAt).to.be.instanceOf(Date) + expect(usersAfter[6].featuresUpdatedAt).to.be.undefined + + const result2 = await runScript([]) + + expect(result2.stdout).to.contain( + 'Found 0 users with compileTimeout <= 60s && != 20s' + ) + expect(result2.stdout).to.contain( + 'Found 4 users with compileTimeout == 20s' + ) + }) + }) + + describe('fixup script', function () { + const usersInput = { + timeout20s1: { + features: { compileTimeout: 20 }, + }, + timeout20s2: { + features: { compileTimeout: 20 }, + featuresUpdatedAt: new Date('2023-01-01'), + }, + timeout20s3: { + features: { compileTimeout: 20 }, + featuresUpdatedAt: new Date('2025-01-01'), + }, + timeout240s1: { + features: { compileTimeout: 240 }, + }, + timeout240s2: { + features: { compileTimeout: 240 }, + featuresUpdatedAt: new Date('2023-01-01'), + }, + timeout240s3: { + features: { compileTimeout: 240 }, + featuresUpdatedAt: new Date('2025-01-01'), + }, + } + + const usersKeys = Object.keys(usersInput) + const userIds = {} + + beforeEach('insert users', async function () { + const usersInsertedValues = await db.users.insertMany( + usersKeys.map(key => ({ + ...usersInput[key], + email: `${key}@example.com`, + })) + ) + usersKeys.forEach( + (key, index) => (userIds[key] = usersInsertedValues.insertedIds[index]) + ) + }) + afterEach('clear users', async function () { + await db.users.deleteMany({}) + }) + + it('gives correct counts in dry mode', async function () { + const users = await db.users.find().toArray() + expect(users).to.have.lengthOf(usersKeys.length) + const result = await runFixupScript([]) + expect(result.stderr).to.contain( + 'Doing dry run. Add --commit to commit changes' + ) + expect(result.stdout).to.contain( + 'Found 2 users needing their featuresUpdatedAt updated' + ) + expect(result.stdout).not.to.contain('Updated 2 records') + const usersAfter = await db.users.find().toArray() + expect(usersAfter).to.deep.equal(users) + }) + + it("updates users featuresUpdatedAt when '--commit' is set", async function () { + const FEATURES_UPDATED_AT = new Date('2024-04-16T12:41:00Z') + const users = await db.users.find().toArray() + expect(users).to.have.lengthOf(usersKeys.length) + const result = await runFixupScript(['--commit']) + expect(result.stdout).to.contain( + 'Found 2 users needing their featuresUpdatedAt updated' + ) + expect(result.stdout).to.contain('Updated 2 records') + const usersAfter = await db.users.find().toArray() + expect(usersAfter).to.deep.equal([ + { + _id: userIds.timeout20s1, + email: 'timeout20s1@example.com', + features: { compileTimeout: 20 }, + featuresUpdatedAt: FEATURES_UPDATED_AT, + }, + { + _id: userIds.timeout20s2, + email: 'timeout20s2@example.com', + features: { compileTimeout: 20 }, + featuresUpdatedAt: FEATURES_UPDATED_AT, + }, + { + _id: userIds.timeout20s3, + email: 'timeout20s3@example.com', + features: { compileTimeout: 20 }, + featuresUpdatedAt: new Date('2025-01-01'), + }, + { + _id: userIds.timeout240s1, + email: 'timeout240s1@example.com', + features: { compileTimeout: 240 }, + }, + { + _id: userIds.timeout240s2, + email: 'timeout240s2@example.com', + features: { compileTimeout: 240 }, + featuresUpdatedAt: new Date('2023-01-01'), + }, + { + _id: userIds.timeout240s3, + email: 'timeout240s3@example.com', + features: { compileTimeout: 240 }, + featuresUpdatedAt: new Date('2025-01-01'), + }, + ]) + + const result2 = await runFixupScript([]) + + expect(result2.stdout).to.contain( + 'Found 0 users needing their featuresUpdatedAt updated' + ) + }) }) })