diff --git a/services/web/app/src/Features/User/UserAuditLogHandler.mjs b/services/web/app/src/Features/User/UserAuditLogHandler.mjs index be5ac971dd..ee9d355450 100644 --- a/services/web/app/src/Features/User/UserAuditLogHandler.mjs +++ b/services/web/app/src/Features/User/UserAuditLogHandler.mjs @@ -5,6 +5,7 @@ import { callbackify } from 'node:util' import SubscriptionLocator from '../Subscription/SubscriptionLocator.mjs' function _canHaveNoIpAddressId(operation, info) { + if (operation === 'add-email' && info.script) return true if (operation === 'join-group-subscription') return true if (operation === 'leave-group-subscription') return true if (operation === 'must-reset-password-set') return true @@ -15,6 +16,7 @@ function _canHaveNoIpAddressId(operation, info) { } function _canHaveNoInitiatorId(operation, info) { + if (operation === 'add-email' && info.script) return true if (operation === 'reset-password') return true if (operation === 'unlink-sso' && info.providerId === 'collabratec') return true diff --git a/services/web/app/src/Features/User/UserUpdater.mjs b/services/web/app/src/Features/User/UserUpdater.mjs index b45eb9a31a..7356790d86 100644 --- a/services/web/app/src/Features/User/UserUpdater.mjs +++ b/services/web/app/src/Features/User/UserUpdater.mjs @@ -101,6 +101,7 @@ async function addEmailAddress(userId, newEmail, affiliationOptions, auditLog) { auditLog.initiatorId, auditLog.ipAddress, { + ...auditLog.info, newSecondaryEmail: newEmail, } ) diff --git a/services/web/scripts/re_add_deleted_emails.mjs b/services/web/scripts/re_add_deleted_emails.mjs new file mode 100644 index 0000000000..e910073c5a --- /dev/null +++ b/services/web/scripts/re_add_deleted_emails.mjs @@ -0,0 +1,288 @@ +// @ts-check + +import minimist from 'minimist' +import { db, ObjectId } from '../app/src/infrastructure/mongodb.js' +import fs from 'node:fs/promises' +import * as csv from 'csv' +import { promisify } from 'node:util' +import { scriptRunner } from './lib/ScriptRunner.mjs' +import Errors from '../app/src/Features/Errors/Errors.js' +import UserGetter from '../app/src/Features/User/UserGetter.mjs' +import { READ_PREFERENCE_SECONDARY } from '@overleaf/mongo-utils/batchedUpdate.js' +import UserUpdater from '../app/src/Features/User/UserUpdater.mjs' +import EmailHelper from '../app/src/Features/Helpers/EmailHelper.js' +import AsyncLocalStorage from '../app/src/infrastructure/AsyncLocalStorage.js' +import AnalyticsManager from '../app/src/Features/Analytics/AnalyticsManager.mjs' +import UserAuditLogHandler from '../app/src/Features/User/UserAuditLogHandler.mjs' +import InstitutionsAPI from '../app/src/Features/Institutions/InstitutionsAPI.mjs' +import OError from '@overleaf/o-error' +import EmailChangeHelper from '../app/src/Features/Analytics/EmailChangeHelper.mjs' +import logger from '@overleaf/logger' + +const CSV_FILENAME = '/tmp/re_add_deleted_emails.csv' + +/** + * @type {(csvString: string) => Promise} + */ +const parseAsync = promisify(csv.parse) + +function usage() { + console.log('Usage: node re_add_deleted_emails.mjs [options]') + console.log( + 'fix wrongly removed emails by remove_unconfirmed_emails (see 2025-11-05 email removal)' + ) + console.log( + 'run this script with a CSV containing user IDs and emails to re-add save in ', + CSV_FILENAME + ) + console.log('Options:') + console.log(' --commit apply the changes') + process.exit(0) +} + +const { commit, help } = minimist(process.argv.slice(2), { + boolean: ['commit', 'help'], + alias: { help: 'h' }, + default: { commit: false }, +}) + +/** + * @param {string} email + */ +async function isEmailUsed(email) { + try { + await UserGetter.promises.ensureUniqueEmailAddress(email) + return false + } catch (err) { + if (err instanceof Errors.EmailExistsError) { + return true + } + throw err + } +} + +async function consumeCsvFile(trackProgress) { + console.time('re_add_deleted_emails') + + const csvContent = await fs.readFile(CSV_FILENAME, 'utf8') + const rows = await parseAsync(csvContent) + rows.shift() // Remove header row + + const emailsByUserId = {} + for (const [userId, email] of rows) { + if (!EmailHelper.parseEmail(email)) { + throw new Error(`invalid email ${email}`) + } + + if (!emailsByUserId[userId]) { + emailsByUserId[userId] = [] + } + emailsByUserId[userId].push(email) + } + + const userIds = Object.keys(emailsByUserId) + + const counts = { + /** @type {string[]} */ + processedUsers: [], + /** @type {string[]} */ + userNotFound: [], + /** @type {string[]} */ + emailsInUse: [], + /** @type {string[]} */ + alreadyOk: [], + /** @type {string[]} */ + primary: [], + /** @type {string[]} */ + secondary: [], + /** @type {string[]} */ + addedEmails: [], + } + + console.log('Total emails in the CSV:', rows.length) + console.log('Total users in the CSV:', userIds.length) + + for (const userId of userIds) { + const candidateEmails = emailsByUserId[userId] + + const user = await db.users.findOne( + { _id: new ObjectId(userId) }, + { readPreference: READ_PREFERENCE_SECONDARY } + ) + if (!user) { + counts.userNotFound.push(userId) + continue + } + + for (const email of candidateEmails) { + if (user.emails.some(item => item.email === email)) { + counts.alreadyOk.push(email) + continue + } + + const isUsed = await isEmailUsed(email) + const isOwnPrimary = user.email === email + + if (isUsed && !isOwnPrimary) { + counts.emailsInUse.push(email) + continue + } + + if (user.email === email) counts.primary.push(email) + else counts.secondary.push(email) + + if (commit) { + const auditLog = { + initiatorId: null, + ipAddress: null, + info: { + script: true, + note: 'fix wrongly removed unconfirmed secondary email', + }, + } + if (isOwnPrimary) { + // can't use addEmailAddress for primary email because ensureUniqueEmailAddress will throw + // using an override instead + await addEmailAddressOverride(user._id, email, {}, auditLog) + } else { + await UserUpdater.promises.addEmailAddress( + user._id, + email, + {}, + auditLog + ) + } + await UserUpdater.promises.confirmEmail(user._id, email) + } + counts.addedEmails.push(email) + } + + counts.processedUsers.push(userId) + trackProgress( + `Processed users: ${counts.processedUsers.length}/${userIds.length}` + ) + } + + console.log() + if (!commit) { + console.log('Dry-run, use --commit to apply changes') + console.log('This would be the result:') + console.log() + } + + console.log('Total emails in the CSV:', rows.length) + console.log('Total users in the CSV:', userIds.length) + console.log() + console.log('Users not found:', counts.userNotFound.length) + console.log('Users not found:', JSON.stringify(counts.userNotFound)) + console.log() + console.log('Already OK:', counts.alreadyOk.length) + console.log('Already OK:', JSON.stringify(counts.alreadyOk)) + console.log() + console.log('Already in use:', counts.emailsInUse.length) + console.log('Already in use:', JSON.stringify(counts.emailsInUse)) + console.log() + console.log('Primary:', counts.primary.length) + console.log('Primary:', JSON.stringify(counts.primary)) + console.log() + console.log('Secondary:', counts.secondary.length) + console.log('Secondary:', JSON.stringify(counts.secondary)) + console.log() + console.log('Added emails:', counts.addedEmails.length) + console.log('Added emails:', JSON.stringify(counts.addedEmails)) + console.log() + console.log() + console.timeEnd('re_add_deleted_emails') + console.log() +} + +async function main(trackProgress) { + if (help) { + return usage() + } + await consumeCsvFile(trackProgress) +} + +try { + await scriptRunner(main) + process.exit(0) +} catch (error) { + console.error(error) + process.exit(1) +} + +async function addEmailAddressOverride( + userId, + newEmail, + affiliationOptions, + auditLog +) { + AsyncLocalStorage.removeItem('userFullEmails') + newEmail = EmailHelper.parseEmail(newEmail) + if (!newEmail) { + throw new Error('invalid email') + } + + // Bypass ensureUniqueEmailAddress when re-adding primary emails + // await UserGetter.promises.ensureUniqueEmailAddress(newEmail) + + AnalyticsManager.recordEventForUserInBackground( + userId, + 'secondary-email-added' + ) + + await UserAuditLogHandler.promises.addEntry( + userId, + 'add-email', + auditLog.initiatorId, + auditLog.ipAddress, + { + ...auditLog.info, + newSecondaryEmail: newEmail, + } + ) + + try { + await InstitutionsAPI.promises.addAffiliation( + userId, + newEmail, + affiliationOptions + ) + } catch (error) { + throw OError.tag(error, 'problem adding affiliation while adding email') + } + + const createdAt = new Date() + let res + try { + const reversedHostname = newEmail.split('@')[1].split('').reverse().join('') + const update = { + $push: { + emails: { email: newEmail, createdAt, reversedHostname }, + }, + } + res = await UserUpdater.promises.updateUser( + { _id: userId, 'emails.email': { $ne: newEmail } }, + update + ) + } catch (error) { + throw OError.tag(error, 'problem updating users emails') + } + + if (res.matchedCount !== 1) { + return + } + + try { + await EmailChangeHelper.registerEmailCreation(userId, newEmail, { + // @ts-expect-error - This is copied from UserUpdater.mjs + createdAt: new Date(), + emailCreatedAt: createdAt, + }) + } catch (error) { + logger.warn( + { error, userId, newEmail }, + 'Error registering email creation with analytics' + ) + } +} diff --git a/services/web/scripts/remove_unconfirmed_emails.mjs b/services/web/scripts/remove_unconfirmed_emails.mjs index 6582ddca94..279f09fedc 100644 --- a/services/web/scripts/remove_unconfirmed_emails.mjs +++ b/services/web/scripts/remove_unconfirmed_emails.mjs @@ -146,15 +146,15 @@ async function consumeCsvFile() { console.log('Total users in the CSV:', userIds.length) for (const userId of userIds) { - const emailsToRemove = emailsByUserId[userId] + const emailsToRemoveCandidates = emailsByUserId[userId] const user = await db.users.findOne({ _id: new ObjectId(userId) }) if (!user) { - skippedEmail.userNotFound += emailsToRemove.length + skippedEmail.userNotFound += emailsToRemoveCandidates.length continue } - const emailsToRemoveNow = emailsToRemove.filter(email => { + const emailsToRemove = emailsToRemoveCandidates.filter(email => { const currentEmail = user.emails.find(e => e.email === email) if (!currentEmail) { skippedEmail.nowRemoved++ @@ -171,9 +171,9 @@ async function consumeCsvFile() { return true }) - removedEmailsCount += emailsToRemoveNow.length + removedEmailsCount += emailsToRemove.length - if (commit && emailsToRemoveNow.length > 0) { + if (commit && emailsToRemove.length > 0) { for (const email of emailsToRemove) { await UserAuditLogHandler.promises.addEntry( userId, diff --git a/services/web/test/acceptance/src/ReAddDeletedEmailsTests.mjs b/services/web/test/acceptance/src/ReAddDeletedEmailsTests.mjs new file mode 100644 index 0000000000..928d5b573d --- /dev/null +++ b/services/web/test/acceptance/src/ReAddDeletedEmailsTests.mjs @@ -0,0 +1,275 @@ +import { promisify } from 'node:util' +import { exec } from 'node:child_process' +import { expect } from 'chai' +import { filterOutput } from './helpers/settings.mjs' +import { db, ObjectId } from '../../../app/src/infrastructure/mongodb.js' +import fs from 'node:fs/promises' + +const CSV_FILENAME = '/tmp/re_add_deleted_emails.csv' + +async function runScript(commit) { + const result = await promisify(exec)( + ['node', 'scripts/re_add_deleted_emails.mjs', commit && '--commit'] + .filter(Boolean) + .join(' ') + ) + + return { + ...result, + stdout: result.stdout.split('\n').filter(filterOutput), + } +} + +/** + * @param {[string, string[]][]} userEmails + */ +const createUsers = async userEmails => + Promise.all( + userEmails.map(async ([email, emails]) => { + const _id = new ObjectId() + await db.users.insertOne({ + _id, + email, + emails: emails.map(email => ({ email })), + features: {}, + }) + return _id + }) + ) + +async function generateCsv(users) { + const text = 'User ID,Email' + const userRows = users.map(user => { + return `${user._id.toString()},${user.email}` + }) + await fs.writeFile(CSV_FILENAME, [text, ...userRows].join('\n')) +} + +describe('scripts/re_add_deleted_emails', function () { + let userIds + + afterEach(async function () { + try { + await fs.unlink(CSV_FILENAME) + } catch (err) { + // Ignore errors if file doesn't exist + } + }) + + describe('when user IDs dont match', function () { + beforeEach(async function () { + userIds = await createUsers([['mismatch1@xmpl.com', []]]) + await generateCsv([{ _id: new ObjectId(), email: 'mismatch2@xmpl.com' }]) + }) + + it('doesnt add new emails', async function () { + const { stdout } = await runScript(true) + expect(stdout).to.include('Total emails in the CSV: 1') + expect(stdout).to.include('Total users in the CSV: 1') + expect(stdout).to.include('Users not found: 1') + expect(stdout).to.include('Added emails: 0') + const updatedUser = await db.users.findOne({ _id: userIds[0] }) + expect(updatedUser.emails).to.have.length(0) + }) + }) + + describe('when email address is invalid', function () { + beforeEach(async function () { + userIds = await createUsers([['user@xmpl.com', []]]) + await generateCsv([{ _id: userIds[0], email: 'inv@lid@xmpl.com' }]) + }) + + it('throws', async function () { + await expect(runScript(true)).to.eventually.be.rejectedWith( + 'invalid email' + ) + const updatedUser = await db.users.findOne({ _id: userIds[0] }) + expect(updatedUser.emails).to.have.length(0) + }) + it('throws even without --commit', async function () { + await expect(runScript(false)).to.eventually.be.rejectedWith( + 'invalid email' + ) + const updatedUser = await db.users.findOne({ _id: userIds[0] }) + expect(updatedUser.emails).to.have.length(0) + }) + }) + + describe('when new email is used by another user', function () { + beforeEach(async function () { + userIds = await createUsers([ + ['user1@xmpl.com', []], + ['user2@xmpl.com', ['new-email@xmpl.com']], + ]) + await generateCsv([{ _id: userIds[0], email: 'new-email@xmpl.com' }]) + }) + + it('doesnt add new emails', async function () { + const { stdout } = await runScript(true) + expect(stdout).to.include('Total emails in the CSV: 1') + expect(stdout).to.include('Total users in the CSV: 1') + expect(stdout).to.include('Users not found: 0') + expect(stdout).to.include('Already in use: ["new-email@xmpl.com"]') + expect(stdout).to.include('Added emails: 0') + const updatedUser = await db.users.findOne({ _id: userIds[0] }) + expect(updatedUser.emails).to.have.length(0) + }) + }) + + describe('when the user has 0 email in the array', function () { + beforeEach(async function () { + userIds = await createUsers([['user@xmpl.com', []]]) + }) + + it('adds the primary email to the user', async function () { + await generateCsv([{ _id: userIds[0], email: 'user@xmpl.com' }]) + const { stdout } = await runScript(true) + expect(stdout).to.include('Total emails in the CSV: 1') + expect(stdout).to.include('Total users in the CSV: 1') + expect(stdout).to.include('Users not found: 0') + expect(stdout).to.include('Added emails: 1') + const updatedUser = await db.users.findOne({ _id: userIds[0] }) + expect(updatedUser.emails).to.have.length(1) + expect(updatedUser.emails[0].email).to.equal('user@xmpl.com') + expect(updatedUser.emails[0].reversedHostname).to.equal('moc.lpmx') + expect(updatedUser.emails[0].confirmedAt).to.be.an.instanceof(Date) + expect(updatedUser.emails[0].createdAt).to.be.an.instanceof(Date) + expect(updatedUser.emails[0].reconfirmedAt).to.be.an.instanceof(Date) + const auditLogs = await db.userAuditLogEntries + .find({ userId: userIds[0] }) + .toArray() + expect(auditLogs).to.have.length(1) + expect(auditLogs[0].operation).to.equal('add-email') + expect(auditLogs[0].info).to.deep.include({ + script: true, + note: 'fix wrongly removed unconfirmed secondary email', + newSecondaryEmail: 'user@xmpl.com', + }) + }) + + it('adds the secondary email to the user', async function () { + await generateCsv([{ _id: userIds[0], email: 'new-email@xmpl.com' }]) + const { stdout } = await runScript(true) + expect(stdout).to.include('Total emails in the CSV: 1') + expect(stdout).to.include('Total users in the CSV: 1') + expect(stdout).to.include('Users not found: 0') + expect(stdout).to.include('Added emails: 1') + const updatedUser = await db.users.findOne({ _id: userIds[0] }) + expect(updatedUser.emails).to.have.length(1) + expect(updatedUser.emails[0].email).to.equal('new-email@xmpl.com') + expect(updatedUser.emails[0].reversedHostname).to.equal('moc.lpmx') + expect(updatedUser.emails[0].confirmedAt).to.be.an.instanceof(Date) + expect(updatedUser.emails[0].createdAt).to.be.an.instanceof(Date) + expect(updatedUser.emails[0].reconfirmedAt).to.be.an.instanceof(Date) + const auditLogs = await db.userAuditLogEntries + .find({ userId: userIds[0] }) + .toArray() + expect(auditLogs).to.have.length(1) + expect(auditLogs[0].operation).to.equal('add-email') + expect(auditLogs[0].info).to.deep.include({ + script: true, + note: 'fix wrongly removed unconfirmed secondary email', + newSecondaryEmail: 'new-email@xmpl.com', + }) + }) + + it('doesnt add new emails without --commit', async function () { + await generateCsv([{ _id: userIds[0], email: 'new-email@xmpl.com' }]) + const { stdout } = await runScript(false) + expect(stdout).to.include('Dry-run, use --commit to apply changes') + const updatedUser = await db.users.findOne({ _id: userIds[0] }) + expect(updatedUser.emails).to.have.length(0) + }) + }) + describe('when the user has several emails in the array', function () { + beforeEach(async function () { + userIds = await createUsers([ + [ + 'user@xmpl.com', + ['email1@xmpl.com', 'email2@xmpl.com', 'email3@xmpl.com'], + ], + ]) + await generateCsv([{ _id: userIds[0], email: 'new-email@xmpl.com' }]) + }) + + it('adds the email to the user', async function () { + const { stdout } = await runScript(true) + + expect(stdout).to.include('Total emails in the CSV: 1') + expect(stdout).to.include('Total users in the CSV: 1') + expect(stdout).to.include('Users not found: 0') + expect(stdout).to.include('Added emails: 1') + const updatedUser = await db.users.findOne({ _id: userIds[0] }) + expect(updatedUser.emails).to.have.length(4) + + expect(updatedUser.emails[3].email).to.equal('new-email@xmpl.com') + expect(updatedUser.emails[3].reversedHostname).to.equal('moc.lpmx') + expect(updatedUser.emails[3].confirmedAt).to.be.an.instanceof(Date) + expect(updatedUser.emails[3].createdAt).to.be.an.instanceof(Date) + expect(updatedUser.emails[3].reconfirmedAt).to.be.an.instanceof(Date) + }) + + it('doesnt add new emails without --commit', async function () { + const { stdout } = await runScript(false) + expect(stdout).to.include('Dry-run, use --commit to apply changes') + const updatedUser = await db.users.findOne({ _id: userIds[0] }) + expect(updatedUser.emails).to.have.length(3) + }) + }) + + describe('all of the above', function () { + beforeEach(async function () { + userIds = await createUsers([ + ['user0@xmpl.com', []], + ['user1@xmpl.com', []], + ['user2@xmpl.com', ['a@xmpl.com', 'b@xmpl.com', 'c@xmpl.com']], + ['user3@xmpl.com', ['d@xmpl.com', 'e@xmpl.com', 'f@xmpl.com']], + ['user4@xmpl.com', ['x@xmpl.com', 'y@xmpl.com', 'z@xmpl.com']], + ['user5@xmpl.com', ['u@xmpl.com', 'v@xmpl.com', 'w@xmpl.com']], + ]) + await generateCsv([ + { _id: userIds[1], email: 'new1@xmpl.com' }, + { _id: userIds[1], email: 'user1@xmpl.com' }, + { _id: userIds[1], email: 'new2@xmpl.com' }, + { _id: userIds[2], email: 'new3@xmpl.com' }, + { _id: userIds[2], email: 'new4@xmpl.com' }, + { _id: userIds[2], email: 'user2@xmpl.com' }, + { _id: userIds[2], email: 'user3@xmpl.com' }, + { _id: userIds[3], email: 'd@xmpl.com' }, + { _id: userIds[5], email: 'a@xmpl.com' }, + { _id: new ObjectId(), email: 'a@xmpl.com' }, + ]) + }) + + it('updates users', async function () { + const { stdout } = await runScript(true) + expect(stdout).to.include('Total emails in the CSV: 10') + expect(stdout).to.include('Total users in the CSV: 5') + expect(stdout).to.include('Users not found: 1') + expect(stdout).to.include( + 'Already in use: ["user3@xmpl.com","a@xmpl.com"]' + ) + expect(stdout).to.include('Already OK: ["d@xmpl.com"]') + expect(stdout).to.include('Primary: ["user1@xmpl.com","user2@xmpl.com"]') + expect(stdout).to.include( + 'Secondary: ["new1@xmpl.com","new2@xmpl.com","new3@xmpl.com","new4@xmpl.com"]' + ) + expect(stdout).to.include( + 'Added emails: ["new1@xmpl.com","user1@xmpl.com","new2@xmpl.com","new3@xmpl.com","new4@xmpl.com","user2@xmpl.com"]' + ) + + const user0 = await db.users.findOne({ _id: userIds[0] }) + const user1 = await db.users.findOne({ _id: userIds[1] }) + const user2 = await db.users.findOne({ _id: userIds[2] }) + const user3 = await db.users.findOne({ _id: userIds[3] }) + const user4 = await db.users.findOne({ _id: userIds[4] }) + const user5 = await db.users.findOne({ _id: userIds[5] }) + expect(user0.emails).to.have.length(0) + expect(user1.emails).to.have.length(3) // new1, user1, new2 + expect(user2.emails).to.have.length(6) // a, b, c, new3, new4, user2 + expect(user3.emails).to.have.length(3) // d, e, f + expect(user4.emails).to.have.length(3) // x, y, z + expect(user5.emails).to.have.length(3) // u, v, w + }) + }) +})