[web] script to update group members via CSV (#24861)

* [web] script to update group members via CSV

GitOrigin-RevId: 973d1bdb1180af008608e14e1ff31af83e47f630

services/web/app/src/Features/Subscription/SubscriptionGroupHandler.js

@@ -1,11 +1,15 @@
 const { callbackify } = require('util')
+const _ = require('lodash')
+const OError = require('@overleaf/o-error')
 const SubscriptionUpdater = require('./SubscriptionUpdater')
 const SubscriptionLocator = require('./SubscriptionLocator')
 const SubscriptionController = require('./SubscriptionController')
 const { Subscription } = require('../../models/Subscription')
+const { User } = require('../../models/User')
 const RecurlyClient = require('./RecurlyClient')
 const PlansLocator = require('./PlansLocator')
 const SubscriptionHandler = require('./SubscriptionHandler')
+const TeamInvitesHandler = require('./TeamInvitesHandler')
 const GroupPlansData = require('./GroupPlansData')
 const Modules = require('../../infrastructure/Modules')
 const { MEMBERS_LIMIT_ADD_ON_CODE } = require('./PaymentProviderEntities')
@@ -14,6 +18,8 @@ const {
   PendingChangeError,
   InactiveError,
 } = require('./Errors')
+const EmailHelper = require('../Helpers/EmailHelper')
+const { InvalidEmailError } = require('../Errors/Errors')
 
 async function removeUserFromGroup(subscriptionId, userIdToRemove) {
   await SubscriptionUpdater.promises.removeUserFromGroup(
@@ -328,6 +334,125 @@ async function upgradeGroupPlan(ownerId) {
   )
 }
 
+async function updateGroupMembersBulk(
+  inviterId,
+  subscriptionId,
+  emailList,
+  options = {}
+) {
+  const { removeMembersNotIncluded, commit } = options
+
+  // remove duplications and empty values
+  emailList = _.uniq(_.compact(emailList))
+
+  const invalidEmails = emailList.filter(
+    email => !EmailHelper.parseEmail(email)
+  )
+
+  if (invalidEmails.length > 0) {
+    throw new InvalidEmailError('email not valid', {
+      invalidEmails,
+    })
+  }
+
+  const subscription = await Subscription.findOne({
+    _id: subscriptionId,
+  }).exec()
+
+  const existingUserData = await User.find(
+    {
+      _id: { $in: subscription.member_ids },
+    },
+    { _id: 1, email: 1, 'emails.email': 1 }
+  ).exec()
+
+  const existingUsers = existingUserData.map(user => ({
+    _id: user._id,
+    emails: user.emails?.map(user => user.email),
+  }))
+
+  const currentMemberEmails = _.flatten(
+    existingUsers
+      .filter(userData => userData.emails?.length > 0)
+      .map(user => user.emails)
+  )
+
+  const currentInvites =
+    subscription.teamInvites?.map(invite => invite.email) || []
+  if (subscription.invited_emails?.length > 0) {
+    currentInvites.push(...subscription.invited_emails)
+  }
+
+  const invitesToSend = _.difference(
+    emailList,
+    currentMemberEmails.concat(currentInvites)
+  )
+
+  let membersToRemove
+  let invitesToRevoke
+  let newTotalCount
+
+  if (!removeMembersNotIncluded) {
+    membersToRemove = []
+    invitesToRevoke = []
+    newTotalCount =
+      existingUsers.length + currentInvites.length + invitesToSend.length
+  } else {
+    membersToRemove = []
+    for (const existingUser of existingUsers) {
+      if (_.intersection(existingUser.emails, emailList).length === 0) {
+        membersToRemove.push(existingUser._id)
+      }
+    }
+    const invitesToMaintain = _.intersection(emailList, currentInvites)
+    invitesToRevoke = _.difference(currentInvites, invitesToMaintain)
+    newTotalCount =
+      existingUsers.length -
+      membersToRemove.length +
+      invitesToMaintain.length +
+      invitesToSend.length
+  }
+
+  const result = {
+    emailsToSendInvite: invitesToSend,
+    emailsToRevokeInvite: invitesToRevoke,
+    membersToRemove,
+    currentMemberCount: existingUsers.length,
+    newTotalCount,
+    membersLimit: subscription.membersLimit,
+  }
+
+  if (commit) {
+    if (newTotalCount > subscription.membersLimit) {
+      const { currentMemberCount, newTotalCount, membersLimit } = result
+      throw new OError('limit reached', {
+        currentMemberCount,
+        newTotalCount,
+        membersLimit,
+      })
+    }
+    for (const email of invitesToSend) {
+      await TeamInvitesHandler.promises.createInvite(
+        inviterId,
+        subscription,
+        email
+      )
+    }
+    for (const email of invitesToRevoke) {
+      await TeamInvitesHandler.promises.revokeInvite(
+        inviterId,
+        subscription,
+        email
+      )
+    }
+    for (const user of membersToRemove) {
+      await removeUserFromGroup(subscription._id, user._id)
+    }
+  }
+
+  return result
+}
+
 module.exports = {
   removeUserFromGroup: callbackify(removeUserFromGroup),
   replaceUserReferencesInGroups: callbackify(replaceUserReferencesInGroups),
@@ -344,6 +469,7 @@ module.exports = {
   getGroupPlanUpgradePreview: callbackify(getGroupPlanUpgradePreview),
   upgradeGroupPlan: callbackify(upgradeGroupPlan),
   checkBillingInfoExistence: callbackify(checkBillingInfoExistence),
+  updateGroupMembersBulk: callbackify(updateGroupMembersBulk),
   promises: {
     removeUserFromGroup,
     replaceUserReferencesInGroups,
@@ -360,5 +486,6 @@ module.exports = {
     getGroupPlanUpgradePreview,
     upgradeGroupPlan,
     checkBillingInfoExistence,
+    updateGroupMembersBulk,
   },
 }

services/web/scripts/add_subscription_members_csv.mjs

@@ -0,0 +1,202 @@
+import fs from 'node:fs'
+import minimist from 'minimist'
+import { parse } from 'csv'
+import Stream from 'node:stream/promises'
+import SubscriptionGroupHandler from '../app/src/Features/Subscription/SubscriptionGroupHandler.js'
+import { Subscription } from '../app/src/models/Subscription.js'
+import { InvalidEmailError } from '../app/src/Features/Errors/Errors.js'
+
+function usage() {
+  console.log(
+    'Usage: node scripts/add_subscription_members_csv.mjs -f <filename> -i <inviter_id> -s <subscription_id> [options]'
+  )
+  console.log('Required arguments:')
+  console.log(
+    '  -s, --subscriptionId <id>        The ID of the subscription to update'
+  )
+  console.log(
+    '  -i, --inviterId <id>             The ID of the user sending the invites'
+  )
+  console.log(
+    '  -f, --filename <filename>        The path to the file to read data from'
+  )
+  console.log('Options:')
+  console.log(
+    '  --commit, -c                     Whether changes should be committed to the DB invites should be sent/revoked'
+  )
+  console.log(
+    '  --removeMembersNotIncluded -r    Remove members that are not in the CSV. Disabled when managed users are enabled for the subscription'
+  )
+  console.log(
+    '  --verbose, -v                    Prints detailed information about the affected group members'
+  )
+  console.log('  -h, --help                       Show this help message')
+  process.exit(0)
+}
+
+let {
+  commit,
+  removeMembersNotIncluded,
+  inviterId,
+  subscriptionId,
+  filename,
+  help,
+  verbose,
+} = minimist(process.argv.slice(2), {
+  string: ['filename', 'subscriptionId', 'inviterId'],
+  boolean: ['commit', 'removeMembersNotIncluded', 'help', 'verbose'],
+  alias: {
+    commit: 'c',
+    removeMembersNotIncluded: 'r',
+    filename: 'f',
+    help: 'h',
+    inviterId: 'i',
+    subscriptionId: 's',
+    verbose: 'v',
+  },
+  default: {
+    commit: false,
+    removeMembersNotIncluded: false,
+    help: false,
+    verbose: false,
+  },
+})
+
+const EMAIL_FIELD = 'email'
+
+if (help) {
+  usage()
+  process.exit(0)
+}
+
+if (!subscriptionId || !inviterId || !filename) {
+  usage()
+  process.exit(1)
+}
+
+async function processRows(rows) {
+  const emailList = []
+  for await (const row of rows) {
+    const email = row[EMAIL_FIELD]
+    if (email) {
+      emailList.push(email)
+    }
+  }
+  if (emailList.length === 0) {
+    console.error(`CSV error: 'email' column doesn't exist or it's empty'`)
+    process.exit(1)
+  }
+
+  let previewResult
+
+  try {
+    previewResult =
+      await SubscriptionGroupHandler.promises.updateGroupMembersBulk(
+        inviterId,
+        subscriptionId,
+        emailList,
+        { removeMembersNotIncluded }
+      )
+  } catch (error) {
+    if (error instanceof InvalidEmailError) {
+      console.error(`${filename} contains invalid email addresses:`)
+      console.error(error.info?.invalidEmails.join(','))
+      process.exit(1)
+    } else {
+      throw error
+    }
+  }
+
+  console.log('Result Preview:')
+  logResult(previewResult)
+
+  if (previewResult.newTotalCount > previewResult.membersLimit) {
+    console.warn(
+      'WARNING: the invite list has reached the membership limit (newTotalCount > membersLimit)'
+    )
+    if (commit) {
+      console.error(`Invites won't be sent and users won't be deleted`)
+    }
+    process.exit(1)
+  }
+
+  if (!commit) {
+    console.log(
+      'this is a dry-run, use the --commit option to send the invite and make any DB changes'
+    )
+    return
+  }
+
+  console.log(
+    `Sending invites to ${previewResult.emailsToSendInvite.length} email addresses`
+  )
+
+  if (previewResult.membersToRemove > 0) {
+    console.log(
+      `${previewResult.membersToRemove.length} members will be removed from the group`
+    )
+  }
+
+  const commitResult =
+    await SubscriptionGroupHandler.promises.updateGroupMembersBulk(
+      inviterId,
+      subscriptionId,
+      emailList,
+      { removeMembersNotIncluded, commit }
+    )
+
+  console.log('Result:')
+  logResult(commitResult)
+}
+
+function logResult(result) {
+  console.log(
+    JSON.stringify(
+      {
+        ...result,
+        emailsToSendInvite: verbose
+          ? result.emailsToSendInvite
+          : result.emailsToSendInvite.length,
+        membersToRemove: verbose
+          ? result.membersToRemove
+          : result.membersToRemove.length,
+        emailsToRevokeInvite: verbose
+          ? result.emailsToRevokeInvite
+          : result.emailsToRevokeInvite.length,
+      },
+      null,
+      2
+    )
+  )
+}
+
+async function main() {
+  const subscription = await Subscription.findOne({
+    _id: subscriptionId,
+  }).exec()
+  if (!subscription) {
+    console.error(`subscription with id=${subscriptionId} not found`)
+    process.exit(1)
+  }
+  if (subscription.managedUsersEnabled && removeMembersNotIncluded) {
+    console.warn(
+      `subscription with id=${subscriptionId} has 'managedUsersEnabled=true'` +
+        `'--removeMembersNotIncluded' has been disabled`
+    )
+    removeMembersNotIncluded = false
+  }
+  await Stream.pipeline(
+    fs.createReadStream(filename),
+    parse({
+      columns: true,
+    }),
+    processRows
+  )
+}
+
+main()
+  .then(() => process.exit(0))
+  .catch(error => {
+    console.error(error)
+    process.exit(1)
+  })

services/web/test/unit/src/Subscription/SubscriptionGroupHandlerTests.js

@@ -1,7 +1,11 @@
 const SandboxedModule = require('sandboxed-module')
+const { ObjectId } = require('mongodb-legacy')
 const sinon = require('sinon')
 const { expect } = require('chai')
 const MockRequest = require('../helpers/MockRequest')
+const {
+  InvalidEmailError,
+} = require('../../../../app/src/Features/Errors/Errors')
 const modulePath =
   '../../../../app/src/Features/Subscription/SubscriptionGroupHandler'
 
@@ -27,6 +31,7 @@ describe('SubscriptionGroupHandler', function () {
       admin_id: this.adminUser_id,
       manager_ids: [this.adminUser_id],
       _id: this.subscription_id,
+      membersLimit: 100,
     }
 
     this.changeRequest = {
@@ -109,6 +114,10 @@ describe('SubscriptionGroupHandler', function () {
       findOne: sinon.stub().returns({ exec: sinon.stub().resolves }),
     }
 
+    this.User = {
+      find: sinon.stub().returns({ exec: sinon.stub().resolves }),
+    }
+
     this.SessionManager = {
       getLoggedInUserId: sinon.stub().returns(this.user._id),
     }
@@ -152,6 +161,13 @@ describe('SubscriptionGroupHandler', function () {
       },
     }
 
+    this.TeamInvitesHandler = {
+      promises: {
+        revokeInvite: sinon.stub().resolves(),
+        createInvite: sinon.stub().resolves(),
+      },
+    }
+
     this.GroupPlansData = {
       enterprise: {
         collaborator: {
@@ -194,9 +210,13 @@ describe('SubscriptionGroupHandler', function () {
         './SubscriptionLocator': this.SubscriptionLocator,
         './SubscriptionController': this.SubscriptionController,
         './SubscriptionHandler': this.SubscriptionHandler,
+        './TeamInvitesHandler': this.TeamInvitesHandler,
         '../../models/Subscription': {
           Subscription: this.Subscription,
         },
+        '../../models/User': {
+          User: this.User,
+        },
         './RecurlyClient': this.RecurlyClient,
         './PlansLocator': this.PlansLocator,
         './PaymentProviderEntities': this.PaymentProviderEntities,
@@ -861,4 +881,329 @@ describe('SubscriptionGroupHandler', function () {
       this.RecurlyClient.promises.getPaymentMethod.should.not.have.been.called
     })
   })
+
+  describe('updateGroupMembersBulk', function () {
+    const inviterId = new ObjectId()
+
+    let members
+    let emailList
+    let callUpdateGroupMembersBulk
+
+    beforeEach(function () {
+      members = [
+        {
+          _id: new ObjectId(),
+          email: 'user1@example.com',
+          emails: [{ email: 'user1@example.com' }],
+        },
+        {
+          _id: new ObjectId(),
+          email: 'user2-alias@example.com',
+          emails: [
+            {
+              email: 'user2-alias@example.com',
+            },
+            {
+              email: 'user2@example.com',
+            },
+          ],
+        },
+        {
+          _id: new ObjectId(),
+          email: 'user3@example.com',
+          emails: [{ email: 'user3@example.com' }],
+        },
+      ]
+
+      emailList = [
+        'user1@example.com',
+        'user2@example.com',
+        'new-user@example.com', // primary email of existing user
+        'new-user-2@example.com', // secondary email of existing user
+      ]
+      callUpdateGroupMembersBulk = async (options = {}) => {
+        this.Subscription.findOne = sinon
+          .stub()
+          .returns({ exec: sinon.stub().resolves(this.subscription) })
+
+        this.User.find = sinon
+          .stub()
+          .returns({ exec: sinon.stub().resolves(members) })
+
+        return await this.Handler.promises.updateGroupMembersBulk(
+          inviterId,
+          this.subscription._id,
+          emailList,
+          options
+        )
+      }
+    })
+
+    it('throws an error when any of the emails is invalid', async function () {
+      emailList.push('invalid@email')
+
+      await expect(
+        callUpdateGroupMembersBulk({ commit: true })
+      ).to.be.rejectedWith(InvalidEmailError)
+    })
+
+    describe('with commit = false', function () {
+      describe('with removeMembersNotIncluded = false', function () {
+        it('should preview zero users to delete, and should not send invites', async function () {
+          const result = await callUpdateGroupMembersBulk()
+
+          expect(result).to.deep.equal({
+            emailsToSendInvite: [
+              'new-user@example.com',
+              'new-user-2@example.com',
+            ],
+            emailsToRevokeInvite: [],
+            membersToRemove: [],
+            currentMemberCount: 3,
+            newTotalCount: 5,
+            membersLimit: this.subscription.membersLimit,
+          })
+
+          expect(this.TeamInvitesHandler.promises.createInvite).not.to.have.been
+            .called
+
+          expect(this.SubscriptionUpdater.promises.removeUserFromGroup).not.to
+            .have.been.called
+        })
+      })
+
+      describe('with removeMembersNotIncluded = true', function () {
+        it('should preview the users to be deleted, and should not send invites', async function () {
+          const result = await callUpdateGroupMembersBulk({
+            removeMembersNotIncluded: true,
+          })
+
+          expect(result).to.deep.equal({
+            emailsToSendInvite: [
+              'new-user@example.com',
+              'new-user-2@example.com',
+            ],
+            emailsToRevokeInvite: [],
+            membersToRemove: [members[2]._id],
+            currentMemberCount: 3,
+            newTotalCount: 4,
+            membersLimit: this.subscription.membersLimit,
+          })
+
+          expect(this.TeamInvitesHandler.promises.createInvite).not.to.have.been
+            .called
+
+          expect(this.SubscriptionUpdater.promises.removeUserFromGroup).not.to
+            .have.been.called
+        })
+
+        it('should preview but not revoke invites to emails that are no longer invited', async function () {
+          this.subscription.teamInvites = [
+            { email: 'new-user@example.com' },
+            { email: 'no-longer-invited@example.com' },
+          ]
+
+          const result = await callUpdateGroupMembersBulk({
+            removeMembersNotIncluded: true,
+          })
+
+          expect(result.emailsToRevokeInvite).to.deep.equal([
+            'no-longer-invited@example.com',
+          ])
+
+          expect(this.TeamInvitesHandler.promises.revokeInvite).not.to.have.been
+            .called
+        })
+      })
+
+      it('does not throw an error when the member limit is reached', async function () {
+        this.subscription.membersLimit = 3
+        const result = await callUpdateGroupMembersBulk()
+
+        expect(result.membersLimit).to.equal(3)
+        expect(result.newTotalCount).to.equal(5)
+      })
+    })
+
+    describe('with commit = true', function () {
+      describe('with removeMembersNotIncluded = false', function () {
+        it('should preview zero users to delete, and should send invites', async function () {
+          const result = await callUpdateGroupMembersBulk({ commit: true })
+
+          expect(result).to.deep.equal({
+            emailsToSendInvite: [
+              'new-user@example.com',
+              'new-user-2@example.com',
+            ],
+            emailsToRevokeInvite: [],
+            membersToRemove: [],
+            currentMemberCount: 3,
+            newTotalCount: 5,
+            membersLimit: this.subscription.membersLimit,
+          })
+
+          expect(this.SubscriptionUpdater.promises.removeUserFromGroup).not.to
+            .have.been.called
+
+          expect(
+            this.TeamInvitesHandler.promises.createInvite.callCount
+          ).to.equal(2)
+
+          expect(
+            this.TeamInvitesHandler.promises.createInvite
+          ).to.have.been.calledWith(
+            inviterId,
+            this.subscription,
+            'new-user@example.com'
+          )
+
+          expect(
+            this.TeamInvitesHandler.promises.createInvite
+          ).to.have.been.calledWith(
+            inviterId,
+            this.subscription,
+            'new-user-2@example.com'
+          )
+        })
+
+        it('should not send invites to emails already invited', async function () {
+          this.subscription.teamInvites = [{ email: 'new-user@example.com' }]
+
+          const result = await callUpdateGroupMembersBulk({ commit: true })
+
+          expect(result.emailsToSendInvite).to.deep.equal([
+            'new-user-2@example.com',
+          ])
+
+          expect(
+            this.TeamInvitesHandler.promises.createInvite.callCount
+          ).to.equal(1)
+
+          expect(
+            this.TeamInvitesHandler.promises.createInvite
+          ).to.have.been.calledWith(
+            inviterId,
+            this.subscription,
+            'new-user-2@example.com'
+          )
+        })
+
+        it('should preview and not revoke invites to emails that are no longer invited', async function () {
+          this.subscription.teamInvites = [
+            { email: 'new-user@example.com' },
+            { email: 'no-longer-invited@example.com' },
+          ]
+
+          const result = await callUpdateGroupMembersBulk({
+            commit: true,
+          })
+
+          expect(result.emailsToRevokeInvite).to.deep.equal([])
+
+          expect(this.TeamInvitesHandler.promises.revokeInvite).not.to.have.been
+            .called
+        })
+      })
+
+      describe('with removeMembersNotIncluded = true', function () {
+        it('should remove users from group, and should send invites', async function () {
+          const result = await callUpdateGroupMembersBulk({
+            commit: true,
+            removeMembersNotIncluded: true,
+          })
+
+          expect(result).to.deep.equal({
+            emailsToSendInvite: [
+              'new-user@example.com',
+              'new-user-2@example.com',
+            ],
+            emailsToRevokeInvite: [],
+            membersToRemove: [members[2]._id],
+            currentMemberCount: 3,
+            newTotalCount: 4,
+            membersLimit: this.subscription.membersLimit,
+          })
+
+          expect(
+            this.SubscriptionUpdater.promises.removeUserFromGroup.callCount
+          ).to.equal(1)
+
+          expect(
+            this.SubscriptionUpdater.promises.removeUserFromGroup
+          ).to.have.been.calledWith(this.subscription._id, members[2]._id)
+
+          expect(
+            this.TeamInvitesHandler.promises.createInvite.callCount
+          ).to.equal(2)
+
+          expect(
+            this.TeamInvitesHandler.promises.createInvite
+          ).to.have.been.calledWith(
+            inviterId,
+            this.subscription,
+            'new-user@example.com'
+          )
+
+          expect(
+            this.TeamInvitesHandler.promises.createInvite
+          ).to.have.been.calledWith(
+            inviterId,
+            this.subscription,
+            'new-user-2@example.com'
+          )
+        })
+
+        it('should send invites and revoke invites to emails no longer invited', async function () {
+          this.subscription.teamInvites = [
+            { email: 'new-user@example.com' },
+            { email: 'no-longer-invited@example.com' },
+          ]
+
+          const result = await callUpdateGroupMembersBulk({
+            commit: true,
+            removeMembersNotIncluded: true,
+          })
+
+          expect(result.emailsToSendInvite).to.deep.equal([
+            'new-user-2@example.com',
+          ])
+
+          expect(result.emailsToRevokeInvite).to.deep.equal([
+            'no-longer-invited@example.com',
+          ])
+
+          expect(
+            this.TeamInvitesHandler.promises.createInvite.callCount
+          ).to.equal(1)
+
+          expect(
+            this.TeamInvitesHandler.promises.createInvite
+          ).to.have.been.calledWith(
+            inviterId,
+            this.subscription,
+            'new-user-2@example.com'
+          )
+
+          expect(
+            this.TeamInvitesHandler.promises.revokeInvite.callCount
+          ).to.equal(1)
+
+          expect(
+            this.TeamInvitesHandler.promises.revokeInvite
+          ).to.have.been.calledWith(
+            inviterId,
+            this.subscription,
+            'no-longer-invited@example.com'
+          )
+        })
+      })
+
+      it('throws an error when the member limit is reached', async function () {
+        this.subscription.membersLimit = 3
+        await expect(
+          callUpdateGroupMembersBulk({ commit: true })
+        ).to.be.rejectedWith('limit reached')
+      })
+    })
+  })
 })