Merge pull request #32594 from overleaf/rh-cio-migration-mapping

Add customer.io fields for migration comms and marketing initiatives

GitOrigin-RevId: f11ffee255d9582cbfd4c7e285bd6690c0cf1e3c
This commit is contained in:
roo hutton
2026-04-16 12:30:59 +01:00
committed by Copybot
parent cae558af69
commit c02ba36b83
12 changed files with 2097 additions and 55 deletions

View File

@@ -7,5 +7,6 @@ module.exports = {
'require-script-runner': require('./require-script-runner'), 'require-script-runner': require('./require-script-runner'),
'require-vi-doMock-valid-path': require('./require-vi-doMock-valid-path'), 'require-vi-doMock-valid-path': require('./require-vi-doMock-valid-path'),
'require-loading-label': require('./require-loading-label'), 'require-loading-label': require('./require-loading-label'),
'require-cio-snake-case-properties': require('./require-cio-snake-case-properties'),
}, },
} }

View File

@@ -0,0 +1,111 @@
'use strict'
const SNAKE_CASE_RE = /^[a-z][a-z0-9]*(_[a-z0-9]+)*$/
function isSnakeCase(name) {
return SNAKE_CASE_RE.test(name)
}
function getStaticKeyName(property) {
if (property.computed) return null
if (property.key.type === 'Identifier') return property.key.name
if (property.key.type === 'Literal' && typeof property.key.value === 'string')
return property.key.value
return null
}
/**
* Check if a node is a call to CustomerIoHandler.updateUserAttributes()
* and return the attributes argument (2nd argument)
*/
function getUpdateUserAttributesArg(node) {
if (
node.callee.type === 'MemberExpression' &&
node.callee.object.type === 'Identifier' &&
node.callee.object.name === 'CustomerIoHandler' &&
node.callee.property.name === 'updateUserAttributes' &&
node.arguments[1]?.type === 'ObjectExpression'
) {
return node.arguments[1]
}
return null
}
/**
* Check if a node is a call to Modules[.promises].hooks.fire('setUserProperties', ...)
* and return the attributes argument (3rd argument)
*/
function getSetUserPropertiesArg(node) {
const callee = node.callee
if (callee.type !== 'MemberExpression' || callee.property.name !== 'fire') {
return null
}
// Check first argument is 'setUserProperties'
if (
!node.arguments[0] ||
node.arguments[0].type !== 'Literal' ||
node.arguments[0].value !== 'setUserProperties'
) {
return null
}
// Match: Modules.hooks.fire or Modules.promises.hooks.fire
const obj = callee.object
if (obj.type === 'MemberExpression' && obj.property.name === 'hooks') {
const parent = obj.object
// Modules.hooks
if (parent.type === 'Identifier' && parent.name === 'Modules') {
if (node.arguments[2]?.type === 'ObjectExpression') {
return node.arguments[2]
}
}
// Modules.promises.hooks
if (
parent.type === 'MemberExpression' &&
parent.property.name === 'promises' &&
parent.object.type === 'Identifier' &&
parent.object.name === 'Modules'
) {
if (node.arguments[2]?.type === 'ObjectExpression') {
return node.arguments[2]
}
}
}
return null
}
module.exports = {
meta: {
type: 'problem',
docs: {
description:
'Enforce snake_case for Customer.io user property attribute names',
},
},
create(context) {
return {
CallExpression(node) {
const attrsNode =
getUpdateUserAttributesArg(node) || getSetUserPropertiesArg(node)
if (!attrsNode) return
for (const property of attrsNode.properties) {
if (property.type === 'SpreadElement') continue
const keyName = getStaticKeyName(property)
if (keyName === null) continue // skip computed/dynamic keys
if (!isSnakeCase(keyName)) {
context.report({
node: property.key,
message: `Customer.io attribute '{{name}}' must be in snake_case.`,
data: { name: keyName },
})
}
}
},
}
},
}

View File

@@ -4,6 +4,7 @@ const noUnnecessaryTrans = require('./no-unnecessary-trans')
const shouldUnescapeTrans = require('./should-unescape-trans') const shouldUnescapeTrans = require('./should-unescape-trans')
const noGeneratedEditorThemes = require('./no-generated-editor-themes') const noGeneratedEditorThemes = require('./no-generated-editor-themes')
const viDoMockValidPath = require('./require-vi-doMock-valid-path') const viDoMockValidPath = require('./require-vi-doMock-valid-path')
const requireCioSnakeCaseProperties = require('./require-cio-snake-case-properties')
const ruleTester = new RuleTester({ const ruleTester = new RuleTester({
parser: require.resolve('@typescript-eslint/parser'), parser: require.resolve('@typescript-eslint/parser'),
@@ -166,3 +167,103 @@ ruleTester.run('domock-require-valid-path', viDoMockValidPath, {
}, },
], ],
}) })
ruleTester.run(
'require-cio-snake-case-properties',
requireCioSnakeCaseProperties,
{
valid: [
// updateUserAttributes with snake_case keys
{
code: `CustomerIoHandler.updateUserAttributes(userId, { plan_type: 'free', group_size: 10 })`,
},
// Modules.promises.hooks.fire with snake_case keys
{
code: `Modules.promises.hooks.fire('setUserProperties', userId, { plan_type: 'free', last_active: 123 })`,
},
// Modules.hooks.fire with snake_case keys
{
code: `Modules.hooks.fire('setUserProperties', userId, { plan_type: 'free' })`,
},
// Single-word keys are valid snake_case
{
code: `CustomerIoHandler.updateUserAttributes(userId, { email: 'a@b.com', role: 'admin' })`,
},
// Computed/dynamic keys are skipped
{
code: `CustomerIoHandler.updateUserAttributes(userId, { [dynamicKey]: true })`,
},
// Spread elements are skipped
{
code: `CustomerIoHandler.updateUserAttributes(userId, { ...existingAttrs })`,
},
// Unrelated function calls are not checked
{
code: `SomeOtherHandler.updateUserAttributes(userId, { camelCase: true })`,
},
// fire() with a different event name is not checked
{
code: `Modules.promises.hooks.fire('someOtherEvent', userId, { camelCase: true })`,
},
],
invalid: [
// camelCase key in updateUserAttributes
{
code: `CustomerIoHandler.updateUserAttributes(userId, { planType: 'free' })`,
errors: [
{
message: `Customer.io attribute 'planType' must be in snake_case.`,
},
],
},
// kebab-case string key
{
code: `CustomerIoHandler.updateUserAttributes(userId, { 'plan-type': 'free' })`,
errors: [
{
message: `Customer.io attribute 'plan-type' must be in snake_case.`,
},
],
},
// PascalCase key
{
code: `CustomerIoHandler.updateUserAttributes(userId, { PlanType: 'free' })`,
errors: [
{
message: `Customer.io attribute 'PlanType' must be in snake_case.`,
},
],
},
// camelCase in Modules.promises.hooks.fire
{
code: `Modules.promises.hooks.fire('setUserProperties', userId, { planType: 'free' })`,
errors: [
{
message: `Customer.io attribute 'planType' must be in snake_case.`,
},
],
},
// camelCase in Modules.hooks.fire
{
code: `Modules.hooks.fire('setUserProperties', userId, { planType: 'free' })`,
errors: [
{
message: `Customer.io attribute 'planType' must be in snake_case.`,
},
],
},
// Multiple invalid keys report multiple errors
{
code: `CustomerIoHandler.updateUserAttributes(userId, { planType: 'free', groupSize: 10, plan_term: 'annual' })`,
errors: [
{
message: `Customer.io attribute 'planType' must be in snake_case.`,
},
{
message: `Customer.io attribute 'groupSize' must be in snake_case.`,
},
],
},
],
}
)

View File

@@ -29,6 +29,7 @@ module.exports = {
'import/no-extraneous-dependencies': 'error', 'import/no-extraneous-dependencies': 'error',
'@overleaf/prefer-kebab-url': 'error', '@overleaf/prefer-kebab-url': 'error',
'@overleaf/require-cio-snake-case-properties': 'error',
// disable some TypeScript rules // disable some TypeScript rules
'@typescript-eslint/no-var-requires': 'off', '@typescript-eslint/no-var-requires': 'off',

View File

@@ -34,7 +34,7 @@ import { OnboardingDataCollection } from '../../models/OnboardingDataCollection.
import UserSettingsHelper from './UserSettingsHelper.mjs' import UserSettingsHelper from './UserSettingsHelper.mjs'
/** /**
* @import { GetProjectsRequest, GetProjectsResponse, AllUsersProjects, MongoProject, FormattedProject, MongoTag } from "./types" * @import { GetProjectsRequest, GetProjectsResponse, AllUsersProjects, MongoProject, FormattedProject, MongoTag, SubscriptionRecord } from "./types"
* @import { Project, ProjectApi, ProjectAccessLevel, Filters, Page, Sort, UserRef } from "../../../../types/project/dashboard/api" * @import { Project, ProjectApi, ProjectAccessLevel, Filters, Page, Sort, UserRef } from "../../../../types/project/dashboard/api"
* @import { Affiliation } from "../../../../types/affiliation" * @import { Affiliation } from "../../../../types/affiliation"
* @import { Source } from "../Authorization/types" * @import { Source } from "../Authorization/types"
@@ -554,26 +554,26 @@ async function projectListPage(req, res, next) {
Modules.promises.hooks Modules.promises.hooks
.fire('setUserProperties', userId, { .fire('setUserProperties', userId, {
overleafId: userId, overleaf_id: userId,
lastActive: user.lastActive last_active: user.lastActive
? Math.floor(user.lastActive.getTime() / 1000) ? Math.floor(user.lastActive.getTime() / 1000)
: null, : null,
signUpDate: user.signUpDate sign_up_date: user.signUpDate
? Math.floor(user.signUpDate.getTime() / 1000) ? Math.floor(user.signUpDate.getTime() / 1000)
: null, : null,
...(usersBestSubscription?.type && { ...(usersBestSubscription?.type && {
'best-subscription-type': usersBestSubscription.type, best_subscription_type: usersBestSubscription.type,
}), }),
aiBlocked, ai_blocked: aiBlocked,
hasAiAssist, has_ai_assist: hasAiAssist,
...(subjectArea && { subjectArea }), ...(subjectArea && { subject_area: subjectArea }),
...(role && { role }), ...(role && { role }),
...(primaryOccupation && { primaryOccupation }), ...(primaryOccupation && { primary_occupation: primaryOccupation }),
...(usedLatex && { usedLatex }), ...(usedLatex && { used_latex: usedLatex }),
...(countryCode && { country: countryCode }), ...(countryCode && { country: countryCode }),
...(commonsInstitution && { commonsInstitution }), ...(commonsInstitution && { commons_institution: commonsInstitution }),
...(groupRole && { groupRole }), ...(groupRole && { group_role: groupRole }),
isManagedUser: Boolean(user.enrollment?.managedBy), is_managed_user: Boolean(user.enrollment?.managedBy),
...(user.email && { email: user.email }), ...(user.email && { email: user.email }),
}) })
.catch(err => { .catch(err => {

View File

@@ -0,0 +1,408 @@
import Settings from '@overleaf/settings'
import { AI_ADD_ON_CODE, isStandaloneAiAddOnPlanCode } from './AiHelper.mjs'
import FeaturesHelper from './FeaturesHelper.mjs'
const INACTIVE_NEXT_RENEWAL_DATE_STATES = new Set([
'canceled',
'cancelled',
'expired',
])
const PENDING_CANCELLATION_STATES = new Set(['canceled', 'cancelled'])
function getSubscriptionState(subscription) {
return (
subscription?.recurlyStatus?.state || subscription?.paymentProvider?.state
)
}
function toUnixTimestamp(dateValue) {
if (!dateValue) {
return null
}
const date = new Date(dateValue)
if (Number.isNaN(date.getTime())) {
return null
}
return Math.floor(date.getTime() / 1000)
}
function normalizePlanType(bestSubscription) {
if (!bestSubscription) {
return null
}
if (['standalone-ai-add-on', 'commons'].includes(bestSubscription.type)) {
return bestSubscription.type
}
const planCode = bestSubscription.plan?.planCode
const isGroupPlan = bestSubscription.plan?.groupPlan === true
if (!planCode) {
return bestSubscription.type || null
}
if (planCode.startsWith('v1_')) {
return 'v1'
}
if (planCode.includes('student')) {
return 'student'
}
if (planCode.includes('professional')) {
return isGroupPlan ? 'group-professional' : 'professional'
}
if (planCode.includes('collaborator')) {
return isGroupPlan ? 'group-standard' : 'standard'
}
if (planCode.includes('personal')) {
return 'personal'
}
if (isGroupPlan) {
return 'group-standard'
}
return planCode
}
function getFriendlyPlanName(planType) {
if (!planType) {
return null
}
const friendlyPlanNames = {
free: 'Free',
personal: 'Personal',
standard: 'Standard',
professional: 'Pro',
student: 'Student',
commons: 'Commons',
'group-standard': 'Group Standard',
'group-professional': 'Group Pro',
'standalone-ai-add-on': 'AI Assist add-on',
v1: 'Legacy',
}
if (friendlyPlanNames[planType]) {
return friendlyPlanNames[planType]
}
return planType
}
function getPlanCadence(bestSubscription) {
if (!bestSubscription?.plan) {
return null
}
return bestSubscription.plan.annual ? 'annual' : 'monthly'
}
function getPlanCadenceFromPlanCode(planCode) {
if (!planCode) {
return null
}
const plan = Settings.plans.find(candidate => candidate.planCode === planCode)
if (plan) {
return plan.annual ? 'annual' : 'monthly'
}
if (planCode.includes('annual')) {
return 'annual'
}
if (isStandaloneAiAddOnPlanCode(planCode)) {
return 'monthly'
}
return null
}
function getNextRenewalDateFromPaymentRecord(paymentRecord) {
const subscriptionState = paymentRecord?.subscription?.state
if (INACTIVE_NEXT_RENEWAL_DATE_STATES.has(subscriptionState)) {
return null
}
return toUnixTimestamp(paymentRecord?.subscription?.periodEnd)
}
function shouldClearNextRenewalDate(subscription) {
if (!subscription) {
return true
}
return INACTIVE_NEXT_RENEWAL_DATE_STATES.has(
getSubscriptionState(subscription)
)
}
function getExpiryDateFromPaymentRecord(paymentRecord) {
const subscriptionState = paymentRecord?.subscription?.state
if (!PENDING_CANCELLATION_STATES.has(subscriptionState)) {
return null
}
const expiryDate = toUnixTimestamp(paymentRecord?.subscription?.periodEnd)
if (expiryDate == null) {
return null
}
return expiryDate > Math.floor(Date.now() / 1000) ? expiryDate : null
}
function shouldClearExpiryDate(subscription) {
if (!subscription) {
return true
}
return !PENDING_CANCELLATION_STATES.has(getSubscriptionState(subscription))
}
function hasIndividualAiAssistAddOn(individualSubscription, paymentRecord) {
if (
!individualSubscription ||
individualSubscription.groupPlan ||
isStandaloneAiAddOnPlanCode(individualSubscription.planCode)
) {
return false
}
return Boolean(
paymentRecord?.subscription?.addOns?.some(
addOn => addOn.code === AI_ADD_ON_CODE
)
)
}
function getAiPlanType(
bestSubscription,
individualSubscription,
paymentRecord,
writefullData
) {
if (
bestSubscription?.type === 'standalone-ai-add-on' ||
isStandaloneAiAddOnPlanCode(individualSubscription?.planCode)
) {
return 'ai-assist-standalone'
}
if (hasIndividualAiAssistAddOn(individualSubscription, paymentRecord)) {
return 'ai-assist-add-on'
}
if (writefullData?.isPremium) {
return 'writefull-premium'
}
return 'none'
}
function getAiPlanCadence(
aiPlan,
bestSubscription,
individualSubscription,
paymentRecord
) {
if (aiPlan === 'ai-assist-standalone') {
return (
getPlanCadenceFromPlanCode(individualSubscription?.planCode) ||
getPlanCadenceFromPlanCode(paymentRecord?.subscription?.planCode)
)
}
if (aiPlan === 'ai-assist-add-on') {
return (
getPlanCadence(bestSubscription) ||
getPlanCadenceFromPlanCode(individualSubscription?.planCode)
)
}
return null
}
function hasPlanAiEnabled(plan) {
if (!plan?.features) {
return false
}
return (
plan.features.aiUsageQuota === Settings.aiFeatures.unlimitedQuota ||
plan.features.aiErrorAssistant === true
)
}
function getGroupAiEnabled(
memberGroupSubscriptions = [],
managedGroupSubscriptions = [],
userIsMemberOfGroupSubscription
) {
if (!userIsMemberOfGroupSubscription) {
return null
}
const allGroupSubscriptions = [
...memberGroupSubscriptions,
...managedGroupSubscriptions,
]
return allGroupSubscriptions.some(subscription => {
const plan = Settings.plans.find(
candidate => candidate.planCode === subscription.planCode
)
return hasPlanAiEnabled(plan)
})
}
function getGroupSize(
bestSubscription,
memberGroupSubscriptions = [],
managedGroupSubscriptions = []
) {
const allGroupSubscriptions = [
...memberGroupSubscriptions,
...managedGroupSubscriptions,
]
if (allGroupSubscriptions.length === 0) {
return null
}
const matchingBestGroupSubscription =
bestSubscription?.type === 'group'
? allGroupSubscriptions.find(
subscription =>
subscription.planCode === bestSubscription.plan?.planCode &&
subscription.teamName === bestSubscription.subscription?.teamName
)
: null
if (matchingBestGroupSubscription?.membersLimit != null) {
return matchingBestGroupSubscription.membersLimit
}
if (bestSubscription?.subscription?.membersLimit != null) {
return bestSubscription.subscription.membersLimit
}
if (
bestSubscription?.plan?.groupPlan &&
bestSubscription.plan.membersLimit != null
) {
return bestSubscription.plan.membersLimit
}
return allGroupSubscriptions.reduce((largestGroupSize, subscription) => {
const plan = Settings.plans.find(
candidate => candidate.planCode === subscription.planCode
)
const groupSize = subscription.membersLimit ?? plan?.membersLimit ?? 0
return Math.max(largestGroupSize, groupSize)
}, 0)
}
function shouldUseCommonsBestSubscription(
hasCommons,
bestSubscription,
commonsPlan
) {
if (!hasCommons) {
return false
}
if (bestSubscription == null) {
return true
}
return FeaturesHelper.isFeatureSetBetter(
commonsPlan?.features || {},
bestSubscription.plan?.features || {}
)
}
/**
* Compute plan-related user properties for sending to customer.io.
*/
function getPlanProperties({
bestSubscription,
individualSubscription,
individualPaymentRecord,
memberGroupSubscriptions,
managedGroupSubscriptions,
userIsMemberOfGroupSubscription,
writefullData,
}) {
const planType = normalizePlanType(bestSubscription)
const displayPlanType = getFriendlyPlanName(planType)
const planTermLabel = getPlanCadence(bestSubscription)
const aiPlan = getAiPlanType(
bestSubscription,
individualSubscription,
individualPaymentRecord,
writefullData
)
const aiPlanTermLabel = getAiPlanCadence(
aiPlan,
bestSubscription,
individualSubscription,
individualPaymentRecord
)
const groupAiEnabled = getGroupAiEnabled(
memberGroupSubscriptions,
managedGroupSubscriptions,
userIsMemberOfGroupSubscription
)
const nextRenewalDate = getNextRenewalDateFromPaymentRecord(
individualPaymentRecord
)
const expiryDate = getExpiryDateFromPaymentRecord(individualPaymentRecord)
const groupSizeValue = getGroupSize(
bestSubscription,
memberGroupSubscriptions,
managedGroupSubscriptions
)
const nextRenewalDateTrait =
nextRenewalDate ??
(shouldClearNextRenewalDate(individualSubscription) ? '' : undefined)
const expiryDateTrait =
expiryDate ??
(shouldClearExpiryDate(individualSubscription) ? '' : undefined)
const properties = {
ai_plan: aiPlan,
}
if (planType) properties.plan_type = planType
if (displayPlanType) properties.display_plan_type = displayPlanType
if (planTermLabel) properties.plan_term_label = planTermLabel
if (aiPlanTermLabel) properties.ai_plan_term_label = aiPlanTermLabel
if (groupAiEnabled !== null) properties.group_ai_enabled = groupAiEnabled
if (nextRenewalDateTrait !== undefined)
properties.next_renewal_date = nextRenewalDateTrait
if (expiryDateTrait !== undefined) properties.expiry_date = expiryDateTrait
if (groupSizeValue !== null) properties.group_size = groupSizeValue
return properties
}
export default {
normalizePlanType,
getFriendlyPlanName,
getNextRenewalDateFromPaymentRecord,
getExpiryDateFromPaymentRecord,
getAiPlanType,
getAiPlanCadence,
hasPlanAiEnabled,
shouldUseCommonsBestSubscription,
getPlanProperties,
}

View File

@@ -16,6 +16,7 @@ import AnalyticsManager from '../Analytics/AnalyticsManager.mjs'
import Queues from '../../infrastructure/Queues.mjs' import Queues from '../../infrastructure/Queues.mjs'
import Modules from '../../infrastructure/Modules.mjs' import Modules from '../../infrastructure/Modules.mjs'
import SubscriptionViewModelBuilder from './SubscriptionViewModelBuilder.mjs' import SubscriptionViewModelBuilder from './SubscriptionViewModelBuilder.mjs'
import CustomerIoPlanHelpers from './CustomerIoPlanHelpers.mjs'
import { AI_ADD_ON_CODE } from './AiHelper.mjs' import { AI_ADD_ON_CODE } from './AiHelper.mjs'
import { fetchNothing } from '@overleaf/fetch-utils' import { fetchNothing } from '@overleaf/fetch-utils'
import SplitTestHandler from '../SplitTests/SplitTestHandler.mjs' import SplitTestHandler from '../SplitTests/SplitTestHandler.mjs'
@@ -58,18 +59,12 @@ async function refreshFeatures(userId, reason) {
const { features: newFeatures, featuresChanged } = const { features: newFeatures, featuresChanged } =
await UserFeaturesUpdater.promises.updateFeatures(userId, features) await UserFeaturesUpdater.promises.updateFeatures(userId, features)
const bestSubscriptionType = await _getBestSubscriptionType(userId) _updateCustomerIoSubscriptionProperties(user, features).catch(err => {
logger.warn(
Modules.promises.hooks { err, userId },
.fire('setUserProperties', userId, { 'Failed to update subscription properties in customer.io'
features, )
'best-subscription-type': bestSubscriptionType, })
overleafId: userId,
...(user.email && { email: user.email }),
})
.catch(err => {
logger.error({ err, userId }, 'Failed to sync features to customer.io')
})
if (oldFeatures.dropbox === true && features.dropbox === false) { if (oldFeatures.dropbox === true && features.dropbox === false) {
logger.debug({ userId }, '[FeaturesUpdater] must unlink dropbox') logger.debug({ userId }, '[FeaturesUpdater] must unlink dropbox')
@@ -133,20 +128,61 @@ async function refreshFeatures(userId, reason) {
return { features: newFeatures, featuresChanged } return { features: newFeatures, featuresChanged }
} }
async function _getBestSubscriptionType(userId) { async function _updateCustomerIoSubscriptionProperties(user, features) {
try { const userId = user._id
const { bestSubscription } = const {
await SubscriptionViewModelBuilder.promises.getUsersSubscriptionDetails({ bestSubscription,
_id: userId, individualSubscription,
}) memberGroupSubscriptions,
return bestSubscription?.type || 'free' managedGroupSubscriptions,
} catch (err) { } = await SubscriptionViewModelBuilder.promises.getUsersSubscriptionDetails({
logger.warn( _id: userId,
{ err, userId }, })
'Failed to calculate best-subscription-type for customer.io'
) const userIsMemberOfGroupSubscription =
return 'free' memberGroupSubscriptions.length > 0 || managedGroupSubscriptions.length > 0
let individualPaymentRecord = null
if (individualSubscription && !individualSubscription.groupPlan) {
try {
;[individualPaymentRecord] = await Modules.promises.hooks.fire(
'getPaymentFromRecordPromise',
individualSubscription
)
} catch (error) {
logger.warn(
{ err: error, userId },
'Failed to load payment record for customer.io subscription properties'
)
}
} }
let writefullData = null
try {
writefullData = await UserGetter.promises.getWritefullData(userId)
} catch (error) {
logger.warn(
{ err: error, userId },
'Failed to load writefull data for customer.io subscription properties'
)
}
const planProperties = CustomerIoPlanHelpers.getPlanProperties({
bestSubscription,
individualSubscription,
individualPaymentRecord,
memberGroupSubscriptions,
managedGroupSubscriptions,
userIsMemberOfGroupSubscription,
writefullData,
})
await Modules.promises.hooks.fire('setUserProperties', userId, {
...planProperties,
features,
overleaf_id: userId,
...(user.email && { email: user.email }),
})
} }
/** /**

View File

@@ -0,0 +1,465 @@
#!/usr/bin/env node
/**
* This script reads a CSV file (output from migrate_mailchimp_to_cio.mjs or
* export_active_subscription_users_csv.mjs) and
* makes batch identify requests to Customer.io using the CDP Analytics node library.
*
* Usage:
* node scripts/batch_identify_to_cio.mjs --input INPUT-FILE [OPTIONS]
*
* Example:
* node scripts/batch_identify_to_cio.mjs --input /tmp/customerio_import.csv
* CUSTOMER_IO_API_KEY=xxx node scripts/batch_identify_to_cio.mjs --input /tmp/customerio_import.csv --commit
*
* Resuming after failure:
* CUSTOMER_IO_API_KEY=xxx node scripts/batch_identify_to_cio.mjs --input /tmp/customerio_import.csv --commit --skip 50000
*
* Options:
* --input, -i PATH Input CSV file (from migrate_mailchimp_to_cio.mjs) (required)
* --commit Actually send to Customer.io (default is dry-run)
* --skip N Skip the first N rows (for resuming after failure)
* --batch-size N Maximum items per batch (default: 1000)
* --help, -h Show this help message
*
* Environment Variables:
* CUSTOMER_IO_API_KEY Customer.io CDP API key (required when using --commit)
*
* CSV Input Format (from migrate_mailchimp_to_cio.mjs):
* - email: Subscriber email address
* - overleafId: Overleaf user ID (from mongo_id)
* - created_at: Unix timestamp
* - cio_subscription_preferences.topics.<topic_id>: 'true' if subscribed
* - labsExperiments: JSON array of experiment names
*
* CSV Input Format (from export_active_subscription_users_csv.mjs):
* - user_id: Overleaf user ID
* - email: Subscriber email address
* - plan_type, display_plan_type, pre_migration_plan_type,
* pre_migration_display_plan_type, plan_term, ai_plan, ai_plan_term,
* next_renewal_date, expiry_date, group_ai_enabled, group_role
*
* Rate Limiting:
* - Max 3000 requests per 3 seconds (Customer.io limit)
* - Progress is logged every 10000 rows for recovery purposes
*
* Notes:
* - Users are identified by email address (used as userId)
* - The library handles batching internally via maxEventsInBatch
*/
import fs, { createReadStream } from 'node:fs'
import * as csv from 'csv'
import minimist from 'minimist'
import { Analytics } from '@customerio/cdp-analytics-node'
import { scriptRunner } from './lib/ScriptRunner.mjs'
const DEFAULT_BATCH_SIZE = 1000
const RATE_LIMIT_REQUESTS = 3000
const RATE_LIMIT_WINDOW_MS = 3000
const PROGRESS_LOG_INTERVAL = 10000
function usage() {
console.error(`Usage: node scripts/batch_identify_to_cio.mjs --input INPUT-FILE [OPTIONS]
Options:
--input, -i PATH Input CSV file (from migrate_mailchimp_to_cio.mjs) (required)
--commit Actually send to Customer.io (default is dry-run)
--skip N Skip the first N rows (for resuming after failure)
--batch-size N Maximum items per batch (default: ${DEFAULT_BATCH_SIZE})
--help, -h Show this help message
Environment Variables:
CUSTOMER_IO_API_KEY Customer.io CDP API key (required when using --commit)
`)
process.exit(1)
}
/**
* Simple rate limiter that enforces max requests per time window
*/
class RateLimiter {
constructor(maxRequests, windowMs) {
this.maxRequests = maxRequests
this.windowMs = windowMs
this.requests = []
}
async waitIfNeeded() {
const now = Date.now()
// Remove requests outside the current window
this.requests = this.requests.filter(t => now - t < this.windowMs)
if (this.requests.length >= this.maxRequests) {
// Wait until the oldest request falls outside the window
const oldestRequest = this.requests[0]
const waitTime = this.windowMs - (now - oldestRequest) + 10 // +10ms buffer
await new Promise(resolve => setTimeout(resolve, waitTime))
// Clean up again after waiting
this.requests = this.requests.filter(t => Date.now() - t < this.windowMs)
}
this.requests.push(Date.now())
}
}
/**
* Create a Customer.io Analytics client
*/
function createCioClient(batchSize) {
const apiKey = process.env.CUSTOMER_IO_API_KEY
if (!apiKey) {
throw new Error(
'CUSTOMER_IO_API_KEY environment variable is required. ' +
'Set it to your Customer.io CDP API key.'
)
}
return new Analytics({
writeKey: apiKey,
host: 'https://cdp.customer.io',
maxEventsInBatch: batchSize,
})
}
/**
* Convert a CSV row to a Customer.io identify payload
*/
function parseOptionalBoolean(value) {
if (value == null || value === '') {
return undefined
}
const normalized = String(value).trim().toLowerCase()
if (normalized === 'true') {
return true
}
if (normalized === 'false') {
return false
}
return undefined
}
function parseOptionalInt(value) {
if (value == null || value === '') {
return undefined
}
const parsed = parseInt(value, 10)
return Number.isNaN(parsed) ? undefined : parsed
}
function getFirstDefinedValue(row, columnNames) {
for (const columnName of columnNames) {
if (Object.prototype.hasOwnProperty.call(row, columnName)) {
return row[columnName]
}
}
return undefined
}
function rowToIdentifyPayload(row) {
const email = getFirstDefinedValue(row, ['email'])
if (!email) {
return null
}
const traits = {}
const overleafUserId = getFirstDefinedValue(row, [
'user_id',
'userId',
'overleafId',
])
if (email) {
traits.email = email
}
if (overleafUserId) {
traits.overleaf_id = overleafUserId
}
// Add created_at if present (keep as unix timestamp)
const createdAtValue = getFirstDefinedValue(row, ['created_at'])
if (createdAtValue) {
const createdAt = parseOptionalInt(createdAtValue)
if (createdAt !== undefined) {
traits.created_at = createdAt
}
}
// Add subscription status fields when present (from export_active_subscription_users_csv.mjs)
const stringTraitMappings = [
{
columnNames: ['plan_type', 'planType'],
traitName: 'plan_type',
},
{
columnNames: ['display_plan_type', 'displayPlanType'],
traitName: 'display_plan_type',
},
{
columnNames: ['pre_migration_plan_type', 'preMigrationPlanType'],
traitName: 'pre_migration_plan_type',
},
{
columnNames: [
'pre_migration_display_plan_type',
'preMigrationDisplayPlanType',
],
traitName: 'pre_migration_display_plan_type',
},
{
columnNames: ['plan_term', 'planTerm', 'plan_term_label'],
traitName: 'plan_term',
},
{
columnNames: ['ai_plan', 'aiPlan'],
traitName: 'ai_plan',
},
{
columnNames: ['ai_plan_term', 'aiPlanTerm', 'ai_plan_term_label'],
traitName: 'ai_plan_term',
},
{
columnNames: ['group_role', 'groupRole'],
traitName: 'group_role',
},
]
for (const { columnNames, traitName } of stringTraitMappings) {
const value = getFirstDefinedValue(row, columnNames)
if (value) {
traits[traitName] = value
}
}
const nextRenewalDateValue = getFirstDefinedValue(row, [
'next_renewal_date',
'nextRenewalDate',
])
if (nextRenewalDateValue !== undefined) {
if (nextRenewalDateValue === '') {
traits.next_renewal_date = ''
} else {
const nextRenewalDate = parseOptionalInt(nextRenewalDateValue)
if (nextRenewalDate !== undefined) {
traits.next_renewal_date = nextRenewalDate
}
}
}
const expiryDateValue = getFirstDefinedValue(row, [
'expiry_date',
'expiryDate',
])
if (expiryDateValue !== undefined) {
if (expiryDateValue === '') {
traits.expiry_date = ''
} else {
const expiryDate = parseOptionalInt(expiryDateValue)
if (expiryDate !== undefined) {
traits.expiry_date = expiryDate
}
}
}
const groupAiEnabledValue = getFirstDefinedValue(row, [
'group_ai_enabled',
'groupAIEnabled',
])
if (groupAiEnabledValue !== undefined) {
const groupAiEnabled = parseOptionalBoolean(groupAiEnabledValue)
if (groupAiEnabled !== undefined) {
traits.group_ai_enabled = groupAiEnabled
}
}
// Add subscription preferences
for (const key of Object.keys(row)) {
if (key.startsWith('cio_subscription_preferences.topics.')) {
if (row[key] === 'true') {
traits[key] = true
}
}
}
// Add labsExperiments if present
if (row.labsExperiments) {
try {
traits.labsExperiments = JSON.parse(row.labsExperiments)
} catch {
// If it's not valid JSON, store as-is
traits.labsExperiments = row.labsExperiments
}
}
return {
// Prefer stable Overleaf user id when available, otherwise fall back to email
userId: overleafUserId || email,
email,
traits,
}
}
/**
* Create a CSV parser stream
*/
function createCsvParser(inputPath) {
return createReadStream(inputPath).pipe(
csv.parse({
columns: true,
skip_empty_lines: true,
relax_column_count: true,
})
)
}
/**
* Main script function
*/
function main() {
const argv = minimist(process.argv.slice(2), {
string: ['input', 'batch-size', 'skip'],
boolean: ['commit', 'help'],
alias: {
i: 'input',
h: 'help',
},
})
if (argv.help) {
usage()
}
const inputPath = argv.input
const commit = argv.commit
const dryRun = !commit
const batchSize = parseInt(argv['batch-size'], 10) || DEFAULT_BATCH_SIZE
const skipRows = parseInt(argv.skip, 10) || 0
if (!inputPath) {
console.error('Error: --input is required')
usage()
}
if (!fs.existsSync(inputPath)) {
console.error(`Error: Input file not found: ${inputPath}`)
process.exit(1)
}
if (commit && !process.env.CUSTOMER_IO_API_KEY) {
console.error(
'Error: CUSTOMER_IO_API_KEY environment variable is required when using --commit'
)
process.exit(1)
}
scriptRunner(
async trackProgress => {
await trackProgress('Starting batch identify to Customer.io...')
await trackProgress(`Input: ${inputPath}`)
await trackProgress(`Batch size limit: ${batchSize}`)
if (skipRows > 0) {
await trackProgress(`Skipping first ${skipRows} rows`)
}
if (dryRun) {
await trackProgress('DRY RUN MODE - no requests will be sent')
}
const rateLimiter = new RateLimiter(
RATE_LIMIT_REQUESTS,
RATE_LIMIT_WINDOW_MS
)
const client = dryRun ? null : createCioClient(batchSize)
// Listen to the 'error' event
if (!dryRun) {
client.on('error', err => {
console.error('cdp-analytics-node error occurred:')
console.error('Code:', err.code)
console.error('Reason:', err.reason)
if (err.ctx) {
console.error('Context:', err.ctx)
}
})
}
let rowNumber = 0
let processedCount = 0
let skippedCount = 0
let lastProgressLog = 0
let loggedFirstIdentifyPayload = false
const parser = createCsvParser(inputPath)
try {
for await (const row of parser) {
rowNumber++
// Skip rows if resuming
if (rowNumber <= skipRows) {
continue
}
const payload = rowToIdentifyPayload(row)
if (!payload) {
skippedCount++
continue
}
if (dryRun && !loggedFirstIdentifyPayload) {
await trackProgress(
`First identify payload (dry run): ${JSON.stringify(payload)}`
)
loggedFirstIdentifyPayload = true
}
if (!dryRun) {
await rateLimiter.waitIfNeeded()
client.identify(payload)
}
processedCount++
// Log progress periodically
if (processedCount - lastProgressLog >= PROGRESS_LOG_INTERVAL) {
await trackProgress(
`Progress: row ${rowNumber}, sent ${processedCount} requests (${skippedCount} skipped)`
)
lastProgressLog = processedCount
}
}
} catch (error) {
await trackProgress(
`ERROR at row ${rowNumber}: ${error.message}. Resume with --skip ${rowNumber - 1}`
)
throw error
}
if (!dryRun && client) {
await trackProgress('Flushing remaining requests...')
await client.closeAndFlush()
}
await trackProgress(
`Completed: processed ${processedCount} rows (${skippedCount} skipped due to missing email)`
)
if (dryRun) {
await trackProgress(
`DRY RUN complete - would have sent ${processedCount} identify requests`
)
}
process.exit(0)
},
{ inputPath, dryRun, batchSize, skipRows }
).catch(err => {
console.error(err)
process.exit(1)
})
}
main()

View File

@@ -0,0 +1,789 @@
import fs from 'node:fs'
import { Parser as CSVParser } from 'json2csv'
import minimist from 'minimist'
import pLimit from 'p-limit'
import Settings from '@overleaf/settings'
import { scriptRunner } from './lib/ScriptRunner.mjs'
import {
db,
ObjectId,
READ_PREFERENCE_SECONDARY,
} from '../app/src/infrastructure/mongodb.mjs'
import PaymentService from '../modules/subscriptions/app/src/PaymentService.mjs'
import FeaturesHelper from '../app/src/Features/Subscription/FeaturesHelper.mjs'
import CustomerIoPlanHelpers from '../app/src/Features/Subscription/CustomerIoPlanHelpers.mjs'
import { isStandaloneAiAddOnPlanCode } from '../app/src/Features/Subscription/AiHelper.mjs'
import InstitutionsGetter from '../app/src/Features/Institutions/InstitutionsGetter.mjs'
const CSV_FIELDS = [
{ label: 'user_id', value: 'userId' },
{ label: 'email', value: 'email' },
{ label: 'plan_type', value: 'planType' },
{ label: 'display_plan_type', value: 'displayPlanType' },
{
label: 'pre_migration_plan_type',
value: 'preMigrationPlanType',
},
{
label: 'pre_migration_display_plan_type',
value: 'preMigrationDisplayPlanType',
},
{ label: 'plan_term', value: 'planTerm' },
{ label: 'ai_plan', value: 'aiPlan' },
{ label: 'ai_plan_term', value: 'aiPlanTerm' },
{ label: 'next_renewal_date', value: 'nextRenewalDate' },
{ label: 'expiry_date', value: 'expiryDate' },
{ label: 'group_ai_enabled', value: 'groupAIEnabled' },
{ label: 'group_role', value: 'groupRole' },
]
const CSV_FIELD_NAMES = CSV_FIELDS.map(field => field.value)
const ACTIVE_SUBSCRIPTION_STATES = ['active', 'trialing']
const COMMONS_PLAN = getPlan(Settings.institutionPlanCode)
function usage() {
console.log(`
Usage:
node scripts/export_active_subscription_users_csv.mjs [options]
Options:
--outputPath <path> Output CSV path (default: /tmp/active_subscription_users.csv)
--concurrency <number> Concurrent payment-provider lookups (default: 5)
--batchSize <number> Number of users processed per batch (default: 500)
--resumeAfterUserId <id> Resume processing strictly after this user id
--checkpointInterval <n> Log resumable checkpoint every N processed users (default: 10000)
--append Append to existing output file (for resume runs)
--help Show this message
`)
}
function parseArgs() {
const args = minimist(process.argv.slice(2), {
string: ['outputPath', 'resumeAfterUserId'],
boolean: ['help', 'append'],
default: {
outputPath: '/tmp/active_subscription_users.csv',
concurrency: 5,
batchSize: 500,
checkpointInterval: 10000,
append: false,
help: false,
},
})
if (args.help) {
usage()
process.exit(0)
}
return args
}
function getPlan(planCode) {
return Settings.plans.find(plan => plan.planCode === planCode) || null
}
function getPlanType(subscription) {
if (isStandaloneAiAddOnPlanCode(subscription.planCode)) {
return 'standalone-ai-add-on'
}
return subscription.groupPlan ? 'group' : 'individual'
}
function getPlanCadence(subscription, plan) {
if (plan != null) {
return plan.annual ? 'annual' : 'monthly'
}
if (isStandaloneAiAddOnPlanCode(subscription.planCode)) {
return subscription.planCode.includes('annual') ? 'annual' : 'monthly'
}
return ''
}
function userHasPremiumAiFeatures(user) {
return (
user?.features?.aiErrorAssistant === true ||
user?.features?.aiUsageQuota === Settings.aiFeatures.unlimitedQuota
)
}
async function userHasCurrentInstitutionLicence(userId, commonsCache) {
if (commonsCache.has(userId)) {
return commonsCache.get(userId)
}
try {
const institutions =
await InstitutionsGetter.promises.getCurrentInstitutionsWithLicence(
userId
)
const hasCommons = Boolean(institutions?.length)
commonsCache.set(userId, hasCommons)
return hasCommons
} catch (error) {
console.warn(
`Failed to evaluate commons licence for user ${userId}: ${error.message}`
)
commonsCache.set(userId, false)
return false
}
}
function isMemberOfGroupSubscription(candidates) {
return candidates.some(candidate => candidate.subscription?.groupPlan)
}
function getAiPlanForUser({
bestSubscription,
planType,
individualSubscription,
paymentRecord,
user,
userIsMemberOfGroupSubscription,
userHasActiveOverleafSubscription,
}) {
const baseAiPlan = CustomerIoPlanHelpers.getAiPlanType(
bestSubscription,
individualSubscription,
paymentRecord,
user?.writefull,
userIsMemberOfGroupSubscription
)
if (baseAiPlan !== 'none') {
return baseAiPlan
}
if (!userHasActiveOverleafSubscription && userHasPremiumAiFeatures(user)) {
return 'ai-assist'
}
return 'none'
}
function getPaymentState(subscription) {
if (subscription?.recurlyStatus?.state) {
return subscription.recurlyStatus.state
}
if (subscription?.paymentProvider?.state) {
return subscription.paymentProvider.state
}
return null
}
function isActivePaidSubscription(subscription) {
const hasRecurlySubscription = Boolean(subscription?.recurlySubscription_id)
const hasStripeSubscription = Boolean(
subscription?.paymentProvider?.subscriptionId
)
if (!hasRecurlySubscription && !hasStripeSubscription) {
return false
}
return ACTIVE_SUBSCRIPTION_STATES.includes(getPaymentState(subscription))
}
function getGroupAiEnabled(candidates) {
const groupCandidates = candidates.filter(
candidate => getPlanType(candidate.subscription) === 'group'
)
if (groupCandidates.length === 0) {
return ''
}
return groupCandidates.some(candidate =>
CustomerIoPlanHelpers.hasPlanAiEnabled(candidate.plan)
)
}
function getGroupRole(candidates, userId) {
const groupCandidates = candidates.filter(
candidate => candidate.subscription?.groupPlan
)
if (groupCandidates.length === 0) {
return ''
}
const isGroupAdminOrManager = groupCandidates.some(candidate => {
const subscription = candidate.subscription
const adminId = subscription?.admin_id?.toString()
const managerIds = (subscription?.manager_ids || []).map(id =>
id?.toString()
)
return adminId === userId || managerIds.includes(userId)
})
return isGroupAdminOrManager ? 'admin' : 'member'
}
function chooseBestCandidate(candidates) {
let best = null
for (const candidate of candidates) {
if (best == null) {
best = candidate
continue
}
const candidateType = getPlanType(candidate.subscription)
const bestType = getPlanType(best.subscription)
if (candidateType === 'standalone-ai-add-on' && bestType !== 'free') {
continue
}
if (
candidateType !== 'standalone-ai-add-on' &&
bestType === 'standalone-ai-add-on'
) {
best = candidate
continue
}
if (
FeaturesHelper.isFeatureSetBetter(
candidate.plan?.features || {},
best.plan?.features || {}
)
) {
best = candidate
}
}
return best
}
function getSubscriptionQuery() {
return {
$or: [
{
recurlySubscription_id: { $exists: true, $nin: ['', null] },
'recurlyStatus.state': { $in: ACTIVE_SUBSCRIPTION_STATES },
},
{
'paymentProvider.subscriptionId': { $exists: true, $nin: ['', null] },
'paymentProvider.state': { $in: ACTIVE_SUBSCRIPTION_STATES },
},
],
}
}
function getSubscriptionProjection() {
return {
_id: 1,
admin_id: 1,
manager_ids: 1,
member_ids: 1,
planCode: 1,
groupPlan: 1,
recurlySubscription_id: 1,
recurlyStatus: 1,
paymentProvider: 1,
addOns: 1,
}
}
function getSupplementaryUserQuery() {
return {
$or: [
{ 'writefull.isPremium': true },
{ 'features.aiErrorAssistant': true },
{ 'features.aiUsageQuota': Settings.aiFeatures.unlimitedQuota },
{
emails: {
$elemMatch: {
confirmedAt: { $exists: true },
'affiliation.institution.confirmed': true,
'affiliation.licence': { $exists: true, $ne: 'free' },
'affiliation.pastReconfirmDate': { $ne: true },
},
},
},
],
}
}
function getTargetUserIdsPipeline(resumeAfterUserId) {
const pipeline = [
{ $match: getSubscriptionQuery() },
{
$project: {
participantIds: {
$setUnion: [
[{ $ifNull: ['$admin_id', null] }],
{ $ifNull: ['$manager_ids', []] },
{ $ifNull: ['$member_ids', []] },
],
},
},
},
{ $unwind: '$participantIds' },
{ $match: { participantIds: { $ne: null } } },
{ $group: { _id: '$participantIds' } },
{
$unionWith: {
coll: 'users',
pipeline: [
{ $match: getSupplementaryUserQuery() },
{ $project: { _id: 1 } },
],
},
},
{ $group: { _id: '$_id' } },
]
if (resumeAfterUserId) {
let resumeId = resumeAfterUserId
try {
resumeId = new ObjectId(resumeAfterUserId)
} catch {
// leave as-is for non-ObjectId identifiers
}
pipeline.push({ $match: { _id: { $gt: resumeId } } })
}
pipeline.push({ $sort: { _id: 1 } })
return pipeline
}
function getTargetUserIdsCursor(resumeAfterUserId) {
return db.subscriptions.aggregate(
getTargetUserIdsPipeline(resumeAfterUserId),
{
allowDiskUse: true,
readPreference: READ_PREFERENCE_SECONDARY,
}
)
}
function isInvalidOrMissingSubscriptionError(error) {
const message = String(error?.message || '').toLowerCase()
return (
message.includes('no such subscription') ||
message.includes('invalid subscription') ||
message.includes('subscription not found')
)
}
async function getSubscriptionPaymentInfo(
userId,
subscription,
paymentInfoCache
) {
const subscriptionId = subscription._id.toString()
if (paymentInfoCache.has(subscriptionId)) {
return {
...paymentInfoCache.get(subscriptionId),
skip: false,
}
}
try {
const paymentRecord =
await PaymentService.promises.getPaymentFromRecord(subscription)
const nextRenewalDate =
CustomerIoPlanHelpers.getNextRenewalDateFromPaymentRecord(
paymentRecord
) || ''
const paymentInfo = { nextRenewalDate, paymentRecord }
paymentInfoCache.set(subscriptionId, paymentInfo)
return { ...paymentInfo, skip: false }
} catch (error) {
if (isInvalidOrMissingSubscriptionError(error)) {
console.warn(
`Skipping user ${userId}: invalid/missing payment-provider subscription for subscription ${subscriptionId} (${error.message})`
)
return { nextRenewalDate: '', paymentRecord: null, skip: true }
}
console.error(
`Failed to get renewal date for subscription ${subscriptionId}:`,
error.message
)
const paymentInfo = { nextRenewalDate: '', paymentRecord: null }
paymentInfoCache.set(subscriptionId, paymentInfo)
return { ...paymentInfo, skip: false }
}
}
async function getUsersByIds(userIds) {
const objectIds = []
for (const userId of userIds) {
try {
objectIds.push(new ObjectId(userId))
} catch {
// ignore invalid ObjectId strings
}
}
const idFilters = []
if (objectIds.length > 0) {
idFilters.push({ _id: { $in: objectIds } })
}
if (userIds.length > 0) {
idFilters.push({ _id: { $in: userIds } })
}
if (idFilters.length === 0) {
return new Map()
}
const users = await db.users
.find(idFilters.length === 1 ? idFilters[0] : { $or: idFilters }, {
projection: {
_id: 1,
email: 1,
features: 1,
writefull: 1,
},
readPreference: READ_PREFERENCE_SECONDARY,
})
.toArray()
return new Map(users.map(user => [user._id.toString(), user]))
}
async function getActiveSubscriptionsForUserIds(userIds) {
const objectIds = []
for (const userId of userIds) {
try {
objectIds.push(new ObjectId(userId))
} catch {
// ignore invalid ObjectId strings
}
}
const idValues = objectIds.length > 0 ? objectIds : userIds
return await db.subscriptions
.find(
{
...getSubscriptionQuery(),
$or: [
{ admin_id: { $in: idValues } },
{ manager_ids: { $in: idValues } },
{ member_ids: { $in: idValues } },
],
},
{
projection: getSubscriptionProjection(),
readPreference: READ_PREFERENCE_SECONDARY,
}
)
.toArray()
}
function buildUserCandidatesMap(subscriptions, allowedUserIds) {
const userCandidates = new Map()
const allowedUserIdsSet = new Set(allowedUserIds)
function addCandidate(userId, candidate) {
if (!userId || !allowedUserIdsSet.has(userId)) {
return
}
if (!userCandidates.has(userId)) {
userCandidates.set(userId, [])
}
userCandidates.get(userId).push(candidate)
}
for (const subscription of subscriptions) {
const plan = getPlan(subscription.planCode)
const baseCandidate = { subscription, plan }
const adminId = subscription.admin_id?.toString()
addCandidate(adminId, baseCandidate)
if (Array.isArray(subscription.manager_ids)) {
for (const managerIdRaw of subscription.manager_ids) {
addCandidate(managerIdRaw?.toString(), baseCandidate)
}
}
if (subscription.groupPlan && Array.isArray(subscription.member_ids)) {
for (const memberIdRaw of subscription.member_ids) {
addCandidate(memberIdRaw?.toString(), baseCandidate)
}
}
}
return userCandidates
}
function writeCsvRows(writeStream, rows, includeHeader) {
if (rows.length === 0) {
return
}
const csvParser = new CSVParser({
fields: includeHeader ? CSV_FIELDS : CSV_FIELD_NAMES,
header: includeHeader,
eol: '\n',
})
writeStream.write(`${csvParser.parse(rows)}\n`)
}
function writeCsvHeader(writeStream) {
writeStream.write(`${CSV_FIELDS.map(field => field.label).join(',')}\n`)
}
async function processUserBatch({
userIds,
writeStream,
paymentInfoCache,
commonsCache,
limit,
trackProgress,
totalUsersCount,
globalState,
checkpointInterval,
}) {
const usersById = await getUsersByIds(userIds)
const subscriptions = await getActiveSubscriptionsForUserIds(userIds)
const activeSubscriptions = subscriptions.filter(isActivePaidSubscription)
const userCandidates = buildUserCandidatesMap(activeSubscriptions, userIds)
const rows = await Promise.all(
userIds.map(userId =>
limit(async () => {
const user = usersById.get(userId)
if (!user?.email) {
return null
}
const candidates = userCandidates.get(userId) || []
const bestCandidate = chooseBestCandidate(candidates)
const hasCommons = await userHasCurrentInstitutionLicence(
userId,
commonsCache
)
const commonsBeatsBestSubscription =
CustomerIoPlanHelpers.shouldUseCommonsBestSubscription(
hasCommons,
bestCandidate,
COMMONS_PLAN
)
const resolvedSubscription = commonsBeatsBestSubscription
? null
: bestCandidate?.subscription
const resolvedPlan = commonsBeatsBestSubscription
? COMMONS_PLAN
: bestCandidate?.plan
const bestSubscriptionForPlanType = commonsBeatsBestSubscription
? { type: 'commons', plan: COMMONS_PLAN }
: resolvedSubscription
? {
type: getPlanType(resolvedSubscription),
plan: resolvedPlan,
}
: { type: 'free' }
const planType = CustomerIoPlanHelpers.normalizePlanType(
bestSubscriptionForPlanType
)
const displayPlanType =
CustomerIoPlanHelpers.getFriendlyPlanName(planType) || ''
const planTerm = resolvedSubscription
? getPlanCadence(resolvedSubscription, resolvedPlan)
: ''
const userIsMemberOfGroupSubscription =
isMemberOfGroupSubscription(candidates)
const userHasActiveOverleafSubscription = bestCandidate != null
let nextRenewalDate = ''
let expiryDate = ''
let paymentRecord = null
if (resolvedSubscription) {
const paymentInfo = await getSubscriptionPaymentInfo(
userId,
resolvedSubscription,
paymentInfoCache
)
if (paymentInfo.skip) {
return null
}
nextRenewalDate = paymentInfo.nextRenewalDate
paymentRecord = paymentInfo.paymentRecord
expiryDate =
CustomerIoPlanHelpers.getExpiryDateFromPaymentRecord(
paymentRecord
) || ''
}
const aiPlan = getAiPlanForUser({
bestSubscription: bestSubscriptionForPlanType,
planType,
individualSubscription: resolvedSubscription,
paymentRecord,
user,
userIsMemberOfGroupSubscription,
userHasActiveOverleafSubscription,
})
const aiPlanTerm = CustomerIoPlanHelpers.getAiPlanCadence(
aiPlan,
bestSubscriptionForPlanType,
resolvedSubscription,
paymentRecord
)
const groupAIEnabled = getGroupAiEnabled(candidates)
const groupRole = getGroupRole(candidates, userId)
return {
userId,
email: user.email,
planType,
displayPlanType,
preMigrationPlanType: planType,
preMigrationDisplayPlanType: displayPlanType,
planTerm,
aiPlan,
aiPlanTerm: aiPlanTerm || '',
nextRenewalDate,
expiryDate,
groupAIEnabled,
groupRole,
}
})
)
)
const csvRows = []
for (const row of rows) {
globalState.processedCount += 1
if (row) {
csvRows.push(row)
globalState.writtenCount += 1
} else {
globalState.skippedCount += 1
}
}
writeCsvRows(writeStream, csvRows, globalState.shouldWriteHeader)
if (csvRows.length > 0) {
globalState.shouldWriteHeader = false
}
globalState.lastProcessedUserId = userIds[userIds.length - 1]
if (globalState.processedCount % checkpointInterval < userIds.length) {
await trackProgress(
`Checkpoint: processed=${globalState.processedCount}/${totalUsersCount || '?'} written=${globalState.writtenCount} skipped=${globalState.skippedCount} resumeAfterUserId=${globalState.lastProcessedUserId}`
)
}
}
async function main(trackProgress) {
const {
outputPath,
concurrency,
batchSize,
resumeAfterUserId,
checkpointInterval,
append,
} = parseArgs()
await trackProgress(
'Building target user cursor (subscriptions + supplementary users)'
)
if (resumeAfterUserId) {
await trackProgress(
`Resume enabled: starting after user id ${resumeAfterUserId}`
)
}
const paymentInfoCache = new Map()
const commonsCache = new Map()
const limit = pLimit(Number(concurrency) || 5)
const resolvedBatchSize = Math.max(1, Number(batchSize) || 500)
const resolvedCheckpointInterval = Math.max(
100,
Number(checkpointInterval) || 10000
)
const outputFileExists = fs.existsSync(outputPath)
const outputFileHasContent =
outputFileExists && fs.statSync(outputPath).size > 0
const writeStream = fs.createWriteStream(outputPath, {
flags: append ? 'a' : 'w',
})
const globalState = {
processedCount: 0,
writtenCount: 0,
skippedCount: 0,
lastProcessedUserId: resumeAfterUserId || null,
shouldWriteHeader: !append || !outputFileHasContent,
}
let pendingUserIds = []
const targetUserIdsCursor = getTargetUserIdsCursor(resumeAfterUserId)
for await (const doc of targetUserIdsCursor) {
pendingUserIds.push(doc._id.toString())
if (pendingUserIds.length >= resolvedBatchSize) {
await processUserBatch({
userIds: pendingUserIds,
writeStream,
paymentInfoCache,
commonsCache,
limit,
trackProgress,
totalUsersCount: null,
globalState,
checkpointInterval: resolvedCheckpointInterval,
})
pendingUserIds = []
}
}
if (pendingUserIds.length > 0) {
await processUserBatch({
userIds: pendingUserIds,
writeStream,
paymentInfoCache,
commonsCache,
limit,
trackProgress,
totalUsersCount: null,
globalState,
checkpointInterval: resolvedCheckpointInterval,
})
}
if (globalState.shouldWriteHeader) {
writeCsvHeader(writeStream)
globalState.shouldWriteHeader = false
}
writeStream.end()
await trackProgress(
`Final checkpoint: processed=${globalState.processedCount} written=${globalState.writtenCount} skipped=${globalState.skippedCount} resumeAfterUserId=${globalState.lastProcessedUserId}`
)
await trackProgress(`CSV generated: ${outputPath}`)
console.log(`✅ Export complete: ${outputPath}`)
console.log(`Rows written: ${globalState.writtenCount}`)
}
try {
await scriptRunner(main)
process.exit(0)
} catch (error) {
console.error(error)
process.exit(1)
}

View File

@@ -467,6 +467,34 @@ describe('ProjectListController', function () {
await ctx.ProjectListController.projectListPage(ctx.req, ctx.res) await ctx.ProjectListController.projectListPage(ctx.req, ctx.res)
}) })
it('should send groupRole to customer.io for group admins', async function (ctx) {
ctx.Features.hasFeature.withArgs('saas').returns(true)
ctx.SubscriptionViewModelBuilder.promises.getUsersSubscriptionDetails.resolves(
{
bestSubscription: { type: 'free' },
individualSubscription: null,
memberGroupSubscriptions: [],
managedGroupSubscriptions: [
{
planCode: 'group_professional',
membersLimit: 12,
},
],
}
)
ctx.res.render = () => {}
await ctx.ProjectListController.projectListPage(ctx.req, ctx.res)
expect(ctx.Modules.promises.hooks.fire).to.have.been.calledWith(
'setUserProperties',
ctx.user._id,
sinon.match({
group_role: 'admin',
})
)
})
it('should show INR Banner for Indian users with free account', async function (ctx) { it('should show INR Banner for Indian users with free account', async function (ctx) {
// usersBestSubscription is only available when saas feature is present // usersBestSubscription is only available when saas feature is present
ctx.Features.hasFeature.withArgs('saas').returns(true) ctx.Features.hasFeature.withArgs('saas').returns(true)

View File

@@ -9,6 +9,7 @@ const MODULE_PATH = '../../../../app/src/Features/Subscription/FeaturesUpdater'
describe('FeaturesUpdater', function () { describe('FeaturesUpdater', function () {
beforeEach(async function (ctx) { beforeEach(async function (ctx) {
ctx.renewalDate = new Date('2099-04-01T00:00:00Z')
ctx.v1UserId = 12345 ctx.v1UserId = 12345
ctx.user = { ctx.user = {
_id: new ObjectId(), _id: new ObjectId(),
@@ -17,7 +18,12 @@ describe('FeaturesUpdater', function () {
} }
ctx.aiAddOn = { addOnCode: AI_ADD_ON_CODE, quantity: 1 } ctx.aiAddOn = { addOnCode: AI_ADD_ON_CODE, quantity: 1 }
ctx.subscriptions = { ctx.subscriptions = {
individual: { planCode: 'individual-plan' }, individual: {
planCode: 'individual-plan',
groupPlan: false,
recurlySubscription_id: 'sub-individual',
recurlyStatus: { state: 'active' },
},
group1: { planCode: 'group-plan-1', groupPlan: true }, group1: { planCode: 'group-plan-1', groupPlan: true },
group2: { planCode: 'group-plan-2', groupPlan: true }, group2: { planCode: 'group-plan-2', groupPlan: true },
noDropbox: { planCode: 'no-dropbox' }, noDropbox: { planCode: 'no-dropbox' },
@@ -115,6 +121,7 @@ describe('FeaturesUpdater', function () {
ctx.UserGetter = { ctx.UserGetter = {
promises: { promises: {
getUser: sinon.stub().resolves(null), getUser: sinon.stub().resolves(null),
getWritefullData: sinon.stub().resolves(null),
}, },
} }
ctx.UserGetter.promises.getUser.withArgs(ctx.user._id).resolves(ctx.user) ctx.UserGetter.promises.getUser.withArgs(ctx.user._id).resolves(ctx.user)
@@ -126,12 +133,25 @@ describe('FeaturesUpdater', function () {
setUserPropertyForUserInBackground: sinon.stub(), setUserPropertyForUserInBackground: sinon.stub(),
} }
ctx.Modules = { ctx.Modules = {
promises: { hooks: { fire: sinon.stub().resolves() } }, promises: { hooks: { fire: sinon.stub().resolves([]) } },
} }
ctx.Modules.promises.hooks.fire
.withArgs('getPaymentFromRecordPromise', ctx.subscriptions.individual)
.resolves([
{
subscription: {
state: 'active',
periodEnd: ctx.renewalDate,
},
},
])
ctx.SubscriptionViewModelBuilder = { ctx.SubscriptionViewModelBuilder = {
promises: { promises: {
getUsersSubscriptionDetails: sinon.stub().resolves({ getUsersSubscriptionDetails: sinon.stub().resolves({
bestSubscription: { type: 'individual' }, bestSubscription: { type: 'individual' },
individualSubscription: ctx.subscriptions.individual,
memberGroupSubscriptions: [],
managedGroupSubscriptions: [],
}), }),
}, },
} }
@@ -419,36 +439,115 @@ describe('FeaturesUpdater', function () {
).to.have.been.calledWith(ctx.user._id, 'feature-set', 'all') ).to.have.been.calledWith(ctx.user._id, 'feature-set', 'all')
}) })
it('should sync features to customer.io', function (ctx) { it('should sync subscription properties to customer.io', function (ctx) {
expect(ctx.Modules.promises.hooks.fire).to.have.been.calledWith( expect(ctx.Modules.promises.hooks.fire).to.have.been.calledWith(
'setUserProperties', 'setUserProperties',
ctx.user._id, ctx.user._id,
{ sinon.match({
features: ctx.Settings.features.all, plan_type: 'individual',
'best-subscription-type': 'individual', display_plan_type: 'individual',
overleafId: ctx.user._id, ai_plan: 'none',
} next_renewal_date: Math.floor(ctx.renewalDate.getTime() / 1000),
expiry_date: '',
features: sinon.match.object,
})
) )
}) })
}) })
describe('when user has an email', function () { describe('when the individual subscription has a pending cancellation', function () {
beforeEach(async function (ctx) { beforeEach(async function (ctx) {
ctx.user.email = 'user@example.com' const pendingCancellationSubscription = {
...ctx.subscriptions.individual,
recurlyStatus: { state: 'canceled' },
}
ctx.SubscriptionViewModelBuilder.promises.getUsersSubscriptionDetails.resolves(
{
bestSubscription: { type: 'individual' },
individualSubscription: pendingCancellationSubscription,
memberGroupSubscriptions: [],
managedGroupSubscriptions: [],
}
)
ctx.Modules.promises.hooks.fire
.withArgs(
'getPaymentFromRecordPromise',
pendingCancellationSubscription
)
.resolves([
{
subscription: {
state: 'canceled',
periodEnd: ctx.renewalDate,
},
},
])
await ctx.FeaturesUpdater.promises.refreshFeatures(ctx.user._id, 'test') await ctx.FeaturesUpdater.promises.refreshFeatures(ctx.user._id, 'test')
}) })
it('should include email in customer.io properties', function (ctx) { it('should sync expiry_date and blank next_renewal_date in customer.io', function (ctx) {
expect(ctx.Modules.promises.hooks.fire).to.have.been.calledWith( expect(ctx.Modules.promises.hooks.fire).to.have.been.calledWith(
'setUserProperties', 'setUserProperties',
ctx.user._id, ctx.user._id,
sinon.match({
plan_type: 'individual',
display_plan_type: 'individual',
ai_plan: 'none',
next_renewal_date: '',
expiry_date: Math.floor(ctx.renewalDate.getTime() / 1000),
features: sinon.match.object,
})
)
})
})
describe('when the user is in a group subscription', function () {
beforeEach(async function (ctx) {
ctx.SubscriptionViewModelBuilder.promises.getUsersSubscriptionDetails.resolves(
{ {
features: ctx.Settings.features.all, bestSubscription: {
'best-subscription-type': 'individual', type: 'group',
overleafId: ctx.user._id, plan: {
email: 'user@example.com', planCode: 'group-plan-1',
groupPlan: true,
membersLimit: 5,
},
subscription: {
teamName: 'Team Alpha',
},
},
memberGroupSubscriptions: [
{
planCode: 'group-plan-1',
teamName: 'Team Alpha',
membersLimit: 8,
},
],
managedGroupSubscriptions: [],
individualSubscription: null,
} }
) )
await ctx.FeaturesUpdater.promises.refreshFeatures(ctx.user._id, 'test')
})
it('should sync groupSize to customer.io', function (ctx) {
expect(ctx.Modules.promises.hooks.fire).to.have.been.calledWith(
'setUserProperties',
ctx.user._id,
sinon.match({
plan_type: 'group-standard',
display_plan_type: 'Group Standard',
plan_term_label: 'monthly',
ai_plan: 'none',
group_ai_enabled: false,
group_size: 8,
next_renewal_date: '',
expiry_date: '',
features: sinon.match.object,
overleaf_id: ctx.user._id,
})
)
}) })
}) })

View File

@@ -1,6 +1,5 @@
import { vi, expect } from 'vitest' import { vi, expect } from 'vitest'
import sinon from 'sinon' import sinon from 'sinon'
import ProjectHelper from '../../../../app/src/Features/Project/ProjectHelper.mjs'
const modulePath = const modulePath =
'../../../../app/src/Features/Templates/TemplatesController.mjs' '../../../../app/src/Features/Templates/TemplatesController.mjs'
@@ -9,8 +8,12 @@ describe('TemplatesController', function () {
beforeEach(async function (ctx) { beforeEach(async function (ctx) {
ctx.user_id = 'user-id' ctx.user_id = 'user-id'
ctx.ProjectHelper = {
compilerFromV1Engine: sinon.stub(),
}
vi.doMock('../../../../app/src/Features/Project/ProjectHelper', () => ({ vi.doMock('../../../../app/src/Features/Project/ProjectHelper', () => ({
default: ProjectHelper, default: ctx.ProjectHelper,
})) }))
vi.doMock( vi.doMock(