From b517df66726f3c4c60f5b159b4b30a99fa16fc00 Mon Sep 17 00:00:00 2001 From: Simon Gardner Date: Mon, 26 Jan 2026 10:56:44 +0000 Subject: [PATCH] Merge pull request #30958 from overleaf/slg-recurly-stripe-migration-30684 add script for migrating customer metadata from recurly to stripe GitOrigin-RevId: 019413eda20cef2e09c9cc278a8806fa244fe019 --- ...te_recurly_customers_to_stripe.helpers.mjs | 364 +++++ ..._customers_to_stripe.helpers.node.test.mjs | 232 +++ .../scripts/recurly/list_recurly_accounts.mjs | 293 ++++ .../migrate_recurly_customers_to_stripe.mjs | 1378 +++++++++++++++++ 4 files changed, 2267 insertions(+) create mode 100644 services/web/scripts/helpers/migrate_recurly_customers_to_stripe.helpers.mjs create mode 100644 services/web/scripts/helpers/migrate_recurly_customers_to_stripe.helpers.node.test.mjs create mode 100644 services/web/scripts/recurly/list_recurly_accounts.mjs create mode 100644 services/web/scripts/recurly/migrate_recurly_customers_to_stripe.mjs diff --git a/services/web/scripts/helpers/migrate_recurly_customers_to_stripe.helpers.mjs b/services/web/scripts/helpers/migrate_recurly_customers_to_stripe.helpers.mjs new file mode 100644 index 0000000000..52799eaa8c --- /dev/null +++ b/services/web/scripts/helpers/migrate_recurly_customers_to_stripe.helpers.mjs @@ -0,0 +1,364 @@ +/* eslint-disable @overleaf/require-script-runner */ +import lodash from 'lodash' + +/* + * This helper can be used by migrate_recurly_customers_to_stripe.mjs + * + * This file can be deleted once the Recurly to Stripe migration is complete. + */ + +const { isEqual } = lodash + +export function coalesceOrEqualOrThrow(a, b, fieldName) { + const isSetA = !!a + const isSetB = !!b + + if (isSetA && isSetB && a !== b) { + throw new Error( + `Field ${fieldName}: Primary and fallback values are both set but differ (${a} != ${b})` + ) + } + + return isSetA ? a : b +} + +function normalizeName(firstName, lastName) { + const first = (firstName || '').trim() + const last = (lastName || '').trim() + const full = `${first} ${last}`.trim() + return full || null +} + +/** + * Extract and coalesce customer name from Recurly data. + * + * Atomic behavior: first+last name are taken from the same source. + * + * Coalesce/equality behavior: + * - Prefer billingInfo name when both first+last are present. + * - Fall back to account name otherwise. + * - If both billingInfo and account have a complete (first+last) name and they differ, throw. + * + * @param {object} account - Recurly account object + * @param {object|null} billingInfo - Recurly billing info object + * @returns {string|null} + */ +export function coalesceOrEqualOrThrowName(account, billingInfo) { + const billingHasFullName = !!(billingInfo?.firstName && billingInfo?.lastName) + const accountHasFullName = !!(account?.firstName && account?.lastName) + + const billingName = billingHasFullName + ? normalizeName(billingInfo.firstName, billingInfo.lastName) + : null + const accountName = accountHasFullName + ? normalizeName(account?.firstName, account?.lastName) + : null + + if (billingHasFullName && accountHasFullName && billingName !== accountName) { + throw new Error( + `Name differs between billingInfo and account (${billingName} != ${accountName})` + ) + } + + return billingName ?? accountName +} + +/** + * Extract and coalesce VAT number from Recurly data. + * + * Coalesce/equality behavior: + * - Prefer billingInfo.vatNumber when set. + * - Fall back to account.vatNumber otherwise. + * - If both are set but differ, throw. + * + * @param {object} account - Recurly account object + * @param {object|null} billingInfo - Recurly billing info object + * @returns {string|null} + */ +export function coalesceOrThrowVATNumber(account, billingInfo) { + const billingVat = billingInfo?.vatNumber?.trim() || null + const accountVat = account?.vatNumber?.trim() || null + return coalesceOrEqualOrThrow(billingVat, accountVat, 'vatNumber') +} + +/** + * Normalize a Recurly address into a Stripe AddressParam. + * + * Recurly address field names appear in multiple shapes depending on SDK/version + * and serialization (e.g. street1/street2/postalCode vs line1/line2/postal_code). + * + * @param {any} address + * @returns {import('stripe').Stripe.AddressParam|null} + */ +export function normalizeRecurlyAddressToStripe(address) { + if (!address) return null + + const line1 = (address.street1 || '').trim() + // eslint-disable-next-line camelcase + const postal_code = (address.postalCode || '').trim() + const country = String(address.country || '') + .trim() + .toUpperCase() + + // Only send an address if it has enough data to be plausibly accepted/usable by Stripe. + // eslint-disable-next-line camelcase + if (!line1 || !postal_code || !country) return null + if (!/^[A-Z]{2}$/.test(country)) return null + + const line2 = (address.street2 || '').trim() + const city = (address.city || '').trim() + const state = (address.region || '').trim() + + return { + line1, + ...(line2 ? { line2 } : {}), + ...(city ? { city } : {}), + ...(state ? { state } : {}), + // eslint-disable-next-line camelcase + postal_code, + country, + } +} + +/** + * Extract address from Recurly data. + * + * Prefers billingInfo address as this is what the customer entered during checkout. + * Falls back to account address for manually-created accounts or legacy data. + * + * @param {object} account - Recurly account object + * @param {object|null} billingInfo - Recurly billing info object + * @returns {import('stripe').Stripe.AddressParam|null} + */ +export function coalesceOrEqualOrThrowAddress(account, billingInfo) { + const billingAddress = normalizeRecurlyAddressToStripe(billingInfo?.address) + const accountAddress = normalizeRecurlyAddressToStripe(account?.address) + + const isBillingAddressValid = !!billingAddress + const isAccountAddressValid = !!accountAddress + + if (!isBillingAddressValid && !isAccountAddressValid) return null + if (isBillingAddressValid && !isAccountAddressValid) return billingAddress + if (!isBillingAddressValid && isAccountAddressValid) return accountAddress + + if (!isEqual(billingAddress, accountAddress)) { + throw new Error('Billing address and account address differ') + } + + return billingAddress +} + +/** + * EU member state country codes for VAT purposes. + */ +export const EU_VAT_COUNTRIES = [ + 'AT', // Austria + 'BE', // Belgium + 'BG', // Bulgaria + 'CY', // Cyprus + 'CZ', // Czech Republic + 'DE', // Germany + 'DK', // Denmark + 'EE', // Estonia + 'ES', // Spain + 'FI', // Finland + 'FR', // France + 'GR', // Greece + 'HR', // Croatia + 'HU', // Hungary + 'IE', // Ireland + 'IT', // Italy + 'LT', // Lithuania + 'LU', // Luxembourg + 'LV', // Latvia + 'MT', // Malta + 'NL', // Netherlands + 'PL', // Poland + 'PT', // Portugal + 'RO', // Romania + 'SE', // Sweden + 'SI', // Slovenia + 'SK', // Slovakia +] + +function caProvinceFromPostalCode(postalCode) { + if (!postalCode) return null + const m = String(postalCode) + .trim() + .toUpperCase() + .match(/^([A-Z])/) + if (!m) return null + const c = m[1] + + if (c === 'A') return 'NL' + if (c === 'B') return 'NS' + if (c === 'C') return 'PE' + if (c === 'E') return 'NB' + if (c === 'G' || c === 'H' || c === 'J') return 'QC' + if (c === 'K' || c === 'L' || c === 'M' || c === 'N' || c === 'P') return 'ON' + if (c === 'R') return 'MB' + if (c === 'S') return 'SK' + if (c === 'T') return 'AB' + if (c === 'V') return 'BC' + if (c === 'Y') return 'YT' + if (c === 'X') return 'NT_NU' // ambiguous without more info + return null +} + +/** + * Determine the Stripe tax ID type for Canada based on the tax ID value format. + * + * Source reference: + * - Stripe docs list supported types + example formats (Canada section) + * https://docs.stripe.com/billing/customer/tax-ids + * + * + * If the value doesn't clearly match one of Stripe's documented example formats, + * return null so the caller can handle it manually. + * + * @param {string|undefined|null} taxIdValue + * @returns {'ca_bn'|'ca_gst_hst'|'ca_pst_bc'|'ca_pst_mb'|'ca_pst_sk'|'ca_qst'|null} + */ +export function getCanadaTaxIdType(taxIdValue, postalCode) { + if (!taxIdValue) return null + + // Normalize for CRA/Stripe examples which may contain spaces (e.g. "123456789 RT 0001") + const normalized = String(taxIdValue).trim().toUpperCase().replace(/\s+/g, '') + if (!normalized) return null + + // GST/HST: CRA defines program account numbers as BN (9 digits) + program id (2 letters) + reference (4 digits). + // CRA explicitly shows GST/HST program account number as: 123456789 RT 0001 (or without spaces). + // Safe because "RT" in this exact position is the GST/HST program identifier. + // source: https://www.canada.ca/en/revenue-agency/services/tax/businesses/topics/business-registration/business-number-program-account/need-program-accounts.htm + if (/^\d{9}RT\d{4}$/.test(normalized)) return 'ca_gst_hst' + + // Québec QST: Stripe support doc gives exact structure and example 1234567891TQ0001, + // and Revenu Québec states QST registration numbers include the letters "TQ". + // Safe because "TQ" in this exact position is distinctive. + // source: https://support.stripe.com/questions/quebec-sales-tax-information + if (/^\d{10}TQ\d{4}$/.test(normalized)) return 'ca_qst' + + // British Columbia PST: BC Gov explicitly states PST number format is PST-1234-5678. + // Safe because it has the "PST-" prefix + hyphen groups. + // source: https://www2.gov.bc.ca/gov/content/taxes/sales-taxes/pst/register + if (/^PST-\d{4}-\d{4}$/.test(normalized)) return 'ca_pst_bc' + + const prov = caProvinceFromPostalCode(postalCode) + + // Ambiguous numeric-only cases: require province inferred from postal code + // Manitoba: allow dashed 123456-7 and undashed 1234567, but ONLY if prov==MB + if ( + (/^\d{6}-\d$/.test(normalized) || /^\d{7}$/.test(normalized)) && + prov === 'MB' + ) { + return 'ca_pst_mb' + } + + // Saskatchewan: 7 digits ONLY if prov==SK + if (/^\d{7}$/.test(normalized) && prov === 'SK') { + return 'ca_pst_sk' + } + + // Canadian BN: CRA says the GST/HST program account number starts with a 9-digit BN, and some workflows ask for only those 9 digits. + // Stripe has ca_bn, but a bare 9-digit value is ambiguous (BN-only vs “first 9 digits of GST/HST account” vs other). + // -> Not safe to classify from format alone. Only classify as ca_bn if your *input field* is explicitly “BN”. + // if (/^\d{9}$/.test(normalized)) return 'ca_bn' + + return null +} + +/** + * Get the Stripe tax ID type for a given country + tax ID value. + * + * Note: for some countries (e.g. Canada) the type depends on the tax ID value format, + * not just the country. + * + * @param {string|undefined|null} country - ISO 3166-1 alpha-2 + * @param {string|undefined|null} taxIdValue + * @param {string|undefined|null} postalCode - used for Canada province inference + * @returns {string|null} - Stripe tax ID type, or null if country/type unsupported + */ +// TODO: this function is naive - we need more than just country to determine tax ID type +// for example canada has multiple types (BN, QST, GST/HST) depending on province (inferred from postal code) +export function getTaxIdType(country, taxIdValue, postalCode) { + if (!country) return null + + const upperCountry = String(country).toUpperCase() + + // EU VAT + if (EU_VAT_COUNTRIES.includes(upperCountry)) { + return 'eu_vat' + } + + // Canada + if (upperCountry === 'CA') { + return getCanadaTaxIdType(taxIdValue, postalCode) + } + + // Country-specific tax IDs (all Stripe-supported types) + // See: https://docs.stripe.com/api/tax_ids/create#create_tax_id-type + const countryTaxIdTypes = { + // Europe (non-EU) + GB: 'gb_vat', + // CH: 'ch_vat', + // NO: 'no_vat', + // IS: 'is_vat', + // LI: 'li_uid', + // TR: 'tr_tin', + + // // Americas + US: 'us_ein', + // CA: 'ca_bn', // this is more complex, see getCanadaTaxIdType() + // MX: 'mx_rfc', + // BR: 'br_cnpj', + // CL: 'cl_tin', + // CO: 'co_nit', + // AR: 'ar_cuit', + // BO: 'bo_tin', + // CR: 'cr_tin', + // DO: 'do_rcn', + // EC: 'ec_ruc', + // PE: 'pe_ruc', + // UY: 'uy_ruc', + // VE: 've_rif', + // SV: 'sv_nit', + + // // Asia-Pacific + // AU: 'au_abn', + // NZ: 'nz_gst', + // JP: 'jp_cn', + // KR: 'kr_brn', + // CN: 'cn_tin', + // HK: 'hk_br', + // TW: 'tw_vat', + // SG: 'sg_gst', + // MY: 'my_sst', + // TH: 'th_vat', + // ID: 'id_npwp', + // PH: 'ph_tin', + // IN: 'in_gst', + // VN: 'vn_tin', + + // // Middle East + // AE: 'ae_trn', + // SA: 'sa_vat', + // BH: 'bh_vat', + // OM: 'om_vat', + // IL: 'il_vat', + + // // Africa + // ZA: 'za_vat', + // EG: 'eg_tin', + // KE: 'ke_pin', + // NG: 'ng_tin', + + // // Other + // GE: 'ge_vat', + // UA: 'ua_vat', + // RS: 'rs_pib', + // MD: 'md_vat', + // AD: 'ad_nrt', + } + + return countryTaxIdTypes[upperCountry] || null +} diff --git a/services/web/scripts/helpers/migrate_recurly_customers_to_stripe.helpers.node.test.mjs b/services/web/scripts/helpers/migrate_recurly_customers_to_stripe.helpers.node.test.mjs new file mode 100644 index 0000000000..afeeeac0b7 --- /dev/null +++ b/services/web/scripts/helpers/migrate_recurly_customers_to_stripe.helpers.node.test.mjs @@ -0,0 +1,232 @@ +/* eslint-disable @overleaf/require-script-runner */ +import test from 'node:test' +import assert from 'node:assert/strict' + +/* + * This test can be run from the services/web directory with: + * + * node --test scripts/helpers/migrate_recurly_customers_to_stripe.helpers.node.test.mjs + * + * It can be deleted once the Recurly to Stripe migration is complete. + */ + +import { + coalesceOrEqualOrThrow, + coalesceOrEqualOrThrowAddress, + coalesceOrEqualOrThrowName, + coalesceOrThrowVATNumber, + getCanadaTaxIdType, +} from './migrate_recurly_customers_to_stripe.helpers.mjs' + +test('coalesceOrEqualOrThrow returns primary when set', () => { + assert.equal(coalesceOrEqualOrThrow('a', undefined, 'field'), 'a') +}) + +test('coalesceOrEqualOrThrow returns fallback when primary is unset', () => { + assert.equal(coalesceOrEqualOrThrow(undefined, 'b', 'field'), 'b') +}) + +test('coalesceOrEqualOrThrow returns value when both are set and equal', () => { + assert.equal(coalesceOrEqualOrThrow('same', 'same', 'field'), 'same') +}) + +test('coalesceOrEqualOrThrow throws when both are set but differ', () => { + assert.throws( + () => coalesceOrEqualOrThrow('a', 'b', 'field'), + /Primary and fallback values are both set but differ/ + ) +}) + +test('coalesceOrEqualOrThrowAddress returns null when neither is valid', () => { + assert.equal(coalesceOrEqualOrThrowAddress({}, null), null) + assert.equal( + coalesceOrEqualOrThrowAddress( + { address: { street1: '', postalCode: '', country: '' } }, + { address: { street1: '', postalCode: '', country: '' } } + ), + null + ) + + assert.equal( + coalesceOrEqualOrThrowAddress( + { address: { street1: ' ', postalCode: ' ', country: ' ' } }, + null + ), + null + ) +}) + +test('coalesceOrEqualOrThrowAddress returns account when billing invalid', () => { + const account = { + address: { street1: '1 Road', postalCode: 'ABC', country: 'GB' }, + } + const billingInfo = { + address: { street1: '', postalCode: 'ABC', country: 'GB' }, + } + assert.deepEqual(coalesceOrEqualOrThrowAddress(account, billingInfo), { + line1: '1 Road', + postal_code: 'ABC', + country: 'GB', + }) +}) + +test('coalesceOrEqualOrThrowAddress returns billing when account invalid', () => { + const account = { + address: { street1: '', postalCode: 'ABC', country: 'GB' }, + } + const billingInfo = { + address: { street1: '1 Road', postalCode: 'ABC', country: 'GB' }, + } + assert.deepEqual(coalesceOrEqualOrThrowAddress(account, billingInfo), { + line1: '1 Road', + postal_code: 'ABC', + country: 'GB', + }) +}) + +test('coalesceOrEqualOrThrowAddress returns billing when both valid+equal', () => { + const addr = { street1: '1 Road', postalCode: 'ABC', country: 'GB' } + assert.deepEqual( + coalesceOrEqualOrThrowAddress({ address: { ...addr } }, { address: addr }), + { line1: '1 Road', postal_code: 'ABC', country: 'GB' } + ) +}) + +test('coalesceOrEqualOrThrowAddress normalizes Recurly-style address fields', () => { + const billingInfo = { + address: { + street1: 'as', + street2: '', + city: '', + region: '', + postalCode: '12312', + country: 'AI', + }, + } + + assert.deepEqual(coalesceOrEqualOrThrowAddress({}, billingInfo), { + line1: 'as', + postal_code: '12312', + country: 'AI', + }) +}) + +test('coalesceOrEqualOrThrowAddress throws when both valid but differ', () => { + const account = { + address: { street1: '1 Road', postalCode: 'ABC', country: 'GB' }, + } + const billingInfo = { + address: { street1: '2 Road', postalCode: 'ABC', country: 'GB' }, + } + assert.throws( + () => coalesceOrEqualOrThrowAddress(account, billingInfo), + /Billing address and account address differ/ + ) +}) + +test('coalesceOrEqualOrThrowName returns billingInfo name when both sources match', () => { + const account = { firstName: 'Alice', lastName: 'Billing' } + const billingInfo = { firstName: 'Alice', lastName: 'Billing' } + assert.equal( + coalesceOrEqualOrThrowName(account, billingInfo), + 'Alice Billing' + ) +}) + +test('coalesceOrEqualOrThrowName prefers billingInfo when billingInfo is full but account is not', () => { + const account = { firstName: 'Alice', lastName: '' } + const billingInfo = { firstName: 'Alice', lastName: 'Billing' } + assert.equal( + coalesceOrEqualOrThrowName(account, billingInfo), + 'Alice Billing' + ) +}) + +test('coalesceOrEqualOrThrowName falls back to account when billingInfo missing last name', () => { + const account = { firstName: 'Alice', lastName: 'Account' } + const billingInfo = { firstName: 'Alice', lastName: '' } + assert.equal( + coalesceOrEqualOrThrowName(account, billingInfo), + 'Alice Account' + ) +}) + +test('coalesceOrEqualOrThrowName returns null when both sources are empty', () => { + assert.equal(coalesceOrEqualOrThrowName({}, null), null) + assert.equal( + coalesceOrEqualOrThrowName({ firstName: '', lastName: '' }, null), + null + ) +}) + +test('coalesceOrEqualOrThrowName throws when both full names are present but differ', () => { + const account = { firstName: 'Alice', lastName: 'Account' } + const billingInfo = { firstName: 'Alice', lastName: 'Billing' } + assert.throws( + () => coalesceOrEqualOrThrowName(account, billingInfo), + /Name differs between billingInfo and account/ + ) +}) + +test('coalesceOrThrowVATNumber returns billingInfo VAT when set', () => { + const account = { vatNumber: '' } + const billingInfo = { vatNumber: 'BILL456' } + assert.equal(coalesceOrThrowVATNumber(account, billingInfo), 'BILL456') +}) + +test('coalesceOrThrowVATNumber returns account VAT when billingInfo VAT unset', () => { + const account = { vatNumber: 'ACCT123' } + const billingInfo = { vatNumber: '' } + assert.equal(coalesceOrThrowVATNumber(account, billingInfo), 'ACCT123') +}) + +test('coalesceOrThrowVATNumber returns null when neither is set', () => { + assert.equal(coalesceOrThrowVATNumber({}, null), null) + assert.equal( + coalesceOrThrowVATNumber({ vatNumber: '' }, { vatNumber: '' }), + null + ) +}) + +test('coalesceOrThrowVATNumber treats trimmed values as equal', () => { + const account = { vatNumber: ' GB123 ' } + const billingInfo = { vatNumber: 'GB123' } + assert.equal(coalesceOrThrowVATNumber(account, billingInfo), 'GB123') +}) + +test('coalesceOrThrowVATNumber throws when both are set but differ', () => { + const account = { vatNumber: 'GB123' } + const billingInfo = { vatNumber: 'DE999' } + assert.throws( + () => coalesceOrThrowVATNumber(account, billingInfo), + /Field vatNumber: Primary and fallback values are both set but differ/ + ) +}) + +test('getCanadaTaxIdType returns ca_gst_hst for GST/HST format', () => { + assert.equal(getCanadaTaxIdType('123456789RT0002', null), 'ca_gst_hst') + assert.equal(getCanadaTaxIdType(' 123456789rt0002 ', null), 'ca_gst_hst') +}) + +test('getCanadaTaxIdType returns ca_qst for Quebec QST format', () => { + assert.equal(getCanadaTaxIdType('1234567890TQ1234', null), 'ca_qst') + assert.equal(getCanadaTaxIdType('1234567890tq1234', null), 'ca_qst') +}) + +test('getCanadaTaxIdType returns PST types for documented provincial formats', () => { + assert.equal(getCanadaTaxIdType('PST-1234-5678', null), 'ca_pst_bc') + assert.equal(getCanadaTaxIdType('123456-7', 'R3C 4T3'), 'ca_pst_mb') + assert.equal(getCanadaTaxIdType('1234567', 'S7K 3J8'), 'ca_pst_sk') +}) + +test('getCanadaTaxIdType returns ca_bn for 9-digit BN format', () => { + // assert.equal(getCanadaTaxIdType('123456789', null), 'ca_bn') + assert.equal(getCanadaTaxIdType('123456789', null), null) // TODO: improve function get definitive ca_bn vs ca_gst_hst +}) + +test('getCanadaTaxIdType returns null when format is unknown/ambiguous', () => { + assert.equal(getCanadaTaxIdType('', null), null) + assert.equal(getCanadaTaxIdType(null, null), null) + assert.equal(getCanadaTaxIdType('RT0002', null), null) + assert.equal(getCanadaTaxIdType('PST12345678', null), null) +}) diff --git a/services/web/scripts/recurly/list_recurly_accounts.mjs b/services/web/scripts/recurly/list_recurly_accounts.mjs new file mode 100644 index 0000000000..da1d7d939c --- /dev/null +++ b/services/web/scripts/recurly/list_recurly_accounts.mjs @@ -0,0 +1,293 @@ +/** + * List Recurly accounts and output as CSV for use with migrate_recurly_customers_to_stripe.mjs + * + * Useful for generating list of customers for testing purposes. + * + * This script can be deleted once the Recurly to Stripe migration is complete. + * + * Usage: + * node scripts/recurly/list_recurly_accounts.mjs --limit 100 --output test_customers.csv + * + * Options: + * --limit N Number of accounts to fetch (default: 100) + * --output FILE Output CSV file (required) + * --stripe-account Account ID to use for target_stripe_account column + * --verbose Enable debug logging + */ + +import Settings from '@overleaf/settings' +import recurly from 'recurly' +import minimist from 'minimist' +import fs from 'node:fs' +import { scriptRunner } from '../lib/ScriptRunner.mjs' + +import { normalizeRecurlyAddressToStripe } from '../helpers/migrate_recurly_customers_to_stripe.helpers.mjs' + +const recurlyApiKey = + process.env.RECURLY_API_KEY || Settings.apis?.recurly?.apiKey +if (!recurlyApiKey) { + throw new Error( + 'Recurly API key is not set. Set RECURLY_API_KEY env var or configure Settings.apis.recurly.apiKey' + ) +} + +const client = new recurly.Client(recurlyApiKey) + +function usage() { + console.error( + 'List Recurly accounts and output as CSV for use with migrate_recurly_customers_to_stripe.mjs' + ) + console.error('') + console.error('Usage:') + console.error( + ' node scripts/recurly/list_recurly_accounts.mjs --output [options]' + ) + console.error('') + console.error('Options:') + console.error( + ' --limit, -l N Number of accounts to fetch (default: 100)' + ) + console.error(' --output, -o FILE Output CSV file (required)') + console.error( + ' --stripe-account, -s ID Target Stripe account ID for all rows' + ) + console.error(' --verbose, -v Enable debug logging') + console.error(' --help, -h Show this help message') +} + +function parseArgs() { + return minimist(process.argv.slice(2), { + alias: { + o: 'output', + l: 'limit', + s: 'stripe-account', + v: 'verbose', + h: 'help', + }, + default: { limit: 100 }, + string: ['output', 'stripe-account'], + boolean: ['verbose'], + }) +} + +async function main(trackProgress) { + const args = parseArgs() + + const DEBUG = !!args.verbose + function debug(message, context = {}) { + if (!DEBUG) return + const contextStr = + Object.keys(context).length > 0 ? ` ${JSON.stringify(context)}` : '' + console.log(`[DEBUG] ${message}${contextStr}`) + } + + if (args.help || args.h) { + usage() + process.exit(0) + } + + if (!args.output) { + usage() + console.error('') + console.error('Error: --output is required') + process.exit(1) + } + + const limit = parseInt(args.limit, 10) + const targetStripeAccount = + args['stripe-account'] || 'REPLACE_WITH_STRIPE_ACCOUNT_ID' + + if (!Number.isFinite(limit) || limit <= 0) { + throw new Error(`Invalid --limit: ${args.limit}`) + } + if (limit < 2) { + throw new Error( + 'Invalid --limit: must be >= 2 to satisfy VAT constraints (>=1 VAT account but <=50% VAT overall)' + ) + } + + await trackProgress(`Fetching up to ${limit} accounts from Recurly...`) + await trackProgress(`Target Stripe account: ${targetStripeAccount}`) + if (DEBUG) { + await trackProgress('Debug logging enabled') + } + + const vatCandidates = [] + const nonVatCandidates = [] + + let scanned = 0 + let acceptedWithAddress = 0 + let rejectedNoAddress = 0 + let rejectedVatOverCap = 0 + let billingInfoFetched = 0 + let billingInfoNotFound = 0 + let billingInfoOtherError = 0 + let usedBillingAddress = 0 + let usedBillingVatNumber = 0 + + const vatCap = Math.floor(limit / 2) + + // List accounts with pagination - Recurly returns a Pager, need to call .each() + // Options must be wrapped in { params: { ... } } + const accountsPager = client.listAccounts({ + params: { limit: Math.min(limit, 200) }, + }) + for await (const account of accountsPager.each()) { + scanned++ + + const recurlyAccountCode = account.code + const row = { + recurly_account_code: recurlyAccountCode, + target_stripe_account: targetStripeAccount, + stripe_customer_id: '', // Empty - no existing Stripe customer + email: account.email, + state: account.state, + } + + let address = account.address + let vatNumber = + typeof account.vatNumber === 'string' ? account.vatNumber : null + + // Fetch billing info only if needed for address/vat detection + if (!normalizeRecurlyAddressToStripe(address) || !vatNumber) { + try { + billingInfoFetched++ + const billingInfo = await client.getBillingInfo( + `code-${recurlyAccountCode}` + ) + if (!address && billingInfo?.address) { + address = billingInfo.address + usedBillingAddress++ + } + if (!vatNumber && billingInfo?.vatNumber) { + vatNumber = billingInfo.vatNumber + usedBillingVatNumber++ + } + } catch (error) { + if (!(error instanceof recurly.errors.NotFoundError)) { + billingInfoOtherError++ + throw error + } + billingInfoNotFound++ + } + } + + if (!normalizeRecurlyAddressToStripe(address)) { + rejectedNoAddress++ + debug('Rejected account: no valid address', { + scanned, + recurlyAccountCode, + rejectedNoAddress, + }) + continue + } + + acceptedWithAddress++ + + const hasVat = !!(typeof vatNumber === 'string' && vatNumber.trim()) + if (hasVat) { + // Enforce VAT upper bound while scanning. + if (vatCandidates.length >= vatCap) { + rejectedVatOverCap++ + debug('Rejected account: VAT over cap', { + scanned, + recurlyAccountCode, + vatCandidates: vatCandidates.length, + vatCap, + rejectedVatOverCap, + }) + continue + } + vatCandidates.push(row) + } else { + nonVatCandidates.push(row) + } + + // Stop once we can satisfy constraints. + const vatToTake = Math.min(vatCap, vatCandidates.length) + const needsAtLeastOneVat = vatCandidates.length >= 1 + const nonVatNeeded = limit - Math.max(1, vatToTake) + if (needsAtLeastOneVat && nonVatCandidates.length >= nonVatNeeded) { + debug('Stopping early: constraints satisfied', { + scanned, + vatCandidates: vatCandidates.length, + nonVatCandidates: nonVatCandidates.length, + vatCap, + nonVatNeeded, + }) + break + } + + if (scanned % 25 === 0) { + await trackProgress( + `Scanned ${scanned} accounts (acceptedWithAddress=${acceptedWithAddress}, vat=${vatCandidates.length}, nonVat=${nonVatCandidates.length})` + ) + debug('Progress', { + scanned, + acceptedWithAddress, + rejectedNoAddress, + rejectedVatOverCap, + vatCandidates: vatCandidates.length, + nonVatCandidates: nonVatCandidates.length, + billingInfoFetched, + billingInfoNotFound, + billingInfoOtherError, + usedBillingAddress, + usedBillingVatNumber, + }) + } + } + + if (vatCandidates.length < 1) { + throw new Error( + `Unable to find any accounts with VAT numbers (scanned=${scanned}, acceptedWithAddress=${acceptedWithAddress}, rejectedNoAddress=${rejectedNoAddress})` + ) + } + + const vatToTake = Math.max(1, Math.min(vatCap, vatCandidates.length)) + const nonVatToTake = limit - vatToTake + if (nonVatCandidates.length < nonVatToTake) { + throw new Error( + `Unable to satisfy VAT ratio constraint: need ${nonVatToTake} non-VAT + ${vatToTake} VAT, but have nonVat=${nonVatCandidates.length}, vat=${vatCandidates.length} (scanned=${scanned}, rejectedNoAddress=${rejectedNoAddress}, rejectedVatOverCap=${rejectedVatOverCap})` + ) + } + + const accounts = [ + ...vatCandidates.slice(0, vatToTake), + ...nonVatCandidates.slice(0, nonVatToTake), + ] + + await trackProgress( + `Selected ${accounts.length} accounts (vat=${vatToTake}, nonVat=${nonVatToTake}, scanned=${scanned}, rejectedNoAddress=${rejectedNoAddress})` + ) + + // Output CSV + const csvHeader = + 'recurly_account_code,target_stripe_account,stripe_customer_id' + const csvRows = accounts.map( + a => + `${a.recurly_account_code},${a.target_stripe_account},${a.stripe_customer_id}` + ) + const csvContent = [csvHeader, ...csvRows].join('\n') + '\n' + + fs.writeFileSync(args.output, csvContent) + await trackProgress(`Wrote ${accounts.length} accounts to ${args.output}`) + + // Output a summary + const states = {} + accounts.forEach(a => { + states[a.state] = (states[a.state] || 0) + 1 + }) + await trackProgress('Account states:') + for (const [state, stateCount] of Object.entries(states)) { + await trackProgress(` ${state}: ${stateCount}`) + } +} + +try { + await scriptRunner(main) + process.exit(0) +} catch (err) { + console.error('Error:', err.message) + process.exit(1) +} diff --git a/services/web/scripts/recurly/migrate_recurly_customers_to_stripe.mjs b/services/web/scripts/recurly/migrate_recurly_customers_to_stripe.mjs new file mode 100644 index 0000000000..52d9b0ec47 --- /dev/null +++ b/services/web/scripts/recurly/migrate_recurly_customers_to_stripe.mjs @@ -0,0 +1,1378 @@ +#!/usr/bin/env node + +/** + * This script updates existing Stripe customer records with data from Recurly. + * + * It can be deleted once the Recurly to Stripe migration is complete. + * + * PREREQUISITE: Customers must already exist in the target Stripe account (created via PAN import + * or other process). This script updates them with additional data from Recurly. + * + * RESUMABLE EXECUTION: + * This script is designed to be re-runnable. If it fails partway through, you can fix + * the issue and re-run with the same arguments. It will: + * 1. Load already-processed records from the success output file + * 2. Skip any records that were already successfully processed + * 3. Re-attempt any records not in the success file (including previous failures) + * + * To force a fresh start, use --restart flag or delete the success output file. + * + * Input CSV format: + * recurly_account_code,target_stripe_account,stripe_customer_id + * + * Where: + * - recurly_account_code: The Recurly account code (also the Overleaf user ID) + * - target_stripe_account: The target Stripe service name ('stripe-us' or 'stripe-uk') + * - stripe_customer_id: The Stripe customer ID (required - customers must already exist) + * + * Output files: + * --output (success file): Records that were successfully updated + * Format: recurly_account_code,target_stripe_account,stripe_customer_id + * + * _skipped_no_stripe_id.csv: Records skipped because stripe_customer_id was missing + * Format: recurly_account_code,target_stripe_account,stripe_customer_id + * + * _errors.csv: Records that failed (overwritten each run) + * Format: recurly_account_code,target_stripe_account,stripe_customer_id,error + * + * _stripe.json (dry-run only): Stripe customer update params + * Format: Array of { recurly_account_code, target_stripe_account, stripe_customer_id, updateParams } + * + * Resume behavior: + * - Records in the success file are SKIPPED (already done) + * - Records in the errors file are RE-PROCESSED (will be retried) + * - After each run, the errors file contains ONLY the failures from that run + * - Successfully retried records are moved from errors to success file + * + * Usage: + * # Dry run (no changes made, outputs _stripe.json with what would be updated) + * node scripts/recurly/migrate_recurly_customers_to_stripe.mjs --input customers.csv --output results.csv + * + * # Commit changes + * node scripts/recurly/migrate_recurly_customers_to_stripe.mjs --input customers.csv --output results.csv --commit + * + * # Resume after failure (just run the same command again) + * node scripts/recurly/migrate_recurly_customers_to_stripe.mjs --input customers.csv --output results.csv --commit + * + * Options: + * --input, -i Path to input CSV file + * --output, -o Path to success output CSV file + * --commit Actually update customers in Stripe (default: dry-run mode) + * --verbose, -v Enable debug logging + * --restart Ignore existing output files and start fresh + * + * + * Note, prior to running this script, environment variables must have been loaded from config/local.env + * + * ``` + * set +a + * source ../../config/local.env + * set -a + * ``` + */ + +import Settings from '@overleaf/settings' +import Stripe from 'stripe' +import recurly from 'recurly' +import minimist from 'minimist' +import fs from 'node:fs' +import * as csv from 'csv' +import { setTimeout } from 'node:timers/promises' +import { scriptRunner } from '../lib/ScriptRunner.mjs' + +import { + coalesceOrEqualOrThrowAddress, + coalesceOrEqualOrThrowName, + coalesceOrThrowVATNumber, + getTaxIdType, +} from '../helpers/migrate_recurly_customers_to_stripe.helpers.mjs' + +// ============================================================================= +// STRIPE CLIENT SETUP +// ============================================================================= + +const stripeClients = {} + +/** + * Get a Stripe client by region ("us" or "uk"). + * + * This intentionally mirrors the Stripe SDK construction used by subscriptions + * (fetch http client + telemetry disabled), but without importing the full + * subscriptions Stripe client module (which pulls in unrelated app code). + */ +function getRegionClient(region) { + const regionLower = String(region || '') + .trim() + .toLowerCase() + + if (regionLower !== 'us' && regionLower !== 'uk') { + throw new Error( + `Unknown Stripe region: ${region}. Expected stripe-us or stripe-uk.` + ) + } + + if (stripeClients[regionLower]) return stripeClients[regionLower] + + const secretKey = + regionLower === 'us' + ? Settings.apis?.stripeUS?.secretKey || + process.env.STRIPE_OL_SECRET_KEY || + process.env.STRIPE_OL_US_SECRET_KEY + : Settings.apis?.stripeUK?.secretKey || + process.env.STRIPE_OL_UK_SECRET_KEY + + if (!secretKey || !String(secretKey).trim()) { + throw new Error( + `No Stripe secret key configured for region ${regionLower}. ` + + `Configure Settings.apis.stripeUS/stripeUK.secretKey or set ` + + `${ + regionLower === 'us' + ? 'STRIPE_OL_SECRET_KEY (or legacy STRIPE_OL_US_SECRET_KEY)' + : 'STRIPE_OL_UK_SECRET_KEY' + }.` + ) + } + + stripeClients[regionLower] = new Stripe(secretKey, { + httpClient: Stripe.createFetchHttpClient(), + telemetry: false, + }) + + return stripeClients[regionLower] +} + +// ============================================================================= +// RECURLY CLIENT SETUP +// ============================================================================= + +const recurlyApiKey = + process.env.RECURLY_API_KEY || Settings.apis?.recurly?.apiKey +if (!recurlyApiKey) { + throw new Error( + 'Recurly API key is not set. Set RECURLY_API_KEY env var or configure Settings.apis.recurly.apiKey' + ) +} +const recurlyClient = new recurly.Client(recurlyApiKey) + +// ============================================================================= +// LOGGING UTILITIES +// ============================================================================= + +/** + * Get ISO timestamp for logging + */ +function timestamp() { + return new Date().toISOString() +} + +/** + * Log a warning message with timestamp + */ +function logWarn(message, context = {}) { + const contextStr = + Object.keys(context).length > 0 ? ` ${JSON.stringify(context)}` : '' + console.warn(`[${timestamp()}] WARN: ${message}${contextStr}`) +} + +/** + * Log an error message with timestamp and optional stack trace + */ +function logError(message, error = null, context = {}) { + const contextStr = + Object.keys(context).length > 0 ? ` ${JSON.stringify(context)}` : '' + console.error(`[${timestamp()}] ERROR: ${message}${contextStr}`) + if (error?.stack) { + console.error(`[${timestamp()}] STACK: ${error.stack}`) + } +} + +/** + * Debug mode flag - controlled by --verbose/-v CLI arg. + * (Intentionally not controlled via env var to avoid accidental noisy logs.) + */ +let DEBUG_MODE = false + +/** + * Log a message with timestamp. + * + * By default, logs at INFO level. + * When { verboseOnly: true }, only logs when DEBUG_MODE is enabled. + */ +function logDebug(message, context = {}, { verboseOnly = false } = {}) { + if (verboseOnly && !DEBUG_MODE) return + const contextStr = + Object.keys(context).length > 0 ? ` ${JSON.stringify(context)}` : '' + const level = verboseOnly ? 'DEBUG' : 'INFO' + console.log(`[${timestamp()}] ${level}: ${message}${contextStr}`) +} + +// ============================================================================= +// RESUME FUNCTIONALITY +// ============================================================================= + +/** + * Load previously successfully processed records from the success output file. + * Returns a Set of recurly_account_codes that have been successfully processed. + * + * Only records in the SUCCESS file are skipped. Records in the errors file + * (or not in any file) will be processed/re-attempted. + * + * @param {string} successOutputPath - Path to the success output CSV file + * @returns {Promise>} + */ +async function loadSuccessfullyProcessed(successOutputPath) { + const processed = new Set() + + if (!fs.existsSync(successOutputPath)) { + logDebug('No existing success file found, starting fresh', { + successOutputPath, + }) + return processed + } + + logDebug('Loading previously successful records from success file', { + successOutputPath, + }) + + return new Promise((resolve, reject) => { + fs.createReadStream(successOutputPath) + .pipe(csv.parse({ columns: true, trim: true })) + .on('data', row => { + if (row.recurly_account_code) { + processed.add(row.recurly_account_code) + } + }) + .on('end', () => { + logDebug('Loaded previously successful records', { + count: processed.size, + }) + resolve(processed) + }) + .on('error', err => { + logError('Failed to read success file', err, { successOutputPath }) + reject(err) + }) + }) +} + +/** + * Helper to write a CSV row with proper escaping + */ +// TODO: consider using a CSV library +function formatCsvRow(columns, row) { + const values = columns.map(col => { + const val = row[col] ?? '' + // Escape CSV values that contain commas, quotes, or newlines + if ( + typeof val === 'string' && + (val.includes(',') || val.includes('"') || val.includes('\n')) + ) { + return `"${val.replace(/"/g, '""')}"` + } + return val + }) + return values.join(',') + '\n' +} + +/** + * Create output writers for success, error, and skipped files. + * + * Success file: Append-only, contains all successfully updated records + * Errors file: Overwritten each run, contains only failures from this run + * Skipped file: Overwritten each run, contains records skipped because they have no stripe_customer_id + * + * @param {string} successPath - Path to the success output CSV file + * @param {string} errorsPath - Path to the errors output CSV file + * @param {string} skippedPath - Path to the skipped_no_stripe_id output CSV file + * @param {boolean} restart - If true, truncate existing files + * @returns {{ writeSuccess: (row: object) => void, writeError: (row: object) => void, writeSkipped: (row: object) => void, close: () => Promise }} + */ +function createOutputWriters( + successPath, + errorsPath, + skippedPath, + restart = false, + { enableSuccessFile = true } = {} +) { + // Success file columns + const successColumns = [ + 'recurly_account_code', + 'target_stripe_account', + 'stripe_customer_id', + ] + + // Errors file columns (includes error message) + const errorsColumns = [ + 'recurly_account_code', + 'target_stripe_account', + 'stripe_customer_id', + 'error', + ] + + // Skipped file columns (same as success, stripe_customer_id is the existing one) + const skippedColumns = [ + 'recurly_account_code', + 'target_stripe_account', + 'stripe_customer_id', + ] + + // Success file: append mode (unless restart) + // NOTE: In dry-run mode, we intentionally do NOT create or write to the success file, + // because commit mode uses it for resume/skip behavior. + const successStream = enableSuccessFile + ? (() => { + const successExists = fs.existsSync(successPath) + const successFlags = restart ? 'w' : 'a' + const stream = fs.createWriteStream(successPath, { + flags: successFlags, + }) + if (restart || !successExists) { + stream.write(successColumns.join(',') + '\n') + } + return stream + })() + : null + + // Errors file: always overwrite (contains only this run's errors) + const errorsStream = fs.createWriteStream(errorsPath, { flags: 'w' }) + errorsStream.write(errorsColumns.join(',') + '\n') + + // Skipped file: always overwrite (contains only this run's skipped) + const skippedStream = fs.createWriteStream(skippedPath, { flags: 'w' }) + skippedStream.write(skippedColumns.join(',') + '\n') + + function writeSuccess(row) { + if (!successStream) return + successStream.write(formatCsvRow(successColumns, row)) + } + + function writeError(row) { + errorsStream.write(formatCsvRow(errorsColumns, row)) + } + + function writeSkipped(row) { + skippedStream.write(formatCsvRow(skippedColumns, row)) + } + + async function close() { + if (successStream) successStream.end() + errorsStream.end() + skippedStream.end() + + const closers = [ + new Promise((resolve, reject) => { + errorsStream.on('finish', resolve) + errorsStream.on('error', reject) + }), + new Promise((resolve, reject) => { + skippedStream.on('finish', resolve) + skippedStream.on('error', reject) + }), + ] + + if (successStream) { + closers.unshift( + new Promise((resolve, reject) => { + successStream.on('finish', resolve) + successStream.on('error', reject) + }) + ) + } + + await Promise.all(closers) + } + + return { writeSuccess, writeError, writeSkipped, close } +} + +/** + * Get the errors file path from the success file path + */ +function getErrorsPath(successPath) { + return successPath.replace(/\.csv$/, '_errors.csv') +} + +/** + * Get the skipped_no_stripe_id file path from the success file path + */ +function getSkippedPath(successPath) { + return successPath.replace(/\.csv$/, '_skipped_no_stripe_id.csv') +} + +/** + * Get the stripe.json file path from the success file path (for dry-run mode) + */ +function getStripeJsonPath(successPath) { + return successPath.replace(/\.csv$/, '_stripe.json') +} + +// ============================================================================= +// RATE LIMITING +// ============================================================================= + +/** + * Rate limiter using sliding window algorithm. + * + * Rate limits (conservative targets, leaving headroom): + * - Recurly: 2000 requests per 5 minutes → target 1500/5min = 300/min = 5/sec + * https://support.recurly.com/hc/en-us/articles/360034160731-What-Are-Recurly-s-API-Rate-Limits + * - Stripe: 100 requests per second → target 50/sec (plenty of headroom) + * https://docs.stripe.com/rate-limits + * + * Recurly is the bottleneck. With 2 Recurly calls per customer (getAccount, getBillingInfo), + * we can process ~2.5 customers/second = ~150 customers/minute = ~9000 customers/hour. + * For 150K customers, expect ~17 hours at full throughput. + */ + +class RateLimiter { + /** + * @param {string} name - Name for logging + * @param {number} maxRequests - Maximum requests allowed in the window + * @param {number} windowMs - Window size in milliseconds + */ + constructor(name, maxRequests, windowMs) { + this.name = name + this.maxRequests = maxRequests + this.windowMs = windowMs + this.requests = [] // timestamps of recent requests + this.totalRequests = 0 + } + + /** + * Wait if necessary to stay within rate limits, then record the request. + */ + async throttle() { + const now = Date.now() + + // Remove requests outside the window + const windowStart = now - this.windowMs + this.requests = this.requests.filter(ts => ts > windowStart) + + // If at limit, wait until the oldest request exits the window + if (this.requests.length >= this.maxRequests) { + const oldestRequest = this.requests[0] + const waitTime = oldestRequest - windowStart + 1 + if (waitTime > 0) { + logDebug( + `Rate limit throttle for ${this.name}`, + { + waitMs: waitTime, + currentRequests: this.requests.length, + maxRequests: this.maxRequests, + }, + { verboseOnly: true } + ) + await setTimeout(waitTime) + } + // Clean up again after waiting + const newNow = Date.now() + this.requests = this.requests.filter(ts => ts > newNow - this.windowMs) + } + + // Record this request + this.requests.push(Date.now()) + this.totalRequests++ + } + + /** + * Get current rate (requests per second over the last window) + */ + getCurrentRate() { + const now = Date.now() + const windowStart = now - this.windowMs + const recentRequests = this.requests.filter(ts => ts > windowStart).length + return (recentRequests / this.windowMs) * 1000 // requests per second + } + + getStats() { + return { + name: this.name, + totalRequests: this.totalRequests, + currentWindowRequests: this.requests.length, + maxRequests: this.maxRequests, + currentRate: this.getCurrentRate().toFixed(2) + '/sec', + } + } +} + +// Recurly: 2000 requests per 5 minutes, target 1500 (75% of limit) for safety margin +const recurlyRateLimiter = new RateLimiter('Recurly', 1500, 5 * 60 * 1000) + +// Stripe: 100 requests per second, target 50 (50% of limit) - much more headroom +// Using 10-second window with 500 requests = 50/sec average +const stripeRateLimiter = new RateLimiter('Stripe', 500, 10 * 1000) + +/** + * Throttle before making a Recurly API call + */ +async function throttleRecurly() { + await recurlyRateLimiter.throttle() +} + +/** + * Throttle before making a Stripe API call + */ +async function throttleStripe() { + await stripeRateLimiter.throttle() +} + +/** + * Get rate limiter statistics for logging + */ +function getRateLimiterStats() { + return { + recurly: recurlyRateLimiter.getStats(), + stripe: stripeRateLimiter.getStats(), + } +} + +// ============================================================================= +// DATA TRANSFORMATION +// ============================================================================= + +/** + * Fetch Recurly account data for a given account code. + * + * @param {string} accountCode - The Recurly account code (Overleaf user ID) + * @returns {Promise<{account: object, billingInfo: object|null}>} + */ +async function fetchRecurlyData(accountCode) { + await throttleRecurly() + const account = await recurlyClient.getAccount(`code-${accountCode}`) + + let billingInfo = null + try { + await throttleRecurly() + billingInfo = await recurlyClient.getBillingInfo(`code-${accountCode}`) + } catch (error) { + // Billing info may not exist for manually billed customers + if (error instanceof recurly.errors.NotFoundError) { + // This is expected for some customers + } else { + throw error + } + } + + return { account, billingInfo } +} + +/** + * Fetch existing customer from the target Stripe account by ID. + * + * @param {Stripe} stripeClient - The Stripe client for the target account + * @param {string} stripeCustomerId - The Stripe customer ID + * @returns {Promise} + * @throws {Error} If customer is not found or is deleted + */ +async function fetchTargetStripeCustomer(stripeClient, stripeCustomerId) { + await throttleStripe() + const customer = await stripeClient.customers.retrieve(stripeCustomerId) + + if (customer.deleted) { + throw new Error(`Stripe customer ${stripeCustomerId} has been deleted`) + } + + return customer +} + +/** + * Replace a customer's tax IDs (delete any existing, then create the desired one). + * + * This makes re-runs more predictable for customers where a tax ID was created + * before a later step failed. + */ +async function replaceCustomerTaxIds( + stripeClient, + stripeCustomerId, + { taxIdType, vatNumber }, + context +) { + // Stripe customers can have multiple tax IDs. For this migration, we want a single + // authoritative tax ID derived from Recurly, so we remove any existing ones first. + const existingTaxIds = [] + + let startingAfter + while (true) { + await throttleStripe() + const page = await stripeClient.customers.listTaxIds(stripeCustomerId, { + limit: 100, + ...(startingAfter ? { starting_after: startingAfter } : {}), + }) + + existingTaxIds.push(...page.data) + + if (!page.has_more || page.data.length === 0) break + startingAfter = page.data[page.data.length - 1].id + } + + if (existingTaxIds.length > 0) { + logDebug( + 'Deleting existing Stripe tax IDs before creating new one', + { + ...context, + existingTaxIdCount: existingTaxIds.length, + }, + { verboseOnly: true } + ) + + for (const taxId of existingTaxIds) { + await throttleStripe() + await stripeClient.customers.deleteTaxId(stripeCustomerId, taxId.id) + } + } + + await throttleStripe() + return await stripeClient.customers.createTaxId(stripeCustomerId, { + type: taxIdType, + value: vatNumber, + }) +} + +/** + * Extract company name from Recurly data. + * + * Prefers billingInfo company as this is what the customer entered during checkout. + * Falls back to account company for manually-created accounts or legacy data. + * + * @param {object} account - Recurly account object + * @param {object|null} billingInfo - Recurly billing info object + * @returns {string|null} + */ +function extractCompanyName(account, billingInfo) { + // Prefer billing info company (entered during checkout) + if (billingInfo?.company) { + return billingInfo.company + } + // Fall back to account-level company (legacy or manually set) + if (account.company) { + return account.company + } + return null +} + +// ============================================================================= +// MAIN PROCESSING +// ============================================================================= + +/** + * Process a single customer row from the input CSV. + * + * Customers are expected to already exist in the target Stripe account + * (created via PAN import). This function updates them with additional + * data from Recurly. + * + * @param {object} row - CSV row with recurly_account_code, target_stripe_account, stripe_customer_id + * @param {number} rowNumber - The row number in the input file (for logging) + * @param {boolean} commit - Whether to actually update the customer + * @returns {Promise} - Result row for output CSV + */ +async function processCustomer(row, rowNumber, commit) { + const { + recurly_account_code: recurlyAccountCode, + target_stripe_account: targetStripeAccount, + stripe_customer_id: stripeCustomerId, + } = row + + const context = { + rowNumber, + recurlyAccountCode, + targetStripeAccount, + stripeCustomerId, + } + + const result = { + recurly_account_code: recurlyAccountCode, + target_stripe_account: targetStripeAccount, + stripe_customer_id: stripeCustomerId || '', + outcome: '', // 'updated', 'dry_run', 'skipped_no_stripe_id', or 'error' + error: '', + customerParams: null, // Stripe customer params (for dry-run output) + } + + try { + // Validate required fields + if (!recurlyAccountCode) { + throw new Error('Missing required field: recurly_account_code') + } + if (!targetStripeAccount) { + throw new Error('Missing required field: target_stripe_account') + } + + // TODO: In a later phase, records without stripe_customer_id will be + // handled differently (e.g., PayPal customers who need re-authorization). + // For now, skip them since we're only processing card customers that + // were imported via PAN import and have a stripe_customer_id. + if (!stripeCustomerId) { + result.outcome = 'skipped_no_stripe_id' + logDebug( + 'Skipping - no stripe_customer_id (not imported via PAN)', + { + ...context, + }, + { verboseOnly: true } + ) + return result + } + + // Get Stripe client for target account + logDebug( + 'Getting Stripe client', + { ...context, step: 'get_stripe_client' }, + { verboseOnly: true } + ) + // get Stripe client for the target account (strip 'stripe-' prefix if present) + const region = String(targetStripeAccount || '') + .trim() + .toLowerCase() + .replace(/^stripe-/, '') + const stripeClient = getRegionClient(region) + + // Fetch Recurly data + logDebug( + 'Fetching Recurly data', + { ...context, step: 'fetch_recurly' }, + { verboseOnly: true } + ) + const { account, billingInfo } = await fetchRecurlyData(recurlyAccountCode) + + logDebug( + 'Fetched Recurly account', + { + ...context, + email: account.email, + hasBillingInfo: !!billingInfo, + paymentMethod: billingInfo?.paypalBillingAgreementId + ? 'paypal' + : billingInfo?.cardType || 'none', + account, + billingInfo, + }, + { verboseOnly: true } + ) + + // TODO: Handle tax exemption + CC emails in later phases of the migration. + const hasCcEmails = + typeof account.ccEmails === 'string' + ? !!account.ccEmails.trim() + : !!account.ccEmails + if (hasCcEmails) { + logDebug( + 'Found CC emails on Recurly account - aborting', + { + ...context, + ccEmails: account.ccEmails, + }, + { verboseOnly: true } + ) + throw new Error( + 'Customer has ccEmails set in Recurly, but this migration does not yet handle CC invoice emails' + ) + } + if (account.exemptionCertificate) { + logDebug( + 'Found tax exemption certificate on Recurly account - aborting', + { + ...context, + exemptionCertificate: account.exemptionCertificate, + }, + { verboseOnly: true } + ) + throw new Error( + 'Customer appears to be tax exempt in Recurly, but this migration does not yet handle tax exemption status' + ) + } + + // Fetch existing customer from target Stripe account + logDebug( + 'Fetching existing Stripe customer', + { + ...context, + step: 'fetch_stripe_customer', + }, + { verboseOnly: true } + ) + const existingCustomer = await fetchTargetStripeCustomer( + stripeClient, + stripeCustomerId + ) + + logDebug( + 'Found existing Stripe customer', + { + ...context, + stripeEmail: existingCustomer.email, + stripeName: existingCustomer.name, + }, + { verboseOnly: true } + ) + + // Extract VAT number and country for tax ID creation + const vatNumber = coalesceOrThrowVATNumber(account, billingInfo) + let taxIdType = null + let country = null + let createdTaxId = null + + // Validate VAT number can be created if present + if (vatNumber) { + // We need to extract address first to get the country + const tempAddress = coalesceOrEqualOrThrowAddress(account, billingInfo) + country = tempAddress?.country + if (!country) { + throw new Error( + `Customer has VAT number (${vatNumber}) but no country in address - cannot determine tax ID type` + ) + } + taxIdType = getTaxIdType(country, vatNumber, tempAddress?.postal_code) + if (!taxIdType) { + throw new Error( + `Unable to determine tax id type for (${vatNumber}), country ${country}, postal_code ${tempAddress?.postal_code}` + ) + } + logDebug( + 'Will create tax ID', + { + ...context, + vatNumber, + country, + taxIdType, + }, + { verboseOnly: true } + ) + } + + if (commit) { + // Create tax ID first (validate it works before updating customer) + if (vatNumber && taxIdType) { + logDebug( + 'Creating tax ID', + { + ...context, + step: 'create_tax_id', + taxIdType, + vatNumber, + }, + { verboseOnly: true } + ) + + createdTaxId = await replaceCustomerTaxIds( + stripeClient, + stripeCustomerId, + { taxIdType, vatNumber }, + context + ) + logDebug( + 'Successfully created tax ID', + { + ...context, + taxId: createdTaxId.id, + taxIdType: createdTaxId.type, + taxIdValue: createdTaxId.value, + }, + { verboseOnly: true } + ) + } + } + + // Transform Recurly data to Stripe customer update params + logDebug( + 'Transforming Recurly data to Stripe params', + { + ...context, + step: 'transform', + }, + { verboseOnly: true } + ) + + const name = coalesceOrEqualOrThrowName(account, billingInfo) + const address = coalesceOrEqualOrThrowAddress(account, billingInfo) + const companyName = extractCompanyName(account, billingInfo) + + // TODO: Handle tax exempt status + // Recurly has account.exemptionCertificate for tax exemption + // Stripe has customer.tax_exempt: 'none' | 'exempt' | 'reverse' + // Need to determine when customers are tax exempt and how to map. + // Current Stripe checkout only allows tax exemption for US customers with EIN (us_ein). + // if (account.exemptionCertificate) { + // customerParams.tax_exempt = 'exempt' + // } + + // TODO: Handle CC emails + // Recurly account.ccEmails field contains additional notification emails. + // Stripe doesn't have a direct equivalent. + // Options: + // 1. Store in metadata (limited to 500 chars per value) + // 2. Handle via application logic outside of Stripe + // if (account.ccEmails) { + // customerParams.metadata.ccEmails = account.ccEmails + // } + + /** @type {Record} */ + const metadata = {} + if (account.createdAt) { + metadata.recurlyCreatedAt = account.createdAt.toISOString() + } + + /** @type {Stripe.CustomerUpdateParams} */ + const customerParams = { + email: account.email, + name, + metadata, + ...(address ? { address } : {}), + ...(companyName ? { business_name: companyName } : {}), + } + + logDebug( + 'Transformed customer params', + { + ...context, + params: customerParams, + }, + { verboseOnly: true } + ) + + if (commit) { + // Update customer in Stripe + logDebug( + 'Updating Stripe customer', + { + ...context, + step: 'update_customer', + }, + { verboseOnly: true } + ) + await throttleStripe() + await stripeClient.customers.update(stripeCustomerId, customerParams) + + result.outcome = 'updated' + logDebug( + 'Successfully updated Stripe customer', + { + ...context, + }, + { verboseOnly: true } + ) + } else { + result.outcome = 'dry_run' + result.customerParams = { + ...customerParams, + // Include tax ID info in dry-run output for review + _taxId: vatNumber + ? { + type: taxIdType, + value: vatNumber, + country, + createdTaxId, + } + : null, + } + logDebug( + 'DRY RUN: Would update Stripe customer', + { + ...context, + email: account.email, + taxId: vatNumber ? { type: taxIdType, value: vatNumber } : null, + }, + { verboseOnly: true } + ) + } + } catch (error) { + result.outcome = 'error' + // Include more error details + const errorDetails = [] + errorDetails.push(error.message) + if (error.code) errorDetails.push(`code=${error.code}`) + if (error.type) errorDetails.push(`type=${error.type}`) + if (error.statusCode) errorDetails.push(`statusCode=${error.statusCode}`) + result.error = errorDetails.join('; ') + + logError('Failed to process customer', error, context) + } + + return result +} + +function usage() { + console.error('Script to migrate Recurly customers to Stripe') + console.error('') + console.error('RESUMABLE: This script can be re-run after failures.') + console.error( + ' It will skip successfully processed records and retry failures.' + ) + console.error('') + console.error('Usage:') + console.error( + ' node scripts/recurly/migrate_recurly_customers_to_stripe.mjs [options]' + ) + console.error('') + console.error('Options:') + console.error(' --input, -i Path to input CSV file (required)') + console.error( + ' --output, -o Path to SUCCESS output CSV file (required)' + ) + console.error( + ' --commit Actually update customers in Stripe (default: dry-run)' + ) + console.error(' --verbose, -v Enable debug logging') + console.error( + ' --restart Ignore existing output files and start fresh' + ) + console.error('') + console.error('Input CSV format:') + console.error( + ' recurly_account_code,target_stripe_account,stripe_customer_id' + ) + console.error('') + console.error('Output files:') + console.error(' SUCCESS file (--output): Successfully updated customers') + console.error( + ' Format: recurly_account_code,target_stripe_account,stripe_customer_id' + ) + console.error('') + console.error( + ' SKIPPED file (_skipped_no_stripe_id.csv): Records without stripe_customer_id' + ) + console.error( + ' Format: recurly_account_code,target_stripe_account,stripe_customer_id' + ) + console.error('') + console.error( + ' ERRORS file (_errors.csv): Records that failed THIS run' + ) + console.error( + ' Format: recurly_account_code,target_stripe_account,stripe_customer_id,error' + ) + console.error('') + console.error( + ' STRIPE JSON (_stripe.json): Dry-run only - customer params that would be used for update' + ) + console.error('') + console.error('Resume behavior:') + console.error(' - Records in SUCCESS file are SKIPPED (already done)') + console.error( + ' - Records in ERRORS file are RE-PROCESSED (retried each run)' + ) + console.error( + ' - After each run, ERRORS file contains ONLY failures from that run' + ) + console.error( + ' - Use --restart to force processing all records from scratch' + ) +} + +function parseArgs() { + return minimist(process.argv.slice(2), { + alias: { i: 'input', o: 'output', h: 'help', v: 'verbose' }, + string: ['input', 'output'], + boolean: ['commit', 'verbose', 'help', 'restart'], + default: { commit: false, verbose: false, restart: false }, + }) +} + +async function main(trackProgress) { + const args = parseArgs() + const { + input: inputPath, + output: successOutputPath, + commit, + verbose, + help, + restart, + } = args + + // Set DEBUG_MODE only from CLI arg (--verbose/-v) + DEBUG_MODE = !!verbose + + if (help || !inputPath || !successOutputPath) { + usage() + process.exit(help ? 0 : 1) + } + + const errorsOutputPath = getErrorsPath(successOutputPath) + const skippedOutputPath = getSkippedPath(successOutputPath) + const stripeJsonPath = getStripeJsonPath(successOutputPath) + + const mode = commit ? 'COMMIT MODE' : 'DRY RUN MODE' + logDebug(`Starting migration in ${mode}`, { + inputPath, + successOutputPath, + errorsOutputPath, + skippedOutputPath, + ...(commit ? {} : { stripeJsonPath }), + }) + await trackProgress(`Starting migration in ${mode}`) + + // Load previously successfully processed records (for resume functionality). + // IMPORTANT: commit mode uses the success file for resume/skip behavior. + // Dry-run mode does NOT read the success file. + let previouslyProcessed = new Set() + if (commit && !restart) { + try { + previouslyProcessed = await loadSuccessfullyProcessed(successOutputPath) + if (previouslyProcessed.size > 0) { + logDebug( + `Will skip ${previouslyProcessed.size} previously successful records` + ) + await trackProgress( + `Resuming: will skip ${previouslyProcessed.size} previously successful records` + ) + } + } catch (err) { + logWarn('Could not load previous success file, starting fresh', { + error: err.message, + }) + } + } else if (restart) { + logDebug('Restart flag set, ignoring existing output files') + await trackProgress('Restart mode: processing all records from scratch') + } + + // Create output writers. + // In dry-run mode, we intentionally do NOT write to the success file, because + // commit mode uses it for resume/skip behavior. + const { + writeSuccess, + writeError, + writeSkipped, + close: closeOutputs, + } = commit + ? createOutputWriters( + successOutputPath, + errorsOutputPath, + skippedOutputPath, + restart, + { enableSuccessFile: true } + ) + : createOutputWriters( + successOutputPath, + errorsOutputPath, + skippedOutputPath, + true, + { enableSuccessFile: false } + ) + + // For dry-run mode, collect Stripe customer params to write to JSON + const stripeCustomerParams = [] + + // Statistics + let totalInInput = 0 + let processedThisRun = 0 + let skippedPreviouslyProcessed = 0 + let updatedCount = 0 + let skippedNoStripeIdCount = 0 + let errorCount = 0 + let dryRunCount = 0 + + // Track errors for final summary (just the account codes, not full results - memory efficient) + const errorAccountCodes = [] + + logDebug('Beginning to process input file', { inputPath }) + + // Process input CSV - true streaming (no collecting results in memory) + const inputStream = fs.createReadStream(inputPath) + const parser = csv.parse({ columns: true, trim: true }) + + inputStream.pipe(parser) + + let rowNumber = 0 + for await (const row of parser) { + rowNumber++ + totalInInput++ + + const accountCode = row.recurly_account_code + + // Check if already successfully processed in a previous run + if (previouslyProcessed.has(accountCode)) { + skippedPreviouslyProcessed++ + logDebug( + 'Skipping previously successful record', + { + rowNumber, + accountCode, + }, + { verboseOnly: true } + ) + continue + } + + // Process this customer + const result = await processCustomer(row, rowNumber, commit) + + processedThisRun++ + + // Write to appropriate output file based on outcome + if (result.outcome === 'error') { + writeError(result) + errorCount++ + errorAccountCodes.push(accountCode) + } else if (result.outcome === 'skipped_no_stripe_id') { + writeSkipped(result) + skippedNoStripeIdCount++ + } else { + writeSuccess(result) + // Update statistics and collect dry-run data + if (result.outcome === 'updated') { + updatedCount++ + } else if (result.outcome === 'dry_run') { + dryRunCount++ + // Collect customer params for stripe.json output + if (result.customerParams) { + stripeCustomerParams.push({ + recurly_account_code: result.recurly_account_code, + target_stripe_account: result.target_stripe_account, + customerParams: result.customerParams, + }) + } + } + } + + // Progress update every 1000 customers (or 100 in debug mode) + const progressInterval = DEBUG_MODE ? 100 : 1000 + if (processedThisRun % progressInterval === 0) { + const rateLimiterStats = getRateLimiterStats() + const progress = { + rowNumber, + processedThisRun, + updated: updatedCount, + dryRun: dryRunCount, + skippedNoStripeId: skippedNoStripeIdCount, + errors: errorCount, + skippedPrevious: skippedPreviouslyProcessed, + recurlyRate: rateLimiterStats.recurly.currentRate, + stripeRate: rateLimiterStats.stripe.currentRate, + } + logDebug('Progress update', progress) + await trackProgress( + `Progress: row ${rowNumber}, ${processedThisRun} processed this run, ${errorCount} errors` + ) + } + } + + // Close output streams + await closeOutputs() + + // Write stripe.json file in dry-run mode + if (!commit && stripeCustomerParams.length > 0) { + await fs.promises.writeFile( + stripeJsonPath, + JSON.stringify(stripeCustomerParams, null, 2) + ) + logDebug( + `Wrote ${stripeCustomerParams.length} customer params to ${stripeJsonPath}` + ) + } + + // Final summary + const totalSuccessful = commit + ? previouslyProcessed.size + updatedCount + : previouslyProcessed.size + const finalRateLimiterStats = getRateLimiterStats() + + logDebug('=== FINAL SUMMARY ===') + logDebug(`Input file total rows: ${totalInInput}`) + logDebug(`Previously successful (skipped): ${skippedPreviouslyProcessed}`) + logDebug(`Processed this run: ${processedThisRun}`) + logDebug( + ` - ${commit ? 'Updated' : 'Would update'}: ${commit ? updatedCount : dryRunCount}` + ) + logDebug(` - Skipped (no stripe_customer_id): ${skippedNoStripeIdCount}`) + logDebug(` - Errors: ${errorCount}`) + if (commit) { + logDebug(`Total in success file: ${totalSuccessful}`) + } + logDebug(`Total in skipped file: ${skippedNoStripeIdCount}`) + logDebug(`Total in errors file: ${errorCount}`) + logDebug( + `API calls - Recurly: ${finalRateLimiterStats.recurly.totalRequests}, Stripe: ${finalRateLimiterStats.stripe.totalRequests}` + ) + + await trackProgress('=== FINAL SUMMARY ===') + await trackProgress(`Input file total rows: ${totalInInput}`) + await trackProgress( + `Previously successful (skipped): ${skippedPreviouslyProcessed}` + ) + await trackProgress(`Processed this run: ${processedThisRun}`) + await trackProgress( + ` - ${commit ? 'Updated' : 'Would update'}: ${commit ? updatedCount : dryRunCount}` + ) + await trackProgress( + ` - Skipped (no stripe_customer_id): ${skippedNoStripeIdCount}` + ) + await trackProgress(` - Errors: ${errorCount}`) + await trackProgress('') + if (commit) { + await trackProgress( + `Success file: ${successOutputPath} (${totalSuccessful} records)` + ) + } else { + await trackProgress( + `Success file: ${successOutputPath} (not modified in dry-run mode)` + ) + } + await trackProgress( + `Skipped file: ${skippedOutputPath} (${skippedNoStripeIdCount} records)` + ) + await trackProgress( + `Errors file: ${errorsOutputPath} (${errorCount} records)` + ) + + if (!commit && dryRunCount > 0) { + await trackProgress('') + await trackProgress( + `Stripe params file: ${stripeJsonPath} (${stripeCustomerParams.length} records)` + ) + await trackProgress( + 'To actually update customers, run the script with --commit flag' + ) + + logDebug('Dry-run params file written', { + stripeJsonPath, + records: stripeCustomerParams.length, + }) + } + + // Log error account codes for easy reference + if (errorCount > 0) { + logWarn(`${errorCount} records failed and are in the errors file.`) + logWarn('Failed account codes:', { + first20: errorAccountCodes.slice(0, 20), + totalErrors: errorAccountCodes.length, + }) + await trackProgress('') + await trackProgress( + `${errorCount} records failed. Re-run the script to retry them.` + ) + await trackProgress( + `Failed accounts (first 20): ${errorAccountCodes.slice(0, 20).join(', ')}` + ) + } + + // Success/warning based on errors + if (errorCount === 0) { + logDebug('Migration completed successfully', { mode }) + await trackProgress(`Migration completed successfully in ${mode}`) + + // If no errors and errors file exists but is empty (just header), note that + if (fs.existsSync(errorsOutputPath)) { + await trackProgress( + `Errors file is empty (header only) - all records processed successfully!` + ) + } + } else { + logWarn('Migration completed with errors', { mode, errorCount }) + await trackProgress( + `Migration completed with ${errorCount} errors in ${mode}` + ) + } + + // Return exit code based on whether there were errors + return errorCount === 0 ? 0 : 1 +} + +// Execute the script using the runner +try { + const exitCode = await scriptRunner(main) + process.exit(exitCode ?? 0) +} catch (error) { + logError('Script failed with unhandled error', error) + process.exit(1) +}