diff --git a/services/web/config/settings.defaults.js b/services/web/config/settings.defaults.js index c4be8ec61f..103289f38a 100644 --- a/services/web/config/settings.defaults.js +++ b/services/web/config/settings.defaults.js @@ -789,7 +789,12 @@ module.exports = { editorLeftMenuSync: [], }, - moduleImportSequence: ['launchpad', 'server-ce-scripts', 'user-activate'], + moduleImportSequence: [ + 'launchpad', + 'server-ce-scripts', + 'user-activate', + 'history-migration', + ], csp: { enabled: process.env.CSP_ENABLED === 'true', diff --git a/services/web/scripts/history/HistoryUpgradeHelper.js b/services/web/modules/history-migration/app/src/HistoryUpgradeHelper.js similarity index 86% rename from services/web/scripts/history/HistoryUpgradeHelper.js rename to services/web/modules/history-migration/app/src/HistoryUpgradeHelper.js index 4fb48364cb..d8eb291bcf 100644 --- a/services/web/scripts/history/HistoryUpgradeHelper.js +++ b/services/web/modules/history-migration/app/src/HistoryUpgradeHelper.js @@ -1,12 +1,12 @@ const { ReadPreference, ObjectId } = require('mongodb') -const { db } = require('../../app/src/infrastructure/mongodb') +const { db } = require('../../../../app/src/infrastructure/mongodb') const Settings = require('@overleaf/settings') -const ProjectHistoryHandler = require('../../app/src/Features/Project/ProjectHistoryHandler') -const HistoryManager = require('../../app/src/Features/History/HistoryManager') -const ProjectHistoryController = require('../../modules/admin-panel/app/src/ProjectHistoryController') -const ProjectEntityHandler = require('../../app/src/Features/Project/ProjectEntityHandler') -const ProjectEntityUpdateHandler = require('../../app/src/Features/Project/ProjectEntityUpdateHandler') +const ProjectHistoryHandler = require('../../../../app/src/Features/Project/ProjectHistoryHandler') +const HistoryManager = require('../../../../app/src/Features/History/HistoryManager') +const ProjectHistoryController = require('../../../admin-panel/app/src/ProjectHistoryController') +const ProjectEntityHandler = require('../../../../app/src/Features/Project/ProjectEntityHandler') +const ProjectEntityUpdateHandler = require('../../../../app/src/Features/Project/ProjectEntityUpdateHandler') // Timestamp of when 'Enable history for SL in background' release const ID_WHEN_FULL_PROJECT_HISTORY_ENABLED = '5a8d8a370000000000000000' @@ -16,6 +16,21 @@ const OBJECT_ID_WHEN_FULL_PROJECT_HISTORY_ENABLED = new ObjectId( const DATETIME_WHEN_FULL_PROJECT_HISTORY_ENABLED = OBJECT_ID_WHEN_FULL_PROJECT_HISTORY_ENABLED.getTimestamp() +async function countProjects(query = {}) { + const count = await db.projects.count(query) + return count +} + +async function countDocHistory(query = {}) { + const count = await db.docHistory.count(query) + return count +} + +async function findProjects(query = {}, projection = {}) { + const projects = await db.projects.find(query).project(projection).toArray() + return projects +} + async function determineProjectHistoryType(project) { if (project.overleaf && project.overleaf.history) { if (project.overleaf.history.upgradeFailed) { @@ -212,18 +227,28 @@ async function doUpgradeForNoneWithoutConversion(project) { return result } -async function doUpgradeForNoneWithConversion(project) { +async function doUpgradeForNoneWithConversion(project, options = {}) { const result = {} const projectId = project._id // migrateProjectHistory expects project id as a string const projectIdString = project._id.toString() try { - await ProjectHistoryController.migrateProjectHistory(projectIdString) + if (options.convertLargeDocsToFile) { + result.convertedDocCount = await convertLargeDocsToFile( + projectId, + options.userId + ) + } + await ProjectHistoryController.migrateProjectHistory( + projectIdString, + options.migrationOptions + ) } catch (err) { // if migrateProjectHistory fails, it cleans up by deleting // the history and unsetting the history id // therefore a failed project will still look like a 'None with conversion' project result.error = err + // We set a failed flag so future runs of the script don't automatically retry await db.projects.updateOne( { _id: projectId }, { @@ -238,7 +263,8 @@ async function doUpgradeForNoneWithConversion(project) { { _id: projectId }, { $set: { - 'overleaf.history.upgradeReason': `none-with-conversion`, + 'overleaf.history.upgradeReason': + `none-with-conversion` + options.reason ? `/${options.reason}` : ``, }, $unset: { 'overleaf.history.upgradeFailed': true, @@ -328,6 +354,9 @@ function docIsTooLarge(estimatedSize, lines, maxDocLength) { } module.exports = { + countProjects, + countDocHistory, + findProjects, determineProjectHistoryType, getUpgradeFunctionForType, upgradeProject, diff --git a/services/web/modules/history-migration/index.js b/services/web/modules/history-migration/index.js new file mode 100644 index 0000000000..4ba52ba2c8 --- /dev/null +++ b/services/web/modules/history-migration/index.js @@ -0,0 +1 @@ +module.exports = {} diff --git a/services/web/scripts/history/count_project_history_categories.js b/services/web/scripts/history/count_project_history_categories.js index e3c54a3f27..8b15735230 100644 --- a/services/web/scripts/history/count_project_history_categories.js +++ b/services/web/scripts/history/count_project_history_categories.js @@ -11,7 +11,10 @@ process.env.MONGO_SOCKET_TIMEOUT = const { promiseMapWithLimit } = require('../../app/src/util/promises') const { batchedUpdate } = require('../helpers/batchedUpdate') -const { determineProjectHistoryType } = require('./HistoryUpgradeHelper') +const { + determineProjectHistoryType, + countProjects, +} = require('../../modules/history-migration/app/src/HistoryUpgradeHelper') const COUNT = { V2: 0, @@ -22,6 +25,8 @@ const COUNT = { NoneWithTemporaryHistory: 0, UpgradeFailed: 0, ConversionFailed: 0, + MigratedProjects: 0, + TotalProjects: 0, } async function processBatch(_, projects) { @@ -60,6 +65,10 @@ async function main() { projection, options ) + COUNT.MigratedProjects = await countProjects({ + 'overleaf.history.display': true, + }) + COUNT.TotalProjects = await countProjects() console.log('Final') console.log(COUNT) } diff --git a/services/web/scripts/history/upgrade_none_with_conversion_if_sl_history.js b/services/web/scripts/history/upgrade_none_with_conversion_if_sl_history.js index a92f55eb98..e33174a654 100644 --- a/services/web/scripts/history/upgrade_none_with_conversion_if_sl_history.js +++ b/services/web/scripts/history/upgrade_none_with_conversion_if_sl_history.js @@ -29,12 +29,15 @@ const USER_ID = process.env.USER_ID const CONVERT_LARGE_DOCS_TO_FILE = process.env.CONVERT_LARGE_DOCS_TO_FILE === 'true' -const { ObjectId, ReadPreference } = require('mongodb') +const { ObjectId } = require('mongodb') const { db, waitForDb } = require('../../app/src/infrastructure/mongodb') const { promiseMapWithLimit } = require('../../app/src/util/promises') const { batchedUpdate } = require('../helpers/batchedUpdate') -const ProjectHistoryController = require('../../modules/admin-panel/app/src/ProjectHistoryController') -const HistoryUpgradeHelper = require('./HistoryUpgradeHelper') +const { + anyDocHistoryExists, + anyDocHistoryIndexExists, + doUpgradeForNoneWithConversion, +} = require('../../modules/history-migration/app/src/HistoryUpgradeHelper') console.log({ DRY_RUN, @@ -111,99 +114,49 @@ async function processProject(project) { } } } - const anyDocHistory = await anyDocHistoryExists(project) - if (anyDocHistory) { - return await doUpgradeForNoneWithConversion(project) - } - const anyDocHistoryIndex = await anyDocHistoryIndexExists(project) - if (anyDocHistoryIndex) { - return await doUpgradeForNoneWithConversion(project) - } -} - -async function doUpgradeForNoneWithConversion(project) { if (RESULT.failed >= MAX_FAILURES) { return } if (MAX_UPGRADES_TO_ATTEMPT && RESULT.attempted >= MAX_UPGRADES_TO_ATTEMPT) { return } - RESULT.attempted += 1 - const projectId = project._id - // migrateProjectHistory expects project id as a string - const projectIdString = project._id.toString() - if (!DRY_RUN) { - try { - if (CONVERT_LARGE_DOCS_TO_FILE) { - const convertedDocCount = - await HistoryUpgradeHelper.convertLargeDocsToFile(projectId, USER_ID) - console.log( - `converted ${convertedDocCount} large docs to binary files for project ${projectId}` - ) - } - await ProjectHistoryController.migrateProjectHistory(projectIdString, { + const anyDocHistoryOrIndex = + (await anyDocHistoryExists(project)) || + (await anyDocHistoryIndexExists(project)) + if (anyDocHistoryOrIndex) { + RESULT.attempted += 1 + if (DRY_RUN) { + return + } + const result = await doUpgradeForNoneWithConversion(project, { + migrationOptions: { archiveOnFailure: ARCHIVE_ON_FAILURE, fixInvalidCharacters: FIX_INVALID_CHARACTERS, forceNewHistoryOnFailure: FORCE_NEW_HISTORY_ON_FAILURE, importZipFilePath: IMPORT_ZIP_FILE_PATH, cutoffDate: CUTOFF_DATE, - }) - } catch (err) { - // if migrateProjectHistory fails, it cleans up by deleting - // the history and unsetting the history id - // therefore a failed project will still look like a 'None with conversion' project - RESULT.failed += 1 - console.error(`project ${projectId} FAILED with error: `, err) - // We set a failed flag so future runs of the script don't automatically retry - await db.projects.updateOne( - { _id: projectId }, - { - $set: { - 'overleaf.history.conversionFailed': true, - }, - } + }, + convertLargeDocsToFile: CONVERT_LARGE_DOCS_TO_FILE, + userId: USER_ID, + reason: `${SCRIPT_VERSION}`, + }) + if (result.convertedDocCount) { + console.log( + `project ${project._id} converted ${result.convertedDocCount} docs to filestore` ) - return } - await db.projects.updateOne( - { _id: projectId }, - { - $set: { - 'overleaf.history.upgradeReason': `none-with-conversion/${SCRIPT_VERSION}`, - }, - $unset: { - 'overleaf.history.upgradeFailed': true, - 'overleaf.history.conversionFailed': true, - }, + if (result.error) { + console.error(`project ${project._id} FAILED with error: `, result.error) + RESULT.failed += 1 + } else if (result.upgraded) { + if (VERBOSE_LOGGING) { + console.log( + `project ${project._id} converted and upgraded to full project history` + ) } - ) - } - if (VERBOSE_LOGGING) { - console.log( - `project ${projectId} converted and upgraded to full project history` - ) - } - RESULT.projectsUpgraded += 1 -} - -async function anyDocHistoryExists(project) { - return await db.docHistory.findOne( - { project_id: { $eq: project._id } }, - { - projection: { _id: 1 }, - readPreference: ReadPreference.SECONDARY, + RESULT.projectsUpgraded += 1 } - ) -} - -async function anyDocHistoryIndexExists(project) { - return await db.docHistoryIndex.findOne( - { project_id: { $eq: project._id } }, - { - projection: { _id: 1 }, - readPreference: ReadPreference.SECONDARY, - } - ) + } } async function main() { diff --git a/services/web/scripts/history/upgrade_project.js b/services/web/scripts/history/upgrade_project.js index 87767bf55a..5bf653903a 100644 --- a/services/web/scripts/history/upgrade_project.js +++ b/services/web/scripts/history/upgrade_project.js @@ -1,6 +1,8 @@ const { ReadPreference, ObjectId } = require('mongodb') const { db, waitForDb } = require('../../app/src/infrastructure/mongodb') -const { upgradeProject } = require('./HistoryUpgradeHelper') +const { + upgradeProject, +} = require('../../modules/history-migration/app/src/HistoryUpgradeHelper') async function processProject(project) { const result = await upgradeProject(project)