mirror of
https://github.com/yu-i-i/overleaf-cep.git
synced 2026-11-14 14:13:43 +01:00
Merge pull request #13327 from overleaf/msm-web-track-changes-cleanup
[web] Cleanup track-changes from web GitOrigin-RevId: 8cef709ec5e91e4ffe8cd06826038ed84f36ef67
This commit is contained in:
Miguel Serrano
@@ -147,7 +147,6 @@ async function deleteDocFromRedis(projectId, docId) {
|
||||
`UnflushedTime:{${docId}}`,
|
||||
`Pathname:{${docId}}`,
|
||||
`ProjectHistoryId:{${docId}}`,
|
||||
`ProjectHistoryType:{${docId}}`,
|
||||
`PendingUpdates:{${docId}}`,
|
||||
`lastUpdatedAt:{${docId}}`,
|
||||
`lastUpdatedBy:{${docId}}`
|
||||
|
||||
@@ -1,60 +0,0 @@
|
||||
const { waitForDb, db } = require('../../app/src/infrastructure/mongodb')
|
||||
|
||||
async function main() {
|
||||
await checkAllProjectsAreMigrated()
|
||||
await setAllowDowngradeToFalse()
|
||||
await deleteHistoryCollections()
|
||||
console.log('Legacy history data cleaned up successfully')
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
async function checkAllProjectsAreMigrated() {
|
||||
console.log('checking all projects are migrated to Full Project History')
|
||||
|
||||
const count = await db.projects.countDocuments({
|
||||
'overleaf.history.display': { $ne: true },
|
||||
})
|
||||
|
||||
if (count === 0) {
|
||||
console.log('All projects are migrated to Full Project History')
|
||||
} else {
|
||||
console.error(
|
||||
`There are ${count} projects that are not migrated to Full Project History` +
|
||||
` please complete the migration before running this script again.`
|
||||
)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
async function setAllowDowngradeToFalse() {
|
||||
console.log('unsetting `allowDowngrade` flag in all projects')
|
||||
await db.projects.updateMany(
|
||||
{
|
||||
'overleaf.history.id': { $exists: true },
|
||||
'overleaf.history.allowDowngrade': true,
|
||||
},
|
||||
{ $unset: { 'overleaf.history.allowDowngrade': 1 } }
|
||||
)
|
||||
console.log('unsetting `allowDowngrade` flag in all projects - Done')
|
||||
}
|
||||
|
||||
async function deleteHistoryCollections() {
|
||||
console.log('removing `docHistory` data')
|
||||
await db.docHistory.deleteMany({})
|
||||
console.log('removing `docHistory` data - Done')
|
||||
|
||||
console.log('removing `docHistoryIndex` data')
|
||||
await db.docHistoryIndex.deleteMany({})
|
||||
console.log('removing `docHistoryIndex` data - Done')
|
||||
|
||||
console.log('removing `projectHistoryMetaData` data')
|
||||
await db.projectHistoryMetaData.deleteMany({})
|
||||
console.log('removing `projectHistoryMetaData` data - Done')
|
||||
}
|
||||
|
||||
waitForDb()
|
||||
.then(main)
|
||||
.catch(err => {
|
||||
console.error(err)
|
||||
process.exit(1)
|
||||
})
|
||||
@@ -1,84 +0,0 @@
|
||||
const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true'
|
||||
const VERBOSE_PROJECT_NAMES = process.env.VERBOSE_PROJECT_NAMES === 'true'
|
||||
const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 50
|
||||
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 500
|
||||
const USE_QUERY_HINT = process.env.USE_QUERY_HINT !== 'false'
|
||||
// persist fallback in order to keep batchedUpdate in-sync
|
||||
process.env.BATCH_SIZE = BATCH_SIZE
|
||||
// raise mongo timeout to 1hr if otherwise unspecified
|
||||
process.env.MONGO_SOCKET_TIMEOUT =
|
||||
parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000
|
||||
|
||||
const { promiseMapWithLimit } = require('../../app/src/util/promises')
|
||||
const { batchedUpdate } = require('../helpers/batchedUpdate')
|
||||
const {
|
||||
determineProjectHistoryType,
|
||||
countProjects,
|
||||
} = require('../../modules/history-migration/app/src/HistoryUpgradeHelper')
|
||||
|
||||
const COUNT = {
|
||||
V2: 0,
|
||||
V1WithoutConversion: 0,
|
||||
V1WithConversion: 0,
|
||||
NoneWithoutConversion: 0,
|
||||
NoneWithConversion: 0,
|
||||
NoneWithTemporaryHistory: 0,
|
||||
UpgradeFailed: 0,
|
||||
ConversionFailed: 0,
|
||||
MigratedProjects: 0,
|
||||
TotalProjects: 0,
|
||||
}
|
||||
|
||||
async function processBatch(projects) {
|
||||
await promiseMapWithLimit(WRITE_CONCURRENCY, projects, processProject)
|
||||
console.log(COUNT)
|
||||
}
|
||||
|
||||
async function processProject(project) {
|
||||
const historyType = await determineProjectHistoryType(project)
|
||||
if (VERBOSE_LOGGING) {
|
||||
console.log(
|
||||
`project ${
|
||||
project[VERBOSE_PROJECT_NAMES ? 'name' : '_id']
|
||||
} is type ${historyType}`
|
||||
)
|
||||
}
|
||||
COUNT[historyType] += 1
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const projection = {
|
||||
_id: 1,
|
||||
overleaf: 1,
|
||||
}
|
||||
const options = {}
|
||||
if (USE_QUERY_HINT) {
|
||||
options.hint = { _id: 1 }
|
||||
}
|
||||
if (VERBOSE_PROJECT_NAMES) {
|
||||
projection.name = 1
|
||||
}
|
||||
await batchedUpdate(
|
||||
'projects',
|
||||
{ 'overleaf.history.display': { $ne: true } },
|
||||
processBatch,
|
||||
projection,
|
||||
options
|
||||
)
|
||||
COUNT.MigratedProjects = await countProjects({
|
||||
'overleaf.history.display': true,
|
||||
})
|
||||
COUNT.TotalProjects = await countProjects()
|
||||
console.log('Final')
|
||||
console.log(COUNT)
|
||||
}
|
||||
|
||||
main()
|
||||
.then(() => {
|
||||
console.error('Done.')
|
||||
process.exit(0)
|
||||
})
|
||||
.catch(error => {
|
||||
console.error({ error })
|
||||
process.exit(1)
|
||||
})
|
||||
@@ -1,328 +0,0 @@
|
||||
// Script to debug the track-changes history of the documents in a project.
|
||||
// Usage:
|
||||
// node debug_history.js --project-id=<project_id>
|
||||
// node debug_history.js --all # to check all unmigrated projects
|
||||
//
|
||||
// Example output:
|
||||
// $ node scripts/debug_history.js --project-id=63ff3adc06177192f18a6b38
|
||||
// Using default settings from /overleaf/services/track-changes/config/settings.defaults.js
|
||||
// Set UV_THREADPOOL_SIZE=16
|
||||
// project 63ff3adc06177192f18a6b38 docId 63ff3adc06177192f18a6b3d OK
|
||||
// project 63ff3adc06177192f18a6b38 docId 63ff3b08de41e3b0989c1720 FAILED
|
||||
// {"action":"rewinding","version":7,"meta":{"start_ts":1677671465447,"end_ts":1677671465447,"user_id":"632ae106f9a6dd002505765b"},
|
||||
// "ops":[{"action":"rewindOp","contentLength":24,"op":{"p":32,"d":6},"errors":[{"message":"invalid offset rewinding delete,
|
||||
// truncating to content length","op":{"p":32,"d":6},"contentLength":24}]}],"status":"failed"}
|
||||
|
||||
/* eslint-disable camelcase */
|
||||
const TrackChangesMongoDb = require('../../../track-changes/app/js/mongodb')
|
||||
const { waitForDb } = require('../../app/src/infrastructure/mongodb')
|
||||
const {
|
||||
findProjects,
|
||||
} = require('../../modules/history-migration/app/src/HistoryUpgradeHelper')
|
||||
const PackManager = require('../../../track-changes/app/js/PackManager')
|
||||
const {
|
||||
packsAreDuplicated,
|
||||
} = require('../../../track-changes/app/js/util/PackUtils')
|
||||
const {
|
||||
ConsistencyError,
|
||||
} = require('../../../track-changes/app/js/DiffGenerator')
|
||||
const DocumentUpdaterManager = require('../../../track-changes/app/js/DocumentUpdaterManager')
|
||||
const DocstoreManager = require('../../../track-changes/app/js/DocstoreManager')
|
||||
const Errors = require('../../../track-changes/app/js/Errors')
|
||||
const minimist = require('minimist')
|
||||
const util = require('util')
|
||||
const logger = require('@overleaf/logger')
|
||||
logger.initialize('debug-history')
|
||||
// disable logging to stdout from internal modules
|
||||
logger.logger.streams = []
|
||||
|
||||
const options = {
|
||||
boolean: ['all', 'verbose', 'raw', 'help'],
|
||||
string: ['project-id'],
|
||||
alias: {
|
||||
'project-id': 'p',
|
||||
verbose: 'v',
|
||||
raw: 'r',
|
||||
help: 'h',
|
||||
all: 'a',
|
||||
},
|
||||
default: {},
|
||||
}
|
||||
const argv = minimist(process.argv.slice(2), options)
|
||||
|
||||
function usage() {
|
||||
console.log(
|
||||
`Usage: ${process.argv[1]} [--project-id=<project_id> | --all] [--verbose] [--raw]`
|
||||
)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
// look in docstore or docupdater for the latest version of the document
|
||||
async function getLatestContent(projectId, docId, lastUpdateVersion) {
|
||||
const [docstoreContent, docstoreVersion] =
|
||||
await DocstoreManager.promises.peekDocument(projectId, docId)
|
||||
|
||||
// if docstore is out of date, check for a newer version in docupdater
|
||||
// and return that instead
|
||||
if (docstoreVersion <= lastUpdateVersion) {
|
||||
const [docupdaterContent, docupdaterVersion] =
|
||||
await DocumentUpdaterManager.promises.peekDocument(projectId, docId)
|
||||
if (docupdaterVersion > docstoreVersion) {
|
||||
return [docupdaterContent, docupdaterVersion]
|
||||
}
|
||||
}
|
||||
|
||||
return [docstoreContent, docstoreVersion]
|
||||
}
|
||||
|
||||
// This class is used to write a record of all the operations that have been applied to a document
|
||||
class LogAppliedOps {
|
||||
constructor() {
|
||||
this.result = []
|
||||
}
|
||||
|
||||
// used to log the initial state of the document
|
||||
start(action, latestContent, version) {
|
||||
this.result.push({
|
||||
action,
|
||||
latestContentLength: latestContent.length,
|
||||
latestContent: argv.raw ? latestContent : undefined,
|
||||
version,
|
||||
})
|
||||
}
|
||||
|
||||
// used to log a new document update
|
||||
update(action, update) {
|
||||
this._finalize()
|
||||
this.opResults = []
|
||||
this.currentResult = {
|
||||
action,
|
||||
version: update.v,
|
||||
meta: update.meta,
|
||||
ops: this.opResults,
|
||||
}
|
||||
this.result.push(this.currentResult)
|
||||
}
|
||||
|
||||
// used to log an operation that has been applied to the document
|
||||
op(action, content, op) {
|
||||
this.currentOp = {
|
||||
action,
|
||||
contentLength: content.length,
|
||||
content: argv.raw ? content : undefined,
|
||||
op: this._filterOp(op),
|
||||
}
|
||||
this.opResults.push(this.currentOp)
|
||||
}
|
||||
|
||||
// used to log an error that occurred while applying an operation
|
||||
opError(message, content, op, err) {
|
||||
this.currentOp.errors = this.currentOp.errors || []
|
||||
this.currentOp.errors.push({
|
||||
message,
|
||||
op: this._filterOp(op),
|
||||
contentLength: content.length,
|
||||
content: argv.raw ? content : undefined,
|
||||
err,
|
||||
})
|
||||
}
|
||||
|
||||
// sets the status of the current update to 'success' or 'failed'
|
||||
// depending on whether any errors were logged
|
||||
_finalize() {
|
||||
if (!this.currentResult) {
|
||||
return
|
||||
}
|
||||
const errors = this.opResults.some(op => op.errors)
|
||||
this.currentResult.status = errors ? 'failed' : 'success'
|
||||
}
|
||||
|
||||
// returns the final result of the log
|
||||
end() {
|
||||
this._finalize()
|
||||
return this.result
|
||||
}
|
||||
|
||||
// Returns a new object with the same keys as op, but with the i and d
|
||||
// fields replaced by their lengths when present. This is used to filter
|
||||
// out the contents of the i and d fields of an operation, to redact
|
||||
// document content.
|
||||
_filterOp(op) {
|
||||
const newOp = {}
|
||||
for (const key of Object.keys(op)) {
|
||||
if (!argv.raw && (key === 'i' || key === 'd')) {
|
||||
newOp[key] = op[key].length
|
||||
} else {
|
||||
newOp[key] = op[key]
|
||||
}
|
||||
}
|
||||
return newOp
|
||||
}
|
||||
}
|
||||
|
||||
// This is the rewindOp function from track-changes, modified to log
|
||||
// the operation and any errors.
|
||||
function rewindOp(content, op, log) {
|
||||
if (op.i != null) {
|
||||
// ShareJS will accept an op where p > content.length when applied,
|
||||
// and it applies as though p == content.length. However, the op is
|
||||
// passed to us with the original p > content.length. Detect if that
|
||||
// is the case with this op, and shift p back appropriately to match
|
||||
// ShareJS if so.
|
||||
let { p } = op
|
||||
const maxP = content.length - op.i.length
|
||||
if (p > maxP) {
|
||||
log.opError(
|
||||
'invalid offset rewinding insert, truncating to content length',
|
||||
content,
|
||||
op
|
||||
)
|
||||
p = maxP
|
||||
}
|
||||
const textToBeRemoved = content.slice(p, p + op.i.length)
|
||||
if (op.i !== textToBeRemoved) {
|
||||
log.opError(
|
||||
'inserted content does not match text to be removed',
|
||||
content,
|
||||
op
|
||||
)
|
||||
throw new ConsistencyError(
|
||||
`Inserted content, '${op.i}', does not match text to be removed, '${textToBeRemoved}'`
|
||||
)
|
||||
}
|
||||
return content.slice(0, p) + content.slice(p + op.i.length)
|
||||
} else if (op.d != null) {
|
||||
if (op.p > content.length) {
|
||||
log.opError(
|
||||
'invalid offset rewinding delete, truncating to content length',
|
||||
content,
|
||||
op
|
||||
)
|
||||
}
|
||||
return content.slice(0, op.p) + op.d + content.slice(op.p)
|
||||
} else {
|
||||
return content
|
||||
}
|
||||
}
|
||||
|
||||
// This is the rewindDoc function from track-changes, modified to log all
|
||||
// operations that are applied to the document.
|
||||
async function rewindDoc(projectId, docId) {
|
||||
const log = new LogAppliedOps()
|
||||
// Prepare to rewind content
|
||||
const docIterator = await PackManager.promises.makeDocIterator(docId)
|
||||
const getUpdate = util.promisify(docIterator.next).bind(docIterator)
|
||||
|
||||
const lastUpdate = await getUpdate()
|
||||
if (!lastUpdate) {
|
||||
return null
|
||||
}
|
||||
|
||||
const lastUpdateVersion = lastUpdate.v
|
||||
|
||||
let latestContent
|
||||
let version
|
||||
try {
|
||||
;[latestContent, version] = await getLatestContent(
|
||||
projectId,
|
||||
docId,
|
||||
lastUpdateVersion
|
||||
)
|
||||
} catch (err) {
|
||||
if (err instanceof Errors.NotFoundError) {
|
||||
// Doc not found in docstore. We can't build its history
|
||||
return null
|
||||
} else {
|
||||
throw err
|
||||
}
|
||||
}
|
||||
log.start('load-doc', latestContent, version)
|
||||
|
||||
let content = latestContent
|
||||
let update = lastUpdate
|
||||
let previousUpdate = null
|
||||
|
||||
while (update) {
|
||||
if (packsAreDuplicated(update, previousUpdate)) {
|
||||
previousUpdate = update
|
||||
update = await getUpdate()
|
||||
continue
|
||||
}
|
||||
log.update('rewinding', update)
|
||||
for (let i = update.op.length - 1; i >= 0; i--) {
|
||||
const op = update.op[i]
|
||||
if (op.broken === true) {
|
||||
log.op('skipped', content, op)
|
||||
continue
|
||||
}
|
||||
try {
|
||||
log.op('rewindOp', content, op)
|
||||
content = rewindOp(content, op, log)
|
||||
} catch (e) {
|
||||
if (e instanceof ConsistencyError && (i = update.op.length - 1)) {
|
||||
// catch known case where the last op in an array has been
|
||||
// merged into a later op
|
||||
op.broken = true
|
||||
log.opError('marking broken', content, op)
|
||||
} else {
|
||||
log.opError('failed', content, op, e)
|
||||
}
|
||||
}
|
||||
}
|
||||
previousUpdate = update
|
||||
update = await getUpdate()
|
||||
}
|
||||
return log.end()
|
||||
}
|
||||
|
||||
async function main() {
|
||||
// Get a list of projects to migrate
|
||||
let projectIds = []
|
||||
if (argv.all) {
|
||||
const projectsToMigrate = await findProjects(
|
||||
{ 'overleaf.history.display': { $ne: true } },
|
||||
{ _id: 1, overleaf: 1 }
|
||||
)
|
||||
projectIds = projectsToMigrate.map(p => p._id.toString())
|
||||
console.log('Unmigrated projects', projectIds.length)
|
||||
} else if (argv['project-id']) {
|
||||
projectIds = [argv['project-id']]
|
||||
} else {
|
||||
usage()
|
||||
process.exit(1)
|
||||
}
|
||||
let errorCount = 0
|
||||
for (const projectId of projectIds) {
|
||||
const docIds = await PackManager.promises.findAllDocsInProject(projectId)
|
||||
if (!docIds.length) {
|
||||
console.log('No docs found for project', projectId)
|
||||
}
|
||||
let projectErrorCount = 0
|
||||
for (const docId of docIds) {
|
||||
const result = await rewindDoc(projectId, docId)
|
||||
const failed = result.filter(r => r.status === 'failed')
|
||||
errorCount += failed.length
|
||||
if (argv.verbose) {
|
||||
console.log(JSON.stringify({ projectId, docId, result }, null, 2))
|
||||
} else if (failed.length > 0) {
|
||||
console.log('project', projectId, 'docId', docId, 'FAILED')
|
||||
for (const f of failed) {
|
||||
console.log(JSON.stringify(f))
|
||||
}
|
||||
projectErrorCount += failed.length
|
||||
}
|
||||
}
|
||||
if (projectErrorCount === 0 && !argv.verbose) {
|
||||
console.log('project', projectId, 'docs', docIds.length, 'OK')
|
||||
}
|
||||
}
|
||||
process.exit(errorCount > 0 ? 1 : 0)
|
||||
}
|
||||
|
||||
waitForDb()
|
||||
.then(TrackChangesMongoDb.waitForDb)
|
||||
.then(main)
|
||||
.catch(err => {
|
||||
console.error(err)
|
||||
process.exit(1)
|
||||
})
|
||||
@@ -1,81 +0,0 @@
|
||||
const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true'
|
||||
const DRY_RUN = process.env.DRY_RUN !== 'false'
|
||||
process.env.MONGO_SOCKET_TIMEOUT =
|
||||
parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000
|
||||
|
||||
const PROJECT_ID = process.env.PROJECT_ID
|
||||
|
||||
const { ObjectId } = require('mongodb')
|
||||
const {
|
||||
db,
|
||||
waitForDb,
|
||||
READ_PREFERENCE_SECONDARY,
|
||||
} = require('../../app/src/infrastructure/mongodb')
|
||||
const ProjectHistoryHandler = require('../../app/src/Features/Project/ProjectHistoryHandler')
|
||||
|
||||
console.log({
|
||||
DRY_RUN,
|
||||
VERBOSE_LOGGING,
|
||||
PROJECT_ID,
|
||||
})
|
||||
|
||||
let INTERRUPT = false
|
||||
|
||||
async function processProject(project) {
|
||||
if (INTERRUPT) {
|
||||
return
|
||||
}
|
||||
if (!shouldPreserveHistory(project)) {
|
||||
console.log(
|
||||
`project ${project._id} does not have preserveHistory:true, skipping`
|
||||
)
|
||||
return
|
||||
}
|
||||
if (!DRY_RUN) {
|
||||
await ProjectHistoryHandler.promises.downgradeHistory(project._id)
|
||||
}
|
||||
if (VERBOSE_LOGGING) {
|
||||
console.log(`project ${project._id} downgraded to track-changes`)
|
||||
}
|
||||
}
|
||||
|
||||
async function shouldPreserveHistory(project) {
|
||||
return await db.projectHistoryMetaData.findOne(
|
||||
{
|
||||
$and: [
|
||||
{ project_id: { $eq: project._id } },
|
||||
{ preserveHistory: { $eq: true } },
|
||||
],
|
||||
},
|
||||
{ readPreference: READ_PREFERENCE_SECONDARY }
|
||||
)
|
||||
}
|
||||
|
||||
async function main() {
|
||||
if (PROJECT_ID) {
|
||||
await waitForDb()
|
||||
const project = await db.projects.findOne({ _id: ObjectId(PROJECT_ID) })
|
||||
await processProject(project)
|
||||
} else {
|
||||
console.log('PROJECT_ID environment value is needed.')
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// Upgrading history is not atomic, if we quit out mid-initialisation
|
||||
// then history could get into a broken state
|
||||
// Instead, skip any unprocessed projects and exit() at end of the batch.
|
||||
process.on('SIGINT', function () {
|
||||
console.log('Caught SIGINT, waiting for in process downgrades to complete')
|
||||
INTERRUPT = true
|
||||
})
|
||||
|
||||
main()
|
||||
.then(() => {
|
||||
console.error('Done.')
|
||||
process.exit(0)
|
||||
})
|
||||
.catch(error => {
|
||||
console.error({ error })
|
||||
process.exit(1)
|
||||
})
|
||||
@@ -1,287 +0,0 @@
|
||||
// raise mongo timeout to 1hr if otherwise unspecified
|
||||
process.env.MONGO_SOCKET_TIMEOUT =
|
||||
parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000
|
||||
|
||||
const fs = require('fs')
|
||||
|
||||
if (fs.existsSync('/etc/container_environment.json')) {
|
||||
try {
|
||||
const envData = JSON.parse(
|
||||
fs.readFileSync('/etc/container_environment.json', 'utf8')
|
||||
)
|
||||
for (const [key, value] of Object.entries(envData)) {
|
||||
process.env[key] = value
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(
|
||||
'cannot read /etc/container_environment.json, the script needs to be run as root',
|
||||
err
|
||||
)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
const VERSION = '0.9.0-cli'
|
||||
const {
|
||||
countProjects,
|
||||
countDocHistory,
|
||||
upgradeProject,
|
||||
findProjects,
|
||||
} = require('../../modules/history-migration/app/src/HistoryUpgradeHelper')
|
||||
const { waitForDb } = require('../../app/src/infrastructure/mongodb')
|
||||
const minimist = require('minimist')
|
||||
const util = require('util')
|
||||
const pLimit = require('p-limit')
|
||||
const logger = require('@overleaf/logger')
|
||||
logger.initialize('history-migration')
|
||||
// disable logging to stdout from internal modules
|
||||
logger.logger.streams = []
|
||||
|
||||
const DEFAULT_OUTPUT_FILE = `history-migration-${new Date()
|
||||
.toISOString()
|
||||
.replace(/[:.]/g, '_')}.log`
|
||||
|
||||
const argv = minimist(process.argv.slice(2), {
|
||||
boolean: [
|
||||
'verbose',
|
||||
'fix-invalid-characters',
|
||||
'convert-large-docs-to-file',
|
||||
'import-broken-history-as-zip',
|
||||
'force-upgrade-on-failure',
|
||||
'dry-run',
|
||||
'use-query-hint',
|
||||
'retry-failed',
|
||||
'archive-on-failure',
|
||||
'force-clean',
|
||||
],
|
||||
string: ['output', 'user-id'],
|
||||
alias: {
|
||||
verbose: 'v',
|
||||
output: 'o',
|
||||
'dry-run': 'd',
|
||||
concurrency: 'j',
|
||||
'use-query-hint': 'q',
|
||||
'retry-failed': 'r',
|
||||
'archive-on-failure': 'a',
|
||||
},
|
||||
default: {
|
||||
output: DEFAULT_OUTPUT_FILE,
|
||||
concurrency: 1,
|
||||
'batch-size': 100,
|
||||
'max-upgrades-to-attempt': false,
|
||||
'max-failures': 50,
|
||||
},
|
||||
})
|
||||
|
||||
let INTERRUPT = false
|
||||
|
||||
async function findProjectsToMigrate() {
|
||||
console.log('History Migration Statistics')
|
||||
|
||||
// Show statistics about the number of projects to migrate
|
||||
const migratedProjects = await countProjects({
|
||||
'overleaf.history.display': true,
|
||||
})
|
||||
const totalProjects = await countProjects()
|
||||
console.log('Migrated Projects : ', migratedProjects)
|
||||
console.log('Total Projects : ', totalProjects)
|
||||
console.log('Remaining Projects : ', totalProjects - migratedProjects)
|
||||
|
||||
if (migratedProjects === totalProjects) {
|
||||
console.log('All projects have been migrated')
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
// Get a list of projects to migrate
|
||||
const projectsToMigrate = await findProjects(
|
||||
{ 'overleaf.history.display': { $ne: true } },
|
||||
{ _id: 1, overleaf: 1 }
|
||||
)
|
||||
|
||||
// Show statistics for docHistory collection
|
||||
const docHistoryWithoutProjectId = await countDocHistory({
|
||||
project_id: { $exists: false },
|
||||
})
|
||||
|
||||
if (docHistoryWithoutProjectId > 0) {
|
||||
console.log(
|
||||
`WARNING: docHistory collection contains ${docHistoryWithoutProjectId} records without project_id`
|
||||
)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
return projectsToMigrate
|
||||
}
|
||||
|
||||
function createProgressBar() {
|
||||
const startTime = new Date()
|
||||
return function progressBar(current, total, msg) {
|
||||
const barLength = 20
|
||||
const percentage = Math.floor((current / total) * 100)
|
||||
const bar = '='.repeat(percentage / (100 / barLength))
|
||||
const empty = ' '.repeat(barLength - bar.length)
|
||||
const elapsed = new Date() - startTime
|
||||
// convert elapsed time to hours, minutes, seconds
|
||||
const ss = Math.floor((elapsed / 1000) % 60)
|
||||
.toString()
|
||||
.padStart(2, '0')
|
||||
const mm = Math.floor((elapsed / (1000 * 60)) % 60)
|
||||
.toString()
|
||||
.padStart(2, '0')
|
||||
const hh = Math.floor(elapsed / (1000 * 60 * 60))
|
||||
.toString()
|
||||
.padStart(2, '0')
|
||||
process.stdout.write(
|
||||
`\r${hh}:${mm}:${ss} |${bar}${empty}| ${percentage}% (${current}/${total}) ${msg}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
async function migrateProjects(projectsToMigrate) {
|
||||
let projectsMigrated = 0
|
||||
let projectsFailed = 0
|
||||
|
||||
console.log('Starting migration...')
|
||||
if (argv.concurrency > 1) {
|
||||
console.log(`Using ${argv.concurrency} concurrent migrations`)
|
||||
}
|
||||
// send log output for each migration to a file
|
||||
const output = fs.createWriteStream(argv.output, { flags: 'a' })
|
||||
console.log(`Writing log output to ${process.cwd()}/${argv.output}`)
|
||||
const logger = new console.Console({ stdout: output })
|
||||
function logJson(obj) {
|
||||
logger.log(JSON.stringify(obj))
|
||||
}
|
||||
// limit the number of concurrent migrations
|
||||
const limit = pLimit(argv.concurrency)
|
||||
const jobs = []
|
||||
// throttle progress reporting to 2x per second
|
||||
const progressBar = createProgressBar()
|
||||
let i = 0
|
||||
const N = projectsToMigrate.length
|
||||
const progressBarTimer = setInterval(() => {
|
||||
if (INTERRUPT) {
|
||||
return // don't update the progress bar if we're shutting down
|
||||
}
|
||||
progressBar(
|
||||
i,
|
||||
N,
|
||||
`Migrated: ${projectsMigrated}, Failed: ${projectsFailed}`
|
||||
)
|
||||
}, 500)
|
||||
|
||||
const options = {
|
||||
migrationOptions: {
|
||||
archiveOnFailure: argv['import-broken-history-as-zip'],
|
||||
fixInvalidCharacters: argv['fix-invalid-characters'],
|
||||
forceNewHistoryOnFailure: argv['force-upgrade-on-failure'],
|
||||
},
|
||||
convertLargeDocsToFile: argv['convert-large-docs-to-file'],
|
||||
userId: argv['user-id'],
|
||||
reason: VERSION,
|
||||
forceClean: argv['force-clean'],
|
||||
}
|
||||
async function _migrateProject(project) {
|
||||
if (INTERRUPT) {
|
||||
return // don't start any new jobs if we're shutting down
|
||||
}
|
||||
const startTime = new Date()
|
||||
try {
|
||||
const result = await upgradeProject(project._id, options)
|
||||
i++
|
||||
if (INTERRUPT && limit.activeCount > 1) {
|
||||
// an interrupt was requested while this job was running
|
||||
// report that we're waiting for the remaining jobs to finish
|
||||
console.log(
|
||||
`Waiting for remaining ${
|
||||
limit.activeCount - 1
|
||||
} active jobs to finish\r`
|
||||
)
|
||||
}
|
||||
if (result.error) {
|
||||
// failed to migrate this project
|
||||
logJson({
|
||||
project_id: project._id,
|
||||
result,
|
||||
stack: result.error.stack,
|
||||
startTime,
|
||||
endTime: new Date(),
|
||||
})
|
||||
projectsFailed++
|
||||
} else {
|
||||
// successfully migrated this project
|
||||
logJson({
|
||||
project_id: project._id,
|
||||
result,
|
||||
startTime,
|
||||
endTime: new Date(),
|
||||
})
|
||||
projectsMigrated++
|
||||
}
|
||||
} catch (err) {
|
||||
// unexpected error from the migration
|
||||
projectsFailed++
|
||||
logJson({
|
||||
project_id: project._id,
|
||||
exception: util.inspect(err),
|
||||
startTime,
|
||||
endTime: new Date(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
for (const project of projectsToMigrate) {
|
||||
jobs.push(limit(_migrateProject, project))
|
||||
}
|
||||
// wait for all the queued jobs to complete
|
||||
await Promise.all(jobs)
|
||||
clearInterval(progressBarTimer)
|
||||
progressBar(i, N, `Migrated: ${projectsMigrated}, Failed: ${projectsFailed}`)
|
||||
process.stdout.write('\n')
|
||||
return { projectsMigrated, projectsFailed }
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const projectsToMigrate = await findProjectsToMigrate()
|
||||
if (argv['dry-run']) {
|
||||
console.log('Dry run, exiting')
|
||||
process.exit(0)
|
||||
}
|
||||
const { projectsMigrated, projectsFailed } = await migrateProjects(
|
||||
projectsToMigrate
|
||||
)
|
||||
console.log('Projects migrated: ', projectsMigrated)
|
||||
console.log('Projects failed: ', projectsFailed)
|
||||
if (projectsFailed > 0) {
|
||||
console.log('------------------------------------------------------')
|
||||
console.log(`Log output written to ${process.cwd()}/${argv.output}`)
|
||||
console.log(
|
||||
'Please check the log for errors. Attach the content of the file when contacting support.'
|
||||
)
|
||||
console.log('------------------------------------------------------')
|
||||
}
|
||||
if (INTERRUPT) {
|
||||
console.log('Migration interrupted, please run again to continue.')
|
||||
} else if (projectsFailed === 0) {
|
||||
console.log(`All projects migrated successfully.`)
|
||||
}
|
||||
console.log('Done.')
|
||||
process.exit(projectsFailed > 0 ? 1 : 0)
|
||||
}
|
||||
|
||||
// Upgrading history is not atomic, if we quit out mid-initialisation
|
||||
// then history could get into a broken state
|
||||
// Instead, skip any unprocessed projects and exit() at end of the batch.
|
||||
process.on('SIGINT', function () {
|
||||
console.log(
|
||||
'\nCaught SIGINT, waiting for all in-progess upgrades to complete'
|
||||
)
|
||||
INTERRUPT = true
|
||||
})
|
||||
|
||||
waitForDb()
|
||||
.then(main)
|
||||
.catch(err => {
|
||||
console.error(err)
|
||||
process.exit(1)
|
||||
})
|
||||
@@ -1,111 +0,0 @@
|
||||
const DRY_RUN = process.env.DRY_RUN !== 'false'
|
||||
const PROJECT_ID = process.env.PROJECT_ID
|
||||
const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true'
|
||||
const VERBOSE_PROJECT_NAMES = process.env.VERBOSE_PROJECT_NAMES === 'true'
|
||||
const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 50
|
||||
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 500
|
||||
// persist fallback in order to keep batchedUpdate in-sync
|
||||
process.env.BATCH_SIZE = BATCH_SIZE
|
||||
process.env.VERBOSE_LOGGING = VERBOSE_LOGGING
|
||||
|
||||
const { ObjectId } = require('mongodb')
|
||||
const { db, waitForDb } = require('../../app/src/infrastructure/mongodb')
|
||||
const { batchedUpdate } = require('../helpers/batchedUpdate')
|
||||
const { promiseMapWithLimit } = require('../../app/src/util/promises')
|
||||
|
||||
const count = {
|
||||
projects: 0,
|
||||
projectsWithIncorrectRevDocs: 0,
|
||||
totalIncorrectRevDocs: 0,
|
||||
totalNanRevDocs: 0,
|
||||
totalNullRevDocs: 0,
|
||||
totalUndefinedRevDocs: 0,
|
||||
convertedRevs: 0,
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const projection = {
|
||||
_id: 1,
|
||||
}
|
||||
|
||||
if (VERBOSE_PROJECT_NAMES) {
|
||||
projection.name = 1
|
||||
}
|
||||
|
||||
const options = {}
|
||||
|
||||
if (PROJECT_ID) {
|
||||
const project = await db.projects.findOne({ _id: ObjectId(PROJECT_ID) })
|
||||
await processProject(project)
|
||||
} else {
|
||||
await batchedUpdate(
|
||||
'projects',
|
||||
{ 'overleaf.history.display': { $ne: true } },
|
||||
processBatch,
|
||||
projection,
|
||||
options
|
||||
)
|
||||
}
|
||||
console.log('Final')
|
||||
}
|
||||
|
||||
async function processBatch(projects) {
|
||||
await promiseMapWithLimit(WRITE_CONCURRENCY, projects, processProject)
|
||||
}
|
||||
|
||||
async function processProject(project) {
|
||||
count.projects++
|
||||
|
||||
const docs = await db.docs
|
||||
.find(
|
||||
{
|
||||
project_id: project._id,
|
||||
$or: [{ rev: null }, { rev: NaN }],
|
||||
},
|
||||
{ _id: 1, rev: 1 }
|
||||
)
|
||||
.toArray()
|
||||
|
||||
if (!docs || docs.length <= 0) {
|
||||
return
|
||||
}
|
||||
|
||||
if (VERBOSE_LOGGING) {
|
||||
console.log(
|
||||
`Found ${docs.length} incorrect doc.rev for project ${
|
||||
project[VERBOSE_PROJECT_NAMES ? 'name' : '_id']
|
||||
}`
|
||||
)
|
||||
}
|
||||
|
||||
count.projectsWithIncorrectRevDocs++
|
||||
count.totalIncorrectRevDocs += docs.length
|
||||
|
||||
for (const doc of docs) {
|
||||
if (doc.rev === undefined) {
|
||||
count.totalUndefinedRevDocs++
|
||||
} else if (doc.rev === null) {
|
||||
count.totalNullRevDocs++
|
||||
} else if (isNaN(doc.rev)) {
|
||||
count.totalNanRevDocs++
|
||||
} else {
|
||||
console.error(`unknown 'rev' value: ${doc.rev}`)
|
||||
}
|
||||
if (!DRY_RUN) {
|
||||
console.log(`fixing rev of doc ${doc.id} from '${doc.rev}' to 0`)
|
||||
await db.docs.updateOne({ _id: doc._id }, { $set: { rev: 0 } })
|
||||
count.convertedRevs++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
waitForDb()
|
||||
.then(main)
|
||||
.then(() => {
|
||||
console.log(count)
|
||||
process.exit(0)
|
||||
})
|
||||
.catch(err => {
|
||||
console.log('Something went wrong!', err)
|
||||
process.exit(1)
|
||||
})
|
||||
@@ -1,133 +0,0 @@
|
||||
const { promisify } = require('util')
|
||||
const { ObjectId } = require('mongodb')
|
||||
const {
|
||||
db,
|
||||
waitForDb,
|
||||
READ_PREFERENCE_SECONDARY,
|
||||
} = require('../../app/src/infrastructure/mongodb')
|
||||
const sleep = promisify(setTimeout)
|
||||
const _ = require('lodash')
|
||||
|
||||
const NOW_IN_S = Date.now() / 1000
|
||||
const ONE_WEEK_IN_S = 60 * 60 * 24 * 7
|
||||
const TEN_SECONDS = 10 * 1000
|
||||
|
||||
function getSecondsFromObjectId(id) {
|
||||
return id.getTimestamp().getTime() / 1000
|
||||
}
|
||||
|
||||
async function main(options) {
|
||||
if (!options) {
|
||||
options = {}
|
||||
}
|
||||
_.defaults(options, {
|
||||
projectId: process.env.PROJECT_ID,
|
||||
dryRun: process.env.DRY_RUN !== 'false',
|
||||
verboseLogging: process.env.VERBOSE_LOGGING === 'true',
|
||||
firstProjectId: process.env.FIRST_PROJECT_ID
|
||||
? ObjectId(process.env.FIRST_PROJECT_ID)
|
||||
: ObjectId('4b3d3b3d0000000000000000'), // timestamped to 2010-01-01T00:01:01.000Z
|
||||
incrementByS: parseInt(process.env.INCREMENT_BY_S, 10) || ONE_WEEK_IN_S,
|
||||
batchSize: parseInt(process.env.BATCH_SIZE, 10) || 1000,
|
||||
stopAtS: parseInt(process.env.STOP_AT_S, 10) || NOW_IN_S,
|
||||
letUserDoubleCheckInputsFor:
|
||||
parseInt(process.env.LET_USER_DOUBLE_CHECK_INPUTS_FOR, 10) || TEN_SECONDS,
|
||||
})
|
||||
|
||||
if (options.projectId) {
|
||||
await waitForDb()
|
||||
const { modifiedCount } = await db.projects.updateOne(
|
||||
{
|
||||
_id: ObjectId(options.projectId),
|
||||
'overleaf.history.allowDowngrade': true,
|
||||
},
|
||||
{ $unset: { 'overleaf.history.allowDowngrade': 1 } }
|
||||
)
|
||||
console.log(`modifiedCount: ${modifiedCount}`)
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
await letUserDoubleCheckInputs(options)
|
||||
await waitForDb()
|
||||
|
||||
let startId = options.firstProjectId
|
||||
|
||||
let totalProcessed = 0
|
||||
while (getSecondsFromObjectId(startId) <= options.stopAtS) {
|
||||
let batchProcessed = 0
|
||||
const end = getSecondsFromObjectId(startId) + options.incrementByS
|
||||
let endId = ObjectId.createFromTime(end)
|
||||
const query = {
|
||||
_id: {
|
||||
// include edge
|
||||
$gte: startId,
|
||||
// exclude edge
|
||||
$lt: endId,
|
||||
},
|
||||
'overleaf.history.allowDowngrade': true,
|
||||
}
|
||||
const projects = await db.projects
|
||||
.find(query, { readPreference: READ_PREFERENCE_SECONDARY })
|
||||
.project({ _id: 1 })
|
||||
.limit(options.batchSize)
|
||||
.toArray()
|
||||
|
||||
if (projects.length) {
|
||||
const projectIds = projects.map(project => project._id)
|
||||
if (options.verboseLogging) {
|
||||
console.log(
|
||||
`Processing projects with ids: ${JSON.stringify(projectIds)}`
|
||||
)
|
||||
} else {
|
||||
console.log(`Processing ${projects.length} projects`)
|
||||
}
|
||||
|
||||
if (!options.dryRun) {
|
||||
await db.projects.updateMany(
|
||||
{ _id: { $in: projectIds } },
|
||||
{ $unset: { 'overleaf.history.allowDowngrade': 1 } }
|
||||
)
|
||||
} else {
|
||||
console.log(
|
||||
`skipping update of ${projectIds.length} projects in dry-run mode`
|
||||
)
|
||||
}
|
||||
|
||||
totalProcessed += projectIds.length
|
||||
batchProcessed += projectIds.length
|
||||
|
||||
if (projects.length === options.batchSize) {
|
||||
endId = projects[projects.length - 1]._id
|
||||
}
|
||||
}
|
||||
console.error(
|
||||
`Processed ${batchProcessed} from ${startId} until ${endId} (${totalProcessed} processed in total)`
|
||||
)
|
||||
|
||||
startId = endId
|
||||
}
|
||||
}
|
||||
|
||||
async function letUserDoubleCheckInputs(options) {
|
||||
console.error('Options:', JSON.stringify(options, null, 2))
|
||||
console.error(
|
||||
'Waiting for you to double check inputs for',
|
||||
options.letUserDoubleCheckInputsFor,
|
||||
'ms'
|
||||
)
|
||||
await sleep(options.letUserDoubleCheckInputsFor)
|
||||
}
|
||||
|
||||
module.exports = main
|
||||
|
||||
if (require.main === module) {
|
||||
main()
|
||||
.then(() => {
|
||||
console.error('Done.')
|
||||
process.exit(0)
|
||||
})
|
||||
.catch(error => {
|
||||
console.error({ error })
|
||||
process.exit(1)
|
||||
})
|
||||
}
|
||||
@@ -1,208 +0,0 @@
|
||||
const SCRIPT_VERSION = 4
|
||||
const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true'
|
||||
const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10
|
||||
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 100
|
||||
const DRY_RUN = process.env.DRY_RUN !== 'false'
|
||||
const USE_QUERY_HINT = process.env.USE_QUERY_HINT !== 'false'
|
||||
const RETRY_FAILED = process.env.RETRY_FAILED === 'true'
|
||||
const MAX_UPGRADES_TO_ATTEMPT =
|
||||
parseInt(process.env.MAX_UPGRADES_TO_ATTEMPT, 10) || false
|
||||
const MAX_FAILURES = parseInt(process.env.MAX_FAILURES, 10) || 50
|
||||
const ARCHIVE_ON_FAILURE = process.env.ARCHIVE_ON_FAILURE === 'true'
|
||||
const FIX_INVALID_CHARACTERS = process.env.FIX_INVALID_CHARACTERS === 'true'
|
||||
const FORCE_NEW_HISTORY_ON_FAILURE =
|
||||
process.env.FORCE_NEW_HISTORY_ON_FAILURE === 'true'
|
||||
const IMPORT_ZIP_FILE_PATH = process.env.IMPORT_ZIP_FILE_PATH
|
||||
const CUTOFF_DATE = process.env.CUTOFF_DATE
|
||||
? new Date(process.env.CUTOFF_DATE)
|
||||
: undefined
|
||||
// persist fallback in order to keep batchedUpdate in-sync
|
||||
process.env.BATCH_SIZE = BATCH_SIZE
|
||||
// raise mongo timeout to 1hr if otherwise unspecified
|
||||
process.env.MONGO_SOCKET_TIMEOUT =
|
||||
parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000
|
||||
|
||||
const PROJECT_ID = process.env.PROJECT_ID
|
||||
|
||||
// User id is required to move large documents to filestore
|
||||
const USER_ID = process.env.USER_ID
|
||||
const CONVERT_LARGE_DOCS_TO_FILE =
|
||||
process.env.CONVERT_LARGE_DOCS_TO_FILE === 'true'
|
||||
|
||||
const { ObjectId } = require('mongodb')
|
||||
const { db, waitForDb } = require('../../app/src/infrastructure/mongodb')
|
||||
const { promiseMapWithLimit } = require('../../app/src/util/promises')
|
||||
const { batchedUpdate } = require('../helpers/batchedUpdate')
|
||||
const {
|
||||
anyDocHistoryExists,
|
||||
anyDocHistoryIndexExists,
|
||||
doUpgradeForNoneWithConversion,
|
||||
} = require('../../modules/history-migration/app/src/HistoryUpgradeHelper')
|
||||
|
||||
console.log({
|
||||
DRY_RUN,
|
||||
VERBOSE_LOGGING,
|
||||
WRITE_CONCURRENCY,
|
||||
BATCH_SIZE,
|
||||
MAX_UPGRADES_TO_ATTEMPT,
|
||||
MAX_FAILURES,
|
||||
USE_QUERY_HINT,
|
||||
RETRY_FAILED,
|
||||
ARCHIVE_ON_FAILURE,
|
||||
PROJECT_ID,
|
||||
FIX_INVALID_CHARACTERS,
|
||||
FORCE_NEW_HISTORY_ON_FAILURE,
|
||||
CONVERT_LARGE_DOCS_TO_FILE,
|
||||
USER_ID,
|
||||
IMPORT_ZIP_FILE_PATH,
|
||||
CUTOFF_DATE,
|
||||
})
|
||||
|
||||
const RESULT = {
|
||||
DRY_RUN,
|
||||
attempted: 0,
|
||||
projectsUpgraded: 0,
|
||||
failed: 0,
|
||||
continueFrom: null,
|
||||
}
|
||||
|
||||
let INTERRUPT = false
|
||||
|
||||
async function processBatch(projects) {
|
||||
if (projects.length && projects[0]._id) {
|
||||
RESULT.continueFrom = projects[0]._id
|
||||
}
|
||||
await promiseMapWithLimit(WRITE_CONCURRENCY, projects, processProject)
|
||||
console.log(RESULT)
|
||||
if (INTERRUPT) {
|
||||
// ctrl+c
|
||||
console.log('Terminated by SIGINT')
|
||||
process.exit(0)
|
||||
}
|
||||
if (RESULT.failed >= MAX_FAILURES) {
|
||||
console.log(`MAX_FAILURES limit (${MAX_FAILURES}) reached. Stopping.`)
|
||||
process.exit(0)
|
||||
}
|
||||
if (MAX_UPGRADES_TO_ATTEMPT && RESULT.attempted >= MAX_UPGRADES_TO_ATTEMPT) {
|
||||
console.log(
|
||||
`MAX_UPGRADES_TO_ATTEMPT limit (${MAX_UPGRADES_TO_ATTEMPT}) reached. Stopping.`
|
||||
)
|
||||
process.exit(0)
|
||||
}
|
||||
}
|
||||
|
||||
async function processProject(project) {
|
||||
if (INTERRUPT) {
|
||||
return
|
||||
}
|
||||
if (project.overleaf && project.overleaf.history) {
|
||||
// projects we're upgrading like this should never have a history id
|
||||
if (project.overleaf.history.id) {
|
||||
return
|
||||
}
|
||||
if (
|
||||
project.overleaf.history.conversionFailed ||
|
||||
project.overleaf.history.upgradeFailed
|
||||
) {
|
||||
if (project.overleaf.history.zipFileArchivedInProject) {
|
||||
return // always give up if we have uploaded the zipfile to the project
|
||||
}
|
||||
if (!RETRY_FAILED) {
|
||||
// we don't want to attempt upgrade on projects
|
||||
// that have been previously attempted and failed
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
if (RESULT.failed >= MAX_FAILURES) {
|
||||
return
|
||||
}
|
||||
if (MAX_UPGRADES_TO_ATTEMPT && RESULT.attempted >= MAX_UPGRADES_TO_ATTEMPT) {
|
||||
return
|
||||
}
|
||||
const anyDocHistoryOrIndex =
|
||||
(await anyDocHistoryExists(project)) ||
|
||||
(await anyDocHistoryIndexExists(project))
|
||||
if (anyDocHistoryOrIndex) {
|
||||
RESULT.attempted += 1
|
||||
if (DRY_RUN) {
|
||||
return
|
||||
}
|
||||
const result = await doUpgradeForNoneWithConversion(project, {
|
||||
migrationOptions: {
|
||||
archiveOnFailure: ARCHIVE_ON_FAILURE,
|
||||
fixInvalidCharacters: FIX_INVALID_CHARACTERS,
|
||||
forceNewHistoryOnFailure: FORCE_NEW_HISTORY_ON_FAILURE,
|
||||
importZipFilePath: IMPORT_ZIP_FILE_PATH,
|
||||
cutoffDate: CUTOFF_DATE,
|
||||
},
|
||||
convertLargeDocsToFile: CONVERT_LARGE_DOCS_TO_FILE,
|
||||
userId: USER_ID,
|
||||
reason: `${SCRIPT_VERSION}`,
|
||||
})
|
||||
if (result.convertedDocCount) {
|
||||
console.log(
|
||||
`project ${project._id} converted ${result.convertedDocCount} docs to filestore`
|
||||
)
|
||||
}
|
||||
if (result.error) {
|
||||
console.error(`project ${project._id} FAILED with error: `, result.error)
|
||||
RESULT.failed += 1
|
||||
} else if (result.upgraded) {
|
||||
if (VERBOSE_LOGGING) {
|
||||
console.log(
|
||||
`project ${project._id} converted and upgraded to full project history`
|
||||
)
|
||||
}
|
||||
RESULT.projectsUpgraded += 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
if (PROJECT_ID) {
|
||||
await waitForDb()
|
||||
const project = await db.projects.findOne({ _id: ObjectId(PROJECT_ID) })
|
||||
await processProject(project)
|
||||
} else {
|
||||
const projection = {
|
||||
_id: 1,
|
||||
overleaf: 1,
|
||||
}
|
||||
const options = {}
|
||||
if (USE_QUERY_HINT) {
|
||||
options.hint = { _id: 1 }
|
||||
}
|
||||
await batchedUpdate(
|
||||
'projects',
|
||||
// we originally used
|
||||
// 'overleaf.history.id': { $exists: false }
|
||||
// but display false is indexed and contains all the above,
|
||||
// it can be faster to skip projects with a history ID than to use a query
|
||||
{ 'overleaf.history.display': { $ne: true } },
|
||||
processBatch,
|
||||
projection,
|
||||
options
|
||||
)
|
||||
}
|
||||
console.log('Final')
|
||||
console.log(RESULT)
|
||||
}
|
||||
|
||||
// Upgrading history is not atomic, if we quit out mid-initialisation
|
||||
// then history could get into a broken state
|
||||
// Instead, skip any unprocessed projects and exit() at end of the batch.
|
||||
process.on('SIGINT', function () {
|
||||
console.log('Caught SIGINT, waiting for in process upgrades to complete')
|
||||
INTERRUPT = true
|
||||
})
|
||||
|
||||
main()
|
||||
.then(() => {
|
||||
console.error('Done.')
|
||||
process.exit(0)
|
||||
})
|
||||
.catch(error => {
|
||||
console.error({ error })
|
||||
process.exit(1)
|
||||
})
|
||||
@@ -1,232 +0,0 @@
|
||||
const SCRIPT_VERSION = 3
|
||||
const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true'
|
||||
const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10
|
||||
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 100
|
||||
const DRY_RUN = process.env.DRY_RUN !== 'false'
|
||||
const USE_QUERY_HINT = process.env.USE_QUERY_HINT !== 'false'
|
||||
const RETRY_FAILED = process.env.RETRY_FAILED === 'true'
|
||||
const MAX_UPGRADES_TO_ATTEMPT =
|
||||
parseInt(process.env.MAX_UPGRADES_TO_ATTEMPT, 10) || false
|
||||
const MAX_FAILURES = parseInt(process.env.MAX_FAILURES, 10) || 50
|
||||
// persist fallback in order to keep batchedUpdate in-sync
|
||||
process.env.BATCH_SIZE = BATCH_SIZE
|
||||
// raise mongo timeout to 1hr if otherwise unspecified
|
||||
process.env.MONGO_SOCKET_TIMEOUT =
|
||||
parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000
|
||||
|
||||
const {
|
||||
db,
|
||||
READ_PREFERENCE_SECONDARY,
|
||||
} = require('../../app/src/infrastructure/mongodb')
|
||||
const { promiseMapWithLimit } = require('../../app/src/util/promises')
|
||||
const { batchedUpdate } = require('../helpers/batchedUpdate')
|
||||
const ProjectHistoryHandler = require('../../app/src/Features/Project/ProjectHistoryHandler')
|
||||
const HistoryManager = require('../../app/src/Features/History/HistoryManager')
|
||||
|
||||
console.log({
|
||||
DRY_RUN,
|
||||
VERBOSE_LOGGING,
|
||||
WRITE_CONCURRENCY,
|
||||
BATCH_SIZE,
|
||||
MAX_UPGRADES_TO_ATTEMPT,
|
||||
MAX_FAILURES,
|
||||
USE_QUERY_HINT,
|
||||
RETRY_FAILED,
|
||||
})
|
||||
|
||||
const RESULT = {
|
||||
DRY_RUN,
|
||||
attempted: 0,
|
||||
projectsUpgraded: 0,
|
||||
failed: 0,
|
||||
continueFrom: null,
|
||||
}
|
||||
|
||||
let INTERRUPT = false
|
||||
|
||||
async function processBatch(projects) {
|
||||
if (projects.length && projects[0]._id) {
|
||||
RESULT.continueFrom = projects[0]._id
|
||||
}
|
||||
await promiseMapWithLimit(WRITE_CONCURRENCY, projects, processProject)
|
||||
console.log(RESULT)
|
||||
if (INTERRUPT) {
|
||||
// ctrl+c
|
||||
console.log('Terminated by SIGINT')
|
||||
process.exit(0)
|
||||
}
|
||||
if (RESULT.failed >= MAX_FAILURES) {
|
||||
console.log(`MAX_FAILURES limit (${MAX_FAILURES}) reached. Stopping.`)
|
||||
process.exit(0)
|
||||
}
|
||||
if (MAX_UPGRADES_TO_ATTEMPT && RESULT.attempted >= MAX_UPGRADES_TO_ATTEMPT) {
|
||||
console.log(
|
||||
`MAX_UPGRADES_TO_ATTEMPT limit (${MAX_UPGRADES_TO_ATTEMPT}) reached. Stopping.`
|
||||
)
|
||||
process.exit(0)
|
||||
}
|
||||
}
|
||||
|
||||
async function processProject(project) {
|
||||
if (INTERRUPT) {
|
||||
return
|
||||
}
|
||||
// If upgradeFailed, skip unless we're explicitly retrying failed upgrades
|
||||
if (
|
||||
project.overleaf &&
|
||||
project.overleaf.history &&
|
||||
project.overleaf.history.upgradeFailed
|
||||
) {
|
||||
if (RETRY_FAILED) {
|
||||
return await doUpgradeForNoneWithoutConversion(project)
|
||||
} else {
|
||||
return
|
||||
}
|
||||
}
|
||||
// Skip any projects with a history ID, these are v1
|
||||
if (
|
||||
project.overleaf &&
|
||||
project.overleaf.history &&
|
||||
project.overleaf.history.id
|
||||
) {
|
||||
return
|
||||
}
|
||||
const anyDocHistory = await anyDocHistoryExists(project)
|
||||
if (anyDocHistory) {
|
||||
return
|
||||
}
|
||||
const anyDocHistoryIndex = await anyDocHistoryIndexExists(project)
|
||||
if (anyDocHistoryIndex) {
|
||||
return
|
||||
}
|
||||
await doUpgradeForNoneWithoutConversion(project)
|
||||
}
|
||||
|
||||
async function doUpgradeForNoneWithoutConversion(project) {
|
||||
if (RESULT.failed >= MAX_FAILURES) {
|
||||
return
|
||||
}
|
||||
if (MAX_UPGRADES_TO_ATTEMPT && RESULT.attempted >= MAX_UPGRADES_TO_ATTEMPT) {
|
||||
return
|
||||
}
|
||||
RESULT.attempted += 1
|
||||
const projectId = project._id
|
||||
if (!DRY_RUN) {
|
||||
// ensureHistoryExistsForProject resyncs project
|
||||
// Change to 'peek'ing the doc when resyncing should
|
||||
// be rolled out prior to using this script
|
||||
try {
|
||||
// Logic originally from ProjectHistoryHandler.ensureHistoryExistsForProject
|
||||
// However sends a force resync project to project history instead
|
||||
// of a resync request to doc-updater
|
||||
let historyId = await ProjectHistoryHandler.promises.getHistoryId(
|
||||
projectId
|
||||
)
|
||||
if (historyId == null) {
|
||||
historyId = await HistoryManager.promises.initializeProject(projectId)
|
||||
if (historyId != null) {
|
||||
await ProjectHistoryHandler.promises.setHistoryId(
|
||||
projectId,
|
||||
historyId
|
||||
)
|
||||
}
|
||||
}
|
||||
await HistoryManager.promises.resyncProject(projectId, {
|
||||
force: true,
|
||||
origin: { kind: 'history-migration' },
|
||||
})
|
||||
await HistoryManager.promises.flushProject(projectId)
|
||||
} catch (err) {
|
||||
RESULT.failed += 1
|
||||
console.error(`project ${project._id} FAILED with error: `, err)
|
||||
await db.projects.updateOne(
|
||||
{ _id: project._id },
|
||||
{
|
||||
$set: {
|
||||
'overleaf.history.upgradeFailed': true,
|
||||
},
|
||||
}
|
||||
)
|
||||
return
|
||||
}
|
||||
await db.projects.updateOne(
|
||||
{ _id: project._id },
|
||||
{
|
||||
$set: {
|
||||
'overleaf.history.display': true,
|
||||
'overleaf.history.upgradedAt': new Date(),
|
||||
'overleaf.history.upgradeReason': `none-without-sl-history/${SCRIPT_VERSION}`,
|
||||
},
|
||||
$unset: {
|
||||
'overleaf.history.upgradeFailed': true,
|
||||
},
|
||||
}
|
||||
)
|
||||
}
|
||||
if (VERBOSE_LOGGING) {
|
||||
console.log(`project ${project._id} converted to full project history`)
|
||||
}
|
||||
RESULT.projectsUpgraded += 1
|
||||
}
|
||||
|
||||
async function anyDocHistoryExists(project) {
|
||||
return await db.docHistory.findOne(
|
||||
{ project_id: { $eq: project._id } },
|
||||
{
|
||||
projection: { _id: 1 },
|
||||
readPreference: READ_PREFERENCE_SECONDARY,
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
async function anyDocHistoryIndexExists(project) {
|
||||
return await db.docHistoryIndex.findOne(
|
||||
{ project_id: { $eq: project._id } },
|
||||
{
|
||||
projection: { _id: 1 },
|
||||
readPreference: READ_PREFERENCE_SECONDARY,
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const projection = {
|
||||
_id: 1,
|
||||
overleaf: 1,
|
||||
}
|
||||
const options = {}
|
||||
if (USE_QUERY_HINT) {
|
||||
options.hint = { _id: 1 }
|
||||
}
|
||||
await batchedUpdate(
|
||||
'projects',
|
||||
// we originally used
|
||||
// 'overleaf.history.id': { $exists: false }
|
||||
// but display false is indexed and contains all the above,
|
||||
// plus we want to be able to retry failed upgrades with a history id
|
||||
{ 'overleaf.history.display': { $ne: true } },
|
||||
processBatch,
|
||||
projection,
|
||||
options
|
||||
)
|
||||
console.log('Final')
|
||||
console.log(RESULT)
|
||||
}
|
||||
|
||||
// Upgrading history is not atomic, if we quit out mid-initialisation
|
||||
// then history could get into a broken state
|
||||
// Instead, skip any unprocessed projects and exit() at end of the batch.
|
||||
process.on('SIGINT', function () {
|
||||
console.log('Caught SIGINT, waiting for in process upgrades to complete')
|
||||
INTERRUPT = true
|
||||
})
|
||||
|
||||
main()
|
||||
.then(() => {
|
||||
console.error('Done.')
|
||||
process.exit(0)
|
||||
})
|
||||
.catch(error => {
|
||||
console.error({ error })
|
||||
process.exit(1)
|
||||
})
|
||||
@@ -1,44 +0,0 @@
|
||||
const { ObjectId } = require('mongodb')
|
||||
const {
|
||||
db,
|
||||
waitForDb,
|
||||
READ_PREFERENCE_SECONDARY,
|
||||
} = require('../../app/src/infrastructure/mongodb')
|
||||
const {
|
||||
upgradeProject,
|
||||
} = require('../../modules/history-migration/app/src/HistoryUpgradeHelper')
|
||||
|
||||
async function processProject(project) {
|
||||
const result = await upgradeProject(project)
|
||||
console.log(result)
|
||||
}
|
||||
|
||||
async function main() {
|
||||
await waitForDb()
|
||||
const args = process.argv.slice(2)
|
||||
const projectId = args[0]
|
||||
const query = { _id: ObjectId(projectId) }
|
||||
const projection = {
|
||||
_id: 1,
|
||||
overleaf: 1,
|
||||
}
|
||||
const options = {
|
||||
projection,
|
||||
readPreference: READ_PREFERENCE_SECONDARY,
|
||||
}
|
||||
const project = await db.projects.findOne(query, options)
|
||||
if (project) {
|
||||
await processProject(project)
|
||||
} else {
|
||||
console.error(`project ${projectId} not found`)
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
.then(() => {
|
||||
process.exit(0)
|
||||
})
|
||||
.catch(error => {
|
||||
console.error({ error })
|
||||
process.exit(1)
|
||||
})
|
||||
@@ -1,243 +0,0 @@
|
||||
const SCRIPT_VERSION = 1
|
||||
const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true'
|
||||
const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10
|
||||
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 100
|
||||
const DRY_RUN = process.env.DRY_RUN !== 'false'
|
||||
const USE_QUERY_HINT = process.env.USE_QUERY_HINT !== 'false'
|
||||
const RETRY_FAILED = process.env.RETRY_FAILED === 'true'
|
||||
const MAX_UPGRADES_TO_ATTEMPT =
|
||||
parseInt(process.env.MAX_UPGRADES_TO_ATTEMPT, 10) || false
|
||||
const MAX_FAILURES = parseInt(process.env.MAX_FAILURES, 10) || 50
|
||||
// persist fallback in order to keep batchedUpdate in-sync
|
||||
process.env.BATCH_SIZE = BATCH_SIZE
|
||||
// raise mongo timeout to 1hr if otherwise unspecified
|
||||
process.env.MONGO_SOCKET_TIMEOUT =
|
||||
parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000
|
||||
|
||||
const PROJECT_ID = process.env.PROJECT_ID
|
||||
|
||||
const { ObjectId } = require('mongodb')
|
||||
const {
|
||||
db,
|
||||
waitForDb,
|
||||
READ_PREFERENCE_SECONDARY,
|
||||
} = require('../../app/src/infrastructure/mongodb')
|
||||
const { promiseMapWithLimit } = require('../../app/src/util/promises')
|
||||
const { batchedUpdate } = require('../helpers/batchedUpdate')
|
||||
const ProjectHistoryController = require('../../modules/history-migration/app/src/ProjectHistoryController')
|
||||
|
||||
console.log({
|
||||
DRY_RUN,
|
||||
VERBOSE_LOGGING,
|
||||
WRITE_CONCURRENCY,
|
||||
BATCH_SIZE,
|
||||
MAX_UPGRADES_TO_ATTEMPT,
|
||||
MAX_FAILURES,
|
||||
USE_QUERY_HINT,
|
||||
RETRY_FAILED,
|
||||
PROJECT_ID,
|
||||
})
|
||||
|
||||
const RESULT = {
|
||||
DRY_RUN,
|
||||
attempted: 0,
|
||||
projectsUpgraded: 0,
|
||||
failed: 0,
|
||||
continueFrom: null,
|
||||
}
|
||||
|
||||
let INTERRUPT = false
|
||||
|
||||
async function processBatch(projects) {
|
||||
if (projects.length && projects[0]._id) {
|
||||
RESULT.continueFrom = projects[0]._id
|
||||
}
|
||||
await promiseMapWithLimit(WRITE_CONCURRENCY, projects, processProject)
|
||||
console.log(RESULT)
|
||||
if (INTERRUPT) {
|
||||
// ctrl+c
|
||||
console.log('Terminated by SIGINT')
|
||||
process.exit(0)
|
||||
}
|
||||
if (RESULT.failed >= MAX_FAILURES) {
|
||||
console.log(`MAX_FAILURES limit (${MAX_FAILURES}) reached. Stopping.`)
|
||||
process.exit(0)
|
||||
}
|
||||
if (MAX_UPGRADES_TO_ATTEMPT && RESULT.attempted >= MAX_UPGRADES_TO_ATTEMPT) {
|
||||
console.log(
|
||||
`MAX_UPGRADES_TO_ATTEMPT limit (${MAX_UPGRADES_TO_ATTEMPT}) reached. Stopping.`
|
||||
)
|
||||
process.exit(0)
|
||||
}
|
||||
}
|
||||
|
||||
async function processProject(project) {
|
||||
if (INTERRUPT) {
|
||||
return
|
||||
}
|
||||
// skip safety check if we want to retry failed upgrades
|
||||
if (!RETRY_FAILED) {
|
||||
if (project.overleaf && project.overleaf.history) {
|
||||
if (
|
||||
project.overleaf.history.conversionFailed ||
|
||||
project.overleaf.history.upgradeFailed
|
||||
) {
|
||||
// we don't want to attempt upgrade on projects
|
||||
// that have been previously attempted and failed
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
const preserveHistory = await shouldPreserveHistory(project)
|
||||
if (preserveHistory) {
|
||||
const anyDocHistory = await anyDocHistoryExists(project)
|
||||
if (anyDocHistory) {
|
||||
return await doUpgradeForV1WithConversion(project)
|
||||
}
|
||||
const anyDocHistoryIndex = await anyDocHistoryIndexExists(project)
|
||||
if (anyDocHistoryIndex) {
|
||||
return await doUpgradeForV1WithConversion(project)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function doUpgradeForV1WithConversion(project) {
|
||||
if (RESULT.failed >= MAX_FAILURES) {
|
||||
return
|
||||
}
|
||||
if (MAX_UPGRADES_TO_ATTEMPT && RESULT.attempted >= MAX_UPGRADES_TO_ATTEMPT) {
|
||||
return
|
||||
}
|
||||
RESULT.attempted += 1
|
||||
const projectId = project._id
|
||||
// migrateProjectHistory expects project id as a string
|
||||
const projectIdString = project._id.toString()
|
||||
if (!DRY_RUN) {
|
||||
try {
|
||||
// We treat these essentially as None projects, the V1 history is irrelevant,
|
||||
// so we will delete it, and do a conversion as if we're a None project
|
||||
await ProjectHistoryController.deleteProjectHistory(projectIdString)
|
||||
if (VERBOSE_LOGGING) {
|
||||
console.log(
|
||||
`project ${projectId} existing full project history deleted`
|
||||
)
|
||||
}
|
||||
await ProjectHistoryController.migrateProjectHistory(projectIdString)
|
||||
} catch (err) {
|
||||
// if migrateProjectHistory fails, it cleans up by deleting
|
||||
// the history and unsetting the history id
|
||||
// therefore a failed project will still look like a 'None with conversion' project
|
||||
RESULT.failed += 1
|
||||
console.error(`project ${projectId} FAILED with error: `, err)
|
||||
// We set a failed flag so future runs of the script don't automatically retry
|
||||
await db.projects.updateOne(
|
||||
{ _id: projectId },
|
||||
{
|
||||
$set: {
|
||||
'overleaf.history.conversionFailed': true,
|
||||
},
|
||||
}
|
||||
)
|
||||
return
|
||||
}
|
||||
await db.projects.updateOne(
|
||||
{ _id: projectId },
|
||||
{
|
||||
$set: {
|
||||
'overleaf.history.upgradeReason': `v1-with-conversion/${SCRIPT_VERSION}`,
|
||||
},
|
||||
$unset: {
|
||||
'overleaf.history.upgradeFailed': true,
|
||||
'overleaf.history.conversionFailed': true,
|
||||
},
|
||||
}
|
||||
)
|
||||
}
|
||||
if (VERBOSE_LOGGING) {
|
||||
console.log(
|
||||
`project ${projectId} converted and upgraded to full project history`
|
||||
)
|
||||
}
|
||||
RESULT.projectsUpgraded += 1
|
||||
}
|
||||
|
||||
async function shouldPreserveHistory(project) {
|
||||
return await db.projectHistoryMetaData.findOne(
|
||||
{
|
||||
$and: [
|
||||
{ project_id: { $eq: project._id } },
|
||||
{ preserveHistory: { $eq: true } },
|
||||
],
|
||||
},
|
||||
{ readPreference: READ_PREFERENCE_SECONDARY }
|
||||
)
|
||||
}
|
||||
|
||||
async function anyDocHistoryExists(project) {
|
||||
return await db.docHistory.findOne(
|
||||
{ project_id: { $eq: project._id } },
|
||||
{
|
||||
projection: { _id: 1 },
|
||||
readPreference: READ_PREFERENCE_SECONDARY,
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
async function anyDocHistoryIndexExists(project) {
|
||||
return await db.docHistoryIndex.findOne(
|
||||
{ project_id: { $eq: project._id } },
|
||||
{
|
||||
projection: { _id: 1 },
|
||||
readPreference: READ_PREFERENCE_SECONDARY,
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
async function main() {
|
||||
if (PROJECT_ID) {
|
||||
await waitForDb()
|
||||
const project = await db.projects.findOne({ _id: ObjectId(PROJECT_ID) })
|
||||
await processProject(project)
|
||||
} else {
|
||||
const projection = {
|
||||
_id: 1,
|
||||
overleaf: 1,
|
||||
}
|
||||
const options = {}
|
||||
if (USE_QUERY_HINT) {
|
||||
options.hint = { _id: 1 }
|
||||
}
|
||||
await batchedUpdate(
|
||||
'projects',
|
||||
{
|
||||
$and: [
|
||||
{ 'overleaf.history.display': { $ne: true } },
|
||||
{ 'overleaf.history.id': { $exists: true } },
|
||||
],
|
||||
},
|
||||
processBatch,
|
||||
projection,
|
||||
options
|
||||
)
|
||||
console.log('Final')
|
||||
console.log(RESULT)
|
||||
}
|
||||
}
|
||||
|
||||
// Upgrading history is not atomic, if we quit out mid-initialisation
|
||||
// then history could get into a broken state
|
||||
// Instead, skip any unprocessed projects and exit() at end of the batch.
|
||||
process.on('SIGINT', function () {
|
||||
console.log('Caught SIGINT, waiting for in process upgrades to complete')
|
||||
INTERRUPT = true
|
||||
})
|
||||
|
||||
main()
|
||||
.then(() => {
|
||||
console.error('Done.')
|
||||
process.exit(0)
|
||||
})
|
||||
.catch(error => {
|
||||
console.error({ error })
|
||||
process.exit(1)
|
||||
})
|
||||
-165
@@ -1,165 +0,0 @@
|
||||
const SCRIPT_VERSION = 2
|
||||
const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true'
|
||||
const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10
|
||||
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 100
|
||||
const DRY_RUN = process.env.DRY_RUN !== 'false'
|
||||
const USE_QUERY_HINT = process.env.USE_QUERY_HINT !== 'false'
|
||||
// persist fallback in order to keep batchedUpdate in-sync
|
||||
process.env.BATCH_SIZE = BATCH_SIZE
|
||||
// raise mongo timeout to 1hr if otherwise unspecified
|
||||
process.env.MONGO_SOCKET_TIMEOUT =
|
||||
parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000
|
||||
|
||||
const { ObjectId } = require('mongodb')
|
||||
const {
|
||||
db,
|
||||
READ_PREFERENCE_SECONDARY,
|
||||
} = require('../../app/src/infrastructure/mongodb')
|
||||
const { promiseMapWithLimit } = require('../../app/src/util/promises')
|
||||
const { batchedUpdate } = require('../helpers/batchedUpdate')
|
||||
|
||||
console.log({
|
||||
DRY_RUN,
|
||||
VERBOSE_LOGGING,
|
||||
WRITE_CONCURRENCY,
|
||||
BATCH_SIZE,
|
||||
USE_QUERY_HINT,
|
||||
})
|
||||
|
||||
const RESULT = {
|
||||
DRY_RUN,
|
||||
projectsUpgraded: 0,
|
||||
}
|
||||
|
||||
const ID_WHEN_FULL_PROJECT_HISTORY_ENABLED = '5a8d8a370000000000000000'
|
||||
const OBJECT_ID_WHEN_FULL_PROJECT_HISTORY_ENABLED = new ObjectId(
|
||||
ID_WHEN_FULL_PROJECT_HISTORY_ENABLED
|
||||
)
|
||||
const DATETIME_WHEN_FULL_PROJECT_HISTORY_ENABLED =
|
||||
OBJECT_ID_WHEN_FULL_PROJECT_HISTORY_ENABLED.getTimestamp()
|
||||
|
||||
// set a default BATCH_LAST_ID at our cutoff point if none set
|
||||
// we still check against this cut off point later, even if
|
||||
// BATCH_LAST_ID is set to something problematic
|
||||
if (!process.env.BATCH_LAST_ID) {
|
||||
process.env.BATCH_LAST_ID = ID_WHEN_FULL_PROJECT_HISTORY_ENABLED
|
||||
}
|
||||
|
||||
async function processBatch(projects) {
|
||||
await promiseMapWithLimit(WRITE_CONCURRENCY, projects, processProject)
|
||||
console.log(RESULT)
|
||||
}
|
||||
|
||||
async function processProject(project) {
|
||||
// safety check
|
||||
if (
|
||||
project.overleaf &&
|
||||
project.overleaf.history &&
|
||||
project.overleaf.history.upgradeFailed
|
||||
) {
|
||||
// a failed history upgrade might look like a v1 project, but history may be broken
|
||||
return
|
||||
}
|
||||
if (!projectCreatedAfterFullProjectHistoryEnabled(project)) {
|
||||
return
|
||||
}
|
||||
// if they have SL history, continue to send to both history systems (for now)
|
||||
const anyDocHistory = await anyDocHistoryExists(project)
|
||||
if (anyDocHistory) {
|
||||
return await doUpgradeForV1WithoutConversion(project, true)
|
||||
}
|
||||
const anyDocHistoryIndex = await anyDocHistoryIndexExists(project)
|
||||
if (anyDocHistoryIndex) {
|
||||
return await doUpgradeForV1WithoutConversion(project, true)
|
||||
}
|
||||
// or if no sl history, nothing to 'downgrade' to
|
||||
return await doUpgradeForV1WithoutConversion(project, false)
|
||||
}
|
||||
|
||||
function projectCreatedAfterFullProjectHistoryEnabled(project) {
|
||||
return (
|
||||
project._id.getTimestamp() >= DATETIME_WHEN_FULL_PROJECT_HISTORY_ENABLED
|
||||
)
|
||||
}
|
||||
|
||||
async function doUpgradeForV1WithoutConversion(project, allowDowngrade) {
|
||||
const setProperties = {
|
||||
'overleaf.history.display': true,
|
||||
'overleaf.history.upgradedAt': new Date(),
|
||||
'overleaf.history.upgradeReason': `v1-after-fph/${SCRIPT_VERSION}`,
|
||||
}
|
||||
if (allowDowngrade) {
|
||||
setProperties['overleaf.history.allowDowngrade'] = true
|
||||
}
|
||||
if (!DRY_RUN) {
|
||||
await db.projects.updateOne(
|
||||
{ _id: project._id },
|
||||
{
|
||||
$set: setProperties,
|
||||
}
|
||||
)
|
||||
}
|
||||
if (VERBOSE_LOGGING) {
|
||||
console.log(
|
||||
`project ${project._id} converted to full project history${
|
||||
allowDowngrade ? ', with allowDowngrade' : ''
|
||||
}`
|
||||
)
|
||||
}
|
||||
RESULT.projectsUpgraded += 1
|
||||
}
|
||||
|
||||
async function anyDocHistoryExists(project) {
|
||||
return await db.docHistory.findOne(
|
||||
{ project_id: { $eq: project._id } },
|
||||
{
|
||||
projection: { _id: 1 },
|
||||
readPreference: READ_PREFERENCE_SECONDARY,
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
async function anyDocHistoryIndexExists(project) {
|
||||
return await db.docHistoryIndex.findOne(
|
||||
{ project_id: { $eq: project._id } },
|
||||
{
|
||||
projection: { _id: 1 },
|
||||
readPreference: READ_PREFERENCE_SECONDARY,
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const projection = {
|
||||
_id: 1,
|
||||
overleaf: 1,
|
||||
}
|
||||
const options = {}
|
||||
if (USE_QUERY_HINT) {
|
||||
options.hint = { _id: 1 }
|
||||
}
|
||||
await batchedUpdate(
|
||||
'projects',
|
||||
{
|
||||
$and: [
|
||||
{ 'overleaf.history.display': { $ne: true } },
|
||||
{ 'overleaf.history.id': { $exists: true } },
|
||||
],
|
||||
},
|
||||
processBatch,
|
||||
projection,
|
||||
options
|
||||
)
|
||||
console.log('Final')
|
||||
console.log(RESULT)
|
||||
}
|
||||
|
||||
main()
|
||||
.then(() => {
|
||||
console.error('Done.')
|
||||
process.exit(0)
|
||||
})
|
||||
.catch(error => {
|
||||
console.error({ error })
|
||||
process.exit(1)
|
||||
})
|
||||
@@ -1,161 +0,0 @@
|
||||
const SCRIPT_VERSION = 3
|
||||
const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true'
|
||||
const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10
|
||||
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 100
|
||||
const DRY_RUN = process.env.DRY_RUN !== 'false'
|
||||
const USE_QUERY_HINT = process.env.USE_QUERY_HINT !== 'false'
|
||||
const UPGRADE_FAILED_WITH_EMPTY_HISTORY =
|
||||
process.env.UPGRADE_FAILED_WITH_EMPTY_HISTORY === 'true'
|
||||
// persist fallback in order to keep batchedUpdate in-sync
|
||||
process.env.BATCH_SIZE = BATCH_SIZE
|
||||
// raise mongo timeout to 1hr if otherwise unspecified
|
||||
process.env.MONGO_SOCKET_TIMEOUT =
|
||||
parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000
|
||||
|
||||
const {
|
||||
db,
|
||||
READ_PREFERENCE_SECONDARY,
|
||||
} = require('../../app/src/infrastructure/mongodb')
|
||||
const { promiseMapWithLimit } = require('../../app/src/util/promises')
|
||||
const { batchedUpdate } = require('../helpers/batchedUpdate')
|
||||
|
||||
console.log({
|
||||
DRY_RUN,
|
||||
VERBOSE_LOGGING,
|
||||
WRITE_CONCURRENCY,
|
||||
BATCH_SIZE,
|
||||
USE_QUERY_HINT,
|
||||
UPGRADE_FAILED_WITH_EMPTY_HISTORY,
|
||||
})
|
||||
|
||||
const RESULT = {
|
||||
DRY_RUN,
|
||||
projectsUpgraded: 0,
|
||||
}
|
||||
|
||||
async function processBatch(projects) {
|
||||
await promiseMapWithLimit(WRITE_CONCURRENCY, projects, processProject)
|
||||
console.log(RESULT)
|
||||
}
|
||||
|
||||
async function processProject(project) {
|
||||
// safety check if history exists and there was a failed upgrade
|
||||
const anyDocHistory = await anyDocHistoryExists(project)
|
||||
const anyDocHistoryIndex = await anyDocHistoryIndexExists(project)
|
||||
if (
|
||||
project.overleaf &&
|
||||
project.overleaf.history &&
|
||||
project.overleaf.history.upgradeFailed
|
||||
) {
|
||||
const emptyHistory = !anyDocHistory && !anyDocHistoryIndex
|
||||
if (emptyHistory && UPGRADE_FAILED_WITH_EMPTY_HISTORY) {
|
||||
console.log(
|
||||
`upgrading previously failed project ${project._id} with empty history`
|
||||
)
|
||||
} else {
|
||||
// a failed history upgrade might look like a v1 project, but history may be broken
|
||||
return
|
||||
}
|
||||
}
|
||||
const preserveHistory = await shouldPreserveHistory(project)
|
||||
if (preserveHistory) {
|
||||
// if we need to preserve history, then we must bail out if history exists
|
||||
if (anyDocHistory) {
|
||||
return
|
||||
}
|
||||
if (anyDocHistoryIndex) {
|
||||
return
|
||||
}
|
||||
return await doUpgradeForV1WithoutConversion(project)
|
||||
} else {
|
||||
// if preserveHistory false, then max 7 days of SL history
|
||||
// but v1 already record to both histories, so safe to upgrade
|
||||
return await doUpgradeForV1WithoutConversion(project)
|
||||
}
|
||||
}
|
||||
|
||||
async function doUpgradeForV1WithoutConversion(project) {
|
||||
if (!DRY_RUN) {
|
||||
await db.projects.updateOne(
|
||||
{ _id: project._id },
|
||||
{
|
||||
$set: {
|
||||
'overleaf.history.display': true,
|
||||
'overleaf.history.upgradedAt': new Date(),
|
||||
'overleaf.history.upgradeReason': `v1-without-sl-history/${SCRIPT_VERSION}`,
|
||||
},
|
||||
}
|
||||
)
|
||||
}
|
||||
if (VERBOSE_LOGGING) {
|
||||
console.log(`project ${project._id} converted to full project history`)
|
||||
}
|
||||
RESULT.projectsUpgraded += 1
|
||||
}
|
||||
|
||||
async function shouldPreserveHistory(project) {
|
||||
return await db.projectHistoryMetaData.findOne(
|
||||
{
|
||||
$and: [
|
||||
{ project_id: { $eq: project._id } },
|
||||
{ preserveHistory: { $eq: true } },
|
||||
],
|
||||
},
|
||||
{ readPreference: READ_PREFERENCE_SECONDARY }
|
||||
)
|
||||
}
|
||||
|
||||
async function anyDocHistoryExists(project) {
|
||||
return await db.docHistory.findOne(
|
||||
{ project_id: { $eq: project._id } },
|
||||
{
|
||||
projection: { _id: 1 },
|
||||
readPreference: READ_PREFERENCE_SECONDARY,
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
async function anyDocHistoryIndexExists(project) {
|
||||
return await db.docHistoryIndex.findOne(
|
||||
{ project_id: { $eq: project._id } },
|
||||
{
|
||||
projection: { _id: 1 },
|
||||
readPreference: READ_PREFERENCE_SECONDARY,
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const projection = {
|
||||
_id: 1,
|
||||
overleaf: 1,
|
||||
}
|
||||
const options = {}
|
||||
if (USE_QUERY_HINT) {
|
||||
options.hint = { _id: 1 }
|
||||
}
|
||||
await batchedUpdate(
|
||||
'projects',
|
||||
{
|
||||
$and: [
|
||||
{ 'overleaf.history.display': { $ne: true } },
|
||||
{ 'overleaf.history.id': { $exists: true } },
|
||||
],
|
||||
},
|
||||
processBatch,
|
||||
projection,
|
||||
options
|
||||
)
|
||||
console.log('Final')
|
||||
console.log(RESULT)
|
||||
}
|
||||
|
||||
main()
|
||||
.then(() => {
|
||||
console.error('Done.')
|
||||
process.exit(0)
|
||||
})
|
||||
.catch(error => {
|
||||
console.error({ error })
|
||||
process.exit(1)
|
||||
})
|
||||
@@ -165,7 +165,6 @@ async function deleteDocFromRedis(projectId, docId) {
|
||||
`UnflushedTime:{${docId}}`,
|
||||
`Pathname:{${docId}}`,
|
||||
`ProjectHistoryId:{${docId}}`,
|
||||
`ProjectHistoryType:{${docId}}`,
|
||||
`PendingUpdates:{${docId}}`,
|
||||
`lastUpdatedAt:{${docId}}`,
|
||||
`lastUpdatedBy:{${docId}}`
|
||||
|
||||
Reference in New Issue
Block a user