From 7af4980a4d2d927d06a7af1006e89145bf60d98b Mon Sep 17 00:00:00 2001 From: andrew rumble Date: Mon, 22 Jul 2024 11:32:48 +0100 Subject: [PATCH] Promisify ProjectEntityUpdateHandler functions Promisify wrapWithLock Promisify getDocContext Promisify updateDocLines Promisify setRootDoc Promisify unsetRootDoc Promisify addDoc and addDocWithRanges Promisify addFile Promisify upsertFile Promisify upsertDocWithPath Promisify deleteEntity Promisify deleteEntityWithPath Promisify mkdirp and mkdirpWithExactCase Promisify moveEntity Promisify renameEntity Promisify addFolder Promisify convertDocToFile Promisify resyncProjectHistory Promisify _addDocAndSendToTpds Promisify _replaceFile Promisify _checkFiletree Promisify _cleanupEntity Promisify _updateProjectStructureWithDeletedEntity Promisify _cleanUpDoc and _cleanUpFile Promisify upsertFileWithPath Promisify upsertDoc Promisify _uploadFile Promisify _addFileAndSendToTpds GitOrigin-RevId: 56a7de0ae79efedd5ad48abae2db22a4028c0c12 --- .../Project/ProjectEntityUpdateHandler.js | 2442 +++++++---------- 1 file changed, 1038 insertions(+), 1404 deletions(-) diff --git a/services/web/app/src/Features/Project/ProjectEntityUpdateHandler.js b/services/web/app/src/Features/Project/ProjectEntityUpdateHandler.js index 15d7a3fb98..24adcc3053 100644 --- a/services/web/app/src/Features/Project/ProjectEntityUpdateHandler.js +++ b/services/web/app/src/Features/Project/ProjectEntityUpdateHandler.js @@ -1,6 +1,5 @@ const _ = require('lodash') const OError = require('@overleaf/o-error') -const async = require('async') const logger = require('@overleaf/logger') const Settings = require('@overleaf/settings') const Path = require('path') @@ -37,308 +36,250 @@ function wrapWithLock(methodWithoutLock, lockManager = LockManager) { // sequentially. In particular the updates must be made in mongo and sent to // the doc-updater in the same order. if (typeof methodWithoutLock === 'function') { - const methodWithLock = (projectId, ...rest) => { - const adjustedLength = Math.max(rest.length, 1) - const args = rest.slice(0, adjustedLength - 1) - const callback = rest[adjustedLength - 1] - lockManager.runWithLock( - LOCK_NAMESPACE, - projectId, - cb => methodWithoutLock(projectId, ...args, cb), - callback + const methodWithLock = async (projectId, ...rest) => { + return lockManager.promises.runWithLock(LOCK_NAMESPACE, projectId, () => + methodWithoutLock(projectId, ...rest) ) } methodWithLock.withoutLock = methodWithoutLock return methodWithLock } else { // handle case with separate setup and locked stages - const wrapWithSetup = methodWithoutLock.beforeLock // a function to set things up before the lock - const mainTask = methodWithoutLock.withLock // function to execute inside the lock - const methodWithLock = wrapWithSetup((projectId, ...rest) => { - const adjustedLength = Math.max(rest.length, 1) - const args = rest.slice(0, adjustedLength - 1) - const callback = rest[adjustedLength - 1] - lockManager.runWithLock( - LOCK_NAMESPACE, - projectId, - cb => mainTask(projectId, ...args, cb), - callback + const mainTask = methodWithoutLock.withLock + const methodWithLock = async (projectId, ...rest) => { + const arg = await methodWithoutLock.beforeLock(projectId, ...rest) + return lockManager.promises.runWithLock(LOCK_NAMESPACE, projectId, () => + mainTask(arg) ) - }) - methodWithLock.withoutLock = wrapWithSetup(mainTask) + } + methodWithLock.withoutLock = async (...args) => { + return await mainTask(await methodWithoutLock.beforeLock(...args)) + } methodWithLock.beforeLock = methodWithoutLock.beforeLock methodWithLock.mainTask = methodWithoutLock.withLock return methodWithLock } } -function getDocContext(projectId, docId, callback) { - ProjectGetter.getProject( - projectId, - { name: true, rootFolder: true }, - (err, project) => { - if (err) { - return callback( - OError.tag(err, 'error fetching project', { +async function getDocContext(projectId, docId) { + let project + try { + project = await ProjectGetter.promises.getProject(projectId, { + name: true, + rootFolder: true, + }) + } catch (err) { + throw OError.tag(err, 'error fetching project', { + projectId, + }) + } + + if (!project) { + throw new Errors.NotFoundError('project not found') + } + try { + const { path, folder } = await ProjectLocator.promises.findElement({ + project, + element_id: docId, + type: 'docs', + }) + return { + projectName: project.name, + isDeletedDoc: false, + path: path.fileSystem, + folder, + } + } catch (err) { + if (err instanceof Errors.NotFoundError) { + // (Soft-)Deleted docs are removed from the file-tree (rootFolder). + // docstore can tell whether it exists and is (soft)-deleted. + let isDeletedDoc + try { + isDeletedDoc = await DocstoreManager.promises.isDocDeleted( + projectId, + docId + ) + if (!isDeletedDoc) { + // NOTE: This can happen while we delete a doc: + // 1. web will update the projects entry + // 2. web triggers flushes to tpds/doc-updater + // 3. web triggers (soft)-delete in docstore + // Specifically when an update comes in after 1 + // and before 3 completes. + logger.debug( + { projectId, docId }, + 'updating doc that is in process of getting soft-deleted' + ) + } + return { + projectName: project.name, + isDeletedDoc: true, + path: null, + folder: null, + } + } catch (error) { + if (error instanceof Errors.NotFoundError) { + logger.warn( + { projectId, docId }, + 'doc not found while updating doc lines' + ) + throw error + } + throw OError.tag( + error, + 'error checking deletion status with docstore', + { projectId, - }) + docId, + } ) } - if (!project) { - return callback(new Errors.NotFoundError('project not found')) - } - ProjectLocator.findElement( - { project, element_id: docId, type: 'docs' }, - (err, doc, path, folder) => { - if (err && err instanceof Errors.NotFoundError) { - // (Soft-)Deleted docs are removed from the file-tree (rootFolder). - // docstore can tell whether it exists and is (soft)-deleted. - DocstoreManager.isDocDeleted( - projectId, - docId, - (err, isDeletedDoc) => { - if (err && err instanceof Errors.NotFoundError) { - logger.warn( - { projectId, docId }, - 'doc not found while updating doc lines' - ) - callback(err) - } else if (err) { - callback( - OError.tag( - err, - 'error checking deletion status with docstore', - { projectId, docId } - ) - ) - } else { - if (!isDeletedDoc) { - // NOTE: This can happen while we delete a doc: - // 1. web will update the projects entry - // 2. web triggers flushes to tpds/doc-updater - // 3. web triggers (soft)-delete in docstore - // Specifically when an update comes in after 1 - // and before 3 completes. - logger.debug( - { projectId, docId }, - 'updating doc that is in process of getting soft-deleted' - ) - } - callback(null, { - projectName: project.name, - isDeletedDoc: true, - path: null, - folder: null, - }) - } - } - ) - } else if (err) { - callback( - OError.tag(err, 'error finding doc in rootFolder', { - docId, - projectId, - }) - ) - } else { - callback(null, { - projectName: project.name, - isDeletedDoc: false, - path: path.fileSystem, - folder, - }) - } - } - ) + } else { + throw OError.tag(err, 'error finding doc in rootFolder', { + docId, + projectId, + }) } - ) + } } -function updateDocLines( +async function updateDocLines( projectId, docId, lines, version, ranges, lastUpdatedAt, - lastUpdatedBy, - callback + lastUpdatedBy ) { - getDocContext(projectId, docId, (err, ctx) => { - if (err && err instanceof Errors.NotFoundError) { + let ctx + try { + ctx = await getDocContext(projectId, docId) + } catch (error) { + if (error instanceof Errors.NotFoundError) { // Do not allow an update to a doc which has never exist on this project logger.warn( { docId, projectId }, 'project or doc not found while updating doc lines' ) - return callback(err) } - if (err) { - return callback(err) - } - const { projectName, isDeletedDoc, path, folder } = ctx - logger.debug({ projectId, docId }, 'telling docstore manager to update doc') - DocstoreManager.updateDoc( + + throw error + } + const { projectName, isDeletedDoc, path, folder } = ctx + logger.debug({ projectId, docId }, 'telling docstore manager to update doc') + let modified, rev + try { + ;({ modified, rev } = await DocstoreManager.promises.updateDoc( projectId, docId, lines, version, - ranges, - (err, modified, rev) => { - if (err != null) { - OError.tag(err, 'error sending doc to docstore', { - docId, - projectId, - }) - return callback(err) - } - logger.debug( - { projectId, docId, modified }, - 'finished updating doc lines' - ) - // path will only be present if the doc is not deleted - if (!modified || isDeletedDoc) { - return callback(null, { rev }) - } - // Don't need to block for marking as updated - ProjectUpdateHandler.promises - .markAsUpdated(projectId, lastUpdatedAt, lastUpdatedBy) - .catch(error => { - logger.error({ error }, 'failed to mark project as updated') - }) - TpdsUpdateSender.addDoc( - { - projectId, - path, - docId, - projectName, - rev, - folderId: folder?._id, - }, - err => { - if (err) { - return callback(err) - } - callback(null, { rev, modified }) - } - ) - } - ) + ranges + )) + } catch (err) { + throw OError.tag(err, 'error sending doc to docstore', { docId, projectId }) + } + // path will only be present if the doc is not deleted + if (!modified || isDeletedDoc) { + return { rev } + } + // Don't need to block for marking as updated + ProjectUpdateHandler.promises + .markAsUpdated(projectId, lastUpdatedAt, lastUpdatedBy) + .catch(error => { + logger.error({ error }, 'failed to mark project as updated') + }) + await TpdsUpdateSender.promises.addDoc({ + projectId, + path, + docId, + projectName, + rev, + folderId: folder?._id, }) + return { rev, modified } } -function setRootDoc(projectId, newRootDocID, callback) { +async function setRootDoc(projectId, newRootDocID) { logger.debug({ projectId, rootDocId: newRootDocID }, 'setting root doc') if (projectId == null || newRootDocID == null) { - return callback( - new Errors.InvalidError('missing arguments (project or doc)') + throw new Errors.InvalidError('missing arguments (project or doc)') + } + const docPath = + await ProjectEntityHandler.promises.getDocPathByProjectIdAndDocId( + projectId, + newRootDocID + ) + if (ProjectEntityUpdateHandler.isPathValidForRootDoc(docPath)) { + await Project.updateOne( + { _id: projectId }, + { rootDoc_id: newRootDocID } + ).exec() + } else { + throw new Errors.UnsupportedFileTypeError( + 'invalid file extension for root doc' ) } - ProjectEntityHandler.getDocPathByProjectIdAndDocId( - projectId, - newRootDocID, - (err, docPath) => { - if (err != null) { - return callback(err) - } - if (ProjectEntityUpdateHandler.isPathValidForRootDoc(docPath)) { - // Ignore spurious floating promises warning until we promisify - // eslint-disable-next-line @typescript-eslint/no-floating-promises - Project.updateOne( - { _id: projectId }, - { rootDoc_id: newRootDocID }, - {}, - callback - ) - } else { - callback( - new Errors.UnsupportedFileTypeError( - 'invalid file extension for root doc' - ) - ) - } - } - ) } -function unsetRootDoc(projectId, callback) { +async function unsetRootDoc(projectId) { logger.debug({ projectId }, 'removing root doc') - // Ignore spurious floating promises warning until we promisify - // eslint-disable-next-line @typescript-eslint/no-floating-promises - Project.updateOne( + await Project.updateOne( { _id: projectId }, - { $unset: { rootDoc_id: true } }, - {}, - callback - ) + { $unset: { rootDoc_id: true } } + ).exec() } -function addDoc( - projectId, - folderId, - docName, - docLines, - userId, - source, - callback -) { - ProjectEntityUpdateHandler.addDocWithRanges( +async function addDoc(projectId, folderId, docName, docLines, userId, source) { + return await ProjectEntityUpdateHandler.promises.addDocWithRanges( projectId, folderId, docName, docLines, {}, userId, - source, - callback + source ) } const addDocWithRanges = wrapWithLock({ - beforeLock(next) { - return function ( + async beforeLock( + projectId, + folderId, + docName, + docLines, + ranges, + userId, + source + ) { + if (!SafePath.isCleanFilename(docName)) { + throw new Errors.InvalidNameError('invalid element name') + } + // Put doc in docstore first, so that if it errors, we don't have a doc_id in the project + // which hasn't been created in docstore. + const doc = new Doc({ name: docName }) + const { rev } = await DocstoreManager.promises.updateDoc( + projectId.toString(), + doc._id.toString(), + docLines, + 0, + ranges + ) + + doc.rev = rev + return { projectId, folderId, + doc, docName, docLines, ranges, userId, source, - callback - ) { - if (!SafePath.isCleanFilename(docName)) { - return callback(new Errors.InvalidNameError('invalid element name')) - } - // Put doc in docstore first, so that if it errors, we don't have a doc_id in the project - // which hasn't been created in docstore. - const doc = new Doc({ name: docName }) - DocstoreManager.updateDoc( - projectId.toString(), - doc._id.toString(), - docLines, - 0, - ranges, - (err, modified, rev) => { - if (err != null) { - return callback(err) - } - doc.rev = rev - next( - projectId, - folderId, - doc, - docName, - docLines, - ranges, - userId, - source, - callback - ) - } - ) } }, - withLock( + async withLock({ projectId, folderId, doc, @@ -347,553 +288,439 @@ const addDocWithRanges = wrapWithLock({ ranges, userId, source, - callback - ) { - ProjectEntityUpdateHandler._addDocAndSendToTpds( + }) { + const { result, project } = + await ProjectEntityUpdateHandler._addDocAndSendToTpds( + projectId, + folderId, + doc + ) + const docPath = result?.path?.fileSystem + const projectHistoryId = project?.overleaf?.history?.id + const newDocs = [ + { + doc, + path: docPath, + docLines: docLines.join('\n'), + ranges, + }, + ] + await DocumentUpdaterHandler.promises.updateProjectStructure( projectId, - folderId, - doc, - (err, result, project) => { - if (err != null) { - return callback(err) - } - const docPath = result && result.path && result.path.fileSystem - const projectHistoryId = - project.overleaf && - project.overleaf.history && - project.overleaf.history.id - const newDocs = [ - { - doc, - path: docPath, - docLines: docLines.join('\n'), - ranges, - }, - ] - DocumentUpdaterHandler.updateProjectStructure( - projectId, - projectHistoryId, - userId, - { newDocs, newProject: project }, - source, - error => { - if (error != null) { - return callback(error) - } - callback(null, doc, folderId || project.rootFolder[0]._id) - } - ) - } + projectHistoryId, + userId, + { newDocs, newProject: project }, + source ) + return { doc, folderId: folderId || project.rootFolder[0]._id } }, }) const addFile = wrapWithLock({ - beforeLock(next) { - return function ( - projectId, - folderId, - fileName, - fsPath, - linkedFileData, - userId, - source, - callback - ) { - if (!SafePath.isCleanFilename(fileName)) { - return callback(new Errors.InvalidNameError('invalid element name')) - } - ProjectEntityUpdateHandler._uploadFile( - projectId, - folderId, - fileName, - fsPath, - linkedFileData, - (error, fileStoreUrl, fileRef) => { - if (error != null) { - return callback(error) - } - next( - projectId, - folderId, - fileName, - fsPath, - linkedFileData, - userId, - fileRef, - fileStoreUrl, - source, - callback - ) - } - ) - } - }, - withLock( + async beforeLock( projectId, folderId, fileName, fsPath, linkedFileData, userId, + source + ) { + if (!SafePath.isCleanFilename(fileName)) { + throw new Errors.InvalidNameError('invalid element name') + } + const { url, fileRef } = await ProjectEntityUpdateHandler._uploadFile( + projectId, + folderId, + fileName, + fsPath, + linkedFileData + ) + + return { + projectId, + folderId, + userId, + fileRef, + fileStoreUrl: url, + source, + } + }, + async withLock({ + projectId, + folderId, + userId, fileRef, fileStoreUrl, source, - callback - ) { - ProjectEntityUpdateHandler._addFileAndSendToTpds( + }) { + const { result, project } = + await ProjectEntityUpdateHandler._addFileAndSendToTpds( + projectId, + folderId, + fileRef + ) + const projectHistoryId = project.overleaf?.history?.id + const newFiles = [ + { + file: fileRef, + path: result && result.path && result.path.fileSystem, + url: fileStoreUrl, + }, + ] + await DocumentUpdaterHandler.promises.updateProjectStructure( projectId, - folderId, - fileRef, - (err, result, project) => { - if (err != null) { - return callback(err) - } - const projectHistoryId = - project.overleaf && - project.overleaf.history && - project.overleaf.history.id - const newFiles = [ - { - file: fileRef, - path: result && result.path && result.path.fileSystem, - url: fileStoreUrl, - }, - ] - DocumentUpdaterHandler.updateProjectStructure( - projectId, - projectHistoryId, - userId, - { newFiles, newProject: project }, - source, - error => { - if (error != null) { - return callback(error) - } - ProjectUpdateHandler.promises - .markAsUpdated(projectId, new Date(), userId) - .catch(error => { - logger.error({ error }, 'failed to mark project as updated') - }) - callback(null, fileRef, folderId) - } - ) - } + projectHistoryId, + userId, + { newFiles, newProject: project }, + source ) + + ProjectUpdateHandler.promises + .markAsUpdated(projectId, new Date(), userId) + .catch(error => { + logger.error({ error }, 'failed to mark project as updated') + }) + return { fileRef, folderId } }, }) const upsertDoc = wrapWithLock( - function (projectId, folderId, docName, docLines, source, userId, callback) { + async function (projectId, folderId, docName, docLines, source, userId) { if (!SafePath.isCleanFilename(docName)) { - return callback(new Errors.InvalidNameError('invalid element name')) + throw new Errors.InvalidNameError('invalid element name') } - ProjectLocator.findElement( - { project_id: projectId, element_id: folderId, type: 'folder' }, - (error, folder, folderPath) => { - if (error != null) { - if (error instanceof Errors.NotFoundError && folder == null) { - return callback(new Error('folder_not_found')) - } - return callback(error) - } - if (folder == null) { - return callback(new Error("Couldn't find folder")) - } - const existingDoc = folder.docs.find(({ name }) => name === docName) - const existingFile = folder.fileRefs.find( - ({ name }) => name === docName - ) - if (existingFile) { - const doc = new Doc({ name: docName }) - const filePath = `${folderPath.fileSystem}/${existingFile.name}` - DocstoreManager.updateDoc( - projectId.toString(), - doc._id.toString(), - docLines, - 0, - {}, - (err, modified, rev) => { - if (err != null) { - return callback(err) - } - doc.rev = rev - ProjectEntityMongoUpdateHandler.replaceFileWithDoc( - projectId, - existingFile._id, - doc, - (err, project) => { - if (err) { - return callback(err) - } - TpdsUpdateSender.addDoc( - { - projectId, - docId: doc._id, - path: filePath, - projectName: project.name, - rev: existingFile.rev + 1, - folderId, - }, - err => { - if (err) { - return callback(err) - } - const projectHistoryId = - project.overleaf && - project.overleaf.history && - project.overleaf.history.id - const newDocs = [ - { - doc, - path: filePath, - docLines: docLines.join('\n'), - }, - ] - const oldFiles = [ - { - file: existingFile, - path: filePath, - }, - ] - DocumentUpdaterHandler.updateProjectStructure( - projectId, - projectHistoryId, - userId, - { oldFiles, newDocs, newProject: project }, - source, - error => { - if (error != null) { - return callback(error) - } - EditorRealTimeController.emitToRoom( - projectId, - 'removeEntity', - existingFile._id, - 'convertFileToDoc' - ) - callback(null, doc, true) - } - ) - } - ) - } - ) - } - ) - } else if (existingDoc) { - DocumentUpdaterHandler.setDocument( - projectId, - existingDoc._id, - userId, - docLines, - source, - (err, result) => { - if (err != null) { - return callback(err) - } - logger.debug( - { projectId, docId: existingDoc._id }, - 'notifying users that the document has been updated' - ) - // there is no need to flush the doc to mongo at this point as docupdater - // flushes it as part of setDoc. - // - // combine rev from response with existing doc metadata - callback(null, { ...existingDoc, ...result }, existingDoc == null) - } - ) - } else { - ProjectEntityUpdateHandler.addDocWithRanges.withoutLock( - projectId, - folderId, - docName, - docLines, - {}, - userId, - source, - (err, doc) => { - if (err != null) { - return callback(err) - } - callback(null, doc, existingDoc == null) - } - ) - } + let element, folderPath + try { + ;({ element, path: folderPath } = + await ProjectLocator.promises.findElement({ + project_id: projectId, + element_id: folderId, + type: 'folder', + })) + } catch (error) { + if (error instanceof Errors.NotFoundError) { + throw new Error('folder_not_found') } - ) + throw error + } + + if (element == null) { + throw new Error("Couldn't find folder") + } + + const existingDoc = element.docs.find(({ name }) => name === docName) + const existingFile = element.fileRefs.find(({ name }) => name === docName) + if (existingFile) { + const doc = new Doc({ name: docName }) + const filePath = `${folderPath.fileSystem}/${existingFile.name}` + const { rev } = await DocstoreManager.promises.updateDoc( + projectId.toString(), + doc._id.toString(), + docLines, + 0, + {} + ) + + doc.rev = rev + const project = + await ProjectEntityMongoUpdateHandler.promises.replaceFileWithDoc( + projectId, + existingFile._id, + doc + ) + + await TpdsUpdateSender.promises.addDoc({ + projectId, + docId: doc._id, + path: filePath, + projectName: project.name, + rev: existingFile.rev + 1, + folderId, + }) + + const projectHistoryId = + project.overleaf && + project.overleaf.history && + project.overleaf.history.id + const newDocs = [ + { + doc, + path: filePath, + docLines: docLines.join('\n'), + }, + ] + const oldFiles = [ + { + file: existingFile, + path: filePath, + }, + ] + await DocumentUpdaterHandler.promises.updateProjectStructure( + projectId, + projectHistoryId, + userId, + { oldFiles, newDocs, newProject: project }, + source + ) + + EditorRealTimeController.emitToRoom( + projectId, + 'removeEntity', + existingFile._id, + 'convertFileToDoc' + ) + return { doc, isNew: true } + } else if (existingDoc) { + const result = await DocumentUpdaterHandler.promises.setDocument( + projectId, + existingDoc._id, + userId, + docLines, + source + ) + logger.debug( + { projectId, docId: existingDoc._id }, + 'notifying users that the document has been updated' + ) + // there is no need to flush the doc to mongo at this point as docupdater + // flushes it as part of setDoc. + // + // combine rev from response with existing doc metadata + return { + doc: { ...existingDoc, ...result }, + isNew: existingDoc == null, + } + } else { + const { doc } = + await ProjectEntityUpdateHandler.promises.addDocWithRanges.withoutLock( + projectId, + folderId, + docName, + docLines, + {}, + userId, + source + ) + + return { doc, isNew: existingDoc == null } + } } ) const upsertFile = wrapWithLock({ - beforeLock(next) { - return function ( - projectId, - folderId, - fileName, - fsPath, - linkedFileData, - userId, - source, - callback - ) { - if (!SafePath.isCleanFilename(fileName)) { - return callback(new Errors.InvalidNameError('invalid element name')) - } - // create a new file - const fileArgs = { - name: fileName, - linkedFileData, - } - FileStoreHandler.uploadFileFromDisk( - projectId, - fileArgs, - fsPath, - (err, fileStoreUrl, fileRef) => { - if (err != null) { - return callback(err) - } - next( - projectId, - folderId, - fileName, - fsPath, - linkedFileData, - userId, - fileRef, - fileStoreUrl, - source, - callback - ) - } - ) - } - }, - withLock( + async beforeLock( projectId, folderId, fileName, fsPath, linkedFileData, userId, - newFileRef, + source + ) { + if (!SafePath.isCleanFilename(fileName)) { + throw new Errors.InvalidNameError('invalid element name') + } + // create a new file + const fileArgs = { + name: fileName, + linkedFileData, + } + const { url, fileRef } = await FileStoreHandler.promises.uploadFileFromDisk( + projectId, + fileArgs, + fsPath + ) + + return { + projectId, + folderId, + fileName, + fsPath, + linkedFileData, + userId, + fileRef, + fileStoreUrl: url, + source, + } + }, + async withLock({ + projectId, + folderId, + fileName, + userId, + fileRef, fileStoreUrl, source, - callback - ) { - ProjectLocator.findElement( - { project_id: projectId, element_id: folderId, type: 'folder' }, - (error, folder) => { - if (error != null) { - if (error instanceof Errors.NotFoundError && folder == null) { - return callback(new Error('folder_not_found')) - } - return callback(error) - } - if (folder == null) { - return callback(new Error("Couldn't find folder")) - } - const existingFile = folder.fileRefs.find( - ({ name }) => name === fileName - ) - const existingDoc = folder.docs.find(({ name }) => name === fileName) - - if (existingDoc) { - ProjectLocator.findElement( - { - project_id: projectId, - element_id: existingDoc._id, - type: 'doc', - }, - (err, doc, path) => { - if (err) { - return callback(new Error('coudnt find existing file')) - } - ProjectEntityMongoUpdateHandler.replaceDocWithFile( - projectId, - existingDoc._id, - newFileRef, - (err, project) => { - if (err) { - return callback(err) - } - const projectHistoryId = - project.overleaf && - project.overleaf.history && - project.overleaf.history.id - TpdsUpdateSender.addFile( - { - projectId: project._id, - fileId: newFileRef._id, - path: path.fileSystem, - rev: newFileRef.rev, - projectName: project.name, - folderId, - }, - err => { - if (err) { - return callback(err) - } - DocumentUpdaterHandler.updateProjectStructure( - projectId, - projectHistoryId, - userId, - { - oldDocs: [ - { doc: existingDoc, path: path.fileSystem }, - ], - - newFiles: [ - { - file: newFileRef, - path: path.fileSystem, - url: fileStoreUrl, - }, - ], - newProject: project, - }, - source, - err => { - if (err) { - return callback(err) - } - EditorRealTimeController.emitToRoom( - projectId, - 'removeEntity', - existingDoc._id, - 'convertDocToFile' - ) - callback(null, newFileRef, true, existingFile) - } - ) - } - ) - } - ) - } - ) - } else if (existingFile) { - ProjectEntityUpdateHandler._replaceFile( - projectId, - existingFile._id, - fsPath, - linkedFileData, - userId, - newFileRef, - fileStoreUrl, - folderId, - source, - (err, newFileRef) => { - if (err != null) { - return callback(err) - } - callback(null, newFileRef, existingFile == null, existingFile) - } - ) - } else { - // this calls directly into the addFile main task (without the beforeLock part) - ProjectEntityUpdateHandler.addFile.mainTask( - projectId, - folderId, - fileName, - fsPath, - linkedFileData, - userId, - newFileRef, - fileStoreUrl, - source, - err => { - if (err != null) { - return callback(err) - } - callback(null, newFileRef, existingFile == null, existingFile) - } - ) - } + }) { + let element + try { + ;({ element } = await ProjectLocator.promises.findElement({ + project_id: projectId, + element_id: folderId, + type: 'folder', + })) + } catch (error) { + if (error instanceof Errors.NotFoundError) { + throw new Error('folder_not_found') } - ) + throw error + } + + if (element == null) { + throw new Error("Couldn't find folder") + } + const existingFile = element.fileRefs.find(({ name }) => name === fileName) + const existingDoc = element.docs.find(({ name }) => name === fileName) + + if (existingDoc) { + let path + try { + ;({ path } = await ProjectLocator.promises.findElement({ + project_id: projectId, + element_id: existingDoc._id, + type: 'doc', + })) + } catch (err) { + throw new Error("couldn't find existing file") + } + const project = + await ProjectEntityMongoUpdateHandler.promises.replaceDocWithFile( + projectId, + existingDoc._id, + fileRef + ) + const projectHistoryId = project.overleaf?.history?.id + await TpdsUpdateSender.promises.addFile({ + projectId: project._id, + fileId: fileRef._id, + path: path.fileSystem, + rev: fileRef.rev, + projectName: project.name, + folderId, + }) + await DocumentUpdaterHandler.promises.updateProjectStructure( + projectId, + projectHistoryId, + userId, + { + oldDocs: [{ doc: existingDoc, path: path.fileSystem }], + + newFiles: [ + { + file: fileRef, + path: path.fileSystem, + url: fileStoreUrl, + }, + ], + newProject: project, + }, + source + ) + EditorRealTimeController.emitToRoom( + projectId, + 'removeEntity', + existingDoc._id, + 'convertDocToFile' + ) + return { fileRef, isNew: true, oldFileRef: existingFile } + } else if (existingFile) { + await ProjectEntityUpdateHandler._replaceFile( + projectId, + existingFile._id, + userId, + fileRef, + fileStoreUrl, + folderId, + source + ) + + return { fileRef, isNew: false, oldFileRef: existingFile } + } else { + // this calls directly into the addFile main task (without the beforeLock part) + await ProjectEntityUpdateHandler.promises.addFile.mainTask({ + projectId, + folderId, + userId, + fileRef, + fileStoreUrl, + source, + }) + + return { + fileRef, + isNew: existingFile == null, + oldFileRef: existingFile, + } + } }, }) const upsertDocWithPath = wrapWithLock( - function (projectId, elementPath, docLines, source, userId, callback) { + async function (projectId, elementPath, docLines, source, userId) { if (!SafePath.isCleanPath(elementPath)) { - return callback(new Errors.InvalidNameError('invalid element name')) + throw new Errors.InvalidNameError('invalid element name') } const docName = Path.basename(elementPath) const folderPath = Path.dirname(elementPath) - ProjectEntityUpdateHandler.mkdirp.withoutLock( - projectId, - folderPath, - (err, newFolders, folder) => { - if (err != null) { - return callback(err) - } - ProjectEntityUpdateHandler.upsertDoc.withoutLock( - projectId, - folder._id, - docName, - docLines, - source, - userId, - (err, doc, isNewDoc) => { - if (err != null) { - return callback(err) - } - callback(null, doc, isNewDoc, newFolders, folder) - } - ) - } - ) + const { newFolders, folder } = + await ProjectEntityUpdateHandler.promises.mkdirp.withoutLock( + projectId, + folderPath + ) + const { isNew, doc } = + await ProjectEntityUpdateHandler.promises.upsertDoc.withoutLock( + projectId, + folder._id, + docName, + docLines, + source, + userId + ) + + return { doc, isNew, newFolders, folder } } ) const upsertFileWithPath = wrapWithLock({ - beforeLock(next) { - return function ( + async beforeLock( + projectId, + elementPath, + fsPath, + linkedFileData, + userId, + source + ) { + if (!SafePath.isCleanPath(elementPath)) { + throw new Errors.InvalidNameError('invalid element name') + } + const fileName = Path.basename(elementPath) + const folderPath = Path.dirname(elementPath) + // create a new file + const fileArgs = { + name: fileName, + linkedFileData, + } + const { url: fileStoreUrl, fileRef } = + await FileStoreHandler.promises.uploadFileFromDisk( + projectId, + fileArgs, + fsPath + ) + + return { projectId, - elementPath, + folderPath, + fileName, fsPath, linkedFileData, userId, + fileRef, + fileStoreUrl, source, - callback - ) { - if (!SafePath.isCleanPath(elementPath)) { - return callback(new Errors.InvalidNameError('invalid element name')) - } - const fileName = Path.basename(elementPath) - const folderPath = Path.dirname(elementPath) - // create a new file - const fileArgs = { - name: fileName, - linkedFileData, - } - FileStoreHandler.uploadFileFromDisk( - projectId, - fileArgs, - fsPath, - (err, fileStoreUrl, fileRef) => { - if (err != null) { - return callback(err) - } - next( - projectId, - folderPath, - fileName, - fsPath, - linkedFileData, - userId, - fileRef, - fileStoreUrl, - source, - callback - ) - } - ) } }, - withLock( + async withLock({ projectId, folderPath, fileName, @@ -903,166 +730,146 @@ const upsertFileWithPath = wrapWithLock({ fileRef, fileStoreUrl, source, - callback - ) { - ProjectEntityUpdateHandler.mkdirp.withoutLock( + }) { + const { newFolders, folder } = + await ProjectEntityUpdateHandler.promises.mkdirp.withoutLock( + projectId, + folderPath + ) + // this calls directly into the upsertFile main task (without the beforeLock part) + const { + fileRef: newFileRef, + isNew, + oldFileRef, + } = await ProjectEntityUpdateHandler.promises.upsertFile.mainTask({ projectId, - folderPath, - (err, newFolders, folder) => { - if (err != null) { - return callback(err) - } - // this calls directly into the upsertFile main task (without the beforeLock part) - ProjectEntityUpdateHandler.upsertFile.mainTask( - projectId, - folder._id, - fileName, - fsPath, - linkedFileData, - userId, - fileRef, - fileStoreUrl, - source, - (err, newFile, isNewFile, existingFile) => { - if (err != null) { - return callback(err) - } - callback(null, newFile, isNewFile, existingFile, newFolders, folder) - } - ) - } - ) + folderId: folder._id, + fileName, + fsPath, + linkedFileData, + userId, + fileRef, + fileStoreUrl, + source, + }) + + return { + fileRef: newFileRef, + isNew, + oldFileRef, + newFolders, + folder, + } }, }) const deleteEntity = wrapWithLock( - function (projectId, entityId, entityType, userId, source, callback) { + async function (projectId, entityId, entityType, userId, source, callback) { logger.debug({ entityId, entityType, projectId }, 'deleting project entity') if (entityType == null) { logger.warn({ err: 'No entityType set', projectId, entityId }) - return callback(new Error('No entityType set')) + throw new Error('No entityType set') } entityType = entityType.toLowerCase() - ProjectEntityMongoUpdateHandler.deleteEntity( + const { entity, path, projectBeforeDeletion, newProject } = + await ProjectEntityMongoUpdateHandler.promises.deleteEntity( + projectId, + entityId, + entityType + ) + const subtreeListing = await ProjectEntityUpdateHandler._cleanUpEntity( + projectBeforeDeletion, + newProject, + entity, + entityType, + path.fileSystem, + userId, + source + ) + + const subtreeEntityIds = subtreeListing.map(entry => + entry.entity._id.toString() + ) + await TpdsUpdateSender.promises.deleteEntity({ projectId, + path: path.fileSystem, + projectName: projectBeforeDeletion.name, entityId, entityType, - (error, entity, path, projectBeforeDeletion, newProject) => { - if (error != null) { - return callback(error) - } - ProjectEntityUpdateHandler._cleanUpEntity( - projectBeforeDeletion, - newProject, - entity, - entityType, - path.fileSystem, - userId, - source, - (error, subtreeListing) => { - if (error != null) { - return callback(error) - } - const subtreeEntityIds = subtreeListing.map(entry => - entry.entity._id.toString() - ) - TpdsUpdateSender.deleteEntity( - { - projectId, - path: path.fileSystem, - projectName: projectBeforeDeletion.name, - entityId, - entityType, - subtreeEntityIds, - }, - error => { - if (error != null) { - return callback(error) - } - callback(null, entityId) - } - ) - } - ) - } - ) + subtreeEntityIds, + }) + + return entityId } ) const deleteEntityWithPath = wrapWithLock( - (projectId, path, userId, source, callback) => - ProjectLocator.findElementByPath( - { project_id: projectId, path, exactCaseMatch: true }, - (err, element, type) => { - if (err != null) { - return callback(err) - } - if (element == null) { - return callback(new Errors.NotFoundError('project not found')) - } - ProjectEntityUpdateHandler.deleteEntity.withoutLock( - projectId, - element._id, - type, - userId, - source, - callback - ) - } + async (projectId, path, userId, source) => { + const { element, type } = await ProjectLocator.promises.findElementByPath({ + project_id: projectId, + path, + exactCaseMatch: true, + }) + if (element == null) { + throw new Errors.NotFoundError('project not found') + } + return await ProjectEntityUpdateHandler.promises.deleteEntity.withoutLock( + projectId, + element._id, + type, + userId, + source ) + } ) -const mkdirp = wrapWithLock(function (projectId, path, callback) { +const mkdirp = wrapWithLock(async function (projectId, path) { for (const folder of path.split('/')) { if (folder.length > 0 && !SafePath.isCleanFilename(folder)) { - return callback(new Errors.InvalidNameError('invalid element name')) + throw new Errors.InvalidNameError('invalid element name') } } - ProjectEntityMongoUpdateHandler.mkdirp( + return await ProjectEntityMongoUpdateHandler.promises.mkdirp( projectId, path, - { exactCaseMatch: false }, - callback + { exactCaseMatch: false } ) }) -const mkdirpWithExactCase = wrapWithLock(function (projectId, path, callback) { +const mkdirpWithExactCase = wrapWithLock(async function (projectId, path) { for (const folder of path.split('/')) { if (folder.length > 0 && !SafePath.isCleanFilename(folder)) { - return callback(new Errors.InvalidNameError('invalid element name')) + throw new Errors.InvalidNameError('invalid element name') } } - ProjectEntityMongoUpdateHandler.mkdirp( + return await ProjectEntityMongoUpdateHandler.promises.mkdirp( projectId, path, - { exactCaseMatch: true }, - callback + { exactCaseMatch: true } ) }) const addFolder = wrapWithLock( - function (projectId, parentFolderId, folderName, callback) { + async function (projectId, parentFolderId, folderName) { if (!SafePath.isCleanFilename(folderName)) { - return callback(new Errors.InvalidNameError('invalid element name')) + throw new Errors.InvalidNameError('invalid element name') } - ProjectEntityMongoUpdateHandler.addFolder( + return await ProjectEntityMongoUpdateHandler.promises.addFolder( projectId, parentFolderId, - folderName, - callback + folderName ) } ) const moveEntity = wrapWithLock( - function ( + async function ( projectId, entityId, destFolderId, entityType, userId, - source, - callback + source ) { logger.debug( { entityType, entityId, projectId, destFolderId }, @@ -1070,67 +877,46 @@ const moveEntity = wrapWithLock( ) if (entityType == null) { logger.warn({ err: 'No entityType set', projectId, entityId }) - return callback(new Error('No entityType set')) + throw new Error('No entityType set') } entityType = entityType.toLowerCase() - DocumentUpdaterHandler.flushProjectToMongo(projectId, err => { - if (err) { - return callback(err) - } - ProjectEntityMongoUpdateHandler.moveEntity( + await DocumentUpdaterHandler.promises.flushProjectToMongo(projectId) + const { project, startPath, endPath, rev, changes } = + await ProjectEntityMongoUpdateHandler.promises.moveEntity( projectId, entityId, destFolderId, - entityType, - (err, project, startPath, endPath, rev, changes) => { - if (err != null) { - return callback(err) - } - const projectHistoryId = - project.overleaf && - project.overleaf.history && - project.overleaf.history.id - TpdsUpdateSender.moveEntity( - { - projectId, - projectName: project.name, - startPath, - endPath, - rev, - entityId, - entityType, - folderId: destFolderId, - }, - err => { - if (err) { - logger.error({ err }, 'error sending tpds update') - } - DocumentUpdaterHandler.updateProjectStructure( - projectId, - projectHistoryId, - userId, - changes, - source, - callback - ) - } - ) - } + entityType ) - }) + + const projectHistoryId = project.overleaf?.history?.id + try { + await TpdsUpdateSender.promises.moveEntity({ + projectId, + projectName: project.name, + startPath, + endPath, + rev, + entityId, + entityType, + folderId: destFolderId, + }) + } catch (err) { + logger.error({ err }, 'error sending tpds update') + } + + return await DocumentUpdaterHandler.promises.updateProjectStructure( + projectId, + projectHistoryId, + userId, + changes, + source + ) } ) const renameEntity = wrapWithLock( - function ( - projectId, - entityId, - entityType, - newName, - userId, - source, - callback - ) { + async function (projectId, entityId, entityType, newName, userId, source) { if (!newName || typeof newName !== 'string') { const err = new OError('invalid newName value', { value: newName, @@ -1142,210 +928,133 @@ const renameEntity = wrapWithLock( source, }) logger.error({ err }, 'Invalid newName passed to renameEntity') - return callback(err) + throw err } if (!SafePath.isCleanFilename(newName)) { - return callback(new Errors.InvalidNameError('invalid element name')) + throw new Errors.InvalidNameError('invalid element name') } logger.debug({ entityId, projectId }, `renaming ${entityType}`) if (entityType == null) { logger.warn({ err: 'No entityType set', projectId, entityId }) - return callback(new Error('No entityType set')) + throw new Error('No entityType set') } entityType = entityType.toLowerCase() - - DocumentUpdaterHandler.flushProjectToMongo(projectId, err => { - if (err) { - return callback(err) - } - ProjectEntityMongoUpdateHandler.renameEntity( + await DocumentUpdaterHandler.promises.flushProjectToMongo(projectId) + const { project, startPath, endPath, rev, changes } = + await ProjectEntityMongoUpdateHandler.promises.renameEntity( projectId, entityId, entityType, - newName, - (err, project, startPath, endPath, rev, changes) => { - if (err != null) { - return callback(err) - } - const projectHistoryId = - project.overleaf && - project.overleaf.history && - project.overleaf.history.id - TpdsUpdateSender.moveEntity( - { - projectId, - projectName: project.name, - startPath, - endPath, - rev, - entityId, - entityType, - folderId: null, // this means the folder has not changed - }, - err => { - if (err) { - logger.error({ err }, 'error sending tpds update') - } - DocumentUpdaterHandler.updateProjectStructure( - projectId, - projectHistoryId, - userId, - changes, - source, - callback - ) - } - ) - } + newName ) - }) + + const projectHistoryId = project.overleaf?.history?.id + try { + await TpdsUpdateSender.promises.moveEntity({ + projectId, + projectName: project.name, + startPath, + endPath, + rev, + entityId, + entityType, + folderId: null, // this means the folder has not changed + }) + } catch (err) { + logger.error({ err }, 'error sending tpds update') + } + return await DocumentUpdaterHandler.promises.updateProjectStructure( + projectId, + projectHistoryId, + userId, + changes, + source + ) } ) // This doesn't directly update project structure, but we need to take the lock // to prevent anything else being queued before the resync update const resyncProjectHistory = wrapWithLock( - (projectId, opts, callback) => - ProjectGetter.getProject( + async (projectId, opts) => { + const project = await ProjectGetter.promises.getProject(projectId, { + rootFolder: true, + overleaf: true, + }) + const projectHistoryId = project.overleaf?.history?.id + if (projectHistoryId == null) { + throw new Errors.ProjectHistoryDisabledError( + `project history not enabled for ${projectId}` + ) + } + + const { docs, files, folders } = + ProjectEntityHandler.getAllEntitiesFromProject(project) + // _checkFileTree() must be passed the folders before docs and + // files + await ProjectEntityUpdateHandler._checkFiletree( projectId, - { rootFolder: true, overleaf: true }, - (error, project) => { - if (error != null) { - return callback(error) - } + projectHistoryId, + [...folders, ...docs, ...files] + ) - const projectHistoryId = - project && - project.overleaf && - project.overleaf.history && - project.overleaf.history.id - if (projectHistoryId == null) { - error = new Errors.ProjectHistoryDisabledError( - `project history not enabled for ${projectId}` - ) - return callback(error) - } - - let docs, files, folders - try { - ;({ docs, files, folders } = - ProjectEntityHandler.getAllEntitiesFromProject(project)) - } catch (error) { - return callback(error) - } - // _checkFileTree() must be passed the folders before docs and - // files - ProjectEntityUpdateHandler._checkFiletree( - projectId, - projectHistoryId, - [...folders, ...docs, ...files], - error => { - if (error) { - return callback(error) - } - - DocumentUpdaterHandler.resyncProjectHistory( - projectId, - projectHistoryId, - docs, - files, - opts, - err => { - if (err) { - return callback(err) - } - if (opts.historyRangesMigration) { - ProjectOptionsHandler.setHistoryRangesSupport( - projectId, - opts.historyRangesMigration === 'forwards', - callback - ) - } else { - callback() - } - } - ) - } - ) - } - ), + await DocumentUpdaterHandler.promises.resyncProjectHistory( + projectId, + projectHistoryId, + docs, + files, + opts + ) + if (opts.historyRangesMigration) { + return await ProjectOptionsHandler.promises.setHistoryRangesSupport( + projectId, + opts.historyRangesMigration === 'forwards' + ) + } + }, LockManager.withTimeout(6 * 60) // use an extended lock for the resync operations ) const convertDocToFile = wrapWithLock({ - beforeLock(next) { - return function (projectId, docId, userId, source, callback) { - DocumentUpdaterHandler.flushDocToMongo(projectId, docId, err => { - if (err) { - return callback(err) - } - ProjectLocator.findElement( - { project_id: projectId, element_id: docId, type: 'doc' }, - (err, doc, path) => { - const docPath = path.fileSystem - if (err) { - return callback(err) - } - DocstoreManager.getDoc( - projectId, - docId, - (err, docLines, rev, version, ranges) => { - if (err) { - return callback(err) - } - if (!_.isEmpty(ranges)) { - return callback(new Errors.DocHasRangesError({})) - } - DocumentUpdaterHandler.deleteDoc(projectId, docId, err => { - if (err) { - return callback(err) - } - FileWriter.writeLinesToDisk( - projectId, - docLines, - (err, fsPath) => { - if (err) { - return callback(err) - } - FileStoreHandler.uploadFileFromDisk( - projectId, - { name: doc.name, rev: rev + 1 }, - fsPath, - (err, fileStoreUrl, fileRef) => { - if (err) { - return callback(err) - } - fs.unlink(fsPath, err => { - if (err) { - logger.warn( - { err, path: fsPath }, - 'failed to clean up temporary file' - ) - } - next( - projectId, - doc, - docPath, - fileRef, - fileStoreUrl, - userId, - source, - callback - ) - }) - } - ) - } - ) - }) - } - ) - } - ) - }) + async beforeLock(projectId, docId, userId, source) { + await DocumentUpdaterHandler.promises.flushDocToMongo(projectId, docId) + const { element: doc, path } = await ProjectLocator.promises.findElement({ + project_id: projectId, + element_id: docId, + type: 'doc', + }) + const docPath = path.fileSystem + const { lines, rev, ranges } = await DocstoreManager.promises.getDoc( + projectId, + docId + ) + if (!_.isEmpty(ranges)) { + throw new Errors.DocHasRangesError({}) + } + await DocumentUpdaterHandler.promises.deleteDoc(projectId, docId, false) + const fsPath = await FileWriter.promises.writeLinesToDisk(projectId, lines) + const { url: fileStoreUrl, fileRef } = + await FileStoreHandler.promises.uploadFileFromDisk( + projectId, + { name: doc.name, rev: rev + 1 }, + fsPath + ) + try { + await fs.promises.unlink(fsPath) + } catch (err) { + logger.warn({ err, path: fsPath }, 'failed to clean up temporary file') + } + return { + projectId, + doc, + path: docPath, + fileRef, + fileStoreUrl, + userId, + source, } }, - withLock( + async withLock({ projectId, doc, path, @@ -1353,283 +1062,258 @@ const convertDocToFile = wrapWithLock({ fileStoreUrl, userId, source, - callback - ) { - ProjectEntityMongoUpdateHandler.replaceDocWithFile( + }) { + const project = + await ProjectEntityMongoUpdateHandler.promises.replaceDocWithFile( + projectId, + doc._id, + fileRef + ) + const projectHistoryId = project.overleaf?.history?.id + await DocumentUpdaterHandler.promises.updateProjectStructure( projectId, - doc._id, - fileRef, - (err, project) => { - if (err) { - return callback(err) - } - const projectHistoryId = - project.overleaf && - project.overleaf.history && - project.overleaf.history.id - DocumentUpdaterHandler.updateProjectStructure( - projectId, - projectHistoryId, - userId, - { - oldDocs: [{ doc, path }], - newFiles: [{ file: fileRef, path, url: fileStoreUrl }], - newProject: project, - }, - source, - err => { - if (err) { - return callback(err) - } - ProjectLocator.findElement( - { - project_id: projectId, - element_id: fileRef._id, - type: 'file', - }, - (err, element, path, folder) => { - if (err) { - return callback(err) - } - EditorRealTimeController.emitToRoom( - projectId, - 'removeEntity', - doc._id, - 'convertDocToFile' - ) - EditorRealTimeController.emitToRoom( - projectId, - 'reciveNewFile', - folder._id, - fileRef, - 'convertDocToFile', - null, - userId - ) - callback(null, fileRef) - } - ) - } - ) - } + projectHistoryId, + userId, + { + oldDocs: [{ doc, path }], + newFiles: [{ file: fileRef, path, url: fileStoreUrl }], + newProject: project, + }, + source ) + const { folder } = await ProjectLocator.promises.findElement({ + project_id: projectId, + element_id: fileRef._id, + type: 'file', + }) + EditorRealTimeController.emitToRoom( + projectId, + 'removeEntity', + doc._id, + 'convertDocToFile' + ) + EditorRealTimeController.emitToRoom( + projectId, + 'reciveNewFile', + folder._id, + fileRef, + 'convertDocToFile', + null, + userId + ) + return fileRef }, }) const ProjectEntityUpdateHandler = { LOCK_NAMESPACE, - addDoc, + addDoc: callbackifyMultiResult(addDoc, ['doc', 'folderId']), - addDocWithRanges, + addDocWithRanges: callbackifyMultiResult(addDocWithRanges, [ + 'doc', + 'folderId', + ]), - addFile, + addFile: callbackifyMultiResult(addFile, ['fileRef', 'folderId']), - addFolder, + addFolder: callbackifyMultiResult(addFolder, ['folder', 'parentFolderId']), - convertDocToFile, + convertDocToFile: callbackify(convertDocToFile), - deleteEntity, + deleteEntity: callbackify(deleteEntity), - deleteEntityWithPath, + deleteEntityWithPath: callbackify(deleteEntityWithPath), - mkdirp, + mkdirp: callbackifyMultiResult(mkdirp, [ + 'newFolders', + 'folder', + 'parentFolder', + ]), - mkdirpWithExactCase, + mkdirpWithExactCase: callbackifyMultiResult(mkdirpWithExactCase, [ + 'newFolders', + 'folder', + 'parentFolder', + ]), - moveEntity, + moveEntity: callbackify(moveEntity), - renameEntity, + renameEntity: callbackify(renameEntity), - resyncProjectHistory, + resyncProjectHistory: callbackify(resyncProjectHistory), - setRootDoc, + setRootDoc: callbackify(setRootDoc), - unsetRootDoc, + unsetRootDoc: callbackify(unsetRootDoc), - updateDocLines, + updateDocLines: callbackify(updateDocLines), - upsertDoc, + upsertDoc: callbackifyMultiResult(upsertDoc, ['doc', 'isNew']), - upsertDocWithPath, + upsertDocWithPath: callbackifyMultiResult(upsertDocWithPath, [ + 'doc', + 'isNew', + 'newFolders', + 'folder', + ]), - upsertFile, + upsertFile: callbackifyMultiResult(upsertFile, [ + 'fileRef', + 'isNew', + 'oldFileRef', + ]), - upsertFileWithPath, + upsertFileWithPath: callbackifyMultiResult(upsertFileWithPath, [ + 'fileRef', + 'isNew', + 'oldFileRef', + 'newFolders', + 'folder', + ]), - _addDocAndSendToTpds(projectId, folderId, doc, callback) { - ProjectEntityMongoUpdateHandler.addDoc( + async _addDocAndSendToTpds(projectId, folderId, doc) { + let result, project + try { + ;({ result, project } = + await ProjectEntityMongoUpdateHandler.promises.addDoc( + projectId, + folderId, + doc + )) + } catch (err) { + throw OError.tag(err, 'error adding file with project', { + projectId, + folderId, + doc_name: doc != null ? doc.name : undefined, + doc_id: doc != null ? doc._id : undefined, + }) + } + + await TpdsUpdateSender.promises.addDoc({ projectId, + docId: doc != null ? doc._id : undefined, + path: result?.path?.fileSystem, + projectName: project.name, + rev: 0, folderId, - doc, - (err, result, project) => { - if (err != null) { - OError.tag(err, 'error adding file with project', { - projectId, - folderId, - doc_name: doc != null ? doc.name : undefined, - doc_id: doc != null ? doc._id : undefined, - }) - return callback(err) - } - TpdsUpdateSender.addDoc( - { - projectId, - docId: doc != null ? doc._id : undefined, - path: result?.path?.fileSystem, - projectName: project.name, - rev: 0, - folderId, - }, - err => { - if (err != null) { - return callback(err) - } - callback(null, result, project) - } - ) - } - ) + }) + return { result, project } }, - _uploadFile(projectId, folderId, fileName, fsPath, linkedFileData, callback) { + async _uploadFile(projectId, folderId, fileName, fsPath, linkedFileData) { if (!SafePath.isCleanFilename(fileName)) { - return callback(new Errors.InvalidNameError('invalid element name')) + throw new Errors.InvalidNameError('invalid element name') } const fileArgs = { name: fileName, linkedFileData, } - FileStoreHandler.uploadFileFromDisk( - projectId, - fileArgs, - fsPath, - (err, fileStoreUrl, fileRef) => { - if (err != null) { - OError.tag(err, 'error uploading image to s3', { - projectId, - folderId, - file_name: fileName, - fileRef, - }) - return callback(err) - } - callback(null, fileStoreUrl, fileRef) - } - ) + try { + return await FileStoreHandler.promises.uploadFileFromDisk( + projectId, + fileArgs, + fsPath + ) + } catch (err) { + throw OError.tag(err, 'error uploading image to s3', { + projectId, + folderId, + file_name: fileName, + }) + } }, - _addFileAndSendToTpds(projectId, folderId, fileRef, callback) { - ProjectEntityMongoUpdateHandler.addFile( + async _addFileAndSendToTpds(projectId, folderId, fileRef) { + let result, project + try { + ;({ result, project } = + await ProjectEntityMongoUpdateHandler.promises.addFile( + projectId, + folderId, + fileRef + )) + } catch (err) { + throw OError.tag(err, 'error adding file with project', { + projectId, + folderId, + file_name: fileRef.name, + fileRef, + }) + } + + await TpdsUpdateSender.promises.addFile({ projectId, + fileId: fileRef._id, + path: result?.path?.fileSystem, + projectName: project.name, + rev: fileRef.rev, folderId, - fileRef, - (err, result, project) => { - if (err != null) { - OError.tag(err, 'error adding file with project', { - projectId, - folderId, - file_name: fileRef.name, - fileRef, - }) - return callback(err) - } - TpdsUpdateSender.addFile( - { - projectId, - fileId: fileRef._id, - path: result?.path?.fileSystem, - projectName: project.name, - rev: fileRef.rev, - folderId, - }, - err => { - if (err != null) { - return callback(err) - } - callback(null, result, project) - } - ) - } - ) + }) + return { result, project } }, - _replaceFile( + async _replaceFile( projectId, fileId, - fsPath, - linkedFileData, userId, newFileRef, fileStoreUrl, folderId, - source, - callback + source ) { - ProjectEntityMongoUpdateHandler.replaceFileWithNew( + const { + oldFileRef, + project, + path, + newProject, + newFileRef: updatedFileRef, + } = await ProjectEntityMongoUpdateHandler.promises.replaceFileWithNew( projectId, fileId, - newFileRef, - (err, oldFileRef, project, path, newProject, newFileRef) => { - if (err != null) { - return callback(err) - } - const oldFiles = [ - { - file: oldFileRef, - path: path.fileSystem, - }, - ] - const newFiles = [ - { - file: newFileRef, - path: path.fileSystem, - url: fileStoreUrl, - }, - ] - const projectHistoryId = - project.overleaf && - project.overleaf.history && - project.overleaf.history.id - TpdsUpdateSender.addFile( - { - projectId: project._id, - fileId: newFileRef._id, - path: path.fileSystem, - rev: newFileRef.rev, - projectName: project.name, - folderId, - }, - err => { - if (err != null) { - return callback(err) - } - ProjectUpdateHandler.promises - .markAsUpdated(projectId, new Date(), userId) - .catch(error => { - logger.error({ error }, 'failed to mark project as updated') - }) - - DocumentUpdaterHandler.updateProjectStructure( - projectId, - projectHistoryId, - userId, - { oldFiles, newFiles, newProject }, - source, - err => { - if (err) { - return callback(err) - } - callback(null, newFileRef) - } - ) - } - ) - } + newFileRef ) + + const oldFiles = [ + { + file: oldFileRef, + path: path.fileSystem, + }, + ] + const newFiles = [ + { + file: updatedFileRef, + path: path.fileSystem, + url: fileStoreUrl, + }, + ] + const projectHistoryId = project.overleaf?.history?.id + await TpdsUpdateSender.promises.addFile({ + projectId: project._id, + fileId: updatedFileRef._id, + path: path.fileSystem, + rev: updatedFileRef.rev, + projectName: project.name, + folderId, + }) + ProjectUpdateHandler.promises + .markAsUpdated(projectId, new Date(), userId) + .catch(error => { + logger.error({ error }, 'failed to mark project as updated') + }) + + await DocumentUpdaterHandler.promises.updateProjectStructure( + projectId, + projectHistoryId, + userId, + { oldFiles, newFiles, newProject }, + source + ) + + return updatedFileRef }, - _checkFiletree(projectId, projectHistoryId, entities, callback) { + async _checkFiletree(projectId, projectHistoryId, entities) { const adjustPathsAfterFolderRename = (oldPath, newPath) => { oldPath = oldPath + '/' newPath = newPath + '/' @@ -1684,11 +1368,8 @@ const ProjectEntityUpdateHandler = { // the case only because getAllEntitiesFromProject() returns folders // in that order and resyncProjectHistory() calls us with the folders // first. - try { - adjustPathsAfterFolderRename(entity.path, newPath) - } catch (error) { - return callback(error) - } + + adjustPathsAfterFolderRename(entity.path, newPath) } } @@ -1697,7 +1378,7 @@ const ProjectEntityUpdateHandler = { } if (renames.length === 0) { - return callback() + return } logger.warn( { @@ -1710,45 +1391,6 @@ const ProjectEntityUpdateHandler = { 'found conflicts or bad filenames in filetree' ) - // rename the duplicate files - const doRename = (rename, cb) => { - const entity = rename.entity - const entityId = entity.folder - ? entity.folder._id - : entity.doc - ? entity.doc._id - : entity.file._id - const entityType = entity.folder ? 'folder' : entity.doc ? 'doc' : 'file' - ProjectEntityMongoUpdateHandler.renameEntity( - projectId, - entityId, - entityType, - rename.newName, - (err, project, startPath, endPath, rev, changes) => { - if (err) { - return cb(err) - } - // update the renamed entity for the resync - entity.path = rename.newPath - if (entityType === 'folder') { - entity.folder.name = rename.newName - } else if (entityType === 'doc') { - entity.doc.name = rename.newName - } else { - entity.file.name = rename.newName - } - DocumentUpdaterHandler.updateProjectStructure( - projectId, - projectHistoryId, - null, - changes, - 'automatic-fix', - cb - ) - } - ) - } - // Avoid conflicts by processing renames in the reverse order. If we have // the following starting situation: // @@ -1768,7 +1410,40 @@ const ProjectEntityUpdateHandler = { // information about existing files. renames.reverse() - async.eachSeries(renames, doRename, callback) + for (const rename of renames) { + // rename the duplicate files + const entity = rename.entity + const entityId = entity.folder + ? entity.folder._id + : entity.doc + ? entity.doc._id + : entity.file._id + const entityType = entity.folder ? 'folder' : entity.doc ? 'doc' : 'file' + const { changes } = + await ProjectEntityMongoUpdateHandler.promises.renameEntity( + projectId, + entityId, + entityType, + rename.newName + ) + + // update the renamed entity for the resync + entity.path = rename.newPath + if (entityType === 'folder') { + entity.folder.name = rename.newName + } else if (entityType === 'doc') { + entity.doc.name = rename.newName + } else { + entity.file.name = rename.newName + } + await DocumentUpdaterHandler.promises.updateProjectStructure( + projectId, + projectHistoryId, + null, + changes, + 'automatic-fix' + ) + } }, findNextAvailablePath(allPaths, candidatePath) { @@ -1798,69 +1473,45 @@ const ProjectEntityUpdateHandler = { return VALID_ROOT_DOC_REGEXP.test(docExtension) }, - _cleanUpEntity( + async _cleanUpEntity( project, newProject, entity, entityType, path, userId, - source, - callback + source ) { const subtreeListing = _listSubtree(entity, entityType, path) - ProjectEntityUpdateHandler._updateProjectStructureWithDeletedEntity( + await ProjectEntityUpdateHandler._updateProjectStructureWithDeletedEntity( project, newProject, subtreeListing, userId, - source, - error => { - if (error != null) { - return callback(error) - } - const jobs = [] - - for (const entry of subtreeListing) { - if (entry.type === 'doc') { - jobs.push(cb => { - ProjectEntityUpdateHandler._cleanUpDoc( - project, - entry.entity, - entry.path, - userId, - cb - ) - }) - } else if (entry.type === 'file') { - jobs.push(cb => { - ProjectEntityUpdateHandler._cleanUpFile( - project, - entry.entity, - entry.path, - userId, - cb - ) - }) - } - } - async.series(jobs, err => { - if (err) { - return callback(err) - } - callback(null, subtreeListing) - }) - } + source ) + + for (const entry of subtreeListing) { + if (entry.type === 'doc') { + await ProjectEntityUpdateHandler._cleanUpDoc( + project, + entry.entity, + entry.path, + userId + ) + } else if (entry.type === 'file') { + await ProjectEntityUpdateHandler._cleanUpFile(project, entry.entity) + } + } + return subtreeListing }, - _updateProjectStructureWithDeletedEntity( + async _updateProjectStructureWithDeletedEntity( project, newProject, subtreeListing, userId, - source, - callback + source ) { const changes = { oldDocs: [], oldFiles: [] } for (const entry of subtreeListing) { @@ -1878,50 +1529,33 @@ const ProjectEntityUpdateHandler = { project.overleaf && project.overleaf.history && project.overleaf.history.id - DocumentUpdaterHandler.updateProjectStructure( + return await DocumentUpdaterHandler.promises.updateProjectStructure( projectId, projectHistoryId, userId, changes, - source, - callback + source ) }, - _cleanUpDoc(project, doc, path, userId, callback) { + async _cleanUpDoc(project, doc) { const projectId = project._id.toString() const docId = doc._id.toString() - const unsetRootDocIfRequired = callback => { - if ( - project.rootDoc_id != null && - project.rootDoc_id.toString() === docId - ) { - ProjectEntityUpdateHandler.unsetRootDoc(projectId, callback) - } else { - callback() - } + if (project.rootDoc_id != null && project.rootDoc_id.toString() === docId) { + await ProjectEntityUpdateHandler.promises.unsetRootDoc(projectId) } - unsetRootDocIfRequired(error => { - if (error != null) { - return callback(error) - } - const { name } = doc - const deletedAt = new Date() - DocstoreManager.deleteDoc(projectId, docId, name, deletedAt, error => { - if (error) { - return callback(error) - } - DocumentUpdaterHandler.deleteDoc(projectId, docId, callback) - }) - }) + const { name } = doc + const deletedAt = new Date() + await DocstoreManager.promises.deleteDoc(projectId, docId, name, deletedAt) + + return await DocumentUpdaterHandler.promises.deleteDoc(projectId, docId) }, - _cleanUpFile(project, file, path, userId, callback) { - ProjectEntityMongoUpdateHandler._insertDeletedFileReference( + async _cleanUpFile(project, file) { + return await ProjectEntityMongoUpdateHandler.promises._insertDeletedFileReference( project._id, - file, - callback + file ) }, }