From 6c8a34c2fe7cb7b0c72d3a5a271a03111bc19ab8 Mon Sep 17 00:00:00 2001 From: Eric Mc Sween Date: Wed, 22 Sep 2021 08:32:16 -0400 Subject: [PATCH] Merge pull request #5168 from overleaf/em-request-pipe-linked-files Fix linked files when they refer to empty files GitOrigin-RevId: 13c751e2679849516e874afdd422cf9e8d16c8d1 --- .../LinkedFiles/ProjectOutputFileAgent.js | 482 ++++++++---------- .../app/src/Features/LinkedFiles/UrlAgent.js | 168 +++--- 2 files changed, 284 insertions(+), 366 deletions(-) diff --git a/services/web/app/src/Features/LinkedFiles/ProjectOutputFileAgent.js b/services/web/app/src/Features/LinkedFiles/ProjectOutputFileAgent.js index 21979e7924..ec11c102f5 100644 --- a/services/web/app/src/Features/LinkedFiles/ProjectOutputFileAgent.js +++ b/services/web/app/src/Features/LinkedFiles/ProjectOutputFileAgent.js @@ -1,21 +1,4 @@ -/* eslint-disable - camelcase, - node/handle-callback-err, - max-len, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -let ProjectOutputFileAgent const AuthorizationManager = require('../Authorization/AuthorizationManager') -const ProjectGetter = require('../Project/ProjectGetter') -const Settings = require('@overleaf/settings') const CompileManager = require('../Compile/CompileManager') const ClsiManager = require('../Compile/ClsiManager') const ProjectFileAgent = require('./ProjectFileAgent') @@ -24,276 +7,235 @@ const { CompileFailedError, BadDataError, AccessDeniedError, - BadEntityTypeError, OutputFileFetchFailedError, } = require('./LinkedFilesErrors') const LinkedFilesHandler = require('./LinkedFilesHandler') -const logger = require('logger-sharelatex') -module.exports = ProjectOutputFileAgent = { - _prepare(project_id, linkedFileData, user_id, callback) { - if (callback == null) { - callback = function (err, linkedFileData) {} +function _prepare(projectId, linkedFileData, userId, callback) { + _checkAuth(projectId, linkedFileData, userId, (err, allowed) => { + if (err) { + return callback(err) } - return this._checkAuth( - project_id, - linkedFileData, - user_id, - (err, allowed) => { - if (err != null) { - return callback(err) - } - if (!allowed) { - return callback(new AccessDeniedError()) - } - if (!this._validate(linkedFileData)) { - return callback(new BadDataError()) - } - return callback(null, linkedFileData) - } - ) - }, - - createLinkedFile( - project_id, - linkedFileData, - name, - parent_folder_id, - user_id, - callback - ) { - if (!this._canCreate(linkedFileData)) { + if (!allowed) { return callback(new AccessDeniedError()) } - linkedFileData = this._sanitizeData(linkedFileData) - return this._prepare( - project_id, - linkedFileData, - user_id, - (err, linkedFileData) => { - if (err != null) { - return callback(err) - } - return this._getFileStream( - linkedFileData, - user_id, - (err, readStream) => { - if (err != null) { - return callback(err) - } - readStream.on('error', callback) - return readStream.on('response', response => { - if (response.statusCode >= 200 && response.statusCode < 300) { - readStream.resume() - return LinkedFilesHandler.importFromStream( - project_id, - readStream, - linkedFileData, - name, - parent_folder_id, - user_id, - function (err, file) { - if (err != null) { - return callback(err) - } - return callback(null, file._id) - } - ) // Created - } else { - err = new OutputFileFetchFailedError( - `Output file fetch failed: ${linkedFileData.build_id}, ${linkedFileData.source_output_file_path}` - ) - err.statusCode = response.statusCode - return callback(err) - } - }) - } - ) - } - ) - }, - - refreshLinkedFile( - project_id, - linkedFileData, - name, - parent_folder_id, - user_id, - callback - ) { - return this._prepare( - project_id, - linkedFileData, - user_id, - (err, linkedFileData) => { - if (err != null) { - return callback(err) - } - return this._compileAndGetFileStream( - linkedFileData, - user_id, - (err, readStream, new_build_id) => { - if (err != null) { - return callback(err) - } - readStream.on('error', callback) - return readStream.on('response', response => { - if (response.statusCode >= 200 && response.statusCode < 300) { - readStream.resume() - linkedFileData.build_id = new_build_id - return LinkedFilesHandler.importFromStream( - project_id, - readStream, - linkedFileData, - name, - parent_folder_id, - user_id, - function (err, file) { - if (err != null) { - return callback(err) - } - return callback(null, file._id) - } - ) // Created - } else { - err = new OutputFileFetchFailedError( - `Output file fetch failed: ${linkedFileData.build_id}, ${linkedFileData.source_output_file_path}` - ) - err.statusCode = response.statusCode - return callback(err) - } - }) - } - ) - } - ) - }, - - _sanitizeData(data) { - return { - provider: data.provider, - source_project_id: data.source_project_id, - source_output_file_path: data.source_output_file_path, - build_id: data.build_id, - } - }, - - _canCreate: ProjectFileAgent._canCreate, - - _getSourceProject: LinkedFilesHandler.getSourceProject, - - _validate(data) { - if (data.v1_source_doc_id != null) { - return ( - data.v1_source_doc_id != null && data.source_output_file_path != null - ) - } else { - return ( - data.source_project_id != null && - data.source_output_file_path != null && - data.build_id != null - ) - } - }, - - _checkAuth(project_id, data, current_user_id, callback) { - if (callback == null) { - callback = function (err, allowed) {} - } - callback = _.once(callback) - if (!this._validate(data)) { + if (!_validate(linkedFileData)) { return callback(new BadDataError()) } - return this._getSourceProject(data, function (err, project) { - if (err != null) { - return callback(err) - } - return AuthorizationManager.canUserReadProject( - current_user_id, - project._id, - null, - function (err, canRead) { - if (err != null) { - return callback(err) - } - return callback(null, canRead) - } - ) - }) - }, + callback(null, linkedFileData) + }) +} - _getFileStream(linkedFileData, user_id, callback) { - if (callback == null) { - callback = function (err, fileStream) {} +function createLinkedFile( + projectId, + linkedFileData, + name, + parentFolderId, + userId, + callback +) { + if (!ProjectFileAgent._canCreate(linkedFileData)) { + return callback(new AccessDeniedError()) + } + linkedFileData = _sanitizeData(linkedFileData) + _prepare(projectId, linkedFileData, userId, (err, linkedFileData) => { + if (err) { + return callback(err) } - callback = _.once(callback) - const { source_output_file_path, build_id } = linkedFileData - return this._getSourceProject(linkedFileData, function (err, project) { - if (err != null) { + _getFileStream(linkedFileData, userId, (err, readStream) => { + if (err) { return callback(err) } - const source_project_id = project._id - return ClsiManager.getOutputFileStream( - source_project_id, - user_id, - build_id, - source_output_file_path, - function (err, readStream) { - if (err != null) { - return callback(err) - } - readStream.pause() - return callback(null, readStream) - } - ) - }) - }, - - _compileAndGetFileStream(linkedFileData, user_id, callback) { - if (callback == null) { - callback = function (err, stream, build_id) {} - } - callback = _.once(callback) - const { source_output_file_path } = linkedFileData - return this._getSourceProject(linkedFileData, function (err, project) { - if (err != null) { - return callback(err) - } - const source_project_id = project._id - return CompileManager.compile( - source_project_id, - user_id, - {}, - function (err, status, outputFiles) { - if (err != null) { - return callback(err) - } - if (status !== 'success') { - return callback(new CompileFailedError()) - } - const outputFile = _.find( - outputFiles, - o => o.path === source_output_file_path - ) - if (outputFile == null) { - return callback(new OutputFileFetchFailedError()) - } - const build_id = outputFile.build - return ClsiManager.getOutputFileStream( - source_project_id, - user_id, - build_id, - source_output_file_path, - function (err, readStream) { - if (err != null) { + readStream.on('error', callback) + readStream.on('response', response => { + if (response.statusCode >= 200 && response.statusCode < 300) { + LinkedFilesHandler.importFromStream( + projectId, + readStream, + linkedFileData, + name, + parentFolderId, + userId, + (err, file) => { + if (err) { return callback(err) } - readStream.pause() - return callback(null, readStream, build_id) + callback(null, file._id) } + ) // Created + } else { + err = new OutputFileFetchFailedError( + `Output file fetch failed: ${linkedFileData.build_id}, ${linkedFileData.source_output_file_path}` ) + err.statusCode = response.statusCode + callback(err) } - ) + }) }) - }, + }) } + +function refreshLinkedFile( + projectId, + linkedFileData, + name, + parentFolderId, + userId, + callback +) { + _prepare(projectId, linkedFileData, userId, (err, linkedFileData) => { + if (err) { + return callback(err) + } + _compileAndGetFileStream( + linkedFileData, + userId, + (err, readStream, newBuildId) => { + if (err) { + return callback(err) + } + readStream.on('error', callback) + readStream.on('response', response => { + if (response.statusCode >= 200 && response.statusCode < 300) { + linkedFileData.build_id = newBuildId + LinkedFilesHandler.importFromStream( + projectId, + readStream, + linkedFileData, + name, + parentFolderId, + userId, + (err, file) => { + if (err) { + return callback(err) + } + callback(null, file._id) + } + ) // Created + } else { + err = new OutputFileFetchFailedError( + `Output file fetch failed: ${linkedFileData.build_id}, ${linkedFileData.source_output_file_path}` + ) + err.statusCode = response.statusCode + callback(err) + } + }) + } + ) + }) +} + +function _sanitizeData(data) { + return { + provider: data.provider, + source_project_id: data.source_project_id, + source_output_file_path: data.source_output_file_path, + build_id: data.build_id, + } +} + +function _validate(data) { + return ( + (data.v1_source_doc_id != null && data.source_output_file_path != null) || + (data.source_project_id != null && + data.source_output_file_path != null && + data.build_id != null) + ) +} + +function _checkAuth(projectId, data, currentUserId, callback) { + callback = _.once(callback) + if (!_validate(data)) { + return callback(new BadDataError()) + } + LinkedFilesHandler.getSourceProject(data, (err, project) => { + if (err) { + return callback(err) + } + AuthorizationManager.canUserReadProject( + currentUserId, + project._id, + null, + (err, canRead) => { + if (err) { + return callback(err) + } + callback(null, canRead) + } + ) + }) +} + +function _getFileStream(linkedFileData, userId, callback) { + callback = _.once(callback) + const { + source_output_file_path: sourceOutputFilePath, + build_id: buildId, + } = linkedFileData + LinkedFilesHandler.getSourceProject(linkedFileData, (err, project) => { + if (err) { + return callback(err) + } + const sourceProjectId = project._id + ClsiManager.getOutputFileStream( + sourceProjectId, + userId, + buildId, + sourceOutputFilePath, + (err, readStream) => { + if (err) { + return callback(err) + } + readStream.pause() + callback(null, readStream) + } + ) + }) +} + +function _compileAndGetFileStream(linkedFileData, userId, callback) { + callback = _.once(callback) + const { source_output_file_path: sourceOutputFilePath } = linkedFileData + LinkedFilesHandler.getSourceProject(linkedFileData, (err, project) => { + if (err) { + return callback(err) + } + const sourceProjectId = project._id + CompileManager.compile( + sourceProjectId, + userId, + {}, + (err, status, outputFiles) => { + if (err) { + return callback(err) + } + if (status !== 'success') { + return callback(new CompileFailedError()) + } + const outputFile = _.find( + outputFiles, + o => o.path === sourceOutputFilePath + ) + if (outputFile == null) { + return callback(new OutputFileFetchFailedError()) + } + const buildId = outputFile.build + ClsiManager.getOutputFileStream( + sourceProjectId, + userId, + buildId, + sourceOutputFilePath, + (err, readStream) => { + if (err) { + return callback(err) + } + readStream.pause() + callback(null, readStream, buildId) + } + ) + } + ) + }) +} + +module.exports = { createLinkedFile, refreshLinkedFile } diff --git a/services/web/app/src/Features/LinkedFiles/UrlAgent.js b/services/web/app/src/Features/LinkedFiles/UrlAgent.js index 2e25447723..1b358b1550 100644 --- a/services/web/app/src/Features/LinkedFiles/UrlAgent.js +++ b/services/web/app/src/Features/LinkedFiles/UrlAgent.js @@ -1,18 +1,3 @@ -/* eslint-disable - camelcase, - node/handle-callback-err, - max-len, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -let UrlAgent const request = require('request') const _ = require('underscore') const urlValidator = require('valid-url') @@ -20,91 +5,82 @@ const { InvalidUrlError, UrlFetchFailedError } = require('./LinkedFilesErrors') const LinkedFilesHandler = require('./LinkedFilesHandler') const UrlHelper = require('../Helpers/UrlHelper') -module.exports = UrlAgent = { - createLinkedFile( - project_id, - linkedFileData, - name, - parent_folder_id, - user_id, - callback - ) { - linkedFileData = this._sanitizeData(linkedFileData) - return this._getUrlStream( - project_id, - linkedFileData, - user_id, - function (err, readStream) { - if (err != null) { - return callback(err) - } - readStream.on('error', callback) - return readStream.on('response', function (response) { - if (response.statusCode >= 200 && response.statusCode < 300) { - readStream.resume() - return LinkedFilesHandler.importFromStream( - project_id, - readStream, - linkedFileData, - name, - parent_folder_id, - user_id, - function (err, file) { - if (err != null) { - return callback(err) - } - return callback(null, file._id) - } - ) // Created - } else { - const error = new UrlFetchFailedError( - `url fetch failed: ${linkedFileData.url}` - ) - error.statusCode = response.statusCode - return callback(error) +function createLinkedFile( + projectId, + linkedFileData, + name, + parentFolderId, + userId, + callback +) { + linkedFileData = _sanitizeData(linkedFileData) + _getUrlStream(projectId, linkedFileData, userId, (err, readStream) => { + if (err) { + return callback(err) + } + readStream.on('error', callback) + readStream.on('response', response => { + if (response.statusCode >= 200 && response.statusCode < 300) { + LinkedFilesHandler.importFromStream( + projectId, + readStream, + linkedFileData, + name, + parentFolderId, + userId, + (err, file) => { + if (err) { + return callback(err) + } + callback(null, file._id) } - }) + ) // Created + } else { + const error = new UrlFetchFailedError( + `url fetch failed: ${linkedFileData.url}` + ) + error.statusCode = response.statusCode + callback(error) } - ) - }, + }) + }) +} - refreshLinkedFile( - project_id, +function refreshLinkedFile( + projectId, + linkedFileData, + name, + parentFolderId, + userId, + callback +) { + createLinkedFile( + projectId, linkedFileData, name, - parent_folder_id, - user_id, + parentFolderId, + userId, callback - ) { - return this.createLinkedFile( - project_id, - linkedFileData, - name, - parent_folder_id, - user_id, - callback - ) - }, - - _sanitizeData(data) { - return { - provider: data.provider, - url: UrlHelper.prependHttpIfNeeded(data.url), - } - }, - - _getUrlStream(project_id, data, current_user_id, callback) { - if (callback == null) { - callback = function (error, fsPath) {} - } - callback = _.once(callback) - let { url } = data - if (!urlValidator.isWebUri(url)) { - return callback(new InvalidUrlError(`invalid url: ${url}`)) - } - url = UrlHelper.wrapUrlWithProxy(url) - const readStream = request.get(url) - readStream.pause() - return callback(null, readStream) - }, + ) } + +function _sanitizeData(data) { + return { + provider: data.provider, + url: UrlHelper.prependHttpIfNeeded(data.url), + } +} + +function _getUrlStream(projectId, data, currentUserId, callback) { + callback = _.once(callback) + let { url } = data + if (!urlValidator.isWebUri(url)) { + return callback(new InvalidUrlError(`invalid url: ${url}`)) + } + url = UrlHelper.wrapUrlWithProxy(url) + const readStream = request.get(url) + readStream.pause() + callback(null, readStream) +} + +module.exports = { createLinkedFile, refreshLinkedFile }