diff --git a/package-lock.json b/package-lock.json index 18a71973ea..0d79170a3d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -39708,6 +39708,7 @@ "services/clsi": { "name": "@overleaf/clsi", "dependencies": { + "@overleaf/fetch-utils": "*", "@overleaf/logger": "*", "@overleaf/metrics": "*", "@overleaf/o-error": "*", @@ -39721,7 +39722,6 @@ "fs-extra": "^10.0.0", "lockfile": "^1.0.4", "lodash": "^4.17.21", - "node-fetch": "^2.6.7", "p-limit": "^3.1.0", "request": "^2.88.2", "send": "^0.17.1", @@ -39732,6 +39732,7 @@ "chai-as-promised": "^7.1.1", "mocha": "^10.2.0", "mock-fs": "^5.1.2", + "node-fetch": "^2.6.7", "sandboxed-module": "^2.0.4", "sinon": "~9.0.1", "sinon-chai": "^3.7.0", @@ -40984,6 +40985,7 @@ "name": "@overleaf/third-party-datastore", "dependencies": { "@overleaf/access-token-encryptor": "*", + "@overleaf/fetch-utils": "*", "@overleaf/logger": "*", "@overleaf/metrics": "*", "@overleaf/o-error": "*", @@ -48997,6 +48999,7 @@ "@overleaf/clsi": { "version": "file:services/clsi", "requires": { + "@overleaf/fetch-utils": "*", "@overleaf/logger": "*", "@overleaf/metrics": "*", "@overleaf/o-error": "*", @@ -50163,6 +50166,7 @@ "version": "file:services/third-party-datastore", "requires": { "@overleaf/access-token-encryptor": "*", + "@overleaf/fetch-utils": "*", "@overleaf/logger": "*", "@overleaf/metrics": "*", "@overleaf/o-error": "*", diff --git a/services/clsi/app/js/UrlFetcher.js b/services/clsi/app/js/UrlFetcher.js index 9d1d6d2103..a580da44fe 100644 --- a/services/clsi/app/js/UrlFetcher.js +++ b/services/clsi/app/js/UrlFetcher.js @@ -1,10 +1,9 @@ const fs = require('fs') const logger = require('@overleaf/logger') const Settings = require('@overleaf/settings') +const { fetchStream } = require('@overleaf/fetch-utils') const { URL } = require('url') -const { promisify } = require('util') -const fetch = require('node-fetch') -const pipeline = promisify(require('stream').pipeline) +const { pipeline } = require('stream/promises') async function pipeUrlToFileWithRetry(url, filePath) { let remainingAttempts = 3 @@ -33,14 +32,13 @@ async function pipeUrlToFile(url, filePath) { url = `${Settings.filestoreDomainOveride}${u.pathname}${u.search}` } - const res = await fetch(url, { signal: AbortSignal.timeout(60 * 1000) }) - if (res.status !== 200) { - throw new Error('non success response: ' + res.statusText) - } + const stream = await fetchStream(url, { + signal: AbortSignal.timeout(60 * 1000), + }) const atomicWrite = filePath + '~' try { - await pipeline(res.body, fs.createWriteStream(atomicWrite)) + await pipeline(stream, fs.createWriteStream(atomicWrite)) await fs.promises.rename(atomicWrite, filePath) } catch (err) { try { diff --git a/services/clsi/package.json b/services/clsi/package.json index 5b7492c022..5b0239b337 100644 --- a/services/clsi/package.json +++ b/services/clsi/package.json @@ -16,6 +16,7 @@ "lint:fix": "eslint --fix ." }, "dependencies": { + "@overleaf/fetch-utils": "*", "@overleaf/logger": "*", "@overleaf/metrics": "*", "@overleaf/o-error": "*", @@ -29,7 +30,6 @@ "fs-extra": "^10.0.0", "lockfile": "^1.0.4", "lodash": "^4.17.21", - "node-fetch": "^2.6.7", "p-limit": "^3.1.0", "request": "^2.88.2", "send": "^0.17.1", @@ -40,6 +40,7 @@ "chai-as-promised": "^7.1.1", "mocha": "^10.2.0", "mock-fs": "^5.1.2", + "node-fetch": "^2.6.7", "sandboxed-module": "^2.0.4", "sinon": "~9.0.1", "sinon-chai": "^3.7.0",