From 9365e08b55876db39762ee72cbd9ec8f67278d05 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 15 May 2024 14:46:18 +0100 Subject: [PATCH] Merge pull request #16775 from overleaf/bg-clsi-timeout-use-fetchutils-connect-timeout Update UrlFetcher to use custom agent with connect timeout GitOrigin-RevId: c92dc27a0e682b0b79859b3521bf64cbf8485345 --- services/clsi/app/js/UrlFetcher.js | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/services/clsi/app/js/UrlFetcher.js b/services/clsi/app/js/UrlFetcher.js index efae8b6777..8d604d267b 100644 --- a/services/clsi/app/js/UrlFetcher.js +++ b/services/clsi/app/js/UrlFetcher.js @@ -1,11 +1,19 @@ const fs = require('fs') const logger = require('@overleaf/logger') const Settings = require('@overleaf/settings') -const { fetchStream } = require('@overleaf/fetch-utils') +const { + CustomHttpAgent, + CustomHttpsAgent, + fetchStream, +} = require('@overleaf/fetch-utils') const { URL } = require('url') const { pipeline } = require('stream/promises') const Metrics = require('./Metrics') +const MAX_CONNECT_TIME = 1000 +const httpAgent = new CustomHttpAgent({ connectTimeout: MAX_CONNECT_TIME }) +const httpsAgent = new CustomHttpsAgent({ connectTimeout: MAX_CONNECT_TIME }) + async function pipeUrlToFileWithRetry(url, filePath) { let remainingAttempts = 3 let lastErr @@ -40,6 +48,10 @@ async function pipeUrlToFile(url, filePath) { const stream = await fetchStream(url, { signal: AbortSignal.timeout(60 * 1000), + // provide a function to get the agent for each request + // as there may be multiple requests with different protocols + // due to redirects. + agent: _url => (_url.protocol === 'https:' ? httpsAgent : httpAgent), }) const atomicWrite = filePath + '~'