Merge pull request #21677 from overleaf/msm-register-hide-sso-login

[web] Hide 'Login with SSO' when accepting SSO group invite

GitOrigin-RevId: fd4c327bbb838209773b4e7b1c41f6e3a355ad8b

CONTRIBUTING.md

@@ -14,6 +14,8 @@ When submitting an issue please describe the issue as clearly as possible, inclu
 reproduce the bug, which situations it appears in, what you expected to happen, and what actually happens.
 If you can include a screenshot for front end issues that is very helpful.
 
+**Note**: If you are using [www.overleaf.com](www.overleaf.com) and have a problem, or if you would like to request a new feature, please contact the Support team at support@overleaf.com. Raise an issue here only to report bugs in the Community Edition release of Overleaf.
+
 Pull Requests
 -------------
 
@@ -34,7 +36,7 @@ Please see [our security policy](https://github.com/overleaf/overleaf/security/p
 Contributor License Agreement
 -----------------------------
 
-Before we can accept any contributions of code, we need you to agree to our 
+Before we can accept any contributions of code, we need you to agree to our
 [Contributor License Agreement](https://docs.google.com/forms/d/e/1FAIpQLSef79XH3mb7yIiMzZw-yALEegS-wyFetvjTiNBfZvf_IHD2KA/viewform?usp=sf_link).
 This is to ensure that you own the copyright of your contribution, and that you
 agree to give us a license to use it in both the open source version, and the version

README.md

@@ -45,8 +45,8 @@ This repo contains two dockerfiles, [`Dockerfile-base`](server-ce/Dockerfile-bas
 `sharelatex/sharelatex-base` image, and [`Dockerfile`](server-ce/Dockerfile) which builds the
 `sharelatex/sharelatex` (or "community") image.
 
-The Base image generally contains the basic dependencies like `wget` and
-`aspell`, plus `texlive`. We split this out because it's a pretty heavy set of
+The Base image generally contains the basic dependencies like `wget`, plus `texlive`.
+We split this out because it's a pretty heavy set of
 dependencies, and it's nice to not have to rebuild all of that every time.
 
 The `sharelatex/sharelatex` image extends the base image and adds the actual Overleaf code

develop/dev.env

@@ -13,7 +13,7 @@ NOTIFICATIONS_HOST=notifications
 PROJECT_HISTORY_HOST=project-history
 REALTIME_HOST=real-time
 REDIS_HOST=redis
-SPELLING_HOST=spelling
+SESSION_SECRET=foo
 WEBPACK_HOST=webpack
 WEB_API_PASSWORD=overleaf
 WEB_API_USER=overleaf

develop/docker-compose.yml

@@ -8,7 +8,6 @@ volumes:
   mongo-data:
   redis-data:
   sharelatex-data:
-  spelling-cache:
   web-data:
   history-v1-buckets:
 
@@ -124,15 +123,6 @@ services:
     volumes:
       - redis-data:/data
 
-  spelling:
-    build:
-      context: ..
-      dockerfile: services/spelling/Dockerfile
-    env_file:
-      - dev.env
-    volumes:
-      - spelling-cache:/overleaf/services/spelling/cache
-
   web:
     build:
       context: ..
@@ -146,7 +136,7 @@ services:
       - EMAIL_CONFIRMATION_DISABLED=true
       - NODE_ENV=development
       - OVERLEAF_ALLOW_PUBLIC_ACCESS=true
-    command: ["node", "app.js"]
+    command: ["node", "app.mjs"]
     volumes:
       - sharelatex-data:/var/lib/overleaf
       - web-data:/overleaf/services/web/data
@@ -163,13 +153,12 @@ services:
       - notifications
       - project-history
       - real-time
-      - spelling
 
   webpack:
     build:
       context: ..
       dockerfile: services/web/Dockerfile
-      target: dev
+      target: webpack
     command: ["npx", "webpack", "serve", "--config", "webpack.config.dev-env.js"]
     ports:
       - "127.0.0.1:80:3808"

libraries/access-token-encryptor/.nvmrc

@@ -1 +1 @@
-18.20.2
+20.18.0

libraries/access-token-encryptor/buildscript.txt

@@ -5,6 +5,6 @@ access-token-encryptor
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=18.20.2
+--node-version=20.18.0
 --public-repo=False
 --script-version=4.5.0

libraries/access-token-encryptor/lib/js/AccessTokenEncryptor.js

@@ -1,5 +1,5 @@
-const { promisify } = require('util')
-const crypto = require('crypto')
+const { promisify } = require('node:util')
+const crypto = require('node:crypto')
 
 const ALGORITHM = 'aes-256-ctr'
 

libraries/access-token-encryptor/test/setup.js

@@ -1,4 +1,13 @@
 const chai = require('chai')
 const chaiAsPromised = require('chai-as-promised')
+const SandboxedModule = require('sandboxed-module')
 
 chai.use(chaiAsPromised)
+
+SandboxedModule.configure({
+  sourceTransformers: {
+    removeNodePrefix: function (source) {
+      return source.replace(/require\(['"]node:/g, "require('")
+    },
+  },
+})

libraries/fetch-utils/.nvmrc

@@ -1 +1 @@
-18.20.2
+20.18.0

libraries/fetch-utils/buildscript.txt

@@ -5,6 +5,6 @@ fetch-utils
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=18.20.2
+--node-version=20.18.0
 --public-repo=False
 --script-version=4.5.0

libraries/fetch-utils/index.js

@@ -1,9 +1,9 @@
 const _ = require('lodash')
-const { Readable } = require('stream')
+const { Readable } = require('node:stream')
 const OError = require('@overleaf/o-error')
 const fetch = require('node-fetch')
-const http = require('http')
-const https = require('https')
+const http = require('node:http')
+const https = require('node:https')
 
 /**
  * @import { Response } from 'node-fetch'

libraries/fetch-utils/test/unit/FetchUtilsTests.js

@@ -1,7 +1,7 @@
 const { expect } = require('chai')
 const { FetchError, AbortError } = require('node-fetch')
-const { Readable } = require('stream')
-const { once } = require('events')
+const { Readable } = require('node:stream')
+const { once } = require('node:events')
 const { TestServer } = require('./helpers/TestServer')
 const selfsigned = require('selfsigned')
 const {
@@ -24,13 +24,17 @@ const pems = selfsigned.generate(attrs, { days: 365 })
 const PRIVATE_KEY = pems.private
 const PUBLIC_CERT = pems.cert
 
-const dns = require('dns')
+const dns = require('node:dns')
 const _originalLookup = dns.lookup
 // Custom DNS resolver function
 dns.lookup = (hostname, options, callback) => {
   if (hostname === 'example.com') {
     // If the hostname is our test case, return the ip address for the test server
-    callback(null, '127.0.0.1', 4)
+    if (options?.all) {
+      callback(null, [{ address: '127.0.0.1', family: 4 }])
+    } else {
+      callback(null, '127.0.0.1', 4)
+    }
   } else {
     // Otherwise, use the default lookup
     _originalLookup(hostname, options, callback)

libraries/fetch-utils/test/unit/helpers/TestServer.js

@@ -1,9 +1,9 @@
 const express = require('express')
 const bodyParser = require('body-parser')
-const { EventEmitter } = require('events')
-const http = require('http')
-const https = require('https')
-const { promisify } = require('util')
+const { EventEmitter } = require('node:events')
+const http = require('node:http')
+const https = require('node:https')
+const { promisify } = require('node:util')
 
 class TestServer {
   constructor() {

libraries/logger/.nvmrc

@@ -1 +1 @@
-18.20.2
+20.18.0

libraries/logger/buildscript.txt

@@ -5,6 +5,6 @@ logger
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=18.20.2
+--node-version=20.18.0
 --public-repo=False
 --script-version=4.5.0

libraries/logger/log-level-checker.js

@@ -1,5 +1,5 @@
 const { fetchString } = require('@overleaf/fetch-utils')
-const fs = require('fs')
+const fs = require('node:fs')
 
 class LogLevelChecker {
   constructor(logger, defaultLevel) {

libraries/logger/logging-manager.js

@@ -1,7 +1,6 @@
-const Stream = require('stream')
+const Stream = require('node:stream')
 const bunyan = require('bunyan')
 const GCPManager = require('./gcp-manager')
-const SentryManager = require('./sentry-manager')
 const Serializers = require('./serializers')
 const {
   FileLogLevelChecker,
@@ -33,10 +32,6 @@ const LoggingManager = {
     return this
   },
 
-  initializeErrorReporting(dsn, options) {
-    this.sentryManager = new SentryManager()
-  },
-
   /**
    * @param {Record<string, any>|string} attributes - Attributes to log (nice serialization for err, req, res)
    * @param {string} [message] - Optional message
@@ -68,9 +63,6 @@ const LoggingManager = {
       })
     }
     this.logger.error(attributes, message, ...Array.from(args))
-    if (this.sentryManager) {
-      this.sentryManager.captureExceptionRateLimited(attributes, message)
-    }
   },
 
   /**
@@ -98,9 +90,6 @@ const LoggingManager = {
    */
   fatal(attributes, message) {
     this.logger.fatal(attributes, message)
-    if (this.sentryManager) {
-      this.sentryManager.captureException(attributes, message, 'fatal')
-    }
   },
 
   _getOutputStreamConfig() {

libraries/logger/package.json

@@ -23,7 +23,6 @@
     "@google-cloud/logging-bunyan": "^5.1.0",
     "@overleaf/fetch-utils": "*",
     "@overleaf/o-error": "*",
-    "@sentry/node": "^6.13.2",
     "bunyan": "^1.8.14"
   },
   "devDependencies": {

libraries/logger/sentry-manager.js

@@ -1,106 +0,0 @@
-const Serializers = require('./serializers')
-const RATE_LIMIT_MAX_ERRORS = 5
-const RATE_LIMIT_INTERVAL_MS = 60000
-
-class SentryManager {
-  constructor(dsn, options) {
-    this.Sentry = require('@sentry/node')
-    this.Sentry.init({ dsn, ...options })
-    // for rate limiting on sentry reporting
-    this.lastErrorTimeStamp = 0
-    this.lastErrorCount = 0
-  }
-
-  captureExceptionRateLimited(attributes, message) {
-    const now = Date.now()
-    // have we recently reported an error?
-    const recentSentryReport =
-      now - this.lastErrorTimeStamp < RATE_LIMIT_INTERVAL_MS
-    // if so, increment the error count
-    if (recentSentryReport) {
-      this.lastErrorCount++
-    } else {
-      this.lastErrorCount = 0
-      this.lastErrorTimeStamp = now
-    }
-    // only report 5 errors every minute to avoid overload
-    if (this.lastErrorCount < RATE_LIMIT_MAX_ERRORS) {
-      // add a note if the rate limit has been hit
-      const note =
-        this.lastErrorCount + 1 === RATE_LIMIT_MAX_ERRORS
-          ? '(rate limited)'
-          : ''
-      // report the exception
-      this.captureException(attributes, message, `error${note}`)
-    }
-  }
-
-  captureException(attributes, message, level) {
-    // handle case of logger.error "message"
-    if (typeof attributes === 'string') {
-      attributes = { err: new Error(attributes) }
-    }
-
-    // extract any error object
-    let error = Serializers.err(attributes.err || attributes.error)
-
-    // avoid reporting errors twice
-    for (const key in attributes) {
-      const value = attributes[key]
-      if (value instanceof Error && value.reportedToSentry) {
-        return
-      }
-    }
-
-    // include our log message in the error report
-    if (error == null) {
-      if (typeof message === 'string') {
-        error = { message }
-      }
-    } else if (message != null) {
-      attributes.description = message
-    }
-
-    // report the error
-    if (error != null) {
-      // capture attributes and use *_id objects as tags
-      const tags = {}
-      const extra = {}
-      for (const key in attributes) {
-        let value = attributes[key]
-        if (Serializers[key]) {
-          value = Serializers[key](value)
-        }
-        if (key.match(/_id/) && typeof value === 'string') {
-          tags[key] = value
-        }
-        extra[key] = value
-      }
-
-      // OError integration
-      extra.info = error.info
-      delete error.info
-
-      // Sentry wants to receive an Error instance.
-      const errInstance = new Error(error.message)
-      Object.assign(errInstance, error)
-
-      try {
-        // send the error to sentry
-        this.Sentry.captureException(errInstance, { tags, extra, level })
-
-        // put a flag on the errors to avoid reporting them multiple times
-        for (const key in attributes) {
-          const value = attributes[key]
-          if (value instanceof Error) {
-            value.reportedToSentry = true
-          }
-        }
-      } catch (err) {
-        // ignore Sentry errors
-      }
-    }
-  }
-}
-
-module.exports = SentryManager

libraries/logger/test/setup.js

@@ -8,4 +8,9 @@ chai.use(sinonChai)
 
 SandboxedModule.configure({
   globals: { Buffer, JSON, console, process },
+  sourceTransformers: {
+    removeNodePrefix: function (source) {
+      return source.replace(/require\(['"]node:/g, "require('")
+    },
+  },
 })

libraries/logger/test/unit/log-level-checker-tests.js

@@ -1,5 +1,5 @@
-const Path = require('path')
-const { promisify } = require('util')
+const Path = require('node:path')
+const { promisify } = require('node:util')
 const SandboxedModule = require('sandboxed-module')
 const sinon = require('sinon')
 const { expect } = require('chai')

libraries/logger/test/unit/logging-manager-tests.js

@@ -1,7 +1,7 @@
 const SandboxedModule = require('sandboxed-module')
 const bunyan = require('bunyan')
 const { expect } = require('chai')
-const path = require('path')
+const path = require('node:path')
 const sinon = require('sinon')
 
 const MODULE_PATH = path.join(__dirname, '../../logging-manager.js')
@@ -43,20 +43,14 @@ describe('LoggingManager', function () {
         .stub()
         .returns(this.GCEMetadataLogLevelChecker),
     }
-    this.SentryManager = {
-      captureException: sinon.stub(),
-      captureExceptionRateLimited: sinon.stub(),
-    }
     this.LoggingManager = SandboxedModule.require(MODULE_PATH, {
       requires: {
         bunyan: this.Bunyan,
         './log-level-checker': this.LogLevelChecker,
-        './sentry-manager': sinon.stub().returns(this.SentryManager),
       },
     })
     this.loggerName = 'test'
     this.logger = this.LoggingManager.initialize(this.loggerName)
-    this.logger.initializeErrorReporting('test_dsn')
   })
 
   afterEach(function () {
@@ -160,13 +154,6 @@ describe('LoggingManager', function () {
     })
   })
 
-  describe('logger.error', function () {
-    it('should report errors to Sentry', function () {
-      this.logger.error({ foo: 'bar' }, 'message')
-      expect(this.SentryManager.captureExceptionRateLimited).to.have.been.called
-    })
-  })
-
   describe('ringbuffer', function () {
     beforeEach(function () {
       this.logBufferMock = [

libraries/logger/test/unit/sentry-manager-tests.js

@@ -1,247 +0,0 @@
-const Path = require('path')
-const SandboxedModule = require('sandboxed-module')
-const { expect } = require('chai')
-const sinon = require('sinon')
-
-const MODULE_PATH = Path.join(__dirname, '../../sentry-manager.js')
-
-describe('SentryManager', function () {
-  beforeEach(function () {
-    this.clock = sinon.useFakeTimers(Date.now())
-    this.Sentry = {
-      init: sinon.stub(),
-      captureException: sinon.stub(),
-    }
-    this.SentryManager = SandboxedModule.require(MODULE_PATH, {
-      requires: {
-        '@sentry/node': this.Sentry,
-      },
-    })
-    this.sentryManager = new this.SentryManager('test_dsn')
-  })
-
-  afterEach(function () {
-    this.clock.restore()
-  })
-
-  describe('captureExceptionRateLimited', function () {
-    it('should report a single error to sentry', function () {
-      this.sentryManager.captureExceptionRateLimited({ foo: 'bar' }, 'message')
-      expect(this.Sentry.captureException).to.have.been.calledOnce
-    })
-
-    it('should report the same error to sentry only once', function () {
-      const error1 = new Error('this is the error')
-      this.sentryManager.captureExceptionRateLimited(
-        { foo: error1 },
-        'first message'
-      )
-      this.sentryManager.captureExceptionRateLimited(
-        { bar: error1 },
-        'second message'
-      )
-      expect(this.Sentry.captureException).to.have.been.calledOnce
-    })
-
-    it('should report two different errors to sentry individually', function () {
-      const error1 = new Error('this is the error')
-      const error2 = new Error('this is the error')
-      this.sentryManager.captureExceptionRateLimited(
-        { foo: error1 },
-        'first message'
-      )
-      this.sentryManager.captureExceptionRateLimited(
-        { bar: error2 },
-        'second message'
-      )
-      expect(this.Sentry.captureException).to.have.been.calledTwice
-    })
-
-    it('for multiple errors should only report a maximum of 5 errors to sentry', function () {
-      for (let i = 0; i < 10; i++) {
-        this.sentryManager.captureExceptionRateLimited(
-          { foo: 'bar' },
-          'message'
-        )
-      }
-      expect(this.Sentry.captureException).to.have.callCount(5)
-    })
-
-    it('for multiple errors with a minute delay should report 10 errors to sentry', function () {
-      for (let i = 0; i < 10; i++) {
-        this.sentryManager.captureExceptionRateLimited(
-          { foo: 'bar' },
-          'message'
-        )
-      }
-      expect(this.Sentry.captureException).to.have.callCount(5)
-
-      // allow a minute to pass
-      this.clock.tick(61 * 1000)
-
-      for (let i = 0; i < 10; i++) {
-        this.sentryManager.captureExceptionRateLimited(
-          { foo: 'bar' },
-          'message'
-        )
-      }
-      expect(this.Sentry.captureException).to.have.callCount(10)
-    })
-  })
-
-  describe('captureException', function () {
-    it('should remove the path from fs errors', function () {
-      const fsError = new Error(
-        "Error: ENOENT: no such file or directory, stat '/tmp/3279b8d0-da10-11e8-8255-efd98985942b'"
-      )
-      fsError.path = '/tmp/3279b8d0-da10-11e8-8255-efd98985942b'
-      this.sentryManager.captureException({ err: fsError }, 'message', 'error')
-      expect(this.Sentry.captureException).to.have.been.calledWith(
-        sinon.match.has(
-          'message',
-          'Error: ENOENT: no such file or directory, stat'
-        )
-      )
-    })
-
-    it('should sanitize error', function () {
-      const err = {
-        name: 'CustomError',
-        message: 'hello',
-        _oErrorTags: [{ stack: 'here:1', info: { one: 1 } }],
-        stack: 'here:0',
-        info: { key: 'value' },
-        code: 42,
-        signal: 9,
-        path: '/foo',
-      }
-      this.sentryManager.captureException({ err }, 'message', 'error')
-      const expectedErr = {
-        name: 'CustomError',
-        message: 'hello',
-        stack: 'here:0\nhere:1',
-        code: 42,
-        signal: 9,
-        path: '/foo',
-      }
-      expect(this.Sentry.captureException).to.have.been.calledWith(
-        sinon.match(expectedErr),
-        sinon.match({
-          tags: sinon.match.any,
-          level: sinon.match.any,
-          extra: {
-            description: 'message',
-            info: sinon.match({
-              one: 1,
-              key: 'value',
-            }),
-          },
-        })
-      )
-      // Chai is very picky with comparing Error instances. Go the long way of comparing all the fields manually.
-      const gotErr = this.Sentry.captureException.args[0][0]
-      for (const [key, wanted] of Object.entries(expectedErr)) {
-        expect(gotErr).to.have.property(key, wanted)
-      }
-    })
-    it('should sanitize request', function () {
-      const req = {
-        ip: '1.2.3.4',
-        method: 'GET',
-        url: '/foo',
-        headers: {
-          referer: 'abc',
-          'content-length': 1337,
-          'user-agent': 'curl',
-          authorization: '42',
-        },
-      }
-      this.sentryManager.captureException({ req }, 'message', 'error')
-      const expectedReq = {
-        remoteAddress: '1.2.3.4',
-        method: 'GET',
-        url: '/foo',
-        headers: {
-          referer: 'abc',
-          'content-length': 1337,
-          'user-agent': 'curl',
-        },
-      }
-      expect(this.Sentry.captureException).to.have.been.calledWith(
-        sinon.match({
-          message: 'message',
-        }),
-        sinon.match({
-          tags: sinon.match.any,
-          level: sinon.match.any,
-          extra: {
-            info: sinon.match.any,
-            req: expectedReq,
-          },
-        })
-      )
-      expect(this.Sentry.captureException.args[0][1].extra.req).to.deep.equal(
-        expectedReq
-      )
-    })
-    it('should sanitize response', function () {
-      const res = {
-        statusCode: 417,
-        body: Buffer.from('foo'),
-        getHeader(key) {
-          expect(key).to.be.oneOf(['content-length'])
-          if (key === 'content-length') return 1337
-        },
-      }
-      this.sentryManager.captureException({ res }, 'message', 'error')
-      const expectedRes = {
-        statusCode: 417,
-        headers: {
-          'content-length': 1337,
-        },
-      }
-      expect(this.Sentry.captureException).to.have.been.calledWith(
-        sinon.match({
-          message: 'message',
-        }),
-        sinon.match({
-          tags: sinon.match.any,
-          level: sinon.match.any,
-          extra: {
-            info: sinon.match.any,
-            res: expectedRes,
-          },
-        })
-      )
-      expect(this.Sentry.captureException.args[0][1].extra.res).to.deep.equal(
-        expectedRes
-      )
-    })
-
-    describe('reportedToSentry', function () {
-      it('should mark the error as reported to sentry', function () {
-        const err = new Error()
-        this.sentryManager.captureException({ err }, 'message')
-        expect(this.Sentry.captureException).to.have.been.called
-        expect(err.reportedToSentry).to.equal(true)
-      })
-
-      it('should mark two errors as reported to sentry', function () {
-        const err1 = new Error()
-        const err2 = new Error()
-        this.sentryManager.captureException({ err: err1, err2 }, 'message')
-        expect(this.Sentry.captureException).to.have.been.called
-        expect(err1.reportedToSentry).to.equal(true)
-        expect(err2.reportedToSentry).to.equal(true)
-      })
-
-      it('should not mark arbitrary objects as reported to sentry', function () {
-        const err = new Error()
-        const ctx = { foo: 'bar' }
-        this.sentryManager.captureException({ err, ctx }, 'message')
-        expect(this.Sentry.captureException).to.have.been.called
-        expect(ctx.reportedToSentry).not.to.exist
-      })
-    })
-  })
-})

libraries/metrics/.nvmrc

@@ -1 +1 @@
-18.20.2
+20.18.0

libraries/metrics/buildscript.txt

@@ -5,6 +5,6 @@ metrics
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=18.20.2
+--node-version=20.18.0
 --public-repo=False
 --script-version=4.5.0

libraries/metrics/initialize.js

@@ -88,7 +88,7 @@ function initializeProfileAgent() {
 }
 
 function initializePrometheus() {
-  const os = require('os')
+  const os = require('node:os')
   const promClient = require('prom-client')
   promClient.register.setDefaultLabels({ app: APP_NAME, host: os.hostname() })
   promClient.collectDefaultMetrics({ timeout: 5000, prefix: '' })

libraries/metrics/leaked_sockets.js

@@ -6,8 +6,8 @@
  * logged along with the corresponding information from /proc/net/tcp.
  */
 
-const fs = require('fs')
-const diagnosticsChannel = require('diagnostics_channel')
+const fs = require('node:fs')
+const diagnosticsChannel = require('node:diagnostics_channel')
 
 const SOCKET_MONITOR_INTERVAL = 60 * 1000
 // set the threshold for logging leaked sockets in minutes, defaults to 15

libraries/metrics/open_sockets.js

@@ -11,13 +11,13 @@ const seconds = 1000
 // In Node 0.10 the default is 5, which means only 5 open connections at one.
 // Node 0.12 has a default of Infinity. Make sure we have no limit set,
 // regardless of Node version.
-require('http').globalAgent.maxSockets = Infinity
-require('https').globalAgent.maxSockets = Infinity
+require('node:http').globalAgent.maxSockets = Infinity
+require('node:https').globalAgent.maxSockets = Infinity
 
-const SOCKETS_HTTP = require('http').globalAgent.sockets
-const SOCKETS_HTTPS = require('https').globalAgent.sockets
-const FREE_SOCKETS_HTTP = require('http').globalAgent.freeSockets
-const FREE_SOCKETS_HTTPS = require('https').globalAgent.freeSockets
+const SOCKETS_HTTP = require('node:http').globalAgent.sockets
+const SOCKETS_HTTPS = require('node:https').globalAgent.sockets
+const FREE_SOCKETS_HTTP = require('node:http').globalAgent.freeSockets
+const FREE_SOCKETS_HTTPS = require('node:https').globalAgent.freeSockets
 
 // keep track of set gauges and reset them in the next collection cycle
 const SEEN_HOSTS_HTTP = new Set()

libraries/metrics/test/acceptance/metrics_tests.js

@@ -1,6 +1,6 @@
-const { promisify } = require('util')
-const os = require('os')
-const http = require('http')
+const { promisify } = require('node:util')
+const os = require('node:os')
+const http = require('node:http')
 const { expect } = require('chai')
 const Metrics = require('../..')
 
@@ -316,7 +316,7 @@ async function checkSummaryValues(key, values) {
   for (const quantile of Object.keys(values)) {
     expect(found[quantile]).to.be.within(
       values[quantile] - 5,
-      values[quantile] + 5,
+      values[quantile] + 15,
       `quantile: ${quantile}`
     )
   }

libraries/metrics/test/unit/js/event_loop.js

@@ -5,7 +5,7 @@
  */
 const chai = require('chai')
 const { expect } = chai
-const path = require('path')
+const path = require('node:path')
 const modulePath = path.join(__dirname, '../../../event_loop.js')
 const SandboxedModule = require('sandboxed-module')
 const sinon = require('sinon')

libraries/metrics/test/unit/js/http.js

@@ -1,4 +1,4 @@
-const Path = require('path')
+const Path = require('node:path')
 const SandboxedModule = require('sandboxed-module')
 const sinon = require('sinon')
 

libraries/mongo-utils/.dockerignore

@@ -0,0 +1 @@
+node_modules/

libraries/mongo-utils/.gitignore

@@ -0,0 +1,3 @@
+
+# managed by monorepo$ bin/update_build_scripts
+.npmrc

libraries/mongo-utils/.nvmrc

@@ -0,0 +1 @@
+20.18.0

libraries/mongo-utils/batchedUpdate.js

@@ -1,9 +1,11 @@
-const { ObjectId } = require('mongodb-legacy')
-const {
-  db,
-  waitForDb,
-  READ_PREFERENCE_SECONDARY,
-} = require('../../app/src/infrastructure/mongodb')
+// @ts-check
+/* eslint-disable no-console */
+const { ObjectId, ReadPreference } = require('mongodb')
+
+const READ_PREFERENCE_SECONDARY =
+  process.env.MONGO_HAS_SECONDARIES === 'true'
+    ? ReadPreference.secondary.mode
+    : ReadPreference.secondaryPreferred.mode
 
 const ONE_MONTH_IN_MS = 1000 * 60 * 60 * 24 * 31
 let ID_EDGE_PAST
@@ -15,16 +17,38 @@ let BATCH_RANGE_START
 let BATCH_RANGE_END
 let BATCH_MAX_TIME_SPAN_IN_MS
 
+/**
+ * @typedef {import("mongodb").Collection} Collection
+ * @typedef {import("mongodb-legacy").Collection} LegacyCollection
+ * @typedef {import("mongodb").Document} Document
+ * @typedef {import("mongodb").FindOptions} FindOptions
+ * @typedef {import("mongodb").UpdateFilter<Document>} UpdateDocument
+ */
+
+/**
+ * @typedef {Object} BatchedUpdateOptions
+ * @property {string} [BATCH_DESCENDING]
+ * @property {string} [BATCH_LAST_ID]
+ * @property {string} [BATCH_MAX_TIME_SPAN_IN_MS]
+ * @property {string} [BATCH_RANGE_END]
+ * @property {string} [BATCH_RANGE_START]
+ * @property {string} [BATCH_SIZE]
+ * @property {string} [VERBOSE_LOGGING]
+ */
+
+/**
+ * @param {BatchedUpdateOptions} options
+ */
 function refreshGlobalOptionsForBatchedUpdate(options = {}) {
   options = Object.assign({}, options, process.env)
 
   BATCH_DESCENDING = options.BATCH_DESCENDING === 'true'
-  BATCH_SIZE = parseInt(options.BATCH_SIZE, 10) || 1000
+  BATCH_SIZE = parseInt(options.BATCH_SIZE || '1000', 10) || 1000
   VERBOSE_LOGGING = options.VERBOSE_LOGGING === 'true'
   if (options.BATCH_LAST_ID) {
-    BATCH_RANGE_START = new ObjectId(options.BATCH_LAST_ID)
+    BATCH_RANGE_START = objectIdFromInput(options.BATCH_LAST_ID)
   } else if (options.BATCH_RANGE_START) {
-    BATCH_RANGE_START = new ObjectId(options.BATCH_RANGE_START)
+    BATCH_RANGE_START = objectIdFromInput(options.BATCH_RANGE_START)
   } else {
     if (BATCH_DESCENDING) {
       BATCH_RANGE_START = ID_EDGE_FUTURE
@@ -32,10 +56,12 @@ function refreshGlobalOptionsForBatchedUpdate(options = {}) {
       BATCH_RANGE_START = ID_EDGE_PAST
     }
   }
-  BATCH_MAX_TIME_SPAN_IN_MS =
-    parseInt(options.BATCH_MAX_TIME_SPAN_IN_MS, 10) || ONE_MONTH_IN_MS
+  BATCH_MAX_TIME_SPAN_IN_MS = parseInt(
+    options.BATCH_MAX_TIME_SPAN_IN_MS || ONE_MONTH_IN_MS.toString(),
+    10
+  )
   if (options.BATCH_RANGE_END) {
-    BATCH_RANGE_END = new ObjectId(options.BATCH_RANGE_END)
+    BATCH_RANGE_END = objectIdFromInput(options.BATCH_RANGE_END)
   } else {
     if (BATCH_DESCENDING) {
       BATCH_RANGE_END = ID_EDGE_PAST
@@ -45,23 +71,32 @@ function refreshGlobalOptionsForBatchedUpdate(options = {}) {
   }
 }
 
-async function getNextBatch({
+/**
+ * @param {Collection | LegacyCollection} collection
+ * @param {Document} query
+ * @param {ObjectId} start
+ * @param {ObjectId} end
+ * @param {Document} projection
+ * @param {FindOptions} findOptions
+ * @return {Promise<Array<Document>>}
+ */
+async function getNextBatch(
   collection,
   query,
   start,
   end,
   projection,
-  findOptions,
-}) {
+  findOptions
+) {
   if (BATCH_DESCENDING) {
     query._id = {
       $gt: end,
-      $lt: start,
+      $lte: start,
     }
   } else {
     query._id = {
       $gt: start,
-      $lt: end,
+      $lte: end,
     }
   }
   return await collection
@@ -72,21 +107,61 @@ async function getNextBatch({
     .toArray()
 }
 
+/**
+ * @param {Collection | LegacyCollection} collection
+ * @param {Array<Document>} nextBatch
+ * @param {UpdateDocument} update
+ * @return {Promise<void>}
+ */
 async function performUpdate(collection, nextBatch, update) {
-  return collection.updateMany(
+  await collection.updateMany(
     { _id: { $in: nextBatch.map(entry => entry._id) } },
     update
   )
 }
 
+/**
+ * @param {string} input
+ * @return {ObjectId}
+ */
+function objectIdFromInput(input) {
+  if (input.includes('T')) {
+    const t = new Date(input).getTime()
+    if (Number.isNaN(t)) throw new Error(`${input} is not a valid date`)
+    return objectIdFromMs(t)
+  } else {
+    return new ObjectId(input)
+  }
+}
+
+/**
+ * @param {ObjectId} objectId
+ * @return {string}
+ */
+function renderObjectId(objectId) {
+  return `${objectId} (${objectId.getTimestamp().toISOString()})`
+}
+
+/**
+ * @param {number} ms
+ * @return {ObjectId}
+ */
 function objectIdFromMs(ms) {
   return ObjectId.createFromTime(ms / 1000)
 }
 
+/**
+ * @param {ObjectId} id
+ * @return {number}
+ */
 function getMsFromObjectId(id) {
   return id.getTimestamp().getTime()
 }
 
+/**
+ * @param {ObjectId} start
+ * @return {ObjectId}
+ */
 function getNextEnd(start) {
   let end
   if (BATCH_DESCENDING) {
@@ -103,6 +178,10 @@ function getNextEnd(start) {
   return end
 }
 
+/**
+ * @param {Collection | LegacyCollection} collection
+ * @return {Promise<ObjectId|null>}
+ */
 async function getIdEdgePast(collection) {
   const [first] = await collection
     .find({})
@@ -111,24 +190,32 @@ async function getIdEdgePast(collection) {
     .limit(1)
     .toArray()
   if (!first) return null
-  // Go 1s further into the past in order to include the first entry via
+  // Go one second further into the past in order to include the first entry via
   // first._id > ID_EDGE_PAST
   return objectIdFromMs(Math.max(0, getMsFromObjectId(first._id) - 1000))
 }
 
+/**
+ * @param {Collection | LegacyCollection} collection
+ * @param {Document} query
+ * @param {UpdateDocument | ((batch: Array<Document>) => Promise<void>)} update
+ * @param {Document} [projection]
+ * @param {FindOptions} [findOptions]
+ * @param {BatchedUpdateOptions} [batchedUpdateOptions]
+ */
 async function batchedUpdate(
-  collectionName,
+  collection,
   query,
   update,
   projection,
   findOptions,
   batchedUpdateOptions
 ) {
-  await waitForDb()
-  const collection = db[collectionName]
   ID_EDGE_PAST = await getIdEdgePast(collection)
   if (!ID_EDGE_PAST) {
-    console.warn(`The collection ${collectionName} appears to be empty.`)
+    console.warn(
+      `The collection ${collection.collectionName} appears to be empty.`
+    )
     return 0
   }
   refreshGlobalOptionsForBatchedUpdate(batchedUpdateOptions)
@@ -143,14 +230,14 @@ async function batchedUpdate(
 
   while (start !== BATCH_RANGE_END) {
     let end = getNextEnd(start)
-    nextBatch = await getNextBatch({
+    nextBatch = await getNextBatch(
       collection,
       query,
       start,
       end,
       projection,
-      findOptions,
-    })
+      findOptions
+    )
     if (nextBatch.length > 0) {
       end = nextBatch[nextBatch.length - 1]._id
       updated += nextBatch.length
@@ -162,7 +249,7 @@ async function batchedUpdate(
           )}`
         )
       } else {
-        console.error(`Running update on batch ending ${end}`)
+        console.error(`Running update on batch ending ${renderObjectId(end)}`)
       }
 
       if (typeof update === 'function') {
@@ -171,20 +258,36 @@ async function batchedUpdate(
         await performUpdate(collection, nextBatch, update)
       }
     }
-    console.error(`Completed batch ending ${end}`)
+    console.error(`Completed batch ending ${renderObjectId(end)}`)
     start = end
   }
   return updated
 }
 
+/**
+ * @param {Collection | LegacyCollection} collection
+ * @param {Document} query
+ * @param {UpdateDocument | ((batch: Array<Object>) => Promise<void>)} update
+ * @param {Document} [projection]
+ * @param {FindOptions} [findOptions]
+ * @param {BatchedUpdateOptions} [batchedUpdateOptions]
+ */
 function batchedUpdateWithResultHandling(
   collection,
   query,
   update,
   projection,
-  options
+  findOptions,
+  batchedUpdateOptions
 ) {
-  batchedUpdate(collection, query, update, projection, options)
+  batchedUpdate(
+    collection,
+    query,
+    update,
+    projection,
+    findOptions,
+    batchedUpdateOptions
+  )
     .then(processed => {
       console.error({ processed })
       process.exit(0)
@@ -196,6 +299,9 @@ function batchedUpdateWithResultHandling(
 }
 
 module.exports = {
+  READ_PREFERENCE_SECONDARY,
+  objectIdFromInput,
+  renderObjectId,
   batchedUpdate,
   batchedUpdateWithResultHandling,
 }

libraries/mongo-utils/buildscript.txt

@@ -0,0 +1,10 @@
+mongo-utils
+--dependencies=None
+--docker-repos=gcr.io/overleaf-ops
+--env-add=
+--env-pass-through=
+--esmock-loader=False
+--is-library=True
+--node-version=20.18.0
+--public-repo=False
+--script-version=4.5.0

libraries/mongo-utils/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "@overleaf/mongo-utils",
+  "version": "0.0.1",
+  "description": "utilities to help working with mongo",
+  "main": "index.js",
+  "scripts": {
+    "test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
+    "test:unit": "mocha --exit test/**/*.{js,cjs}",
+    "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
+    "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
+    "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
+    "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
+    "test:ci": "npm run test:unit",
+    "types:check": "tsc --noEmit"
+  },
+  "author": "Overleaf (https://www.overleaf.com)",
+  "license": "AGPL-3.0-only",
+  "dependencies": {
+    "mongodb": "6.10.0",
+    "mongodb-legacy": "6.1.3"
+  },
+  "devDependencies": {
+    "chai": "^4.3.6",
+    "mocha": "^10.2.0",
+    "sandboxed-module": "^2.0.4",
+    "sinon": "^9.2.4",
+    "sinon-chai": "^3.7.0",
+    "typescript": "^5.0.4"
+  }
+}

libraries/mongo-utils/test/setup.js

@@ -0,0 +1,11 @@
+const chai = require('chai')
+const sinonChai = require('sinon-chai')
+const SandboxedModule = require('sandboxed-module')
+
+// Chai configuration
+chai.should()
+chai.use(sinonChai)
+
+SandboxedModule.configure({
+  globals: { Buffer, JSON, console, process },
+})

libraries/mongo-utils/tsconfig.json

@@ -0,0 +1,7 @@
+{
+  "extends": "../../tsconfig.backend.json",
+  "include": [
+    "**/*.js",
+    "**/*.cjs"
+  ]
+}

libraries/o-error/.nvmrc

@@ -1 +1 @@
-18.20.2
+20.18.0

libraries/o-error/buildscript.txt

@@ -5,6 +5,6 @@ o-error
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=18.20.2
+--node-version=20.18.0
 --public-repo=False
 --script-version=4.5.0

libraries/o-error/doc/demo.js

@@ -70,7 +70,7 @@ sayHi3(43, (err, result) => {
   }
 })
 
-const promisify = require('util').promisify
+const promisify = require('node:util').promisify
 demoDatabase.findUserAsync = promisify(demoDatabase.findUser)
 
 async function sayHi4NoHandling(userId) {

libraries/o-error/test/o-error-util.test.js

@@ -1,5 +1,5 @@
 const { expect } = require('chai')
-const { promisify } = require('util')
+const { promisify } = require('node:util')
 
 const OError = require('..')
 

libraries/o-error/test/support/index.js

@@ -23,7 +23,7 @@ exports.expectError = function OErrorExpectError(e, expected) {
   ).to.be.true
 
   expect(
-    require('util').types.isNativeError(e),
+    require('node:util').types.isNativeError(e),
     'error should be recognised by util.types.isNativeError'
   ).to.be.true
 

libraries/object-persistor/.nvmrc

@@ -1 +1 @@
-18.20.2
+20.18.0

libraries/object-persistor/buildscript.txt

@@ -5,6 +5,6 @@ object-persistor
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=18.20.2
+--node-version=20.18.0
 --public-repo=False
 --script-version=4.5.0

libraries/object-persistor/package.json

@@ -24,7 +24,8 @@
     "@overleaf/logger": "*",
     "@overleaf/metrics": "*",
     "@overleaf/o-error": "*",
-    "aws-sdk": "^2.718.0",
+    "@overleaf/stream-utils": "*",
+    "aws-sdk": "^2.1691.0",
     "fast-crc32c": "overleaf/node-fast-crc32c#aae6b2a4c7a7a159395df9cc6c38dfde702d6f51",
     "glob": "^7.1.6",
     "range-parser": "^1.2.1",
@@ -35,7 +36,7 @@
     "chai-as-promised": "^7.1.1",
     "mocha": "^10.2.0",
     "mock-fs": "^5.2.0",
-    "mongodb": "^6.1.0",
+    "mongodb": "6.10.0",
     "sandboxed-module": "^2.0.4",
     "sinon": "^9.2.4",
     "sinon-chai": "^3.7.0",

libraries/object-persistor/src/AbstractPersistor.js

@@ -1,6 +1,12 @@
 const { NotImplementedError } = require('./Errors')
 
 module.exports = class AbstractPersistor {
+  /**
+   * @param location
+   * @param target
+   * @param {string} source
+   * @return {Promise<void>}
+   */
   async sendFile(location, target, source) {
     throw new NotImplementedError('method not implemented in persistor', {
       method: 'sendFile',
@@ -10,6 +16,13 @@ module.exports = class AbstractPersistor {
     })
   }
 
+  /**
+   * @param location
+   * @param target
+   * @param {NodeJS.ReadableStream} sourceStream
+   * @param {Object} opts
+   * @return {Promise<void>}
+   */
   async sendStream(location, target, sourceStream, opts = {}) {
     throw new NotImplementedError('method not implemented in persistor', {
       method: 'sendStream',
@@ -22,12 +35,12 @@ module.exports = class AbstractPersistor {
   /**
    * @param location
    * @param name
-   * @param {Object} opts
-   * @param {Number} opts.start
-   * @param {Number} opts.end
-   * @return {Promise<Readable>}
+   * @param {Object} [opts]
+   * @param {Number} [opts.start]
+   * @param {Number} [opts.end]
+   * @return {Promise<NodeJS.ReadableStream>}
    */
-  async getObjectStream(location, name, opts) {
+  async getObjectStream(location, name, opts = {}) {
     throw new NotImplementedError('method not implemented in persistor', {
       method: 'getObjectStream',
       location,
@@ -36,6 +49,11 @@ module.exports = class AbstractPersistor {
     })
   }
 
+  /**
+   * @param {string} location
+   * @param {string} name
+   * @return {Promise<string>}
+   */
   async getRedirectUrl(location, name) {
     throw new NotImplementedError('method not implemented in persistor', {
       method: 'getRedirectUrl',
@@ -44,7 +62,13 @@ module.exports = class AbstractPersistor {
     })
   }
 
-  async getObjectSize(location, name) {
+  /**
+   * @param {string} location
+   * @param {string} name
+   * @param {Object} opts
+   * @return {Promise<number>}
+   */
+  async getObjectSize(location, name, opts) {
     throw new NotImplementedError('method not implemented in persistor', {
       method: 'getObjectSize',
       location,
@@ -52,7 +76,13 @@ module.exports = class AbstractPersistor {
     })
   }
 
-  async getObjectMd5Hash(location, name) {
+  /**
+   * @param {string} location
+   * @param {string} name
+   * @param {Object} opts
+   * @return {Promise<string>}
+   */
+  async getObjectMd5Hash(location, name, opts) {
     throw new NotImplementedError('method not implemented in persistor', {
       method: 'getObjectMd5Hash',
       location,
@@ -60,7 +90,14 @@ module.exports = class AbstractPersistor {
     })
   }
 
-  async copyObject(location, fromName, toName) {
+  /**
+   * @param {string} location
+   * @param {string} fromName
+   * @param {string} toName
+   * @param {Object} opts
+   * @return {Promise<void>}
+   */
+  async copyObject(location, fromName, toName, opts) {
     throw new NotImplementedError('method not implemented in persistor', {
       method: 'copyObject',
       location,
@@ -69,6 +106,11 @@ module.exports = class AbstractPersistor {
     })
   }
 
+  /**
+   * @param {string} location
+   * @param {string} name
+   * @return {Promise<void>}
+   */
   async deleteObject(location, name) {
     throw new NotImplementedError('method not implemented in persistor', {
       method: 'deleteObject',
@@ -77,7 +119,13 @@ module.exports = class AbstractPersistor {
     })
   }
 
-  async deleteDirectory(location, name) {
+  /**
+   * @param {string} location
+   * @param {string} name
+   * @param {string} [continuationToken]
+   * @return {Promise<void>}
+   */
+  async deleteDirectory(location, name, continuationToken) {
     throw new NotImplementedError('method not implemented in persistor', {
       method: 'deleteDirectory',
       location,
@@ -85,7 +133,13 @@ module.exports = class AbstractPersistor {
     })
   }
 
-  async checkIfObjectExists(location, name) {
+  /**
+   * @param {string} location
+   * @param {string} name
+   * @param {Object} opts
+   * @return {Promise<boolean>}
+   */
+  async checkIfObjectExists(location, name, opts) {
     throw new NotImplementedError('method not implemented in persistor', {
       method: 'checkIfObjectExists',
       location,
@@ -93,7 +147,13 @@ module.exports = class AbstractPersistor {
     })
   }
 
-  async directorySize(location, name) {
+  /**
+   * @param {string} location
+   * @param {string} name
+   * @param {string} [continuationToken]
+   * @return {Promise<number>}
+   */
+  async directorySize(location, name, continuationToken) {
     throw new NotImplementedError('method not implemented in persistor', {
       method: 'directorySize',
       location,

libraries/object-persistor/src/Errors.js

@@ -5,6 +5,8 @@ class WriteError extends OError {}
 class ReadError extends OError {}
 class SettingsError extends OError {}
 class NotImplementedError extends OError {}
+class AlreadyWrittenError extends OError {}
+class NoKEKMatchedError extends OError {}
 
 module.exports = {
   NotFoundError,
@@ -12,4 +14,6 @@ module.exports = {
   ReadError,
   SettingsError,
   NotImplementedError,
+  AlreadyWrittenError,
+  NoKEKMatchedError,
 }

libraries/object-persistor/src/FSPersistor.js

@@ -1,20 +1,26 @@
-const crypto = require('crypto')
-const fs = require('fs')
-const fsPromises = require('fs/promises')
+const crypto = require('node:crypto')
+const fs = require('node:fs')
+const fsPromises = require('node:fs/promises')
 const globCallbacks = require('glob')
-const Path = require('path')
-const { PassThrough } = require('stream')
-const { pipeline } = require('stream/promises')
-const { promisify } = require('util')
+const Path = require('node:path')
+const { PassThrough } = require('node:stream')
+const { pipeline } = require('node:stream/promises')
+const { promisify } = require('node:util')
 
 const AbstractPersistor = require('./AbstractPersistor')
-const { ReadError, WriteError } = require('./Errors')
+const { ReadError, WriteError, NotImplementedError } = require('./Errors')
 const PersistorHelper = require('./PersistorHelper')
 
 const glob = promisify(globCallbacks)
 
 module.exports = class FSPersistor extends AbstractPersistor {
   constructor(settings = {}) {
+    if (settings.storageClass) {
+      throw new NotImplementedError(
+        'FS backend does not support storage classes'
+      )
+    }
+
     super()
     this.useSubdirectories = Boolean(settings.useSubdirectories)
   }
@@ -36,6 +42,14 @@ module.exports = class FSPersistor extends AbstractPersistor {
   }
 
   async sendStream(location, target, sourceStream, opts = {}) {
+    if (opts.ifNoneMatch === '*') {
+      // The standard library only has fs.rename(), which does not support exclusive flags.
+      // Refuse to act on this write operation.
+      throw new NotImplementedError(
+        'Overwrite protection required by caller, but it is not available is FS backend. Configure GCS or S3 backend instead, get in touch with support for further information.'
+      )
+    }
+
     const targetPath = this._getFsPath(location, target)
 
     try {
@@ -55,7 +69,7 @@ module.exports = class FSPersistor extends AbstractPersistor {
       throw PersistorHelper.wrapError(
         err,
         'failed to write stream',
-        { location, target },
+        { location, target, ifNoneMatch: opts.ifNoneMatch },
         WriteError
       )
     }
@@ -63,6 +77,11 @@ module.exports = class FSPersistor extends AbstractPersistor {
 
   // opts may be {start: Number, end: Number}
   async getObjectStream(location, name, opts = {}) {
+    if (opts.autoGunzip) {
+      throw new NotImplementedError(
+        'opts.autoGunzip is not supported by FS backend. Configure GCS or S3 backend instead, get in touch with support for further information.'
+      )
+    }
     const observer = new PersistorHelper.ObserverStream({
       metric: 'fs.ingress', // ingress to us from disk
       bucket: location,

libraries/object-persistor/src/GcsPersistor.js

@@ -1,17 +1,28 @@
-const fs = require('fs')
-const { pipeline } = require('stream/promises')
-const { PassThrough } = require('stream')
+const fs = require('node:fs')
+const { pipeline } = require('node:stream/promises')
+const { PassThrough } = require('node:stream')
 const { Storage, IdempotencyStrategy } = require('@google-cloud/storage')
-const { WriteError, ReadError, NotFoundError } = require('./Errors')
+const {
+  WriteError,
+  ReadError,
+  NotFoundError,
+  NotImplementedError,
+} = require('./Errors')
 const asyncPool = require('tiny-async-pool')
 const AbstractPersistor = require('./AbstractPersistor')
 const PersistorHelper = require('./PersistorHelper')
 const Logger = require('@overleaf/logger')
+const zlib = require('node:zlib')
 
 module.exports = class GcsPersistor extends AbstractPersistor {
   constructor(settings) {
-    super()
+    if (settings.storageClass) {
+      throw new NotImplementedError(
+        'Use default bucket class for GCS instead of settings.storageClass'
+      )
+    }
 
+    super()
     this.settings = settings
 
     // endpoint settings will be null by default except for tests
@@ -78,10 +89,14 @@ module.exports = class GcsPersistor extends AbstractPersistor {
         writeOptions.metadata = writeOptions.metadata || {}
         writeOptions.metadata.contentEncoding = opts.contentEncoding
       }
+      const fileOptions = {}
+      if (opts.ifNoneMatch === '*') {
+        fileOptions.generation = 0
+      }
 
       const uploadStream = this.storage
         .bucket(bucketName)
-        .file(key)
+        .file(key, fileOptions)
         .createWriteStream(writeOptions)
 
       await pipeline(readStream, observer, uploadStream)
@@ -97,7 +112,7 @@ module.exports = class GcsPersistor extends AbstractPersistor {
       throw PersistorHelper.wrapError(
         err,
         'upload to GCS failed',
-        { bucketName, key },
+        { bucketName, key, ifNoneMatch: opts.ifNoneMatch },
         WriteError
       )
     }
@@ -113,12 +128,14 @@ module.exports = class GcsPersistor extends AbstractPersistor {
       .file(key)
       .createReadStream({ decompress: false, ...opts })
 
+    let contentEncoding
     try {
       await new Promise((resolve, reject) => {
         stream.on('response', res => {
           switch (res.statusCode) {
             case 200: // full response
             case 206: // partial response
+              contentEncoding = res.headers['content-encoding']
               return resolve()
             case 404:
               return reject(new NotFoundError())
@@ -139,7 +156,11 @@ module.exports = class GcsPersistor extends AbstractPersistor {
     }
     // Return a PassThrough stream with a minimal interface. It will buffer until the caller starts reading. It will emit errors from the source stream (Stream.pipeline passes errors along).
     const pass = new PassThrough()
-    pipeline(stream, observer, pass).catch(() => {})
+    const transformer = []
+    if (contentEncoding === 'gzip' && opts.autoGunzip) {
+      transformer.push(zlib.createGunzip())
+    }
+    pipeline(stream, observer, ...transformer, pass).catch(() => {})
     return pass
   }
 
@@ -176,7 +197,7 @@ module.exports = class GcsPersistor extends AbstractPersistor {
         .bucket(bucketName)
         .file(key)
         .getMetadata()
-      return metadata.size
+      return parseInt(metadata.size, 10)
     } catch (err) {
       throw PersistorHelper.wrapError(
         err,
@@ -285,7 +306,10 @@ module.exports = class GcsPersistor extends AbstractPersistor {
       )
     }
 
-    return files.reduce((acc, file) => Number(file.metadata.size) + acc, 0)
+    return files.reduce(
+      (acc, file) => parseInt(file.metadata.size, 10) + acc,
+      0
+    )
   }
 
   async checkIfObjectExists(bucketName, key) {

libraries/object-persistor/src/MigrationPersistor.js

@@ -1,8 +1,8 @@
 const AbstractPersistor = require('./AbstractPersistor')
 const Logger = require('@overleaf/logger')
 const Metrics = require('@overleaf/metrics')
-const Stream = require('stream')
-const { pipeline } = require('stream/promises')
+const Stream = require('node:stream')
+const { pipeline } = require('node:stream/promises')
 const { NotFoundError, WriteError } = require('./Errors')
 
 // Persistor that wraps two other persistors. Talks to the 'primary' by default,

libraries/object-persistor/src/PerProjectEncryptedS3Persistor.js

@@ -0,0 +1,458 @@
+// @ts-check
+const Crypto = require('node:crypto')
+const Stream = require('node:stream')
+const fs = require('node:fs')
+const { promisify } = require('node:util')
+const { WritableBuffer } = require('@overleaf/stream-utils')
+const { S3Persistor, SSECOptions } = require('./S3Persistor.js')
+const {
+  AlreadyWrittenError,
+  NoKEKMatchedError,
+  NotFoundError,
+  NotImplementedError,
+  ReadError,
+} = require('./Errors')
+const logger = require('@overleaf/logger')
+const Path = require('node:path')
+
+const generateKey = promisify(Crypto.generateKey)
+const hkdf = promisify(Crypto.hkdf)
+
+const AES256_KEY_LENGTH = 32
+
+/**
+ * @typedef {import('aws-sdk').AWSError} AWSError
+ */
+
+/**
+ * @typedef {Object} Settings
+ * @property {boolean} automaticallyRotateDEKEncryption
+ * @property {string} dataEncryptionKeyBucketName
+ * @property {boolean} ignoreErrorsFromDEKReEncryption
+ * @property {(bucketName: string, path: string) => string} pathToProjectFolder
+ * @property {() => Promise<Array<RootKeyEncryptionKey>>} getRootKeyEncryptionKeys
+ */
+
+/**
+ * Helper function to make TS happy when accessing error properties
+ * AWSError is not an actual class, so we cannot use instanceof.
+ * @param {any} err
+ * @return {err is AWSError}
+ */
+function isAWSError(err) {
+  return !!err
+}
+
+/**
+ * @param {any} err
+ * @return {boolean}
+ */
+function isForbiddenError(err) {
+  if (!err || !(err instanceof ReadError || err instanceof NotFoundError)) {
+    return false
+  }
+  const cause = err.cause
+  if (!isAWSError(cause)) return false
+  return cause.statusCode === 403
+}
+
+class RootKeyEncryptionKey {
+  /** @type {Buffer} */
+  #keyEncryptionKey
+  /** @type {Buffer} */
+  #salt
+
+  /**
+   * @param {Buffer} keyEncryptionKey
+   * @param {Buffer} salt
+   */
+  constructor(keyEncryptionKey, salt) {
+    if (keyEncryptionKey.byteLength !== AES256_KEY_LENGTH) {
+      throw new Error(`kek is not ${AES256_KEY_LENGTH} bytes long`)
+    }
+    this.#keyEncryptionKey = keyEncryptionKey
+    this.#salt = salt
+  }
+
+  /**
+   * @param {string} prefix
+   * @return {Promise<SSECOptions>}
+   */
+  async forProject(prefix) {
+    return new SSECOptions(
+      Buffer.from(
+        await hkdf(
+          'sha256',
+          this.#keyEncryptionKey,
+          this.#salt,
+          prefix,
+          AES256_KEY_LENGTH
+        )
+      )
+    )
+  }
+}
+
+class PerProjectEncryptedS3Persistor extends S3Persistor {
+  /** @type {Settings} */
+  #settings
+  /** @type {Promise<Array<RootKeyEncryptionKey>>} */
+  #availableKeyEncryptionKeysPromise
+
+  /**
+   * @param {Settings} settings
+   */
+  constructor(settings) {
+    if (!settings.dataEncryptionKeyBucketName) {
+      throw new Error('settings.dataEncryptionKeyBucketName is missing')
+    }
+    super(settings)
+    this.#settings = settings
+    this.#availableKeyEncryptionKeysPromise = settings
+      .getRootKeyEncryptionKeys()
+      .then(rootKEKs => {
+        if (rootKEKs.length === 0) throw new Error('no root kek provided')
+        return rootKEKs
+      })
+  }
+
+  async ensureKeyEncryptionKeysLoaded() {
+    await this.#availableKeyEncryptionKeysPromise
+  }
+
+  /**
+   * @param {string} bucketName
+   * @param {string} path
+   * @return {{dekPath: string, projectFolder: string}}
+   */
+  #buildProjectPaths(bucketName, path) {
+    const projectFolder = this.#settings.pathToProjectFolder(bucketName, path)
+    const dekPath = Path.join(projectFolder, 'dek')
+    return { projectFolder, dekPath }
+  }
+
+  /**
+   * @param {string} projectFolder
+   * @return {Promise<SSECOptions>}
+   */
+  async #getCurrentKeyEncryptionKey(projectFolder) {
+    const [currentRootKEK] = await this.#availableKeyEncryptionKeysPromise
+    return await currentRootKEK.forProject(projectFolder)
+  }
+
+  /**
+   * @param {string} bucketName
+   * @param {string} path
+   */
+  async getDataEncryptionKeySize(bucketName, path) {
+    const { projectFolder, dekPath } = this.#buildProjectPaths(bucketName, path)
+    for (const rootKEK of await this.#availableKeyEncryptionKeysPromise) {
+      const ssecOptions = await rootKEK.forProject(projectFolder)
+      try {
+        return await super.getObjectSize(
+          this.#settings.dataEncryptionKeyBucketName,
+          dekPath,
+          { ssecOptions }
+        )
+      } catch (err) {
+        if (isForbiddenError(err)) continue
+        throw err
+      }
+    }
+    throw new NoKEKMatchedError('no kek matched')
+  }
+
+  /**
+   * @param {string} bucketName
+   * @param {string} path
+   * @return {Promise<CachedPerProjectEncryptedS3Persistor>}
+   */
+  async forProject(bucketName, path) {
+    return new CachedPerProjectEncryptedS3Persistor(
+      this,
+      await this.#getDataEncryptionKeyOptions(bucketName, path)
+    )
+  }
+
+  /**
+   * @param {string} bucketName
+   * @param {string} path
+   * @return {Promise<CachedPerProjectEncryptedS3Persistor>}
+   */
+  async forProjectRO(bucketName, path) {
+    return new CachedPerProjectEncryptedS3Persistor(
+      this,
+      await this.#getExistingDataEncryptionKeyOptions(bucketName, path)
+    )
+  }
+
+  /**
+   * @param {string} bucketName
+   * @param {string} path
+   * @return {Promise<CachedPerProjectEncryptedS3Persistor>}
+   */
+  async generateDataEncryptionKey(bucketName, path) {
+    return new CachedPerProjectEncryptedS3Persistor(
+      this,
+      await this.#generateDataEncryptionKeyOptions(bucketName, path)
+    )
+  }
+
+  /**
+   * @param {string} bucketName
+   * @param {string} path
+   * @return {Promise<SSECOptions>}
+   */
+  async #generateDataEncryptionKeyOptions(bucketName, path) {
+    const dataEncryptionKey = (
+      await generateKey('aes', { length: 256 })
+    ).export()
+    const { projectFolder, dekPath } = this.#buildProjectPaths(bucketName, path)
+    await super.sendStream(
+      this.#settings.dataEncryptionKeyBucketName,
+      dekPath,
+      Stream.Readable.from([dataEncryptionKey]),
+      {
+        // Do not overwrite any objects if already created
+        ifNoneMatch: '*',
+        ssecOptions: await this.#getCurrentKeyEncryptionKey(projectFolder),
+        contentLength: 32,
+      }
+    )
+    return new SSECOptions(dataEncryptionKey)
+  }
+
+  /**
+   * @param {string} bucketName
+   * @param {string} path
+   * @return {Promise<SSECOptions>}
+   */
+  async #getExistingDataEncryptionKeyOptions(bucketName, path) {
+    const { projectFolder, dekPath } = this.#buildProjectPaths(bucketName, path)
+    let res
+    let kekIndex = 0
+    for (const rootKEK of await this.#availableKeyEncryptionKeysPromise) {
+      const ssecOptions = await rootKEK.forProject(projectFolder)
+      try {
+        res = await super.getObjectStream(
+          this.#settings.dataEncryptionKeyBucketName,
+          dekPath,
+          { ssecOptions }
+        )
+        break
+      } catch (err) {
+        if (isForbiddenError(err)) {
+          kekIndex++
+          continue
+        }
+        throw err
+      }
+    }
+    if (!res) throw new NoKEKMatchedError('no kek matched')
+    const buf = new WritableBuffer()
+    await Stream.promises.pipeline(res, buf)
+
+    if (kekIndex !== 0 && this.#settings.automaticallyRotateDEKEncryption) {
+      const ssecOptions = await this.#getCurrentKeyEncryptionKey(projectFolder)
+      try {
+        await super.sendStream(
+          this.#settings.dataEncryptionKeyBucketName,
+          dekPath,
+          Stream.Readable.from([buf.getContents()]),
+          { ssecOptions }
+        )
+      } catch (err) {
+        if (this.#settings.ignoreErrorsFromDEKReEncryption) {
+          logger.warn({ err, dekPath }, 'failed to persist re-encrypted DEK')
+        } else {
+          throw err
+        }
+      }
+    }
+
+    return new SSECOptions(buf.getContents())
+  }
+
+  /**
+   * @param {string} bucketName
+   * @param {string} path
+   * @return {Promise<SSECOptions>}
+   */
+  async #getDataEncryptionKeyOptions(bucketName, path) {
+    try {
+      return await this.#getExistingDataEncryptionKeyOptions(bucketName, path)
+    } catch (err) {
+      if (err instanceof NotFoundError) {
+        try {
+          return await this.#generateDataEncryptionKeyOptions(bucketName, path)
+        } catch (err2) {
+          if (err2 instanceof AlreadyWrittenError) {
+            // Concurrent initial write
+            return await this.#getExistingDataEncryptionKeyOptions(
+              bucketName,
+              path
+            )
+          }
+          throw err2
+        }
+      }
+      throw err
+    }
+  }
+
+  async sendStream(bucketName, path, sourceStream, opts = {}) {
+    const ssecOptions =
+      opts.ssecOptions ||
+      (await this.#getDataEncryptionKeyOptions(bucketName, path))
+    return await super.sendStream(bucketName, path, sourceStream, {
+      ...opts,
+      ssecOptions,
+    })
+  }
+
+  async getObjectStream(bucketName, path, opts = {}) {
+    const ssecOptions =
+      opts.ssecOptions ||
+      (await this.#getExistingDataEncryptionKeyOptions(bucketName, path))
+    return await super.getObjectStream(bucketName, path, {
+      ...opts,
+      ssecOptions,
+    })
+  }
+
+  async getObjectSize(bucketName, path, opts = {}) {
+    const ssecOptions =
+      opts.ssecOptions ||
+      (await this.#getExistingDataEncryptionKeyOptions(bucketName, path))
+    return await super.getObjectSize(bucketName, path, { ...opts, ssecOptions })
+  }
+
+  async getObjectStorageClass(bucketName, path, opts = {}) {
+    const ssecOptions =
+      opts.ssecOptions ||
+      (await this.#getExistingDataEncryptionKeyOptions(bucketName, path))
+    return await super.getObjectStorageClass(bucketName, path, {
+      ...opts,
+      ssecOptions,
+    })
+  }
+
+  async directorySize(bucketName, path, continuationToken) {
+    // Note: Listing a bucket does not require SSE-C credentials.
+    return await super.directorySize(bucketName, path, continuationToken)
+  }
+
+  async deleteDirectory(bucketName, path, continuationToken) {
+    // Note: Listing/Deleting a prefix does not require SSE-C credentials.
+    await super.deleteDirectory(bucketName, path, continuationToken)
+    const { projectFolder, dekPath } = this.#buildProjectPaths(bucketName, path)
+    if (projectFolder === path) {
+      await super.deleteObject(
+        this.#settings.dataEncryptionKeyBucketName,
+        dekPath
+      )
+    }
+  }
+
+  async getObjectMd5Hash(bucketName, path, opts = {}) {
+    // The ETag in object metadata is not the MD5 content hash, skip the HEAD request.
+    opts = { ...opts, etagIsNotMD5: true }
+    return await super.getObjectMd5Hash(bucketName, path, opts)
+  }
+
+  async copyObject(bucketName, sourcePath, destinationPath, opts = {}) {
+    const ssecOptions =
+      opts.ssecOptions ||
+      (await this.#getDataEncryptionKeyOptions(bucketName, destinationPath))
+    const ssecSrcOptions =
+      opts.ssecSrcOptions ||
+      (await this.#getExistingDataEncryptionKeyOptions(bucketName, sourcePath))
+    return await super.copyObject(bucketName, sourcePath, destinationPath, {
+      ...opts,
+      ssecOptions,
+      ssecSrcOptions,
+    })
+  }
+
+  /**
+   * @param {string} bucketName
+   * @param {string} path
+   * @return {Promise<string>}
+   */
+  async getRedirectUrl(bucketName, path) {
+    throw new NotImplementedError('signed links are not supported with SSE-C')
+  }
+}
+
+/**
+ * Helper class for batch updates to avoid repeated fetching of the project path.
+ *
+ * A general "cache" for project keys is another alternative. For now, use a helper class.
+ */
+class CachedPerProjectEncryptedS3Persistor {
+  /** @type SSECOptions */
+  #projectKeyOptions
+  /** @type PerProjectEncryptedS3Persistor */
+  #parent
+
+  /**
+   * @param {PerProjectEncryptedS3Persistor} parent
+   * @param {SSECOptions} projectKeyOptions
+   */
+  constructor(parent, projectKeyOptions) {
+    this.#parent = parent
+    this.#projectKeyOptions = projectKeyOptions
+  }
+
+  /**
+   * @param {string} bucketName
+   * @param {string} path
+   * @param {string} fsPath
+   */
+  async sendFile(bucketName, path, fsPath) {
+    return await this.sendStream(bucketName, path, fs.createReadStream(fsPath))
+  }
+
+  /**
+   * @param {string} bucketName
+   * @param {string} path
+   * @param {NodeJS.ReadableStream} sourceStream
+   * @param {Object} opts
+   * @param {string} [opts.contentType]
+   * @param {string} [opts.contentEncoding]
+   * @param {number} [opts.contentLength]
+   * @param {'*'} [opts.ifNoneMatch]
+   * @param {SSECOptions} [opts.ssecOptions]
+   * @param {string} [opts.sourceMd5]
+   * @return {Promise<void>}
+   */
+  async sendStream(bucketName, path, sourceStream, opts = {}) {
+    return await this.#parent.sendStream(bucketName, path, sourceStream, {
+      ...opts,
+      ssecOptions: this.#projectKeyOptions,
+    })
+  }
+
+  /**
+   * @param {string} bucketName
+   * @param {string} path
+   * @param {Object} opts
+   * @param {number} [opts.start]
+   * @param {number} [opts.end]
+   * @param {boolean} [opts.autoGunzip]
+   * @param {SSECOptions} [opts.ssecOptions]
+   * @return {Promise<NodeJS.ReadableStream>}
+   */
+  async getObjectStream(bucketName, path, opts = {}) {
+    return await this.#parent.getObjectStream(bucketName, path, {
+      ...opts,
+      ssecOptions: this.#projectKeyOptions,
+    })
+  }
+}
+
+module.exports = {
+  PerProjectEncryptedS3Persistor,
+  CachedPerProjectEncryptedS3Persistor,
+  RootKeyEncryptionKey,
+}

libraries/object-persistor/src/PersistorFactory.js

@@ -1,15 +1,20 @@
 const Logger = require('@overleaf/logger')
 const { SettingsError } = require('./Errors')
 const GcsPersistor = require('./GcsPersistor')
-const S3Persistor = require('./S3Persistor')
+const { S3Persistor } = require('./S3Persistor')
 const FSPersistor = require('./FSPersistor')
 const MigrationPersistor = require('./MigrationPersistor')
+const {
+  PerProjectEncryptedS3Persistor,
+} = require('./PerProjectEncryptedS3Persistor')
 
 function getPersistor(backend, settings) {
   switch (backend) {
     case 'aws-sdk':
     case 's3':
       return new S3Persistor(settings.s3)
+    case 's3SSEC':
+      return new PerProjectEncryptedS3Persistor(settings.s3SSEC)
     case 'fs':
       return new FSPersistor({
         useSubdirectories: settings.useSubdirectories,

libraries/object-persistor/src/PersistorHelper.js

@@ -1,9 +1,9 @@
-const Crypto = require('crypto')
-const Stream = require('stream')
-const { pipeline } = require('stream/promises')
+const Crypto = require('node:crypto')
+const Stream = require('node:stream')
+const { pipeline } = require('node:stream/promises')
 const Logger = require('@overleaf/logger')
 const Metrics = require('@overleaf/metrics')
-const { WriteError, NotFoundError } = require('./Errors')
+const { WriteError, NotFoundError, AlreadyWrittenError } = require('./Errors')
 
 const _128KiB = 128 * 1024
 const TIMING_BUCKETS = [
@@ -26,12 +26,14 @@ const SIZE_BUCKETS = [
  */
 class ObserverStream extends Stream.Transform {
   /**
-   * @param {string} metric prefix for metrics
-   * @param {string} bucket name of source/target bucket
-   * @param {string} hash optional hash algorithm, e.g. 'md5'
+   * @param {Object} opts
+   * @param {string} opts.metric prefix for metrics
+   * @param {string} opts.bucket name of source/target bucket
+   * @param {string} [opts.hash] optional hash algorithm, e.g. 'md5'
    */
-  constructor({ metric, bucket, hash = '' }) {
+  constructor(opts) {
     super({ autoDestroy: true })
+    const { metric, bucket, hash = '' } = opts
 
     this.bytes = 0
     this.start = performance.now()
@@ -138,6 +140,10 @@ async function verifyMd5(persistor, bucket, key, sourceMd5, destMd5 = null) {
 }
 
 function wrapError(error, message, params, ErrorType) {
+  params = {
+    ...params,
+    cause: error,
+  }
   if (
     error instanceof NotFoundError ||
     ['NoSuchKey', 'NotFound', 404, 'AccessDenied', 'ENOENT'].includes(
@@ -146,6 +152,13 @@ function wrapError(error, message, params, ErrorType) {
     (error.response && error.response.statusCode === 404)
   ) {
     return new NotFoundError('no such file', params, error)
+  } else if (
+    params.ifNoneMatch === '*' &&
+    (error.code === 'PreconditionFailed' ||
+      error.response?.statusCode === 412 ||
+      error instanceof AlreadyWrittenError)
+  ) {
+    return new AlreadyWrittenError(message, params, error)
   } else {
     return new ErrorType(message, params, error)
   }

libraries/object-persistor/src/S3Persistor.js

@@ -1,5 +1,6 @@
-const http = require('http')
-const https = require('https')
+// @ts-check
+const http = require('node:http')
+const https = require('node:https')
 if (http.globalAgent.maxSockets < 300) {
   http.globalAgent.maxSockets = 300
 }
@@ -7,27 +8,92 @@ if (https.globalAgent.maxSockets < 300) {
   https.globalAgent.maxSockets = 300
 }
 
+const Crypto = require('node:crypto')
 const Metrics = require('@overleaf/metrics')
 const AbstractPersistor = require('./AbstractPersistor')
 const PersistorHelper = require('./PersistorHelper')
 
-const { pipeline, PassThrough } = require('stream')
-const fs = require('fs')
+const { pipeline, PassThrough } = require('node:stream')
+const fs = require('node:fs')
 const S3 = require('aws-sdk/clients/s3')
-const { URL } = require('url')
+const { URL } = require('node:url')
 const { WriteError, ReadError, NotFoundError } = require('./Errors')
+const zlib = require('node:zlib')
+
+/**
+ * Wrapper with private fields to avoid revealing them on console, JSON.stringify or similar.
+ */
+class SSECOptions {
+  #keyAsBuffer
+  #keyMD5
+
+  /**
+   * @param {Buffer} keyAsBuffer
+   */
+  constructor(keyAsBuffer) {
+    this.#keyAsBuffer = keyAsBuffer
+    this.#keyMD5 = Crypto.createHash('md5').update(keyAsBuffer).digest('base64')
+  }
+
+  getPutOptions() {
+    return {
+      SSECustomerKey: this.#keyAsBuffer,
+      SSECustomerKeyMD5: this.#keyMD5,
+      SSECustomerAlgorithm: 'AES256',
+    }
+  }
+
+  getGetOptions() {
+    return {
+      SSECustomerKey: this.#keyAsBuffer,
+      SSECustomerKeyMD5: this.#keyMD5,
+      SSECustomerAlgorithm: 'AES256',
+    }
+  }
+
+  getCopyOptions() {
+    return {
+      CopySourceSSECustomerKey: this.#keyAsBuffer,
+      CopySourceSSECustomerKeyMD5: this.#keyMD5,
+      CopySourceSSECustomerAlgorithm: 'AES256',
+    }
+  }
+}
+
+class S3Persistor extends AbstractPersistor {
+  /** @type {Map<string, S3>} */
+  #clients = new Map()
 
-module.exports = class S3Persistor extends AbstractPersistor {
   constructor(settings = {}) {
     super()
 
+    settings.storageClass = settings.storageClass || {}
     this.settings = settings
   }
 
+  /**
+   * @param {string} bucketName
+   * @param {string} key
+   * @param {string} fsPath
+   * @return {Promise<void>}
+   */
   async sendFile(bucketName, key, fsPath) {
-    return await this.sendStream(bucketName, key, fs.createReadStream(fsPath))
+    await this.sendStream(bucketName, key, fs.createReadStream(fsPath))
   }
 
+  /**
+   * @param {string} bucketName
+   * @param {string} key
+   * @param {NodeJS.ReadableStream} readStream
+   * @param {Object} opts
+   * @param {string} [opts.contentType]
+   * @param {string} [opts.contentEncoding]
+   * @param {number} [opts.contentLength]
+   * @param {'*'} [opts.ifNoneMatch]
+   * @param {SSECOptions} [opts.ssecOptions]
+   * @param {string} [opts.sourceMd5]
+   * @return {Promise<void>}
+   */
   async sendStream(bucketName, key, readStream, opts = {}) {
     try {
       const observeOptions = {
@@ -39,42 +105,71 @@ module.exports = class S3Persistor extends AbstractPersistor {
       // observer will catch errors, clean up and log a warning
       pipeline(readStream, observer, () => {})
 
-      // if we have an md5 hash, pass this to S3 to verify the upload
+      /** @type {S3.PutObjectRequest} */
       const uploadOptions = {
         Bucket: bucketName,
         Key: key,
         Body: observer,
       }
 
+      if (this.settings.storageClass[bucketName]) {
+        uploadOptions.StorageClass = this.settings.storageClass[bucketName]
+      }
+
       if (opts.contentType) {
         uploadOptions.ContentType = opts.contentType
       }
       if (opts.contentEncoding) {
         uploadOptions.ContentEncoding = opts.contentEncoding
       }
+      if (opts.contentLength) {
+        uploadOptions.ContentLength = opts.contentLength
+      }
+      if (opts.ifNoneMatch === '*') {
+        uploadOptions.IfNoneMatch = '*'
+      }
+      if (opts.ssecOptions) {
+        Object.assign(uploadOptions, opts.ssecOptions.getPutOptions())
+      }
 
       // if we have an md5 hash, pass this to S3 to verify the upload - otherwise
       // we rely on the S3 client's checksum calculation to validate the upload
-      const clientOptions = {}
+      let computeChecksums = false
       if (opts.sourceMd5) {
         uploadOptions.ContentMD5 = PersistorHelper.hexToBase64(opts.sourceMd5)
       } else {
-        clientOptions.computeChecksums = true
+        computeChecksums = true
       }
 
-      await this._getClientForBucket(bucketName, clientOptions)
-        .upload(uploadOptions, { partSize: this.settings.partSize })
-        .promise()
+      if (this.settings.disableMultiPartUpload) {
+        await this._getClientForBucket(bucketName, computeChecksums)
+          .putObject(uploadOptions)
+          .promise()
+      } else {
+        await this._getClientForBucket(bucketName, computeChecksums)
+          .upload(uploadOptions, { partSize: this.settings.partSize })
+          .promise()
+      }
     } catch (err) {
       throw PersistorHelper.wrapError(
         err,
         'upload to S3 failed',
-        { bucketName, key },
+        { bucketName, key, ifNoneMatch: opts.ifNoneMatch },
         WriteError
       )
     }
   }
 
+  /**
+   * @param {string} bucketName
+   * @param {string} key
+   * @param {Object} [opts]
+   * @param {number} [opts.start]
+   * @param {number} [opts.end]
+   * @param {boolean} [opts.autoGunzip]
+   * @param {SSECOptions} [opts.ssecOptions]
+   * @return {Promise<NodeJS.ReadableStream>}
+   */
   async getObjectStream(bucketName, key, opts) {
     opts = opts || {}
 
@@ -85,6 +180,9 @@ module.exports = class S3Persistor extends AbstractPersistor {
     if (opts.start != null && opts.end != null) {
       params.Range = `bytes=${opts.start}-${opts.end}`
     }
+    if (opts.ssecOptions) {
+      Object.assign(params, opts.ssecOptions.getGetOptions())
+    }
     const observer = new PersistorHelper.ObserverStream({
       metric: 's3.ingress', // ingress from S3 to us
       bucket: bucketName,
@@ -93,18 +191,21 @@ module.exports = class S3Persistor extends AbstractPersistor {
     const req = this._getClientForBucket(bucketName).getObject(params)
     const stream = req.createReadStream()
 
+    let contentEncoding
     try {
       await new Promise((resolve, reject) => {
-        req.on('httpHeaders', statusCode => {
+        req.on('httpHeaders', (statusCode, headers) => {
           switch (statusCode) {
             case 200: // full response
             case 206: // partial response
-              return resolve()
-            case 403: // AccessDenied is handled the same as NoSuchKey
+              contentEncoding = headers['content-encoding']
+              return resolve(undefined)
+            case 403: // AccessDenied
+              return // handled by stream.on('error') handler below
             case 404: // NoSuchKey
-              return reject(new NotFoundError())
+              return reject(new NotFoundError('not found'))
             default:
-              return reject(new Error('non success status: ' + statusCode))
+            // handled by stream.on('error') handler below
           }
         })
         // The AWS SDK is forwarding any errors from the request to the stream.
@@ -122,23 +223,32 @@ module.exports = class S3Persistor extends AbstractPersistor {
     }
     // Return a PassThrough stream with a minimal interface. It will buffer until the caller starts reading. It will emit errors from the source stream (Stream.pipeline passes errors along).
     const pass = new PassThrough()
-    pipeline(stream, observer, pass, err => {
+    const transformer = []
+    if (contentEncoding === 'gzip' && opts.autoGunzip) {
+      transformer.push(zlib.createGunzip())
+    }
+    pipeline(stream, observer, ...transformer, pass, err => {
       if (err) req.abort()
     })
     return pass
   }
 
+  /**
+   * @param {string} bucketName
+   * @param {string} key
+   * @return {Promise<string>}
+   */
   async getRedirectUrl(bucketName, key) {
     const expiresSeconds = Math.round(this.settings.signedUrlExpiryInMs / 1000)
     try {
-      const url = await this._getClientForBucket(
-        bucketName
-      ).getSignedUrlPromise('getObject', {
-        Bucket: bucketName,
-        Key: key,
-        Expires: expiresSeconds,
-      })
-      return url
+      return await this._getClientForBucket(bucketName).getSignedUrlPromise(
+        'getObject',
+        {
+          Bucket: bucketName,
+          Key: key,
+          Expires: expiresSeconds,
+        }
+      )
     } catch (err) {
       throw PersistorHelper.wrapError(
         err,
@@ -149,6 +259,12 @@ module.exports = class S3Persistor extends AbstractPersistor {
     }
   }
 
+  /**
+   * @param {string} bucketName
+   * @param {string} key
+   * @param {string} [continuationToken]
+   * @return {Promise<void>}
+   */
   async deleteDirectory(bucketName, key, continuationToken) {
     let response
     const options = { Bucket: bucketName, Prefix: key }
@@ -169,8 +285,8 @@ module.exports = class S3Persistor extends AbstractPersistor {
       )
     }
 
-    const objects = response.Contents.map(item => ({ Key: item.Key }))
-    if (objects.length) {
+    const objects = response.Contents?.map(item => ({ Key: item.Key || '' }))
+    if (objects?.length) {
       try {
         await this._getClientForBucket(bucketName)
           .deleteObjects({
@@ -200,12 +316,22 @@ module.exports = class S3Persistor extends AbstractPersistor {
     }
   }
 
-  async getObjectSize(bucketName, key) {
+  /**
+   * @param {string} bucketName
+   * @param {string} key
+   * @param {Object} opts
+   * @param {SSECOptions} [opts.ssecOptions]
+   * @return {Promise<S3.HeadObjectOutput>}
+   */
+  async #headObject(bucketName, key, opts = {}) {
+    const params = { Bucket: bucketName, Key: key }
+    if (opts.ssecOptions) {
+      Object.assign(params, opts.ssecOptions.getGetOptions())
+    }
     try {
-      const response = await this._getClientForBucket(bucketName)
-        .headObject({ Bucket: bucketName, Key: key })
+      return await this._getClientForBucket(bucketName)
+        .headObject(params)
         .promise()
-      return response.ContentLength
     } catch (err) {
       throw PersistorHelper.wrapError(
         err,
@@ -216,19 +342,51 @@ module.exports = class S3Persistor extends AbstractPersistor {
     }
   }
 
-  async getObjectMd5Hash(bucketName, key) {
+  /**
+   * @param {string} bucketName
+   * @param {string} key
+   * @param {Object} opts
+   * @param {SSECOptions} [opts.ssecOptions]
+   * @return {Promise<number>}
+   */
+  async getObjectSize(bucketName, key, opts = {}) {
+    const response = await this.#headObject(bucketName, key, opts)
+    return response.ContentLength || 0
+  }
+
+  /**
+   * @param {string} bucketName
+   * @param {string} key
+   * @param {Object} opts
+   * @param {SSECOptions} [opts.ssecOptions]
+   * @return {Promise<string | undefined>}
+   */
+  async getObjectStorageClass(bucketName, key, opts = {}) {
+    const response = await this.#headObject(bucketName, key, opts)
+    return response.StorageClass
+  }
+
+  /**
+   * @param {string} bucketName
+   * @param {string} key
+   * @param {Object} opts
+   * @param {SSECOptions} [opts.ssecOptions]
+   * @param {boolean} [opts.etagIsNotMD5]
+   * @return {Promise<string>}
+   */
+  async getObjectMd5Hash(bucketName, key, opts = {}) {
     try {
-      const response = await this._getClientForBucket(bucketName)
-        .headObject({ Bucket: bucketName, Key: key })
-        .promise()
-      const md5 = S3Persistor._md5FromResponse(response)
-      if (md5) {
-        return md5
+      if (!opts.etagIsNotMD5) {
+        const response = await this.#headObject(bucketName, key, opts)
+        const md5 = S3Persistor._md5FromResponse(response)
+        if (md5) {
+          return md5
+        }
       }
       // etag is not in md5 format
       Metrics.inc('s3.md5Download')
       return await PersistorHelper.calculateStreamMd5(
-        await this.getObjectStream(bucketName, key)
+        await this.getObjectStream(bucketName, key, opts)
       )
     } catch (err) {
       throw PersistorHelper.wrapError(
@@ -240,6 +398,11 @@ module.exports = class S3Persistor extends AbstractPersistor {
     }
   }
 
+  /**
+   * @param {string} bucketName
+   * @param {string} key
+   * @return {Promise<void>}
+   */
   async deleteObject(bucketName, key) {
     try {
       await this._getClientForBucket(bucketName)
@@ -256,12 +419,27 @@ module.exports = class S3Persistor extends AbstractPersistor {
     }
   }
 
-  async copyObject(bucketName, sourceKey, destKey) {
+  /**
+   * @param {string} bucketName
+   * @param {string} sourceKey
+   * @param {string} destKey
+   * @param {Object} opts
+   * @param {SSECOptions} [opts.ssecSrcOptions]
+   * @param {SSECOptions} [opts.ssecOptions]
+   * @return {Promise<void>}
+   */
+  async copyObject(bucketName, sourceKey, destKey, opts = {}) {
     const params = {
       Bucket: bucketName,
       Key: destKey,
       CopySource: `${bucketName}/${sourceKey}`,
     }
+    if (opts.ssecSrcOptions) {
+      Object.assign(params, opts.ssecSrcOptions.getCopyOptions())
+    }
+    if (opts.ssecOptions) {
+      Object.assign(params, opts.ssecOptions.getPutOptions())
+    }
     try {
       await this._getClientForBucket(bucketName).copyObject(params).promise()
     } catch (err) {
@@ -274,9 +452,16 @@ module.exports = class S3Persistor extends AbstractPersistor {
     }
   }
 
-  async checkIfObjectExists(bucketName, key) {
+  /**
+   * @param {string} bucketName
+   * @param {string} key
+   * @param {Object} opts
+   * @param {SSECOptions} [opts.ssecOptions]
+   * @return {Promise<boolean>}
+   */
+  async checkIfObjectExists(bucketName, key, opts) {
     try {
-      await this.getObjectSize(bucketName, key)
+      await this.getObjectSize(bucketName, key, opts)
       return true
     } catch (err) {
       if (err instanceof NotFoundError) {
@@ -291,6 +476,12 @@ module.exports = class S3Persistor extends AbstractPersistor {
     }
   }
 
+  /**
+   * @param {string} bucketName
+   * @param {string} key
+   * @param {string} [continuationToken]
+   * @return {Promise<number>}
+   */
   async directorySize(bucketName, key, continuationToken) {
     try {
       const options = {
@@ -304,7 +495,8 @@ module.exports = class S3Persistor extends AbstractPersistor {
         .listObjectsV2(options)
         .promise()
 
-      const size = response.Contents.reduce((acc, item) => item.Size + acc, 0)
+      const size =
+        response.Contents?.reduce((acc, item) => (item.Size || 0) + acc, 0) || 0
       if (response.IsTruncated) {
         return (
           size +
@@ -326,15 +518,38 @@ module.exports = class S3Persistor extends AbstractPersistor {
     }
   }
 
-  _getClientForBucket(bucket, clientOptions) {
-    return new S3(
-      this._buildClientOptions(
-        this.settings.bucketCreds?.[bucket],
-        clientOptions
+  /**
+   * @param {string} bucket
+   * @param {boolean} computeChecksums
+   * @return {S3}
+   * @private
+   */
+  _getClientForBucket(bucket, computeChecksums = false) {
+    /** @type {S3.Types.ClientConfiguration} */
+    const clientOptions = {}
+    const cacheKey = `${bucket}:${computeChecksums}`
+    if (computeChecksums) {
+      clientOptions.computeChecksums = true
+    }
+    let client = this.#clients.get(cacheKey)
+    if (!client) {
+      client = new S3(
+        this._buildClientOptions(
+          this.settings.bucketCreds?.[bucket],
+          clientOptions
+        )
       )
-    )
+      this.#clients.set(cacheKey, client)
+    }
+    return client
   }
 
+  /**
+   * @param {Object} bucketCredentials
+   * @param {S3.Types.ClientConfiguration} clientOptions
+   * @return {S3.Types.ClientConfiguration}
+   * @private
+   */
   _buildClientOptions(bucketCredentials, clientOptions) {
     const options = clientOptions || {}
 
@@ -356,7 +571,7 @@ module.exports = class S3Persistor extends AbstractPersistor {
     if (this.settings.endpoint) {
       const endpoint = new URL(this.settings.endpoint)
       options.endpoint = this.settings.endpoint
-      options.sslEnabled = endpoint.protocol === 'https'
+      options.sslEnabled = endpoint.protocol === 'https:'
     }
 
     // path-style access is only used for acceptance tests
@@ -370,9 +585,22 @@ module.exports = class S3Persistor extends AbstractPersistor {
       }
     }
 
+    if (options.sslEnabled && this.settings.ca && !options.httpOptions?.agent) {
+      options.httpOptions = options.httpOptions || {}
+      options.httpOptions.agent = new https.Agent({
+        rejectUnauthorized: true,
+        ca: this.settings.ca,
+      })
+    }
+
     return options
   }
 
+  /**
+   * @param {S3.HeadObjectOutput} response
+   * @return {string|null}
+   * @private
+   */
   static _md5FromResponse(response) {
     const md5 = (response.ETag || '').replace(/[ "]/g, '')
     if (!md5.match(/^[a-f0-9]{32}$/)) {
@@ -382,3 +610,8 @@ module.exports = class S3Persistor extends AbstractPersistor {
     return md5
   }
 }
+
+module.exports = {
+  S3Persistor,
+  SSECOptions,
+}

libraries/object-persistor/test/Init.js

@@ -25,4 +25,9 @@ SandboxedModule.configure({
     },
   },
   globals: { Buffer, Math, console, process, URL },
+  sourceTransformers: {
+    removeNodePrefix: function (source) {
+      return source.replace(/require\(['"]node:/g, "require('")
+    },
+  },
 })

libraries/object-persistor/test/unit/FSPersistorTests.js

@@ -1,10 +1,10 @@
-const crypto = require('crypto')
+const crypto = require('node:crypto')
 const { expect } = require('chai')
 const mockFs = require('mock-fs')
-const fs = require('fs')
-const fsPromises = require('fs/promises')
-const Path = require('path')
-const StreamPromises = require('stream/promises')
+const fs = require('node:fs')
+const fsPromises = require('node:fs/promises')
+const Path = require('node:path')
+const StreamPromises = require('node:stream/promises')
 const SandboxedModule = require('sandboxed-module')
 const Errors = require('../../src/Errors')
 

libraries/object-persistor/test/unit/GcsPersistorTests.js

@@ -1,4 +1,4 @@
-const { EventEmitter } = require('events')
+const { EventEmitter } = require('node:events')
 const sinon = require('sinon')
 const chai = require('chai')
 const { expect } = chai
@@ -45,11 +45,11 @@ describe('GcsPersistorTests', function () {
 
     files = [
       {
-        metadata: { size: 11, md5Hash: '/////wAAAAD/////AAAAAA==' },
+        metadata: { size: '11', md5Hash: '/////wAAAAD/////AAAAAA==' },
         delete: sinon.stub(),
       },
       {
-        metadata: { size: 22, md5Hash: '/////wAAAAD/////AAAAAA==' },
+        metadata: { size: '22', md5Hash: '/////wAAAAD/////AAAAAA==' },
         delete: sinon.stub(),
       },
     ]
@@ -63,7 +63,7 @@ describe('GcsPersistorTests', function () {
 
       read() {
         if (this.err) return this.emit('error', this.err)
-        this.emit('response', { statusCode: this.statusCode })
+        this.emit('response', { statusCode: this.statusCode, headers: {} })
       }
     }
 
@@ -302,7 +302,7 @@ describe('GcsPersistorTests', function () {
       })
 
       it('should return the object size', function () {
-        expect(size).to.equal(files[0].metadata.size)
+        expect(size).to.equal(11)
       })
 
       it('should pass the bucket and key to GCS', function () {

libraries/object-persistor/test/unit/PersistorFactoryTests.js

@@ -1,7 +1,7 @@
 const chai = require('chai')
 const { expect } = chai
 const SandboxedModule = require('sandboxed-module')
-const StreamPromises = require('stream/promises')
+const StreamPromises = require('node:stream/promises')
 
 const MODULE_PATH = '../../src/PersistorFactory.js'
 
@@ -32,7 +32,7 @@ describe('PersistorManager', function () {
     Settings = {}
     const requires = {
       './GcsPersistor': GcsPersistor,
-      './S3Persistor': S3Persistor,
+      './S3Persistor': { S3Persistor },
       './FSPersistor': FSPersistor,
       '@overleaf/logger': {
         info() {},

libraries/object-persistor/test/unit/S3PersistorTests.js

@@ -3,7 +3,7 @@ const chai = require('chai')
 const { expect } = chai
 const SandboxedModule = require('sandboxed-module')
 const Errors = require('../../src/Errors')
-const { EventEmitter } = require('events')
+const { EventEmitter } = require('node:events')
 
 const MODULE_PATH = '../../src/S3Persistor.js'
 
@@ -91,8 +91,22 @@ describe('S3PersistorTests', function () {
 
       createReadStream() {
         setTimeout(() => {
+          if (this.notFoundSSEC) {
+            // special case for AWS S3: 404 NoSuchKey wrapped in a 400. A single request received a single response, and multiple httpHeaders events are triggered. Don't ask.
+            this.emit('httpHeaders', 400, {})
+            this.emit('httpHeaders', 404, {})
+            ReadStream.emit('error', S3NotFoundError)
+            return
+          }
+
           if (this.err) return ReadStream.emit('error', this.err)
-          this.emit('httpHeaders', this.statusCode)
+          this.emit('httpHeaders', this.statusCode, {})
+          if (this.statusCode === 403) {
+            ReadStream.emit('error', S3AccessDeniedError)
+          }
+          if (this.statusCode === 404) {
+            ReadStream.emit('error', S3NotFoundError)
+          }
         })
         return ReadStream
       }
@@ -133,7 +147,7 @@ describe('S3PersistorTests', function () {
       deleteObjects: sinon.stub().returns(EmptyPromise),
       getSignedUrlPromise: sinon.stub().resolves(redirectUrl),
     }
-    S3 = sinon.stub().returns(S3Client)
+    S3 = sinon.stub().callsFake(() => Object.assign({}, S3Client))
 
     Hash = {
       end: sinon.stub(),
@@ -159,7 +173,7 @@ describe('S3PersistorTests', function () {
         crypto,
       },
       globals: { console, Buffer },
-    }))(settings)
+    }).S3Persistor)(settings)
   })
 
   describe('getObjectStream', function () {
@@ -338,6 +352,34 @@ describe('S3PersistorTests', function () {
       })
     })
 
+    describe("when the file doesn't exist -- SSEC", function () {
+      let error, stream
+
+      beforeEach(async function () {
+        S3GetObjectRequest.notFoundSSEC = 404
+        try {
+          stream = await S3Persistor.getObjectStream(bucket, key)
+        } catch (err) {
+          error = err
+        }
+      })
+      it('does not return a stream', function () {
+        expect(stream).not.to.exist
+      })
+
+      it('throws a NotFoundError', function () {
+        expect(error).to.be.an.instanceOf(Errors.NotFoundError)
+      })
+
+      it('wraps the error', function () {
+        expect(error.cause).to.exist
+      })
+
+      it('stores the bucket and key in the error', function () {
+        expect(error.info).to.include({ bucketName: bucket, key })
+      })
+    })
+
     describe('when access to the file is denied', function () {
       let error, stream
 
@@ -359,7 +401,7 @@ describe('S3PersistorTests', function () {
       })
 
       it('wraps the error', function () {
-        expect(error.cause).to.exist
+        expect(error.cause).to.equal(S3AccessDeniedError)
       })
 
       it('stores the bucket and key in the error', function () {
@@ -985,4 +1027,22 @@ describe('S3PersistorTests', function () {
       })
     })
   })
+
+  describe('_getClientForBucket', function () {
+    it('should return same instance for same bucket', function () {
+      const a = S3Persistor._getClientForBucket('foo')
+      const b = S3Persistor._getClientForBucket('foo')
+      expect(a).to.equal(b)
+    })
+    it('should return different instance for different bucket', function () {
+      const a = S3Persistor._getClientForBucket('foo')
+      const b = S3Persistor._getClientForBucket('bar')
+      expect(a).to.not.equal(b)
+    })
+    it('should return different instance for same bucket different computeChecksums', function () {
+      const a = S3Persistor._getClientForBucket('foo', false)
+      const b = S3Persistor._getClientForBucket('foo', true)
+      expect(a).to.not.equal(b)
+    })
+  })
 })

libraries/overleaf-editor-core/.nvmrc

@@ -1 +1 @@
-18.20.2
+20.18.0

libraries/overleaf-editor-core/buildscript.txt

@@ -5,6 +5,6 @@ overleaf-editor-core
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=18.20.2
+--node-version=20.18.0
 --public-repo=False
 --script-version=4.5.0

libraries/overleaf-editor-core/lib/blob.js

@@ -40,6 +40,11 @@ class Blob {
 
   static NotFoundError = NotFoundError
 
+  /**
+   * @param {string} hash
+   * @param {number} byteLength
+   * @param {number} [stringLength]
+   */
   constructor(hash, byteLength, stringLength) {
     this.setHash(hash)
     this.setByteLength(byteLength)
@@ -63,14 +68,14 @@ class Blob {
 
   /**
    * Hex hash.
-   * @return {?String}
+   * @return {String}
    */
   getHash() {
     return this.hash
   }
 
   setHash(hash) {
-    assert.maybe.match(hash, Blob.HEX_HASH_RX, 'bad hash')
+    assert.match(hash, Blob.HEX_HASH_RX, 'bad hash')
     this.hash = hash
   }
 
@@ -83,13 +88,13 @@ class Blob {
   }
 
   setByteLength(byteLength) {
-    assert.maybe.integer(byteLength, 'bad byteLength')
+    assert.integer(byteLength, 'bad byteLength')
     this.byteLength = byteLength
   }
 
   /**
    * Utf-8 length of the blob content, if it appears to be valid UTF-8.
-   * @return {?number}
+   * @return {number|undefined}
    */
   getStringLength() {
     return this.stringLength

libraries/overleaf-editor-core/lib/change_note.js

@@ -10,7 +10,7 @@ const Change = require('./change')
 class ChangeNote {
   /**
    * @param {number} baseVersion the new base version for the change
-   * @param {?Change} change
+   * @param {Change} [change]
    */
   constructor(baseVersion, change) {
     assert.integer(baseVersion, 'bad baseVersion')

libraries/overleaf-editor-core/lib/file.js

@@ -95,7 +95,7 @@ class File {
 
   /**
    * @param  {number} byteLength
-   * @param  {number?} stringLength
+   * @param  {number} [stringLength]
    * @param  {Object} [metadata]
    * @return {File}
    */

libraries/overleaf-editor-core/lib/file_data/index.js

@@ -47,7 +47,7 @@ class FileData {
 
   /** @see File.createHollow
    * @param {number} byteLength
-   * @param {number|null} stringLength
+   * @param {number} [stringLength]
    */
   static createHollow(byteLength, stringLength) {
     if (stringLength == null) {
@@ -63,20 +63,14 @@ class FileData {
    */
   static createLazyFromBlobs(blob, rangesBlob) {
     assert.instance(blob, Blob, 'FileData: bad blob')
-    if (blob.getStringLength() == null) {
-      return new BinaryFileData(
-        // TODO(das7pad): see call-sites
-        // @ts-ignore
-        blob.getHash(),
-        blob.getByteLength()
-      )
+    const stringLength = blob.getStringLength()
+    if (stringLength == null) {
+      return new BinaryFileData(blob.getHash(), blob.getByteLength())
     }
     return new LazyStringFileData(
-      // TODO(das7pad): see call-sites
-      // @ts-ignore
       blob.getHash(),
       rangesBlob?.getHash(),
-      blob.getStringLength()
+      stringLength
     )
   }
 

libraries/overleaf-editor-core/lib/file_data/string_file_data.js

@@ -142,12 +142,8 @@ class StringFileData extends FileData {
         trackedChanges: this.trackedChanges.toRaw(),
       }
       const rangesBlob = await blobStore.putObject(ranges)
-      // TODO(das7pad): Provide interface that guarantees hash exists?
-      // @ts-ignore
       return { hash: blob.getHash(), rangesHash: rangesBlob.getHash() }
     }
-    // TODO(das7pad): Provide interface that guarantees hash exists?
-    // @ts-ignore
     return { hash: blob.getHash() }
   }
 }

libraries/overleaf-editor-core/lib/history.js

@@ -109,7 +109,7 @@ class History {
    * @param {BlobStore} blobStore
    * @param {number} [concurrency] applies separately to files, changes and
    *                               operations
-   * @return {Promise.<Object>}
+   * @return {Promise<import('overleaf-editor-core/lib/types').RawHistory>}
    */
   async store(blobStore, concurrency) {
     assert.maybe.number(concurrency, 'bad concurrency')

libraries/overleaf-editor-core/lib/v2_doc_versions.js

@@ -27,7 +27,7 @@ class V2DocVersions {
   }
 
   /**
-   * @return {?RawV2DocVersions}
+   * @return {RawV2DocVersions|null}
    */
   toRaw() {
     if (!this.data) return null

libraries/promise-utils/.nvmrc

@@ -1 +1 @@
-18.20.2
+20.18.0

libraries/promise-utils/buildscript.txt

@@ -5,6 +5,6 @@ promise-utils
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=18.20.2
+--node-version=20.18.0
 --public-repo=False
 --script-version=4.5.0

libraries/promise-utils/index.js

@@ -1,4 +1,4 @@
-const { promisify, callbackify } = require('util')
+const { promisify, callbackify } = require('node:util')
 const pLimit = require('p-limit')
 
 module.exports = {
@@ -253,8 +253,14 @@ function expressifyErrorHandler(fn) {
  * Map values in `array` with the async function `fn`
  *
  * Limit the number of unresolved promises to `concurrency`.
+ * @template T
+ * @template V
+ * @param {number} concurrency
+ * @param {Array<T>} array
+ * @param {(arg: T) => Promise<V>} fn
+ * @return {Promise<Array<Awaited<V>>>}
  */
-function promiseMapWithLimit(concurrency, array, fn) {
+async function promiseMapWithLimit(concurrency, array, fn) {
   const limit = pLimit(concurrency)
-  return Promise.all(array.map(x => limit(() => fn(x))))
+  return await Promise.all(array.map(x => limit(() => fn(x))))
 }

libraries/ranges-tracker/.nvmrc

@@ -1 +1 @@
-18.20.2
+20.18.0

libraries/ranges-tracker/buildscript.txt

@@ -5,6 +5,6 @@ ranges-tracker
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=18.20.2
+--node-version=20.18.0
 --public-repo=False
 --script-version=4.5.0

libraries/redis-wrapper/.nvmrc

@@ -1 +1 @@
-18.20.2
+20.18.0

libraries/redis-wrapper/RedisLocker.js

@@ -1,8 +1,8 @@
-const { promisify } = require('util')
+const { promisify } = require('node:util')
 const metrics = require('@overleaf/metrics')
 const logger = require('@overleaf/logger')
-const os = require('os')
-const crypto = require('crypto')
+const os = require('node:os')
+const crypto = require('node:crypto')
 
 const HOST = os.hostname()
 const PID = process.pid

libraries/redis-wrapper/RedisWebLocker.js

@@ -1,8 +1,8 @@
-const { callbackify, promisify } = require('util')
+const { callbackify, promisify } = require('node:util')
 const metrics = require('@overleaf/metrics')
 const logger = require('@overleaf/logger')
-const os = require('os')
-const crypto = require('crypto')
+const os = require('node:os')
+const crypto = require('node:crypto')
 const async = require('async')
 
 const HOST = os.hostname()

libraries/redis-wrapper/buildscript.txt

@@ -5,6 +5,6 @@ redis-wrapper
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=18.20.2
+--node-version=20.18.0
 --public-repo=False
 --script-version=4.5.0

libraries/redis-wrapper/index.js

@@ -1,6 +1,6 @@
-const crypto = require('crypto')
-const os = require('os')
-const { promisify } = require('util')
+const crypto = require('node:crypto')
+const os = require('node:os')
+const { promisify } = require('node:util')
 
 const Redis = require('ioredis')
 

libraries/redis-wrapper/test/setup.js

@@ -0,0 +1,9 @@
+const SandboxedModule = require('sandboxed-module')
+
+SandboxedModule.configure({
+  sourceTransformers: {
+    removeNodePrefix: function (source) {
+      return source.replace(/require\(['"]node:/g, "require('")
+    },
+  },
+})

libraries/redis-wrapper/test/unit/src/test.js

@@ -6,8 +6,8 @@
  */
 require('chai').should()
 const SandboxedModule = require('sandboxed-module')
-const assert = require('assert')
-const path = require('path')
+const assert = require('node:assert')
+const path = require('node:path')
 const sinon = require('sinon')
 const modulePath = path.join(__dirname, './../../../index.js')
 const redisLockerModulePath = path.join(__dirname, './../../../RedisLocker.js')

libraries/settings/.nvmrc

@@ -1 +1 @@
-18.20.2
+20.18.0

libraries/settings/Settings.js

@@ -1,6 +1,6 @@
 /* eslint-disable no-console */
-const fs = require('fs')
-const Path = require('path')
+const fs = require('node:fs')
+const Path = require('node:path')
 const { merge } = require('./merge')
 
 const CWD = process.cwd()

libraries/settings/buildscript.txt

@@ -5,6 +5,6 @@ settings
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=18.20.2
+--node-version=20.18.0
 --public-repo=False
 --script-version=4.5.0

libraries/stream-utils/.nvmrc

@@ -1 +1 @@
-18.20.2
+20.18.0

libraries/stream-utils/buildscript.txt

@@ -5,6 +5,6 @@ stream-utils
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=18.20.2
+--node-version=20.18.0
 --public-repo=False
 --script-version=4.5.0

libraries/stream-utils/index.js

@@ -1,8 +1,8 @@
-const { Writable, Readable, PassThrough, Transform } = require('stream')
+const { Writable, Readable, PassThrough, Transform } = require('node:stream')
 
 /**
  * A writable stream that stores all data written to it in a node Buffer.
- * @extends stream.Writable
+ * @extends Writable
  * @example
  * const { WritableBuffer } = require('@overleaf/stream-utils')
  * const bufferStream = new WritableBuffer()
@@ -43,7 +43,7 @@ class WritableBuffer extends Writable {
 
 /**
  * A readable stream created from a string.
- * @extends stream.Readable
+ * @extends Readable
  * @example
  * const { ReadableString } = require('@overleaf/stream-utils')
  * const stringStream = new ReadableString('hello world')
@@ -66,7 +66,7 @@ class SizeExceededError extends Error {}
 
 /**
  * Limited size stream which will emit a SizeExceededError if the size is exceeded
- * @extends stream.Transform
+ * @extends Transform
  */
 class LimitedStream extends Transform {
   constructor(maxSize) {
@@ -93,7 +93,7 @@ class AbortError extends Error {}
 
 /**
  * TimeoutStream which will emit an AbortError if it exceeds a user specified timeout
- * @extends stream.PassThrough
+ * @extends PassThrough
  */
 class TimeoutStream extends PassThrough {
   constructor(timeout) {
@@ -111,7 +111,7 @@ class TimeoutStream extends PassThrough {
 
 /**
  * LoggerStream which will call the provided logger function when the stream exceeds a user specified limit. It will call the provided function again when flushing the stream and it exceeded the user specified limit before.
- * @extends stream.Transform
+ * @extends Transform
  */
 class LoggerStream extends Transform {
   /**

package.json

@@ -5,6 +5,8 @@
     "patch-package": "^8.0.0"
   },
   "devDependencies": {
+    "@types/chai": "^4.3.0",
+    "@types/chai-as-promised": "^7.1.8",
     "@types/mocha": "^10.0.6",
     "@typescript-eslint/eslint-plugin": "^8.0.0",
     "@typescript-eslint/parser": "^8.0.0",
@@ -19,6 +21,7 @@
     "eslint-plugin-node": "^11.1.0",
     "eslint-plugin-prettier": "^4.0.0",
     "eslint-plugin-promise": "^6.0.0",
+    "eslint-plugin-unicorn": "^56.0.0",
     "prettier": "3.3.3",
     "typescript": "^5.5.4"
   },
@@ -63,7 +66,6 @@
     "services/project-history",
     "services/real-time",
     "services/references",
-    "services/spelling",
     "services/templates",
     "services/third-party-datastore",
     "services/third-party-references",

server-ce/.eslintrc

@@ -4,6 +4,9 @@
     "standard",
     "prettier"
   ],
+  "plugins": [
+    "unicorn"
+  ],
   "parserOptions": {
     "ecmaVersion": 2020
   },
@@ -12,7 +15,8 @@
   },
   "rules": {
     // Do not allow importing of implicit dependencies.
-    "import/no-extraneous-dependencies": "error"
+    "import/no-extraneous-dependencies": "error",
+    "unicorn/prefer-node-protocol": "error"
   },
   "overrides": [
     // Extra rules for Cypress tests

server-ce/Dockerfile-base

@@ -2,7 +2,7 @@
 # Overleaf Base Image (sharelatex/sharelatex-base)
 # --------------------------------------------------
 
-FROM phusion/baseimage:jammy-1.0.4
+FROM phusion/baseimage:noble-1.0.0
 
 # Makes sure LuaTex cache is writable
 # -----------------------------------
@@ -25,13 +25,12 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
       unattended-upgrades \
       build-essential wget net-tools unzip time imagemagick optipng strace nginx git python3 python-is-python3 zlib1g-dev libpcre3-dev gettext-base libwww-perl ca-certificates curl gnupg \
       qpdf \
-      aspell aspell-en aspell-af aspell-am aspell-ar aspell-ar-large aspell-bg aspell-bn aspell-br aspell-ca aspell-cs aspell-cy aspell-da aspell-de aspell-de-1901 aspell-el aspell-eo aspell-es aspell-et aspell-eu-es aspell-fa aspell-fo aspell-fr aspell-ga aspell-gl-minimos aspell-gu aspell-he aspell-hi aspell-hr aspell-hsb aspell-hu aspell-hy aspell-id aspell-is aspell-it aspell-kk aspell-kn aspell-ku aspell-lt aspell-lv aspell-ml aspell-mr aspell-nl aspell-no aspell-nr aspell-ns  aspell-pa aspell-pl aspell-pt aspell-pt-br aspell-ro aspell-ru aspell-sk aspell-sl aspell-ss aspell-st aspell-sv aspell-tl aspell-tn aspell-ts aspell-uk aspell-uz aspell-xh aspell-zu \
 # upgrade base-image, batch all the upgrades together, rather than installing them on-by-one (which is slow!)
 &&  unattended-upgrade --verbose --no-minimal-upgrade-steps \
 # install Node.js https://github.com/nodesource/distributions#nodejs
 &&  mkdir -p /etc/apt/keyrings \
 &&  curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
-&&  echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list \
+&&  echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list \
 &&  apt-get update \
 &&  apt-get install -y nodejs \
     \

server-ce/bin/grunt

@@ -11,22 +11,22 @@ cd /overleaf/services/web
 case "$TASK" in
   user:create-admin)
     echo "The grunt command is deprecated, run the create-user script using node instead"
-    node modules/server-ce-scripts/scripts/create-user --admin "$@"
+    node modules/server-ce-scripts/scripts/create-user.mjs --admin "$@"
     ;;
 
   user:delete)
     echo "The grunt command is deprecated, run the delete-user script using node instead"
-    node modules/server-ce-scripts/scripts/delete-user "$@"
+    node modules/server-ce-scripts/scripts/delete-user.mjs "$@"
     ;;
 
   check:mongo)
     echo "The grunt command is deprecated, run the check-mongodb script using node instead"
-    node modules/server-ce-scripts/scripts/check-mongodb
+    node modules/server-ce-scripts/scripts/check-mongodb.mjs
     ;;
 
   check:redis)
     echo "The grunt command is deprecated, run the check-redis script using node instead"
-    node modules/server-ce-scripts/scripts/check-redis
+    node modules/server-ce-scripts/scripts/check-redis.mjs
     ;;
 
   *)

server-ce/config/env.sh

@@ -9,6 +9,5 @@ export HISTORY_V1_HOST=127.0.0.1
 export NOTIFICATIONS_HOST=127.0.0.1
 export PROJECT_HISTORY_HOST=127.0.0.1
 export REALTIME_HOST=127.0.0.1
-export SPELLING_HOST=127.0.0.1
 export WEB_HOST=127.0.0.1
 export WEB_API_HOST=127.0.0.1

server-ce/config/settings.js

@@ -212,6 +212,11 @@ const settings = {
     enabled: process.env.OVERLEAF_CSP_ENABLED !== 'false',
   },
 
+  rateLimit: {
+    subnetRateLimiterDisabled:
+      process.env.SUBNET_RATE_LIMITER_DISABLED !== 'false',
+  },
+
   // These credentials are used for authenticating api requests
   // between services that may need to go over public channels
   httpAuthUsers,
@@ -326,6 +331,10 @@ if (process.env.OVERLEAF_HEADER_EXTRAS != null) {
   }
 }
 
+if (process.env.OVERLEAF_LOGIN_SUPPORT_TEXT != null) {
+  settings.nav.login_support_text = process.env.OVERLEAF_LOGIN_SUPPORT_TEXT
+}
+
 // Sending Email
 // -------------
 //

server-ce/hotfix/5.2.1/Dockerfile

@@ -0,0 +1,5 @@
+FROM sharelatex/sharelatex:5.2.0
+
+# Subnet rate limiter fix
+COPY pr_21327.patch /
+RUN cd / && patch -p0 < pr_21327.patch && rm pr_21327.patch

server-ce/hotfix/5.2.1/pr_21327.patch

@@ -0,0 +1,36 @@
+--- overleaf/services/web/app/src/infrastructure/RateLimiter.js
++++ overleaf/services/web/app/src/infrastructure/RateLimiter.js
+@@ -39,7 +39,7 @@ class RateLimiter {
+       keyPrefix: `rate-limit:${name}`,
+       storeClient: rclient,
+     })
+-    if (opts.subnetPoints) {
++    if (opts.subnetPoints && !Settings.rateLimit?.subnetRateLimiterDisabled) {
+       this._subnetRateLimiter = new RateLimiterFlexible.RateLimiterRedis({
+         ...opts,
+         points: opts.subnetPoints,
+--- overleaf/services/web/config/settings.defaults.js
++++ overleaf/services/web/config/settings.defaults.js
+@@ -777,6 +777,8 @@ module.exports = {
+   reloadModuleViewsOnEachRequest: process.env.NODE_ENV === 'development',
+
+   rateLimit: {
++    subnetRateLimiterDisabled:
++      process.env.SUBNET_RATE_LIMITER_DISABLED === 'true',
+     autoCompile: {
+       everyone: process.env.RATE_LIMIT_AUTO_COMPILE_EVERYONE || 100,
+       standard: process.env.RATE_LIMIT_AUTO_COMPILE_STANDARD || 25,
+--- etc/overleaf/settings.js
++++ etc/overleaf/settings.js
+@@ -212,6 +212,11 @@ const settings = {
+     enabled: process.env.OVERLEAF_CSP_ENABLED !== 'false',
+   },
+
++  rateLimit: {
++    subnetRateLimiterDisabled:
++      process.env.SUBNET_RATE_LIMITER_DISABLED !== 'false',
++  },
++
+   // These credentials are used for authenticating api requests
+   // between services that may need to go over public channels
+   httpAuthUsers,

server-ce/init_preshutdown_scripts/00_close_site

@@ -12,12 +12,12 @@ echo "closed" > "${SITE_MAINTENANCE_FILE}"
 sleep 5
 
 # giving a grace period of 5 seconds for users before disconnecting them and start shutting down
-cd /overleaf/services/web && node scripts/disconnect_all_users.js --delay-in-seconds=5 >> /var/log/overleaf/web.log 2>&1
+cd /overleaf/services/web && node scripts/disconnect_all_users.mjs --delay-in-seconds=5 >> /var/log/overleaf/web.log 2>&1
 
 EXIT_CODE="$?"
 if [ $EXIT_CODE -ne 0 ]
 then
-  echo "scripts/disconnect_all_users.js failed with exit code $EXIT_CODE"
+  echo "scripts/disconnect_all_users.mjs failed with exit code $EXIT_CODE"
   exit 1
 fi