README for extended CE

- Replace free-text license input with a select box
- Improve visual presentation of modals and enhance keyboard interaction

.editorconfig

@@ -1,25 +0,0 @@
-root = true
-
-[*]
-charset = utf-8
-indent_style = space
-indent_size = 2
-end_of_line = lf
-insert_final_newline = true
-trim_trailing_whitespace = true
-
-[Makefile]
-indent_style = tab
-
-[*.go]
-indent_style = tab
-
-[*.{pug,coffee}]
-indent_style = tab
-
-[*.{pug,patch}]
-trim_trailing_whitespace = false
-
-[Jenkinsfile]
-insert_final_newline = false
-max_line_length = off

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,44 @@
+<!--
+
+Note: If you are using www.overleaf.com and have a problem, 
+      or if you would like to request a new feature please contact
+      the support team at support@overleaf.com
+      
+      This form should only be used to report bugs in the 
+      Community Edition release of Overleaf.
+
+-->
+
+
+
+<!-- BUG REPORT TEMPLATE -->
+
+## Steps to Reproduce
+<!-- Describe the steps leading up to when / where you found the bug. -->
+<!-- Screenshots may be helpful here. -->
+
+1.
+2.
+3.
+
+## Expected Behaviour
+<!-- What should have happened when you completed the steps above? -->
+
+## Observed Behaviour
+<!-- What actually happened when you completed the steps above? -->
+<!-- Screenshots may be helpful here. -->
+
+## Context
+<!-- How has this issue affected you? What were you trying to accomplish? -->
+
+## Technical Info
+<!-- Provide any technical details that may be applicable (or N/A if not applicable). -->
+
+* URL:
+* Browser Name and version:
+* Operating System and version (desktop or mobile):
+* Signed in as:
+* Project and/or file:
+
+## Analysis
+<!--- Optionally, document investigation of / suggest a fix for the bug, e.g. 'comes from this line / commit' -->

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,49 +0,0 @@
----
-name: Bug report
-about: Report a bug
-title: ''
-labels: type:bug
-assignees: ''
-
----
-
-<!--
-
-Note: This form should only be used to report bugs in the
-      Extended CE release of Overleaf.
-
--->
-
-
-
-<!-- BUG REPORT TEMPLATE -->
-
-## Steps to Reproduce
-<!-- Describe the steps leading up to when / where you found the bug. -->
-<!-- Screenshots may be helpful here. -->
-
-1.
-2.
-3.
-
-## Expected Behaviour
-<!-- What should have happened when you completed the steps above? -->
-
-## Observed Behaviour
-<!-- What actually happened when you completed the steps above? -->
-<!-- Screenshots may be helpful here. -->
-
-## Context
-<!-- How has this issue affected you? What were you trying to accomplish? -->
-
-## Technical Info
-<!-- Provide any technical details that may be applicable (or N/A if not applicable). -->
-
-* URL:
-* Browser Name and version:
-* Operating System and version (desktop or mobile):
-* Signed in as:
-* Project and/or file:
-
-## Analysis
-<!--- Optionally, document investigation of / suggest a fix for the bug, e.g. 'comes from this line / commit' -->

.github/PULL_REQUEST_TEMPLATE.md

@@ -4,3 +4,8 @@
 
 ## Related issues / Pull Requests
 <!-- Fixes #xyz, Contributes to #xyz, Related to #xyz-->
+
+
+## Contributor Agreement
+
+- [ ] I confirm I have signed the [Contributor License Agreement](https://github.com/overleaf/overleaf/blob/master/CONTRIBUTING.md#contributor-license-agreement)

CONTRIBUTING.md

@@ -1,33 +1,43 @@
-Contributing to Overleaf CE+
-============================
+Contributing to Overleaf
+==========================
 
 Thank you for reading this! If you'd like to report a bug or join in the development
-of Overleaf CE+, then here are some notes on how to do that.
+of Overleaf, then here are some notes on how to do that.
 
 Reporting bugs and opening issues
 ---------------------------------
 
-If you'd like to report a bug or open an issue, please **[check if there is an existing issue](https://github.com/yu-i-i/overleaf-cep).**
+If you'd like to report a bug or open an issue, please **[check if there is an existing issue](https://github.com/overleaf/overleaf/issues).**
 If there is then please add any more information that you have, or give it a 👍.
 
 When submitting an issue please describe the issue as clearly as possible, including how to
 reproduce the bug, which situations it appears in, what you expected to happen, and what actually happens.
 If you can include a screenshot for front end issues that is very helpful.
 
+**Note**: If you are using [www.overleaf.com](www.overleaf.com) and have a problem, or if you would like to request a new feature, please contact the Support team at support@overleaf.com. Raise an issue here only to report bugs in the Community Edition release of Overleaf.
 
 Pull Requests
 -------------
 
-See [Overleaf wiki](https://github.com/overleaf/overleaf/wiki)
+See [our wiki](https://github.com/overleaf/overleaf/wiki)
 for how to manage the Overleaf development environment and for our developer guidelines.
 
 We love pull requests, so be bold with them! Don't be afraid of going ahead
 and changing something, or adding a new feature. We're very happy to work with you
 to get your changes merged into Overleaf.
 
-If you're looking for something to work on, have a look at the [open issues](https://github.com/yu-i-i/overleaf-cep/issues).
+If you're looking for something to work on, have a look at the [open issues](https://github.com/overleaf/overleaf/issues).
 
 Security
 --------
 
-Please see [Overleaf security policy](https://github.com/overleaf/overleaf/security/policy) if you would like to report a potential security vulnerability.
+Please see [our security policy](https://github.com/overleaf/overleaf/security/policy) if you would like to report a potential security vulnerability.
+
+Contributor License Agreement
+-----------------------------
+
+Before we can accept any contributions of code, we need you to agree to our
+[Contributor License Agreement](https://docs.google.com/forms/d/e/1FAIpQLSef79XH3mb7yIiMzZw-yALEegS-wyFetvjTiNBfZvf_IHD2KA/viewform?usp=sf_link).
+This is to ensure that you own the copyright of your contribution, and that you
+agree to give us a license to use it in both the open source version, and the version
+of Overleaf running at www.overleaf.com, which may have additional changes.

CREDITS.md

@@ -1,70 +0,0 @@
-# CREDITS
-
-This project includes ideas and adapted code from several open-source projects.\
-In most cases the original code has been modified, optimized, or extended.
-
-## Symbol palette
-
-The symbol palette feature is based on the original Overleaf implementation:
-
-https://github.com/overleaf/web/tree/master/frontend/js/features/symbol-palette
-
-The original code was slightly improved, particularly in the parts related to keyboard input.
-
-## LDAP authentication
-
-LDAP authentication was inspired by:
-
-https://github.com/smhaller/ldap-overleaf-sl
-
-The project provided the idea and part of the implementation for adding LDAP users to the user's contacts.
-
-## Autocomplete of reference keys
-
-Autocomplete of reference keys is based on:
-
-https://github.com/lcpu-club/overleaf
-
-The implementation largely follows the referenced code, except that a different `.bib` file parser is used.
-
-## Real-time track changes and comments
-
-The Track Changes and Comments feature exists largely in the original Overleaf codebase.\
-The missing parts were implemented based on:
-
-https://github.com/ertuil/overleaf
-
-The referenced code was fixed, optimized, and extended.
-
-## Sandboxed compiles
-
-The Sandboxed Compiles feature was until recently largely present in the original Overleaf codebase.\
-The missing parts were implemented in this project.
-
-## Import file from external URL
-
-The "From External URL" feature exists in the original Overleaf code.\
-The missing proxy component was implemented in this project.
-
-## Git integration
-
-Git integration is based on:
-
-https://github.com/ayaka-notes/overleaf-pro/tree/feat-git-bridge
-
-The backend was completely rewritten.\
-The frontend uses the referenced code, which was optimized and improved.
-
-## Original Features
-
-The following features are original work implemented in this project:
-
-- Template Gallery
-- Advanced administrator tools for managing user accounts and projects
-- SAML authentication
-- OpenID Connect authentication
-
-## Acknowledgments
-
-Thanks to the users of the project for valuable feedback, suggestions,
-and help in identifying and fixing bugs.

README.md

@@ -21,21 +21,19 @@
 
 [Overleaf](https://www.overleaf.com) is an open-source online real-time collaborative LaTeX editor. Overleaf runs a hosted version at [www.overleaf.com](https://www.overleaf.com), but you can also run your own local version, and contribute to the development of Overleaf.
 
-## Extended Community Edition (CE+)
+## Extended Community Edition
 
 The present "extended" version of Overleaf CE includes:
 
-- Template gallery
-- Sandboxed compiles with TeX Live image selection
+- Template Gallery
+- Sandboxed Compiles with TeX Live image selection
 - LDAP authentication
 - SAML authentication
 - OpenID Connect authentication
 - Real-time track changes and comments
 - Autocomplete of reference keys
-- Symbol palette
-- Import file from external URL
-- Advanced administrator tools for managing user accounts and projects
-- Git integration
+- Symbol Palette
+- "From External URL" feature
 
 > [!CAUTION]
 > Overleaf Community Edition is intended for use in environments where **all** users are trusted. Community Edition is **not** appropriate for scenarios where isolation of users is required due to Sandbox Compiles not being available. When not using Sandboxed Compiles, users have full read and write access to the `sharelatex` container resources (filesystem, network, environment variables) when running LaTeX compiles. 
@@ -75,14 +73,11 @@ folder are added.
 
 ## Authors
 
-[The Overleaf Team](https://www.overleaf.com/about)\
-[yu-i-i](https://github.com/yu-i-i/overleaf-cep) — CE extensions; references to adapted code are listed in [`CREDITS`](CREDITS.md)
+[The Overleaf Team](https://www.overleaf.com/about)  
+[yu-i-i](https://github.com/yu-i-i/overleaf-cep) — Extensions for CE unless otherwise noted
 
 ## License
 
 The code in this repository is released under the GNU AFFERO GENERAL PUBLIC LICENSE, version 3. A copy can be found in the [`LICENSE`](LICENSE) file.
 
-Copyright (c) Overleaf, 2014-2026.\
-Copyright (c) yu-i-i, 2024-2026, for CE extensions.
-
-Portions of the code are derived from other open-source projects; see [`CREDITS`](CREDITS.md).
+Copyright (c) Overleaf, 2014-2025.

develop/README.md

@@ -42,7 +42,7 @@ To do this, use the included `bin/dev` script:
 bin/dev
 ```
 
-This will start all services using `node --watch`, which will automatically monitor the code and restart the services as necessary.
+This will start all services using `nodemon`, which will automatically monitor the code and restart the services as necessary.
 
 To improve performance, you can start only a subset of the services in development mode by providing a space-separated list to the `bin/dev` script:
 

develop/dev.env

@@ -13,13 +13,10 @@ MONGO_HOST=mongo
 MONGO_URL=mongodb://mongo/sharelatex?directConnection=true
 NOTIFICATIONS_HOST=notifications
 PROJECT_HISTORY_HOST=project-history
-QUEUES_REDIS_HOST=redis
-DSMP_REDIS_HOST=redis
 REALTIME_HOST=real-time
 REDIS_HOST=redis
 REFERENCES_HOST=references
 SESSION_SECRET=foo
-V1_HISTORY_HOST=history-v1
 WEBPACK_HOST=webpack
 WEB_API_PASSWORD=overleaf
 WEB_API_USER=overleaf

develop/docker-compose.dev.yml

@@ -88,17 +88,17 @@ services:
     volumes:
       - ../services/linked-url-proxy/app:/overleaf/services/linked-url-proxy/app
       - ../services/linked-url-proxy/config:/overleaf/services/linked-url-proxy/config
-      - ../services/linked-url-proxy/app.mjs:/overleaf/services/linked-url-proxy/app.mjs
+      - ../services/linked-url-proxy/app.js:/overleaf/services/linked-url-proxy/app.js
 
   notifications:
-    command: ["node", "--watch", "app.ts"]
+    command: ["node", "--watch", "app.js"]
     environment:
       - NODE_OPTIONS=--inspect=0.0.0.0:9229
     ports:
       - "127.0.0.1:9236:9229"
     volumes:
       - ../services/notifications/app:/overleaf/services/notifications/app
-      - ../services/notifications/app.ts:/overleaf/services/notifications/app.ts
+      - ../services/notifications/app.js:/overleaf/services/notifications/app.js
       - ../services/notifications/config:/overleaf/services/notifications/config
 
   project-history:
@@ -135,7 +135,7 @@ services:
       - ../services/references/app.js:/overleaf/services/references/app.js
 
   web:
-    command: ["node", "--watch", "app.mjs", "--watch-locales"]
+    command: ["node", "--watch", "app.js", "--watch-locales"]
     environment:
       - NODE_OPTIONS=--inspect=0.0.0.0:9229
     ports:

develop/docker-compose.yml

@@ -25,10 +25,10 @@ services:
     env_file:
       - dev.env
     environment:
+      - DOCKER_RUNNER=true
       - TEXLIVE_IMAGE=texlive-full # docker build texlive -t texlive-full
-      - SANDBOXED_COMPILES=true
-      - SANDBOXED_COMPILES_HOST_DIR_COMPILES=${PWD}/compiles
-      - SANDBOXED_COMPILES_HOST_DIR_OUTPUT=${PWD}/output
+      - COMPILES_HOST_DIR=${PWD}/compiles
+      - OUTPUT_HOST_DIR=${PWD}/output
     user: root
     volumes:
       - ${PWD}/compiles:/overleaf/services/clsi/compiles
@@ -69,6 +69,7 @@ services:
       - filestore-public-files:/overleaf/services/filestore/public_files
       - filestore-template-files:/overleaf/services/filestore/template_files
       - filestore-uploads:/overleaf/services/filestore/uploads
+      - filestore-user-files:/overleaf/services/filestore/user_files
 
   history-v1:
     build:
@@ -129,7 +130,7 @@ services:
       dockerfile: services/real-time/Dockerfile
     env_file:
       - dev.env
-
+ 
   redis:
     image: redis:5
     ports:
@@ -187,18 +188,3 @@ services:
       - "127.0.0.1:80:3808"
     volumes:
       - ./webpack.config.dev-env.js:/overleaf/services/web/webpack.config.dev-env.js
-
-  git-bridge:
-    image: quay.io/sharelatex/git-bridge:latest
-    expose:
-      - "8000"
-    ports:
-      - "8000:8000"
-    environment:
-      GIT_BRIDGE_API_BASE_URL: "http://web:3000/api/v0"
-      GIT_BRIDGE_OAUTH2_SERVER: "http://web:3000"
-      GIT_BRIDGE_POSTBACK_BASE_URL: "http://git-bridge:8000"
-      GIT_BRIDGE_ROOT_DIR: "/data/git-bridge"
-      LOG_LEVEL: "DEBUG"
-    volumes:
-      - ./data/git-bridge:/data/git-bridge

develop/webpack.config.dev-env.js

@@ -14,11 +14,6 @@ module.exports = merge(base, {
         target: 'http://real-time:3026',
         ws: true,
       },
-      {
-        context: '/git/**',
-        target: 'http://git-bridge:8000',
-        pathRewrite: { '^/git': '' }
-      },
       {
         context: ['!**/*.js', '!**/*.css', '!**/*.json'],
         target: 'http://web:3000',

docker-compose.yml

@@ -1,3 +1,4 @@
+version: '2.2'
 services:
     sharelatex:
         restart: always

libraries/access-token-encryptor/.mocharc.cjs

@@ -1,13 +0,0 @@
-let reporterOptions = {}
-if (process.env.CI) {
-  reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
-    'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
-  }
-}
-const all = {
-  require: 'test/setup.js',
-  ...reporterOptions,
-}
-
-module.exports = all

libraries/access-token-encryptor/.mocharc.json

@@ -0,0 +1,6 @@
+{
+  "ui": "bdd",
+  "recursive": "true",
+  "reporter": "spec",
+  "require": "test/setup.js"
+}

libraries/access-token-encryptor/.nvmrc

@@ -1 +1 @@
-22.18.0
+22.15.1

libraries/access-token-encryptor/buildscript.txt

@@ -1,9 +1,10 @@
 access-token-encryptor
 --dependencies=None
+--docker-repos=gcr.io/overleaf-ops
 --env-add=
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=22.18.0
---pipeline-owner=🚉 Platform
+--node-version=22.15.1
 --public-repo=False
+--script-version=4.7.0

libraries/access-token-encryptor/package.json

@@ -5,8 +5,8 @@
   "main": "index.js",
   "scripts": {
     "test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
-    "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
-    "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
+    "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
+    "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
     "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
     "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
     "test:ci": "npm run test:unit",
@@ -22,8 +22,6 @@
     "chai": "^4.3.6",
     "chai-as-promised": "^7.1.1",
     "mocha": "^11.1.0",
-    "mocha-junit-reporter": "^2.2.1",
-    "mocha-multi-reporters": "^1.5.1",
     "sandboxed-module": "^2.0.4",
     "typescript": "^5.0.4"
   }

libraries/access-token-encryptor/test/mocha-multi-reporters.cjs

@@ -1,9 +0,0 @@
-module.exports = {
-  reporterEnabled: 'spec, mocha-junit-reporter',
-  mochaJunitReporterReporterOptions: {
-    mochaFile: `reports/junit-mocha-${process.env.MOCHA_GREP}.xml`,
-    includePending: true,
-    jenkinsMode: true,
-    output: true,
-  },
-}

libraries/access-token-encryptor/tsconfig.json

@@ -2,7 +2,6 @@
   "extends": "../../tsconfig.backend.json",
   "include": [
     "**/*.js",
-    "**/*.cjs",
-    "**/*.ts"
+    "**/*.cjs"
   ]
 }

libraries/eslint-plugin/index.js

@@ -1,11 +0,0 @@
-module.exports = {
-  rules: {
-    'no-unnecessary-trans': require('./no-unnecessary-trans'),
-    'prefer-kebab-url': require('./prefer-kebab-url'),
-    'should-unescape-trans': require('./should-unescape-trans'),
-    'no-generated-editor-themes': require('./no-generated-editor-themes'),
-    'require-script-runner': require('./require-script-runner'),
-    'require-vi-doMock-valid-path': require('./require-vi-doMock-valid-path'),
-    'require-loading-label': require('./require-loading-label'),
-  },
-}

libraries/eslint-plugin/no-generated-editor-themes.js

@@ -1,21 +0,0 @@
-module.exports = {
-  meta: {
-    type: 'error',
-    docs: {
-      description:
-        'Prohibit CodeMirror themes that are generated in a function',
-    },
-  },
-  create(context) {
-    return {
-      ':matches(ArrowFunctionExpression, FunctionDeclaration, FunctionExpression) CallExpression > MemberExpression[object.name="EditorView"]:matches([property.name="theme"],[property.name="baseTheme"])'(
-        node
-      ) {
-        context.report({
-          node,
-          message: `EditorView.theme and EditorView.baseTheme each add CSS to the page for every instance of the theme. Store the theme in a variable and reuse it instead.`,
-        })
-      },
-    }
-  },
-}

libraries/eslint-plugin/no-unnecessary-trans.js

@@ -1,43 +0,0 @@
-module.exports = {
-  meta: {
-    type: 'problem',
-    fixable: 'code',
-    docs: {
-      description: 'Prohibit Trans with no components or values',
-    },
-  },
-  create(context) {
-    return {
-      'JSXOpeningElement[name.name="Trans"]'(node) {
-        const attributes = new Map(
-          node.attributes.map(attr => [attr.name.name, attr])
-        )
-
-        if (!attributes.has('components')) {
-          if (node.parent.children.length > 0) {
-            context.report({
-              node,
-              message: `Trans components must not have child elements`,
-            })
-          } else if (attributes.has('values')) {
-            context.report({
-              node,
-              message: `Use t('…') when there are no components`,
-            })
-          } else {
-            context.report({
-              node,
-              message: `Use t('…') when there are no components`,
-              fix(fixer) {
-                const i18nKey = attributes.get('i18nKey').value.value
-
-                // Note: Prettier can fix indentation
-                return fixer.replaceText(node.parent, `{t('${i18nKey}')}`)
-              },
-            })
-          }
-        }
-      },
-    }
-  },
-}

libraries/eslint-plugin/package.json

@@ -1,17 +0,0 @@
-{
-  "name": "@overleaf/eslint-plugin",
-  "version": "0.1.0",
-  "author": "Overleaf (https://www.overleaf.com)",
-  "license": "AGPL-3.0-only",
-  "main": "index.js",
-  "dependencies": {
-    "eslint": "^8.51.0",
-    "lodash": "^4.17.21"
-  },
-  "devDependencies": {
-    "@typescript-eslint/parser": "^8.50.0"
-  },
-  "scripts": {
-    "test": "node rules.test.js"
-  }
-}

libraries/eslint-plugin/prefer-kebab-url-ignore.js

@@ -1,83 +0,0 @@
-// URL parts should be kebab-case, but we didn't have this rule in the past.
-// The ESLint rule `prefer-kebab-url` will ignore these "legacy" URL parts.
-
-const ignoreWords = {
-  snake: new Set([
-    'clear_saml_data',
-    'confirm_link',
-    'confirm_university_domain',
-    'create_recurly_account',
-    'current_history_content',
-    'current_user',
-    'default_email',
-    'disable_managed_users',
-    'doc_snapshot',
-    'enable_history_ranges_support',
-    'features_override',
-    'generate_password_reset_url',
-    'get_assignment',
-    'get_clone',
-    'health_check',
-    'institutional_emails',
-    'latest_template',
-    'link_after_saml_response',
-    'linked_file',
-    'metrics_segmentation',
-    'new_users',
-    'no_autostart_post_gateway',
-    'personal_info',
-    'planned_maintenance',
-    'refresh_features',
-    'register_admin',
-    'register_ldap_admin',
-    'register_saml_admin',
-    'restore_file',
-    'revert_file',
-    'saved_vers',
-    'send_test_email',
-    'session_maintenance',
-    'set_in_session',
-    'sign_in_to_link',
-    'split_test',
-    'sso_configuration_test',
-    'sso_email',
-    'sso_enrollment',
-    'track_changes',
-    'update_admin',
-    'user_details',
-  ]),
-  camel: new Set([
-    'addWorkflowScope',
-    'aiErrorAssistant',
-    'beginAuth',
-    'brandVariationId',
-    'closeEditor',
-    'completeRegistration',
-    'deactivateOldProjects',
-    'deletedSubscription',
-    'disconnectAllUsers',
-    'editingSession',
-    'emailSubscription',
-    'enableManagedUsers',
-    'externalCollaboration',
-    'flushProjectToTpds',
-    'indexAll',
-    'offboardManagedUser',
-    'openEditor',
-    'perfTest',
-    'pollDropboxForUser',
-    'resendInvite',
-    'resendManagedUserInvite',
-    'salesContactForm',
-    'showSupport',
-  ]),
-  other: new Set([
-    'Project',
-    'disableSSO',
-    'enableSSO',
-    'resendSSOLinkInvite',
-    'usersCSV',
-  ]),
-}
-
-module.exports = { ignoreWords }

libraries/eslint-plugin/prefer-kebab-url.js

@@ -1,91 +0,0 @@
-const _ = require('lodash')
-const { ignoreWords } = require('./prefer-kebab-url-ignore')
-
-const removeTextBetweenBrackets = text => {
-  while (text.includes('[') || text.includes('(')) {
-    text = text.replaceAll(/\[[^[\]]*]/g, '')
-    text = text.replaceAll(/\([^()]*\)/g, '')
-  }
-  return text
-}
-
-const shouldIgnoreWord = str =>
-  str.includes(':') ||
-  str.includes('(') ||
-  str === '*' ||
-  str.match(/^[a-z0-9.]+$/) ||
-  ignoreWords.snake.has(str) ||
-  ignoreWords.camel.has(str) ||
-  ignoreWords.other.has(str)
-
-const getSuggestion = routePath => {
-  if (typeof routePath === 'string') {
-    const kebabed = routePath
-      .split('/')
-      .map(word => (shouldIgnoreWord(word) ? word : _.kebabCase(word)))
-      .join('/')
-    return kebabed === routePath ? null : `'${kebabed}'`
-  }
-
-  if (routePath instanceof RegExp) {
-    const words = removeTextBetweenBrackets(routePath.source).match(/[\w-]+/g)
-    if (!words) return routePath
-
-    let newSource = routePath.source
-    for (const word of words) {
-      if (!shouldIgnoreWord(word)) {
-        newSource = newSource.replaceAll(
-          new RegExp(`\\b${word}\\b`, 'g'),
-          _.kebabCase(word)
-        )
-      }
-    }
-
-    const kebabed = new RegExp(newSource, routePath.flags)
-    return kebabed.source.toString() === routePath.source.toString()
-      ? null
-      : kebabed
-  }
-}
-
-module.exports = {
-  meta: {
-    type: 'problem',
-    fixable: 'code',
-    hasSuggestions: true,
-    docs: {
-      description: 'Enforce using kebab-case for URL paths',
-    },
-  },
-  create: context => ({
-    CallExpression(node) {
-      if (
-        node.callee.type === 'MemberExpression' &&
-        node.arguments[0]?.type === 'Literal' &&
-        [/app/i, /router/i].some(callee =>
-          typeof callee === 'string'
-            ? node.callee.object.name === callee
-            : callee.test(node.callee.object.name)
-        ) &&
-        ['get', 'post', 'put', 'delete'].includes(node.callee.property.name)
-      ) {
-        const routePath = node.arguments[0].value
-
-        const suggestion = getSuggestion(routePath)
-
-        if (suggestion) {
-          context.report({
-            node: node.arguments[0],
-            message: 'Route path should be in kebab-case.',
-            suggest: [
-              {
-                desc: `Change to kebab-case: ${suggestion}`,
-                fix: fixer => fixer.replaceText(node.arguments[0], suggestion),
-              },
-            ],
-          })
-        }
-      }
-    },
-  }),
-}

libraries/eslint-plugin/require-loading-label.js

@@ -1,49 +0,0 @@
-module.exports = {
-  meta: {
-    type: 'problem',
-    fixable: null,
-    docs: {
-      description:
-        'Require loadingLabel prop when isLoading is specified on OLButton',
-    },
-    schema: [],
-  },
-  create(context) {
-    return {
-      'JSXOpeningElement[name.name="OLButton"]'(node) {
-        const attributes = new Map(
-          node.attributes.map(attr => [attr.name?.name, attr])
-        )
-
-        const isLoadingAttr = attributes.get('isLoading')
-        const loadingLabelAttr = attributes.get('loadingLabel')
-
-        if (isLoadingAttr && !loadingLabelAttr) {
-          const isLoadingValue = isLoadingAttr.value
-
-          if (
-            !isLoadingValue ||
-            (isLoadingValue.type === 'JSXExpressionContainer' &&
-              isLoadingValue.expression.type === 'Literal' &&
-              isLoadingValue.expression.value === true)
-          ) {
-            context.report({
-              node: isLoadingAttr,
-              message:
-                'Button with isLoading prop must also specify loadingLabel',
-            })
-          } else if (
-            isLoadingValue.type === 'JSXExpressionContainer' &&
-            isLoadingValue.expression.type !== 'Literal'
-          ) {
-            context.report({
-              node: isLoadingAttr,
-              message:
-                'Button with isLoading prop must also specify loadingLabel',
-            })
-          }
-        }
-      },
-    }
-  },
-}

libraries/eslint-plugin/require-script-runner.js

@@ -1,28 +0,0 @@
-module.exports = {
-  meta: {
-    type: 'suggestion',
-    docs: {
-      description: 'Require Script Runner for scripts',
-    },
-  },
-  create(context) {
-    let hasImport = false
-
-    return {
-      ImportDeclaration(node) {
-        if (node.source.value.endsWith('lib/ScriptRunner.mjs')) {
-          hasImport = true
-        }
-      },
-      'Program:exit'() {
-        if (!hasImport) {
-          context.report({
-            loc: { line: 1, column: 0 },
-            message:
-              'Please use Script Runner for scripts. Refer to the developer manual (https://manual.dev-overleaf.com/development/code/web_scripts/#monitor-script-execution-and-usage-with-script-runner) for more information.',
-          })
-        }
-      },
-    }
-  },
-}

libraries/eslint-plugin/require-vi-doMock-valid-path.js

@@ -1,138 +0,0 @@
-const path = require('node:path')
-const fs = require('node:fs')
-
-module.exports = {
-  meta: {
-    type: 'problem',
-    docs: {
-      description: 'Ensure vi.doMock first argument is a resolvable path.',
-      category: 'Best Practices',
-      recommended: false,
-      url: '',
-    },
-    fixable: 'code',
-    hasSuggestions: true,
-    schema: [],
-    messages: {
-      unresolvablePath:
-        'The path "{{pathValue}}" in vi.doMock() cannot be resolved relative to the current file.',
-      notAStringLiteral:
-        'The first argument of vi.doMock() must be (or resolve to) a string literal representing a path.',
-      noArguments: 'vi.doMock() called with no arguments.',
-    },
-  },
-  create(context) {
-    const currentFilePath = context.getFilename()
-    // ESLint can sometimes pass <text> or <input> for snippets not in a file
-    if (currentFilePath === '<text>' || currentFilePath === '<input>') {
-      return {}
-    }
-    const currentDirectory = path.dirname(currentFilePath)
-
-    function canResolve(modulePath) {
-      try {
-        require.resolve(path.resolve(currentDirectory, modulePath))
-        return true
-      } catch (e) {
-        const absolutePath = path.resolve(currentDirectory, modulePath)
-        const extensions = [
-          '',
-          '.js',
-          '.mjs',
-          '.ts',
-          '.jsx',
-          '.tsx',
-          '.json',
-          '.node',
-          '/index.js',
-          '/index.ts',
-        ] // Add common extensions
-        for (const ext of extensions) {
-          if (fs.existsSync(absolutePath + ext)) {
-            return true
-          }
-        }
-        return false
-      }
-    }
-
-    return {
-      CallExpression(node) {
-        if (
-          node.callee.type === 'MemberExpression' &&
-          node.callee.object.type === 'Identifier' &&
-          node.callee.object.name === 'vi' &&
-          node.callee.property.type === 'Identifier' &&
-          node.callee.property.name === 'doMock'
-        ) {
-          if (node.arguments.length === 0) {
-            context.report({
-              node,
-              messageId: 'noArguments',
-            })
-            return
-          }
-
-          const firstArg = node.arguments[0]
-          let pathValue = firstArg.value
-
-          if (
-            firstArg.type !== 'Literal' ||
-            typeof firstArg.value !== 'string'
-          ) {
-            if (firstArg.type === 'Identifier') {
-              const variable = context
-                .getScope()
-                .variables.find(v => v.name === firstArg.name)
-              if (
-                variable &&
-                variable.defs.length > 0 &&
-                variable.defs[0].node.init &&
-                variable.defs[0].node.init.type === 'Literal' &&
-                typeof variable.defs[0].node.init.value === 'string'
-              ) {
-                pathValue = variable.defs[0].node.init.value
-                if (canResolve(pathValue)) {
-                  return
-                }
-                // If the first argument was a variable that didn't resolve then we can't auto-fix it
-              }
-            }
-            context.report({
-              node: firstArg,
-              messageId: 'notAStringLiteral',
-            })
-            return
-          }
-
-          if (!pathValue.startsWith('.')) {
-            return
-          }
-
-          if (!canResolve(pathValue)) {
-            const mjsPath = pathValue.replace('.js', '.mjs')
-            const additionalReportOptions = {}
-            if (canResolve(mjsPath)) {
-              additionalReportOptions.fix = fixer =>
-                fixer.replaceText(firstArg, `'${mjsPath}'`)
-              additionalReportOptions.suggest = [
-                {
-                  desc: `Replace with "${pathValue.replace('.js', '.mjs')}"`,
-                  fix: fixer => fixer.replaceText(firstArg, `'${mjsPath}'`),
-                },
-              ]
-            }
-            context.report({
-              node: firstArg,
-              messageId: 'unresolvablePath',
-              data: {
-                pathValue,
-              },
-              ...additionalReportOptions,
-            })
-          }
-        }
-      },
-    }
-  },
-}

libraries/eslint-plugin/rules.test.js

@@ -1,168 +0,0 @@
-const { RuleTester } = require('eslint')
-const preferKebabUrl = require('./prefer-kebab-url')
-const noUnnecessaryTrans = require('./no-unnecessary-trans')
-const shouldUnescapeTrans = require('./should-unescape-trans')
-const noGeneratedEditorThemes = require('./no-generated-editor-themes')
-const viDoMockValidPath = require('./require-vi-doMock-valid-path')
-
-const ruleTester = new RuleTester({
-  parser: require.resolve('@typescript-eslint/parser'),
-  parserOptions: {
-    ecmaVersion: 'latest',
-    ecmaFeatures: { jsx: true },
-  },
-})
-
-ruleTester.run('prefer-kebab-url', preferKebabUrl, {
-  valid: [
-    { code: `app.get('/foo-bar')` },
-    { code: `app.get('/foo-bar/:id')` },
-    { code: `router.post('/foo-bar')` },
-    { code: `router.get('/foo-bar/:id/:name/:age')` },
-    { code: `webRouter.get('/foo-bar/:user_id/(ProjectName)/get-info')` },
-    { code: `webApp.post('/foo-bar/:user_id/(ProjectName)/get-info')` },
-    {
-      code: `router.get(/^\\/download\\/project\\/([^/]*)\\/output\\/output\\.pdf$/)`,
-    },
-    {
-      code: `webRouter.get(/^\\/project\\/([^/]*)\\/user\\/([0-9a-f]+)\\/build\\/([0-9a-f-]+)\\/output\\/(.*)$/)`,
-    },
-  ],
-  invalid: [
-    {
-      code: `app.get('/fooBar')`,
-      errors: [{ message: 'Route path should be in kebab-case.' }],
-    },
-    {
-      code: `app.get('/fooBar/:id')`,
-      errors: [{ message: 'Route path should be in kebab-case.' }],
-    },
-    {
-      code: `webRouter.get('/foo_bar/:id/FooBar/:name/fooBar')`,
-      errors: [{ message: 'Route path should be in kebab-case.' }],
-    },
-    {
-      code: `router.get(/^\\/downLoad\\/pro-ject\\/([^/]*)\\/OutPut\\/out-put\\.pdf$/)`,
-      errors: [{ message: 'Route path should be in kebab-case.' }],
-    },
-  ],
-})
-
-ruleTester.run('no-unnecessary-trans', noUnnecessaryTrans, {
-  valid: [
-    { code: `<Trans i18nKey="test" components={{ strong: <strong/> }}/>` },
-  ],
-  invalid: [
-    {
-      code: `<Trans i18nKey="test" values={{ test: 'foo '}}/>`,
-      errors: [{ message: `Use t('…') when there are no components` }],
-    },
-    {
-      code: `<Trans i18nKey="test" />`,
-      errors: [{ message: `Use t('…') when there are no components` }],
-      output: `{t('test')}`,
-    },
-  ],
-})
-
-ruleTester.run('should-unescape-trans', shouldUnescapeTrans, {
-  valid: [
-    {
-      code: `<Trans i18nKey="test" components={{ strong: <strong/> }}/>`,
-    },
-    {
-      code: `<Trans i18nKey="test" values={{ foo: 'bar' }} components={{ strong: <strong/> }} shouldUnescape tOptions={{ interpolation: { escapeValue: true } }}/>`,
-    },
-  ],
-  invalid: [
-    {
-      code: `<Trans i18nKey="test" values={{ foo: 'bar' }} components={{ strong: <strong/> }} />`,
-      errors: [{ message: 'Trans with values must have shouldUnescape' }],
-      output: `<Trans i18nKey="test" values={{ foo: 'bar' }}\nshouldUnescape components={{ strong: <strong/> }} />`,
-    },
-    {
-      code: `<Trans i18nKey="test" values={{ foo: 'bar' }} components={{ strong: <strong/> }} shouldUnescape />`,
-      errors: [
-        {
-          message:
-            'Trans with shouldUnescape must have tOptions.interpolation.escapeValue',
-        },
-      ],
-      output: `<Trans i18nKey="test" values={{ foo: 'bar' }} components={{ strong: <strong/> }} shouldUnescape\ntOptions={{ interpolation: { escapeValue: true } }} />`,
-    },
-  ],
-})
-
-const noGeneratedEditorThemesError =
-  'EditorView.theme and EditorView.baseTheme each add CSS to the page for every instance of the theme. Store the theme in a variable and reuse it instead.'
-ruleTester.run('no-generated-editor-themes', noGeneratedEditorThemes, {
-  valid: [
-    {
-      code: `EditorView.theme({ '.cm-editor': { color: 'black' } })`,
-    },
-    {
-      code: `const theme = EditorView.theme({ '.cm-editor': { color: 'black' } })`,
-    },
-  ],
-  invalid: [
-    {
-      code: `function createTheme() { return EditorView.theme({ '.cm-editor': { color: 'black' } }) }`,
-      errors: [
-        {
-          message: noGeneratedEditorThemesError,
-        },
-      ],
-    },
-    {
-      code: `() => EditorView.theme({ '.cm-editor': { color: 'black' } })`,
-      errors: [
-        {
-          message: noGeneratedEditorThemesError,
-        },
-      ],
-    },
-    {
-      code: `class Foo { createTheme() { return EditorView.theme({ '.cm-editor': { color: 'black' } }) } }`,
-      errors: [
-        {
-          message: noGeneratedEditorThemesError,
-        },
-      ],
-    },
-  ],
-})
-
-ruleTester.run('domock-require-valid-path', viDoMockValidPath, {
-  valid: [
-    {
-      code: 'vi.doMock("./require-vi-doMock-valid-path.js")',
-      filename: __filename,
-    },
-    {
-      code: 'const filename = "./require-vi-doMock-valid-path.js"; vi.doMock(filename);',
-      filename: __filename,
-    },
-  ],
-  invalid: [
-    {
-      code: "vi.doMock('./require-vi-doMock-valid-path2')",
-      filename: __filename,
-      errors: [
-        {
-          message:
-            'The path "./require-vi-doMock-valid-path2" in vi.doMock() cannot be resolved relative to the current file.',
-        },
-      ],
-    },
-    {
-      code: 'const filename = "./require-vi-doMock-valid-path2.js"; vi.doMock(filename);',
-      filename: __filename,
-      errors: [
-        {
-          message:
-            'The first argument of vi.doMock() must be (or resolve to) a string literal representing a path.',
-        },
-      ],
-    },
-  ],
-})

libraries/eslint-plugin/should-unescape-trans.js

@@ -1,60 +0,0 @@
-module.exports = {
-  meta: {
-    type: 'problem',
-    fixable: 'code',
-    docs: {
-      description: 'Ensure that Trans with values has shouldUnescape',
-    },
-  },
-  create(context) {
-    return {
-      'JSXOpeningElement[name.name="Trans"]'(node) {
-        const attributes = new Map(
-          node.attributes.map(attr => [attr.name.name, attr])
-        )
-
-        if (attributes.has('values') && !attributes.has('shouldUnescape')) {
-          context.report({
-            node,
-            message: 'Trans with values must have shouldUnescape',
-            fix(fixer) {
-              return fixer.insertTextAfter(
-                attributes.get('values'),
-                '\nshouldUnescape' // Note: Prettier can fix indentation
-              )
-            },
-          })
-        }
-
-        if (attributes.has('values') && attributes.has('shouldUnescape')) {
-          const tOptions = attributes.get('tOptions')
-          if (!tOptions) {
-            context.report({
-              node,
-              message:
-                'Trans with shouldUnescape must have tOptions.interpolation.escapeValue',
-              fix(fixer) {
-                return fixer.insertTextAfter(
-                  attributes.get('shouldUnescape'),
-                  '\ntOptions={{ interpolation: { escapeValue: true } }}' // Note: Prettier can fix indentation
-                )
-              },
-            })
-          } else {
-            const property = tOptions.value.expression.properties
-              .find(p => p.key.name === 'interpolation')
-              ?.value.properties.find(p => p.key.name === 'escapeValue')
-
-            if (property?.value.value !== true) {
-              context.report({
-                node,
-                message:
-                  'Trans with shouldUnescape must have tOptions.interpolation.escapeValue set to true',
-              })
-            }
-          }
-        }
-      },
-    }
-  },
-}

libraries/eslint-plugin/tsconfig.json

@@ -1,6 +0,0 @@
-{
-  "extends": "../../tsconfig.backend.json",
-  "include": [
-    "**/*.js",
-  ]
-}

libraries/fetch-utils/.mocharc.cjs

@@ -1,13 +0,0 @@
-let reporterOptions = {}
-if (process.env.CI) {
-  reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
-    'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
-  }
-}
-const all = {
-  require: 'test/setup.js',
-  ...reporterOptions,
-}
-
-module.exports = all

libraries/fetch-utils/.mocharc.json

@@ -0,0 +1,6 @@
+{
+  "ui": "bdd",
+  "recursive": "true",
+  "reporter": "spec",
+  "require": "test/setup.js"
+}

libraries/fetch-utils/.nvmrc

@@ -1 +1 @@
-22.18.0
+22.15.1

libraries/fetch-utils/buildscript.txt

@@ -1,9 +1,10 @@
 fetch-utils
 --dependencies=None
+--docker-repos=gcr.io/overleaf-ops
 --env-add=
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=22.18.0
---pipeline-owner=🚉 Platform
+--node-version=22.15.1
 --public-repo=False
+--script-version=4.7.0

libraries/fetch-utils/package.json

@@ -5,8 +5,8 @@
   "main": "index.js",
   "scripts": {
     "test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
-    "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
-    "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
+    "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
+    "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
     "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
     "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
     "test:ci": "npm run test:unit",
@@ -16,14 +16,12 @@
   "author": "Overleaf (https://www.overleaf.com)",
   "license": "AGPL-3.0-only",
   "devDependencies": {
-    "@types/node-fetch": "^2.6.13",
-    "body-parser": "1.20.4",
+    "@types/node-fetch": "^2.6.11",
+    "body-parser": "^1.20.3",
     "chai": "^4.3.6",
     "chai-as-promised": "^7.1.1",
-    "express": "4.22.1",
+    "express": "^4.21.2",
     "mocha": "^11.1.0",
-    "mocha-junit-reporter": "^2.2.1",
-    "mocha-multi-reporters": "^1.5.1",
     "typescript": "^5.0.4"
   },
   "dependencies": {

libraries/fetch-utils/test/mocha-multi-reporters.cjs

@@ -1,9 +0,0 @@
-module.exports = {
-  reporterEnabled: 'spec, mocha-junit-reporter',
-  mochaJunitReporterReporterOptions: {
-    mochaFile: `reports/junit-mocha-${process.env.MOCHA_GREP}.xml`,
-    includePending: true,
-    jenkinsMode: true,
-    output: true,
-  },
-}

libraries/fetch-utils/tsconfig.json

@@ -2,7 +2,6 @@
   "extends": "../../tsconfig.backend.json",
   "include": [
     "**/*.js",
-    "**/*.cjs",
-    "**/*.ts"
+    "**/*.cjs"
   ]
 }

libraries/logger/.mocharc.cjs

@@ -1,13 +0,0 @@
-let reporterOptions = {}
-if (process.env.CI) {
-  reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
-    'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
-  }
-}
-const all = {
-  require: 'test/setup.js',
-  ...reporterOptions,
-}
-
-module.exports = all

libraries/logger/.mocharc.json

@@ -0,0 +1,3 @@
+{
+  "require": "test/setup.js"
+}

libraries/logger/.nvmrc

@@ -1 +1 @@
-22.18.0
+22.15.1

libraries/logger/buildscript.txt

@@ -1,9 +1,10 @@
 logger
 --dependencies=None
+--docker-repos=gcr.io/overleaf-ops
 --env-add=
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=22.18.0
---pipeline-owner=🚉 Platform
+--node-version=22.15.1
 --public-repo=False
+--script-version=4.7.0

libraries/logger/logging-manager.js

@@ -11,7 +11,7 @@ const LoggingManager = {
   /**
    * @param {string} name - The name of the logger
    */
-  initialize(name, options = {}) {
+  initialize(name) {
     this.isProduction =
       (process.env.NODE_ENV || '').toLowerCase() === 'production'
     const isTest = (process.env.NODE_ENV || '').toLowerCase() === 'test'
@@ -27,7 +27,7 @@ const LoggingManager = {
         req: Serializers.req,
         res: Serializers.res,
       },
-      streams: options.streams ?? [this._getOutputStreamConfig()],
+      streams: [this._getOutputStreamConfig()],
     })
     this._setupRingBuffer()
     this._setupLogLevelChecker()

libraries/logger/package.json

@@ -13,8 +13,8 @@
     "test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
     "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
     "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
-    "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
-    "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
+    "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
+    "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
     "test:ci": "npm run test:unit",
     "test:unit": "mocha --exit test/**/*.{js,cjs}",
     "types:check": "tsc --noEmit"
@@ -28,8 +28,6 @@
   "devDependencies": {
     "chai": "^4.3.6",
     "mocha": "^11.1.0",
-    "mocha-junit-reporter": "^2.2.1",
-    "mocha-multi-reporters": "^1.5.1",
     "sandboxed-module": "^2.0.4",
     "sinon": "^9.2.4",
     "sinon-chai": "^3.7.0",

libraries/logger/test-log-recorder.js

@@ -1,20 +0,0 @@
-const logger = require('./')
-const bunyan = require('bunyan')
-const serializers = require('./serializers')
-
-function testLogRecorder() {
-  const currentTest = this.currentTest
-  for (const level of ['error', 'fatal']) {
-    logger[level] = (info, msg) => {
-      const entry = { level, ...info, msg }
-      for (const [name, fn] of Object.entries(serializers)) {
-        if (name in entry) entry[name] = fn(entry[name])
-      }
-      currentTest.consoleErrors = (currentTest.consoleErrors || []).concat(
-        JSON.stringify(info, bunyan.safeCycles())
-      )
-    }
-  }
-}
-
-module.exports = testLogRecorder

libraries/logger/test/mocha-multi-reporters.cjs

@@ -1,9 +0,0 @@
-module.exports = {
-  reporterEnabled: 'spec, mocha-junit-reporter',
-  mochaJunitReporterReporterOptions: {
-    mochaFile: `reports/junit-mocha-${process.env.MOCHA_GREP}.xml`,
-    includePending: true,
-    jenkinsMode: true,
-    output: true,
-  },
-}

libraries/logger/tsconfig.json

@@ -2,7 +2,6 @@
   "extends": "../../tsconfig.backend.json",
   "include": [
     "**/*.js",
-    "**/*.cjs",
-    "**/*.ts"
+    "**/*.cjs"
   ]
 }

libraries/metrics/.mocharc.cjs

@@ -1,13 +0,0 @@
-let reporterOptions = {}
-if (process.env.CI) {
-  reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
-    'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
-  }
-}
-const all = {
-  require: 'test/setup.js',
-  ...reporterOptions,
-}
-
-module.exports = all

libraries/metrics/.nvmrc

@@ -1 +1 @@
-22.18.0
+22.15.1

libraries/metrics/buildscript.txt

@@ -1,9 +1,10 @@
 metrics
 --dependencies=None
+--docker-repos=gcr.io/overleaf-ops
 --env-add=
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=22.18.0
---pipeline-owner=🚉 Platform
+--node-version=22.15.1
 --public-repo=False
+--script-version=4.7.0

libraries/metrics/http.js

@@ -75,4 +75,4 @@ class RequestLogger {
   }
 }
 
-module.exports = { monitor, RequestLogger }
+module.exports.monitor = monitor

libraries/metrics/index.js

@@ -97,12 +97,6 @@ function histogram(key, value, buckets, labels = {}) {
 }
 
 class Timer {
-  /**
-   * @param {string} key
-   * @param {number} sampleRate
-   * @param {Record<string, any>} labels
-   * @param {Array<number>} buckets
-   */
   constructor(key, sampleRate = 1, labels = {}, buckets = undefined) {
     if (typeof sampleRate === 'object') {
       // called with (key, labels, buckets)

libraries/metrics/package.json

@@ -24,17 +24,15 @@
     "bunyan": "^1.0.0",
     "chai": "^4.3.6",
     "mocha": "^11.1.0",
-    "mocha-junit-reporter": "^2.2.1",
-    "mocha-multi-reporters": "^1.5.1",
     "sandboxed-module": "^2.0.4",
     "sinon": "^9.2.4",
     "typescript": "^5.0.4"
   },
   "scripts": {
-    "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
-    "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
+    "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
+    "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
     "test:unit": "mocha --exit test/**/*.{js,cjs}",
-    "test:acceptance": "mocha --recursive --exit --grep=$MOCHA_GREP test/acceptance",
+    "test:acceptance": "mocha --reporter spec --recursive --exit --grep=$MOCHA_GREP test/acceptance",
     "test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
     "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
     "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",

libraries/metrics/test/mocha-multi-reporters.cjs

@@ -1,9 +0,0 @@
-module.exports = {
-  reporterEnabled: 'spec, mocha-junit-reporter',
-  mochaJunitReporterReporterOptions: {
-    mochaFile: `reports/junit-mocha-${process.env.MOCHA_GREP}.xml`,
-    includePending: true,
-    jenkinsMode: true,
-    output: true,
-  },
-}

libraries/metrics/test/setup.js

@@ -1 +0,0 @@
-// no setup yet

libraries/metrics/tsconfig.json

@@ -2,7 +2,6 @@
   "extends": "../../tsconfig.backend.json",
   "include": [
     "**/*.js",
-    "**/*.cjs",
-    "**/*.ts"
+    "**/*.cjs"
   ]
 }

libraries/mongo-utils/.nvmrc

@@ -1 +1 @@
-22.18.0
+22.15.1

libraries/mongo-utils/buildscript.txt

@@ -1,9 +1,10 @@
 mongo-utils
 --dependencies=None
+--docker-repos=gcr.io/overleaf-ops
 --env-add=
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=22.18.0
---pipeline-owner=🚉 Platform
+--node-version=22.15.1
 --public-repo=False
+--script-version=4.7.0

libraries/mongo-utils/index.js

@@ -1,5 +0,0 @@
-// @ts-check
-
-const { cleanupTestDatabase, dropTestDatabase } = require('./test-utils')
-
-module.exports = { cleanupTestDatabase, dropTestDatabase }

libraries/mongo-utils/package.json

@@ -6,8 +6,8 @@
   "scripts": {
     "test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
     "test:unit": "mocha --exit test/**/*.{js,cjs}",
-    "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
-    "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
+    "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
+    "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
     "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
     "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
     "test:ci": "npm run test:unit",
@@ -22,8 +22,6 @@
   "devDependencies": {
     "chai": "^4.3.6",
     "mocha": "^11.1.0",
-    "mocha-junit-reporter": "^2.2.1",
-    "mocha-multi-reporters": "^1.5.1",
     "sandboxed-module": "^2.0.4",
     "sinon": "^9.2.4",
     "sinon-chai": "^3.7.0",

libraries/mongo-utils/test-utils.js

@@ -1,55 +0,0 @@
-// @ts-check
-
-/**
- * @import { MongoClient } from 'mongodb'
- * @import { MongoClient as LegacyMongoClient } from 'mongodb-legacy'
- */
-
-/**
- * Delete all data from the test Mongo database
- *
- * This doesn't drop the collections, so indexes are preserved.
- *
- * @param {MongoClient | LegacyMongoClient} mongoClient
- */
-async function cleanupTestDatabase(mongoClient) {
-  ensureTestDatabase(mongoClient)
-  const db = mongoClient.db()
-  const allCollections = await db.collections()
-  const collections = allCollections.filter(
-    coll => coll.collectionName !== 'migrations'
-  )
-  await Promise.all(collections.map(coll => coll.deleteMany({})))
-}
-
-/**
- * Drop the test Monto database
- *
- * This drops the whole database, including indexes.
- *
- * @param {MongoClient | LegacyMongoClient } mongoClient
- */
-async function dropTestDatabase(mongoClient) {
-  ensureTestDatabase(mongoClient)
-  await mongoClient.db().dropDatabase()
-}
-/**
- * Ensure that the given client is connected to a test database.
- *
- * This should be called before performing destructive operations on the test
- * database.
- *
- * @param {MongoClient | LegacyMongoClient } mongoClient
- */
-function ensureTestDatabase(mongoClient) {
-  const dbName = mongoClient.db().databaseName
-  const env = process.env.NODE_ENV
-
-  if (dbName !== 'test-overleaf' || env !== 'test') {
-    throw new Error(
-      `Refusing to clear database '${dbName}' in environment '${env}'`
-    )
-  }
-}
-
-module.exports = { cleanupTestDatabase, dropTestDatabase }

libraries/mongo-utils/test/setup.js

@@ -0,0 +1,11 @@
+const chai = require('chai')
+const sinonChai = require('sinon-chai')
+const SandboxedModule = require('sandboxed-module')
+
+// Chai configuration
+chai.should()
+chai.use(sinonChai)
+
+SandboxedModule.configure({
+  globals: { Buffer, JSON, console, process },
+})

libraries/mongo-utils/tsconfig.json

@@ -2,7 +2,6 @@
   "extends": "../../tsconfig.backend.json",
   "include": [
     "**/*.js",
-    "**/*.cjs",
-    "**/*.ts"
+    "**/*.cjs"
   ]
 }

libraries/o-error/.editorconfig

@@ -0,0 +1,9 @@
+root = true
+
+[*]
+charset = utf-8
+indent_style = space
+indent_size = 2
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true

libraries/o-error/.mocharc.cjs

@@ -1,13 +0,0 @@
-let reporterOptions = {}
-if (process.env.CI) {
-  reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
-    'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
-  }
-}
-const all = {
-  require: 'test/setup.js',
-  ...reporterOptions,
-}
-
-module.exports = all

libraries/o-error/.nvmrc

@@ -1 +1 @@
-22.18.0
+22.15.1

libraries/o-error/buildscript.txt

@@ -1,9 +1,10 @@
 o-error
 --dependencies=None
+--docker-repos=gcr.io/overleaf-ops
 --env-add=
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=22.18.0
---pipeline-owner=🚉 Platform
+--node-version=22.15.1
 --public-repo=False
+--script-version=4.7.0

libraries/o-error/package.json

@@ -18,8 +18,8 @@
   ],
   "scripts": {
     "build": "npm run --silent test",
-    "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
-    "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
+    "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
+    "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
     "test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
     "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
     "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
@@ -32,11 +32,9 @@
   "repository": "github:overleaf/o-error",
   "devDependencies": {
     "@types/chai": "^4.3.0",
-    "@types/node": "^18.19.123",
+    "@types/node": "^18.17.4",
     "chai": "^4.3.6",
     "mocha": "^11.1.0",
-    "mocha-junit-reporter": "^2.2.1",
-    "mocha-multi-reporters": "^1.5.1",
     "typescript": "^5.0.4"
   }
 }

libraries/o-error/test/mocha-multi-reporters.cjs

@@ -1,9 +0,0 @@
-module.exports = {
-  reporterEnabled: 'spec, mocha-junit-reporter',
-  mochaJunitReporterReporterOptions: {
-    mochaFile: `reports/junit-mocha-${process.env.MOCHA_GREP}.xml`,
-    includePending: true,
-    jenkinsMode: true,
-    output: true,
-  },
-}

libraries/o-error/test/o-error-util.test.js

@@ -1,7 +1,7 @@
 const { expect } = require('chai')
 const { promisify } = require('node:util')
 
-const OError = require('../..')
+const OError = require('..')
 
 const {
   expectError,

libraries/o-error/test/o-error.test.js

@@ -1,6 +1,6 @@
 const { expect } = require('chai')
 
-const OError = require('../..')
+const OError = require('..')
 const {
   expectError,
   expectFullStackWithoutStackFramesToEqual,

libraries/o-error/test/setup.js

@@ -1 +0,0 @@
-// no setup yet

libraries/o-error/test/support/index.js

@@ -0,0 +1,61 @@
+const { expect } = require('chai')
+
+const OError = require('../..')
+
+/**
+ * @param {Error} e
+ * @param {any} expected
+ */
+exports.expectError = function OErrorExpectError(e, expected) {
+  expect(
+    e.name,
+    "error should set the name property to the error's name"
+  ).to.equal(expected.name)
+
+  expect(
+    e instanceof expected.klass,
+    'error should be an instance of the error type'
+  ).to.be.true
+
+  expect(
+    e instanceof Error,
+    'error should be an instance of the built-in Error type'
+  ).to.be.true
+
+  expect(
+    require('node:util').types.isNativeError(e),
+    'error should be recognised by util.types.isNativeError'
+  ).to.be.true
+
+  expect(
+    e.toString(),
+    'toString should return the default error message formatting'
+  ).to.equal(expected.message)
+
+  expect(e.stack, 'error should have a stack trace').to.not.be.empty
+
+  expect(
+    /** @type {string} */ (e.stack).split('\n')[0],
+    'stack should start with the default error message formatting'
+  ).to.match(new RegExp(`^${expected.name}:`))
+
+  expect(
+    /** @type {string} */ (e.stack).split('\n')[1],
+    'first stack frame should be the function where the error was thrown'
+  ).to.match(expected.firstFrameRx)
+}
+
+/**
+ * @param {Error} error
+ * @param {String[]} expected
+ */
+exports.expectFullStackWithoutStackFramesToEqual = function (error, expected) {
+  const fullStack = OError.getFullStack(error)
+  const fullStackWithoutFrames = fullStack
+    .split('\n')
+    .filter(line => !/^\s+at\s/.test(line))
+  expect(
+    fullStackWithoutFrames,
+    'full stack without frames should equal'
+  ).to.deep.equal(expected)
+}

libraries/o-error/test/unit/support/index.js

@@ -1,61 +0,0 @@
-const { expect } = require('chai')
-
-const OError = require('../../..')
-
-/**
- * @param {Error} e
- * @param {any} expected
- */
-exports.expectError = function OErrorExpectError(e, expected) {
-  expect(
-    e.name,
-    "error should set the name property to the error's name"
-  ).to.equal(expected.name)
-
-  expect(
-    e instanceof expected.klass,
-    'error should be an instance of the error type'
-  ).to.be.true
-
-  expect(
-    e instanceof Error,
-    'error should be an instance of the built-in Error type'
-  ).to.be.true
-
-  expect(
-    require('node:util').types.isNativeError(e),
-    'error should be recognised by util.types.isNativeError'
-  ).to.be.true
-
-  expect(
-    e.toString(),
-    'toString should return the default error message formatting'
-  ).to.equal(expected.message)
-
-  expect(e.stack, 'error should have a stack trace').to.not.be.empty
-
-  expect(
-    /** @type {string} */ (e.stack).split('\n')[0],
-    'stack should start with the default error message formatting'
-  ).to.match(new RegExp(`^${expected.name}:`))
-
-  expect(
-    /** @type {string} */ (e.stack).split('\n')[1],
-    'first stack frame should be the function where the error was thrown'
-  ).to.match(expected.firstFrameRx)
-}
-
-/**
- * @param {Error} error
- * @param {String[]} expected
- */
-exports.expectFullStackWithoutStackFramesToEqual = function (error, expected) {
-  const fullStack = OError.getFullStack(error)
-  const fullStackWithoutFrames = fullStack
-    .split('\n')
-    .filter(line => !/^\s+at\s/.test(line))
-  expect(
-    fullStackWithoutFrames,
-    'full stack without frames should equal'
-  ).to.deep.equal(expected)
-}

libraries/o-error/tsconfig.json

@@ -2,7 +2,6 @@
   "extends": "../../tsconfig.backend.json",
   "include": [
     "**/*.js",
-    "**/*.cjs",
-    "**/*.ts"
+    "**/*.cjs"
   ]
 }

libraries/object-persistor/.mocharc.cjs

@@ -1,13 +0,0 @@
-let reporterOptions = {}
-if (process.env.CI) {
-  reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
-    'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
-  }
-}
-const all = {
-  require: 'test/setup.js',
-  ...reporterOptions,
-}
-
-module.exports = all

libraries/object-persistor/.mocharc.json

@@ -0,0 +1,6 @@
+{
+  "ui": "bdd",
+  "recursive": "true",
+  "reporter": "spec",
+  "require": "./test/Init"
+}

libraries/object-persistor/.nvmrc

@@ -1 +1 @@
-22.18.0
+22.15.1

libraries/object-persistor/README.md

@@ -46,7 +46,7 @@ Uploads a stream to the backend.
 - `key`: The key for the uploaded object
 - `readStream`: The data stream to upload
 - `opts` (optional):
-    - `sourceMd5`: The md5 hash of the source data, if known. The uploaded data will be compared against this and the operation will fail if it does not match. If omitted, the md5 is calculated as the data is uploaded instead, and verified against the backend. This is not supported in `S3Persistor` as it performs [its own integrity protections](https://aws.amazon.com/blogs/aws/introducing-default-data-integrity-protections-for-new-objects-in-amazon-s3/). Setting `sourceMd5` with `S3Persistor` will result in an error being thrown.
+    - `sourceMd5`: The md5 hash of the source data, if known. The uploaded data will be compared against this and the operation will fail if it does not match. If omitted, the md5 is calculated as the data is uploaded instead, and verified against the backend.
     - `contentType`: The content type to write in the object metadata
     - `contentEncoding`: The content encoding to write in the object metadata
 
@@ -264,10 +264,8 @@ For the `FS` persistor, the `bucketName` should be the full path to the folder o
 - `s3.key` (required): The AWS access key ID
 - `s3.secret` (required): The AWS secret access key
 - `s3.partSize`: The part size for S3 uploads. Defaults to 100 megabytes.
-- `s3.httpOptions`: HTTP Options passed to the [`NodeHttpHandler` constructor](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/Package/-smithy-node-http-handler/Class/NodeHttpHandler/)
-  - For backwards compatibility reasons, the `timeout` property that was passed to the [S3 constructor](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#constructor-property) before migrating to AWS SDK v3 is now passed to the `NodeHttpHandler` constructor as `connectionTimeout`.
-- `s3.maxRetries` (legacy): The number of times the S3 client will retry in case of an error
-- `s3.maxAttempts`: The number of times the S3 client will attempt to perform the operation in case there are errors. Default value is 3.
+- `s3.httpOptions`: HTTP options passed directly to the [S3 constructor](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#constructor-property).
+- `s3.maxRetries`: The number of times the S3 client will retry in case of an error
 - `s3.endpoint`: For testing - overrides the S3 endpoint to use a different service (e.g. a fake S3 server)
 - `s3.pathStyle`: For testing - use old path-style URLs, for services that do not support subdomain-based access
 
@@ -282,6 +280,12 @@ For the `FS` persistor, the `bucketName` should be the full path to the folder o
 }
 ```
 
+#### Notes
+
+In order for server-side MD5 generation to work, uploads must be below the `partSize`. Otherwise a multipart upload will be used, and the S3 `eTag` which is used to retrieve the MD5 will not be the MD5 hash of the uploaded object. In these cases, we download the data and calculate the MD5 manually.
+
+For verification during upload, we use S3's checksum mechanism to verify the integrity of the uploaded data, but when explicitly retrieving the md5 hash this will download the entire object if its size is above the part size.
+
 ### GCS-specific parameters
 
 GCS authentication is configured automatically via the local service account, or the `GOOGLE_APPLICATION_CREDENTIALS` environment variable.

libraries/object-persistor/buildscript.txt

@@ -1,9 +1,10 @@
 object-persistor
 --dependencies=None
+--docker-repos=gcr.io/overleaf-ops
 --env-add=
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=22.18.0
---pipeline-owner=🚉 Platform
+--node-version=22.15.1
 --public-repo=False
+--script-version=4.7.0

libraries/object-persistor/package.json

@@ -6,8 +6,8 @@
   "scripts": {
     "test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
     "test:unit": "mocha --exit test/**/*.{js,cjs}",
-    "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
-    "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
+    "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
+    "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
     "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
     "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
     "test:ci": "npm run test:unit",
@@ -20,17 +20,14 @@
   "author": "Overleaf (https://www.overleaf.com/)",
   "license": "AGPL-3.0",
   "dependencies": {
-    "@aws-sdk/client-s3": "^3.864.0",
-    "@aws-sdk/lib-storage": "^3.864.0",
-    "@aws-sdk/node-http-handler": "^3.370.0",
-    "@aws-sdk/s3-request-presigner": "^3.864.0",
     "@google-cloud/storage": "^6.10.1",
     "@overleaf/logger": "*",
     "@overleaf/metrics": "*",
     "@overleaf/o-error": "*",
     "@overleaf/stream-utils": "*",
+    "aws-sdk": "^2.1691.0",
     "fast-crc32c": "overleaf/node-fast-crc32c#aae6b2a4c7a7a159395df9cc6c38dfde702d6f51",
-    "glob": "^12.0.0",
+    "glob": "^7.1.6",
     "range-parser": "^1.2.1",
     "tiny-async-pool": "^1.1.0"
   },
@@ -38,8 +35,6 @@
     "chai": "^4.3.6",
     "chai-as-promised": "^7.1.1",
     "mocha": "^11.1.0",
-    "mocha-junit-reporter": "^2.2.1",
-    "mocha-multi-reporters": "^1.5.1",
     "mock-fs": "^5.2.0",
     "mongodb": "6.12.0",
     "sandboxed-module": "^2.0.4",

libraries/object-persistor/src/AbstractPersistor.js

@@ -160,38 +160,4 @@ module.exports = class AbstractPersistor {
       name,
     })
   }
-
-  /**
-   * List objects in a directory, returning the full keys.
-   *
-   * Suitable only for directories where the number of keys is known to be small.
-   *
-   * @param {string} location
-   * @param {string} prefix
-   * @returns {Promise<Array<string>>}
-   */
-  async listDirectoryKeys(location, prefix) {
-    throw new NotImplementedError('method not implemented in persistor', {
-      method: 'listDirectoryKeys',
-      location,
-      prefix,
-    })
-  }
-
-  /**
-   * List objects in a directory, returning key and size information.
-   *
-   * Suitable only for directories where the number of keys is known to be small.
-   *
-   * @param {string} location
-   * @param {string} prefix
-   * @returns {Promise<Array<{key: string, size: number}>>}
-   */
-  async listDirectoryStats(location, prefix) {
-    throw new NotImplementedError('method not implemented in persistor', {
-      method: 'listDirectoryStats',
-      location,
-      prefix,
-    })
-  }
 }

libraries/object-persistor/src/FSPersistor.js

@@ -1,15 +1,18 @@
 const crypto = require('node:crypto')
 const fs = require('node:fs')
 const fsPromises = require('node:fs/promises')
-const { glob } = require('glob')
+const globCallbacks = require('glob')
 const Path = require('node:path')
 const { PassThrough } = require('node:stream')
 const { pipeline } = require('node:stream/promises')
+const { promisify } = require('node:util')
 
 const AbstractPersistor = require('./AbstractPersistor')
 const { ReadError, WriteError, NotImplementedError } = require('./Errors')
 const PersistorHelper = require('./PersistorHelper')
 
+const glob = promisify(globCallbacks)
+
 module.exports = class FSPersistor extends AbstractPersistor {
   constructor(settings = {}) {
     if (settings.storageClass) {
@@ -83,7 +86,7 @@ module.exports = class FSPersistor extends AbstractPersistor {
       metric: 'fs.ingress', // ingress to us from disk
       bucket: location,
     })
-    const fsPath = this._getFsPath(location, name, opts.useSubdirectories)
+    const fsPath = this._getFsPath(location, name)
 
     try {
       opts.fd = await fsPromises.open(fsPath, 'r')
@@ -194,50 +197,6 @@ module.exports = class FSPersistor extends AbstractPersistor {
     }
   }
 
-  async listDirectoryKeys(location, name) {
-    const fsPath = this._getFsPath(location, name)
-    const paths = await this._listDirectory(fsPath)
-
-    // Filter to only return files, not directories
-    const files = []
-    for (const path of paths) {
-      try {
-        const stat = await fsPromises.stat(path)
-        if (stat.isFile()) {
-          files.push(path)
-        }
-      } catch (err) {
-        // ignore files that may have just been deleted
-        if (err.code !== 'ENOENT') {
-          throw err
-        }
-      }
-    }
-    return files
-  }
-
-  async listDirectoryStats(location, name) {
-    const fsPath = this._getFsPath(location, name)
-    const paths = await this._listDirectory(fsPath)
-
-    // Filter to only return files, not directories, with their sizes
-    const stats = []
-    for (const path of paths) {
-      try {
-        const stat = await fsPromises.stat(path)
-        if (stat.isFile()) {
-          stats.push({ key: path, size: stat.size })
-        }
-      } catch (err) {
-        // ignore files that may have just been deleted
-        if (err.code !== 'ENOENT') {
-          throw err
-        }
-      }
-    }
-    return stats
-  }
-
   async checkIfObjectExists(location, name) {
     const fsPath = this._getFsPath(location, name)
     try {
@@ -336,9 +295,9 @@ module.exports = class FSPersistor extends AbstractPersistor {
     await fsPromises.rm(dirPath, { force: true, recursive: true })
   }
 
-  _getFsPath(location, key, useSubdirectories = false) {
+  _getFsPath(location, key) {
     key = key.replace(/\/$/, '')
-    if (!this.useSubdirectories && !useSubdirectories) {
+    if (!this.useSubdirectories) {
       key = key.replace(/\//g, '_')
     }
     return Path.join(location, key)
@@ -346,8 +305,10 @@ module.exports = class FSPersistor extends AbstractPersistor {
 
   async _listDirectory(path) {
     if (this.useSubdirectories) {
+      // eslint-disable-next-line @typescript-eslint/return-await
       return await glob(Path.join(path, '**'))
     } else {
+      // eslint-disable-next-line @typescript-eslint/return-await
       return await glob(`${path}_*`)
     }
   }

libraries/object-persistor/src/GcsPersistor.js

@@ -288,25 +288,23 @@ module.exports = class GcsPersistor extends AbstractPersistor {
     } while (query)
   }
 
-  async #listDirectory(bucketName, prefix) {
+  async directorySize(bucketName, key) {
+    let files
+    const prefix = ensurePrefixIsDirectory(key)
+
     try {
       const [response] = await this.storage
         .bucket(bucketName)
         .getFiles({ prefix })
-      return response
+      files = response
     } catch (err) {
       throw PersistorHelper.wrapError(
         err,
         'failed to list objects in GCS',
-        { bucketName, prefix },
+        { bucketName, key },
         ReadError
       )
     }
-  }
-
-  async directorySize(bucketName, key) {
-    const prefix = ensurePrefixIsDirectory(key)
-    const files = await this.#listDirectory(bucketName, prefix)
 
     return files.reduce(
       (acc, file) => parseInt(file.metadata.size, 10) + acc,
@@ -314,25 +312,6 @@ module.exports = class GcsPersistor extends AbstractPersistor {
     )
   }
 
-  async listDirectoryKeys(bucketName, prefix) {
-    const files = await this.#listDirectory(
-      bucketName,
-      ensurePrefixIsDirectory(prefix)
-    )
-    return files.map(file => file.name)
-  }
-
-  async listDirectoryStats(bucketName, prefix) {
-    const files = await this.#listDirectory(
-      bucketName,
-      ensurePrefixIsDirectory(prefix)
-    )
-    return files.map(file => ({
-      key: file.name,
-      size: parseInt(file.metadata.size, 10),
-    }))
-  }
-
   async checkIfObjectExists(bucketName, key) {
     try {
       const [response] = await this.storage

libraries/object-persistor/src/MigrationPersistor.js

@@ -171,7 +171,19 @@ module.exports = class MigrationPersistor extends AbstractPersistor {
     destKey
   ) {
     try {
-      await this.primaryPersistor.sendStream(destBucket, destKey, stream)
+      let sourceMd5
+      try {
+        sourceMd5 = await this.fallbackPersistor.getObjectMd5Hash(
+          sourceBucket,
+          sourceKey
+        )
+      } catch (err) {
+        Logger.warn(err, 'error getting md5 hash from fallback persistor')
+      }
+
+      await this.primaryPersistor.sendStream(destBucket, destKey, stream, {
+        sourceMd5,
+      })
     } catch (err) {
       const error = new WriteError(
         'unable to copy file to destination persistor',

libraries/object-persistor/src/PerProjectEncryptedS3Persistor.js

@@ -20,6 +20,10 @@ const hkdf = promisify(Crypto.hkdf)
 
 const AES256_KEY_LENGTH = 32
 
+/**
+ * @typedef {import('aws-sdk').AWSError} AWSError
+ */
+
 /**
  * @typedef {Object} Settings
  * @property {boolean} automaticallyRotateDEKEncryption
@@ -30,8 +34,14 @@ const AES256_KEY_LENGTH = 32
  */
 
 /**
- * @typedef {import('./types').ListDirectoryResult} ListDirectoryResult
+ * Helper function to make TS happy when accessing error properties
+ * AWSError is not an actual class, so we cannot use instanceof.
+ * @param {any} err
+ * @return {err is AWSError}
  */
+function isAWSError(err) {
+  return !!err
+}
 
 /**
  * @param {any} err
@@ -41,8 +51,9 @@ function isForbiddenError(err) {
   if (!err || !(err instanceof ReadError || err instanceof NotFoundError)) {
     return false
   }
-  // @ts-ignore
-  return err?.cause.statusCode === 403 || err?.cause.Code === 'AccessDenied'
+  const cause = err.cause
+  if (!isAWSError(cause)) return false
+  return cause.statusCode === 403
 }
 
 class RootKeyEncryptionKey {
@@ -380,9 +391,9 @@ class PerProjectEncryptedS3Persistor extends S3Persistor {
  * A general "cache" for project keys is another alternative. For now, use a helper class.
  */
 class CachedPerProjectEncryptedS3Persistor {
-  /** @type SSECOptions  */
+  /** @type SSECOptions */
   #projectKeyOptions
-  /** @type PerProjectEncryptedS3Persistor  */
+  /** @type PerProjectEncryptedS3Persistor */
   #parent
 
   /**
@@ -413,26 +424,6 @@ class CachedPerProjectEncryptedS3Persistor {
     return await this.#parent.getObjectSize(bucketName, path)
   }
 
-  /**
-   *
-   * @param {string} bucketName
-   * @param {string} path
-   * @return {Promise<string[]>}
-   */
-  async listDirectoryKeys(bucketName, path) {
-    return await this.#parent.listDirectoryKeys(bucketName, path)
-  }
-
-  /**
-   *
-   * @param {string} bucketName
-   * @param {string} path
-   * @return {Promise<Array<{key: string, size: number}>>}
-   */
-  async listDirectoryStats(bucketName, path) {
-    return await this.#parent.listDirectoryStats(bucketName, path)
-  }
-
   /**
    * @param {string} bucketName
    * @param {string} path
@@ -443,6 +434,7 @@ class CachedPerProjectEncryptedS3Persistor {
    * @param {number} [opts.contentLength]
    * @param {'*'} [opts.ifNoneMatch]
    * @param {SSECOptions} [opts.ssecOptions]
+   * @param {string} [opts.sourceMd5]
    * @return {Promise<void>}
    */
   async sendStream(bucketName, path, sourceStream, opts = {}) {

libraries/object-persistor/src/PersistorHelper.js

@@ -144,22 +144,17 @@ function wrapError(error, message, params, ErrorType) {
     ...params,
     cause: error,
   }
-
-  // aws-sdk v3 renames `code` to `Code`, but it's not always present, so we
-  // add a fallback to `name` for compatibility.
-  const errorCode = error.code || error.Code || error.name
-
   if (
     error instanceof NotFoundError ||
     ['NoSuchKey', 'NotFound', 404, 'AccessDenied', 'ENOENT'].includes(
-      errorCode
+      error.code
     ) ||
     (error.response && error.response.statusCode === 404)
   ) {
     return new NotFoundError('no such file', params, error)
   } else if (
     params.ifNoneMatch === '*' &&
-    (errorCode === 'PreconditionFailed' ||
+    (error.code === 'PreconditionFailed' ||
       error.response?.statusCode === 412 ||
       error instanceof AlreadyWrittenError)
   ) {

libraries/object-persistor/src/ProjectKey.js

@@ -1,23 +0,0 @@
-const path = require('node:path')
-
-//
-// The advice in http://docs.aws.amazon.com/AmazonS3/latest/dev/
-// request-rate-perf-considerations.html is to avoid sequential key prefixes,
-// so we reverse the project ID part of the key as they suggest.
-//
-function format(projectId) {
-  const prefix = naiveReverse(pad(projectId))
-  return path.join(prefix.slice(0, 3), prefix.slice(3, 6), prefix.slice(6))
-}
-
-function pad(number) {
-  return (number || 0).toString().padStart(9, '0')
-}
-
-function naiveReverse(string) {
-  return string.split('').reverse().join('')
-}
-
-exports.format = format
-exports.pad = pad
-exports.naiveReverse = naiveReverse

libraries/object-persistor/src/S3Md5Middleware.js

@@ -1,49 +0,0 @@
-const Crypto = require('node:crypto')
-
-/**
- * Creates an S3 client that uses MD5 checksums for DeleteObjects operations
- * https://github.com/aws/aws-sdk-js-v3/blob/main/supplemental-docs/MD5_FALLBACK.md
- */
-const md5Middleware = (next, context) => async args => {
-  if (context.commandName !== 'DeleteObjectsCommand') {
-    return next(args)
-  }
-
-  const headers = args.request.headers
-
-  // Remove any checksum headers added by default middleware
-  // This ensures our Content-MD5 is the primary integrity check
-  Object.keys(headers).forEach(header => {
-    const lowerHeader = header.toLowerCase()
-    if (
-      lowerHeader.startsWith('x-amz-checksum-') ||
-      lowerHeader.startsWith('x-amz-sdk-checksum-')
-    ) {
-      delete headers[header]
-    }
-  })
-
-  if (args.request.body) {
-    const bodyContent = Buffer.from(args.request.body)
-    headers['Content-MD5'] = Crypto.createHash('md5')
-      .update(bodyContent)
-      .digest('base64')
-  }
-
-  return await next(args)
-}
-
-function addMd5Middleware(client) {
-  // Add the middleware relative to the flexible checksums middleware
-  // This ensures it runs after default checksums might be added, but before signing
-  client.middlewareStack.add(md5Middleware, {
-    step: 'build',
-    toMiddleware: 'flexibleChecksumsMiddleware',
-    name: 'addMD5ChecksumForDeleteObjects',
-    tags: ['MD5_FALLBACK'],
-  })
-}
-
-module.exports = {
-  addMd5Middleware,
-}

libraries/object-persistor/src/S3Persistor.js

@@ -15,27 +15,10 @@ const PersistorHelper = require('./PersistorHelper')
 
 const { pipeline, PassThrough } = require('node:stream')
 const fs = require('node:fs')
-const {
-  S3Client,
-  CreateBucketCommand,
-  GetObjectCommand,
-  PutObjectCommand,
-  HeadObjectCommand,
-  DeleteObjectCommand,
-  ListObjectsV2Command,
-  DeleteObjectsCommand,
-  CopyObjectCommand,
-} = require('@aws-sdk/client-s3')
-const { Upload } = require('@aws-sdk/lib-storage')
-const { getSignedUrl } = require('@aws-sdk/s3-request-presigner')
-const { NodeHttpHandler } = require('@aws-sdk/node-http-handler')
+const S3 = require('aws-sdk/clients/s3')
+const { URL } = require('node:url')
 const { WriteError, ReadError, NotFoundError } = require('./Errors')
 const zlib = require('node:zlib')
-const { addMd5Middleware } = require('./S3Md5Middleware')
-
-/**
- * @typedef {import('./types').ListDirectoryResult} ListDirectoryResult
- */
 
 /**
  * Wrapper with private fields to avoid revealing them on console, JSON.stringify or similar.
@@ -78,7 +61,7 @@ class SSECOptions {
 }
 
 class S3Persistor extends AbstractPersistor {
-  /** @type {Map<string, S3Client>} */
+  /** @type {Map<string, S3>} */
   #clients = new Map()
 
   constructor(settings = {}) {
@@ -108,6 +91,7 @@ class S3Persistor extends AbstractPersistor {
    * @param {number} [opts.contentLength]
    * @param {'*'} [opts.ifNoneMatch]
    * @param {SSECOptions} [opts.ssecOptions]
+   * @param {string} [opts.sourceMd5]
    * @return {Promise<void>}
    */
   async sendStream(bucketName, key, readStream, opts = {}) {
@@ -121,7 +105,7 @@ class S3Persistor extends AbstractPersistor {
       // observer will catch errors, clean up and log a warning
       pipeline(readStream, observer, () => {})
 
-      /** @type {import('@aws-sdk/client-s3').PutObjectCommandInput} */
+      /** @type {S3.PutObjectRequest} */
       const uploadOptions = {
         Bucket: bucketName,
         Key: key,
@@ -132,12 +116,6 @@ class S3Persistor extends AbstractPersistor {
         uploadOptions.StorageClass = this.settings.storageClass[bucketName]
       }
 
-      if ('sourceMd5' in opts) {
-        // we fail straight away to prevent the client from wasting CPU/IO precomputing the hash
-        throw new Error(
-          'sourceMd5 option is not supported, S3 provides its own integrity protection mechanism'
-        )
-      }
       if (opts.contentType) {
         uploadOptions.ContentType = opts.contentType
       }
@@ -154,12 +132,24 @@ class S3Persistor extends AbstractPersistor {
         Object.assign(uploadOptions, opts.ssecOptions.getPutOptions())
       }
 
-      const upload = new Upload({
-        client: this._getClientForBucket(bucketName),
-        params: uploadOptions,
-        partSize: this.settings.partSize,
-      })
-      await upload.done()
+      // if we have an md5 hash, pass this to S3 to verify the upload - otherwise
+      // we rely on the S3 client's checksum calculation to validate the upload
+      let computeChecksums = false
+      if (opts.sourceMd5) {
+        uploadOptions.ContentMD5 = PersistorHelper.hexToBase64(opts.sourceMd5)
+      } else {
+        computeChecksums = true
+      }
+
+      if (this.settings.disableMultiPartUpload) {
+        await this._getClientForBucket(bucketName, computeChecksums)
+          .putObject(uploadOptions)
+          .promise()
+      } else {
+        await this._getClientForBucket(bucketName, computeChecksums)
+          .upload(uploadOptions, { partSize: this.settings.partSize })
+          .promise()
+      }
     } catch (err) {
       throw PersistorHelper.wrapError(
         err,
@@ -183,7 +173,6 @@ class S3Persistor extends AbstractPersistor {
   async getObjectStream(bucketName, key, opts) {
     opts = opts || {}
 
-    /** @type {import('@aws-sdk/client-s3').GetObjectCommandInput} */
     const params = {
       Bucket: bucketName,
       Key: key,
@@ -199,18 +188,32 @@ class S3Persistor extends AbstractPersistor {
       bucket: bucketName,
     })
 
-    const abortController = new AbortController()
-    let stream, contentEncoding
+    const req = this._getClientForBucket(bucketName).getObject(params)
+    const stream = req.createReadStream()
+
+    let contentEncoding
     try {
-      const { Body, ContentEncoding } = await this._getClientForBucket(
-        bucketName
-      ).send(new GetObjectCommand(params), {
-        abortSignal: abortController.signal,
+      await new Promise((resolve, reject) => {
+        req.on('httpHeaders', (statusCode, headers) => {
+          switch (statusCode) {
+            case 200: // full response
+            case 206: // partial response
+              contentEncoding = headers['content-encoding']
+              return resolve(undefined)
+            case 403: // AccessDenied
+              return // handled by stream.on('error') handler below
+            case 404: // NoSuchKey
+              return reject(new NotFoundError('not found'))
+            default:
+            // handled by stream.on('error') handler below
+          }
+        })
+        // The AWS SDK is forwarding any errors from the request to the stream.
+        // The AWS SDK is emitting additional errors on the stream ahead of starting to stream.
+        stream.on('error', reject)
+        // The AWS SDK is kicking off the request in the next event loop cycle.
       })
-      stream = Body
-      contentEncoding = ContentEncoding
     } catch (err) {
-      abortController.abort()
       throw PersistorHelper.wrapError(
         err,
         'error reading file from S3',
@@ -224,11 +227,8 @@ class S3Persistor extends AbstractPersistor {
     if (contentEncoding === 'gzip' && opts.autoGunzip) {
       transformer.push(zlib.createGunzip())
     }
-    // @ts-ignore stream (Body) can be undefined in GetObjectCommand
     pipeline(stream, observer, ...transformer, pass, err => {
-      if (err) {
-        abortController.abort()
-      }
+      if (err) req.abort()
     })
     return pass
   }
@@ -241,13 +241,13 @@ class S3Persistor extends AbstractPersistor {
   async getRedirectUrl(bucketName, key) {
     const expiresSeconds = Math.round(this.settings.signedUrlExpiryInMs / 1000)
     try {
-      return await getSignedUrl(
-        this._getClientForBucket(bucketName),
-        new GetObjectCommand({
+      return await this._getClientForBucket(bucketName).getSignedUrlPromise(
+        'getObject',
+        {
           Bucket: bucketName,
           Key: key,
-        }),
-        { expiresIn: expiresSeconds }
+          Expires: expiresSeconds,
+        }
       )
     } catch (err) {
       throw PersistorHelper.wrapError(
@@ -266,23 +266,37 @@ class S3Persistor extends AbstractPersistor {
    * @return {Promise<void>}
    */
   async deleteDirectory(bucketName, key, continuationToken) {
-    const { contents, response } = await this.#listDirectory(
-      bucketName,
-      key,
-      continuationToken
-    )
-    const objects = contents.map(item => ({ Key: item.Key || '' }))
+    let response
+    const options = { Bucket: bucketName, Prefix: key }
+    if (continuationToken) {
+      options.ContinuationToken = continuationToken
+    }
+
+    try {
+      response = await this._getClientForBucket(bucketName)
+        .listObjectsV2(options)
+        .promise()
+    } catch (err) {
+      throw PersistorHelper.wrapError(
+        err,
+        'failed to list objects in S3',
+        { bucketName, key },
+        ReadError
+      )
+    }
+
+    const objects = response.Contents?.map(item => ({ Key: item.Key || '' }))
     if (objects?.length) {
       try {
-        await this._getClientForBucket(bucketName).send(
-          new DeleteObjectsCommand({
+        await this._getClientForBucket(bucketName)
+          .deleteObjects({
             Bucket: bucketName,
             Delete: {
               Objects: objects,
               Quiet: true,
             },
           })
-        )
+          .promise()
       } catch (err) {
         throw PersistorHelper.wrapError(
           err,
@@ -302,42 +316,12 @@ class S3Persistor extends AbstractPersistor {
     }
   }
 
-  /**
-   *
-   * @param {string} bucketName
-   * @param {string} key
-   * @param {string} [continuationToken]
-   * @return {Promise<ListDirectoryResult>}
-   */
-  async #listDirectory(bucketName, key, continuationToken) {
-    let response
-    const options = { Bucket: bucketName, Prefix: key }
-    if (continuationToken) {
-      options.ContinuationToken = continuationToken
-    }
-
-    try {
-      response = await this._getClientForBucket(bucketName).send(
-        new ListObjectsV2Command(options)
-      )
-    } catch (err) {
-      throw PersistorHelper.wrapError(
-        err,
-        'failed to list objects in S3',
-        { bucketName, key },
-        ReadError
-      )
-    }
-
-    return { contents: response.Contents ?? [], response }
-  }
-
   /**
    * @param {string} bucketName
    * @param {string} key
    * @param {Object} opts
    * @param {SSECOptions} [opts.ssecOptions]
-   * @return {Promise<import('@aws-sdk/client-s3').HeadObjectOutput>}
+   * @return {Promise<S3.HeadObjectOutput>}
    */
   async #headObject(bucketName, key, opts = {}) {
     const params = { Bucket: bucketName, Key: key }
@@ -345,8 +329,9 @@ class S3Persistor extends AbstractPersistor {
       Object.assign(params, opts.ssecOptions.getGetOptions())
     }
     try {
-      const client = await this._getClientForBucket(bucketName)
-      return await client.send(new HeadObjectCommand(params))
+      return await this._getClientForBucket(bucketName)
+        .headObject(params)
+        .promise()
     } catch (err) {
       throw PersistorHelper.wrapError(
         err,
@@ -420,9 +405,9 @@ class S3Persistor extends AbstractPersistor {
    */
   async deleteObject(bucketName, key) {
     try {
-      await this._getClientForBucket(bucketName).send(
-        new DeleteObjectCommand({ Bucket: bucketName, Key: key })
-      )
+      await this._getClientForBucket(bucketName)
+        .deleteObject({ Bucket: bucketName, Key: key })
+        .promise()
     } catch (err) {
       // s3 does not give us a NotFoundError here
       throw PersistorHelper.wrapError(
@@ -447,7 +432,7 @@ class S3Persistor extends AbstractPersistor {
     const params = {
       Bucket: bucketName,
       Key: destKey,
-      CopySource: `/${bucketName}/${sourceKey}`,
+      CopySource: `${bucketName}/${sourceKey}`,
     }
     if (opts.ssecSrcOptions) {
       Object.assign(params, opts.ssecSrcOptions.getCopyOptions())
@@ -456,9 +441,7 @@ class S3Persistor extends AbstractPersistor {
       Object.assign(params, opts.ssecOptions.getPutOptions())
     }
     try {
-      await this._getClientForBucket(bucketName).send(
-        new CopyObjectCommand(params)
-      )
+      await this._getClientForBucket(bucketName).copyObject(params).promise()
     } catch (err) {
       throw PersistorHelper.wrapError(
         err,
@@ -508,9 +491,9 @@ class S3Persistor extends AbstractPersistor {
       if (continuationToken) {
         options.ContinuationToken = continuationToken
       }
-      const response = await this._getClientForBucket(bucketName).send(
-        new ListObjectsV2Command(options)
-      )
+      const response = await this._getClientForBucket(bucketName)
+        .listObjectsV2(options)
+        .promise()
 
       const size =
         response.Contents?.reduce((acc, item) => (item.Size || 0) + acc, 0) || 0
@@ -535,95 +518,36 @@ class S3Persistor extends AbstractPersistor {
     }
   }
 
-  /**
-   * @param {string} bucketName
-   * @param {string} prefix
-   * @return {Promise<Array<string>>}
-   */
-  async listDirectoryKeys(bucketName, prefix) {
-    const keys = []
-    let continuationToken
-
-    do {
-      const { contents, response } = await this.#listDirectory(
-        bucketName,
-        prefix,
-        continuationToken
-      )
-      keys.push(...contents.map(item => item.Key || ''))
-      continuationToken = response.IsTruncated
-        ? response.NextContinuationToken
-        : undefined
-    } while (continuationToken)
-
-    return keys
-  }
-
-  /**
-   * @param {string} bucketName
-   * @param {string} prefix
-   * @return {Promise<Array<{key: string, size: number}>>}
-   */
-  async listDirectoryStats(bucketName, prefix) {
-    const stats = []
-    let continuationToken
-
-    do {
-      const { contents, response } = await this.#listDirectory(
-        bucketName,
-        prefix,
-        continuationToken
-      )
-      stats.push(
-        ...contents.map(item => ({
-          key: item.Key || '',
-          size: item.Size ?? -1,
-        }))
-      )
-      continuationToken = response.IsTruncated
-        ? response.NextContinuationToken
-        : undefined
-    } while (continuationToken)
-
-    return stats
-  }
-
   /**
    * @param {string} bucket
-   * @return {S3Client}
+   * @param {boolean} computeChecksums
+   * @return {S3}
    * @private
    */
-  _getClientForBucket(bucket) {
-    /** @type {import('@aws-sdk/client-s3').S3ClientConfig} */
+  _getClientForBucket(bucket, computeChecksums = false) {
+    /** @type {S3.Types.ClientConfiguration} */
     const clientOptions = {}
-    const cacheKey = bucket
-
+    const cacheKey = `${bucket}:${computeChecksums}`
+    if (computeChecksums) {
+      clientOptions.computeChecksums = true
+    }
     let client = this.#clients.get(cacheKey)
     if (!client) {
-      client = new S3Client(
+      client = new S3(
         this._buildClientOptions(
           this.settings.bucketCreds?.[bucket],
           clientOptions
         )
       )
       this.#clients.set(cacheKey, client)
-
-      // Some third-party S3-compatible services (as MinIO) do not support the default checksums
-      // for object integrity in `DeleteObjectsCommand`. We add a fallback for those cases.
-      // https://github.com/aws/aws-sdk-js-v3/blob/main/supplemental-docs/MD5_FALLBACK.md
-      const useMd5Fallback = process.env.DELETE_OBJECTS_MD5_FALLBACK === 'true'
-      if (useMd5Fallback) {
-        addMd5Middleware(client)
-      }
     }
-
     return client
   }
 
   /**
    * @param {Object} bucketCredentials
-   * @param {import('@aws-sdk/client-s3').S3ClientConfig} clientOptions
-   * @return {import('@aws-sdk/client-s3').S3ClientConfig}
+   * @param {S3.Types.ClientConfiguration} clientOptions
+   * @return {S3.Types.ClientConfiguration}
    * @private
    */
   _buildClientOptions(bucketCredentials, clientOptions) {
@@ -641,89 +565,39 @@ class S3Persistor extends AbstractPersistor {
       }
     } else {
       // Use the default credentials provider (process.env -> SSP -> ini -> IAM)
-      // Docs: https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/setting-credentials-node.html
+      // Docs: https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/CredentialProviderChain.html#defaultProviders-property
     }
 
-    // region is required since aws sdk v3
-    options.region =
-      this.settings.region || process.env.AWS_REGION || 'us-east-1'
-
-    let sslEnabled = false
     if (this.settings.endpoint) {
       const endpoint = new URL(this.settings.endpoint)
       options.endpoint = this.settings.endpoint
-      sslEnabled = endpoint.protocol === 'https:'
+      options.sslEnabled = endpoint.protocol === 'https:'
     }
 
     // path-style access is only used for acceptance tests
     if (this.settings.pathStyle) {
-      options.forcePathStyle = true
+      options.s3ForcePathStyle = true
     }
 
-    // maxRetries has been moved to maxAttempts in aws-sdk v3,
-    // we're keeping the existing setting for backwards compatibility
-    if (this.settings.maxRetries) {
-      options.maxAttempts = this.settings.maxRetries + 1
+    for (const opt of ['httpOptions', 'maxRetries', 'region']) {
+      if (this.settings[opt]) {
+        options[opt] = this.settings[opt]
+      }
     }
 
-    const requestHandlerParams = this.settings.httpOptions || {}
-
-    if (sslEnabled && this.settings.ca) {
-      const agent = new https.Agent({
+    if (options.sslEnabled && this.settings.ca && !options.httpOptions?.agent) {
+      options.httpOptions = options.httpOptions || {}
+      options.httpOptions.agent = new https.Agent({
         rejectUnauthorized: true,
         ca: this.settings.ca,
       })
-      Object.assign(requestHandlerParams, {
-        httpAgent: agent,
-        httpsAgent: agent,
-      })
-      options.requestHandler = new NodeHttpHandler({
-        httpAgent: agent,
-        httpsAgent: agent,
-      })
-    }
-
-    const requestHandler = this._buildNodeHttpHandler(requestHandlerParams)
-    if (requestHandler) {
-      options.requestHandler = requestHandler
     }
 
     return options
   }
 
   /**
-   * @param {import('@aws-sdk/node-http-handler').NodeHttpHandlerOptions & { timeout?: number }} params
-   * @private
-   */
-  _buildNodeHttpHandler(params) {
-    const isEmpty = Object.keys(params).length === 0
-    if (isEmpty) {
-      return
-    }
-    // ensures backwards compatibility with aws-sdk v2 httpOptions
-    if (params.timeout) {
-      params.connectionTimeout = params.timeout
-      delete params.timeout
-    }
-    return new NodeHttpHandler(params)
-  }
-
-  // test-only
-  _createBucket(bucketName) {
-    return this._getClientForBucket(bucketName).send(
-      new CreateBucketCommand({ Bucket: bucketName })
-    )
-  }
-
-  // test-only
-  _upload(bucketName, uploadOptions) {
-    return this._getClientForBucket(bucketName).send(
-      new PutObjectCommand(uploadOptions)
-    )
-  }
-
-  /**
-   * @param {import('@aws-sdk/client-s3').HeadObjectOutput} response
+   * @param {S3.HeadObjectOutput} response
    * @return {string|null}
    * @private
    */

libraries/object-persistor/src/types.d.ts

@@ -1,6 +0,0 @@
-import type { ListObjectsV2Output, _Object } from '@aws-sdk/client-s3'
-
-export type ListDirectoryResult = {
-  contents: Array<_Object>
-  response: ListObjectsV2Output
-}

libraries/object-persistor/test/Init.js

@@ -0,0 +1,33 @@
+const SandboxedModule = require('sandboxed-module')
+const chai = require('chai')
+const sinon = require('sinon')
+
+chai.use(require('sinon-chai'))
+chai.use(require('chai-as-promised'))
+
+SandboxedModule.configure({
+  requires: {
+    '@overleaf/logger': {
+      debug() {},
+      log() {},
+      info() {},
+      warn() {},
+      error() {},
+      err() {},
+    },
+    '@overleaf/metrics': {
+      inc: sinon.stub(),
+      count: sinon.stub(),
+      histogram: sinon.stub(),
+      Timer: class Timer {
+        done() {}
+      },
+    },
+  },
+  globals: { Buffer, Math, console, process, URL },
+  sourceTransformers: {
+    removeNodePrefix: function (source) {
+      return source.replace(/require\(['"]node:/g, "require('")
+    },
+  },
+})

libraries/object-persistor/test/mocha-multi-reporters.cjs

@@ -1,9 +0,0 @@
-module.exports = {
-  reporterEnabled: 'spec, mocha-junit-reporter',
-  mochaJunitReporterReporterOptions: {
-    mochaFile: `reports/junit-mocha-${process.env.MOCHA_GREP}.xml`,
-    includePending: true,
-    jenkinsMode: true,
-    output: true,
-  },
-}

libraries/object-persistor/test/setup.js

@@ -1,33 +0,0 @@
-const SandboxedModule = require('sandboxed-module')
-const chai = require('chai')
-const sinon = require('sinon')
-
-chai.use(require('sinon-chai'))
-chai.use(require('chai-as-promised'))
-
-SandboxedModule.configure({
-  requires: {
-    '@overleaf/logger': {
-      debug() {},
-      log() {},
-      info() {},
-      warn() {},
-      error() {},
-      err() {},
-    },
-    '@overleaf/metrics': {
-      inc: sinon.stub(),
-      count: sinon.stub(),
-      histogram: sinon.stub(),
-      Timer: class Timer {
-        done() {}
-      },
-    },
-  },
-  globals: { Buffer, Math, console, process, URL },
-  sourceTransformers: {
-    removeNodePrefix: function (source) {
-      return source.replace(/require\((['"])node:/g, 'require($1')
-    },
-  },
-})

libraries/object-persistor/test/unit/FSPersistorTests.js

@@ -400,56 +400,6 @@ describe('FSPersistorTests', function () {
           ).to.equal(0)
         })
       })
-
-      describe('listDirectoryKeys', function () {
-        beforeEach(async function () {
-          for (const file of Object.values(files)) {
-            await persistor.sendFile(location, file, '/uploads/info.txt')
-          }
-        })
-
-        it('should list directory keys', async function () {
-          const keys = await persistor.listDirectoryKeys(location, 'animals')
-          expect(keys).to.have.lengthOf(2)
-          expect(keys).to.include(scenario.fsPath(files.wombat))
-          expect(keys).to.include(scenario.fsPath(files.giraffe))
-        })
-
-        it('should return empty array for non-existing directories', async function () {
-          const keys = await persistor.listDirectoryKeys(
-            location,
-            'does-not-exist'
-          )
-          expect(keys).to.deep.equal([])
-        })
-      })
-
-      describe('listDirectoryStats', function () {
-        beforeEach(async function () {
-          for (const file of Object.values(files)) {
-            await persistor.sendFile(location, file, '/uploads/info.txt')
-          }
-        })
-
-        it('should list directory stats', async function () {
-          const stats = await persistor.listDirectoryStats(location, 'animals')
-          expect(stats).to.have.lengthOf(2)
-          const keys = stats.map(s => s.key)
-          expect(keys).to.include(scenario.fsPath(files.wombat))
-          expect(keys).to.include(scenario.fsPath(files.giraffe))
-          for (const stat of stats) {
-            expect(stat.size).to.equal(localFiles['/uploads/info.txt'].length)
-          }
-        })
-
-        it('should return empty array for non-existing directories', async function () {
-          const stats = await persistor.listDirectoryStats(
-            location,
-            'does-not-exist'
-          )
-          expect(stats).to.deep.equal([])
-        })
-      })
     })
   }
 })

libraries/object-persistor/test/unit/GcsPersistorTests.js

@@ -723,127 +723,4 @@ describe('GcsPersistorTests', function () {
       })
     })
   })
-
-  describe('listDirectoryKeys', function () {
-    describe('with valid parameters', function () {
-      let keys
-
-      beforeEach(async function () {
-        const filesWithNames = files.map((file, i) => ({
-          ...file,
-          name: i === 0 ? 'llama' : 'hippo',
-        }))
-        GcsBucket.getFiles.resolves([filesWithNames])
-        keys = await GcsPersistor.listDirectoryKeys(bucket, key)
-      })
-
-      it('should list the objects in the directory', function () {
-        expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
-        expect(GcsBucket.getFiles).to.have.been.calledWith({
-          prefix: `${key}/`,
-        })
-      })
-
-      it('should return the keys', function () {
-        expect(keys).to.deep.equal(['llama', 'hippo'])
-      })
-    })
-
-    describe('when there are no files', function () {
-      let keys
-
-      beforeEach(async function () {
-        GcsBucket.getFiles.resolves([[]])
-        keys = await GcsPersistor.listDirectoryKeys(bucket, key)
-      })
-
-      it('should return an empty array', function () {
-        expect(keys).to.deep.equal([])
-      })
-    })
-
-    describe('when there is an error listing the objects', function () {
-      let error
-
-      beforeEach(async function () {
-        GcsBucket.getFiles.rejects(genericError)
-        try {
-          await GcsPersistor.listDirectoryKeys(bucket, key)
-        } catch (err) {
-          error = err
-        }
-      })
-
-      it('should generate a ReadError', function () {
-        expect(error).to.be.an.instanceOf(Errors.ReadError)
-      })
-
-      it('should wrap the error', function () {
-        expect(error.cause).to.equal(genericError)
-      })
-    })
-  })
-
-  describe('listDirectoryStats', function () {
-    describe('with valid parameters', function () {
-      let stats
-
-      beforeEach(async function () {
-        const filesWithNames = files.map((file, i) => ({
-          ...file,
-          name: i === 0 ? 'llama' : 'hippo',
-        }))
-        GcsBucket.getFiles.resolves([filesWithNames])
-        stats = await GcsPersistor.listDirectoryStats(bucket, key)
-      })
-
-      it('should list the objects in the directory', function () {
-        expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
-        expect(GcsBucket.getFiles).to.have.been.calledWith({
-          prefix: `${key}/`,
-        })
-      })
-
-      it('should return the stats', function () {
-        expect(stats).to.deep.equal([
-          { key: 'llama', size: 11 },
-          { key: 'hippo', size: 22 },
-        ])
-      })
-    })
-
-    describe('when there are no files', function () {
-      let stats
-
-      beforeEach(async function () {
-        GcsBucket.getFiles.resolves([[]])
-        stats = await GcsPersistor.listDirectoryStats(bucket, key)
-      })
-
-      it('should return an empty array', function () {
-        expect(stats).to.deep.equal([])
-      })
-    })
-
-    describe('when there is an error listing the objects', function () {
-      let error
-
-      beforeEach(async function () {
-        GcsBucket.getFiles.rejects(genericError)
-        try {
-          await GcsPersistor.listDirectoryStats(bucket, key)
-        } catch (err) {
-          error = err
-        }
-      })
-
-      it('should generate a ReadError', function () {
-        expect(error).to.be.an.instanceOf(Errors.ReadError)
-      })
-
-      it('should wrap the error', function () {
-        expect(error.cause).to.equal(genericError)
-      })
-    })
-  })
 })

libraries/object-persistor/test/unit/MigrationPersistorTests.js

@@ -197,11 +197,19 @@ describe('MigrationPersistorTests', function () {
         expect(fallbackPersistor.getObjectStream).to.have.been.calledOnce
       })
 
+      it('should get the md5 hash from the source', function () {
+        expect(fallbackPersistor.getObjectMd5Hash).to.have.been.calledWith(
+          fallbackBucket,
+          key
+        )
+      })
+
       it('should send a stream to the primary', function () {
         expect(primaryPersistor.sendStream).to.have.been.calledWithExactly(
           bucket,
           key,
-          sinon.match.instanceOf(Stream.PassThrough)
+          sinon.match.instanceOf(Stream.PassThrough),
+          { sourceMd5: md5 }
         )
       })
 
@@ -468,11 +476,19 @@ describe('MigrationPersistorTests', function () {
         ).not.to.have.been.calledWithExactly(fallbackBucket, key)
       })
 
+      it('should get the md5 hash from the source', function () {
+        expect(fallbackPersistor.getObjectMd5Hash).to.have.been.calledWith(
+          fallbackBucket,
+          key
+        )
+      })
+
       it('should send the file to the primary', function () {
         expect(primaryPersistor.sendStream).to.have.been.calledWithExactly(
           bucket,
           destKey,
-          sinon.match.instanceOf(Stream.PassThrough)
+          sinon.match.instanceOf(Stream.PassThrough),
+          { sourceMd5: md5 }
         )
       })
     })