Update README.md

[web] Add option to disable subnet rate limiting (+CE/SP Hotfix `5.2.1`)

GitOrigin-RevId: 78d60c9638cede729dd93c3c2421f55b34c0dbfe

CONTRIBUTING.md

@@ -14,8 +14,6 @@ When submitting an issue please describe the issue as clearly as possible, inclu
 reproduce the bug, which situations it appears in, what you expected to happen, and what actually happens.
 If you can include a screenshot for front end issues that is very helpful.
 
-**Note**: If you are using [www.overleaf.com](www.overleaf.com) and have a problem, or if you would like to request a new feature, please contact the Support team at support@overleaf.com. Raise an issue here only to report bugs in the Community Edition release of Overleaf.
-
 Pull Requests
 -------------
 
@@ -36,7 +34,7 @@ Please see [our security policy](https://github.com/overleaf/overleaf/security/p
 Contributor License Agreement
 -----------------------------
 
-Before we can accept any contributions of code, we need you to agree to our
+Before we can accept any contributions of code, we need you to agree to our 
 [Contributor License Agreement](https://docs.google.com/forms/d/e/1FAIpQLSef79XH3mb7yIiMzZw-yALEegS-wyFetvjTiNBfZvf_IHD2KA/viewform?usp=sf_link).
 This is to ensure that you own the copyright of your contribution, and that you
 agree to give us a license to use it in both the open source version, and the version

README.md

@@ -27,15 +27,12 @@
 
 The present "extended" version of Overleaf CE includes:
 
-- Template Gallery
 - Sandboxed Compiles with TeX Live image selection
 - LDAP authentication
 - SAML authentication
-- OpenID Connect authentication
 - Real-time track changes and comments
 - Autocomplete of reference keys
 - Symbol Palette
-- "From External URL" feature
 
 ## Enterprise
 
@@ -48,6 +45,7 @@ To run a custom image, add a file named docker-compose.override.yml with the fol
 
 ```yml
 ---
+version: '2.2'
 services:
     sharelatex:
         image: sharelatex/sharelatex:ext-ce
@@ -56,24 +54,16 @@ services:
 ```
 Here, the attached volume provides convenient access for the container to the certificates needed for SAML or LDAP authentication.
 
-If you want to build a Docker image of the extended CE based on the upstream v5.4.0 codebase, you can check out the corresponding tag by running:
+If you want to build a Docker image of the extended CE based on the upstream v5.2.1 codebase, you can check out the corresponding tag by running:
 ```
-git checkout v5.4.0-ext-v2
+git checkout v5.2.1-ext
 ```
-After building the image, switch to the latest state of the repository and check the `server-ce/hotfix` directory. If a subdirectory matching your version (e.g., `5.4.0`) exists, build a patched image.
 Alternatively, you can download a prebuilt image from Docker Hub:
 ```
-docker pull overleafcep/sharelatex:5.4.0-ext-v2
+docker pull overleafcep/sharelatex:5.2.1-ext
 ```
 Make sure to update the image name in overleaf-toolkit/config/docker-compose.override.yml accordingly.
 
-### New in `v5.4.0-ext-v2`:
-
-- Added "From External URL" feature
-- Introduced Template Gallery
-- Improved keyboard input experience in the Symbol Palette
-- Fixed compilation error when using the `minted` package
-
 ## Sandboxed Compiles
 
 To enable sandboxed compiles (also known as "Sibling containers"), set the following configuration options in `overleaf-toolkit/config/overleaf.rc`:
@@ -95,29 +85,23 @@ The following environment variables are used to specify which TeX Live images to
 
 Users can select the image for their project in the project menu.
 
-Here is an example where the default TeX Live image is `latest-full` from Docker Hub, but the `TL2024-historic` image can be used for older projects:
+Here is an example where the default TeX Live image is `latest-full` from Docker Hub, but the `TL2023-historic` image can be used for older projects:
 ```
-ALL_TEX_LIVE_DOCKER_IMAGES=texlive/texlive:latest-full, texlive/texlive:TL2024-historic
-ALL_TEX_LIVE_DOCKER_IMAGE_NAMES=TeXLive 2025, TeXLive 2024
+ALL_TEX_LIVE_DOCKER_IMAGES=texlive/texlive:latest-full, texlive/texlive:TL2023-historic
+ALL_TEX_LIVE_DOCKER_IMAGE_NAMES=TeXLive 2024, TeXLive 2023
 TEX_LIVE_DOCKER_IMAGE=texlive/texlive:latest-full
 ```
 For additional details refer to
 [Server Pro: Sandboxed Compiles](https://github.com/overleaf/overleaf/wiki/Server-Pro:-Sandboxed-Compiles) and
 [Toolkit: Sandboxed Compiles](https://github.com/overleaf/toolkit/blob/master/doc/sandboxed-compiles.md).
 
-When the compilation takes place in a dedicated container, it is relatively safe to permit running external commands from inside the TeX
-file during compilation. This is required for packages like `minted`. For this purpose, the following environment variable can be used:
-
-- `TEX_COMPILER_EXTRA_FLAGS`
-    * A list of extra flags for TeX compiler. Example: `-shell-escape -file-line-error`
-
 <details>
 <summary><h4>Sample variables.env file</h4></summary>
 
 ```
 OVERLEAF_APP_NAME="Our Overleaf Instance"
 
-ENABLED_LINKED_FILE_TYPES=project_file,project_output_file,url
+ENABLED_LINKED_FILE_TYPES=project_file,project_output_file
 
 # Enables Thumbnail generation using ImageMagick
 ENABLE_CONVERSIONS=true
@@ -163,119 +147,13 @@ NAV_HIDE_POWERED_BY=true
 ALL_TEX_LIVE_DOCKER_IMAGES=texlive/texlive:latest-full, texlive/texlive:TL2023-historic
 ALL_TEX_LIVE_DOCKER_IMAGE_NAMES=TeXLive 2024, TeXLive 2023
 TEX_LIVE_DOCKER_IMAGE=texlive/texlive:latest-full
-TEX_COMPILER_EXTRA_FLAGS=-shell-escape
 ```
 </details>
 
-## "From External URL" feature
-
-To enable the ["From External URL"](https://www.overleaf.com/learn/how-to/How_to_upload_a_file_using_an_external_URL) feature the variable
-`ENABLED_LINKED_FILE_TYPES` must include `url` type:
-```
-ENABLED_LINKED_FILE_TYPES=project_file,project_output_file,url
-```
-This will extend both the *Add Files* menu and the *Insert Figure* dropdown in the toolbar: the *Add Files* menu allows you to add a 
-file to your project using its URL, while the *Insert Figure* dropdown lets you insert an image into your document directly from its URL.
-
-## Template Gallery
-
-The Template Gallery feature is controlled using the following environment variables:
-
-- `OVERLEAF_TEMPLATE_GALLERY`: Set to `true` to enable the Template Gallery.
-
-- `OVERLEAF_TEMPLATE_CATEGORIES`: An optional whitespace-separated list of zero or more template category keys. Valid keys include:  
-  _`academic-journal book presentation poster cv homework bibliography calendar formal-letter report thesis newsletter`_
-
-  The special key `all` is automatically appended. This key represents a category that includes all templates. 
-
-  For each category key, the following optional environment variables can be set:
-  - `TEMPLATE_<KEY>_NAME`: The display name of the template category.
-  - `TEMPLATE_<KEY>_DESCRIPTION`: A short description of the category.
-
-  If a category key contains a hyphen (`-`), replace it with an underscore (`_`) in the environment variable names.
-
-- `ENABLE_CONVERSIONS`: Must be set to `true` to enable thumbnail and preview generation for templates.
-
-- `OVERLEAF_NON_ADMIN_CAN_PUBLISH_TEMPLATES`: Determines whether non-admin users can publish templates. Defaults to `false`.
-
-To publish a project as a template, select the appropriate option from the project menu. In the form that appears, enter the template details:
-
-
-- The *Title* must be unique.
-- The *Description* field supports Markdown formatting.
-- The *Author* field accepts Markdown-formatted links.
-
-After submission, template details can be edited, or the template can be deleted via the Template Gallery page.  
-Users can manage their own templates; admins can manage any template.
-
-<details>
-<summary><h4>Example</h4></summary>
-
-```
-#################
-#   TEMPLATES   #
-#################
-
-ENABLE_CONVERSIONS=true
-
-OVERLEAF_TEMPLATE_GALLERY=true
-OVERLEAF_NON_ADMIN_CAN_PUBLISH_TEMPLATES=true
-OVERLEAF_TEMPLATE_CATEGORIES=academic-journal book presentation poster cv homework bibliography calendar formal-letter report thesis newsletter
-
-TEMPLATE_ACADEMIC_JOURNAL_NAME=Journal articles
-TEMPLATE_ACADEMIC_JOURNAL_DESCRIPTION=Templates for academic journal submissions, including formats for major publishers and preprint servers.
-
-TEMPLATE_BIBLIOGRAPHY_NAME=Bibliographies
-TEMPLATE_BIBLIOGRAPHY_DESCRIPTION=Styles for creating and managing bibliographies with BibTeX or BibLaTeX. Suitable for use in academic papers, theses, and reports.
-
-TEMPLATE_BOOK_NAME=Books
-TEMPLATE_BOOK_DESCRIPTION=Templates for writing books or long-form documents, including chapter structuring, front matter, and indexing.
-
-TEMPLATE_CALENDAR_NAME=Calendars
-TEMPLATE_CALENDAR_DESCRIPTION=Templates to create yearly, monthly, or weekly calendars. Useful for personal planners or event scheduling.
-
-TEMPLATE_CV_NAME=CVs and résumés
-TEMPLATE_CV_DESCRIPTION=Templates for CVs and résumés with various formats for academic, industry, and creative positions.
-
-TEMPLATE_FORMAL_LETTER_NAME=Formal letters
-TEMPLATE_FORMAL_LETTER_DESCRIPTION=Templates for professional letters, such as cover letters, recommendation letters, and official correspondence.
-
-TEMPLATE_HOMEWORK_NAME=Assignments
-TEMPLATE_HOMEWORK_DESCRIPTION=Templates for homework, coursework, and problem sets. Designed to be clean and well-structured for students and educators.
-
-TEMPLATE_NEWSLETTER_NAME=Newsletters
-TEMPLATE_NEWSLETTER_DESCRIPTION=Templates for creating newsletters with formats for academic, corporate, or community communications.
-
-TEMPLATE_POSTER_NAME=Posters
-TEMPLATE_POSTER_DESCRIPTION=Templates for scientific and academic posters, typically used in conferences and research presentations.
-
-TEMPLATE_PRESENTATION_NAME=Presentations
-TEMPLATE_PRESENTATION_DESCRIPTION=Templates for Beamer and other presentation formats, tailored for academic talks and lectures.
-
-TEMPLATE_REPORT_NAME=Reports
-TEMPLATE_REPORT_DESCRIPTION=Templates for technical, lab, or project reports. Includes sections for figures, tables, and references.
-
-TEMPLATE_THESIS_NAME=Theses
-TEMPLATE_THESIS_DESCRIPTION=Templates for writing theses and dissertations, following institutional formatting and citation guidelines.
-
-TEMPLATE_ALL_NAME=All templates
-TEMPLATE_ALL_DESCRIPTION=Browse a collection of all available LaTeX templates, categorized by document type, style, and purpose.
-
-```
-
-</details>
-
-
 ## Authentication Methods
 
-The following authentication methods are supported: local authentication, LDAP authentication, SAML authentication,
-and OpenID Connect (OIDC) authentication. Local authentication is always active. The environment variable `EXTERNAL_AUTH`
-specifies which external authentication methods are activated. The value of this variable is a list. If the list includes `ldap`, `saml`, or `oidc`,
-then LDAP authentication, SAML authentication, and OIDC authentication will be activated, respectively. 
-
-For example: `EXTERNAL_AUTH=ldap saml oidc`
-
-This configuration activates all available authentication methods, although this is rarely necessary.
+The following authentication methods are supported: Local authentication, LDAP authentication, and SAML authentication. Local authentication is always active.
+To enable LDAP or SAML authentication, the environment variable `EXTERNAL_AUTH` must be set to `ldap` or `saml`, respectively.
 
 <details>
 <summary><h3>Local Authentication</h3></summary>
@@ -304,14 +182,6 @@ It is possible to enforce password restrictions on local users:
 
 Internally, Overleaf LDAP uses the [passport-ldapauth](https://github.com/vesse/passport-ldapauth) library. Most of these configuration options are passed through to the `server` config object which is used to configure `passport-ldapauth`. If you are having issues configuring LDAP, it is worth reading the README for `passport-ldapauth` to understand the configuration it expects.
 
-When using Local and LDAP authentication methods, a user enters a `username` and `password` in the login form. If LDAP authentication is enabled, it is attempted first:
-
-1. An LDAP user is searched for in the LDAP directory using the filter defined by `OVERLEAF_LDAP_SEARCH_FILTER` and authenticated.
-2. If authentication is successful, the Overleaf users database is checked for a user with the primary email address that matches the email address of the authenticated LDAP user:
-   - If a matching user is found, the `hashedPassword` field for this user is deleted (if it exists). This ensures that the user can only log in via LDAP authentication in the future.
-   - If no matching user is found, a new Overleaf user is created using the email, first name, and last name retrieved from the LDAP server.
-3. If LDAP authentication fails or is unsuccessful, local authentication is attempted.
-
 #### Environment Variables
 
 - `OVERLEAF_LDAP_URL` **(required)**
@@ -433,7 +303,7 @@ The above example results in loading into the contacts of the current LDAP user
 ```
 OVERLEAF_APP_NAME="Our Overleaf Instance"
 
-ENABLED_LINKED_FILE_TYPES=project_file,project_output_file,url
+ENABLED_LINKED_FILE_TYPES=project_file,project_output_file
 
 # Enables Thumbnail generation using ImageMagick
 ENABLE_CONVERSIONS=true
@@ -567,43 +437,19 @@ Internally, Overleaf SAML module uses the [passport-saml](https://github.com/nod
 configuration options are passed through to `passport-saml`. If you are having issues configuring SAML, it is worth reading the README
 for `passport-saml` to get a feel for the configuration it expects.
 
-When using the SAML authentication method, a user is redirected to the Identity Provider (IdP) authentication site.
-If the IdP successfully authenticates the user, the Overleaf users database is checked for a record containing a `samlIdentifiers` field structured as follows:
-
-```
-samlIdentifiers: [
-  {
-    externalUserId: "...",
-    providerId: "1",
-    userIdAttribute: "..."
-  }
-]
-```
-The `externalUserId` must match the value of the property specified by `userIdAttribute` in the user profile returned by the IdP server.
-
-If no matching record is found, the database is searched for a user with the primary email address matching the email in the IdP user profile:
-
-- If such a user is found, the `hashedPassword` field is deleted to disable local authentication, and the `samlIdentifiers` field is added.
-- If no matching user is found, a new user is created with the email address and `samlIdentifiers` from the IdP profile.
-
-**Note:** Currently, only one SAML IdP is supported. The `providerId` field in `samlIdentifiers` is fixed to `'1'`.
-
 #### Environment Variables
 
 - `OVERLEAF_SAML_IDENTITY_SERVICE_NAME`
-    * Display name for the identity service, used on the login page (default: `Log in with SAML IdP`).
-
-- `OVERLEAF_SAML_USER_ID_FIELD`
-    * The value of this attribute will be used by Overleaf as the external user ID, defaults to `nameID`.
+    * Display name for the identity service, used on the login page (default: `Login with SAML IdP`).
 
 - `OVERLEAF_SAML_EMAIL_FIELD`
-    * Name of the Email field in user profile, defaults to `nameID`.
+    * Name of the Email field in user profile, default to 'nameID'.
 
 - `OVERLEAF_SAML_FIRST_NAME_FIELD`
-    * Name of the firstName field in user profile, defaults to `givenName`.
+    * Name of the firstName field in user profile, default to 'givenName'.
 
 - `OVERLEAF_SAML_LAST_NAME_FIELD`
-    * Name of the lastName field in user profile, defaults to `lastName`
+    * Name of the lastName field in user profile, default to 'lastName'
 
 - `OVERLEAF_SAML_UPDATE_USER_DETAILS_ON_LOGIN`
     * If set to `true`, updates the user `first_name` and `last_name` field on login,
@@ -615,6 +461,11 @@ If no matching record is found, the database is searched for a user with the pri
       - Example: `https://idp.example.com/simplesaml/saml2/idp/SSOService.php`
       - Azure Example: `https://login.microsoftonline.com/8b26b46a-6dd3-45c7-a104-f883f4db1f6b/saml2`
 
+- `OVERLEAF_SAML_CALLBACK_URL` **(required)**
+    * Callback URL for Overleaf service. Should be the full URL of the `/saml/login/callback` path.
+
+      - Example: `https://my-overleaf-instance.com/saml/login/callback`
+
 - `OVERLEAF_SAML_ISSUER` **(required)**
     * The Issuer name.
 
@@ -652,13 +503,10 @@ If no matching record is found, the database is searched for a user with the pri
 - `OVERLEAF_SAML_ADDITIONAL_AUTHORIZE_PARAMS`
     * JSON dictionary of additional query params to add to 'authorize' requests.
 
-        - Example: `{"some_key": "some_value"}`
+          - Example: `{"some_key": "some_value"}`
 
 - `OVERLEAF_SAML_IDENTIFIER_FORMAT`
-    * Name identifier format to request from the identity provider (default: `urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress`).
-      If using `urn:oasis:names:tc:SAML:2.0:nameid-format:persistent`, ensure the `OVERLEAF_SAML_EMAIL_FIELD` envirionment variable is defined.
-      If `urn:oasis:names:tc:SAML:2.0:nameid-format:transient` is required, you must also define the `OVERLEAF_SAML_ID_FIELD` environment variable,
-      which can, for example, be set to the user's email address.
+    * Name identifier format to request from identity provider (default: `urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress`).
 
 - `OVERLEAF_SAML_ACCEPTED_CLOCK_SKEW_MS`
     * Time in milliseconds of skew that is acceptable between client and server when checking OnBefore and NotOnOrAfter assertion
@@ -686,10 +534,6 @@ If no matching record is found, the database is searched for a user with the pri
     * If `never`, then InResponseTo  won't be validated (default).
     * If `ifPresent`, then InResponseTo will only be validated if present in the incoming SAML response.
 
-- `OVERLEAF_SAML_WANT_ASSERTIONS_SIGNED` and `OVERLEAF_SAML_WANT_AUTHN_RESPONSE_SIGNED`
-    * When set to `true` (default), Overleaf expects the SAML Assertions, respectively the entire SAML Authentication Response, to be signed by the IdP.
-      When both options are `false`, at least one of the assertions or the response must be signed.
-
 - `OVERLEAF_SAML_REQUEST_ID_EXPIRATION_PERIOD_MS`
     * Defines the expiration time when a Request ID generated for a SAML request will not be valid if seen
           in a SAML response in the `InResponseTo` field. Default: 28800000 (8 hours).
@@ -699,13 +543,20 @@ If no matching record is found, the database is searched for a user with the pri
 
       - Example: `https://idp.example.com/simplesaml/saml2/idp/SingleLogoutService.php`
 
+- `OVERLEAF_SAML_LOGOUT_CALLBACK_URL`
+    * Callback URL for IdP initiated logout. Should be the full URL of the `/saml/logot/callback` path.
+       With this value the `Location` attribute in the `SingleLogoutService` elements in the generated service provider metadata is populated with this value.
+
+      - Example: `https://my-overleaf-instance.com/saml/logout/callback`
+
 - `OVERLEAF_SAML_ADDITIONAL_LOGOUT_PARAMS`
     * JSON dictionary of additional query params to add to 'logout' requests.
 
+
 - `OVERLEAF_SAML_IS_ADMIN_FIELD` and `OVERLEAF_SAML_IS_ADMIN_FIELD_VALUE`
     * When both environment variables are set, the login process updates `user.isAdmin = true` if the profile returned by the SAML IdP contains the attribute specified by
      `OVERLEAF_SAML_IS_ADMIN_FIELD` and its value either matches `OVERLEAF_SAML_IS_ADMIN_FIELD_VALUE` or is an array containing `OVERLEAF_SAML_IS_ADMIN_FIELD_VALUE`,
-      otherwise `user.isAdmin` is set to `false`. If either of these variables is not set, then the admin status is only set to `true` during admin user
+      otherwise `user.isAdmin` is set to `false`. If either of these variables is not set, then the admin status is only set to `true` during admin user.
       creation in Launchpad.
 
 #### Metadata for the Identity Provider
@@ -770,7 +621,7 @@ and set as appropriate in your IdP configuration, or send the metadata file to t
 ```
 OVERLEAF_APP_NAME="Our Overleaf Instance"
 
-ENABLED_LINKED_FILE_TYPES=project_file,project_output_file,url
+ENABLED_LINKED_FILE_TYPES=project_file,project_output_file
 
 # Enables Thumbnail generation using ImageMagick
 ENABLE_CONVERSIONS=true
@@ -814,7 +665,7 @@ NAV_HIDE_POWERED_BY=true
 #################
 
 EXTERNAL_AUTH=saml
-OVERLEAF_SAML_IDENTITY_SERVICE_NAME='Log in with My IdP'
+OVERLEAF_SAML_IDENTITY_SERVICE_NAME='Login with My IdP'
 OVERLEAF_SAML_EMAIL_FIELD=mail
 OVERLEAF_SAML_FIRST_NAME_FIELD=givenName
 OVERLEAF_SAML_LAST_NAME_FIELD=sn
@@ -834,169 +685,14 @@ OVERLEAF_SAML_IS_ADMIN_FIELD_VALUE=overleaf.admin@example.com
 </details>
 </details>
 
-<details>
-<summary><h3>OIDC Authentication</h3></summary>
-
-Internally, Overleaf OIDC module uses the [passport-openidconnect](https://github.com/jaredhanson/passport-openidconnect) library.
-If you are having issues configuring OpenID Connect, it is worth reading the README for `passport-openidconnect` to get a feel for the configuration it expects.
-
-
-When using the OIDC authentication method, a user is redirected to the Identity Provider (IdP) authentication site.
-If the IdP successfully authenticates the user, the Overleaf users database is checked for a record containing a `thirdPartyIdentifiers` field structured as follows:
-
-```
-thirdPartyIdentifiers: [
-  {
-    externalUserId: "...",
-    externalData: null,
-    providerId: "..."
-  }
-]
-```
-
-The `externalUserId` must match the user ID in the profile returned by the IdP server (see the `OVERLEAF_OIDC_USER_ID_FIELD` environment variable), and `providerId`
-must match the ID of the OIDC provider (see the `OVERLEAF_OIDC_PROVIDER_ID`).
-
-If no matching record is found, the database is searched for a user with the primary email address matching the email in the IdP user profile:
-- If such a user is found, the `thirdPartyIdentifiers` field is updated.
-- If no matching user is found, a new user is created with the email address and `thirdPartyIdentifiers` from the IdP profile.
-
-In both cases, the user is said to be 'linked' to the external OIDC user. The user can be unlinked from the OIDC provider on the `/user/settings` page.
-
-
-#### Environment Variables
-
-The values of the following five required variables can be found using `.well-known/openid-configuration` endpoint of your OpenID Provider (OP).
-
-- `OVERLEAF_OIDC_ISSUER` **(required)**
-
-- `OVERLEAF_OIDC_AUTHORIZATION_URL` **(required)**
-
-- `OVERLEAF_OIDC_TOKEN_URL` **(required)**
-
-- `OVERLEAF_OIDC_USER_INFO_URL` **(required)**
-
-- `OVERLEAF_OIDC_LOGOUT_URL` **(required)**
-
-The values of the following two required variables will be provided by the admin of your OP
-
-- `OVERLEAF_OIDC_CLIENT_ID` **(required)**
-
-- `OVERLEAF_OIDC_CLIENT_SECRET` **(required)**
-
-- `OVERLEAF_OIDC_SCOPE`
-    * Default: `openid profile email`
-
-- `OVERLEAF_OIDC_PROVIDER_ID`
-    * Arbitrary ID of the OP, defaults to `oidc`.
-
-- `OVERLEAF_OIDC_PROVIDER_NAME`
-    * The name of the OP, used in the `Linked Accounts` section of the `/user/settings` page, defaults to `OIDC Provider`.
-
-- `OVERLEAF_OIDC_IDENTITY_SERVICE_NAME`
-    * Display name for the identity service, used on the login page (default: `Log in with $OVERLEAF_OIDC_PROVIDER_NAME`).
-
-- `OVERLEAF_OIDC_PROVIDER_DESCRIPTION`
-    * Description of OP, used in the `Linked Accounts` section (default: `Log in with $OVERLEAF_OIDC_PROVIDER_NAME`).
-
-- `OVERLEAF_OIDC_PROVIDER_INFO_LINK`
-    * `Learn more` URL in the OP description, default: no `Learn more` link in the description.
-
-- `OVERLEAF_OIDC_PROVIDER_HIDE_NOT_LINKED`
-    * Do not show OP on the `/user/settings` page, if the user's account is not linked with the OP, default `false`.
-
-- `OVERLEAF_OIDC_USER_ID_FIELD`
-    * The value of this attribute will be used by Overleaf as the external user ID, defaults to `id`.
-      Other possible reasonable values are `email` and `username` (corresponding to `preferred_username` OIDC claim).
-
-- `OVERLEAF_OIDC_UPDATE_USER_DETAILS_ON_LOGIN`
-    * If set to `true`, updates the user `first_name` and `last_name` field on login,
-      and disables the user details form on `/user/settings` page.
-
-- `OVERLEAF_OIDC_IS_ADMIN_FIELD` and `OVERLEAF_OIDC_IS_ADMIN_FIELD_VALUE`
-    * When both environment variables are set, the login process updates `user.isAdmin = true` if the profile returned by the OP contains the attribute specified by
-      `OVERLEAF_OIDC_IS_ADMIN_FIELD` and its value matches `OVERLEAF_OIDC_IS_ADMIN_FIELD_VALUE`, otherwise `user.isAdmin` is set to `false`.
-      If `OVERLEAF_OIDC_IS_ADMIN_FIELD` is `email` then the value of the attribute `emails[0].value` is used for match checking.
-
-The redirect URL for your OpenID Provider is `https://my-overleaf-instance.com/oidc/login/callback`.
-
-<details>
-<summary><h4>Sample variables.env file</h4></summary>
-
-```
-OVERLEAF_APP_NAME="Our Overleaf Instance"
-
-ENABLED_LINKED_FILE_TYPES=project_file,project_output_file,url
-
-# Enables Thumbnail generation using ImageMagick
-ENABLE_CONVERSIONS=true
-
-# Disables email confirmation requirement
-EMAIL_CONFIRMATION_DISABLED=true
-
-## Nginx
-# NGINX_WORKER_PROCESSES=4
-# NGINX_WORKER_CONNECTIONS=768
-
-## Set for TLS via nginx-proxy
-# OVERLEAF_BEHIND_PROXY=true
-# OVERLEAF_SECURE_COOKIE=true
-
-OVERLEAF_SITE_URL=http://my-overleaf-instance.com
-OVERLEAF_NAV_TITLE=Our Overleaf Instance
-# OVERLEAF_HEADER_IMAGE_URL=http://somewhere.com/mylogo.png
-OVERLEAF_ADMIN_EMAIL=support@example.com
-
-OVERLEAF_LEFT_FOOTER=[{"text": "Contact your support team", "url": "mailto:support@example.com"}]
-OVERLEAF_RIGHT_FOOTER=[{"text":"Hello, I am on the Right", "url":"https://github.com/yu-i-i/overleaf-cep"}]
-
-OVERLEAF_EMAIL_FROM_ADDRESS=team@example.com
-OVERLEAF_EMAIL_SMTP_HOST=smtp.example.com
-OVERLEAF_EMAIL_SMTP_PORT=587
-OVERLEAF_EMAIL_SMTP_SECURE=false
-# OVERLEAF_EMAIL_SMTP_USER=
-# OVERLEAF_EMAIL_SMTP_PASS=
-# OVERLEAF_EMAIL_SMTP_NAME=
-OVERLEAF_EMAIL_SMTP_LOGGER=false
-OVERLEAF_EMAIL_SMTP_TLS_REJECT_UNAUTH=true
-OVERLEAF_EMAIL_SMTP_IGNORE_TLS=false
-OVERLEAF_CUSTOM_EMAIL_FOOTER=This system is run by department x
-
-OVERLEAF_PROXY_LEARN=true
-NAV_HIDE_POWERED_BY=true
-
-#################
-## OIDC for CE ##
-#################
-
-EXTERNAL_AUTH=oidc
-
-OVERLEAF_OIDC_PROVIDER_ID=oidc
-OVERLEAF_OIDC_ISSUER=https://keycloak.provider.com/realms/example
-OVERLEAF_OIDC_AUTHORIZATION_URL=https://keycloak.provider.com/realms/example/protocol/openid-connect/auth
-OVERLEAF_OIDC_TOKEN_URL=https://keycloak.provider.com/realms/example/protocol/openid-connect/token
-OVERLEAF_OIDC_USER_INFO_URL=https://keycloak.provider.com/realms/example/protocol/openid-connect/userinfo
-OVERLEAF_OIDC_LOGOUT_URL=https://keycloak.provider.com/realms/example/protocol/openid-connect/logout
-OVERLEAF_OIDC_CLIENT_ID=Overleaf-OIDC
-OVERLEAF_OIDC_CLIENT_SECRET=DoNotUseThisATGgaAcTgCcATgGATTACAagGtTCaGcGTAG
-OVERLEAF_OIDC_IDENTITY_SERVICE_NAME='Log in with Keycloak OIDC Provider'
-OVERLEAF_OIDC_PROVIDER_NAME=OIDC Keycloak Provider
-OVERLEAF_OIDC_PROVIDER_INFO_LINK=https://openid.net
-OVERLEAF_OIDC_IS_ADMIN_FIELD=email
-OVERLEAF_OIDC_IS_ADMIN_FIELD_VALUE=overleaf.admin@example.com
-OVERLEAF_OIDC_UPDATE_USER_DETAILS_ON_LOGIN=false
-```
-</details>
-</details>
-
 ## Overleaf Docker Image
 
 This repo contains two dockerfiles, [`Dockerfile-base`](server-ce/Dockerfile-base), which builds the
 `sharelatex/sharelatex-base` image, and [`Dockerfile`](server-ce/Dockerfile) which builds the
 `sharelatex/sharelatex` (or "community") image.
 
-The Base image generally contains the basic dependencies like `wget`, plus `texlive`.
-We split this out because it's a pretty heavy set of
+The Base image generally contains the basic dependencies like `wget` and
+`aspell`, plus `texlive`. We split this out because it's a pretty heavy set of
 dependencies, and it's nice to not have to rebuild all of that every time.
 
 The `sharelatex/sharelatex` image extends the base image and adds the actual Overleaf code
@@ -1020,4 +716,4 @@ Extensions for CE by: [yu-i-i](https://github.com/yu-i-i/overleaf-cep)
 
 The code in this repository is released under the GNU AFFERO GENERAL PUBLIC LICENSE, version 3. A copy can be found in the [`LICENSE`](LICENSE) file.
 
-Copyright (c) Overleaf, 2014-2025.
+Copyright (c) Overleaf, 2014-2024.

bin/shared/mongodb-init-replica-set.js

@@ -1,3 +0,0 @@
-/* eslint-disable no-undef */
-
-rs.initiate({ _id: 'overleaf', members: [{ _id: 0, host: 'mongo:27017' }] })

develop/README.md

@@ -11,6 +11,12 @@ bin/build
 > [!NOTE]
 > If Docker is running out of RAM while building the services in parallel, create a `.env` file in this directory containing `COMPOSE_PARALLEL_LIMIT=1`.
 
+Next, initialize the database:
+
+```shell
+bin/init
+```
+
 Then start the services:
 
 ```shell
@@ -77,7 +83,6 @@ each service:
 | `filestore`        | 9235 |
 | `notifications`    | 9236 |
 | `real-time`        | 9237 |
-| `references`       | 9238 |
 | `history-v1`       | 9239 |
 | `project-history`  | 9240 |
 

develop/bin/init

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+docker compose up --detach mongo
+curl --max-time 10 --retry 5 --retry-delay 5 --retry-all-errors --silent --output /dev/null localhost:27017
+docker compose exec mongo mongosh --eval "rs.initiate({ _id: 'overleaf', members: [{ _id: 0, host: 'mongo:27017' }] })"
+docker compose down mongo

develop/dev.env

@@ -14,7 +14,7 @@ PROJECT_HISTORY_HOST=project-history
 REALTIME_HOST=real-time
 REDIS_HOST=redis
 REFERENCES_HOST=references
-SESSION_SECRET=foo
+SPELLING_HOST=spelling
 WEBPACK_HOST=webpack
 WEB_API_PASSWORD=overleaf
 WEB_API_USER=overleaf

develop/docker-compose.dev.yml

@@ -117,7 +117,7 @@ services:
     environment:
       - NODE_OPTIONS=--inspect=0.0.0.0:9229
     ports:
-      - "127.0.0.1:9238:9229"
+      - "127.0.0.1:9236:9229"
     volumes:
       - ../services/references/app:/overleaf/services/references/app
       - ../services/references/config:/overleaf/services/references/config

develop/docker-compose.yml

@@ -8,6 +8,7 @@ volumes:
   mongo-data:
   redis-data:
   sharelatex-data:
+  spelling-cache:
   web-data:
   history-v1-buckets:
 
@@ -88,20 +89,12 @@ services:
         - history-v1-buckets:/buckets
 
   mongo:
-    image: mongo:6.0
+    image: mongo:5
     command: --replSet overleaf
     ports:
       - "127.0.0.1:27017:27017" # for debugging
     volumes:
       - mongo-data:/data/db
-      - ../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
-    environment:
-      MONGO_INITDB_DATABASE: sharelatex
-    extra_hosts:
-      # Required when using the automatic database setup for initializing the
-      # replica set. This override is not needed when running the setup after
-      # starting up mongo.
-      - mongo:127.0.0.1
 
   notifications:
     build:
@@ -123,7 +116,7 @@ services:
       dockerfile: services/real-time/Dockerfile
     env_file:
       - dev.env
- 
+
   redis:
     image: redis:5
     ports:
@@ -138,6 +131,15 @@ services:
     env_file:
       - dev.env
 
+  spelling:
+    build:
+      context: ..
+      dockerfile: services/spelling/Dockerfile
+    env_file:
+      - dev.env
+    volumes:
+      - spelling-cache:/overleaf/services/spelling/cache
+
   web:
     build:
       context: ..
@@ -147,11 +149,11 @@ services:
       - dev.env
     environment:
       - APP_NAME=Overleaf Community Edition
-      - ENABLED_LINKED_FILE_TYPES=project_file,project_output_file,url
+      - ENABLED_LINKED_FILE_TYPES=project_file,project_output_file
       - EMAIL_CONFIRMATION_DISABLED=true
       - NODE_ENV=development
       - OVERLEAF_ALLOW_PUBLIC_ACCESS=true
-    command: ["node", "app.mjs"]
+    command: ["node", "app.js"]
     volumes:
       - sharelatex-data:/var/lib/overleaf
       - web-data:/overleaf/services/web/data
@@ -169,12 +171,13 @@ services:
       - project-history
       - real-time
       - references
+      - spelling
 
   webpack:
     build:
       context: ..
       dockerfile: services/web/Dockerfile
-      target: webpack
+      target: dev
     command: ["npx", "webpack", "serve", "--config", "webpack.config.dev-env.js"]
     ports:
       - "127.0.0.1:80:3808"

docker-compose.yml

@@ -32,7 +32,7 @@ services:
             OVERLEAF_REDIS_HOST: redis
             REDIS_HOST: redis
 
-            ENABLED_LINKED_FILE_TYPES: 'project_file,project_output_file,url'
+            ENABLED_LINKED_FILE_TYPES: 'project_file,project_output_file'
 
             # Enables Thumbnail generation using ImageMagick
             ENABLE_CONVERSIONS: 'true'
@@ -40,6 +40,10 @@ services:
             # Disables email confirmation requirement
             EMAIL_CONFIRMATION_DISABLED: 'true'
 
+            # temporary fix for LuaLaTex compiles
+            # see https://github.com/overleaf/overleaf/issues/695
+            TEXMFVAR: /var/lib/overleaf/tmp/texmf-var
+
             ## Set for SSL via nginx-proxy
             #VIRTUAL_HOST: 103.112.212.22
 
@@ -98,12 +102,12 @@ services:
 
     mongo:
         restart: always
-        image: mongo:6.0
+        image: mongo:5.0
         container_name: mongo
         command: '--replSet overleaf'
         volumes:
             - ~/mongo_data:/data/db
-            - ./bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
+            - ./mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
         environment:
           MONGO_INITDB_DATABASE: sharelatex
         extra_hosts:
@@ -111,7 +115,7 @@ services:
           # This override is not needed when running the setup after starting up mongo.
           - mongo:127.0.0.1
         healthcheck:
-            test: echo 'db.stats().ok' | mongosh localhost:27017/test --quiet
+            test: echo 'db.stats().ok' | mongo localhost:27017/test --quiet
             interval: 10s
             timeout: 10s
             retries: 5

libraries/access-token-encryptor/.nvmrc

@@ -1 +1 @@
-20.18.2
+18.20.2

libraries/access-token-encryptor/buildscript.txt

@@ -5,6 +5,6 @@ access-token-encryptor
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=20.18.2
+--node-version=18.20.2
 --public-repo=False
 --script-version=4.5.0

libraries/access-token-encryptor/lib/js/AccessTokenEncryptor.js

@@ -1,5 +1,5 @@
-const { promisify } = require('node:util')
-const crypto = require('node:crypto')
+const { promisify } = require('util')
+const crypto = require('crypto')
 
 const ALGORITHM = 'aes-256-ctr'
 

libraries/access-token-encryptor/package.json

@@ -21,7 +21,7 @@
   "devDependencies": {
     "chai": "^4.3.6",
     "chai-as-promised": "^7.1.1",
-    "mocha": "^11.1.0",
+    "mocha": "^10.2.0",
     "sandboxed-module": "^2.0.4",
     "typescript": "^5.0.4"
   }

libraries/access-token-encryptor/test/setup.js

@@ -1,13 +1,4 @@
 const chai = require('chai')
 const chaiAsPromised = require('chai-as-promised')
-const SandboxedModule = require('sandboxed-module')
 
 chai.use(chaiAsPromised)
-
-SandboxedModule.configure({
-  sourceTransformers: {
-    removeNodePrefix: function (source) {
-      return source.replace(/require\(['"]node:/g, "require('")
-    },
-  },
-})

libraries/fetch-utils/.nvmrc

@@ -1 +1 @@
-20.18.2
+18.20.2

libraries/fetch-utils/buildscript.txt

@@ -5,6 +5,6 @@ fetch-utils
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=20.18.2
+--node-version=18.20.2
 --public-repo=False
 --script-version=4.5.0

libraries/fetch-utils/index.js

@@ -1,9 +1,9 @@
 const _ = require('lodash')
-const { Readable } = require('node:stream')
+const { Readable } = require('stream')
 const OError = require('@overleaf/o-error')
 const fetch = require('node-fetch')
-const http = require('node:http')
-const https = require('node:https')
+const http = require('http')
+const https = require('https')
 
 /**
  * @import { Response } from 'node-fetch'
@@ -95,19 +95,6 @@ async function fetchNothing(url, opts = {}) {
  * @throws {RequestFailedError} if the response has a non redirect status code or missing Location header
  */
 async function fetchRedirect(url, opts = {}) {
-  const { location } = await fetchRedirectWithResponse(url, opts)
-  return location
-}
-
-/**
- * Make a request and extract the redirect from the response.
- *
- * @param {string | URL} url - request URL
- * @param {object} opts - fetch options
- * @return {Promise<{location: string, response: Response}>}
- * @throws {RequestFailedError} if the response has a non redirect status code or missing Location header
- */
-async function fetchRedirectWithResponse(url, opts = {}) {
   const { fetchOpts } = parseOpts(opts)
   fetchOpts.redirect = 'manual'
   const response = await performRequest(url, fetchOpts)
@@ -125,7 +112,7 @@ async function fetchRedirectWithResponse(url, opts = {}) {
     )
   }
   await discardResponseBody(response)
-  return { location, response }
+  return location
 }
 
 /**
@@ -310,7 +297,6 @@ module.exports = {
   fetchStreamWithResponse,
   fetchNothing,
   fetchRedirect,
-  fetchRedirectWithResponse,
   fetchString,
   fetchStringWithResponse,
   RequestFailedError,

libraries/fetch-utils/package.json

@@ -20,8 +20,8 @@
     "body-parser": "^1.20.3",
     "chai": "^4.3.6",
     "chai-as-promised": "^7.1.1",
-    "express": "^4.21.2",
-    "mocha": "^11.1.0",
+    "express": "^4.21.0",
+    "mocha": "^10.2.0",
     "typescript": "^5.0.4"
   },
   "dependencies": {

libraries/fetch-utils/test/unit/FetchUtilsTests.js

@@ -1,7 +1,7 @@
 const { expect } = require('chai')
 const { FetchError, AbortError } = require('node-fetch')
-const { Readable } = require('node:stream')
-const { once } = require('node:events')
+const { Readable } = require('stream')
+const { once } = require('events')
 const { TestServer } = require('./helpers/TestServer')
 const selfsigned = require('selfsigned')
 const {
@@ -24,17 +24,13 @@ const pems = selfsigned.generate(attrs, { days: 365 })
 const PRIVATE_KEY = pems.private
 const PUBLIC_CERT = pems.cert
 
-const dns = require('node:dns')
+const dns = require('dns')
 const _originalLookup = dns.lookup
 // Custom DNS resolver function
 dns.lookup = (hostname, options, callback) => {
   if (hostname === 'example.com') {
     // If the hostname is our test case, return the ip address for the test server
-    if (options?.all) {
-      callback(null, [{ address: '127.0.0.1', family: 4 }])
-    } else {
-      callback(null, '127.0.0.1', 4)
-    }
+    callback(null, '127.0.0.1', 4)
   } else {
     // Otherwise, use the default lookup
     _originalLookup(hostname, options, callback)

libraries/fetch-utils/test/unit/helpers/TestServer.js

@@ -1,9 +1,9 @@
 const express = require('express')
 const bodyParser = require('body-parser')
-const { EventEmitter } = require('node:events')
-const http = require('node:http')
-const https = require('node:https')
-const { promisify } = require('node:util')
+const { EventEmitter } = require('events')
+const http = require('http')
+const https = require('https')
+const { promisify } = require('util')
 
 class TestServer {
   constructor() {

libraries/logger/.nvmrc

@@ -1 +1 @@
-20.18.2
+18.20.2

libraries/logger/buildscript.txt

@@ -5,6 +5,6 @@ logger
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=20.18.2
+--node-version=18.20.2
 --public-repo=False
 --script-version=4.5.0

libraries/logger/log-level-checker.js

@@ -1,5 +1,5 @@
 const { fetchString } = require('@overleaf/fetch-utils')
-const fs = require('node:fs')
+const fs = require('fs')
 
 class LogLevelChecker {
   constructor(logger, defaultLevel) {

libraries/logger/logging-manager.js

@@ -1,6 +1,7 @@
-const Stream = require('node:stream')
+const Stream = require('stream')
 const bunyan = require('bunyan')
 const GCPManager = require('./gcp-manager')
+const SentryManager = require('./sentry-manager')
 const Serializers = require('./serializers')
 const {
   FileLogLevelChecker,
@@ -14,10 +15,8 @@ const LoggingManager = {
   initialize(name) {
     this.isProduction =
       (process.env.NODE_ENV || '').toLowerCase() === 'production'
-    const isTest = (process.env.NODE_ENV || '').toLowerCase() === 'test'
     this.defaultLevel =
-      process.env.LOG_LEVEL ||
-      (this.isProduction ? 'info' : isTest ? 'fatal' : 'debug')
+      process.env.LOG_LEVEL || (this.isProduction ? 'info' : 'debug')
     this.loggerName = name
     this.logger = bunyan.createLogger({
       name,
@@ -34,6 +33,10 @@ const LoggingManager = {
     return this
   },
 
+  initializeErrorReporting(dsn, options) {
+    this.sentryManager = new SentryManager()
+  },
+
   /**
    * @param {Record<string, any>|string} attributes - Attributes to log (nice serialization for err, req, res)
    * @param {string} [message] - Optional message
@@ -65,6 +68,9 @@ const LoggingManager = {
       })
     }
     this.logger.error(attributes, message, ...Array.from(args))
+    if (this.sentryManager) {
+      this.sentryManager.captureExceptionRateLimited(attributes, message)
+    }
   },
 
   /**
@@ -92,6 +98,9 @@ const LoggingManager = {
    */
   fatal(attributes, message) {
     this.logger.fatal(attributes, message)
+    if (this.sentryManager) {
+      this.sentryManager.captureException(attributes, message, 'fatal')
+    }
   },
 
   _getOutputStreamConfig() {

libraries/logger/package.json

@@ -23,11 +23,12 @@
     "@google-cloud/logging-bunyan": "^5.1.0",
     "@overleaf/fetch-utils": "*",
     "@overleaf/o-error": "*",
+    "@sentry/node": "^6.13.2",
     "bunyan": "^1.8.14"
   },
   "devDependencies": {
     "chai": "^4.3.6",
-    "mocha": "^11.1.0",
+    "mocha": "^10.2.0",
     "sandboxed-module": "^2.0.4",
     "sinon": "^9.2.4",
     "sinon-chai": "^3.7.0",

libraries/logger/sentry-manager.js

@@ -0,0 +1,106 @@
+const Serializers = require('./serializers')
+const RATE_LIMIT_MAX_ERRORS = 5
+const RATE_LIMIT_INTERVAL_MS = 60000
+
+class SentryManager {
+  constructor(dsn, options) {
+    this.Sentry = require('@sentry/node')
+    this.Sentry.init({ dsn, ...options })
+    // for rate limiting on sentry reporting
+    this.lastErrorTimeStamp = 0
+    this.lastErrorCount = 0
+  }
+
+  captureExceptionRateLimited(attributes, message) {
+    const now = Date.now()
+    // have we recently reported an error?
+    const recentSentryReport =
+      now - this.lastErrorTimeStamp < RATE_LIMIT_INTERVAL_MS
+    // if so, increment the error count
+    if (recentSentryReport) {
+      this.lastErrorCount++
+    } else {
+      this.lastErrorCount = 0
+      this.lastErrorTimeStamp = now
+    }
+    // only report 5 errors every minute to avoid overload
+    if (this.lastErrorCount < RATE_LIMIT_MAX_ERRORS) {
+      // add a note if the rate limit has been hit
+      const note =
+        this.lastErrorCount + 1 === RATE_LIMIT_MAX_ERRORS
+          ? '(rate limited)'
+          : ''
+      // report the exception
+      this.captureException(attributes, message, `error${note}`)
+    }
+  }
+
+  captureException(attributes, message, level) {
+    // handle case of logger.error "message"
+    if (typeof attributes === 'string') {
+      attributes = { err: new Error(attributes) }
+    }
+
+    // extract any error object
+    let error = Serializers.err(attributes.err || attributes.error)
+
+    // avoid reporting errors twice
+    for (const key in attributes) {
+      const value = attributes[key]
+      if (value instanceof Error && value.reportedToSentry) {
+        return
+      }
+    }
+
+    // include our log message in the error report
+    if (error == null) {
+      if (typeof message === 'string') {
+        error = { message }
+      }
+    } else if (message != null) {
+      attributes.description = message
+    }
+
+    // report the error
+    if (error != null) {
+      // capture attributes and use *_id objects as tags
+      const tags = {}
+      const extra = {}
+      for (const key in attributes) {
+        let value = attributes[key]
+        if (Serializers[key]) {
+          value = Serializers[key](value)
+        }
+        if (key.match(/_id/) && typeof value === 'string') {
+          tags[key] = value
+        }
+        extra[key] = value
+      }
+
+      // OError integration
+      extra.info = error.info
+      delete error.info
+
+      // Sentry wants to receive an Error instance.
+      const errInstance = new Error(error.message)
+      Object.assign(errInstance, error)
+
+      try {
+        // send the error to sentry
+        this.Sentry.captureException(errInstance, { tags, extra, level })
+
+        // put a flag on the errors to avoid reporting them multiple times
+        for (const key in attributes) {
+          const value = attributes[key]
+          if (value instanceof Error) {
+            value.reportedToSentry = true
+          }
+        }
+      } catch (err) {
+        // ignore Sentry errors
+      }
+    }
+  }
+}
+
+module.exports = SentryManager

libraries/logger/test/setup.js

@@ -8,9 +8,4 @@ chai.use(sinonChai)
 
 SandboxedModule.configure({
   globals: { Buffer, JSON, console, process },
-  sourceTransformers: {
-    removeNodePrefix: function (source) {
-      return source.replace(/require\(['"]node:/g, "require('")
-    },
-  },
 })

libraries/logger/test/unit/log-level-checker-tests.js

@@ -1,5 +1,5 @@
-const Path = require('node:path')
-const { promisify } = require('node:util')
+const Path = require('path')
+const { promisify } = require('util')
 const SandboxedModule = require('sandboxed-module')
 const sinon = require('sinon')
 const { expect } = require('chai')

libraries/logger/test/unit/logging-manager-tests.js

@@ -1,7 +1,7 @@
 const SandboxedModule = require('sandboxed-module')
 const bunyan = require('bunyan')
 const { expect } = require('chai')
-const path = require('node:path')
+const path = require('path')
 const sinon = require('sinon')
 
 const MODULE_PATH = path.join(__dirname, '../../logging-manager.js')
@@ -43,14 +43,20 @@ describe('LoggingManager', function () {
         .stub()
         .returns(this.GCEMetadataLogLevelChecker),
     }
+    this.SentryManager = {
+      captureException: sinon.stub(),
+      captureExceptionRateLimited: sinon.stub(),
+    }
     this.LoggingManager = SandboxedModule.require(MODULE_PATH, {
       requires: {
         bunyan: this.Bunyan,
         './log-level-checker': this.LogLevelChecker,
+        './sentry-manager': sinon.stub().returns(this.SentryManager),
       },
     })
     this.loggerName = 'test'
     this.logger = this.LoggingManager.initialize(this.loggerName)
+    this.logger.initializeErrorReporting('test_dsn')
   })
 
   afterEach(function () {
@@ -154,6 +160,13 @@ describe('LoggingManager', function () {
     })
   })
 
+  describe('logger.error', function () {
+    it('should report errors to Sentry', function () {
+      this.logger.error({ foo: 'bar' }, 'message')
+      expect(this.SentryManager.captureExceptionRateLimited).to.have.been.called
+    })
+  })
+
   describe('ringbuffer', function () {
     beforeEach(function () {
       this.logBufferMock = [

libraries/logger/test/unit/sentry-manager-tests.js

@@ -0,0 +1,247 @@
+const Path = require('path')
+const SandboxedModule = require('sandboxed-module')
+const { expect } = require('chai')
+const sinon = require('sinon')
+
+const MODULE_PATH = Path.join(__dirname, '../../sentry-manager.js')
+
+describe('SentryManager', function () {
+  beforeEach(function () {
+    this.clock = sinon.useFakeTimers(Date.now())
+    this.Sentry = {
+      init: sinon.stub(),
+      captureException: sinon.stub(),
+    }
+    this.SentryManager = SandboxedModule.require(MODULE_PATH, {
+      requires: {
+        '@sentry/node': this.Sentry,
+      },
+    })
+    this.sentryManager = new this.SentryManager('test_dsn')
+  })
+
+  afterEach(function () {
+    this.clock.restore()
+  })
+
+  describe('captureExceptionRateLimited', function () {
+    it('should report a single error to sentry', function () {
+      this.sentryManager.captureExceptionRateLimited({ foo: 'bar' }, 'message')
+      expect(this.Sentry.captureException).to.have.been.calledOnce
+    })
+
+    it('should report the same error to sentry only once', function () {
+      const error1 = new Error('this is the error')
+      this.sentryManager.captureExceptionRateLimited(
+        { foo: error1 },
+        'first message'
+      )
+      this.sentryManager.captureExceptionRateLimited(
+        { bar: error1 },
+        'second message'
+      )
+      expect(this.Sentry.captureException).to.have.been.calledOnce
+    })
+
+    it('should report two different errors to sentry individually', function () {
+      const error1 = new Error('this is the error')
+      const error2 = new Error('this is the error')
+      this.sentryManager.captureExceptionRateLimited(
+        { foo: error1 },
+        'first message'
+      )
+      this.sentryManager.captureExceptionRateLimited(
+        { bar: error2 },
+        'second message'
+      )
+      expect(this.Sentry.captureException).to.have.been.calledTwice
+    })
+
+    it('for multiple errors should only report a maximum of 5 errors to sentry', function () {
+      for (let i = 0; i < 10; i++) {
+        this.sentryManager.captureExceptionRateLimited(
+          { foo: 'bar' },
+          'message'
+        )
+      }
+      expect(this.Sentry.captureException).to.have.callCount(5)
+    })
+
+    it('for multiple errors with a minute delay should report 10 errors to sentry', function () {
+      for (let i = 0; i < 10; i++) {
+        this.sentryManager.captureExceptionRateLimited(
+          { foo: 'bar' },
+          'message'
+        )
+      }
+      expect(this.Sentry.captureException).to.have.callCount(5)
+
+      // allow a minute to pass
+      this.clock.tick(61 * 1000)
+
+      for (let i = 0; i < 10; i++) {
+        this.sentryManager.captureExceptionRateLimited(
+          { foo: 'bar' },
+          'message'
+        )
+      }
+      expect(this.Sentry.captureException).to.have.callCount(10)
+    })
+  })
+
+  describe('captureException', function () {
+    it('should remove the path from fs errors', function () {
+      const fsError = new Error(
+        "Error: ENOENT: no such file or directory, stat '/tmp/3279b8d0-da10-11e8-8255-efd98985942b'"
+      )
+      fsError.path = '/tmp/3279b8d0-da10-11e8-8255-efd98985942b'
+      this.sentryManager.captureException({ err: fsError }, 'message', 'error')
+      expect(this.Sentry.captureException).to.have.been.calledWith(
+        sinon.match.has(
+          'message',
+          'Error: ENOENT: no such file or directory, stat'
+        )
+      )
+    })
+
+    it('should sanitize error', function () {
+      const err = {
+        name: 'CustomError',
+        message: 'hello',
+        _oErrorTags: [{ stack: 'here:1', info: { one: 1 } }],
+        stack: 'here:0',
+        info: { key: 'value' },
+        code: 42,
+        signal: 9,
+        path: '/foo',
+      }
+      this.sentryManager.captureException({ err }, 'message', 'error')
+      const expectedErr = {
+        name: 'CustomError',
+        message: 'hello',
+        stack: 'here:0\nhere:1',
+        code: 42,
+        signal: 9,
+        path: '/foo',
+      }
+      expect(this.Sentry.captureException).to.have.been.calledWith(
+        sinon.match(expectedErr),
+        sinon.match({
+          tags: sinon.match.any,
+          level: sinon.match.any,
+          extra: {
+            description: 'message',
+            info: sinon.match({
+              one: 1,
+              key: 'value',
+            }),
+          },
+        })
+      )
+      // Chai is very picky with comparing Error instances. Go the long way of comparing all the fields manually.
+      const gotErr = this.Sentry.captureException.args[0][0]
+      for (const [key, wanted] of Object.entries(expectedErr)) {
+        expect(gotErr).to.have.property(key, wanted)
+      }
+    })
+    it('should sanitize request', function () {
+      const req = {
+        ip: '1.2.3.4',
+        method: 'GET',
+        url: '/foo',
+        headers: {
+          referer: 'abc',
+          'content-length': 1337,
+          'user-agent': 'curl',
+          authorization: '42',
+        },
+      }
+      this.sentryManager.captureException({ req }, 'message', 'error')
+      const expectedReq = {
+        remoteAddress: '1.2.3.4',
+        method: 'GET',
+        url: '/foo',
+        headers: {
+          referer: 'abc',
+          'content-length': 1337,
+          'user-agent': 'curl',
+        },
+      }
+      expect(this.Sentry.captureException).to.have.been.calledWith(
+        sinon.match({
+          message: 'message',
+        }),
+        sinon.match({
+          tags: sinon.match.any,
+          level: sinon.match.any,
+          extra: {
+            info: sinon.match.any,
+            req: expectedReq,
+          },
+        })
+      )
+      expect(this.Sentry.captureException.args[0][1].extra.req).to.deep.equal(
+        expectedReq
+      )
+    })
+    it('should sanitize response', function () {
+      const res = {
+        statusCode: 417,
+        body: Buffer.from('foo'),
+        getHeader(key) {
+          expect(key).to.be.oneOf(['content-length'])
+          if (key === 'content-length') return 1337
+        },
+      }
+      this.sentryManager.captureException({ res }, 'message', 'error')
+      const expectedRes = {
+        statusCode: 417,
+        headers: {
+          'content-length': 1337,
+        },
+      }
+      expect(this.Sentry.captureException).to.have.been.calledWith(
+        sinon.match({
+          message: 'message',
+        }),
+        sinon.match({
+          tags: sinon.match.any,
+          level: sinon.match.any,
+          extra: {
+            info: sinon.match.any,
+            res: expectedRes,
+          },
+        })
+      )
+      expect(this.Sentry.captureException.args[0][1].extra.res).to.deep.equal(
+        expectedRes
+      )
+    })
+
+    describe('reportedToSentry', function () {
+      it('should mark the error as reported to sentry', function () {
+        const err = new Error()
+        this.sentryManager.captureException({ err }, 'message')
+        expect(this.Sentry.captureException).to.have.been.called
+        expect(err.reportedToSentry).to.equal(true)
+      })
+
+      it('should mark two errors as reported to sentry', function () {
+        const err1 = new Error()
+        const err2 = new Error()
+        this.sentryManager.captureException({ err: err1, err2 }, 'message')
+        expect(this.Sentry.captureException).to.have.been.called
+        expect(err1.reportedToSentry).to.equal(true)
+        expect(err2.reportedToSentry).to.equal(true)
+      })
+
+      it('should not mark arbitrary objects as reported to sentry', function () {
+        const err = new Error()
+        const ctx = { foo: 'bar' }
+        this.sentryManager.captureException({ err, ctx }, 'message')
+        expect(this.Sentry.captureException).to.have.been.called
+        expect(ctx.reportedToSentry).not.to.exist
+      })
+    })
+  })
+})

libraries/metrics/.nvmrc

@@ -1 +1 @@
-20.18.2
+18.20.2

libraries/metrics/buildscript.txt

@@ -5,6 +5,6 @@ metrics
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=20.18.2
+--node-version=18.20.2
 --public-repo=False
 --script-version=4.5.0

libraries/metrics/initialize.js

@@ -88,7 +88,7 @@ function initializeProfileAgent() {
 }
 
 function initializePrometheus() {
-  const os = require('node:os')
+  const os = require('os')
   const promClient = require('prom-client')
   promClient.register.setDefaultLabels({ app: APP_NAME, host: os.hostname() })
   promClient.collectDefaultMetrics({ timeout: 5000, prefix: '' })

libraries/metrics/leaked_sockets.js

@@ -6,8 +6,8 @@
  * logged along with the corresponding information from /proc/net/tcp.
  */
 
-const fs = require('node:fs')
-const diagnosticsChannel = require('node:diagnostics_channel')
+const fs = require('fs')
+const diagnosticsChannel = require('diagnostics_channel')
 
 const SOCKET_MONITOR_INTERVAL = 60 * 1000
 // set the threshold for logging leaked sockets in minutes, defaults to 15

libraries/metrics/open_sockets.js

@@ -11,13 +11,13 @@ const seconds = 1000
 // In Node 0.10 the default is 5, which means only 5 open connections at one.
 // Node 0.12 has a default of Infinity. Make sure we have no limit set,
 // regardless of Node version.
-require('node:http').globalAgent.maxSockets = Infinity
-require('node:https').globalAgent.maxSockets = Infinity
+require('http').globalAgent.maxSockets = Infinity
+require('https').globalAgent.maxSockets = Infinity
 
-const SOCKETS_HTTP = require('node:http').globalAgent.sockets
-const SOCKETS_HTTPS = require('node:https').globalAgent.sockets
-const FREE_SOCKETS_HTTP = require('node:http').globalAgent.freeSockets
-const FREE_SOCKETS_HTTPS = require('node:https').globalAgent.freeSockets
+const SOCKETS_HTTP = require('http').globalAgent.sockets
+const SOCKETS_HTTPS = require('https').globalAgent.sockets
+const FREE_SOCKETS_HTTP = require('http').globalAgent.freeSockets
+const FREE_SOCKETS_HTTPS = require('https').globalAgent.freeSockets
 
 // keep track of set gauges and reset them in the next collection cycle
 const SEEN_HOSTS_HTTP = new Set()

libraries/metrics/package.json

@@ -23,7 +23,7 @@
   "devDependencies": {
     "bunyan": "^1.0.0",
     "chai": "^4.3.6",
-    "mocha": "^11.1.0",
+    "mocha": "^10.2.0",
     "sandboxed-module": "^2.0.4",
     "sinon": "^9.2.4",
     "typescript": "^5.0.4"

libraries/metrics/test/acceptance/metrics_tests.js

@@ -1,6 +1,6 @@
-const { promisify } = require('node:util')
-const os = require('node:os')
-const http = require('node:http')
+const { promisify } = require('util')
+const os = require('os')
+const http = require('http')
 const { expect } = require('chai')
 const Metrics = require('../..')
 
@@ -316,7 +316,7 @@ async function checkSummaryValues(key, values) {
   for (const quantile of Object.keys(values)) {
     expect(found[quantile]).to.be.within(
       values[quantile] - 5,
-      values[quantile] + 15,
+      values[quantile] + 5,
       `quantile: ${quantile}`
     )
   }

libraries/metrics/test/unit/js/event_loop.js

@@ -5,7 +5,7 @@
  */
 const chai = require('chai')
 const { expect } = chai
-const path = require('node:path')
+const path = require('path')
 const modulePath = path.join(__dirname, '../../../event_loop.js')
 const SandboxedModule = require('sandboxed-module')
 const sinon = require('sinon')

libraries/metrics/test/unit/js/http.js

@@ -1,4 +1,4 @@
-const Path = require('node:path')
+const Path = require('path')
 const SandboxedModule = require('sandboxed-module')
 const sinon = require('sinon')
 

libraries/mongo-utils/.dockerignore

@@ -1 +0,0 @@
-node_modules/

libraries/mongo-utils/.gitignore

@@ -1,3 +0,0 @@
-
-# managed by monorepo$ bin/update_build_scripts
-.npmrc

libraries/mongo-utils/.nvmrc

@@ -1 +0,0 @@
-20.18.2

libraries/mongo-utils/buildscript.txt

@@ -1,10 +0,0 @@
-mongo-utils
---dependencies=None
---docker-repos=gcr.io/overleaf-ops
---env-add=
---env-pass-through=
---esmock-loader=False
---is-library=True
---node-version=20.18.2
---public-repo=False
---script-version=4.5.0

libraries/mongo-utils/package.json

@@ -1,30 +0,0 @@
-{
-  "name": "@overleaf/mongo-utils",
-  "version": "0.0.1",
-  "description": "utilities to help working with mongo",
-  "main": "index.js",
-  "scripts": {
-    "test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
-    "test:unit": "mocha --exit test/**/*.{js,cjs}",
-    "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
-    "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
-    "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
-    "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
-    "test:ci": "npm run test:unit",
-    "types:check": "tsc --noEmit"
-  },
-  "author": "Overleaf (https://www.overleaf.com)",
-  "license": "AGPL-3.0-only",
-  "dependencies": {
-    "mongodb": "6.12.0",
-    "mongodb-legacy": "6.1.3"
-  },
-  "devDependencies": {
-    "chai": "^4.3.6",
-    "mocha": "^11.1.0",
-    "sandboxed-module": "^2.0.4",
-    "sinon": "^9.2.4",
-    "sinon-chai": "^3.7.0",
-    "typescript": "^5.0.4"
-  }
-}

libraries/mongo-utils/test/setup.js

@@ -1,11 +0,0 @@
-const chai = require('chai')
-const sinonChai = require('sinon-chai')
-const SandboxedModule = require('sandboxed-module')
-
-// Chai configuration
-chai.should()
-chai.use(sinonChai)
-
-SandboxedModule.configure({
-  globals: { Buffer, JSON, console, process },
-})

libraries/mongo-utils/tsconfig.json

@@ -1,7 +0,0 @@
-{
-  "extends": "../../tsconfig.backend.json",
-  "include": [
-    "**/*.js",
-    "**/*.cjs"
-  ]
-}

libraries/o-error/.nvmrc

@@ -1 +1 @@
-20.18.2
+18.20.2

libraries/o-error/buildscript.txt

@@ -5,6 +5,6 @@ o-error
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=20.18.2
+--node-version=18.20.2
 --public-repo=False
 --script-version=4.5.0

libraries/o-error/doc/demo.js

@@ -70,7 +70,7 @@ sayHi3(43, (err, result) => {
   }
 })
 
-const promisify = require('node:util').promisify
+const promisify = require('util').promisify
 demoDatabase.findUserAsync = promisify(demoDatabase.findUser)
 
 async function sayHi4NoHandling(userId) {

libraries/o-error/package.json

@@ -34,7 +34,7 @@
     "@types/chai": "^4.3.0",
     "@types/node": "^18.17.4",
     "chai": "^4.3.6",
-    "mocha": "^11.1.0",
+    "mocha": "^10.2.0",
     "typescript": "^5.0.4"
   }
 }

libraries/o-error/test/o-error-util.test.js

@@ -1,5 +1,5 @@
 const { expect } = require('chai')
-const { promisify } = require('node:util')
+const { promisify } = require('util')
 
 const OError = require('..')
 

libraries/o-error/test/support/index.js

@@ -23,7 +23,7 @@ exports.expectError = function OErrorExpectError(e, expected) {
   ).to.be.true
 
   expect(
-    require('node:util').types.isNativeError(e),
+    require('util').types.isNativeError(e),
     'error should be recognised by util.types.isNativeError'
   ).to.be.true
 

libraries/object-persistor/.nvmrc

@@ -1 +1 @@
-20.18.2
+18.20.2

libraries/object-persistor/buildscript.txt

@@ -5,6 +5,6 @@ object-persistor
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=20.18.2
+--node-version=18.20.2
 --public-repo=False
 --script-version=4.5.0

libraries/object-persistor/package.json

@@ -24,8 +24,7 @@
     "@overleaf/logger": "*",
     "@overleaf/metrics": "*",
     "@overleaf/o-error": "*",
-    "@overleaf/stream-utils": "*",
-    "aws-sdk": "^2.1691.0",
+    "aws-sdk": "^2.718.0",
     "fast-crc32c": "overleaf/node-fast-crc32c#aae6b2a4c7a7a159395df9cc6c38dfde702d6f51",
     "glob": "^7.1.6",
     "range-parser": "^1.2.1",
@@ -34,9 +33,9 @@
   "devDependencies": {
     "chai": "^4.3.6",
     "chai-as-promised": "^7.1.1",
-    "mocha": "^11.1.0",
+    "mocha": "^10.2.0",
     "mock-fs": "^5.2.0",
-    "mongodb": "6.12.0",
+    "mongodb": "^6.1.0",
     "sandboxed-module": "^2.0.4",
     "sinon": "^9.2.4",
     "sinon-chai": "^3.7.0",

libraries/object-persistor/src/AbstractPersistor.js

@@ -1,12 +1,6 @@
 const { NotImplementedError } = require('./Errors')
 
 module.exports = class AbstractPersistor {
-  /**
-   * @param location
-   * @param target
-   * @param {string} source
-   * @return {Promise<void>}
-   */
   async sendFile(location, target, source) {
     throw new NotImplementedError('method not implemented in persistor', {
       method: 'sendFile',
@@ -16,13 +10,6 @@ module.exports = class AbstractPersistor {
     })
   }
 
-  /**
-   * @param location
-   * @param target
-   * @param {NodeJS.ReadableStream} sourceStream
-   * @param {Object} opts
-   * @return {Promise<void>}
-   */
   async sendStream(location, target, sourceStream, opts = {}) {
     throw new NotImplementedError('method not implemented in persistor', {
       method: 'sendStream',
@@ -35,12 +22,12 @@ module.exports = class AbstractPersistor {
   /**
    * @param location
    * @param name
-   * @param {Object} [opts]
-   * @param {Number} [opts.start]
-   * @param {Number} [opts.end]
-   * @return {Promise<NodeJS.ReadableStream>}
+   * @param {Object} opts
+   * @param {Number} opts.start
+   * @param {Number} opts.end
+   * @return {Promise<Readable>}
    */
-  async getObjectStream(location, name, opts = {}) {
+  async getObjectStream(location, name, opts) {
     throw new NotImplementedError('method not implemented in persistor', {
       method: 'getObjectStream',
       location,
@@ -49,11 +36,6 @@ module.exports = class AbstractPersistor {
     })
   }
 
-  /**
-   * @param {string} location
-   * @param {string} name
-   * @return {Promise<string>}
-   */
   async getRedirectUrl(location, name) {
     throw new NotImplementedError('method not implemented in persistor', {
       method: 'getRedirectUrl',
@@ -62,13 +44,7 @@ module.exports = class AbstractPersistor {
     })
   }
 
-  /**
-   * @param {string} location
-   * @param {string} name
-   * @param {Object} opts
-   * @return {Promise<number>}
-   */
-  async getObjectSize(location, name, opts) {
+  async getObjectSize(location, name) {
     throw new NotImplementedError('method not implemented in persistor', {
       method: 'getObjectSize',
       location,
@@ -76,13 +52,7 @@ module.exports = class AbstractPersistor {
     })
   }
 
-  /**
-   * @param {string} location
-   * @param {string} name
-   * @param {Object} opts
-   * @return {Promise<string>}
-   */
-  async getObjectMd5Hash(location, name, opts) {
+  async getObjectMd5Hash(location, name) {
     throw new NotImplementedError('method not implemented in persistor', {
       method: 'getObjectMd5Hash',
       location,
@@ -90,14 +60,7 @@ module.exports = class AbstractPersistor {
     })
   }
 
-  /**
-   * @param {string} location
-   * @param {string} fromName
-   * @param {string} toName
-   * @param {Object} opts
-   * @return {Promise<void>}
-   */
-  async copyObject(location, fromName, toName, opts) {
+  async copyObject(location, fromName, toName) {
     throw new NotImplementedError('method not implemented in persistor', {
       method: 'copyObject',
       location,
@@ -106,11 +69,6 @@ module.exports = class AbstractPersistor {
     })
   }
 
-  /**
-   * @param {string} location
-   * @param {string} name
-   * @return {Promise<void>}
-   */
   async deleteObject(location, name) {
     throw new NotImplementedError('method not implemented in persistor', {
       method: 'deleteObject',
@@ -119,13 +77,7 @@ module.exports = class AbstractPersistor {
     })
   }
 
-  /**
-   * @param {string} location
-   * @param {string} name
-   * @param {string} [continuationToken]
-   * @return {Promise<void>}
-   */
-  async deleteDirectory(location, name, continuationToken) {
+  async deleteDirectory(location, name) {
     throw new NotImplementedError('method not implemented in persistor', {
       method: 'deleteDirectory',
       location,
@@ -133,13 +85,7 @@ module.exports = class AbstractPersistor {
     })
   }
 
-  /**
-   * @param {string} location
-   * @param {string} name
-   * @param {Object} opts
-   * @return {Promise<boolean>}
-   */
-  async checkIfObjectExists(location, name, opts) {
+  async checkIfObjectExists(location, name) {
     throw new NotImplementedError('method not implemented in persistor', {
       method: 'checkIfObjectExists',
       location,
@@ -147,13 +93,7 @@ module.exports = class AbstractPersistor {
     })
   }
 
-  /**
-   * @param {string} location
-   * @param {string} name
-   * @param {string} [continuationToken]
-   * @return {Promise<number>}
-   */
-  async directorySize(location, name, continuationToken) {
+  async directorySize(location, name) {
     throw new NotImplementedError('method not implemented in persistor', {
       method: 'directorySize',
       location,

libraries/object-persistor/src/Errors.js

@@ -5,8 +5,6 @@ class WriteError extends OError {}
 class ReadError extends OError {}
 class SettingsError extends OError {}
 class NotImplementedError extends OError {}
-class AlreadyWrittenError extends OError {}
-class NoKEKMatchedError extends OError {}
 
 module.exports = {
   NotFoundError,
@@ -14,6 +12,4 @@ module.exports = {
   ReadError,
   SettingsError,
   NotImplementedError,
-  AlreadyWrittenError,
-  NoKEKMatchedError,
 }

libraries/object-persistor/src/FSPersistor.js

@@ -1,26 +1,20 @@
-const crypto = require('node:crypto')
-const fs = require('node:fs')
-const fsPromises = require('node:fs/promises')
+const crypto = require('crypto')
+const fs = require('fs')
+const fsPromises = require('fs/promises')
 const globCallbacks = require('glob')
-const Path = require('node:path')
-const { PassThrough } = require('node:stream')
-const { pipeline } = require('node:stream/promises')
-const { promisify } = require('node:util')
+const Path = require('path')
+const { PassThrough } = require('stream')
+const { pipeline } = require('stream/promises')
+const { promisify } = require('util')
 
 const AbstractPersistor = require('./AbstractPersistor')
-const { ReadError, WriteError, NotImplementedError } = require('./Errors')
+const { ReadError, WriteError } = require('./Errors')
 const PersistorHelper = require('./PersistorHelper')
 
 const glob = promisify(globCallbacks)
 
 module.exports = class FSPersistor extends AbstractPersistor {
   constructor(settings = {}) {
-    if (settings.storageClass) {
-      throw new NotImplementedError(
-        'FS backend does not support storage classes'
-      )
-    }
-
     super()
     this.useSubdirectories = Boolean(settings.useSubdirectories)
   }
@@ -42,14 +36,6 @@ module.exports = class FSPersistor extends AbstractPersistor {
   }
 
   async sendStream(location, target, sourceStream, opts = {}) {
-    if (opts.ifNoneMatch === '*') {
-      // The standard library only has fs.rename(), which does not support exclusive flags.
-      // Refuse to act on this write operation.
-      throw new NotImplementedError(
-        'Overwrite protection required by caller, but it is not available is FS backend. Configure GCS or S3 backend instead, get in touch with support for further information.'
-      )
-    }
-
     const targetPath = this._getFsPath(location, target)
 
     try {
@@ -69,7 +55,7 @@ module.exports = class FSPersistor extends AbstractPersistor {
       throw PersistorHelper.wrapError(
         err,
         'failed to write stream',
-        { location, target, ifNoneMatch: opts.ifNoneMatch },
+        { location, target },
         WriteError
       )
     }
@@ -77,11 +63,6 @@ module.exports = class FSPersistor extends AbstractPersistor {
 
   // opts may be {start: Number, end: Number}
   async getObjectStream(location, name, opts = {}) {
-    if (opts.autoGunzip) {
-      throw new NotImplementedError(
-        'opts.autoGunzip is not supported by FS backend. Configure GCS or S3 backend instead, get in touch with support for further information.'
-      )
-    }
     const observer = new PersistorHelper.ObserverStream({
       metric: 'fs.ingress', // ingress to us from disk
       bucket: location,

libraries/object-persistor/src/GcsPersistor.js

@@ -1,28 +1,17 @@
-const fs = require('node:fs')
-const { pipeline } = require('node:stream/promises')
-const { PassThrough } = require('node:stream')
+const fs = require('fs')
+const { pipeline } = require('stream/promises')
+const { PassThrough } = require('stream')
 const { Storage, IdempotencyStrategy } = require('@google-cloud/storage')
-const {
-  WriteError,
-  ReadError,
-  NotFoundError,
-  NotImplementedError,
-} = require('./Errors')
+const { WriteError, ReadError, NotFoundError } = require('./Errors')
 const asyncPool = require('tiny-async-pool')
 const AbstractPersistor = require('./AbstractPersistor')
 const PersistorHelper = require('./PersistorHelper')
 const Logger = require('@overleaf/logger')
-const zlib = require('node:zlib')
 
 module.exports = class GcsPersistor extends AbstractPersistor {
   constructor(settings) {
-    if (settings.storageClass) {
-      throw new NotImplementedError(
-        'Use default bucket class for GCS instead of settings.storageClass'
-      )
-    }
-
     super()
+
     this.settings = settings
 
     // endpoint settings will be null by default except for tests
@@ -89,14 +78,10 @@ module.exports = class GcsPersistor extends AbstractPersistor {
         writeOptions.metadata = writeOptions.metadata || {}
         writeOptions.metadata.contentEncoding = opts.contentEncoding
       }
-      const fileOptions = {}
-      if (opts.ifNoneMatch === '*') {
-        fileOptions.generation = 0
-      }
 
       const uploadStream = this.storage
         .bucket(bucketName)
-        .file(key, fileOptions)
+        .file(key)
         .createWriteStream(writeOptions)
 
       await pipeline(readStream, observer, uploadStream)
@@ -112,7 +97,7 @@ module.exports = class GcsPersistor extends AbstractPersistor {
       throw PersistorHelper.wrapError(
         err,
         'upload to GCS failed',
-        { bucketName, key, ifNoneMatch: opts.ifNoneMatch },
+        { bucketName, key },
         WriteError
       )
     }
@@ -128,14 +113,12 @@ module.exports = class GcsPersistor extends AbstractPersistor {
       .file(key)
       .createReadStream({ decompress: false, ...opts })
 
-    let contentEncoding
     try {
       await new Promise((resolve, reject) => {
         stream.on('response', res => {
           switch (res.statusCode) {
             case 200: // full response
             case 206: // partial response
-              contentEncoding = res.headers['content-encoding']
               return resolve()
             case 404:
               return reject(new NotFoundError())
@@ -156,11 +139,7 @@ module.exports = class GcsPersistor extends AbstractPersistor {
     }
     // Return a PassThrough stream with a minimal interface. It will buffer until the caller starts reading. It will emit errors from the source stream (Stream.pipeline passes errors along).
     const pass = new PassThrough()
-    const transformer = []
-    if (contentEncoding === 'gzip' && opts.autoGunzip) {
-      transformer.push(zlib.createGunzip())
-    }
-    pipeline(stream, observer, ...transformer, pass).catch(() => {})
+    pipeline(stream, observer, pass).catch(() => {})
     return pass
   }
 
@@ -197,7 +176,7 @@ module.exports = class GcsPersistor extends AbstractPersistor {
         .bucket(bucketName)
         .file(key)
         .getMetadata()
-      return parseInt(metadata.size, 10)
+      return metadata.size
     } catch (err) {
       throw PersistorHelper.wrapError(
         err,
@@ -306,10 +285,7 @@ module.exports = class GcsPersistor extends AbstractPersistor {
       )
     }
 
-    return files.reduce(
-      (acc, file) => parseInt(file.metadata.size, 10) + acc,
-      0
-    )
+    return files.reduce((acc, file) => Number(file.metadata.size) + acc, 0)
   }
 
   async checkIfObjectExists(bucketName, key) {

libraries/object-persistor/src/MigrationPersistor.js

@@ -1,8 +1,8 @@
 const AbstractPersistor = require('./AbstractPersistor')
 const Logger = require('@overleaf/logger')
 const Metrics = require('@overleaf/metrics')
-const Stream = require('node:stream')
-const { pipeline } = require('node:stream/promises')
+const Stream = require('stream')
+const { pipeline } = require('stream/promises')
 const { NotFoundError, WriteError } = require('./Errors')
 
 // Persistor that wraps two other persistors. Talks to the 'primary' by default,

libraries/object-persistor/src/PerProjectEncryptedS3Persistor.js

@@ -1,469 +0,0 @@
-// @ts-check
-const Crypto = require('node:crypto')
-const Stream = require('node:stream')
-const fs = require('node:fs')
-const { promisify } = require('node:util')
-const { WritableBuffer } = require('@overleaf/stream-utils')
-const { S3Persistor, SSECOptions } = require('./S3Persistor.js')
-const {
-  AlreadyWrittenError,
-  NoKEKMatchedError,
-  NotFoundError,
-  NotImplementedError,
-  ReadError,
-} = require('./Errors')
-const logger = require('@overleaf/logger')
-const Path = require('node:path')
-
-const generateKey = promisify(Crypto.generateKey)
-const hkdf = promisify(Crypto.hkdf)
-
-const AES256_KEY_LENGTH = 32
-
-/**
- * @typedef {import('aws-sdk').AWSError} AWSError
- */
-
-/**
- * @typedef {Object} Settings
- * @property {boolean} automaticallyRotateDEKEncryption
- * @property {string} dataEncryptionKeyBucketName
- * @property {boolean} ignoreErrorsFromDEKReEncryption
- * @property {(bucketName: string, path: string) => string} pathToProjectFolder
- * @property {() => Promise<Array<RootKeyEncryptionKey>>} getRootKeyEncryptionKeys
- */
-
-/**
- * Helper function to make TS happy when accessing error properties
- * AWSError is not an actual class, so we cannot use instanceof.
- * @param {any} err
- * @return {err is AWSError}
- */
-function isAWSError(err) {
-  return !!err
-}
-
-/**
- * @param {any} err
- * @return {boolean}
- */
-function isForbiddenError(err) {
-  if (!err || !(err instanceof ReadError || err instanceof NotFoundError)) {
-    return false
-  }
-  const cause = err.cause
-  if (!isAWSError(cause)) return false
-  return cause.statusCode === 403
-}
-
-class RootKeyEncryptionKey {
-  /** @type {Buffer} */
-  #keyEncryptionKey
-  /** @type {Buffer} */
-  #salt
-
-  /**
-   * @param {Buffer} keyEncryptionKey
-   * @param {Buffer} salt
-   */
-  constructor(keyEncryptionKey, salt) {
-    if (keyEncryptionKey.byteLength !== AES256_KEY_LENGTH) {
-      throw new Error(`kek is not ${AES256_KEY_LENGTH} bytes long`)
-    }
-    this.#keyEncryptionKey = keyEncryptionKey
-    this.#salt = salt
-  }
-
-  /**
-   * @param {string} prefix
-   * @return {Promise<SSECOptions>}
-   */
-  async forProject(prefix) {
-    return new SSECOptions(
-      Buffer.from(
-        await hkdf(
-          'sha256',
-          this.#keyEncryptionKey,
-          this.#salt,
-          prefix,
-          AES256_KEY_LENGTH
-        )
-      )
-    )
-  }
-}
-
-class PerProjectEncryptedS3Persistor extends S3Persistor {
-  /** @type {Settings} */
-  #settings
-  /** @type {Promise<Array<RootKeyEncryptionKey>>} */
-  #availableKeyEncryptionKeysPromise
-
-  /**
-   * @param {Settings} settings
-   */
-  constructor(settings) {
-    if (!settings.dataEncryptionKeyBucketName) {
-      throw new Error('settings.dataEncryptionKeyBucketName is missing')
-    }
-    super(settings)
-    this.#settings = settings
-    this.#availableKeyEncryptionKeysPromise = settings
-      .getRootKeyEncryptionKeys()
-      .then(rootKEKs => {
-        if (rootKEKs.length === 0) throw new Error('no root kek provided')
-        return rootKEKs
-      })
-  }
-
-  async ensureKeyEncryptionKeysLoaded() {
-    await this.#availableKeyEncryptionKeysPromise
-  }
-
-  /**
-   * @param {string} bucketName
-   * @param {string} path
-   * @return {{dekPath: string, projectFolder: string}}
-   */
-  #buildProjectPaths(bucketName, path) {
-    const projectFolder = this.#settings.pathToProjectFolder(bucketName, path)
-    const dekPath = Path.join(projectFolder, 'dek')
-    return { projectFolder, dekPath }
-  }
-
-  /**
-   * @param {string} projectFolder
-   * @return {Promise<SSECOptions>}
-   */
-  async #getCurrentKeyEncryptionKey(projectFolder) {
-    const [currentRootKEK] = await this.#availableKeyEncryptionKeysPromise
-    return await currentRootKEK.forProject(projectFolder)
-  }
-
-  /**
-   * @param {string} bucketName
-   * @param {string} path
-   */
-  async getDataEncryptionKeySize(bucketName, path) {
-    const { projectFolder, dekPath } = this.#buildProjectPaths(bucketName, path)
-    for (const rootKEK of await this.#availableKeyEncryptionKeysPromise) {
-      const ssecOptions = await rootKEK.forProject(projectFolder)
-      try {
-        return await super.getObjectSize(
-          this.#settings.dataEncryptionKeyBucketName,
-          dekPath,
-          { ssecOptions }
-        )
-      } catch (err) {
-        if (isForbiddenError(err)) continue
-        throw err
-      }
-    }
-    throw new NoKEKMatchedError('no kek matched')
-  }
-
-  /**
-   * @param {string} bucketName
-   * @param {string} path
-   * @return {Promise<CachedPerProjectEncryptedS3Persistor>}
-   */
-  async forProject(bucketName, path) {
-    return new CachedPerProjectEncryptedS3Persistor(
-      this,
-      await this.#getDataEncryptionKeyOptions(bucketName, path)
-    )
-  }
-
-  /**
-   * @param {string} bucketName
-   * @param {string} path
-   * @return {Promise<CachedPerProjectEncryptedS3Persistor>}
-   */
-  async forProjectRO(bucketName, path) {
-    return new CachedPerProjectEncryptedS3Persistor(
-      this,
-      await this.#getExistingDataEncryptionKeyOptions(bucketName, path)
-    )
-  }
-
-  /**
-   * @param {string} bucketName
-   * @param {string} path
-   * @return {Promise<CachedPerProjectEncryptedS3Persistor>}
-   */
-  async generateDataEncryptionKey(bucketName, path) {
-    return new CachedPerProjectEncryptedS3Persistor(
-      this,
-      await this.#generateDataEncryptionKeyOptions(bucketName, path)
-    )
-  }
-
-  /**
-   * @param {string} bucketName
-   * @param {string} path
-   * @return {Promise<SSECOptions>}
-   */
-  async #generateDataEncryptionKeyOptions(bucketName, path) {
-    const dataEncryptionKey = (
-      await generateKey('aes', { length: 256 })
-    ).export()
-    const { projectFolder, dekPath } = this.#buildProjectPaths(bucketName, path)
-    await super.sendStream(
-      this.#settings.dataEncryptionKeyBucketName,
-      dekPath,
-      Stream.Readable.from([dataEncryptionKey]),
-      {
-        // Do not overwrite any objects if already created
-        ifNoneMatch: '*',
-        ssecOptions: await this.#getCurrentKeyEncryptionKey(projectFolder),
-        contentLength: 32,
-      }
-    )
-    return new SSECOptions(dataEncryptionKey)
-  }
-
-  /**
-   * @param {string} bucketName
-   * @param {string} path
-   * @return {Promise<SSECOptions>}
-   */
-  async #getExistingDataEncryptionKeyOptions(bucketName, path) {
-    const { projectFolder, dekPath } = this.#buildProjectPaths(bucketName, path)
-    let res
-    let kekIndex = 0
-    for (const rootKEK of await this.#availableKeyEncryptionKeysPromise) {
-      const ssecOptions = await rootKEK.forProject(projectFolder)
-      try {
-        res = await super.getObjectStream(
-          this.#settings.dataEncryptionKeyBucketName,
-          dekPath,
-          { ssecOptions }
-        )
-        break
-      } catch (err) {
-        if (isForbiddenError(err)) {
-          kekIndex++
-          continue
-        }
-        throw err
-      }
-    }
-    if (!res) throw new NoKEKMatchedError('no kek matched')
-    const buf = new WritableBuffer()
-    await Stream.promises.pipeline(res, buf)
-
-    if (kekIndex !== 0 && this.#settings.automaticallyRotateDEKEncryption) {
-      const ssecOptions = await this.#getCurrentKeyEncryptionKey(projectFolder)
-      try {
-        await super.sendStream(
-          this.#settings.dataEncryptionKeyBucketName,
-          dekPath,
-          Stream.Readable.from([buf.getContents()]),
-          { ssecOptions }
-        )
-      } catch (err) {
-        if (this.#settings.ignoreErrorsFromDEKReEncryption) {
-          logger.warn({ err, dekPath }, 'failed to persist re-encrypted DEK')
-        } else {
-          throw err
-        }
-      }
-    }
-
-    return new SSECOptions(buf.getContents())
-  }
-
-  /**
-   * @param {string} bucketName
-   * @param {string} path
-   * @return {Promise<SSECOptions>}
-   */
-  async #getDataEncryptionKeyOptions(bucketName, path) {
-    try {
-      return await this.#getExistingDataEncryptionKeyOptions(bucketName, path)
-    } catch (err) {
-      if (err instanceof NotFoundError) {
-        try {
-          return await this.#generateDataEncryptionKeyOptions(bucketName, path)
-        } catch (err2) {
-          if (err2 instanceof AlreadyWrittenError) {
-            // Concurrent initial write
-            return await this.#getExistingDataEncryptionKeyOptions(
-              bucketName,
-              path
-            )
-          }
-          throw err2
-        }
-      }
-      throw err
-    }
-  }
-
-  async sendStream(bucketName, path, sourceStream, opts = {}) {
-    const ssecOptions =
-      opts.ssecOptions ||
-      (await this.#getDataEncryptionKeyOptions(bucketName, path))
-    return await super.sendStream(bucketName, path, sourceStream, {
-      ...opts,
-      ssecOptions,
-    })
-  }
-
-  async getObjectStream(bucketName, path, opts = {}) {
-    const ssecOptions =
-      opts.ssecOptions ||
-      (await this.#getExistingDataEncryptionKeyOptions(bucketName, path))
-    return await super.getObjectStream(bucketName, path, {
-      ...opts,
-      ssecOptions,
-    })
-  }
-
-  async getObjectSize(bucketName, path, opts = {}) {
-    const ssecOptions =
-      opts.ssecOptions ||
-      (await this.#getExistingDataEncryptionKeyOptions(bucketName, path))
-    return await super.getObjectSize(bucketName, path, { ...opts, ssecOptions })
-  }
-
-  async getObjectStorageClass(bucketName, path, opts = {}) {
-    const ssecOptions =
-      opts.ssecOptions ||
-      (await this.#getExistingDataEncryptionKeyOptions(bucketName, path))
-    return await super.getObjectStorageClass(bucketName, path, {
-      ...opts,
-      ssecOptions,
-    })
-  }
-
-  async directorySize(bucketName, path, continuationToken) {
-    // Note: Listing a bucket does not require SSE-C credentials.
-    return await super.directorySize(bucketName, path, continuationToken)
-  }
-
-  async deleteDirectory(bucketName, path, continuationToken) {
-    // Let [Settings.pathToProjectFolder] validate the project path before deleting things.
-    const { projectFolder, dekPath } = this.#buildProjectPaths(bucketName, path)
-    // Note: Listing/Deleting a prefix does not require SSE-C credentials.
-    await super.deleteDirectory(bucketName, path, continuationToken)
-    if (projectFolder === path) {
-      await super.deleteObject(
-        this.#settings.dataEncryptionKeyBucketName,
-        dekPath
-      )
-    }
-  }
-
-  async getObjectMd5Hash(bucketName, path, opts = {}) {
-    // The ETag in object metadata is not the MD5 content hash, skip the HEAD request.
-    opts = { ...opts, etagIsNotMD5: true }
-    return await super.getObjectMd5Hash(bucketName, path, opts)
-  }
-
-  async copyObject(bucketName, sourcePath, destinationPath, opts = {}) {
-    const ssecOptions =
-      opts.ssecOptions ||
-      (await this.#getDataEncryptionKeyOptions(bucketName, destinationPath))
-    const ssecSrcOptions =
-      opts.ssecSrcOptions ||
-      (await this.#getExistingDataEncryptionKeyOptions(bucketName, sourcePath))
-    return await super.copyObject(bucketName, sourcePath, destinationPath, {
-      ...opts,
-      ssecOptions,
-      ssecSrcOptions,
-    })
-  }
-
-  /**
-   * @param {string} bucketName
-   * @param {string} path
-   * @return {Promise<string>}
-   */
-  async getRedirectUrl(bucketName, path) {
-    throw new NotImplementedError('signed links are not supported with SSE-C')
-  }
-}
-
-/**
- * Helper class for batch updates to avoid repeated fetching of the project path.
- *
- * A general "cache" for project keys is another alternative. For now, use a helper class.
- */
-class CachedPerProjectEncryptedS3Persistor {
-  /** @type SSECOptions */
-  #projectKeyOptions
-  /** @type PerProjectEncryptedS3Persistor */
-  #parent
-
-  /**
-   * @param {PerProjectEncryptedS3Persistor} parent
-   * @param {SSECOptions} projectKeyOptions
-   */
-  constructor(parent, projectKeyOptions) {
-    this.#parent = parent
-    this.#projectKeyOptions = projectKeyOptions
-  }
-
-  /**
-   * @param {string} bucketName
-   * @param {string} path
-   * @param {string} fsPath
-   */
-  async sendFile(bucketName, path, fsPath) {
-    return await this.sendStream(bucketName, path, fs.createReadStream(fsPath))
-  }
-
-  /**
-   *
-   * @param {string} bucketName
-   * @param {string} path
-   * @return {Promise<number>}
-   */
-  async getObjectSize(bucketName, path) {
-    return await this.#parent.getObjectSize(bucketName, path)
-  }
-
-  /**
-   * @param {string} bucketName
-   * @param {string} path
-   * @param {NodeJS.ReadableStream} sourceStream
-   * @param {Object} opts
-   * @param {string} [opts.contentType]
-   * @param {string} [opts.contentEncoding]
-   * @param {number} [opts.contentLength]
-   * @param {'*'} [opts.ifNoneMatch]
-   * @param {SSECOptions} [opts.ssecOptions]
-   * @param {string} [opts.sourceMd5]
-   * @return {Promise<void>}
-   */
-  async sendStream(bucketName, path, sourceStream, opts = {}) {
-    return await this.#parent.sendStream(bucketName, path, sourceStream, {
-      ...opts,
-      ssecOptions: this.#projectKeyOptions,
-    })
-  }
-
-  /**
-   * @param {string} bucketName
-   * @param {string} path
-   * @param {Object} opts
-   * @param {number} [opts.start]
-   * @param {number} [opts.end]
-   * @param {boolean} [opts.autoGunzip]
-   * @param {SSECOptions} [opts.ssecOptions]
-   * @return {Promise<NodeJS.ReadableStream>}
-   */
-  async getObjectStream(bucketName, path, opts = {}) {
-    return await this.#parent.getObjectStream(bucketName, path, {
-      ...opts,
-      ssecOptions: this.#projectKeyOptions,
-    })
-  }
-}
-
-module.exports = {
-  PerProjectEncryptedS3Persistor,
-  CachedPerProjectEncryptedS3Persistor,
-  RootKeyEncryptionKey,
-}

libraries/object-persistor/src/PersistorFactory.js

@@ -1,20 +1,15 @@
 const Logger = require('@overleaf/logger')
 const { SettingsError } = require('./Errors')
 const GcsPersistor = require('./GcsPersistor')
-const { S3Persistor } = require('./S3Persistor')
+const S3Persistor = require('./S3Persistor')
 const FSPersistor = require('./FSPersistor')
 const MigrationPersistor = require('./MigrationPersistor')
-const {
-  PerProjectEncryptedS3Persistor,
-} = require('./PerProjectEncryptedS3Persistor')
 
 function getPersistor(backend, settings) {
   switch (backend) {
     case 'aws-sdk':
     case 's3':
       return new S3Persistor(settings.s3)
-    case 's3SSEC':
-      return new PerProjectEncryptedS3Persistor(settings.s3SSEC)
     case 'fs':
       return new FSPersistor({
         useSubdirectories: settings.useSubdirectories,

libraries/object-persistor/src/PersistorHelper.js

@@ -1,9 +1,9 @@
-const Crypto = require('node:crypto')
-const Stream = require('node:stream')
-const { pipeline } = require('node:stream/promises')
+const Crypto = require('crypto')
+const Stream = require('stream')
+const { pipeline } = require('stream/promises')
 const Logger = require('@overleaf/logger')
 const Metrics = require('@overleaf/metrics')
-const { WriteError, NotFoundError, AlreadyWrittenError } = require('./Errors')
+const { WriteError, NotFoundError } = require('./Errors')
 
 const _128KiB = 128 * 1024
 const TIMING_BUCKETS = [
@@ -26,14 +26,12 @@ const SIZE_BUCKETS = [
  */
 class ObserverStream extends Stream.Transform {
   /**
-   * @param {Object} opts
-   * @param {string} opts.metric prefix for metrics
-   * @param {string} opts.bucket name of source/target bucket
-   * @param {string} [opts.hash] optional hash algorithm, e.g. 'md5'
+   * @param {string} metric prefix for metrics
+   * @param {string} bucket name of source/target bucket
+   * @param {string} hash optional hash algorithm, e.g. 'md5'
    */
-  constructor(opts) {
+  constructor({ metric, bucket, hash = '' }) {
     super({ autoDestroy: true })
-    const { metric, bucket, hash = '' } = opts
 
     this.bytes = 0
     this.start = performance.now()
@@ -140,10 +138,6 @@ async function verifyMd5(persistor, bucket, key, sourceMd5, destMd5 = null) {
 }
 
 function wrapError(error, message, params, ErrorType) {
-  params = {
-    ...params,
-    cause: error,
-  }
   if (
     error instanceof NotFoundError ||
     ['NoSuchKey', 'NotFound', 404, 'AccessDenied', 'ENOENT'].includes(
@@ -152,13 +146,6 @@ function wrapError(error, message, params, ErrorType) {
     (error.response && error.response.statusCode === 404)
   ) {
     return new NotFoundError('no such file', params, error)
-  } else if (
-    params.ifNoneMatch === '*' &&
-    (error.code === 'PreconditionFailed' ||
-      error.response?.statusCode === 412 ||
-      error instanceof AlreadyWrittenError)
-  ) {
-    return new AlreadyWrittenError(message, params, error)
   } else {
     return new ErrorType(message, params, error)
   }

libraries/object-persistor/src/S3Persistor.js

@@ -1,6 +1,5 @@
-// @ts-check
-const http = require('node:http')
-const https = require('node:https')
+const http = require('http')
+const https = require('https')
 if (http.globalAgent.maxSockets < 300) {
   http.globalAgent.maxSockets = 300
 }
@@ -8,92 +7,27 @@ if (https.globalAgent.maxSockets < 300) {
   https.globalAgent.maxSockets = 300
 }
 
-const Crypto = require('node:crypto')
 const Metrics = require('@overleaf/metrics')
 const AbstractPersistor = require('./AbstractPersistor')
 const PersistorHelper = require('./PersistorHelper')
 
-const { pipeline, PassThrough } = require('node:stream')
-const fs = require('node:fs')
+const { pipeline, PassThrough } = require('stream')
+const fs = require('fs')
 const S3 = require('aws-sdk/clients/s3')
-const { URL } = require('node:url')
+const { URL } = require('url')
 const { WriteError, ReadError, NotFoundError } = require('./Errors')
-const zlib = require('node:zlib')
-
-/**
- * Wrapper with private fields to avoid revealing them on console, JSON.stringify or similar.
- */
-class SSECOptions {
-  #keyAsBuffer
-  #keyMD5
-
-  /**
-   * @param {Buffer} keyAsBuffer
-   */
-  constructor(keyAsBuffer) {
-    this.#keyAsBuffer = keyAsBuffer
-    this.#keyMD5 = Crypto.createHash('md5').update(keyAsBuffer).digest('base64')
-  }
-
-  getPutOptions() {
-    return {
-      SSECustomerKey: this.#keyAsBuffer,
-      SSECustomerKeyMD5: this.#keyMD5,
-      SSECustomerAlgorithm: 'AES256',
-    }
-  }
-
-  getGetOptions() {
-    return {
-      SSECustomerKey: this.#keyAsBuffer,
-      SSECustomerKeyMD5: this.#keyMD5,
-      SSECustomerAlgorithm: 'AES256',
-    }
-  }
-
-  getCopyOptions() {
-    return {
-      CopySourceSSECustomerKey: this.#keyAsBuffer,
-      CopySourceSSECustomerKeyMD5: this.#keyMD5,
-      CopySourceSSECustomerAlgorithm: 'AES256',
-    }
-  }
-}
-
-class S3Persistor extends AbstractPersistor {
-  /** @type {Map<string, S3>} */
-  #clients = new Map()
 
+module.exports = class S3Persistor extends AbstractPersistor {
   constructor(settings = {}) {
     super()
 
-    settings.storageClass = settings.storageClass || {}
     this.settings = settings
   }
 
-  /**
-   * @param {string} bucketName
-   * @param {string} key
-   * @param {string} fsPath
-   * @return {Promise<void>}
-   */
   async sendFile(bucketName, key, fsPath) {
-    await this.sendStream(bucketName, key, fs.createReadStream(fsPath))
+    return await this.sendStream(bucketName, key, fs.createReadStream(fsPath))
   }
 
-  /**
-   * @param {string} bucketName
-   * @param {string} key
-   * @param {NodeJS.ReadableStream} readStream
-   * @param {Object} opts
-   * @param {string} [opts.contentType]
-   * @param {string} [opts.contentEncoding]
-   * @param {number} [opts.contentLength]
-   * @param {'*'} [opts.ifNoneMatch]
-   * @param {SSECOptions} [opts.ssecOptions]
-   * @param {string} [opts.sourceMd5]
-   * @return {Promise<void>}
-   */
   async sendStream(bucketName, key, readStream, opts = {}) {
     try {
       const observeOptions = {
@@ -105,71 +39,42 @@ class S3Persistor extends AbstractPersistor {
       // observer will catch errors, clean up and log a warning
       pipeline(readStream, observer, () => {})
 
-      /** @type {S3.PutObjectRequest} */
+      // if we have an md5 hash, pass this to S3 to verify the upload
       const uploadOptions = {
         Bucket: bucketName,
         Key: key,
         Body: observer,
       }
 
-      if (this.settings.storageClass[bucketName]) {
-        uploadOptions.StorageClass = this.settings.storageClass[bucketName]
-      }
-
       if (opts.contentType) {
         uploadOptions.ContentType = opts.contentType
       }
       if (opts.contentEncoding) {
         uploadOptions.ContentEncoding = opts.contentEncoding
       }
-      if (opts.contentLength) {
-        uploadOptions.ContentLength = opts.contentLength
-      }
-      if (opts.ifNoneMatch === '*') {
-        uploadOptions.IfNoneMatch = '*'
-      }
-      if (opts.ssecOptions) {
-        Object.assign(uploadOptions, opts.ssecOptions.getPutOptions())
-      }
 
       // if we have an md5 hash, pass this to S3 to verify the upload - otherwise
       // we rely on the S3 client's checksum calculation to validate the upload
-      let computeChecksums = false
+      const clientOptions = {}
       if (opts.sourceMd5) {
         uploadOptions.ContentMD5 = PersistorHelper.hexToBase64(opts.sourceMd5)
       } else {
-        computeChecksums = true
+        clientOptions.computeChecksums = true
       }
 
-      if (this.settings.disableMultiPartUpload) {
-        await this._getClientForBucket(bucketName, computeChecksums)
-          .putObject(uploadOptions)
-          .promise()
-      } else {
-        await this._getClientForBucket(bucketName, computeChecksums)
-          .upload(uploadOptions, { partSize: this.settings.partSize })
-          .promise()
-      }
+      await this._getClientForBucket(bucketName, clientOptions)
+        .upload(uploadOptions, { partSize: this.settings.partSize })
+        .promise()
     } catch (err) {
       throw PersistorHelper.wrapError(
         err,
         'upload to S3 failed',
-        { bucketName, key, ifNoneMatch: opts.ifNoneMatch },
+        { bucketName, key },
         WriteError
       )
     }
   }
 
-  /**
-   * @param {string} bucketName
-   * @param {string} key
-   * @param {Object} [opts]
-   * @param {number} [opts.start]
-   * @param {number} [opts.end]
-   * @param {boolean} [opts.autoGunzip]
-   * @param {SSECOptions} [opts.ssecOptions]
-   * @return {Promise<NodeJS.ReadableStream>}
-   */
   async getObjectStream(bucketName, key, opts) {
     opts = opts || {}
 
@@ -180,9 +85,6 @@ class S3Persistor extends AbstractPersistor {
     if (opts.start != null && opts.end != null) {
       params.Range = `bytes=${opts.start}-${opts.end}`
     }
-    if (opts.ssecOptions) {
-      Object.assign(params, opts.ssecOptions.getGetOptions())
-    }
     const observer = new PersistorHelper.ObserverStream({
       metric: 's3.ingress', // ingress from S3 to us
       bucket: bucketName,
@@ -191,21 +93,18 @@ class S3Persistor extends AbstractPersistor {
     const req = this._getClientForBucket(bucketName).getObject(params)
     const stream = req.createReadStream()
 
-    let contentEncoding
     try {
       await new Promise((resolve, reject) => {
-        req.on('httpHeaders', (statusCode, headers) => {
+        req.on('httpHeaders', statusCode => {
           switch (statusCode) {
             case 200: // full response
             case 206: // partial response
-              contentEncoding = headers['content-encoding']
-              return resolve(undefined)
-            case 403: // AccessDenied
-              return // handled by stream.on('error') handler below
+              return resolve()
+            case 403: // AccessDenied is handled the same as NoSuchKey
             case 404: // NoSuchKey
-              return reject(new NotFoundError('not found'))
+              return reject(new NotFoundError())
             default:
-            // handled by stream.on('error') handler below
+              return reject(new Error('non success status: ' + statusCode))
           }
         })
         // The AWS SDK is forwarding any errors from the request to the stream.
@@ -223,32 +122,23 @@ class S3Persistor extends AbstractPersistor {
     }
     // Return a PassThrough stream with a minimal interface. It will buffer until the caller starts reading. It will emit errors from the source stream (Stream.pipeline passes errors along).
     const pass = new PassThrough()
-    const transformer = []
-    if (contentEncoding === 'gzip' && opts.autoGunzip) {
-      transformer.push(zlib.createGunzip())
-    }
-    pipeline(stream, observer, ...transformer, pass, err => {
+    pipeline(stream, observer, pass, err => {
       if (err) req.abort()
     })
     return pass
   }
 
-  /**
-   * @param {string} bucketName
-   * @param {string} key
-   * @return {Promise<string>}
-   */
   async getRedirectUrl(bucketName, key) {
     const expiresSeconds = Math.round(this.settings.signedUrlExpiryInMs / 1000)
     try {
-      return await this._getClientForBucket(bucketName).getSignedUrlPromise(
-        'getObject',
-        {
-          Bucket: bucketName,
-          Key: key,
-          Expires: expiresSeconds,
-        }
-      )
+      const url = await this._getClientForBucket(
+        bucketName
+      ).getSignedUrlPromise('getObject', {
+        Bucket: bucketName,
+        Key: key,
+        Expires: expiresSeconds,
+      })
+      return url
     } catch (err) {
       throw PersistorHelper.wrapError(
         err,
@@ -259,12 +149,6 @@ class S3Persistor extends AbstractPersistor {
     }
   }
 
-  /**
-   * @param {string} bucketName
-   * @param {string} key
-   * @param {string} [continuationToken]
-   * @return {Promise<void>}
-   */
   async deleteDirectory(bucketName, key, continuationToken) {
     let response
     const options = { Bucket: bucketName, Prefix: key }
@@ -285,8 +169,8 @@ class S3Persistor extends AbstractPersistor {
       )
     }
 
-    const objects = response.Contents?.map(item => ({ Key: item.Key || '' }))
-    if (objects?.length) {
+    const objects = response.Contents.map(item => ({ Key: item.Key }))
+    if (objects.length) {
       try {
         await this._getClientForBucket(bucketName)
           .deleteObjects({
@@ -316,22 +200,12 @@ class S3Persistor extends AbstractPersistor {
     }
   }
 
-  /**
-   * @param {string} bucketName
-   * @param {string} key
-   * @param {Object} opts
-   * @param {SSECOptions} [opts.ssecOptions]
-   * @return {Promise<S3.HeadObjectOutput>}
-   */
-  async #headObject(bucketName, key, opts = {}) {
-    const params = { Bucket: bucketName, Key: key }
-    if (opts.ssecOptions) {
-      Object.assign(params, opts.ssecOptions.getGetOptions())
-    }
+  async getObjectSize(bucketName, key) {
     try {
-      return await this._getClientForBucket(bucketName)
-        .headObject(params)
+      const response = await this._getClientForBucket(bucketName)
+        .headObject({ Bucket: bucketName, Key: key })
         .promise()
+      return response.ContentLength
     } catch (err) {
       throw PersistorHelper.wrapError(
         err,
@@ -342,51 +216,19 @@ class S3Persistor extends AbstractPersistor {
     }
   }
 
-  /**
-   * @param {string} bucketName
-   * @param {string} key
-   * @param {Object} opts
-   * @param {SSECOptions} [opts.ssecOptions]
-   * @return {Promise<number>}
-   */
-  async getObjectSize(bucketName, key, opts = {}) {
-    const response = await this.#headObject(bucketName, key, opts)
-    return response.ContentLength || 0
-  }
-
-  /**
-   * @param {string} bucketName
-   * @param {string} key
-   * @param {Object} opts
-   * @param {SSECOptions} [opts.ssecOptions]
-   * @return {Promise<string | undefined>}
-   */
-  async getObjectStorageClass(bucketName, key, opts = {}) {
-    const response = await this.#headObject(bucketName, key, opts)
-    return response.StorageClass
-  }
-
-  /**
-   * @param {string} bucketName
-   * @param {string} key
-   * @param {Object} opts
-   * @param {SSECOptions} [opts.ssecOptions]
-   * @param {boolean} [opts.etagIsNotMD5]
-   * @return {Promise<string>}
-   */
-  async getObjectMd5Hash(bucketName, key, opts = {}) {
+  async getObjectMd5Hash(bucketName, key) {
     try {
-      if (!opts.etagIsNotMD5) {
-        const response = await this.#headObject(bucketName, key, opts)
-        const md5 = S3Persistor._md5FromResponse(response)
-        if (md5) {
-          return md5
-        }
+      const response = await this._getClientForBucket(bucketName)
+        .headObject({ Bucket: bucketName, Key: key })
+        .promise()
+      const md5 = S3Persistor._md5FromResponse(response)
+      if (md5) {
+        return md5
       }
       // etag is not in md5 format
       Metrics.inc('s3.md5Download')
       return await PersistorHelper.calculateStreamMd5(
-        await this.getObjectStream(bucketName, key, opts)
+        await this.getObjectStream(bucketName, key)
       )
     } catch (err) {
       throw PersistorHelper.wrapError(
@@ -398,11 +240,6 @@ class S3Persistor extends AbstractPersistor {
     }
   }
 
-  /**
-   * @param {string} bucketName
-   * @param {string} key
-   * @return {Promise<void>}
-   */
   async deleteObject(bucketName, key) {
     try {
       await this._getClientForBucket(bucketName)
@@ -419,27 +256,12 @@ class S3Persistor extends AbstractPersistor {
     }
   }
 
-  /**
-   * @param {string} bucketName
-   * @param {string} sourceKey
-   * @param {string} destKey
-   * @param {Object} opts
-   * @param {SSECOptions} [opts.ssecSrcOptions]
-   * @param {SSECOptions} [opts.ssecOptions]
-   * @return {Promise<void>}
-   */
-  async copyObject(bucketName, sourceKey, destKey, opts = {}) {
+  async copyObject(bucketName, sourceKey, destKey) {
     const params = {
       Bucket: bucketName,
       Key: destKey,
       CopySource: `${bucketName}/${sourceKey}`,
     }
-    if (opts.ssecSrcOptions) {
-      Object.assign(params, opts.ssecSrcOptions.getCopyOptions())
-    }
-    if (opts.ssecOptions) {
-      Object.assign(params, opts.ssecOptions.getPutOptions())
-    }
     try {
       await this._getClientForBucket(bucketName).copyObject(params).promise()
     } catch (err) {
@@ -452,16 +274,9 @@ class S3Persistor extends AbstractPersistor {
     }
   }
 
-  /**
-   * @param {string} bucketName
-   * @param {string} key
-   * @param {Object} opts
-   * @param {SSECOptions} [opts.ssecOptions]
-   * @return {Promise<boolean>}
-   */
-  async checkIfObjectExists(bucketName, key, opts) {
+  async checkIfObjectExists(bucketName, key) {
     try {
-      await this.getObjectSize(bucketName, key, opts)
+      await this.getObjectSize(bucketName, key)
       return true
     } catch (err) {
       if (err instanceof NotFoundError) {
@@ -476,12 +291,6 @@ class S3Persistor extends AbstractPersistor {
     }
   }
 
-  /**
-   * @param {string} bucketName
-   * @param {string} key
-   * @param {string} [continuationToken]
-   * @return {Promise<number>}
-   */
   async directorySize(bucketName, key, continuationToken) {
     try {
       const options = {
@@ -495,8 +304,7 @@ class S3Persistor extends AbstractPersistor {
         .listObjectsV2(options)
         .promise()
 
-      const size =
-        response.Contents?.reduce((acc, item) => (item.Size || 0) + acc, 0) || 0
+      const size = response.Contents.reduce((acc, item) => item.Size + acc, 0)
       if (response.IsTruncated) {
         return (
           size +
@@ -518,38 +326,15 @@ class S3Persistor extends AbstractPersistor {
     }
   }
 
-  /**
-   * @param {string} bucket
-   * @param {boolean} computeChecksums
-   * @return {S3}
-   * @private
-   */
-  _getClientForBucket(bucket, computeChecksums = false) {
-    /** @type {S3.Types.ClientConfiguration} */
-    const clientOptions = {}
-    const cacheKey = `${bucket}:${computeChecksums}`
-    if (computeChecksums) {
-      clientOptions.computeChecksums = true
-    }
-    let client = this.#clients.get(cacheKey)
-    if (!client) {
-      client = new S3(
-        this._buildClientOptions(
-          this.settings.bucketCreds?.[bucket],
-          clientOptions
-        )
+  _getClientForBucket(bucket, clientOptions) {
+    return new S3(
+      this._buildClientOptions(
+        this.settings.bucketCreds?.[bucket],
+        clientOptions
       )
-      this.#clients.set(cacheKey, client)
-    }
-    return client
+    )
   }
 
-  /**
-   * @param {Object} bucketCredentials
-   * @param {S3.Types.ClientConfiguration} clientOptions
-   * @return {S3.Types.ClientConfiguration}
-   * @private
-   */
   _buildClientOptions(bucketCredentials, clientOptions) {
     const options = clientOptions || {}
 
@@ -571,7 +356,7 @@ class S3Persistor extends AbstractPersistor {
     if (this.settings.endpoint) {
       const endpoint = new URL(this.settings.endpoint)
       options.endpoint = this.settings.endpoint
-      options.sslEnabled = endpoint.protocol === 'https:'
+      options.sslEnabled = endpoint.protocol === 'https'
     }
 
     // path-style access is only used for acceptance tests
@@ -585,22 +370,9 @@ class S3Persistor extends AbstractPersistor {
       }
     }
 
-    if (options.sslEnabled && this.settings.ca && !options.httpOptions?.agent) {
-      options.httpOptions = options.httpOptions || {}
-      options.httpOptions.agent = new https.Agent({
-        rejectUnauthorized: true,
-        ca: this.settings.ca,
-      })
-    }
-
     return options
   }
 
-  /**
-   * @param {S3.HeadObjectOutput} response
-   * @return {string|null}
-   * @private
-   */
   static _md5FromResponse(response) {
     const md5 = (response.ETag || '').replace(/[ "]/g, '')
     if (!md5.match(/^[a-f0-9]{32}$/)) {
@@ -610,8 +382,3 @@ class S3Persistor extends AbstractPersistor {
     return md5
   }
 }
-
-module.exports = {
-  S3Persistor,
-  SSECOptions,
-}

libraries/object-persistor/test/Init.js

@@ -25,9 +25,4 @@ SandboxedModule.configure({
     },
   },
   globals: { Buffer, Math, console, process, URL },
-  sourceTransformers: {
-    removeNodePrefix: function (source) {
-      return source.replace(/require\(['"]node:/g, "require('")
-    },
-  },
 })

libraries/object-persistor/test/unit/FSPersistorTests.js

@@ -1,10 +1,10 @@
-const crypto = require('node:crypto')
+const crypto = require('crypto')
 const { expect } = require('chai')
 const mockFs = require('mock-fs')
-const fs = require('node:fs')
-const fsPromises = require('node:fs/promises')
-const Path = require('node:path')
-const StreamPromises = require('node:stream/promises')
+const fs = require('fs')
+const fsPromises = require('fs/promises')
+const Path = require('path')
+const StreamPromises = require('stream/promises')
 const SandboxedModule = require('sandboxed-module')
 const Errors = require('../../src/Errors')
 

libraries/object-persistor/test/unit/GcsPersistorTests.js

@@ -1,4 +1,4 @@
-const { EventEmitter } = require('node:events')
+const { EventEmitter } = require('events')
 const sinon = require('sinon')
 const chai = require('chai')
 const { expect } = chai
@@ -45,11 +45,11 @@ describe('GcsPersistorTests', function () {
 
     files = [
       {
-        metadata: { size: '11', md5Hash: '/////wAAAAD/////AAAAAA==' },
+        metadata: { size: 11, md5Hash: '/////wAAAAD/////AAAAAA==' },
         delete: sinon.stub(),
       },
       {
-        metadata: { size: '22', md5Hash: '/////wAAAAD/////AAAAAA==' },
+        metadata: { size: 22, md5Hash: '/////wAAAAD/////AAAAAA==' },
         delete: sinon.stub(),
       },
     ]
@@ -63,7 +63,7 @@ describe('GcsPersistorTests', function () {
 
       read() {
         if (this.err) return this.emit('error', this.err)
-        this.emit('response', { statusCode: this.statusCode, headers: {} })
+        this.emit('response', { statusCode: this.statusCode })
       }
     }
 
@@ -302,7 +302,7 @@ describe('GcsPersistorTests', function () {
       })
 
       it('should return the object size', function () {
-        expect(size).to.equal(11)
+        expect(size).to.equal(files[0].metadata.size)
       })
 
       it('should pass the bucket and key to GCS', function () {

libraries/object-persistor/test/unit/PersistorFactoryTests.js

@@ -1,7 +1,7 @@
 const chai = require('chai')
 const { expect } = chai
 const SandboxedModule = require('sandboxed-module')
-const StreamPromises = require('node:stream/promises')
+const StreamPromises = require('stream/promises')
 
 const MODULE_PATH = '../../src/PersistorFactory.js'
 
@@ -32,7 +32,7 @@ describe('PersistorManager', function () {
     Settings = {}
     const requires = {
       './GcsPersistor': GcsPersistor,
-      './S3Persistor': { S3Persistor },
+      './S3Persistor': S3Persistor,
       './FSPersistor': FSPersistor,
       '@overleaf/logger': {
         info() {},

libraries/object-persistor/test/unit/S3PersistorTests.js

@@ -3,7 +3,7 @@ const chai = require('chai')
 const { expect } = chai
 const SandboxedModule = require('sandboxed-module')
 const Errors = require('../../src/Errors')
-const { EventEmitter } = require('node:events')
+const { EventEmitter } = require('events')
 
 const MODULE_PATH = '../../src/S3Persistor.js'
 
@@ -91,22 +91,8 @@ describe('S3PersistorTests', function () {
 
       createReadStream() {
         setTimeout(() => {
-          if (this.notFoundSSEC) {
-            // special case for AWS S3: 404 NoSuchKey wrapped in a 400. A single request received a single response, and multiple httpHeaders events are triggered. Don't ask.
-            this.emit('httpHeaders', 400, {})
-            this.emit('httpHeaders', 404, {})
-            ReadStream.emit('error', S3NotFoundError)
-            return
-          }
-
           if (this.err) return ReadStream.emit('error', this.err)
-          this.emit('httpHeaders', this.statusCode, {})
-          if (this.statusCode === 403) {
-            ReadStream.emit('error', S3AccessDeniedError)
-          }
-          if (this.statusCode === 404) {
-            ReadStream.emit('error', S3NotFoundError)
-          }
+          this.emit('httpHeaders', this.statusCode)
         })
         return ReadStream
       }
@@ -147,7 +133,7 @@ describe('S3PersistorTests', function () {
       deleteObjects: sinon.stub().returns(EmptyPromise),
       getSignedUrlPromise: sinon.stub().resolves(redirectUrl),
     }
-    S3 = sinon.stub().callsFake(() => Object.assign({}, S3Client))
+    S3 = sinon.stub().returns(S3Client)
 
     Hash = {
       end: sinon.stub(),
@@ -173,7 +159,7 @@ describe('S3PersistorTests', function () {
         crypto,
       },
       globals: { console, Buffer },
-    }).S3Persistor)(settings)
+    }))(settings)
   })
 
   describe('getObjectStream', function () {
@@ -352,34 +338,6 @@ describe('S3PersistorTests', function () {
       })
     })
 
-    describe("when the file doesn't exist -- SSEC", function () {
-      let error, stream
-
-      beforeEach(async function () {
-        S3GetObjectRequest.notFoundSSEC = 404
-        try {
-          stream = await S3Persistor.getObjectStream(bucket, key)
-        } catch (err) {
-          error = err
-        }
-      })
-      it('does not return a stream', function () {
-        expect(stream).not.to.exist
-      })
-
-      it('throws a NotFoundError', function () {
-        expect(error).to.be.an.instanceOf(Errors.NotFoundError)
-      })
-
-      it('wraps the error', function () {
-        expect(error.cause).to.exist
-      })
-
-      it('stores the bucket and key in the error', function () {
-        expect(error.info).to.include({ bucketName: bucket, key })
-      })
-    })
-
     describe('when access to the file is denied', function () {
       let error, stream
 
@@ -401,7 +359,7 @@ describe('S3PersistorTests', function () {
       })
 
       it('wraps the error', function () {
-        expect(error.cause).to.equal(S3AccessDeniedError)
+        expect(error.cause).to.exist
       })
 
       it('stores the bucket and key in the error', function () {
@@ -1027,22 +985,4 @@ describe('S3PersistorTests', function () {
       })
     })
   })
-
-  describe('_getClientForBucket', function () {
-    it('should return same instance for same bucket', function () {
-      const a = S3Persistor._getClientForBucket('foo')
-      const b = S3Persistor._getClientForBucket('foo')
-      expect(a).to.equal(b)
-    })
-    it('should return different instance for different bucket', function () {
-      const a = S3Persistor._getClientForBucket('foo')
-      const b = S3Persistor._getClientForBucket('bar')
-      expect(a).to.not.equal(b)
-    })
-    it('should return different instance for same bucket different computeChecksums', function () {
-      const a = S3Persistor._getClientForBucket('foo', false)
-      const b = S3Persistor._getClientForBucket('foo', true)
-      expect(a).to.not.equal(b)
-    })
-  })
 })

libraries/overleaf-editor-core/.nvmrc

@@ -1 +1 @@
-20.18.2
+18.20.2

libraries/overleaf-editor-core/buildscript.txt

@@ -5,6 +5,6 @@ overleaf-editor-core
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=20.18.2
+--node-version=18.20.2
 --public-repo=False
 --script-version=4.5.0

libraries/overleaf-editor-core/lib/blob.js

@@ -40,11 +40,6 @@ class Blob {
 
   static NotFoundError = NotFoundError
 
-  /**
-   * @param {string} hash
-   * @param {number} byteLength
-   * @param {number} [stringLength]
-   */
   constructor(hash, byteLength, stringLength) {
     this.setHash(hash)
     this.setByteLength(byteLength)
@@ -68,14 +63,14 @@ class Blob {
 
   /**
    * Hex hash.
-   * @return {String}
+   * @return {?String}
    */
   getHash() {
     return this.hash
   }
 
   setHash(hash) {
-    assert.match(hash, Blob.HEX_HASH_RX, 'bad hash')
+    assert.maybe.match(hash, Blob.HEX_HASH_RX, 'bad hash')
     this.hash = hash
   }
 
@@ -88,13 +83,13 @@ class Blob {
   }
 
   setByteLength(byteLength) {
-    assert.integer(byteLength, 'bad byteLength')
+    assert.maybe.integer(byteLength, 'bad byteLength')
     this.byteLength = byteLength
   }
 
   /**
    * Utf-8 length of the blob content, if it appears to be valid UTF-8.
-   * @return {number|undefined}
+   * @return {?number}
    */
   getStringLength() {
     return this.stringLength

libraries/overleaf-editor-core/lib/change.js

@@ -100,9 +100,6 @@ class Change {
     )
   }
 
-  /**
-   * @return {Operation[]}
-   */
   getOperations() {
     return this.operations
   }
@@ -251,24 +248,6 @@ class Change {
    * @param {boolean} [opts.strict] - Do not ignore recoverable errors
    */
   applyTo(snapshot, opts = {}) {
-    // eslint-disable-next-line no-unused-vars
-    for (const operation of this.iterativelyApplyTo(snapshot, opts)) {
-      // Nothing to do: we're just consuming the iterator for the side effects
-    }
-  }
-
-  /**
-   * Generator that applies this change to a snapshot and yields each
-   * operation after it has been applied.
-   *
-   * Recoverable errors (caused by historical bad data) are ignored unless
-   * opts.strict is true
-   *
-   * @param {Snapshot} snapshot modified in place
-   * @param {object} opts
-   * @param {boolean} [opts.strict] - Do not ignore recoverable errors
-   */
-  *iterativelyApplyTo(snapshot, opts = {}) {
     assert.object(snapshot, 'bad snapshot')
 
     for (const operation of this.operations) {
@@ -282,7 +261,6 @@ class Change {
           throw err
         }
       }
-      yield operation
     }
 
     // update project version if present in change

libraries/overleaf-editor-core/lib/change_note.js

@@ -10,7 +10,7 @@ const Change = require('./change')
 class ChangeNote {
   /**
    * @param {number} baseVersion the new base version for the change
-   * @param {Change} [change]
+   * @param {?Change} change
    */
   constructor(baseVersion, change) {
     assert.integer(baseVersion, 'bad baseVersion')

libraries/overleaf-editor-core/lib/file.js

@@ -95,7 +95,7 @@ class File {
 
   /**
    * @param  {number} byteLength
-   * @param  {number} [stringLength]
+   * @param  {number?} stringLength
    * @param  {Object} [metadata]
    * @return {File}
    */

libraries/overleaf-editor-core/lib/file_data/index.js

@@ -47,7 +47,7 @@ class FileData {
 
   /** @see File.createHollow
    * @param {number} byteLength
-   * @param {number} [stringLength]
+   * @param {number|null} stringLength
    */
   static createHollow(byteLength, stringLength) {
     if (stringLength == null) {
@@ -63,14 +63,20 @@ class FileData {
    */
   static createLazyFromBlobs(blob, rangesBlob) {
     assert.instance(blob, Blob, 'FileData: bad blob')
-    const stringLength = blob.getStringLength()
-    if (stringLength == null) {
-      return new BinaryFileData(blob.getHash(), blob.getByteLength())
+    if (blob.getStringLength() == null) {
+      return new BinaryFileData(
+        // TODO(das7pad): see call-sites
+        // @ts-ignore
+        blob.getHash(),
+        blob.getByteLength()
+      )
     }
     return new LazyStringFileData(
+      // TODO(das7pad): see call-sites
+      // @ts-ignore
       blob.getHash(),
       rangesBlob?.getHash(),
-      stringLength
+      blob.getStringLength()
     )
   }
 

libraries/overleaf-editor-core/lib/file_data/string_file_data.js

@@ -142,8 +142,12 @@ class StringFileData extends FileData {
         trackedChanges: this.trackedChanges.toRaw(),
       }
       const rangesBlob = await blobStore.putObject(ranges)
+      // TODO(das7pad): Provide interface that guarantees hash exists?
+      // @ts-ignore
       return { hash: blob.getHash(), rangesHash: rangesBlob.getHash() }
     }
+    // TODO(das7pad): Provide interface that guarantees hash exists?
+    // @ts-ignore
     return { hash: blob.getHash() }
   }
 }

libraries/overleaf-editor-core/lib/file_map.js

@@ -22,7 +22,7 @@ class NonUniquePathnameError extends PathnameError {
    * @param {string[]} pathnames
    */
   constructor(pathnames) {
-    super('pathnames are not unique', { pathnames })
+    super('pathnames are not unique: ' + pathnames, { pathnames })
     this.pathnames = pathnames
   }
 }
@@ -30,13 +30,9 @@ class NonUniquePathnameError extends PathnameError {
 class BadPathnameError extends PathnameError {
   /**
    * @param {string} pathname
-   * @param {string} reason
    */
-  constructor(pathname, reason) {
-    if (pathname.length > 10) {
-      pathname = pathname.slice(0, 5) + '...' + pathname.slice(-5)
-    }
-    super('invalid pathname', { reason, pathname })
+  constructor(pathname) {
+    super(pathname + ' is not a valid pathname', { pathname })
     this.pathname = pathname
   }
 }
@@ -46,7 +42,7 @@ class PathnameConflictError extends PathnameError {
    * @param {string} pathname
    */
   constructor(pathname) {
-    super('pathname conflicts with another file', { pathname })
+    super(`pathname '${pathname}' conflicts with another file`, { pathname })
     this.pathname = pathname
   }
 }
@@ -56,7 +52,7 @@ class FileNotFoundError extends PathnameError {
    * @param {string} pathname
    */
   constructor(pathname) {
-    super('file does not exist', { pathname })
+    super(`file ${pathname} does not exist`, { pathname })
     this.pathname = pathname
   }
 }
@@ -319,9 +315,8 @@ function checkPathnamesAreUnique(files) {
  */
 function checkPathname(pathname) {
   assert.nonEmptyString(pathname, 'bad pathname')
-  const [isClean, reason] = safePathname.isCleanDebug(pathname)
-  if (isClean) return
-  throw new FileMap.BadPathnameError(pathname, reason)
+  if (safePathname.isClean(pathname)) return
+  throw new FileMap.BadPathnameError(pathname)
 }
 
 /**

libraries/overleaf-editor-core/lib/history.js

@@ -109,7 +109,7 @@ class History {
    * @param {BlobStore} blobStore
    * @param {number} [concurrency] applies separately to files, changes and
    *                               operations
-   * @return {Promise<import('overleaf-editor-core/lib/types').RawHistory>}
+   * @return {Promise.<Object>}
    */
   async store(blobStore, concurrency) {
     assert.maybe.number(concurrency, 'bad concurrency')

libraries/overleaf-editor-core/lib/operation/text_operation.js

@@ -56,34 +56,18 @@ class TextOperation extends EditOperation {
 
   constructor() {
     super()
-
-    /**
-     * When an operation is applied to an input string, you can think of this as
-     * if an imaginary cursor runs over the entire string and skips over some
-     * parts, removes some parts and inserts characters at some positions. These
-     * actions (skip/remove/insert) are stored as an array in the "ops" property.
-     * @type {ScanOp[]}
-     */
+    // When an operation is applied to an input string, you can think of this as
+    // if an imaginary cursor runs over the entire string and skips over some
+    // parts, removes some parts and inserts characters at some positions. These
+    // actions (skip/remove/insert) are stored as an array in the "ops" property.
+    /** @type {ScanOp[]} */
     this.ops = []
-
-    /**
-     * An operation's baseLength is the length of every string the operation
-     * can be applied to.
-     */
+    // An operation's baseLength is the length of every string the operation
+    // can be applied to.
     this.baseLength = 0
-
-    /**
-     * The targetLength is the length of every string that results from applying
-     * the operation on a valid input string.
-     */
+    // The targetLength is the length of every string that results from applying
+    // the operation on a valid input string.
     this.targetLength = 0
-
-    /**
-     * The expected content hash after this operation is applied
-     *
-     * @type {string | null}
-     */
-    this.contentHash = null
   }
 
   /**
@@ -239,12 +223,7 @@ class TextOperation extends EditOperation {
    * @returns {RawTextOperation}
    */
   toJSON() {
-    /** @type {RawTextOperation} */
-    const json = { textOperation: this.ops.map(op => op.toJSON()) }
-    if (this.contentHash != null) {
-      json.contentHash = this.contentHash
-    }
-    return json
+    return { textOperation: this.ops.map(op => op.toJSON()) }
   }
 
   /**
@@ -252,7 +231,7 @@ class TextOperation extends EditOperation {
    * @param {RawTextOperation} obj
    * @returns {TextOperation}
    */
-  static fromJSON = function ({ textOperation: ops, contentHash }) {
+  static fromJSON = function ({ textOperation: ops }) {
     const o = new TextOperation()
     for (const op of ops) {
       if (isRetain(op)) {
@@ -271,9 +250,6 @@ class TextOperation extends EditOperation {
         throw new UnprocessableError('unknown operation: ' + JSON.stringify(op))
       }
     }
-    if (contentHash != null) {
-      o.contentHash = contentHash
-    }
     return o
   }
 

libraries/overleaf-editor-core/lib/safe_pathname.js

@@ -64,57 +64,17 @@ function cleanPart(filename) {
  * @return {String}
  */
 exports.clean = function (pathname) {
-  return exports.cleanDebug(pathname)[0]
-}
-
-/**
- * See clean
- * @param {string} pathname
- * @return {[string,string]}
- */
-exports.cleanDebug = function (pathname) {
-  let prev = pathname
-  let reason = ''
-
-  /**
-   * @param {string} label
-   */
-  function recordReasonIfChanged(label) {
-    if (pathname === prev) return
-    if (reason) reason += ','
-    reason += label
-    prev = pathname
-  }
   pathname = path.normalize(pathname)
-  recordReasonIfChanged('normalize')
-
-  pathname = pathname.replace(/\\/g, '/')
-  recordReasonIfChanged('workaround for IE')
-
-  pathname = pathname.replace(/\/+/g, '/')
-  recordReasonIfChanged('no multiple slashes')
-
-  pathname = pathname.replace(/^(\/.*)$/, '_$1')
-  recordReasonIfChanged('no leading /')
-
-  pathname = pathname.replace(/^(.+)\/$/, '$1')
-  recordReasonIfChanged('no trailing /')
-
-  pathname = pathname.replace(/^ *(.*)$/, '$1')
-  recordReasonIfChanged('no leading spaces')
-
-  pathname = pathname.replace(/^(.*[^ ]) *$/, '$1')
-  recordReasonIfChanged('no trailing spaces')
-
+  pathname = pathname.replace(/\\/g, '/') // workaround for IE
+  pathname = pathname.replace(/\/+/g, '/') // no multiple slashes
+  pathname = pathname.replace(/^(\/.*)$/, '_$1') // no leading /
+  pathname = pathname.replace(/^(.+)\/$/, '$1') // no trailing /
+  pathname = pathname.replace(/^ *(.*)$/, '$1') // no leading spaces
+  pathname = pathname.replace(/^(.*[^ ]) *$/, '$1') // no trailing spaces
   if (pathname.length === 0) pathname = '_'
-  recordReasonIfChanged('empty')
-
   pathname = pathname.split('/').map(cleanPart).join('/')
-  recordReasonIfChanged('cleanPart')
-
   pathname = pathname.replace(BLOCKED_FILE_RX, '@$1')
-  recordReasonIfChanged('BLOCKED_FILE_RX')
-  return [pathname, reason]
+  return pathname
 }
 
 /**
@@ -124,19 +84,9 @@ exports.cleanDebug = function (pathname) {
  * @return {Boolean}
  */
 exports.isClean = function pathnameIsClean(pathname) {
-  return exports.isCleanDebug(pathname)[0]
-}
-
-/**
- * A pathname is clean (see clean) and not too long.
- *
- * @param {string} pathname
- * @return {[boolean,string]}
- */
-exports.isCleanDebug = function (pathname) {
-  if (pathname.length > MAX_PATH) return [false, 'MAX_PATH']
-  if (pathname.length === 0) return [false, 'empty']
-  const [cleanPathname, reason] = exports.cleanDebug(pathname)
-  if (cleanPathname !== pathname) return [false, reason]
-  return [true, '']
+  return (
+    exports.clean(pathname) === pathname &&
+    pathname.length <= MAX_PATH &&
+    pathname.length > 0
+  )
 }

libraries/overleaf-editor-core/lib/snapshot.js

@@ -224,7 +224,7 @@ class Snapshot {
    *
    * @param {string} kind see {File#load}
    * @param {ReadonlyBlobStore} blobStore
-   * @return {Promise<Record<string, File>>} an object where keys are the pathnames and
+   * @return {Promise<Object>} an object where keys are the pathnames and
    * values are the files in the snapshot
    */
   async loadFiles(kind, blobStore) {

libraries/overleaf-editor-core/lib/types.ts

@@ -132,7 +132,6 @@ export type RawScanOp = RawInsertOp | RawRemoveOp | RawRetainOp
 
 export type RawTextOperation = {
   textOperation: RawScanOp[]
-  contentHash?: string
 }
 
 export type RawAddCommentOperation = {

libraries/overleaf-editor-core/lib/v2_doc_versions.js

@@ -27,7 +27,7 @@ class V2DocVersions {
   }
 
   /**
-   * @return {RawV2DocVersions|null}
+   * @return {?RawV2DocVersions}
    */
   toRaw() {
     if (!this.data) return null

libraries/overleaf-editor-core/package.json

@@ -20,7 +20,7 @@
     "@types/check-types": "^7.3.7",
     "@types/path-browserify": "^1.0.2",
     "chai": "^3.3.0",
-    "mocha": "^11.1.0",
+    "mocha": "^10.2.0",
     "sinon": "^9.2.4",
     "typescript": "^5.0.4"
   },

libraries/overleaf-editor-core/test/safe_pathname.test.js

@@ -5,11 +5,10 @@ const ot = require('..')
 const safePathname = ot.safePathname
 
 describe('safePathname', function () {
-  function expectClean(input, output, reason = '') {
+  function expectClean(input, output) {
     // check expected output and also idempotency
-    const [cleanedInput, gotReason] = safePathname.cleanDebug(input)
+    const cleanedInput = safePathname.clean(input)
     expect(cleanedInput).to.equal(output)
-    expect(gotReason).to.equal(reason)
     expect(safePathname.clean(cleanedInput)).to.equal(cleanedInput)
     expect(safePathname.isClean(cleanedInput)).to.be.true
   }
@@ -23,56 +22,44 @@ describe('safePathname', function () {
     expect(safePathname.isClean('rm -rf /')).to.be.falsy
 
     // replace invalid characters with underscores
-    expectClean(
-      'test-s*\u0001\u0002m\u0007st\u0008.jpg',
-      'test-s___m_st_.jpg',
-      'cleanPart'
-    )
+    expectClean('test-s*\u0001\u0002m\u0007st\u0008.jpg', 'test-s___m_st_.jpg')
 
     // keep slashes, normalize paths, replace ..
-    expectClean('./foo', 'foo', 'normalize')
-    expectClean('../foo', '__/foo', 'cleanPart')
-    expectClean('foo/./bar', 'foo/bar', 'normalize')
-    expectClean('foo/../bar', 'bar', 'normalize')
-    expectClean('../../tricky/foo.bar', '__/__/tricky/foo.bar', 'cleanPart')
-    expectClean(
-      'foo/../../tricky/foo.bar',
-      '__/tricky/foo.bar',
-      'normalize,cleanPart'
-    )
-    expectClean('foo/bar/../../tricky/foo.bar', 'tricky/foo.bar', 'normalize')
-    expectClean(
-      'foo/bar/baz/../../tricky/foo.bar',
-      'foo/tricky/foo.bar',
-      'normalize'
-    )
+    expectClean('./foo', 'foo')
+    expectClean('../foo', '__/foo')
+    expectClean('foo/./bar', 'foo/bar')
+    expectClean('foo/../bar', 'bar')
+    expectClean('../../tricky/foo.bar', '__/__/tricky/foo.bar')
+    expectClean('foo/../../tricky/foo.bar', '__/tricky/foo.bar')
+    expectClean('foo/bar/../../tricky/foo.bar', 'tricky/foo.bar')
+    expectClean('foo/bar/baz/../../tricky/foo.bar', 'foo/tricky/foo.bar')
 
     // remove illegal chars even when there is no extension
-    expectClean('**foo', '__foo', 'cleanPart')
+    expectClean('**foo', '__foo')
 
     // remove windows file paths
-    expectClean('c:\\temp\\foo.txt', 'c:/temp/foo.txt', 'workaround for IE')
+    expectClean('c:\\temp\\foo.txt', 'c:/temp/foo.txt')
 
     // do not allow a leading slash (relative paths only)
-    expectClean('/foo', '_/foo', 'no leading /')
-    expectClean('//foo', '_/foo', 'normalize,no leading /')
+    expectClean('/foo', '_/foo')
+    expectClean('//foo', '_/foo')
 
     // do not allow multiple leading slashes
-    expectClean('//foo', '_/foo', 'normalize,no leading /')
+    expectClean('//foo', '_/foo')
 
     // do not allow a trailing slash
-    expectClean('/', '_', 'no leading /,no trailing /')
-    expectClean('foo/', 'foo', 'no trailing /')
-    expectClean('foo.tex/', 'foo.tex', 'no trailing /')
+    expectClean('/', '_')
+    expectClean('foo/', 'foo')
+    expectClean('foo.tex/', 'foo.tex')
 
     // do not allow multiple trailing slashes
-    expectClean('//', '_', 'normalize,no leading /,no trailing /')
-    expectClean('///', '_', 'normalize,no leading /,no trailing /')
-    expectClean('foo//', 'foo', 'normalize,no trailing /')
+    expectClean('//', '_')
+    expectClean('///', '_')
+    expectClean('foo//', 'foo')
 
     // file and folder names that consist of . and .. are not OK
-    expectClean('.', '_', 'cleanPart')
-    expectClean('..', '__', 'cleanPart')
+    expectClean('.', '_')
+    expectClean('..', '__')
     // we will allow name with more dots e.g. ... and ....
     expectClean('...', '...')
     expectClean('....', '....')
@@ -95,10 +82,10 @@ describe('safePathname', function () {
     expectClean('a b.png', 'a b.png')
 
     // leading and trailing spaces are not OK
-    expectClean(' foo', 'foo', 'no leading spaces')
-    expectClean('  foo', 'foo', 'no leading spaces')
-    expectClean('foo ', 'foo', 'no trailing spaces')
-    expectClean('foo  ', 'foo', 'no trailing spaces')
+    expectClean(' foo', 'foo')
+    expectClean('  foo', 'foo')
+    expectClean('foo ', 'foo')
+    expectClean('foo  ', 'foo')
 
     // reserved file names on Windows should not be OK, but we already have
     // some in the old system, so have to allow them for now
@@ -113,14 +100,14 @@ describe('safePathname', function () {
     // there's no particular reason to allow multiple slashes; sometimes people
     // seem to rename files to URLs (https://domain/path) in an attempt to
     // upload a file, and this results in an empty directory name
-    expectClean('foo//bar.png', 'foo/bar.png', 'normalize')
-    expectClean('foo///bar.png', 'foo/bar.png', 'normalize')
+    expectClean('foo//bar.png', 'foo/bar.png')
+    expectClean('foo///bar.png', 'foo/bar.png')
 
     // Check javascript property handling
     expectClean('foo/prototype', 'foo/prototype') // OK as part of a pathname
     expectClean('prototype/test.txt', 'prototype/test.txt')
-    expectClean('prototype', '@prototype', 'BLOCKED_FILE_RX') // not OK as whole pathname
-    expectClean('hasOwnProperty', '@hasOwnProperty', 'BLOCKED_FILE_RX')
-    expectClean('**proto**', '@__proto__', 'cleanPart,BLOCKED_FILE_RX')
+    expectClean('prototype', '@prototype') // not OK as whole pathname
+    expectClean('hasOwnProperty', '@hasOwnProperty')
+    expectClean('**proto**', '@__proto__')
   })
 })

libraries/promise-utils/.nvmrc

@@ -1 +1 @@
-20.18.2
+18.20.2

libraries/promise-utils/buildscript.txt

@@ -5,6 +5,6 @@ promise-utils
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=20.18.2
+--node-version=18.20.2
 --public-repo=False
 --script-version=4.5.0

libraries/promise-utils/index.js

@@ -1,4 +1,4 @@
-const { promisify, callbackify } = require('node:util')
+const { promisify, callbackify } = require('util')
 const pLimit = require('p-limit')
 
 module.exports = {
@@ -13,7 +13,6 @@ module.exports = {
   expressify,
   expressifyErrorHandler,
   promiseMapWithLimit,
-  promiseMapSettledWithLimit,
 }
 
 /**
@@ -254,30 +253,8 @@ function expressifyErrorHandler(fn) {
  * Map values in `array` with the async function `fn`
  *
  * Limit the number of unresolved promises to `concurrency`.
- * @template T
- * @template V
- * @param {number} concurrency
- * @param {Array<T>} array
- * @param {(arg: T) => Promise<V>} fn
- * @return {Promise<Array<Awaited<V>>>}
  */
-async function promiseMapWithLimit(concurrency, array, fn) {
+function promiseMapWithLimit(concurrency, array, fn) {
   const limit = pLimit(concurrency)
-  return await Promise.all(array.map(x => limit(() => fn(x))))
-}
-
-/**
- * Map values in `array` with the async function `fn`
- *
- * Limit the number of unresolved promises to `concurrency`.
- *
- * @template T, U
- * @param {number} concurrency
- * @param {Array<T>} array
- * @param {(T) => Promise<U>} fn
- * @return {Promise<Array<PromiseSettledResult<U>>>}
- */
-function promiseMapSettledWithLimit(concurrency, array, fn) {
-  const limit = pLimit(concurrency)
-  return Promise.allSettled(array.map(x => limit(() => fn(x))))
+  return Promise.all(array.map(x => limit(() => fn(x))))
 }

libraries/promise-utils/package.json

@@ -18,7 +18,7 @@
   "devDependencies": {
     "chai": "^4.3.10",
     "chai-as-promised": "^7.1.1",
-    "mocha": "^11.1.0",
+    "mocha": "^10.2.0",
     "typescript": "^5.0.4"
   },
   "dependencies": {

libraries/ranges-tracker/.nvmrc

@@ -1 +1 @@
-20.18.2
+18.20.2

libraries/ranges-tracker/buildscript.txt

@@ -5,6 +5,6 @@ ranges-tracker
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=20.18.2
+--node-version=18.20.2
 --public-repo=False
 --script-version=4.5.0

libraries/ranges-tracker/index.cjs

@@ -145,7 +145,11 @@ class RangesTracker {
   }
 
   removeChangeId(changeId) {
-    this.removeChangeIds([changeId])
+    const change = this.getChange(changeId)
+    if (change == null) {
+      return
+    }
+    this._removeChange(change)
   }
 
   removeChangeIds(ids) {
@@ -312,7 +316,7 @@ class RangesTracker {
     const movedChanges = []
     const removeChanges = []
     const newChanges = []
-    const trackedDeletesAtOpPosition = []
+
     for (let i = 0; i < this.changes.length; i++) {
       change = this.changes[i]
       const changeStart = change.op.p
@@ -323,15 +327,13 @@ class RangesTracker {
           change.op.p += opLength
           movedChanges.push(change)
         } else if (opStart === changeStart) {
+          // If we are undoing, then we want to cancel any existing delete ranges if we can.
+          // Check if the insert matches the start of the delete, and just remove it from the delete instead if so.
           if (
-            !alreadyMerged &&
             undoing &&
             change.op.d.length >= op.i.length &&
             change.op.d.slice(0, op.i.length) === op.i
           ) {
-            // If we are undoing, then we want to reject any existing tracked delete if we can.
-            // Check if the insert matches the start of the delete, and just
-            // remove it from the delete instead if so.
             change.op.d = change.op.d.slice(op.i.length)
             change.op.p += op.i.length
             if (change.op.d === '') {
@@ -340,25 +342,9 @@ class RangesTracker {
               movedChanges.push(change)
             }
             alreadyMerged = true
-
-            // Any tracked delete that came before this tracked delete
-            // rejection was moved after the incoming insert. Move them back
-            // so that they appear before the tracked delete rejection.
-            for (const trackedDelete of trackedDeletesAtOpPosition) {
-              trackedDelete.op.p -= opLength
-            }
           } else {
-            // We're not rejecting that tracked delete. Move it after the
-            // insert.
             change.op.p += opLength
             movedChanges.push(change)
-
-            // Keep track of tracked deletes that are at the same position as the
-            // insert. If we find a tracked delete to reject, we'll want to
-            // reposition them.
-            if (!alreadyMerged) {
-              trackedDeletesAtOpPosition.push(change)
-            }
           }
         }
       } else if (change.op.i != null) {
@@ -638,11 +624,9 @@ class RangesTracker {
   }
 
   _addOp(op, metadata) {
-    // Don't take a reference to the existing op since we'll modify this in place with future changes
-    op = this._clone(op)
     const change = {
       id: this.newId(),
-      op,
+      op: this._clone(op), // Don't take a reference to the existing op since we'll modify this in place with future changes
       metadata: this._clone(metadata),
     }
     this.changes.push(change)
@@ -665,7 +649,7 @@ class RangesTracker {
   }
 
   _removeChange(change) {
-    this.changes = this.changes.filter(c => c !== change)
+    this.changes = this.changes.filter(c => c.id !== change.id)
     this._markAsDirty(change, 'change', 'removed')
   }
 

libraries/ranges-tracker/package.json

@@ -20,7 +20,7 @@
   },
   "devDependencies": {
     "chai": "^4.3.6",
-    "mocha": "^11.1.0",
+    "mocha": "^10.2.0",
     "typescript": "^5.0.4"
   }
 }

libraries/ranges-tracker/test/unit/ranges-tracker-test.js

@@ -4,7 +4,6 @@ const RangesTracker = require('../..')
 describe('RangesTracker', function () {
   describe('with duplicate change ids', function () {
     beforeEach(function () {
-      this.comments = []
       this.changes = [
         { id: 'id1', op: { p: 1, i: 'hello' } },
         { id: 'id2', op: { p: 10, i: 'world' } },
@@ -27,199 +26,4 @@ describe('RangesTracker', function () {
       expect(this.rangesTracker.changes).to.deep.equal([this.changes[2]])
     })
   })
-
-  describe('with duplicate tracked insert ids', function () {
-    beforeEach(function () {
-      this.comments = []
-      this.changes = [
-        { id: 'id1', op: { p: 10, i: 'one' } },
-        { id: 'id1', op: { p: 20, i: 'two' } },
-        { id: 'id1', op: { p: 30, d: 'three' } },
-      ]
-      this.rangesTracker = new RangesTracker(this.changes, this.comments)
-    })
-
-    it("deleting one tracked insert doesn't delete the others", function () {
-      this.rangesTracker.applyOp({ p: 20, d: 'two' })
-      expect(this.rangesTracker.changes).to.deep.equal([
-        this.changes[0],
-        this.changes[2],
-      ])
-    })
-  })
-
-  describe('with duplicate tracked delete ids', function () {
-    beforeEach(function () {
-      this.comments = []
-      this.changes = [
-        { id: 'id1', op: { p: 10, d: 'one' } },
-        { id: 'id1', op: { p: 20, d: 'two' } },
-        { id: 'id1', op: { p: 30, d: 'three' } },
-      ]
-      this.rangesTracker = new RangesTracker(this.changes, this.comments)
-    })
-
-    it('deleting over tracked deletes in tracked changes mode removes the tracked deletes covered', function () {
-      this.rangesTracker.track_changes = true
-      this.rangesTracker.applyOp({
-        p: 15,
-        d: '567890123456789012345',
-      })
-      expect(this.rangesTracker.changes.map(c => c.op)).to.deep.equal([
-        { p: 10, d: 'one' },
-        { p: 15, d: '56789two0123456789three012345' },
-      ])
-    })
-
-    it('a tracked delete between two tracked deletes joins them into a single tracked delete', function () {
-      this.rangesTracker.track_changes = true
-      this.rangesTracker.applyOp({
-        p: 20,
-        d: '0123456789',
-      })
-      expect(this.rangesTracker.changes.map(c => c.op)).to.deep.equal([
-        { p: 10, d: 'one' },
-        { p: 20, d: 'two0123456789three' },
-      ])
-    })
-
-    it("rejecting one tracked delete doesn't reject the others", function () {
-      this.rangesTracker.track_changes = true
-      this.rangesTracker.applyOp({
-        p: 20,
-        i: 'two',
-        u: true,
-      })
-      expect(this.rangesTracker.changes.map(c => c.op)).to.deep.equal([
-        { p: 10, d: 'one' },
-        { p: 33, d: 'three' },
-      ])
-    })
-
-    it("rejecting all tracked deletes doesn't introduce tracked inserts", function () {
-      this.rangesTracker.track_changes = true
-      this.rangesTracker.applyOp({
-        p: 10,
-        i: 'one',
-        u: true,
-      })
-      this.rangesTracker.applyOp({
-        p: 23,
-        i: 'two',
-        u: true,
-      })
-      this.rangesTracker.applyOp({
-        p: 36,
-        i: 'three',
-        u: true,
-      })
-      expect(this.rangesTracker.changes.map(c => c.op)).to.deep.equal([])
-    })
-  })
-
-  describe('with multiple tracked deletes at the same position', function () {
-    beforeEach(function () {
-      this.comments = []
-      this.changes = [
-        { id: 'id1', op: { p: 33, d: 'before' } },
-        { id: 'id2', op: { p: 50, d: 'right before' } },
-        { id: 'id3', op: { p: 50, d: 'this one' } },
-        { id: 'id4', op: { p: 50, d: 'right after' } },
-        { id: 'id5', op: { p: 75, d: 'long after' } },
-      ]
-      this.rangesTracker = new RangesTracker(this.changes, this.comments)
-    })
-
-    it('preserves the text order when rejecting changes', function () {
-      this.rangesTracker.applyOp(
-        { p: 50, i: 'this one', u: true },
-        { user_id: 'user-id' }
-      )
-      expect(this.rangesTracker.changes).to.deep.equal([
-        { id: 'id1', op: { p: 33, d: 'before' } },
-        { id: 'id2', op: { p: 50, d: 'right before' } },
-        { id: 'id4', op: { p: 58, d: 'right after' } },
-        { id: 'id5', op: { p: 83, d: 'long after' } },
-      ])
-    })
-
-    it('moves all tracked deletes after the insert if not rejecting changes', function () {
-      this.rangesTracker.applyOp(
-        { p: 50, i: 'some other text', u: true, orderedRejections: true },
-        { user_id: 'user-id' }
-      )
-      expect(this.rangesTracker.changes).to.deep.equal([
-        { id: 'id1', op: { p: 33, d: 'before' } },
-        { id: 'id2', op: { p: 65, d: 'right before' } },
-        { id: 'id3', op: { p: 65, d: 'this one' } },
-        { id: 'id4', op: { p: 65, d: 'right after' } },
-        { id: 'id5', op: { p: 90, d: 'long after' } },
-      ])
-    })
-  })
-
-  describe('with multiple tracked deletes at the same position with the same content', function () {
-    beforeEach(function () {
-      this.comments = []
-      this.changes = [
-        { id: 'id1', op: { p: 10, d: 'cat' } },
-        { id: 'id2', op: { p: 10, d: 'giraffe' } },
-        { id: 'id3', op: { p: 10, d: 'cat' } },
-        { id: 'id4', op: { p: 10, d: 'giraffe' } },
-      ]
-      this.rangesTracker = new RangesTracker(this.changes, this.comments)
-    })
-
-    it('removes only the first matching tracked delete', function () {
-      this.rangesTracker.applyOp(
-        { p: 10, i: 'giraffe', u: true },
-        { user_id: 'user-id' }
-      )
-      expect(this.rangesTracker.changes).to.deep.equal([
-        { id: 'id1', op: { p: 10, d: 'cat' } },
-        { id: 'id3', op: { p: 17, d: 'cat' } },
-        { id: 'id4', op: { p: 17, d: 'giraffe' } },
-      ])
-    })
-  })
-
-  describe('with a tracked insert at the same position as a tracked delete', function () {
-    beforeEach(function () {
-      this.comments = []
-      this.changes = [
-        {
-          id: 'id1',
-          op: { p: 5, d: 'before' },
-          metadata: { user_id: 'user-id' },
-        },
-        {
-          id: 'id2',
-          op: { p: 10, d: 'delete' },
-          metadata: { user_id: 'user-id' },
-        },
-        {
-          id: 'id3',
-          op: { p: 10, i: 'insert' },
-          metadata: { user_id: 'user-id' },
-        },
-      ]
-      this.rangesTracker = new RangesTracker(this.changes, this.comments)
-    })
-
-    it('places a tracked insert at the same position before both the delete and the insert', function () {
-      this.rangesTracker.track_changes = true
-      this.rangesTracker.applyOp(
-        { p: 10, i: 'incoming' },
-        { user_id: 'user-id' }
-      )
-      expect(this.rangesTracker.changes.map(change => change.op)).to.deep.equal(
-        [
-          { p: 5, d: 'before' },
-          { p: 10, i: 'incoming' },
-          { p: 18, d: 'delete' },
-          { p: 18, i: 'insert' },
-        ]
-      )
-    })
-  })
 })

libraries/redis-wrapper/.nvmrc

@@ -1 +1 @@
-20.18.2
+18.20.2

libraries/redis-wrapper/RedisLocker.js

@@ -1,8 +1,8 @@
-const { promisify } = require('node:util')
+const { promisify } = require('util')
 const metrics = require('@overleaf/metrics')
 const logger = require('@overleaf/logger')
-const os = require('node:os')
-const crypto = require('node:crypto')
+const os = require('os')
+const crypto = require('crypto')
 
 const HOST = os.hostname()
 const PID = process.pid

libraries/redis-wrapper/RedisWebLocker.js

@@ -1,8 +1,8 @@
-const { callbackify, promisify } = require('node:util')
+const { callbackify, promisify } = require('util')
 const metrics = require('@overleaf/metrics')
 const logger = require('@overleaf/logger')
-const os = require('node:os')
-const crypto = require('node:crypto')
+const os = require('os')
+const crypto = require('crypto')
 const async = require('async')
 
 const HOST = os.hostname()

libraries/redis-wrapper/buildscript.txt

@@ -5,6 +5,6 @@ redis-wrapper
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=20.18.2
+--node-version=18.20.2
 --public-repo=False
 --script-version=4.5.0